148 lines
4.0 KiB
Python
148 lines
4.0 KiB
Python
|
|
import chromadb
|
|
from chromadb.utils import embedding_functions
|
|
|
|
|
|
# ==========================================
|
|
# PENGATURAN
|
|
# ==========================================
|
|
|
|
FOLDER_DATABASE = "./chroma_db"
|
|
|
|
MODEL_EMBEDDING = "paraphrase-multilingual-MiniLM-L12-v2"
|
|
|
|
|
|
# ==========================================
|
|
# 15 PERTANYAAN PENGUJIAN
|
|
# GANTI PERTANYAAN DAN KATA KUNCI
|
|
# SESUAI ISI PDF YANG KAMU GUNAKAN.
|
|
# ==========================================
|
|
|
|
UJI = [
|
|
("Pertanyaan 1 dari dokumen", "kata kunci jawaban 1"),
|
|
("Pertanyaan 2 dari dokumen", "kata kunci jawaban 2"),
|
|
("Pertanyaan 3 dari dokumen", "kata kunci jawaban 3"),
|
|
("Pertanyaan 4 dari dokumen", "kata kunci jawaban 4"),
|
|
("Pertanyaan 5 dari dokumen", "kata kunci jawaban 5"),
|
|
("Pertanyaan 6 dari dokumen", "kata kunci jawaban 6"),
|
|
("Pertanyaan 7 dari dokumen", "kata kunci jawaban 7"),
|
|
("Pertanyaan 8 dari dokumen", "kata kunci jawaban 8"),
|
|
("Pertanyaan 9 dari dokumen", "kata kunci jawaban 9"),
|
|
("Pertanyaan 10 dari dokumen", "kata kunci jawaban 10"),
|
|
("Pertanyaan 11 dari dokumen", "kata kunci jawaban 11"),
|
|
("Pertanyaan 12 dari dokumen", "kata kunci jawaban 12"),
|
|
("Pertanyaan 13 dari dokumen", "kata kunci jawaban 13"),
|
|
("Pertanyaan 14 dari dokumen", "kata kunci jawaban 14"),
|
|
("Pertanyaan 15 dari dokumen", "kata kunci jawaban 15"),
|
|
]
|
|
|
|
|
|
# ==========================================
|
|
# MENGHUBUNGKAN KE CHROMADB
|
|
# ==========================================
|
|
|
|
def hubungkan_database(ukuran):
|
|
client = chromadb.PersistentClient(
|
|
path=FOLDER_DATABASE
|
|
)
|
|
|
|
embedding = (
|
|
embedding_functions.SentenceTransformerEmbeddingFunction(
|
|
model_name=MODEL_EMBEDDING
|
|
)
|
|
)
|
|
|
|
nama_koleksi = f"dokumen_chunk_{ukuran}"
|
|
|
|
koleksi = client.get_collection(
|
|
name=nama_koleksi,
|
|
embedding_function=embedding
|
|
)
|
|
|
|
return koleksi
|
|
|
|
|
|
# ==========================================
|
|
# MENGHITUNG HIT@K DAN MRR
|
|
# ==========================================
|
|
|
|
def evaluasi(koleksi, k):
|
|
benar = 0
|
|
total_reciprocal_rank = 0
|
|
|
|
jumlah_data = koleksi.count()
|
|
|
|
if jumlah_data == 0:
|
|
return 0, 0
|
|
|
|
jumlah_hasil = min(k, jumlah_data)
|
|
|
|
for pertanyaan, kata_kunci in UJI:
|
|
|
|
hasil = koleksi.query(
|
|
query_texts=[pertanyaan],
|
|
n_results=jumlah_hasil
|
|
)
|
|
|
|
daftar_dokumen = hasil["documents"][0]
|
|
|
|
peringkat_benar = None
|
|
|
|
for nomor, dokumen in enumerate(daftar_dokumen, start=1):
|
|
if kata_kunci.lower() in dokumen.lower():
|
|
peringkat_benar = nomor
|
|
break
|
|
|
|
if peringkat_benar is not None:
|
|
benar += 1
|
|
total_reciprocal_rank += 1 / peringkat_benar
|
|
|
|
hit = benar / len(UJI)
|
|
mrr = total_reciprocal_rank / len(UJI)
|
|
|
|
return hit, mrr
|
|
|
|
|
|
# ==========================================
|
|
# PROGRAM UTAMA EVALUASI
|
|
# ==========================================
|
|
|
|
def jalankan_evaluasi():
|
|
print("\n===== EVALUASI RETRIEVAL RAG =====")
|
|
|
|
print("Jumlah pertanyaan:", len(UJI))
|
|
|
|
print("\nHasil perbandingan ukuran chunk:")
|
|
print("-----------------------------------------------")
|
|
print("Ukuran | Hit@1 | Hit@3 | Hit@5 | MRR")
|
|
print("-----------------------------------------------")
|
|
|
|
for ukuran in [500, 1000, 1500]:
|
|
|
|
try:
|
|
koleksi = hubungkan_database(ukuran)
|
|
|
|
hasil_1, mrr_1 = evaluasi(koleksi, 1)
|
|
hasil_3, mrr_3 = evaluasi(koleksi, 3)
|
|
hasil_5, mrr_5 = evaluasi(koleksi, 5)
|
|
|
|
# MRR dihitung menggunakan hasil top-5.
|
|
print(
|
|
f"{ukuran:6} | "
|
|
f"{hasil_1:.2f} | "
|
|
f"{hasil_3:.2f} | "
|
|
f"{hasil_5:.2f} | "
|
|
f"{mrr_5:.2f}"
|
|
)
|
|
|
|
except Exception as error:
|
|
print(f"Chunk {ukuran}: gagal dievaluasi.")
|
|
print("Keterangan:", error)
|
|
|
|
print("-----------------------------------------------")
|
|
print("Evaluasi selesai.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
jalankan_evaluasi()
|