195 lines
4.3 KiB
Python
195 lines
4.3 KiB
Python
|
|
import chromadb
|
|
from chromadb.utils import embedding_functions
|
|
|
|
|
|
# ==========================================
|
|
# PENGATURAN
|
|
# ==========================================
|
|
|
|
FOLDER_DATABASE = "./chroma_db"
|
|
|
|
MODEL_EMBEDDING = "paraphrase-multilingual-MiniLM-L12-v2"
|
|
|
|
|
|
# ==========================================
|
|
# 15 PERTANYAAN PENGUJIAN
|
|
# Berdasarkan materi PDF yang digunakan
|
|
# ==========================================
|
|
|
|
UJI = [
|
|
("Apa kepanjangan dari LLM?", "Large Language Model"),
|
|
|
|
("Bagaimana cara kerja Bag-of-Words?", "frekuensi kata"),
|
|
|
|
("Apa kelemahan Bag-of-Words?", "tidak memahami urutan panjang"),
|
|
|
|
("Apa fungsi Word Embedding?", "vektor"),
|
|
|
|
("Apa contoh metode Word Embedding?", "Word2vec"),
|
|
|
|
("Apa itu RNN dan LSTM?", "RNN / LSTM"),
|
|
|
|
("Apa mekanisme yang digunakan Transformer?", "self-attention"),
|
|
|
|
("Apa judul penelitian yang memperkenalkan Transformer?",
|
|
"Attention Is All You Need"),
|
|
|
|
("Apa contoh model pada era pre-training skala besar?",
|
|
"BERT, GPT-2, GPT-3"),
|
|
|
|
("Apa kemampuan GPT-3 yang ditunjukkan dalam materi?",
|
|
"few-shot"),
|
|
|
|
("Apa yang dimaksud dengan RAG?",
|
|
"kumpulan dokumen"),
|
|
|
|
("Kapan RAG tepat digunakan?",
|
|
"sering berubah"),
|
|
|
|
("Apa kelemahan memasukkan seluruh dokumen ke prompt?",
|
|
"context window"),
|
|
|
|
("Apa teknik yang digunakan untuk memberikan contoh kepada model?",
|
|
"Few-shot"),
|
|
|
|
("Apa materi utama pada P-10?",
|
|
"Embedding & Vector Database"),
|
|
]
|
|
|
|
|
|
# ==========================================
|
|
# MENGHUBUNGKAN KE CHROMADB
|
|
# ==========================================
|
|
|
|
def hubungkan_database(ukuran):
|
|
|
|
client = chromadb.PersistentClient(
|
|
path=FOLDER_DATABASE
|
|
)
|
|
|
|
embedding = (
|
|
embedding_functions.SentenceTransformerEmbeddingFunction(
|
|
model_name=MODEL_EMBEDDING
|
|
)
|
|
)
|
|
|
|
nama_koleksi = f"dokumen_chunk_{ukuran}"
|
|
|
|
koleksi = client.get_collection(
|
|
name=nama_koleksi,
|
|
embedding_function=embedding
|
|
)
|
|
|
|
return koleksi
|
|
|
|
|
|
# ==========================================
|
|
# MENGHITUNG HIT@K DAN MRR
|
|
# ==========================================
|
|
|
|
def evaluasi(koleksi, k):
|
|
|
|
benar = 0
|
|
total_reciprocal_rank = 0
|
|
|
|
jumlah_data = koleksi.count()
|
|
|
|
if jumlah_data == 0:
|
|
return 0, 0
|
|
|
|
jumlah_hasil = min(k, jumlah_data)
|
|
|
|
for pertanyaan, kata_kunci in UJI:
|
|
|
|
hasil = koleksi.query(
|
|
query_texts=[pertanyaan],
|
|
n_results=jumlah_hasil
|
|
)
|
|
|
|
daftar_dokumen = hasil["documents"][0]
|
|
|
|
peringkat_benar = None
|
|
|
|
for nomor, dokumen in enumerate(
|
|
daftar_dokumen,
|
|
start=1
|
|
):
|
|
|
|
if kata_kunci.lower() in dokumen.lower():
|
|
|
|
peringkat_benar = nomor
|
|
break
|
|
|
|
if peringkat_benar is not None:
|
|
|
|
benar += 1
|
|
|
|
total_reciprocal_rank += 1 / peringkat_benar
|
|
|
|
hit = benar / len(UJI)
|
|
|
|
mrr = total_reciprocal_rank / len(UJI)
|
|
|
|
return hit, mrr
|
|
|
|
|
|
# ==========================================
|
|
# PROGRAM UTAMA EVALUASI
|
|
# ==========================================
|
|
|
|
def jalankan_evaluasi():
|
|
|
|
print("\n===== EVALUASI RETRIEVAL RAG =====")
|
|
|
|
print("Jumlah pertanyaan:", len(UJI))
|
|
|
|
print("\nHasil perbandingan ukuran chunk:")
|
|
|
|
print("-----------------------------------------------")
|
|
|
|
print("Ukuran | Hit@1 | Hit@3 | Hit@5 | MRR")
|
|
|
|
print("-----------------------------------------------")
|
|
|
|
for ukuran in [500, 1000, 1500]:
|
|
|
|
try:
|
|
|
|
koleksi = hubungkan_database(ukuran)
|
|
|
|
hasil_1, mrr_1 = evaluasi(koleksi, 1)
|
|
|
|
hasil_3, mrr_3 = evaluasi(koleksi, 3)
|
|
|
|
hasil_5, mrr_5 = evaluasi(koleksi, 5)
|
|
|
|
# MRR ditampilkan berdasarkan hasil top-5.
|
|
|
|
print(
|
|
f"{ukuran:6} | "
|
|
f"{hasil_1:.2f} | "
|
|
f"{hasil_3:.2f} | "
|
|
f"{hasil_5:.2f} | "
|
|
f"{mrr_5:.2f}"
|
|
)
|
|
|
|
except Exception as error:
|
|
|
|
print(f"Chunk {ukuran}: gagal dievaluasi.")
|
|
|
|
print("Keterangan:", error)
|
|
|
|
print("-----------------------------------------------")
|
|
|
|
print("Evaluasi selesai.")
|
|
|
|
|
|
# ==========================================
|
|
# MENJALANKAN PROGRAM
|
|
# ==========================================
|
|
|
|
if __name__ == "__main__":
|
|
|
|
jalankan_evaluasi()
|