Commit
This commit is contained in:
Hijau-dev
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
Lab P-10: RAG Part 1 - Embedding, Chunking, Vector DB
|
||||
Cara pakai:
|
||||
pip install pypdf chromadb sentence-transformers
|
||||
mkdir dokumen # taruh 2-3 PDF di sini
|
||||
|
||||
python rag_lab.py lab1 # Lab 1: eksperimen chunking
|
||||
python rag_lab.py lab2 # Lab 2: indeks + 5 pertanyaan + 2 parafrase
|
||||
python rag_lab.py lab3 # Lab 3: hit@k pada 2 ukuran chunk
|
||||
python rag_lab.py query "pertanyaan" # tanya bebas (opsi --size 500)
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
|
||||
import chromadb
|
||||
from chromadb.utils import embedding_functions
|
||||
from pypdf import PdfReader
|
||||
|
||||
FOLDER = "dokumen"
|
||||
DB_PATH = "chroma_db"
|
||||
# Model multibahasa supaya bahasa Indonesia & parafrase dipahami dengan baik
|
||||
EMBED_MODEL = "paraphrase-multilingual-MiniLM-L12-v2"
|
||||
|
||||
_ef = None
|
||||
_client = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- dasar
|
||||
def baca_pdf(path):
|
||||
"""Baca seluruh teks PDF, rapikan spasi/baris baru."""
|
||||
reader = PdfReader(path)
|
||||
teks = " ".join((hal.extract_text() or "") for hal in reader.pages)
|
||||
return re.sub(r"\s+", " ", teks).strip()
|
||||
|
||||
|
||||
def potong(teks, ukuran=500, overlap_persen=0.15):
|
||||
"""Potong teks per `ukuran` karakter dengan overlap (10-20%)."""
|
||||
overlap = int(ukuran * overlap_persen)
|
||||
langkah = ukuran - overlap
|
||||
return [teks[i:i + ukuran] for i in range(0, len(teks), langkah)
|
||||
if teks[i:i + ukuran].strip()]
|
||||
|
||||
|
||||
def get_client():
|
||||
global _client, _ef
|
||||
if _client is None:
|
||||
_client = chromadb.PersistentClient(path=DB_PATH)
|
||||
_ef = embedding_functions.SentenceTransformerEmbeddingFunction(
|
||||
model_name=EMBED_MODEL)
|
||||
return _client
|
||||
|
||||
|
||||
def daftar_pdf():
|
||||
files = sorted(glob.glob(os.path.join(FOLDER, "*.pdf")))
|
||||
if not files:
|
||||
raise SystemExit(f"Tidak ada PDF di folder '{FOLDER}/'. Letakkan 2-3 PDF dulu.")
|
||||
return files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- indeks
|
||||
def indeks(ukuran, overlap_persen=0.15, verbose=True):
|
||||
"""Bangun koleksi `dok_<ukuran>` dari semua PDF di folder dokumen/."""
|
||||
client = get_client()
|
||||
nama = f"dok_{ukuran}"
|
||||
try:
|
||||
client.delete_collection(nama)
|
||||
except Exception:
|
||||
pass
|
||||
col = client.create_collection(nama, embedding_function=_ef,
|
||||
metadata={"hnsw:space": "cosine"})
|
||||
for path in daftar_pdf():
|
||||
chunks = potong(baca_pdf(path), ukuran, overlap_persen)
|
||||
fname = os.path.basename(path)
|
||||
col.add(
|
||||
ids=[f"{fname}-{i}" for i in range(len(chunks))],
|
||||
documents=chunks,
|
||||
metadatas=[{"sumber": fname, "chunk": i} for i in range(len(chunks))],
|
||||
)
|
||||
if verbose:
|
||||
print(f" [{nama}] {fname}: {len(chunks)} chunk")
|
||||
return col
|
||||
|
||||
|
||||
def get_collection(ukuran):
|
||||
client = get_client()
|
||||
try:
|
||||
return client.get_collection(f"dok_{ukuran}", embedding_function=_ef)
|
||||
except Exception:
|
||||
return indeks(ukuran, verbose=False)
|
||||
|
||||
|
||||
def query(pertanyaan, k=3, ukuran=500, tampil=True):
|
||||
col = get_collection(ukuran)
|
||||
hasil = col.query(query_texts=[pertanyaan], n_results=k)
|
||||
docs = hasil["documents"][0]
|
||||
metas = hasil["metadatas"][0]
|
||||
jarak = hasil["distances"][0]
|
||||
if tampil:
|
||||
print(f"\nTanya: {pertanyaan}")
|
||||
for rank, (d, m, j) in enumerate(zip(docs, metas, jarak), 1):
|
||||
print(f" #{rank} [{m['sumber']} chunk {m['chunk']}] jarak={j:.3f}")
|
||||
print(f" {d[:200]}...")
|
||||
return docs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- lab 1
|
||||
def lab1():
|
||||
path = daftar_pdf()[0]
|
||||
teks = baca_pdf(path)
|
||||
print(f"=== LAB 1 | file: {path} | total {len(teks)} karakter ===")
|
||||
print("\n[1] 500 karakter pertama:\n")
|
||||
print(teks[:500])
|
||||
|
||||
# --- Tabel 1: ringkasan jumlah chunk
|
||||
print("\n[2] Jumlah chunk (overlap 15%)\n")
|
||||
hasil = {}
|
||||
print(f"{'Ukuran':>7} | {'Overlap':>7} | {'Jml chunk':>9} | {'Rata2 panjang':>13} | {'Berakhir titik':>14}")
|
||||
print("-" * 64)
|
||||
for ukuran in (500, 1000, 1500):
|
||||
chunks = potong(teks, ukuran, 0.15)
|
||||
hasil[ukuran] = chunks
|
||||
rata = sum(len(c) for c in chunks) / len(chunks)
|
||||
titik = sum(c.rstrip().endswith(".") for c in chunks)
|
||||
print(f"{ukuran:>7} | {int(ukuran*0.15):>7} | {len(chunks):>9} | "
|
||||
f"{rata:>13.0f} | {titik:>8}/{len(chunks):<5}")
|
||||
|
||||
# --- Tabel 2: 3 chunk acak per ukuran
|
||||
print("\n[3] 3 chunk acak per ukuran (cek: terpotong di tengah kalimat/tabel?)\n")
|
||||
print(f"{'Ukuran':>7} | {'Awal chunk (30 krk)':<32} | {'Akhir chunk (30 krk)':<32} | Titik?")
|
||||
print("-" * 85)
|
||||
for ukuran, chunks in hasil.items():
|
||||
for c in random.sample(chunks, min(3, len(chunks))):
|
||||
awal = c[:30].ljust(30)
|
||||
akhir = c.rstrip()[-30:].rjust(30)
|
||||
ya = "Ya" if c.rstrip().endswith(".") else "Tidak"
|
||||
print(f"{ukuran:>7} | {awal:<32} | ..{akhir:<30} | {ya}")
|
||||
print("-" * 85)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- lab 2
|
||||
PERTANYAAN_LAB2 = [
|
||||
"Wahana apa yang pertama kali mengirim gambar jarak dekat Mars?",
|
||||
"Siapa yang mengusulkan Tes Turing?",
|
||||
"Berapa target porsi energi terbarukan Indonesia tahun 2025?",
|
||||
"Apa itu Narrow AI dan contohnya?",
|
||||
"Apa saja tantangan pengembangan energi terbarukan?",
|
||||
]
|
||||
PERTANYAAN_PARAFRASE = [
|
||||
# tidak memakai kata yang sama dengan dokumen
|
||||
"Pesawat robotik pertama yang memotret planet merah dari dekat",
|
||||
"Sumber listrik dari perut bumi yang jalan nonstop tanpa peduli cuaca",
|
||||
]
|
||||
|
||||
|
||||
def lab2():
|
||||
print("=== LAB 2 | Bangun indeks (ukuran 500) ===")
|
||||
indeks(500)
|
||||
print("\n--- 5 pertanyaan: periksa manual apakah chunk teratas relevan ---")
|
||||
for q in PERTANYAAN_LAB2:
|
||||
query(q, k=3, ukuran=500)
|
||||
print("\n--- 2 pertanyaan parafrase: apakah pencarian semantik tetap menemukan? ---")
|
||||
for q in PERTANYAAN_PARAFRASE:
|
||||
query(q, k=3, ukuran=500)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- lab 3
|
||||
# (pertanyaan, [kata kunci jawaban]) -> hit jika SALAH SATU kata kunci ada di chunk
|
||||
# Sesuaikan jika dokumen Anda berbeda.
|
||||
SOAL_UJI = [
|
||||
("Wahana apa yang pertama mengirim gambar jarak dekat Mars?", ["mariner 4"]),
|
||||
("Kapan Viking 1 dan 2 mendarat di Mars?", ["1976"]),
|
||||
("Apa nama rover pertama yang beroda di Mars?", ["sojourner"]),
|
||||
("Apa temuan utama Curiosity?", ["bahan organik", "kawah gale"]),
|
||||
("Apa program Mars Sample Return?", ["mars sample return", "msr"]),
|
||||
("Siapa yang mengusulkan Tes Turing?", ["alan turing"]),
|
||||
("Siapa yang mencetuskan istilah Artificial Intelligence pada 1956?", ["john mccarthy"]),
|
||||
("Sebutkan contoh Narrow AI.", ["siri", "alexa"]),
|
||||
("Siapa yang dikalahkan IBM Deep Blue?", ["kasparov"]),
|
||||
("Siapa yang dikalahkan AlphaGo?", ["lee sedol"]),
|
||||
("Berapa target bauran EBT nasional tahun 2025?", ["23%"]),
|
||||
("Berapa persen cadangan panas bumi dunia yang dimiliki Indonesia?", ["40%"]),
|
||||
("Berapa potensi tenaga surya Indonesia?", ["207,8"]),
|
||||
("Berapa potensi tenaga air (hidro) Indonesia?", ["75,0"]),
|
||||
("Apa tantangan pengembangan energi terbarukan di Indonesia?",
|
||||
["investasi awal", "intermitensi", "transmisi"]),
|
||||
]
|
||||
|
||||
|
||||
def hit_at_k(ukuran, k):
|
||||
col = get_collection(ukuran)
|
||||
hit, gagal = 0, []
|
||||
for q, kunci in SOAL_UJI:
|
||||
docs = col.query(query_texts=[q], n_results=k)["documents"][0]
|
||||
gabung = " ".join(docs).lower()
|
||||
if any(kw in gabung for kw in kunci):
|
||||
hit += 1
|
||||
else:
|
||||
gagal.append(q)
|
||||
return hit / len(SOAL_UJI), gagal
|
||||
|
||||
|
||||
def lab3():
|
||||
ukuran_list = (500, 1000) # ubah ke (500, 1500) jika mau
|
||||
print("=== LAB 3 | hit@k pada dua ukuran chunk ===")
|
||||
for u in ukuran_list:
|
||||
indeks(u, verbose=False)
|
||||
|
||||
print(f"\n{'ukuran':>8} | " + " | ".join(f"hit@{k}" for k in (1, 3, 5)))
|
||||
print("-" * 36)
|
||||
gagal_semua = {}
|
||||
for u in ukuran_list:
|
||||
baris = []
|
||||
for k in (1, 3, 5):
|
||||
skor, gagal = hit_at_k(u, k)
|
||||
baris.append(f"{skor:.2f}")
|
||||
if k == 1:
|
||||
gagal_semua[u] = gagal
|
||||
print(f"{u:>8} | " + " | ".join(f"{b:>5}" for b in baris))
|
||||
|
||||
print("\nPertanyaan yang GAGAL pada k=1:")
|
||||
for u, g in gagal_semua.items():
|
||||
print(f" ukuran {u}:")
|
||||
for q in g or ["(tidak ada)"]:
|
||||
print(f" - {q}")
|
||||
print("\nTugas Anda: tulis kesimpulan 3-4 kalimat (ukuran mana lebih baik, "
|
||||
"pertanyaan apa gagal, dugaan penyebab).")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("perintah", choices=["lab1", "lab2", "lab3", "query"])
|
||||
ap.add_argument("teks", nargs="?", help="pertanyaan (untuk perintah query)")
|
||||
ap.add_argument("--size", type=int, default=500)
|
||||
ap.add_argument("-k", type=int, default=3)
|
||||
a = ap.parse_args()
|
||||
|
||||
if a.perintah == "lab1":
|
||||
lab1()
|
||||
elif a.perintah == "lab2":
|
||||
lab2()
|
||||
elif a.perintah == "lab3":
|
||||
lab3()
|
||||
else:
|
||||
if not a.teks:
|
||||
raise SystemExit('Contoh: python rag_lab.py query "Siapa Alan Turing?"')
|
||||
query(a.teks, k=a.k, ukuran=a.size)
|
||||
Reference in New Issue
Block a user