commit d74de81c2c9c2ad1d2df799938f0aeb01ee52455 Author: Hijau-dev Date: Mon Nov 2 11:04:04 2026 +0700 Commit diff --git a/dokumen/Eksplorasi Mars.pdf b/dokumen/Eksplorasi Mars.pdf new file mode 100644 index 0000000..a56f491 Binary files /dev/null and b/dokumen/Eksplorasi Mars.pdf differ diff --git a/dokumen/Pengantar Kecerdasan Buatan.pdf b/dokumen/Pengantar Kecerdasan Buatan.pdf new file mode 100644 index 0000000..b561b07 Binary files /dev/null and b/dokumen/Pengantar Kecerdasan Buatan.pdf differ diff --git a/dokumen/Potensi Energi Terbarukan.pdf b/dokumen/Potensi Energi Terbarukan.pdf new file mode 100644 index 0000000..0f856e8 Binary files /dev/null and b/dokumen/Potensi Energi Terbarukan.pdf differ diff --git a/main.py b/main.py new file mode 100644 index 0000000..4ade555 --- /dev/null +++ b/main.py @@ -0,0 +1,250 @@ +""" +Lab P-10: RAG Part 1 - Embedding, Chunking, Vector DB +Cara pakai: + pip install pypdf chromadb sentence-transformers + mkdir dokumen # taruh 2-3 PDF di sini + + python rag_lab.py lab1 # Lab 1: eksperimen chunking + python rag_lab.py lab2 # Lab 2: indeks + 5 pertanyaan + 2 parafrase + python rag_lab.py lab3 # Lab 3: hit@k pada 2 ukuran chunk + python rag_lab.py query "pertanyaan" # tanya bebas (opsi --size 500) +""" +import argparse +import glob +import os +import random +import re + +import chromadb +from chromadb.utils import embedding_functions +from pypdf import PdfReader + +FOLDER = "dokumen" +DB_PATH = "chroma_db" +# Model multibahasa supaya bahasa Indonesia & parafrase dipahami dengan baik +EMBED_MODEL = "paraphrase-multilingual-MiniLM-L12-v2" + +_ef = None +_client = None + + +# ---------------------------------------------------------------- dasar +def baca_pdf(path): + """Baca seluruh teks PDF, rapikan spasi/baris baru.""" + reader = PdfReader(path) + teks = " ".join((hal.extract_text() or "") for hal in reader.pages) + return re.sub(r"\s+", " ", teks).strip() + + +def potong(teks, ukuran=500, overlap_persen=0.15): + """Potong teks per `ukuran` karakter dengan overlap (10-20%).""" + overlap = int(ukuran * overlap_persen) + langkah = ukuran - overlap + return [teks[i:i + ukuran] for i in range(0, len(teks), langkah) + if teks[i:i + ukuran].strip()] + + +def get_client(): + global _client, _ef + if _client is None: + _client = chromadb.PersistentClient(path=DB_PATH) + _ef = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name=EMBED_MODEL) + return _client + + +def daftar_pdf(): + files = sorted(glob.glob(os.path.join(FOLDER, "*.pdf"))) + if not files: + raise SystemExit(f"Tidak ada PDF di folder '{FOLDER}/'. Letakkan 2-3 PDF dulu.") + return files + + +# ---------------------------------------------------------------- indeks +def indeks(ukuran, overlap_persen=0.15, verbose=True): + """Bangun koleksi `dok_` dari semua PDF di folder dokumen/.""" + client = get_client() + nama = f"dok_{ukuran}" + try: + client.delete_collection(nama) + except Exception: + pass + col = client.create_collection(nama, embedding_function=_ef, + metadata={"hnsw:space": "cosine"}) + for path in daftar_pdf(): + chunks = potong(baca_pdf(path), ukuran, overlap_persen) + fname = os.path.basename(path) + col.add( + ids=[f"{fname}-{i}" for i in range(len(chunks))], + documents=chunks, + metadatas=[{"sumber": fname, "chunk": i} for i in range(len(chunks))], + ) + if verbose: + print(f" [{nama}] {fname}: {len(chunks)} chunk") + return col + + +def get_collection(ukuran): + client = get_client() + try: + return client.get_collection(f"dok_{ukuran}", embedding_function=_ef) + except Exception: + return indeks(ukuran, verbose=False) + + +def query(pertanyaan, k=3, ukuran=500, tampil=True): + col = get_collection(ukuran) + hasil = col.query(query_texts=[pertanyaan], n_results=k) + docs = hasil["documents"][0] + metas = hasil["metadatas"][0] + jarak = hasil["distances"][0] + if tampil: + print(f"\nTanya: {pertanyaan}") + for rank, (d, m, j) in enumerate(zip(docs, metas, jarak), 1): + print(f" #{rank} [{m['sumber']} chunk {m['chunk']}] jarak={j:.3f}") + print(f" {d[:200]}...") + return docs + + +# ---------------------------------------------------------------- lab 1 +def lab1(): + path = daftar_pdf()[0] + teks = baca_pdf(path) + print(f"=== LAB 1 | file: {path} | total {len(teks)} karakter ===") + print("\n[1] 500 karakter pertama:\n") + print(teks[:500]) + + # --- Tabel 1: ringkasan jumlah chunk + print("\n[2] Jumlah chunk (overlap 15%)\n") + hasil = {} + print(f"{'Ukuran':>7} | {'Overlap':>7} | {'Jml chunk':>9} | {'Rata2 panjang':>13} | {'Berakhir titik':>14}") + print("-" * 64) + for ukuran in (500, 1000, 1500): + chunks = potong(teks, ukuran, 0.15) + hasil[ukuran] = chunks + rata = sum(len(c) for c in chunks) / len(chunks) + titik = sum(c.rstrip().endswith(".") for c in chunks) + print(f"{ukuran:>7} | {int(ukuran*0.15):>7} | {len(chunks):>9} | " + f"{rata:>13.0f} | {titik:>8}/{len(chunks):<5}") + + # --- Tabel 2: 3 chunk acak per ukuran + print("\n[3] 3 chunk acak per ukuran (cek: terpotong di tengah kalimat/tabel?)\n") + print(f"{'Ukuran':>7} | {'Awal chunk (30 krk)':<32} | {'Akhir chunk (30 krk)':<32} | Titik?") + print("-" * 85) + for ukuran, chunks in hasil.items(): + for c in random.sample(chunks, min(3, len(chunks))): + awal = c[:30].ljust(30) + akhir = c.rstrip()[-30:].rjust(30) + ya = "Ya" if c.rstrip().endswith(".") else "Tidak" + print(f"{ukuran:>7} | {awal:<32} | ..{akhir:<30} | {ya}") + print("-" * 85) + + +# ---------------------------------------------------------------- lab 2 +PERTANYAAN_LAB2 = [ + "Wahana apa yang pertama kali mengirim gambar jarak dekat Mars?", + "Siapa yang mengusulkan Tes Turing?", + "Berapa target porsi energi terbarukan Indonesia tahun 2025?", + "Apa itu Narrow AI dan contohnya?", + "Apa saja tantangan pengembangan energi terbarukan?", +] +PERTANYAAN_PARAFRASE = [ + # tidak memakai kata yang sama dengan dokumen + "Pesawat robotik pertama yang memotret planet merah dari dekat", + "Sumber listrik dari perut bumi yang jalan nonstop tanpa peduli cuaca", +] + + +def lab2(): + print("=== LAB 2 | Bangun indeks (ukuran 500) ===") + indeks(500) + print("\n--- 5 pertanyaan: periksa manual apakah chunk teratas relevan ---") + for q in PERTANYAAN_LAB2: + query(q, k=3, ukuran=500) + print("\n--- 2 pertanyaan parafrase: apakah pencarian semantik tetap menemukan? ---") + for q in PERTANYAAN_PARAFRASE: + query(q, k=3, ukuran=500) + + +# ---------------------------------------------------------------- lab 3 +# (pertanyaan, [kata kunci jawaban]) -> hit jika SALAH SATU kata kunci ada di chunk +# Sesuaikan jika dokumen Anda berbeda. +SOAL_UJI = [ + ("Wahana apa yang pertama mengirim gambar jarak dekat Mars?", ["mariner 4"]), + ("Kapan Viking 1 dan 2 mendarat di Mars?", ["1976"]), + ("Apa nama rover pertama yang beroda di Mars?", ["sojourner"]), + ("Apa temuan utama Curiosity?", ["bahan organik", "kawah gale"]), + ("Apa program Mars Sample Return?", ["mars sample return", "msr"]), + ("Siapa yang mengusulkan Tes Turing?", ["alan turing"]), + ("Siapa yang mencetuskan istilah Artificial Intelligence pada 1956?", ["john mccarthy"]), + ("Sebutkan contoh Narrow AI.", ["siri", "alexa"]), + ("Siapa yang dikalahkan IBM Deep Blue?", ["kasparov"]), + ("Siapa yang dikalahkan AlphaGo?", ["lee sedol"]), + ("Berapa target bauran EBT nasional tahun 2025?", ["23%"]), + ("Berapa persen cadangan panas bumi dunia yang dimiliki Indonesia?", ["40%"]), + ("Berapa potensi tenaga surya Indonesia?", ["207,8"]), + ("Berapa potensi tenaga air (hidro) Indonesia?", ["75,0"]), + ("Apa tantangan pengembangan energi terbarukan di Indonesia?", + ["investasi awal", "intermitensi", "transmisi"]), +] + + +def hit_at_k(ukuran, k): + col = get_collection(ukuran) + hit, gagal = 0, [] + for q, kunci in SOAL_UJI: + docs = col.query(query_texts=[q], n_results=k)["documents"][0] + gabung = " ".join(docs).lower() + if any(kw in gabung for kw in kunci): + hit += 1 + else: + gagal.append(q) + return hit / len(SOAL_UJI), gagal + + +def lab3(): + ukuran_list = (500, 1000) # ubah ke (500, 1500) jika mau + print("=== LAB 3 | hit@k pada dua ukuran chunk ===") + for u in ukuran_list: + indeks(u, verbose=False) + + print(f"\n{'ukuran':>8} | " + " | ".join(f"hit@{k}" for k in (1, 3, 5))) + print("-" * 36) + gagal_semua = {} + for u in ukuran_list: + baris = [] + for k in (1, 3, 5): + skor, gagal = hit_at_k(u, k) + baris.append(f"{skor:.2f}") + if k == 1: + gagal_semua[u] = gagal + print(f"{u:>8} | " + " | ".join(f"{b:>5}" for b in baris)) + + print("\nPertanyaan yang GAGAL pada k=1:") + for u, g in gagal_semua.items(): + print(f" ukuran {u}:") + for q in g or ["(tidak ada)"]: + print(f" - {q}") + print("\nTugas Anda: tulis kesimpulan 3-4 kalimat (ukuran mana lebih baik, " + "pertanyaan apa gagal, dugaan penyebab).") + + +# ---------------------------------------------------------------- main +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("perintah", choices=["lab1", "lab2", "lab3", "query"]) + ap.add_argument("teks", nargs="?", help="pertanyaan (untuk perintah query)") + ap.add_argument("--size", type=int, default=500) + ap.add_argument("-k", type=int, default=3) + a = ap.parse_args() + + if a.perintah == "lab1": + lab1() + elif a.perintah == "lab2": + lab2() + elif a.perintah == "lab3": + lab3() + else: + if not a.teks: + raise SystemExit('Contoh: python rag_lab.py query "Siapa Alan Turing?"') + query(a.teks, k=a.k, ukuran=a.size) \ No newline at end of file