commit 5cfa138cf4b7f4f62172d3185953dfb1b4d96f37 Author: Cliff Julyano S Date: Mon Nov 2 11:00:37 2026 +0700 first commit diff --git a/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/data_level0.bin b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/data_level0.bin new file mode 100644 index 0000000..5049ee9 Binary files /dev/null and b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/data_level0.bin differ diff --git a/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/header.bin b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/header.bin differ diff --git a/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/length.bin b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/length.bin new file mode 100644 index 0000000..8b0b0ee Binary files /dev/null and b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/length.bin differ diff --git a/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/link_lists.bin b/chroma_db/409452d7-2eaf-4e93-89ad-11855feb22f0/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/data_level0.bin b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/data_level0.bin new file mode 100644 index 0000000..35205c1 Binary files /dev/null and b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/data_level0.bin differ diff --git a/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/header.bin b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/header.bin differ diff --git a/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/length.bin b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/length.bin new file mode 100644 index 0000000..5e6ac0d Binary files /dev/null and b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/length.bin differ diff --git a/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/link_lists.bin b/chroma_db/4b8ce07d-9113-492d-86f2-6200fb3af8bd/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/data_level0.bin b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/data_level0.bin new file mode 100644 index 0000000..cbb86ef Binary files /dev/null and b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/data_level0.bin differ diff --git a/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/header.bin b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/header.bin differ diff --git a/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/length.bin b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/length.bin new file mode 100644 index 0000000..0197cff Binary files /dev/null and b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/length.bin differ diff --git a/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/link_lists.bin b/chroma_db/ad289497-1ca0-4b39-9e15-3664b666cffd/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/data_level0.bin b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/data_level0.bin new file mode 100644 index 0000000..dd1aa98 Binary files /dev/null and b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/data_level0.bin differ diff --git a/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/header.bin b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/header.bin differ diff --git a/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/length.bin b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/length.bin new file mode 100644 index 0000000..9578ad1 Binary files /dev/null and b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/length.bin differ diff --git a/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/link_lists.bin b/chroma_db/cd8f63d4-76f3-4171-9eec-c7fada8ba019/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/chroma.sqlite3 b/chroma_db/chroma.sqlite3 new file mode 100644 index 0000000..0d75bc1 Binary files /dev/null and b/chroma_db/chroma.sqlite3 differ diff --git a/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/data_level0.bin b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/data_level0.bin new file mode 100644 index 0000000..a440ecc Binary files /dev/null and b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/data_level0.bin differ diff --git a/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/header.bin b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/header.bin differ diff --git a/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/length.bin b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/length.bin new file mode 100644 index 0000000..4211283 Binary files /dev/null and b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/length.bin differ diff --git a/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/link_lists.bin b/chroma_db/d2ed8f9c-5f2a-4d00-baa9-af23c7271155/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/data_level0.bin b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/data_level0.bin new file mode 100644 index 0000000..5ebca45 Binary files /dev/null and b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/data_level0.bin differ diff --git a/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/header.bin b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/header.bin differ diff --git a/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/length.bin b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/length.bin new file mode 100644 index 0000000..257e43e Binary files /dev/null and b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/length.bin differ diff --git a/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/link_lists.bin b/chroma_db/db7a1f5d-2a4e-43c0-8b94-08ab1e5bc08e/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/data_level0.bin b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/data_level0.bin new file mode 100644 index 0000000..1a976fa Binary files /dev/null and b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/data_level0.bin differ diff --git a/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/header.bin b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/header.bin differ diff --git a/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/length.bin b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/length.bin new file mode 100644 index 0000000..366cb21 Binary files /dev/null and b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/length.bin differ diff --git a/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/link_lists.bin b/chroma_db/def415e4-1383-466a-854b-f4848e693ab9/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/data_level0.bin b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/data_level0.bin new file mode 100644 index 0000000..fde5556 Binary files /dev/null and b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/data_level0.bin differ diff --git a/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/header.bin b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/header.bin differ diff --git a/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/length.bin b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/length.bin new file mode 100644 index 0000000..df48cca Binary files /dev/null and b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/length.bin differ diff --git a/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/link_lists.bin b/chroma_db/f144812f-6127-4760-b0ea-acfa12328696/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/dokumen/statistika.pdf b/dokumen/statistika.pdf new file mode 100644 index 0000000..0fb0c14 Binary files /dev/null and b/dokumen/statistika.pdf differ diff --git a/lab1_chunking.py b/lab1_chunking.py new file mode 100644 index 0000000..b42dd07 --- /dev/null +++ b/lab1_chunking.py @@ -0,0 +1,325 @@ +""" +LAB 1 - EKSPERIMEN CHUNKING (versi 3) + +Membaca PDF, memotongnya dengan 3 ukuran chunk (overlap 15%), lalu +menganalisis kualitas potongan: terpotong di tengah kalimat/kata, +mengandung pola tabel, atau memuat simbol matematika yang rusak. + +Dua metode dibandingkan: + - "tetap" : potong tepat per N karakter (naif, sering memutus kata/kalimat) + - "cerdas" : potong di batas kalimat terdekat (atau batas kata jika tidak + ada), dan awal chunk berikutnya selalu di awal kata. + Panjang chunk = maksimal "ukuran" karakter. +""" + +import random +import re +from pathlib import Path + +from pypdf import PdfReader +from pypdf.errors import PdfReadError + +random.seed(42) # agar sampel acak reproducible + +FOLDER = Path("dokumen") +PENGATURAN = [(500, 75), (1000, 150), (1500, 225)] # (ukuran, overlap) = 15% +JUMLAH_SAMPEL = 3 +JENDELA_CARI = 0.3 # cari batas kalimat/kata di 30% terakhir sebuah chunk + +# Karakter Private Use Area (simbol matematika yang rusak saat ekstraksi) +POLA_SIMBOL_RUSAK = re.compile(r"[-]") +POLA_AKHIR_KALIMAT = re.compile(r"[.!?:;][\"')\]]*$") +POLA_BATAS_KALIMAT = re.compile(r"[.!?][\"')\]]*(?=\s|$)") +POLA_SPASI = re.compile(r"\s") +POLA_KOLOM = re.compile(r"\S\s{3,}\S") # 3+ spasi di antara dua token + + +# ========================================== +# MEMBACA PDF +# ========================================== + +def baca_pdf(path): + """Mengembalikan (teks_gabungan, jumlah_halaman).""" + reader = PdfReader(path) + if reader.is_encrypted: + try: + reader.decrypt("") + except Exception: + raise ValueError("PDF terenkripsi dan tidak bisa dibuka tanpa password") + halaman = [(p.extract_text() or "") for p in reader.pages] + return "\n".join(halaman), len(reader.pages) + + +# ========================================== +# CHUNKING +# ========================================== + +def _validasi(ukuran, tumpang): + if ukuran <= 0: + raise ValueError("Ukuran chunk harus > 0") + if tumpang < 0 or tumpang >= ukuran: + raise ValueError("Overlap harus 0 <= tumpang < ukuran") + + +def potong_tetap(teks, ukuran=800, tumpang=100): + """Potong per N karakter. Mengembalikan daftar (mulai, akhir).""" + _validasi(ukuran, tumpang) + posisi = [] + mulai = 0 + langkah = ukuran - tumpang + while mulai < len(teks): + akhir = min(mulai + ukuran, len(teks)) + if teks[mulai:akhir].strip(): + posisi.append((mulai, akhir)) + if akhir >= len(teks): # hindari chunk ekor yang isinya hanya overlap + break + mulai += langkah + return posisi + + +def _cari_akhir(teks, mulai, batas, ukuran): + """Pilih titik akhir chunk (<= batas): batas kalimat > batas kata > batas keras.""" + awal_cari = max(mulai + 1, batas - int(ukuran * JENDELA_CARI)) + wilayah = teks[awal_cari:batas] + + # 1. batas kalimat terakhir di dalam jendela + cocok = None + for cocok in POLA_BATAS_KALIMAT.finditer(wilayah): + pass + if cocok: + return awal_cari + cocok.end() + + # 2. spasi/baris baru terakhir (chunk berakhir tepat di akhir sebuah kata) + spasi = None + for spasi in POLA_SPASI.finditer(wilayah): + pass + if spasi: + return awal_cari + spasi.start() + + # 3. tidak ada spasi sama sekali (mis. string panjang tanpa jeda) + return batas + + +def potong_cerdas(teks, ukuran=800, tumpang=100): + """Potong di batas kalimat/kata. Mengembalikan daftar (mulai, akhir).""" + _validasi(ukuran, tumpang) + n = len(teks) + posisi = [] + mulai = 0 + while mulai < n and teks[mulai].isspace(): + mulai += 1 + + while mulai < n: + batas = min(mulai + ukuran, n) + akhir = n if batas >= n else _cari_akhir(teks, mulai, batas, ukuran) + + if teks[mulai:akhir].strip(): + posisi.append((mulai, akhir)) + if akhir >= n: + break + + # awal chunk berikutnya = mundur sebesar overlap, lalu geser maju + # ke awal kata terdekat supaya tidak dimulai di tengah kata + baru = max(akhir - tumpang, mulai + 1) + while baru < akhir and not teks[baru - 1].isspace(): + baru += 1 + if baru <= mulai: + baru = akhir + while baru < n and teks[baru].isspace(): + baru += 1 + mulai = baru + return posisi + + +METODE = {"tetap": potong_tetap, "cerdas": potong_cerdas} + + +# ========================================== +# ANALISIS CHUNK +# ========================================== + +def analisis_chunk(teks_penuh, mulai, akhir): + chunk = teks_penuh[mulai:akhir] + isi = chunk.rstrip() + + berakhir_kalimat = bool(POLA_AKHIR_KALIMAT.search(isi)) + + # Terpotong di tengah kata: karakter terakhir chunk dan karakter + # berikutnya di teks asli sama-sama huruf/angka. + ada_lanjutan = akhir < len(teks_penuh) + terpotong_kata = ( + ada_lanjutan + and chunk[-1:].isalnum() + and teks_penuh[akhir:akhir + 1].isalnum() + ) + + baris = [b for b in (x.strip() for x in chunk.splitlines()) if b] + baris_tabel = sum(1 for b in baris if "|" in b or "\t" in b or POLA_KOLOM.search(b)) + ada_tabel = baris_tabel >= 2 + + jumlah_simbol = len(POLA_SIMBOL_RUSAK.findall(chunk)) + + kata = isi.split() + return { + "panjang": len(chunk), + "berakhir_kalimat": berakhir_kalimat, + "terpotong_kata": terpotong_kata, + "ada_tabel": ada_tabel, + "jumlah_simbol_rusak": jumlah_simbol, + "ada_simbol_rusak": jumlah_simbol > 0, + "akhir_kata": " ".join(kata[-3:]) if kata else "", + } + + +def persen(bagian, total): + return f"{bagian / total * 100:.0f}%" if total else "-" + + +def rapikan(teks): + """Ubah baris baru/spasi berlebih jadi satu spasi agar enak dibaca di terminal.""" + return " ".join(teks.split()) + + +# ========================================== +# PROGRAM UTAMA +# ========================================== + +def main(): + if not FOLDER.is_dir(): + raise SystemExit(f"❌ ERROR: Folder '{FOLDER}/' tidak ditemukan!") + + daftar_pdf = sorted(FOLDER.glob("*.pdf")) # sorted -> hasil konsisten + if not daftar_pdf: + raise SystemExit(f"❌ ERROR: Tidak ada file PDF di folder '{FOLDER}/'!") + + print("=" * 70) + print(" LAB 1 — EKSPERIMEN CHUNKING") + print("=" * 70) + print(f"\n📁 Folder : {FOLDER.resolve()}") + print(f"📄 Jumlah PDF : {len(daftar_pdf)} file") + for p in daftar_pdf: + print(f" - {p.name}") + + # ---------- Langkah 1: baca PDF ---------- + pdf_path = daftar_pdf[0] + try: + teks, n_halaman = baca_pdf(pdf_path) + except (PdfReadError, ValueError, OSError) as e: + raise SystemExit(f"❌ ERROR: Gagal membaca '{pdf_path.name}': {e}") + + if not teks.strip(): + raise SystemExit(f"❌ ERROR: PDF '{pdf_path.name}' kosong / hasil scan (butuh OCR)!") + + print("\n" + "=" * 70) + print(" LANGKAH 1 — MEMBACA PDF") + print("=" * 70) + print(f"\n📄 File yang diuji : {pdf_path.name}") + print(f"📏 Total karakter : {len(teks):,}") + print(f"📑 Jumlah halaman : {n_halaman}") + print("\n--- 500 KARAKTER PERTAMA ---") + print(teks[:500]) + print("--- (akhir 500 karakter) ---") + + # ---------- Langkah 2: potong ---------- + print("\n" + "=" * 70) + print(" LANGKAH 2 — POTONG 3 UKURAN (OVERLAP 15%), 2 METODE") + print("=" * 70) + + posisi_semua = {} # (ukuran, metode) -> daftar posisi + print(f"\n{'Ukuran':<10}{'Overlap':<10}{'Chunk (tetap)':<16}{'Chunk (cerdas)':<16}") + print("-" * 52) + for ukuran, overlap in PENGATURAN: + for nama, fungsi in METODE.items(): + posisi_semua[(ukuran, nama)] = fungsi(teks, ukuran, overlap) + print(f"{ukuran:<10}{overlap:<10}" + f"{len(posisi_semua[(ukuran, 'tetap')]):<16}" + f"{len(posisi_semua[(ukuran, 'cerdas')]):<16}") + + # ---------- Langkah 3: contoh chunk (metode cerdas) ---------- + print("\n" + "=" * 70) + print(f" LANGKAH 3 — CONTOH {JUMLAH_SAMPEL} CHUNK ACAK TIAP UKURAN (metode cerdas)") + print("=" * 70) + + for ukuran, _ in PENGATURAN: + posisi = posisi_semua[(ukuran, "cerdas")] + print(f"\n{'#' * 70}\n## UKURAN MAKS = {ukuran} KARAKTER ({len(posisi)} chunk)\n{'#' * 70}") + + for i, (m, a) in enumerate(random.sample(posisi, min(JUMLAH_SAMPEL, len(posisi))), 1): + info = analisis_chunk(teks, m, a) + print(f"\n--- Contoh {i} (posisi {m}–{a}) ---") + print(f" Panjang : {info['panjang']} karakter") + print(f" Berakhir kalimat : {'✅ YA' if info['berakhir_kalimat'] else '❌ TIDAK'}") + print(f" Putus di tg. kata : {'❌ YA' if info['terpotong_kata'] else '✅ tidak'}") + print(f" Pola tabel : {'⚠️ YA' if info['ada_tabel'] else '✅ tidak'}") + simbol = f"⚠️ {info['jumlah_simbol_rusak']} simbol" if info["ada_simbol_rusak"] else "✅ tidak" + print(f" Simbol rusak : {simbol}") + print(f" Awal : {rapikan(teks[m:m + 100])}") + print(f" Akhir : ...{rapikan(teks[max(m, a - 100):a])}") + + # ---------- Langkah 4: rekap (semua chunk) ---------- + print("\n" + "=" * 70) + print(" LANGKAH 4 — REKAP & KESIMPULAN (dihitung dari semua chunk)") + print("=" * 70) + + ringkasan = {} + for (ukuran, nama), posisi in posisi_semua.items(): + semua = [analisis_chunk(teks, m, a) for m, a in posisi] + ringkasan[(ukuran, nama)] = { + "jumlah_chunk": len(semua), + "terpotong_kalimat": sum(not x["berakhir_kalimat"] for x in semua), + "terpotong_kata": sum(x["terpotong_kata"] for x in semua), + "ada_tabel": sum(x["ada_tabel"] for x in semua), + "ada_simbol_rusak": sum(x["ada_simbol_rusak"] for x in semua), + "rata_panjang": sum(x["panjang"] for x in semua) / len(semua), + } + + kolom = (f"{'Ukuran':<8}{'Metode':<8}{'Chunk':<7}{'Rata2 pjg':<11}" + f"{'Putus kalimat':<16}{'Putus kata':<14}{'Tabel':<12}{'Simbol':<10}") + print("\n" + kolom) + print("-" * len(kolom)) + for (ukuran, nama), r in ringkasan.items(): + n = r["jumlah_chunk"] + + def sel(k): + return f"{r[k]} ({persen(r[k], n)})" + + print(f"{ukuran:<8}{nama:<8}{n:<7}{r['rata_panjang']:<11.0f}" + f"{sel('terpotong_kalimat'):<16}{sel('terpotong_kata'):<14}" + f"{sel('ada_tabel'):<12}{sel('ada_simbol_rusak'):<10}") + + def jml(nama, k): + return sum(r[k] for (u, nm), r in ringkasan.items() if nm == nama) + + tot = {nm: jml(nm, "jumlah_chunk") for nm in METODE} + jumlah_cerdas = [ringkasan[(u, "cerdas")]["jumlah_chunk"] for u, _ in PENGATURAN] + + print("\n--- KESIMPULAN OTOMATIS ---") + print(f"1. Jumlah chunk (metode cerdas) per ukuran: {' → '.join(map(str, jumlah_cerdas))} " + f"({'menurun' if jumlah_cerdas == sorted(jumlah_cerdas, reverse=True) else 'tidak monoton'} " + f"seiring ukuran bertambah).") + print(f"2. Chunk putus di tengah kata: tetap {jml('tetap', 'terpotong_kata')}/{tot['tetap']} " + f"({persen(jml('tetap', 'terpotong_kata'), tot['tetap'])}) → cerdas " + f"{jml('cerdas', 'terpotong_kata')}/{tot['cerdas']} " + f"({persen(jml('cerdas', 'terpotong_kata'), tot['cerdas'])}).") + print(f"3. Chunk tidak berakhir di akhir kalimat: tetap {jml('tetap', 'terpotong_kalimat')}/{tot['tetap']} " + f"({persen(jml('tetap', 'terpotong_kalimat'), tot['tetap'])}) → cerdas " + f"{jml('cerdas', 'terpotong_kalimat')}/{tot['cerdas']} " + f"({persen(jml('cerdas', 'terpotong_kalimat'), tot['cerdas'])}).") + print(f"4. Chunk dengan pola tabel (metode cerdas): {jml('cerdas', 'ada_tabel')}/{tot['cerdas']} " + f"({persen(jml('cerdas', 'ada_tabel'), tot['cerdas'])}); " + f"dengan simbol rusak: {jml('cerdas', 'ada_simbol_rusak')}/{tot['cerdas']} " + f"({persen(jml('cerdas', 'ada_simbol_rusak'), tot['cerdas'])}).") + if jml("cerdas", "terpotong_kalimat") / tot["cerdas"] > 0.5: + print("5. Sebagian besar chunk masih tidak berakhir dengan tanda baca akhir kalimat " + "(baris PDF sering berhenti di judul/daftar/tabel); pertimbangkan pemotongan " + "per halaman atau per bagian/judul.") + else: + print("5. Metode cerdas menjaga sebagian besar chunk berakhir di batas kalimat.") + + print("\n" + "=" * 70) + print(" ✅ Eksperimen chunking selesai.") + print("=" * 70) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/lab2_indeks_cari.py b/lab2_indeks_cari.py new file mode 100644 index 0000000..413106a --- /dev/null +++ b/lab2_indeks_cari.py @@ -0,0 +1,386 @@ + +from pathlib import Path +from pypdf import PdfReader +import chromadb +from chromadb.utils import embedding_functions +import random +import time + +# ========================================== +# LAB 2 - BANGUN INDEKS & CARI +# ========================================== + +print("=" * 65) +print("LAB 2 - BANGUN INDEKS & CARI") +print("=" * 65, flush=True) + +# ========================================== +# KONFIGURASI +# ========================================== + +FOLDER_PDF = Path("dokumen") +DATABASE = "./chroma_db" + +UKURAN_CHUNK = 800 +OVERLAP = 100 + +# ========================================== +# FUNGSI MEMBACA PDF +# ========================================== + +def baca_pdf(path): + + reader = PdfReader(path) + + halaman = [] + + for nomor, page in enumerate(reader.pages, start=1): + + teks = page.extract_text() or "" + + halaman.append(teks) + + return "\n".join(halaman) + + +# ========================================== +# FUNGSI CHUNKING +# ========================================== + +def potong(teks, ukuran=800, tumpang=100): + + hasil = [] + mulai = 0 + + if ukuran <= 0 or tumpang < 0 or tumpang >= ukuran: + raise ValueError("Pengaturan chunk tidak valid.") + + while mulai < len(teks): + + chunk = teks[mulai:mulai + ukuran] + + if chunk.strip(): + hasil.append(chunk) + + mulai += ukuran - tumpang + + return hasil + + +# ========================================== +# EMBEDDING MODEL +# ========================================== + +print("\n[1] Memuat model embedding...", flush=True) + +emb = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="paraphrase-multilingual-MiniLM-L12-v2", + device="cpu" +) + +print("Model embedding berhasil dimuat!", flush=True) + + +# ========================================== +# KONEKSI CHROMADB +# ========================================== + +print("\n[2] Menghubungkan database ChromaDB...", flush=True) + +client = chromadb.PersistentClient(path=DATABASE) + +print("Database berhasil dihubungkan!", flush=True) + + +# ========================================== +# LANGKAH 1 - INDEXING PDF +# ========================================== + +print("\n[3] Memeriksa file PDF...", flush=True) + +daftar_pdf = sorted(FOLDER_PDF.glob("*.pdf")) + +if not 2 <= len(daftar_pdf) <= 3: + + raise SystemExit( + f"ERROR: Harus tersedia 2-3 PDF. " + f"Saat ini ditemukan {len(daftar_pdf)} PDF." + ) + +print(f"Ditemukan {len(daftar_pdf)} file PDF.", flush=True) + +# Membaca dan memotong semua PDF +data_index = [] + +for pdf in daftar_pdf: + + print(f"\nMembaca: {pdf.name}", flush=True) + + teks = baca_pdf(pdf) + + if not teks.strip(): + print("PDF tidak memiliki teks yang bisa diekstrak.") + continue + + chunks = potong( + teks, + ukuran=UKURAN_CHUNK, + tumpang=OVERLAP + ) + + data_index.append((pdf, chunks)) + + print( + f"Berhasil membaca {len(chunks)} chunk.", + flush=True + ) + +if not data_index: + raise SystemExit("Tidak ada PDF yang berhasil dibaca.") + +# Menghapus koleksi lama agar tidak ada data duplikat +print("\nMembersihkan indeks lama...", flush=True) + +try: + client.delete_collection("pedoman") +except Exception: + pass + +# Membuat koleksi baru +koleksi = client.create_collection( + name="pedoman", + embedding_function=emb, + metadata={"hnsw:space": "cosine"} +) + +print("Koleksi baru berhasil dibuat!", flush=True) + + +# ========================================== +# MEMASUKKAN DATA KE DATABASE +# ========================================== + +print("\n=== LAPORAN INDEXING ===", flush=True) + +total_chunk = 0 + +for pdf, chunks in data_index: + + ids = [ + f"{pdf.stem}-{i}" + for i in range(len(chunks)) + ] + + metadatas = [ + { + "sumber": pdf.name, + "chunk": i + } + for i in range(len(chunks)) + ] + + print( + f"Mengindeks {pdf.name}...", + flush=True + ) + + # Embedding dilakukan secara bertahap + batch_size = 32 + + for mulai in range(0, len(chunks), batch_size): + + batch_chunks = chunks[mulai:mulai + batch_size] + + batch_ids = ids[mulai:mulai + batch_size] + + batch_meta = metadatas[mulai:mulai + batch_size] + + koleksi.upsert( + ids=batch_ids, + documents=batch_chunks, + metadatas=batch_meta + ) + + print( + f" Proses: {min(mulai + batch_size, len(chunks))}" + f"/{len(chunks)} chunk", + flush=True + ) + + total_chunk += len(chunks) + + print( + f"{pdf.name} -> {len(chunks)} chunk", + flush=True + ) + +print("\nTOTAL SELURUH CHUNK:", total_chunk, flush=True) + +print( + "Jumlah data di ChromaDB:", + koleksi.count(), + flush=True +) + + +# ========================================== +# FUNGSI PENCARIAN QUERY +# ========================================== + +def tampilkan_query(q, label): + + print("\n" + "-" * 65) + print(f"[{label}] {q}") + print("-" * 65, flush=True) + + try: + + jumlah_data = koleksi.count() + + if jumlah_data == 0: + print("Database kosong!") + return + + # Tahap 1: Membuat embedding pertanyaan + print( + " [1/3] Membuat embedding pertanyaan...", + flush=True + ) + + vektor = emb([q]) + + print( + " [2/3] Mencari chunk yang relevan...", + flush=True + ) + + # Tahap 2: Pencarian menggunakan vektor + hasil = koleksi.query( + query_embeddings=vektor, + n_results=min(3, jumlah_data), + include=[ + "documents", + "metadatas", + "distances" + ] + ) + + print( + " [3/3] Menampilkan hasil pencarian...", + flush=True + ) + + dokumen = hasil["documents"][0] + metadata = hasil["metadatas"][0] + jarak = hasil["distances"][0] + + for nomor, (doc, meta, dist) in enumerate( + zip(dokumen, metadata, jarak), + start=1 + ): + + print(f"\n HASIL {nomor}") + + print(f" Sumber : {meta['sumber']}") + print(f" Chunk : {meta['chunk']}") + print(f" Jarak : {dist:.4f}") + + print("\n Isi chunk:") + print(" ", doc[:250].replace("\n", " ")) + + # Evaluasi awal berdasarkan jarak cosine + if dist < 0.3: + print(" Kemiripan: Relatif tinggi") + elif dist < 0.6: + print(" Kemiripan: Sedang, perlu diperiksa") + else: + print(" Kemiripan: Relatif rendah") + + print("\nQuery berhasil diselesaikan!", flush=True) + + except Exception as error: + + print( + f"\nERROR SAAT QUERY: {error}", + flush=True + ) + + +# ========================================== +# LANGKAH 2 - 5 PERTANYAAN LANGSUNG +# ========================================== + +pertanyaan = [ + + "What is the basic concept of probability?", + + "What is the multiplication rule?", + + "What is the addition rule?", + + "What is a sample space?", + + "What is the difference between classical and empirical probability?" + +] + +print("\n" + "=" * 65) +print("LANGKAH 2 - LIMA PERTANYAAN LANGSUNG") +print("=" * 65, flush=True) + +for nomor, q in enumerate(pertanyaan, start=1): + + print( + f"\nMemproses pertanyaan {nomor}/5...", + flush=True + ) + + tampilkan_query(q, "PERTANYAAN LANGSUNG") + + time.sleep(0.3) + + +# ========================================== +# LANGKAH 3 - 2 PERTANYAAN PARAFRASE +# ========================================== + +parafrase = [ + + "How do you find the chance of two events happening together?", + + "What is the collection of all possible outcomes in an experiment?" + +] + +print("\n" + "=" * 65) +print("LANGKAH 3 - DUA PERTANYAAN PARAFRASE") +print("=" * 65, flush=True) + +for nomor, q in enumerate(parafrase, start=1): + + print( + f"\nMemproses parafrase {nomor}/2...", + flush=True + ) + + tampilkan_query(q, "PERTANYAAN PARAFRASE") + + time.sleep(0.3) + + +# ========================================== +# KESIMPULAN +# ========================================== + +print("\n" + "=" * 65) +print("KESIMPULAN EKSPERIMEN LAB 2") +print("=" * 65) + +print(f"Jumlah PDF : {len(data_index)}") +print(f"Total chunk : {total_chunk}") +print(f"Pertanyaan biasa : {len(pertanyaan)}") +print(f"Pertanyaan parafrase: {len(parafrase)}") + +print("\nIndexing dan pencarian selesai!") +print("Silakan periksa relevansi setiap hasil pencarian.") +print("=" * 65, flush=True) diff --git a/lab3_hit_at_k.py b/lab3_hit_at_k.py new file mode 100644 index 0000000..941ba1a --- /dev/null +++ b/lab3_hit_at_k.py @@ -0,0 +1,224 @@ + +from pathlib import Path +from pypdf import PdfReader +import chromadb +from chromadb.utils import embedding_functions + +# ========================================== +# 1. MEMBACA DOKUMEN PDF +# ========================================== + +def baca_pdf(path): + reader = PdfReader(path) + return "\n".join((p.extract_text() or "") for p in reader.pages) + + +# ========================================== +# 2. MEMBAGI DOKUMEN MENJADI CHUNK +# ========================================== + +def potong(teks, ukuran=800, tumpang=100): + hasil = [] + mulai = 0 + + while mulai < len(teks): + hasil.append(teks[mulai:mulai + ukuran]) + mulai += ukuran - tumpang + + return [h for h in hasil if h.strip()] + + +# ========================================== +# 3. PENGATURAN EMBEDDING DAN DATABASE +# ========================================== + +emb = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="paraphrase-multilingual-MiniLM-L12-v2" +) + +client = chromadb.PersistentClient(path="./chroma_db") + + +# ========================================== +# 4. MEMBUAT KOLEKSI +# ========================================== + +def buat_koleksi(nama, ukuran, tumpang): + + try: + client.delete_collection(nama) + except Exception: + pass + + kol = client.get_or_create_collection( + name=nama, + embedding_function=emb, + metadata={"hnsw:space": "cosine"} + ) + + for pdf in Path("dokumen").glob("*.pdf"): + + teks = baca_pdf(pdf) + chunks = potong(teks, ukuran, tumpang) + + if not chunks: + continue + + kol.upsert( + ids=[ + f"{nama}-{pdf.stem}-{i}" + for i in range(len(chunks)) + ], + documents=chunks, + metadatas=[ + {"sumber": pdf.name, "chunk": i} + for i in range(len(chunks)) + ] + ) + + return kol + + +# ========================================== +# 5. 15 PERTANYAAN DAN KATA KUNCI +# ========================================== + +UJI = [ + ("What is a sample space?", "sample space"), + ("What is classical probability?", "classical probability"), + ("What is empirical probability?", "empirical probability"), + ("What is subjective probability?", "subjective probability"), + ("What is a simple event?", "simple event"), + ("What is the multiplication rule?", "multiplication rule"), + ("What is the addition rule?", "addition rule"), + ("What are mutually exclusive events?", "mutually exclusive"), + ("What is conditional probability?", "conditional probability"), + ("What is the law of large numbers?", "law of large numbers"), + ("What is a discrete probability distribution?", "discrete probability distribution"), + ("What is a binomial experiment?", "binomial experiment"), + ("What is the fundamental counting principle?", "fundamental counting principle"), + ("What is the difference between permutations and combinations?", "permutation"), + ("What is a random variable?", "random variable"), +] + + +# ========================================== +# 6. FUNGSI HIT@K +# ========================================== + +def hit_at_k(koleksi, uji, k): + + benar = 0 + jumlah = min(k, koleksi.count()) + + if jumlah == 0: + return 0 + + for pertanyaan, kunci in uji: + + docs = koleksi.query( + query_texts=[pertanyaan], + n_results=jumlah + )["documents"][0] + + if any(kunci.lower() in d.lower() for d in docs): + benar += 1 + + return benar / len(uji) + + +# ========================================== +# 7. MEMBUAT DUA KOLEKSI BERBEDA +# ========================================== + +kol_500 = buat_koleksi("chunk_500", 500, 75) +kol_1500 = buat_koleksi("chunk_1500", 1500, 225) + + +# ========================================== +# 8. MENAMPILKAN TABEL HASIL HIT@K +# ========================================== + +print("\n========== TABEL HIT@K ==========") + +print(f"{'k':<5}{'Chunk 500':<15}{'Chunk 1500':<15}") + +hasil = {} + +for k in (1, 3, 5): + + h500 = hit_at_k(kol_500, UJI, k) + h1500 = hit_at_k(kol_1500, UJI, k) + + hasil[k] = (h500, h1500) + + print(f"{k:<5}{h500:<15.2f}{h1500:<15.2f}") + + +# ========================================== +# 9. MENAMPILKAN PERTANYAAN YANG GAGAL +# ========================================== + +print("\n========== PERTANYAAN GAGAL DI k=3 ==========") + +gagal = [] + +for pertanyaan, kunci in UJI: + + docs_500 = kol_500.query( + query_texts=[pertanyaan], + n_results=min(3, kol_500.count()) + )["documents"][0] + + docs_1500 = kol_1500.query( + query_texts=[pertanyaan], + n_results=min(3, kol_1500.count()) + )["documents"][0] + + ok_500 = any(kunci.lower() in d.lower() for d in docs_500) + ok_1500 = any(kunci.lower() in d.lower() for d in docs_1500) + + if not ok_500 or not ok_1500: + + gagal.append(pertanyaan) + + print(f"\nPertanyaan: {pertanyaan}") + print(f"Kata kunci: {kunci}") + print(f"Chunk 500 : {'Berhasil' if ok_500 else 'Gagal'}") + print(f"Chunk 1500: {'Berhasil' if ok_1500 else 'Gagal'}") + +if not gagal: + print("Semua pertanyaan berhasil ditemukan pada kedua koleksi.") + + +# ========================================== +# 10. KESIMPULAN OTOMATIS +# ========================================== + +print("\n========== KESIMPULAN ==========") + +rata_500 = sum(v[0] for v in hasil.values()) / len(hasil) +rata_1500 = sum(v[1] for v in hasil.values()) / len(hasil) + +print(f"Rata-rata Hit@K Chunk 500 : {rata_500:.2f}") +print(f"Rata-rata Hit@K Chunk 1500 : {rata_1500:.2f}") + +if rata_500 > rata_1500: + print("Chunk 500 memiliki hasil pencarian lebih baik.") +elif rata_1500 > rata_500: + print("Chunk 1500 memiliki hasil pencarian lebih baik.") +else: + print("Kedua ukuran chunk memiliki hasil rata-rata yang sama.") + +print( + f"Jumlah pertanyaan yang gagal pada minimal satu koleksi " + f"di k=3: {len(gagal)} dari {len(UJI)}." +) + +print( + "Kegagalan pencarian dapat disebabkan oleh perbedaan istilah, " + "informasi yang tersebar di beberapa chunk, atau hasil embedding " + "yang kurang relevan." +) + +print("\n========== PENGUJIAN SELESAI ==========")