Tugas P-10: RAG Part 1 (Lab 1-3)

lab1_chunking.py

@@ -0,0 +1,56 @@
+"""Lab 1 - Eksperimen chunking.
+
+Pakai:  python lab1_chunking.py dokumen/namafile.pdf
+Hasil dicetak ke layar dan disimpan ke hasil_lab1.txt (bisa ditempel ke laporan).
+"""
+import random
+import sys
+from umum import baca_pdf, potong
+
+UKURAN = [500, 1000, 1500]
+OVERLAP_PERSEN = 0.15          # 15% (rentang yang disarankan 10-20%)
+AKHIR_KALIMAT = (".", "!", "?", ":", ";")
+
+if len(sys.argv) < 2:
+    sys.exit("Pakai: python lab1_chunking.py dokumen/namafile.pdf")
+
+random.seed(42)                # supaya contoh acak bisa diulang
+teks = baca_pdf(sys.argv[1])
+if not teks.strip():
+    sys.exit("Teks kosong. PDF kemungkinan hasil scan; pakai PDF berbasis teks.")
+
+keluaran = []
+def cetak(s=""):
+    print(s)
+    keluaran.append(s)
+
+cetak("=== Langkah 1: cek teks terbaca ===")
+cetak(f"Total karakter: {len(teks)}")
+cetak(teks[:500])
+cetak()
+
+cetak("=== Langkah 2: jumlah chunk per pengaturan ===")
+cetak(f"{'ukuran':>7} {'overlap':>8} {'jumlah chunk':>13} {'rata2 panjang':>14} {'% tdk berakhir kalimat':>24}")
+hasil_per_ukuran = {}
+for u in UKURAN:
+    t = int(u * OVERLAP_PERSEN)
+    chunks = potong(teks, u, t)
+    terpotong = sum(1 for c in chunks[:-1] if not c.rstrip().endswith(AKHIR_KALIMAT))
+    persen = 100 * terpotong / max(1, len(chunks) - 1)
+    rata = sum(len(c) for c in chunks) / len(chunks)
+    hasil_per_ukuran[u] = (chunks, t)
+    cetak(f"{u:>7} {t:>8} {len(chunks):>13} {rata:>14.0f} {persen:>23.0f}%")
+cetak()
+
+cetak("=== Langkah 3: 3 chunk acak per ukuran (awal 200 + akhir 100 karakter) ===")
+for u, (chunks, t) in hasil_per_ukuran.items():
+    cetak(f"--- ukuran {u} (overlap {t}) ---")
+    for idx in sorted(random.sample(range(len(chunks)), min(3, len(chunks)))):
+        c = chunks[idx].replace("\n", " ")
+        cetak(f"[chunk {idx}] {c[:200]} ... {c[-100:]}")
+        cetak(f"   berakhir di akhir kalimat? {'YA' if chunks[idx].rstrip().endswith(AKHIR_KALIMAT) else 'TIDAK (terpotong)'}")
+    cetak()
+
+with open("hasil_lab1.txt", "w", encoding="utf-8") as f:
+    f.write("\n".join(keluaran))
+print("Disimpan ke hasil_lab1.txt")