""" Lab 1 - Eksperimen Chunking (15 menit) Hanya butuh library pypdf: pip install pypdf Cara jalankan (pilih salah satu): python lab_1.py -> otomatis pakai PDF pertama di folder dokumen/ python lab_1.py dokumen/panduan_akademik.pdf -> pakai PDF tertentu python lab_1.py dokumen/panduan_akademik.pdf --overlap 0.2 """ import argparse import random import re from pathlib import Path from pypdf import PdfReader UKURAN_LIST = [500, 1000, 1500] def baca_pdf(path): """Baca seluruh teks dari satu file PDF.""" reader = PdfReader(str(path)) halaman = [(p.extract_text() or "") for p in reader.pages] teks = "\n".join(halaman) return re.sub(r"[ \t]+", " ", teks).strip() def potong(teks, ukuran=1000, overlap=0.15): """Potong teks jadi chunk berukuran `ukuran` karakter dengan overlap (0-1).""" langkah = max(1, int(ukuran * (1 - overlap))) chunks = [] for mulai in range(0, len(teks), langkah): potongan = teks[mulai:mulai + ukuran].strip() if potongan: chunks.append(potongan) if mulai + ukuran >= len(teks): break return chunks def cari_pdf_default(): daftar = sorted(Path("dokumen").glob("*.pdf")) if not daftar: raise SystemExit("Tidak ada PDF di folder dokumen/. Taruh PDF di sana atau tulis path-nya.") return daftar[0] def main(): p = argparse.ArgumentParser(description="Lab 1: eksperimen chunking") p.add_argument("pdf", nargs="?", help="Path PDF (kosongkan untuk pakai PDF pertama di dokumen/)") p.add_argument("--overlap", type=float, default=0.15, help="Overlap 0.10 - 0.20 (default 0.15)") args = p.parse_args() path = Path(args.pdf) if args.pdf else cari_pdf_default() print(f"File: {path}\n") # 1. Baca PDF, cetak 500 karakter pertama teks = baca_pdf(path) print(f"=== 1. Hasil baca PDF ({len(teks)} karakter total) ===") print(teks[:500]) # 2. Potong dengan tiga ukuran, catat jumlah chunk print("\n=== 2. Jumlah chunk per pengaturan ===") hasil = {} for ukuran in UKURAN_LIST: chunks = potong(teks, ukuran, args.overlap) hasil[ukuran] = chunks print(f" ukuran={ukuran:<5} overlap={args.overlap:.0%} -> {len(chunks)} chunk") # 3. Lihat 3 chunk acak tiap ukuran print("\n=== 3. Tiga chunk acak per ukuran (terpotong di tengah kalimat/tabel?) ===") for ukuran, chunks in hasil.items(): print(f"\n--- ukuran {ukuran} ---") for c in random.sample(chunks, min(3, len(chunks))): awal_rapi = c[0].isupper() or c[0].isdigit() akhir_rapi = c[-1] in ".!?:" print(f"[awal rapi: {awal_rapi} | akhir rapi: {akhir_rapi}]") print(c) print("." * 40) if __name__ == "__main__": main()