84 lines
2.8 KiB
Python
84 lines
2.8 KiB
Python
"""
|
|
Lab 1 - Eksperimen Chunking (15 menit)
|
|
Hanya butuh library pypdf: pip install pypdf
|
|
|
|
Cara jalankan (pilih salah satu):
|
|
python lab_1.py -> otomatis pakai PDF pertama di folder dokumen/
|
|
python lab_1.py dokumen/panduan_akademik.pdf -> pakai PDF tertentu
|
|
python lab_1.py dokumen/panduan_akademik.pdf --overlap 0.2
|
|
"""
|
|
|
|
import argparse
|
|
import random
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from pypdf import PdfReader
|
|
|
|
UKURAN_LIST = [500, 1000, 1500]
|
|
|
|
|
|
def baca_pdf(path):
|
|
"""Baca seluruh teks dari satu file PDF."""
|
|
reader = PdfReader(str(path))
|
|
halaman = [(p.extract_text() or "") for p in reader.pages]
|
|
teks = "\n".join(halaman)
|
|
return re.sub(r"[ \t]+", " ", teks).strip()
|
|
|
|
|
|
def potong(teks, ukuran=1000, overlap=0.15):
|
|
"""Potong teks jadi chunk berukuran `ukuran` karakter dengan overlap (0-1)."""
|
|
langkah = max(1, int(ukuran * (1 - overlap)))
|
|
chunks = []
|
|
for mulai in range(0, len(teks), langkah):
|
|
potongan = teks[mulai:mulai + ukuran].strip()
|
|
if potongan:
|
|
chunks.append(potongan)
|
|
if mulai + ukuran >= len(teks):
|
|
break
|
|
return chunks
|
|
|
|
|
|
def cari_pdf_default():
|
|
daftar = sorted(Path("dokumen").glob("*.pdf"))
|
|
if not daftar:
|
|
raise SystemExit("Tidak ada PDF di folder dokumen/. Taruh PDF di sana atau tulis path-nya.")
|
|
return daftar[0]
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description="Lab 1: eksperimen chunking")
|
|
p.add_argument("pdf", nargs="?", help="Path PDF (kosongkan untuk pakai PDF pertama di dokumen/)")
|
|
p.add_argument("--overlap", type=float, default=0.15, help="Overlap 0.10 - 0.20 (default 0.15)")
|
|
args = p.parse_args()
|
|
|
|
path = Path(args.pdf) if args.pdf else cari_pdf_default()
|
|
print(f"File: {path}\n")
|
|
|
|
# 1. Baca PDF, cetak 500 karakter pertama
|
|
teks = baca_pdf(path)
|
|
print(f"=== 1. Hasil baca PDF ({len(teks)} karakter total) ===")
|
|
print(teks[:500])
|
|
|
|
# 2. Potong dengan tiga ukuran, catat jumlah chunk
|
|
print("\n=== 2. Jumlah chunk per pengaturan ===")
|
|
hasil = {}
|
|
for ukuran in UKURAN_LIST:
|
|
chunks = potong(teks, ukuran, args.overlap)
|
|
hasil[ukuran] = chunks
|
|
print(f" ukuran={ukuran:<5} overlap={args.overlap:.0%} -> {len(chunks)} chunk")
|
|
|
|
# 3. Lihat 3 chunk acak tiap ukuran
|
|
print("\n=== 3. Tiga chunk acak per ukuran (terpotong di tengah kalimat/tabel?) ===")
|
|
for ukuran, chunks in hasil.items():
|
|
print(f"\n--- ukuran {ukuran} ---")
|
|
for c in random.sample(chunks, min(3, len(chunks))):
|
|
awal_rapi = c[0].isupper() or c[0].isdigit()
|
|
akhir_rapi = c[-1] in ".!?:"
|
|
print(f"[awal rapi: {awal_rapi} | akhir rapi: {akhir_rapi}]")
|
|
print(c)
|
|
print("." * 40)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |