57 lines
2.0 KiB
Python
57 lines
2.0 KiB
Python
"""Lab 1 - Eksperimen chunking.
|
|
|
|
Pakai: python lab1_chunking.py dokumen/namafile.pdf
|
|
Hasil dicetak ke layar dan disimpan ke hasil_lab1.txt (bisa ditempel ke laporan).
|
|
"""
|
|
import random
|
|
import sys
|
|
from umum import baca_pdf, potong
|
|
|
|
UKURAN = [500, 1000, 1500]
|
|
OVERLAP_PERSEN = 0.15 # 15% (rentang yang disarankan 10-20%)
|
|
AKHIR_KALIMAT = (".", "!", "?", ":", ";")
|
|
|
|
if len(sys.argv) < 2:
|
|
sys.exit("Pakai: python lab1_chunking.py dokumen/namafile.pdf")
|
|
|
|
random.seed(42) # supaya contoh acak bisa diulang
|
|
teks = baca_pdf(sys.argv[1])
|
|
if not teks.strip():
|
|
sys.exit("Teks kosong. PDF kemungkinan hasil scan; pakai PDF berbasis teks.")
|
|
|
|
keluaran = []
|
|
def cetak(s=""):
|
|
print(s)
|
|
keluaran.append(s)
|
|
|
|
cetak("=== Langkah 1: cek teks terbaca ===")
|
|
cetak(f"Total karakter: {len(teks)}")
|
|
cetak(teks[:500])
|
|
cetak()
|
|
|
|
cetak("=== Langkah 2: jumlah chunk per pengaturan ===")
|
|
cetak(f"{'ukuran':>7} {'overlap':>8} {'jumlah chunk':>13} {'rata2 panjang':>14} {'% tdk berakhir kalimat':>24}")
|
|
hasil_per_ukuran = {}
|
|
for u in UKURAN:
|
|
t = int(u * OVERLAP_PERSEN)
|
|
chunks = potong(teks, u, t)
|
|
terpotong = sum(1 for c in chunks[:-1] if not c.rstrip().endswith(AKHIR_KALIMAT))
|
|
persen = 100 * terpotong / max(1, len(chunks) - 1)
|
|
rata = sum(len(c) for c in chunks) / len(chunks)
|
|
hasil_per_ukuran[u] = (chunks, t)
|
|
cetak(f"{u:>7} {t:>8} {len(chunks):>13} {rata:>14.0f} {persen:>23.0f}%")
|
|
cetak()
|
|
|
|
cetak("=== Langkah 3: 3 chunk acak per ukuran (awal 200 + akhir 100 karakter) ===")
|
|
for u, (chunks, t) in hasil_per_ukuran.items():
|
|
cetak(f"--- ukuran {u} (overlap {t}) ---")
|
|
for idx in sorted(random.sample(range(len(chunks)), min(3, len(chunks)))):
|
|
c = chunks[idx].replace("\n", " ")
|
|
cetak(f"[chunk {idx}] {c[:200]} ... {c[-100:]}")
|
|
cetak(f" berakhir di akhir kalimat? {'YA' if chunks[idx].rstrip().endswith(AKHIR_KALIMAT) else 'TIDAK (terpotong)'}")
|
|
cetak()
|
|
|
|
with open("hasil_lab1.txt", "w", encoding="utf-8") as f:
|
|
f.write("\n".join(keluaran))
|
|
print("Disimpan ke hasil_lab1.txt")
|