31 lines
1.0 KiB
Python
31 lines
1.0 KiB
Python
from pathlib import Path
|
|
from pypdf import PdfReader
|
|
|
|
def baca_pdf(path):
|
|
r = PdfReader(path)
|
|
return "\n".join((p.extract_text() or "") for p in r.pages)
|
|
|
|
def potong(teks, ukuran=800, tumpang=100):
|
|
hasil, mulai = [], 0
|
|
while mulai < len(teks):
|
|
hasil.append(teks[mulai:mulai + ukuran])
|
|
mulai += ukuran - tumpang
|
|
return [h for h in hasil if h.strip()]
|
|
|
|
# Gabungkan semua teks PDF untuk simulasi dokumen besar
|
|
semua_teks = ""
|
|
for pdf in Path("dokumen").glob("*.pdf"):
|
|
semua_teks += baca_pdf(pdf) + "\n\n"
|
|
|
|
print("Total panjang teks semua PDF:", len(semua_teks), "karakter\n")
|
|
|
|
for ukuran in (500, 1000, 1500):
|
|
tumpang = int(ukuran * 0.15) # Overlap 15%
|
|
chunks = potong(semua_teks, ukuran, tumpang)
|
|
print(f"=== Ukuran: {ukuran}, Overlap: {tumpang} ===")
|
|
print(f"Jumlah chunk yang dihasilkan: {len(chunks)}")
|
|
# Cetak 100 karakter pertama dari chunk ke-1 dan terakhir
|
|
print(f"Chunk 1: {repr(chunks[0][:100])}...")
|
|
if len(chunks) > 1:
|
|
print(f"Chunk akhir: {repr(chunks[-1][:100])}...")
|
|
print("-" * 40) |