68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
"""Fungsi bersama untuk Lab P-10 (RAG Part 1)."""
|
|
import re
|
|
from pathlib import Path
|
|
from pypdf import PdfReader
|
|
|
|
MODEL_EMBED = "paraphrase-multilingual-MiniLM-L12-v2" # multilingual, jalan lokal
|
|
DB_PATH = "./chroma_db"
|
|
FOLDER_DOKUMEN = Path("dokumen")
|
|
|
|
|
|
def baca_pdf_per_halaman(path):
|
|
"""Kembalikan list (nomor_halaman, teks). Halaman tanpa teks (scan) dilewati."""
|
|
reader = PdfReader(path)
|
|
hasil = []
|
|
for i, p in enumerate(reader.pages, start=1):
|
|
teks = re.sub(r"\s+", " ", p.extract_text() or "").strip() # rapikan baris baru PDF
|
|
if teks:
|
|
hasil.append((i, teks))
|
|
return hasil
|
|
|
|
|
|
def baca_pdf(path):
|
|
"""Seluruh teks PDF sebagai satu string."""
|
|
return "\n".join(t for _, t in baca_pdf_per_halaman(path))
|
|
|
|
|
|
def potong(teks, ukuran=800, tumpang=100):
|
|
"""Pemotongan ukuran tetap (karakter) dengan overlap."""
|
|
if tumpang >= ukuran:
|
|
raise ValueError("tumpang harus lebih kecil dari ukuran")
|
|
hasil, mulai = [], 0
|
|
while mulai < len(teks):
|
|
hasil.append(teks[mulai:mulai + ukuran])
|
|
if mulai + ukuran >= len(teks): # sisa teks sudah masuk chunk ini; jangan buat chunk ekor
|
|
break
|
|
mulai += ukuran - tumpang
|
|
return [h for h in hasil if h.strip()]
|
|
|
|
|
|
def nama_koleksi(ukuran):
|
|
return f"pedoman_{ukuran}"
|
|
|
|
|
|
def ambil_embedding():
|
|
"""Model embedding multilingual (diunduh sekali, lalu tersimpan di cache)."""
|
|
from chromadb.utils import embedding_functions
|
|
return embedding_functions.SentenceTransformerEmbeddingFunction(model_name=MODEL_EMBED)
|
|
|
|
|
|
def buka_klien():
|
|
import chromadb
|
|
return chromadb.PersistentClient(path=DB_PATH)
|
|
|
|
|
|
def buat_koleksi(klien, nama, emb):
|
|
"""Buat/ambil koleksi dengan jarak cosine (menyesuaikan versi ChromaDB)."""
|
|
try:
|
|
return klien.get_or_create_collection(
|
|
name=nama, embedding_function=emb, metadata={"hnsw:space": "cosine"})
|
|
except Exception:
|
|
return klien.get_or_create_collection(
|
|
name=nama, embedding_function=emb,
|
|
configuration={"hnsw": {"space": "cosine"}})
|
|
|
|
|
|
def ambil_koleksi(klien, nama, emb):
|
|
return klien.get_collection(name=nama, embedding_function=emb)
|