Initial commit
This commit is contained in:
angelicatesvara07-crypto
+55
@@ -0,0 +1,55 @@
|
|||||||
|
|
||||||
|
from reportlab.lib.pagesizes import A4
|
||||||
|
from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
|
||||||
|
from reportlab.lib.styles import getSampleStyleSheet
|
||||||
|
|
||||||
|
dokumen = [
|
||||||
|
{
|
||||||
|
"nama": "akademik.pdf",
|
||||||
|
"judul": "Panduan Akademik Mahasiswa",
|
||||||
|
"isi": [
|
||||||
|
("Cuti Akademik", "Mahasiswa dapat mengajukan cuti akademik paling lama dua semester berturut-turut dengan persetujuan dekan. Permohonan cuti diajukan sebelum masa registrasi semester berjalan dimulai."),
|
||||||
|
("Kehadiran", "Mahasiswa wajib mengikuti minimal 75 persen dari total pertemuan perkuliahan agar dapat mengikuti ujian akhir semester."),
|
||||||
|
("Nilai", "Nilai D pada mata kuliah wajib dapat diperbaiki dengan mengulang mata kuliah tersebut pada semester berikutnya."),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nama": "keuangan.pdf",
|
||||||
|
"judul": "Panduan Administrasi Keuangan",
|
||||||
|
"isi": [
|
||||||
|
("Pembayaran Kuliah", "Pembayaran biaya kuliah dilakukan paling lambat dua minggu sebelum perkuliahan dimulai melalui bagian keuangan universitas."),
|
||||||
|
("Keterlambatan", "Mahasiswa yang terlambat melakukan pembayaran wajib menghubungi bagian administrasi keuangan untuk memperoleh informasi penyelesaian administrasi."),
|
||||||
|
("Bukti Pembayaran", "Mahasiswa harus menyimpan bukti pembayaran sebagai dokumen administrasi selama masa studi."),
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nama": "kemahasiswaan.pdf",
|
||||||
|
"judul": "Panduan Layanan Kemahasiswaan",
|
||||||
|
"isi": [
|
||||||
|
("Beasiswa", "Mahasiswa dapat mengajukan beasiswa dengan memenuhi persyaratan akademik dan administratif yang ditentukan universitas."),
|
||||||
|
("Organisasi", "Mahasiswa diperbolehkan mengikuti organisasi kemahasiswaan selama tidak mengganggu kewajiban akademik."),
|
||||||
|
("Konsultasi Akademik", "Mahasiswa dapat berkonsultasi dengan dosen wali mengenai kesulitan belajar dan perencanaan mata kuliah."),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
styles = getSampleStyleSheet()
|
||||||
|
|
||||||
|
for item in dokumen:
|
||||||
|
path = "dokumen/" + item["nama"]
|
||||||
|
pdf = SimpleDocTemplate(path, pagesize=A4)
|
||||||
|
|
||||||
|
konten = [
|
||||||
|
Paragraph(item["judul"], styles["Title"]),
|
||||||
|
Spacer(1, 20)
|
||||||
|
]
|
||||||
|
|
||||||
|
for judul, isi in item["isi"]:
|
||||||
|
konten.append(Paragraph(judul, styles["Heading2"]))
|
||||||
|
konten.append(Paragraph(isi, styles["BodyText"]))
|
||||||
|
konten.append(Spacer(1, 15))
|
||||||
|
|
||||||
|
pdf.build(konten)
|
||||||
|
print("Berhasil membuat:", path)
|
||||||
|
|
||||||
|
print("\nSemua PDF berhasil dibuat!")
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
import chromadb
|
||||||
|
from chromadb.utils import embedding_functions
|
||||||
|
|
||||||
|
# Panggil model dan database yang sama
|
||||||
|
emb = embedding_functions.SentenceTransformerEmbeddingFunction(
|
||||||
|
model_name="paraphrase-multilingual-MiniLM-L12-v2")
|
||||||
|
client = chromadb.PersistentClient(path="./chroma_db")
|
||||||
|
koleksi = client.get_or_create_collection("pedoman", embedding_function=emb)
|
||||||
|
|
||||||
|
# Daftar pertanyaan uji
|
||||||
|
daftar_pertanyaan = [
|
||||||
|
"Apa syarat cuti akademik?",
|
||||||
|
"Berapa batas kehadiran untuk ujian?",
|
||||||
|
"Bolehkah berhenti kuliah sementara?", # Pertanyaan parafrase (tanpa kata 'cuti')
|
||||||
|
"Apa sanksi jika telat bayar kuliah?" # Pertanyaan parafrase (tanpa kata 'keuangan')
|
||||||
|
]
|
||||||
|
|
||||||
|
for q in daftar_pertanyaan:
|
||||||
|
print("=" * 60)
|
||||||
|
print("PERTANYAAN:", q)
|
||||||
|
# Ambil 2 hasil terdekat
|
||||||
|
hasil = koleksi.query(query_texts=[q], n_results=2)
|
||||||
|
|
||||||
|
for doc, meta, jarak in zip(hasil["documents"][0], hasil["metadatas"][0], hasil["distances"][0]):
|
||||||
|
# Jarak (distance) semakin kecil berarti semakin mirip
|
||||||
|
print(f" [Sumber: {meta['sumber']} | Jarak: {jarak:.3f}]")
|
||||||
|
print(" Teks:", doc[:120].replace("\n", " "), "...")
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
from pypdf import PdfReader
|
||||||
|
|
||||||
|
for pdf in Path("dokumen").glob("*.pdf"):
|
||||||
|
r = PdfReader(pdf)
|
||||||
|
teks = "\n".join((p.extract_text() or "") for p in r.pages)
|
||||||
|
print("=" * 50)
|
||||||
|
print("File :", pdf.name)
|
||||||
|
print("Halaman:", len(r.pages))
|
||||||
|
print("Panjang teks:", len(teks))
|
||||||
|
print("500 karakter pertama:")
|
||||||
|
print(teks[:500])
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,74 @@
|
|||||||
|
%PDF-1.4
|
||||||
|
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||||
|
1 0 obj
|
||||||
|
<<
|
||||||
|
/F1 2 0 R /F2 3 0 R
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
2 0 obj
|
||||||
|
<<
|
||||||
|
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
3 0 obj
|
||||||
|
<<
|
||||||
|
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
4 0 obj
|
||||||
|
<<
|
||||||
|
/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
|
||||||
|
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||||
|
>> /Rotate 0 /Trans <<
|
||||||
|
|
||||||
|
>>
|
||||||
|
/Type /Page
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
5 0 obj
|
||||||
|
<<
|
||||||
|
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
6 0 obj
|
||||||
|
<<
|
||||||
|
/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||||
|
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
7 0 obj
|
||||||
|
<<
|
||||||
|
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
8 0 obj
|
||||||
|
<<
|
||||||
|
/Filter [ /ASCII85Decode /FlateDecode ] /Length 566
|
||||||
|
>>
|
||||||
|
stream
|
||||||
|
Gat=fbAQ&g&4Q?mMHRLkU5H*ae&sq<9cln,rP0gljcMsbTh#=bq[R=CZ%Kta%0;,IEpas>\H6gN6s:ZA^m>Blh?sl&W;_;qZ3%$JE*(;UAn&E`P-?E(-KSEO"Wugi^bmpN'pBUR)O=QWOATbtLdH-MQ0V]GEM[(BJ5ui\n@:o_%<_.R2`T1RN$"^-Tt5g0P6,X^2h.a-LiR7le^79\s3YD?*\:ffo<6!ZeCC7Z\(m>3AIC5=0[r41,SK_5Xe9Wg*fRN:[\71PO3o!Z:j1KTD4jD:<s-=4N?5`U:i#6H$iW&dTV?s"F?AAo2eX;e#Ztf,c/:lLZe]Hb_*ZO_39XQ?_19?&_<H[Yn$onAWttT26mhOsfi]^lrpE<FIcX,7lO>=a0+WHR\WiHB095'1>H6.3W/HlK=YeXIWeA<T+eU(L>L/eOj0a`&q6(/hZ&Aj&d-\k7NT3G\ZkT]c)fcVMZTQFm]+H\\>oW<@`+(pqRU7*&U^c3@\I(>q*aqiV7(^dW+sp>8FYkF(+m3faerOToA6s>kUY:Cm311AZrJP;+(@1!S_X&AN=&G^~>endstream
|
||||||
|
endobj
|
||||||
|
xref
|
||||||
|
0 9
|
||||||
|
0000000000 65535 f
|
||||||
|
0000000061 00000 n
|
||||||
|
0000000102 00000 n
|
||||||
|
0000000209 00000 n
|
||||||
|
0000000321 00000 n
|
||||||
|
0000000524 00000 n
|
||||||
|
0000000592 00000 n
|
||||||
|
0000000872 00000 n
|
||||||
|
0000000931 00000 n
|
||||||
|
trailer
|
||||||
|
<<
|
||||||
|
/ID
|
||||||
|
[<430fe01d3101a78d4a66e001f226174a><430fe01d3101a78d4a66e001f226174a>]
|
||||||
|
% ReportLab generated PDF document -- digest (opensource)
|
||||||
|
|
||||||
|
/Info 6 0 R
|
||||||
|
/Root 5 0 R
|
||||||
|
/Size 9
|
||||||
|
>>
|
||||||
|
startxref
|
||||||
|
1587
|
||||||
|
%%EOF
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
%PDF-1.4
|
||||||
|
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||||
|
1 0 obj
|
||||||
|
<<
|
||||||
|
/F1 2 0 R /F2 3 0 R
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
2 0 obj
|
||||||
|
<<
|
||||||
|
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
3 0 obj
|
||||||
|
<<
|
||||||
|
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
4 0 obj
|
||||||
|
<<
|
||||||
|
/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
|
||||||
|
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||||
|
>> /Rotate 0 /Trans <<
|
||||||
|
|
||||||
|
>>
|
||||||
|
/Type /Page
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
5 0 obj
|
||||||
|
<<
|
||||||
|
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
6 0 obj
|
||||||
|
<<
|
||||||
|
/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||||
|
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
7 0 obj
|
||||||
|
<<
|
||||||
|
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
8 0 obj
|
||||||
|
<<
|
||||||
|
/Filter [ /ASCII85Decode /FlateDecode ] /Length 516
|
||||||
|
>>
|
||||||
|
stream
|
||||||
|
Gatn"d8%PD&4PLPMKqT'Z&:1bO-8]7(cCOg%!3rID7A0R9)O%6.o\3#+u)gGA7t&mSWd7Y!C+H9QNb/]L>,3!@L2K26oYGqT#3qNHg?Mok?,S\'E]d3OsdaNpd18>GZM1l4BSi;&;4kqL;QJo*"2ld8hS-joO#rj2&_kB#9*Te<.ShlA@IBW$K;KRCUcBn@4/8V*4le[lfL]O,bmR@%K4SX1"r8"0-bf_^KisZcDuko4<k<J]][Xkg+Nd>DU:\C9SuuTRDM;X/'XL@&f9jk_+%Gu""SJ@L`-q,OQQ)!<$.<U$sCV&ZiXj.6:+EbkB1Q-n#Sbam@\O$0K=nf";g'FY':\e]NNk:OZ]gs*[ibA7@C3ap@M$N\on3fCB='9d(lY$$Nrs+C>[?cBj.uJ[G#(ZMZ`78r1_T\3(A+Y`c5#t>N`.DJ%meekooi2A1giX^G"JU1+B=\Yf3+p0P.J>b"^rEJof/?C9lNG<gb96h?Q%m35V?!):tE)_oZ="U%\k.V`73~>endstream
|
||||||
|
endobj
|
||||||
|
xref
|
||||||
|
0 9
|
||||||
|
0000000000 65535 f
|
||||||
|
0000000061 00000 n
|
||||||
|
0000000102 00000 n
|
||||||
|
0000000209 00000 n
|
||||||
|
0000000321 00000 n
|
||||||
|
0000000524 00000 n
|
||||||
|
0000000592 00000 n
|
||||||
|
0000000872 00000 n
|
||||||
|
0000000931 00000 n
|
||||||
|
trailer
|
||||||
|
<<
|
||||||
|
/ID
|
||||||
|
[<be4b92fe71d8c47f732a578c48a5c846><be4b92fe71d8c47f732a578c48a5c846>]
|
||||||
|
% ReportLab generated PDF document -- digest (opensource)
|
||||||
|
|
||||||
|
/Info 6 0 R
|
||||||
|
/Root 5 0 R
|
||||||
|
/Size 9
|
||||||
|
>>
|
||||||
|
startxref
|
||||||
|
1537
|
||||||
|
%%EOF
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
%PDF-1.4
|
||||||
|
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||||
|
1 0 obj
|
||||||
|
<<
|
||||||
|
/F1 2 0 R /F2 3 0 R
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
2 0 obj
|
||||||
|
<<
|
||||||
|
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
3 0 obj
|
||||||
|
<<
|
||||||
|
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
4 0 obj
|
||||||
|
<<
|
||||||
|
/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
|
||||||
|
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||||
|
>> /Rotate 0 /Trans <<
|
||||||
|
|
||||||
|
>>
|
||||||
|
/Type /Page
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
5 0 obj
|
||||||
|
<<
|
||||||
|
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
6 0 obj
|
||||||
|
<<
|
||||||
|
/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||||
|
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
7 0 obj
|
||||||
|
<<
|
||||||
|
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||||
|
>>
|
||||||
|
endobj
|
||||||
|
8 0 obj
|
||||||
|
<<
|
||||||
|
/Filter [ /ASCII85Decode /FlateDecode ] /Length 539
|
||||||
|
>>
|
||||||
|
stream
|
||||||
|
Gat=fb>,r/&4Q?lMHL]?8fX#F/4"k^5lrK"0HmS\`-+cnKDP=9dPQBN/Y]Nb7@r,%j!u83,MMUA"aJ8Lp/up%+e+$R7L<TYn<=Br#,9;28=1Z3,iu`#$4]I@T[EqF'n&.r[+m,8+e)MPgbbJ@d5^+i&S)%Rhp<)=*BY%%*;R04:G>drV9rR7p'KD0h!ERqP'9+eC.-dV3=(UUq@B8k]<c=,%c<MqEJ5.NRg)5>hrm(t@%@3Z7W`^=D:Hjf&*N#l5>\f;THJku./A1GL7AI#S6,jq"/_o;!3^ESJq^&$ejZ*6O:;C#X>4GjD+"nUeAU=OK+#NC\#MR@kXT8E9;7Vd7<B^-WN&t+hIl`2a/5d*mh$qW^+p0orWclN)acio-.@G$UAr&*auSPT4QO=mJdrXc=L3dn6a7+IQW\chKi)n62d97iZ11`&PqpMB@@+=1e+$![Vg"J2q(\*k_:E:4"[j.gGtWOF<U^IB("m/.lrLn3DK[XIG%n30"/kPVi?5lLRiFTb`-iY`oAl)3?),9p(\G%*1B~>endstream
|
||||||
|
endobj
|
||||||
|
xref
|
||||||
|
0 9
|
||||||
|
0000000000 65535 f
|
||||||
|
0000000061 00000 n
|
||||||
|
0000000102 00000 n
|
||||||
|
0000000209 00000 n
|
||||||
|
0000000321 00000 n
|
||||||
|
0000000524 00000 n
|
||||||
|
0000000592 00000 n
|
||||||
|
0000000872 00000 n
|
||||||
|
0000000931 00000 n
|
||||||
|
trailer
|
||||||
|
<<
|
||||||
|
/ID
|
||||||
|
[<bb972e76e34dfd7c99f5710a35d79b0d><bb972e76e34dfd7c99f5710a35d79b0d>]
|
||||||
|
% ReportLab generated PDF document -- digest (opensource)
|
||||||
|
|
||||||
|
/Info 6 0 R
|
||||||
|
/Root 5 0 R
|
||||||
|
/Size 9
|
||||||
|
>>
|
||||||
|
startxref
|
||||||
|
1560
|
||||||
|
%%EOF
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import chromadb
|
||||||
|
from chromadb.utils import embedding_functions
|
||||||
|
|
||||||
|
# Panggil DB dan Model
|
||||||
|
emb = embedding_functions.SentenceTransformerEmbeddingFunction(
|
||||||
|
model_name="paraphrase-multilingual-MiniLM-L12-v2")
|
||||||
|
client = chromadb.PersistentClient(path="./chroma_db")
|
||||||
|
koleksi = client.get_or_create_collection("pedoman", embedding_function=emb)
|
||||||
|
|
||||||
|
# Daftar (Pertanyaan, Kata Kunci Jawaban yang HARUS ada di chunk)
|
||||||
|
UJI = [
|
||||||
|
("Apa syarat cuti akademik?", "cuti akademik"),
|
||||||
|
("Berapa lama cuti akademik bisa diajukan?", "dua semester"),
|
||||||
|
("Berapa batas kehadiran untuk ujian?", "75 persen"),
|
||||||
|
("Bolehkah berhenti kuliah sementara?", "cuti akademik"),
|
||||||
|
("Bagaimana cara mengajukan beasiswa?", "beasiswa"),
|
||||||
|
("Apa sanksi jika telat bayar kuliah?", "terlambat melakukan pembayaran"),
|
||||||
|
("Kapan batas pembayaran biaya kuliah?", "dua minggu sebelum"),
|
||||||
|
("Apa yang harus disimpan setelah bayar?", "bukti pembayaran")
|
||||||
|
]
|
||||||
|
|
||||||
|
def hit_at_k(uji, k=3):
|
||||||
|
benar = 0
|
||||||
|
for pertanyaan, kunci in uji:
|
||||||
|
# Ambil k hasil teratas
|
||||||
|
hasil = koleksi.query(query_texts=[pertanyaan], n_results=k)["documents"][0]
|
||||||
|
# Cek apakah kunci ada di salah satu hasil
|
||||||
|
if any(kunci.lower() in d.lower() for d in hasil):
|
||||||
|
benar += 1
|
||||||
|
return benar / len(uji)
|
||||||
|
|
||||||
|
print(f"Total pertanyaan uji: {len(UJI)}\n")
|
||||||
|
print("Mengevaluasi Retrieval...")
|
||||||
|
for k in (1, 3, 5):
|
||||||
|
skor = hit_at_k(UJI, k)
|
||||||
|
print(f"Hit@{k} = {skor:.2f} (Benar {int(skor*len(UJI))} dari {len(UJI)} pertanyaan)")
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
from pypdf import PdfReader
|
||||||
|
import chromadb
|
||||||
|
from chromadb.utils import embedding_functions
|
||||||
|
|
||||||
|
def baca_pdf(path):
|
||||||
|
r = PdfReader(path)
|
||||||
|
return "\n".join((p.extract_text() or "") for p in r.pages)
|
||||||
|
|
||||||
|
def potong(teks, ukuran=500, tumpang=75):
|
||||||
|
hasil, mulai = [], 0
|
||||||
|
while mulai < len(teks):
|
||||||
|
hasil.append(teks[mulai:mulai + ukuran])
|
||||||
|
mulai += ukuran - tumpang
|
||||||
|
return [h for h in hasil if h.strip()]
|
||||||
|
|
||||||
|
print("Menyiapkan model embedding (unduh jika belum ada)...")
|
||||||
|
# Model multilingual agar bisa bahasa Indonesia
|
||||||
|
emb = embedding_functions.SentenceTransformerEmbeddingFunction(
|
||||||
|
model_name="paraphrase-multilingual-MiniLM-L12-v2")
|
||||||
|
|
||||||
|
client = chromadb.PersistentClient(path="./chroma_db")
|
||||||
|
koleksi = client.get_or_create_collection(
|
||||||
|
name="pedoman", embedding_function=emb,
|
||||||
|
metadata={"hnsw:space": "cosine"})
|
||||||
|
|
||||||
|
# Proses tiap PDF
|
||||||
|
for pdf in Path("dokumen").glob("*.pdf"):
|
||||||
|
teks = baca_pdf(pdf)
|
||||||
|
chunks = potong(teks, ukuran=500, tumpang=75) # Pakai ukuran 500
|
||||||
|
koleksi.upsert(
|
||||||
|
ids=[f"{pdf.stem}-{i}" for i in range(len(chunks))],
|
||||||
|
documents=chunks,
|
||||||
|
metadatas=[{"sumber": pdf.name, "chunk": i} for i in range(len(chunks))],
|
||||||
|
)
|
||||||
|
print(f"{pdf.name} → {len(chunks)} chunk disimpan.")
|
||||||
|
|
||||||
|
print("\nSelesai! Indeks tersimpan di folder 'chroma_db'.")
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
from pypdf import PdfReader
|
||||||
|
|
||||||
|
def baca_pdf(path):
|
||||||
|
r = PdfReader(path)
|
||||||
|
return "\n".join((p.extract_text() or "") for p in r.pages)
|
||||||
|
|
||||||
|
def potong(teks, ukuran=800, tumpang=100):
|
||||||
|
hasil, mulai = [], 0
|
||||||
|
while mulai < len(teks):
|
||||||
|
hasil.append(teks[mulai:mulai + ukuran])
|
||||||
|
mulai += ukuran - tumpang
|
||||||
|
return [h for h in hasil if h.strip()]
|
||||||
|
|
||||||
|
# Gabungkan semua teks PDF untuk simulasi dokumen besar
|
||||||
|
semua_teks = ""
|
||||||
|
for pdf in Path("dokumen").glob("*.pdf"):
|
||||||
|
semua_teks += baca_pdf(pdf) + "\n\n"
|
||||||
|
|
||||||
|
print("Total panjang teks semua PDF:", len(semua_teks), "karakter\n")
|
||||||
|
|
||||||
|
for ukuran in (500, 1000, 1500):
|
||||||
|
tumpang = int(ukuran * 0.15) # Overlap 15%
|
||||||
|
chunks = potong(semua_teks, ukuran, tumpang)
|
||||||
|
print(f"=== Ukuran: {ukuran}, Overlap: {tumpang} ===")
|
||||||
|
print(f"Jumlah chunk yang dihasilkan: {len(chunks)}")
|
||||||
|
# Cetak 100 karakter pertama dari chunk ke-1 dan terakhir
|
||||||
|
print(f"Chunk 1: {repr(chunks[0][:100])}...")
|
||||||
|
if len(chunks) > 1:
|
||||||
|
print(f"Chunk akhir: {repr(chunks[-1][:100])}...")
|
||||||
|
print("-" * 40)
|
||||||
Reference in New Issue
Block a user