commit d16eabcfd830cd2174e0eaeaf5f87072af7a8a36 Author: angelicatesvara07-crypto Date: Mon Nov 2 10:50:42 2026 +0700 Initial commit diff --git a/buat_pdf.py b/buat_pdf.py new file mode 100644 index 0000000..0f54312 --- /dev/null +++ b/buat_pdf.py @@ -0,0 +1,55 @@ + +from reportlab.lib.pagesizes import A4 +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer +from reportlab.lib.styles import getSampleStyleSheet + +dokumen = [ + { + "nama": "akademik.pdf", + "judul": "Panduan Akademik Mahasiswa", + "isi": [ + ("Cuti Akademik", "Mahasiswa dapat mengajukan cuti akademik paling lama dua semester berturut-turut dengan persetujuan dekan. Permohonan cuti diajukan sebelum masa registrasi semester berjalan dimulai."), + ("Kehadiran", "Mahasiswa wajib mengikuti minimal 75 persen dari total pertemuan perkuliahan agar dapat mengikuti ujian akhir semester."), + ("Nilai", "Nilai D pada mata kuliah wajib dapat diperbaiki dengan mengulang mata kuliah tersebut pada semester berikutnya."), + ] + }, + { + "nama": "keuangan.pdf", + "judul": "Panduan Administrasi Keuangan", + "isi": [ + ("Pembayaran Kuliah", "Pembayaran biaya kuliah dilakukan paling lambat dua minggu sebelum perkuliahan dimulai melalui bagian keuangan universitas."), + ("Keterlambatan", "Mahasiswa yang terlambat melakukan pembayaran wajib menghubungi bagian administrasi keuangan untuk memperoleh informasi penyelesaian administrasi."), + ("Bukti Pembayaran", "Mahasiswa harus menyimpan bukti pembayaran sebagai dokumen administrasi selama masa studi."), + ] + }, + { + "nama": "kemahasiswaan.pdf", + "judul": "Panduan Layanan Kemahasiswaan", + "isi": [ + ("Beasiswa", "Mahasiswa dapat mengajukan beasiswa dengan memenuhi persyaratan akademik dan administratif yang ditentukan universitas."), + ("Organisasi", "Mahasiswa diperbolehkan mengikuti organisasi kemahasiswaan selama tidak mengganggu kewajiban akademik."), + ("Konsultasi Akademik", "Mahasiswa dapat berkonsultasi dengan dosen wali mengenai kesulitan belajar dan perencanaan mata kuliah."), + ] + } +] + +styles = getSampleStyleSheet() + +for item in dokumen: + path = "dokumen/" + item["nama"] + pdf = SimpleDocTemplate(path, pagesize=A4) + + konten = [ + Paragraph(item["judul"], styles["Title"]), + Spacer(1, 20) + ] + + for judul, isi in item["isi"]: + konten.append(Paragraph(judul, styles["Heading2"])) + konten.append(Paragraph(isi, styles["BodyText"])) + konten.append(Spacer(1, 15)) + + pdf.build(konten) + print("Berhasil membuat:", path) + +print("\nSemua PDF berhasil dibuat!") diff --git a/cari.py b/cari.py new file mode 100644 index 0000000..72f9169 --- /dev/null +++ b/cari.py @@ -0,0 +1,27 @@ +import chromadb +from chromadb.utils import embedding_functions + +# Panggil model dan database yang sama +emb = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="paraphrase-multilingual-MiniLM-L12-v2") +client = chromadb.PersistentClient(path="./chroma_db") +koleksi = client.get_or_create_collection("pedoman", embedding_function=emb) + +# Daftar pertanyaan uji +daftar_pertanyaan = [ + "Apa syarat cuti akademik?", + "Berapa batas kehadiran untuk ujian?", + "Bolehkah berhenti kuliah sementara?", # Pertanyaan parafrase (tanpa kata 'cuti') + "Apa sanksi jika telat bayar kuliah?" # Pertanyaan parafrase (tanpa kata 'keuangan') +] + +for q in daftar_pertanyaan: + print("=" * 60) + print("PERTANYAAN:", q) + # Ambil 2 hasil terdekat + hasil = koleksi.query(query_texts=[q], n_results=2) + + for doc, meta, jarak in zip(hasil["documents"][0], hasil["metadatas"][0], hasil["distances"][0]): + # Jarak (distance) semakin kecil berarti semakin mirip + print(f" [Sumber: {meta['sumber']} | Jarak: {jarak:.3f}]") + print(" Teks:", doc[:120].replace("\n", " "), "...") \ No newline at end of file diff --git a/cek_pdf.py b/cek_pdf.py new file mode 100644 index 0000000..51ad6b4 --- /dev/null +++ b/cek_pdf.py @@ -0,0 +1,12 @@ +from pathlib import Path +from pypdf import PdfReader + +for pdf in Path("dokumen").glob("*.pdf"): + r = PdfReader(pdf) + teks = "\n".join((p.extract_text() or "") for p in r.pages) + print("=" * 50) + print("File :", pdf.name) + print("Halaman:", len(r.pages)) + print("Panjang teks:", len(teks)) + print("500 karakter pertama:") + print(teks[:500]) \ No newline at end of file diff --git a/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/data_level0.bin b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/data_level0.bin new file mode 100644 index 0000000..5efb1b9 Binary files /dev/null and b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/data_level0.bin differ diff --git a/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/header.bin b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/header.bin differ diff --git a/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/length.bin b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/length.bin new file mode 100644 index 0000000..103707d Binary files /dev/null and b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/length.bin differ diff --git a/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/link_lists.bin b/chroma_db/6f7bc640-f0f3-49c9-bb37-3e55ae6d9dcb/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/chroma.sqlite3 b/chroma_db/chroma.sqlite3 new file mode 100644 index 0000000..8aabcf5 Binary files /dev/null and b/chroma_db/chroma.sqlite3 differ diff --git a/dokumen/akademik.pdf b/dokumen/akademik.pdf new file mode 100644 index 0000000..2eb0b1e --- /dev/null +++ b/dokumen/akademik.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +%“Œ‹ž ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 566 +>> +stream +Gat=fbAQ&g&4Q?mMHRLkU5H*ae&sq<9cln,rP0gljcMsbTh#=bq[R=CZ%Kta%0;,IEpas>\H6gN6s:ZA^m>Blh?sl&W;_;qZ3%$JE*(;UAn&E`P-?E(-KSEO"Wugi^bmpN'pBUR)O=QWOATbtLdH-MQ0V]GEM[(BJ5ui\n@:o_%<_.R2`T1RN$"^-Tt5g0P6,X^2h.a-LiR7le^79\s3YD?*\:ffo<6!ZeCC7Z\(m>3AIC5=0[r41,SK_5Xe9Wg*fRN:[\71PO3o!Z:j1KTD4jD:=a0+WHR\WiHB095'1>H6.3W/HlK=YeXIWeAL/eOj0a`&q6(/hZ&Aj&d-\k7NT3G\ZkT]c)fcVMZTQFm]+H\\>oW<@`+(pqRU7*&U^c3@\I(>q*aqiV7(^dW+sp>8FYkF(+m3faerOToA6s>kUY:Cm311AZrJP;+(@1!S_X&AN=&G^~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000524 00000 n +0000000592 00000 n +0000000872 00000 n +0000000931 00000 n +trailer +<< +/ID +[<430fe01d3101a78d4a66e001f226174a><430fe01d3101a78d4a66e001f226174a>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1587 +%%EOF diff --git a/dokumen/kemahasiswaan.pdf b/dokumen/kemahasiswaan.pdf new file mode 100644 index 0000000..f7865d8 --- /dev/null +++ b/dokumen/kemahasiswaan.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +%“Œ‹ž ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 516 +>> +stream +Gatn"d8%PD&4PLPMKqT'Z&:1bO-8]7(cCOg%!3rID7A0R9)O%6.o\3#+u)gGA7t&mSWd7Y!C+H9QNb/]L>,3!@L2K26oYGqT#3qNHg?Mok?,S\'E]d3OsdaNpd18>GZM1l4BSi;&;4kqL;QJo*"2ld8hS-joO#rj2&_kB#9*Te<.ShlA@IBW$K;KRCUcBn@4/8V*4le[lfL]O,bmR@%K4SX1"r8"0-bf_^KisZcDuko4DU:\C9SuuTRDM;X/'XL@&f9jk_+%Gu""SJ@L`-q,OQQ)!<$.[?cBj.uJ[G#(ZMZ`78r1_T\3(A+Y`c5#t>N`.DJ%meekooi2A1giX^G"JU1+B=\Yf3+p0P.J>b"^rEJof/?C9lNGendstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000524 00000 n +0000000592 00000 n +0000000872 00000 n +0000000931 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1537 +%%EOF diff --git a/dokumen/keuangan.pdf b/dokumen/keuangan.pdf new file mode 100644 index 0000000..bbb93d2 --- /dev/null +++ b/dokumen/keuangan.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +%“Œ‹ž ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 539 +>> +stream +Gat=fb>,r/&4Q?lMHL]?8fX#F/4"k^5lrK"0HmS\`-+cnKDP=9dPQBN/Y]Nb7@r,%j!u83,MMUA"aJ8Lp/up%+e+$R7LdrV9rR7p'KD0h!ERqP'9+eC.-dV3=(UUq@B8k]hrm(t@%@3Z7W`^=D:Hjf&*N#l5>\f;THJku./A1GL7AI#S6,jq"/_o;!3^ESJq^&$ejZ*6O:;C#X>4GjD+"nUeAU=OK+#NC\#MR@kXT8E9;7Vd7endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000524 00000 n +0000000592 00000 n +0000000872 00000 n +0000000931 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1560 +%%EOF diff --git a/evaluasi_retrieval.py b/evaluasi_retrieval.py new file mode 100644 index 0000000..ac09f35 --- /dev/null +++ b/evaluasi_retrieval.py @@ -0,0 +1,36 @@ +import chromadb +from chromadb.utils import embedding_functions + +# Panggil DB dan Model +emb = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="paraphrase-multilingual-MiniLM-L12-v2") +client = chromadb.PersistentClient(path="./chroma_db") +koleksi = client.get_or_create_collection("pedoman", embedding_function=emb) + +# Daftar (Pertanyaan, Kata Kunci Jawaban yang HARUS ada di chunk) +UJI = [ + ("Apa syarat cuti akademik?", "cuti akademik"), + ("Berapa lama cuti akademik bisa diajukan?", "dua semester"), + ("Berapa batas kehadiran untuk ujian?", "75 persen"), + ("Bolehkah berhenti kuliah sementara?", "cuti akademik"), + ("Bagaimana cara mengajukan beasiswa?", "beasiswa"), + ("Apa sanksi jika telat bayar kuliah?", "terlambat melakukan pembayaran"), + ("Kapan batas pembayaran biaya kuliah?", "dua minggu sebelum"), + ("Apa yang harus disimpan setelah bayar?", "bukti pembayaran") +] + +def hit_at_k(uji, k=3): + benar = 0 + for pertanyaan, kunci in uji: + # Ambil k hasil teratas + hasil = koleksi.query(query_texts=[pertanyaan], n_results=k)["documents"][0] + # Cek apakah kunci ada di salah satu hasil + if any(kunci.lower() in d.lower() for d in hasil): + benar += 1 + return benar / len(uji) + +print(f"Total pertanyaan uji: {len(UJI)}\n") +print("Mengevaluasi Retrieval...") +for k in (1, 3, 5): + skor = hit_at_k(UJI, k) + print(f"Hit@{k} = {skor:.2f} (Benar {int(skor*len(UJI))} dari {len(UJI)} pertanyaan)") \ No newline at end of file diff --git a/indeks.py b/indeks.py new file mode 100644 index 0000000..f4153ee --- /dev/null +++ b/indeks.py @@ -0,0 +1,38 @@ +from pathlib import Path +from pypdf import PdfReader +import chromadb +from chromadb.utils import embedding_functions + +def baca_pdf(path): + r = PdfReader(path) + return "\n".join((p.extract_text() or "") for p in r.pages) + +def potong(teks, ukuran=500, tumpang=75): + hasil, mulai = [], 0 + while mulai < len(teks): + hasil.append(teks[mulai:mulai + ukuran]) + mulai += ukuran - tumpang + return [h for h in hasil if h.strip()] + +print("Menyiapkan model embedding (unduh jika belum ada)...") +# Model multilingual agar bisa bahasa Indonesia +emb = embedding_functions.SentenceTransformerEmbeddingFunction( + model_name="paraphrase-multilingual-MiniLM-L12-v2") + +client = chromadb.PersistentClient(path="./chroma_db") +koleksi = client.get_or_create_collection( + name="pedoman", embedding_function=emb, + metadata={"hnsw:space": "cosine"}) + +# Proses tiap PDF +for pdf in Path("dokumen").glob("*.pdf"): + teks = baca_pdf(pdf) + chunks = potong(teks, ukuran=500, tumpang=75) # Pakai ukuran 500 + koleksi.upsert( + ids=[f"{pdf.stem}-{i}" for i in range(len(chunks))], + documents=chunks, + metadatas=[{"sumber": pdf.name, "chunk": i} for i in range(len(chunks))], + ) + print(f"{pdf.name} → {len(chunks)} chunk disimpan.") + +print("\nSelesai! Indeks tersimpan di folder 'chroma_db'.") \ No newline at end of file diff --git a/lab1_chunking.py b/lab1_chunking.py new file mode 100644 index 0000000..135875d --- /dev/null +++ b/lab1_chunking.py @@ -0,0 +1,31 @@ +from pathlib import Path +from pypdf import PdfReader + +def baca_pdf(path): + r = PdfReader(path) + return "\n".join((p.extract_text() or "") for p in r.pages) + +def potong(teks, ukuran=800, tumpang=100): + hasil, mulai = [], 0 + while mulai < len(teks): + hasil.append(teks[mulai:mulai + ukuran]) + mulai += ukuran - tumpang + return [h for h in hasil if h.strip()] + +# Gabungkan semua teks PDF untuk simulasi dokumen besar +semua_teks = "" +for pdf in Path("dokumen").glob("*.pdf"): + semua_teks += baca_pdf(pdf) + "\n\n" + +print("Total panjang teks semua PDF:", len(semua_teks), "karakter\n") + +for ukuran in (500, 1000, 1500): + tumpang = int(ukuran * 0.15) # Overlap 15% + chunks = potong(semua_teks, ukuran, tumpang) + print(f"=== Ukuran: {ukuran}, Overlap: {tumpang} ===") + print(f"Jumlah chunk yang dihasilkan: {len(chunks)}") + # Cetak 100 karakter pertama dari chunk ke-1 dan terakhir + print(f"Chunk 1: {repr(chunks[0][:100])}...") + if len(chunks) > 1: + print(f"Chunk akhir: {repr(chunks[-1][:100])}...") + print("-" * 40) \ No newline at end of file