commit 52e042838f741d31f315ce330f82519b2a382224 Author: alex Date: Mon Nov 2 12:01:19 2026 +0700 Update project diff --git a/chroma_db/run_20261102_111104/chroma.sqlite3 b/chroma_db/run_20261102_111104/chroma.sqlite3 new file mode 100644 index 0000000..290ef52 Binary files /dev/null and b/chroma_db/run_20261102_111104/chroma.sqlite3 differ diff --git a/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/data_level0.bin b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/data_level0.bin new file mode 100644 index 0000000..5efb1b9 Binary files /dev/null and b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/data_level0.bin differ diff --git a/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/header.bin b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/header.bin differ diff --git a/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/length.bin b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/length.bin new file mode 100644 index 0000000..cb3e162 Binary files /dev/null and b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/length.bin differ diff --git a/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/link_lists.bin b/chroma_db/run_20261102_111104/e9cea4f3-de01-45f4-9803-5e35fc08c596/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/data_level0.bin b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/data_level0.bin new file mode 100644 index 0000000..1e47b3b Binary files /dev/null and b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/data_level0.bin differ diff --git a/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/header.bin b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/header.bin differ diff --git a/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/length.bin b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/length.bin new file mode 100644 index 0000000..df9850a Binary files /dev/null and b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/length.bin differ diff --git a/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/link_lists.bin b/chroma_db/run_20261102_111133/7992d001-d7f9-4176-b985-e722661225fa/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/data_level0.bin b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/data_level0.bin new file mode 100644 index 0000000..5efb1b9 Binary files /dev/null and b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/data_level0.bin differ diff --git a/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/header.bin b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/header.bin new file mode 100644 index 0000000..bb54792 Binary files /dev/null and b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/header.bin differ diff --git a/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/length.bin b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/length.bin new file mode 100644 index 0000000..cb3e162 Binary files /dev/null and b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/length.bin differ diff --git a/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/link_lists.bin b/chroma_db/run_20261102_111133/97070c02-5683-40e3-9349-4bbefa120438/link_lists.bin new file mode 100644 index 0000000..e69de29 diff --git a/chroma_db/run_20261102_111133/chroma.sqlite3 b/chroma_db/run_20261102_111133/chroma.sqlite3 new file mode 100644 index 0000000..42707e6 Binary files /dev/null and b/chroma_db/run_20261102_111133/chroma.sqlite3 differ diff --git a/dokumen/akademik.pdf b/dokumen/akademik.pdf new file mode 100644 index 0000000..2eb0b1e --- /dev/null +++ b/dokumen/akademik.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +%東京 ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 566 +>> +stream +Gat=fbAQ&g&4Q?mMHRLkU5H*ae&sq<9cln,rP0gljcMsbTh#=bq[R=CZ%Kta%0;,IEpas>\H6gN6s:ZA^m>Blh?sl&W;_;qZ3%$JE*(;UAn&E`P-?E(-KSEO"Wugi^bmpN'pBUR)O=QWOATbtLdH-MQ0V]GEM[(BJ5ui\n@:o_%<_.R2`T1RN$"^-Tt5g0P6,X^2h.a-LiR7le^79\s3YD?*\:ffo<6!ZeCC7Z\(m>3AIC5=0[r41,SK_5Xe9Wg*fRN:[\71PO3o!Z:j1KTD4jD:=a0+WHR\WiHB095'1>H6.3W/HlK=YeXIWeAL/eOj0a`&q6(/hZ&Aj&d-\k7NT3G\ZkT]c)fcVMZTQFm]+H\\>oW<@`+(pqRU7*&U^c3@\I(>q*aqiV7(^dW+sp>8FYkF(+m3faerOToA6s>kUY:Cm311AZrJP;+(@1!S_X&AN=&G^~>endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000524 00000 n +0000000592 00000 n +0000000872 00000 n +0000000931 00000 n +trailer +<< +/ID +[<430fe01d3101a78d4a66e001f226174a><430fe01d3101a78d4a66e001f226174a>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1587 +%%EOF diff --git a/dokumen/kemahasiswaan.pdf b/dokumen/kemahasiswaan.pdf new file mode 100644 index 0000000..f7865d8 --- /dev/null +++ b/dokumen/kemahasiswaan.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +%東京 ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 516 +>> +stream +Gatn"d8%PD&4PLPMKqT'Z&:1bO-8]7(cCOg%!3rID7A0R9)O%6.o\3#+u)gGA7t&mSWd7Y!C+H9QNb/]L>,3!@L2K26oYGqT#3qNHg?Mok?,S\'E]d3OsdaNpd18>GZM1l4BSi;&;4kqL;QJo*"2ld8hS-joO#rj2&_kB#9*Te<.ShlA@IBW$K;KRCUcBn@4/8V*4le[lfL]O,bmR@%K4SX1"r8"0-bf_^KisZcDuko4DU:\C9SuuTRDM;X/'XL@&f9jk_+%Gu""SJ@L`-q,OQQ)!<$.[?cBj.uJ[G#(ZMZ`78r1_T\3(A+Y`c5#t>N`.DJ%meekooi2A1giX^G"JU1+B=\Yf3+p0P.J>b"^rEJof/?C9lNGendstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000524 00000 n +0000000592 00000 n +0000000872 00000 n +0000000931 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1537 +%%EOF diff --git a/dokumen/keuangan.pdf b/dokumen/keuangan.pdf new file mode 100644 index 0000000..bbb93d2 --- /dev/null +++ b/dokumen/keuangan.pdf @@ -0,0 +1,74 @@ +%PDF-1.4 +%東京 ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R /F2 3 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font +>> +endobj +4 0 obj +<< +/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +5 0 obj +<< +/PageMode /UseNone /Pages 7 0 R /Type /Catalog +>> +endobj +6 0 obj +<< +/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False +>> +endobj +7 0 obj +<< +/Count 1 /Kids [ 4 0 R ] /Type /Pages +>> +endobj +8 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 539 +>> +stream +Gat=fb>,r/&4Q?lMHL]?8fX#F/4"k^5lrK"0HmS\`-+cnKDP=9dPQBN/Y]Nb7@r,%j!u83,MMUA"aJ8Lp/up%+e+$R7LdrV9rR7p'KD0h!ERqP'9+eC.-dV3=(UUq@B8k]hrm(t@%@3Z7W`^=D:Hjf&*N#l5>\f;THJku./A1GL7AI#S6,jq"/_o;!3^ESJq^&$ejZ*6O:;C#X>4GjD+"nUeAU=OK+#NC\#MR@kXT8E9;7Vd7endstream +endobj +xref +0 9 +0000000000 65535 f +0000000061 00000 n +0000000102 00000 n +0000000209 00000 n +0000000321 00000 n +0000000524 00000 n +0000000592 00000 n +0000000872 00000 n +0000000931 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 6 0 R +/Root 5 0 R +/Size 9 +>> +startxref +1560 +%%EOF diff --git a/lab_p10.py b/lab_p10.py new file mode 100644 index 0000000..0f85f89 --- /dev/null +++ b/lab_p10.py @@ -0,0 +1,187 @@ +""" +Lab Terbimbing P-10: RAG Part 1 (Chunking, ChromaDB, Hit@k) + +Persiapan: + pip install chromadb sentence-transformers pypdf + Taruh 2-3 PDF berbasis teks di folder dokumen/ + +Jalankan bertahap: + python lab_p10.py lab1 + python lab_p10.py lab2 + python lab_p10.py lab3 +""" +import random +import sys +from datetime import datetime +from pathlib import Path + +import chromadb +from chromadb.utils import embedding_functions +from pypdf import PdfReader + +BASE = Path(__file__).resolve().parent # folder tempat lab_p10.py berada +FOLDER = BASE / "dokumen" +# Setiap kali script dijalankan -> folder indeks BARU (tidak bentrok dengan run lain) +RUN_ID = datetime.now().strftime("%Y%m%d_%H%M%S") +DB_PATH = str(BASE / "chroma_db" / f"run_{RUN_ID}") +MODEL = "paraphrase-multilingual-MiniLM-L12-v2" + + +# ---------- fungsi dasar ---------- +def baca_pdf(path): + reader = PdfReader(path) + return "\n".join((p.extract_text() or "") for p in reader.pages) + + +def potong(teks, ukuran=800, tumpang=100): + hasil, mulai = [], 0 + while mulai < len(teks): + hasil.append(teks[mulai:mulai + ukuran]) + mulai += ukuran - tumpang + return [h for h in hasil if h.strip()] + + +def daftar_pdf(): + pdfs = sorted(FOLDER.glob("*.pdf")) + if not pdfs: + sys.exit(f"Tidak ada PDF di {FOLDER}. Letakkan 2-3 PDF dulu.") + return pdfs + + +def buat_koleksi(nama, ukuran, tumpang): + print(f"Indeks run ini disimpan di: {DB_PATH}") + emb = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=MODEL) + client = chromadb.PersistentClient(path=DB_PATH) + koleksi = client.get_or_create_collection( + name=nama, embedding_function=emb, metadata={"hnsw:space": "cosine"} + ) + for pdf in daftar_pdf(): + chunks = potong(baca_pdf(pdf), ukuran, tumpang) + if not chunks: + print(f"PERINGATAN: {pdf.name} kosong (mungkin PDF scan), dilewati.") + continue + koleksi.upsert( + ids=[f"{pdf.stem}-{i}" for i in range(len(chunks))], + documents=chunks, + metadatas=[{"sumber": pdf.name, "chunk": i} for i in range(len(chunks))], + ) + print(f"[{nama}] {pdf.name} -> {len(chunks)} chunk") + return koleksi + + +# ---------- LAB 1: eksperimen chunking ---------- +def lab1(): + pdf = daftar_pdf()[0] + teks = baca_pdf(pdf) + print(f"File: {pdf.name} | total {len(teks)} karakter") + print("=== 500 karakter pertama ===") + print(teks[:500]) + print("\n(Jika kosong/aneh: PDF hasil scan, butuh OCR.)\n") + + for ukuran in (500, 1000, 1500): + tumpang = int(ukuran * 0.15) # overlap 15% + chunks = potong(teks, ukuran, tumpang) + print(f"\n##### ukuran={ukuran}, overlap={tumpang} -> {len(chunks)} chunk #####") + for c in random.sample(chunks, min(3, len(chunks))): + print("-" * 50) + print(c) + print("-" * 50) + print("\nCatat: jumlah chunk tiap ukuran, dan apakah ada chunk " + "yang terpotong di tengah kalimat/tabel.") + + +# ---------- LAB 2: bangun indeks & cari ---------- +PERTANYAAN_LAB2 = [ + "Berapa jumlah gelas sampel untuk setiap merek air mineral?", + "Apa saja ukuran pemusatan data yang dihitung?", + "Apa tujuan dari praktikum statistik deskriptif ini?", + "Merek air mineral mana yang beratnya paling konsisten?", + "Data apa saja yang termasuk skala pengukuran rasio?", +] +PARAFRASE_LAB2 = [ + "Seberapa lebar sebaran tinggi badan cowok dibanding cewek?", + "Pakai aplikasi apa data hasil timbangan diolah?", +] + + +def tampil(koleksi, daftar): + for q in daftar: + hasil = koleksi.query(query_texts=[q], n_results=3) + print(f"\nQ: {q}") + for doc, meta, jarak in zip(hasil["documents"][0], + hasil["metadatas"][0], + hasil["distances"][0]): + print(f" [{meta['sumber']} ﾂｷ chunk {meta['chunk']}] jarak={jarak:.3f}") + print(" ", doc[:200].replace("\n", " "), "...") + + +def lab2(): + koleksi = buat_koleksi("pedoman", 800, 100) + print("\n===== 5 PERTANYAAN BIASA =====") + tampil(koleksi, PERTANYAAN_LAB2) + print("\n===== 2 PERTANYAAN PARAFRASE =====") + tampil(koleksi, PARAFRASE_LAB2) + print("\nCatat: apakah chunk teratas relevan? Apakah parafrase tetap ketemu?") + + +# ---------- LAB 3: ukur Hit@k (+ MRR) ---------- +UJI = [ + ("Berapa mean berat air mineral Alfa pada data tunggal?", "225,3"), + ("Berapa standar deviasi tinggi badan laki-laki?", "6,65"), + ("Berapa varians tinggi badan perempuan?", "8,97"), + ("Software apa yang dipakai untuk mengolah data?", "Minitab"), + ("Berapa gelas sampel yang ditimbang untuk tiap merek?", "40 gelas"), + ("Merek air mineral apa saja yang diteliti?", "Aqua, Alfamart, dan Vit"), + ("Apa kelebihan boxplot dibanding mean?", "tahan terhadap nilai ekstrim"), + ("Data apa yang termasuk skala nominal?", "NRP, jenis kelamin"), + ("Berapa skewness data kelompok merek Alfa?", "-0,1826"), + ("Berapa IQR tinggi badan laki-laki?", "11,25"), + ("Berapa mean berat air mineral Vit?", "201,75"), + ("Apa itu presisi dalam pengukuran?", "konsistensi hasil pengukuran"), + ("Berapa rata-rata jumlah SKS responden?", "20,765"), + ("Untuk apa statistika inferensial digunakan?", "statistika inferensial digunakan"), + ("Berapa range tinggi badan perempuan?", "8,20"), +] + + +def hit_at_k(koleksi, uji, k): + benar = 0 + for q, kunci in uji: + docs = koleksi.query(query_texts=[q], n_results=k)["documents"][0] + if any(kunci.lower() in d.lower() for d in docs): + benar += 1 + return benar / len(uji) + + +def mrr(koleksi, uji, k=5): + total = 0.0 + for q, kunci in uji: + docs = koleksi.query(query_texts=[q], n_results=k)["documents"][0] + for rank, d in enumerate(docs, 1): + if kunci.lower() in d.lower(): + total += 1 / rank + break + return total / len(uji) + + +def lab3(): + if len(UJI) < 15: + print(f"PERINGATAN: baru {len(UJI)} pertanyaan uji; tugas meminta 15.\n") + pengaturan = {"chunk500": (500, 75), "chunk1000": (1000, 150)} + ringkasan = {} + for nama, (ukuran, tumpang) in pengaturan.items(): + koleksi = buat_koleksi(nama, ukuran, tumpang) + ringkasan[nama] = {f"Hit@{k}": hit_at_k(koleksi, UJI, k) for k in (1, 3, 5)} + ringkasan[nama]["MRR"] = mrr(koleksi, UJI) + + print("\n===== HASIL =====") + for nama, skor in ringkasan.items(): + print(nama, " | ".join(f"{m}={v:.2f}" for m, v in skor.items())) + print("\nTulis kesimpulan 3-4 kalimat: ukuran terbaik, pertanyaan yang gagal, dugaan penyebab.") + + +if __name__ == "__main__": + perintah = sys.argv[1] if len(sys.argv) > 1 else "" + {"lab1": lab1, "lab2": lab2, "lab3": lab3}.get( + perintah, lambda: print(__doc__) + )() \ No newline at end of file