Update project
This commit is contained in:
alex
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,74 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 566
|
||||
>>
|
||||
stream
|
||||
Gat=fbAQ&g&4Q?mMHRLkU5H*ae&sq<9cln,rP0gljcMsbTh#=bq[R=CZ%Kta%0;,IEpas>\H6gN6s:ZA^m>Blh?sl&W;_;qZ3%$JE*(;UAn&E`P-?E(-KSEO"Wugi^bmpN'pBUR)O=QWOATbtLdH-MQ0V]GEM[(BJ5ui\n@:o_%<_.R2`T1RN$"^-Tt5g0P6,X^2h.a-LiR7le^79\s3YD?*\:ffo<6!ZeCC7Z\(m>3AIC5=0[r41,SK_5Xe9Wg*fRN:[\71PO3o!Z:j1KTD4jD:<s-=4N?5`U:i#6H$iW&dTV?s"F?AAo2eX;e#Ztf,c/:lLZe]Hb_*ZO_39XQ?_19?&_<H[Yn$onAWttT26mhOsfi]^lrpE<FIcX,7lO>=a0+WHR\WiHB095'1>H6.3W/HlK=YeXIWeA<T+eU(L>L/eOj0a`&q6(/hZ&Aj&d-\k7NT3G\ZkT]c)fcVMZTQFm]+H\\>oW<@`+(pqRU7*&U^c3@\I(>q*aqiV7(^dW+sp>8FYkF(+m3faerOToA6s>kUY:Cm311AZrJP;+(@1!S_X&AN=&G^~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000524 00000 n
|
||||
0000000592 00000 n
|
||||
0000000872 00000 n
|
||||
0000000931 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<430fe01d3101a78d4a66e001f226174a><430fe01d3101a78d4a66e001f226174a>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
1587
|
||||
%%EOF
|
||||
@@ -0,0 +1,74 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 516
|
||||
>>
|
||||
stream
|
||||
Gatn"d8%PD&4PLPMKqT'Z&:1bO-8]7(cCOg%!3rID7A0R9)O%6.o\3#+u)gGA7t&mSWd7Y!C+H9QNb/]L>,3!@L2K26oYGqT#3qNHg?Mok?,S\'E]d3OsdaNpd18>GZM1l4BSi;&;4kqL;QJo*"2ld8hS-joO#rj2&_kB#9*Te<.ShlA@IBW$K;KRCUcBn@4/8V*4le[lfL]O,bmR@%K4SX1"r8"0-bf_^KisZcDuko4<k<J]][Xkg+Nd>DU:\C9SuuTRDM;X/'XL@&f9jk_+%Gu""SJ@L`-q,OQQ)!<$.<U$sCV&ZiXj.6:+EbkB1Q-n#Sbam@\O$0K=nf";g'FY':\e]NNk:OZ]gs*[ibA7@C3ap@M$N\on3fCB='9d(lY$$Nrs+C>[?cBj.uJ[G#(ZMZ`78r1_T\3(A+Y`c5#t>N`.DJ%meekooi2A1giX^G"JU1+B=\Yf3+p0P.J>b"^rEJof/?C9lNG<gb96h?Q%m35V?!):tE)_oZ="U%\k.V`73~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000524 00000 n
|
||||
0000000592 00000 n
|
||||
0000000872 00000 n
|
||||
0000000931 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<be4b92fe71d8c47f732a578c48a5c846><be4b92fe71d8c47f732a578c48a5c846>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
1537
|
||||
%%EOF
|
||||
@@ -0,0 +1,74 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 8 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 7 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20261102090414+07'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102090414+07'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Count 1 /Kids [ 4 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 539
|
||||
>>
|
||||
stream
|
||||
Gat=fb>,r/&4Q?lMHL]?8fX#F/4"k^5lrK"0HmS\`-+cnKDP=9dPQBN/Y]Nb7@r,%j!u83,MMUA"aJ8Lp/up%+e+$R7L<TYn<=Br#,9;28=1Z3,iu`#$4]I@T[EqF'n&.r[+m,8+e)MPgbbJ@d5^+i&S)%Rhp<)=*BY%%*;R04:G>drV9rR7p'KD0h!ERqP'9+eC.-dV3=(UUq@B8k]<c=,%c<MqEJ5.NRg)5>hrm(t@%@3Z7W`^=D:Hjf&*N#l5>\f;THJku./A1GL7AI#S6,jq"/_o;!3^ESJq^&$ejZ*6O:;C#X>4GjD+"nUeAU=OK+#NC\#MR@kXT8E9;7Vd7<B^-WN&t+hIl`2a/5d*mh$qW^+p0orWclN)acio-.@G$UAr&*auSPT4QO=mJdrXc=L3dn6a7+IQW\chKi)n62d97iZ11`&PqpMB@@+=1e+$![Vg"J2q(\*k_:E:4"[j.gGtWOF<U^IB("m/.lrLn3DK[XIG%n30"/kPVi?5lLRiFTb`-iY`oAl)3?),9p(\G%*1B~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 9
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000524 00000 n
|
||||
0000000592 00000 n
|
||||
0000000872 00000 n
|
||||
0000000931 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<bb972e76e34dfd7c99f5710a35d79b0d><bb972e76e34dfd7c99f5710a35d79b0d>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 6 0 R
|
||||
/Root 5 0 R
|
||||
/Size 9
|
||||
>>
|
||||
startxref
|
||||
1560
|
||||
%%EOF
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
Lab Terbimbing P-10: RAG Part 1 (Chunking, ChromaDB, Hit@k)
|
||||
|
||||
Persiapan:
|
||||
pip install chromadb sentence-transformers pypdf
|
||||
Taruh 2-3 PDF berbasis teks di folder dokumen/
|
||||
|
||||
Jalankan bertahap:
|
||||
python lab_p10.py lab1
|
||||
python lab_p10.py lab2
|
||||
python lab_p10.py lab3
|
||||
"""
|
||||
import random
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import chromadb
|
||||
from chromadb.utils import embedding_functions
|
||||
from pypdf import PdfReader
|
||||
|
||||
BASE = Path(__file__).resolve().parent # folder tempat lab_p10.py berada
|
||||
FOLDER = BASE / "dokumen"
|
||||
# Setiap kali script dijalankan -> folder indeks BARU (tidak bentrok dengan run lain)
|
||||
RUN_ID = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
DB_PATH = str(BASE / "chroma_db" / f"run_{RUN_ID}")
|
||||
MODEL = "paraphrase-multilingual-MiniLM-L12-v2"
|
||||
|
||||
|
||||
# ---------- fungsi dasar ----------
|
||||
def baca_pdf(path):
|
||||
reader = PdfReader(path)
|
||||
return "\n".join((p.extract_text() or "") for p in reader.pages)
|
||||
|
||||
|
||||
def potong(teks, ukuran=800, tumpang=100):
|
||||
hasil, mulai = [], 0
|
||||
while mulai < len(teks):
|
||||
hasil.append(teks[mulai:mulai + ukuran])
|
||||
mulai += ukuran - tumpang
|
||||
return [h for h in hasil if h.strip()]
|
||||
|
||||
|
||||
def daftar_pdf():
|
||||
pdfs = sorted(FOLDER.glob("*.pdf"))
|
||||
if not pdfs:
|
||||
sys.exit(f"Tidak ada PDF di {FOLDER}. Letakkan 2-3 PDF dulu.")
|
||||
return pdfs
|
||||
|
||||
|
||||
def buat_koleksi(nama, ukuran, tumpang):
|
||||
print(f"Indeks run ini disimpan di: {DB_PATH}")
|
||||
emb = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=MODEL)
|
||||
client = chromadb.PersistentClient(path=DB_PATH)
|
||||
koleksi = client.get_or_create_collection(
|
||||
name=nama, embedding_function=emb, metadata={"hnsw:space": "cosine"}
|
||||
)
|
||||
for pdf in daftar_pdf():
|
||||
chunks = potong(baca_pdf(pdf), ukuran, tumpang)
|
||||
if not chunks:
|
||||
print(f"PERINGATAN: {pdf.name} kosong (mungkin PDF scan), dilewati.")
|
||||
continue
|
||||
koleksi.upsert(
|
||||
ids=[f"{pdf.stem}-{i}" for i in range(len(chunks))],
|
||||
documents=chunks,
|
||||
metadatas=[{"sumber": pdf.name, "chunk": i} for i in range(len(chunks))],
|
||||
)
|
||||
print(f"[{nama}] {pdf.name} -> {len(chunks)} chunk")
|
||||
return koleksi
|
||||
|
||||
|
||||
# ---------- LAB 1: eksperimen chunking ----------
|
||||
def lab1():
|
||||
pdf = daftar_pdf()[0]
|
||||
teks = baca_pdf(pdf)
|
||||
print(f"File: {pdf.name} | total {len(teks)} karakter")
|
||||
print("=== 500 karakter pertama ===")
|
||||
print(teks[:500])
|
||||
print("\n(Jika kosong/aneh: PDF hasil scan, butuh OCR.)\n")
|
||||
|
||||
for ukuran in (500, 1000, 1500):
|
||||
tumpang = int(ukuran * 0.15) # overlap 15%
|
||||
chunks = potong(teks, ukuran, tumpang)
|
||||
print(f"\n##### ukuran={ukuran}, overlap={tumpang} -> {len(chunks)} chunk #####")
|
||||
for c in random.sample(chunks, min(3, len(chunks))):
|
||||
print("-" * 50)
|
||||
print(c)
|
||||
print("-" * 50)
|
||||
print("\nCatat: jumlah chunk tiap ukuran, dan apakah ada chunk "
|
||||
"yang terpotong di tengah kalimat/tabel.")
|
||||
|
||||
|
||||
# ---------- LAB 2: bangun indeks & cari ----------
|
||||
PERTANYAAN_LAB2 = [
|
||||
"Berapa jumlah gelas sampel untuk setiap merek air mineral?",
|
||||
"Apa saja ukuran pemusatan data yang dihitung?",
|
||||
"Apa tujuan dari praktikum statistik deskriptif ini?",
|
||||
"Merek air mineral mana yang beratnya paling konsisten?",
|
||||
"Data apa saja yang termasuk skala pengukuran rasio?",
|
||||
]
|
||||
PARAFRASE_LAB2 = [
|
||||
"Seberapa lebar sebaran tinggi badan cowok dibanding cewek?",
|
||||
"Pakai aplikasi apa data hasil timbangan diolah?",
|
||||
]
|
||||
|
||||
|
||||
def tampil(koleksi, daftar):
|
||||
for q in daftar:
|
||||
hasil = koleksi.query(query_texts=[q], n_results=3)
|
||||
print(f"\nQ: {q}")
|
||||
for doc, meta, jarak in zip(hasil["documents"][0],
|
||||
hasil["metadatas"][0],
|
||||
hasil["distances"][0]):
|
||||
print(f" [{meta['sumber']} · chunk {meta['chunk']}] jarak={jarak:.3f}")
|
||||
print(" ", doc[:200].replace("\n", " "), "...")
|
||||
|
||||
|
||||
def lab2():
|
||||
koleksi = buat_koleksi("pedoman", 800, 100)
|
||||
print("\n===== 5 PERTANYAAN BIASA =====")
|
||||
tampil(koleksi, PERTANYAAN_LAB2)
|
||||
print("\n===== 2 PERTANYAAN PARAFRASE =====")
|
||||
tampil(koleksi, PARAFRASE_LAB2)
|
||||
print("\nCatat: apakah chunk teratas relevan? Apakah parafrase tetap ketemu?")
|
||||
|
||||
|
||||
# ---------- LAB 3: ukur Hit@k (+ MRR) ----------
|
||||
UJI = [
|
||||
("Berapa mean berat air mineral Alfa pada data tunggal?", "225,3"),
|
||||
("Berapa standar deviasi tinggi badan laki-laki?", "6,65"),
|
||||
("Berapa varians tinggi badan perempuan?", "8,97"),
|
||||
("Software apa yang dipakai untuk mengolah data?", "Minitab"),
|
||||
("Berapa gelas sampel yang ditimbang untuk tiap merek?", "40 gelas"),
|
||||
("Merek air mineral apa saja yang diteliti?", "Aqua, Alfamart, dan Vit"),
|
||||
("Apa kelebihan boxplot dibanding mean?", "tahan terhadap nilai ekstrim"),
|
||||
("Data apa yang termasuk skala nominal?", "NRP, jenis kelamin"),
|
||||
("Berapa skewness data kelompok merek Alfa?", "-0,1826"),
|
||||
("Berapa IQR tinggi badan laki-laki?", "11,25"),
|
||||
("Berapa mean berat air mineral Vit?", "201,75"),
|
||||
("Apa itu presisi dalam pengukuran?", "konsistensi hasil pengukuran"),
|
||||
("Berapa rata-rata jumlah SKS responden?", "20,765"),
|
||||
("Untuk apa statistika inferensial digunakan?", "statistika inferensial digunakan"),
|
||||
("Berapa range tinggi badan perempuan?", "8,20"),
|
||||
]
|
||||
|
||||
|
||||
def hit_at_k(koleksi, uji, k):
|
||||
benar = 0
|
||||
for q, kunci in uji:
|
||||
docs = koleksi.query(query_texts=[q], n_results=k)["documents"][0]
|
||||
if any(kunci.lower() in d.lower() for d in docs):
|
||||
benar += 1
|
||||
return benar / len(uji)
|
||||
|
||||
|
||||
def mrr(koleksi, uji, k=5):
|
||||
total = 0.0
|
||||
for q, kunci in uji:
|
||||
docs = koleksi.query(query_texts=[q], n_results=k)["documents"][0]
|
||||
for rank, d in enumerate(docs, 1):
|
||||
if kunci.lower() in d.lower():
|
||||
total += 1 / rank
|
||||
break
|
||||
return total / len(uji)
|
||||
|
||||
|
||||
def lab3():
|
||||
if len(UJI) < 15:
|
||||
print(f"PERINGATAN: baru {len(UJI)} pertanyaan uji; tugas meminta 15.\n")
|
||||
pengaturan = {"chunk500": (500, 75), "chunk1000": (1000, 150)}
|
||||
ringkasan = {}
|
||||
for nama, (ukuran, tumpang) in pengaturan.items():
|
||||
koleksi = buat_koleksi(nama, ukuran, tumpang)
|
||||
ringkasan[nama] = {f"Hit@{k}": hit_at_k(koleksi, UJI, k) for k in (1, 3, 5)}
|
||||
ringkasan[nama]["MRR"] = mrr(koleksi, UJI)
|
||||
|
||||
print("\n===== HASIL =====")
|
||||
for nama, skor in ringkasan.items():
|
||||
print(nama, " | ".join(f"{m}={v:.2f}" for m, v in skor.items()))
|
||||
print("\nTulis kesimpulan 3-4 kalimat: ukuran terbaik, pertanyaan yang gagal, dugaan penyebab.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
perintah = sys.argv[1] if len(sys.argv) > 1 else ""
|
||||
{"lab1": lab1, "lab2": lab2, "lab3": lab3}.get(
|
||||
perintah, lambda: print(__doc__)
|
||||
)()
|
||||
Reference in New Issue
Block a user