commit pertama
This commit is contained in:
Rendra
Binary file not shown.
|
After Width: | Height: | Size: 281 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 311 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 306 KiB |
@@ -0,0 +1,93 @@
|
||||
%PDF-1.4
|
||||
%“Œ‹ž ReportLab Generated PDF document (opensource)
|
||||
1 0 obj
|
||||
<<
|
||||
/F1 2 0 R /F2 3 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Contents 9 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 8 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/Contents 10 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 8 0 R /Resources <<
|
||||
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
|
||||
>> /Rotate 0 /Trans <<
|
||||
|
||||
>>
|
||||
/Type /Page
|
||||
>>
|
||||
endobj
|
||||
6 0 obj
|
||||
<<
|
||||
/PageMode /UseNone /Pages 8 0 R /Type /Catalog
|
||||
>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<<
|
||||
/Author (\(anonymous\)) /CreationDate (D:20261102012834+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20261102012834+00'00') /Producer (ReportLab PDF Library - \(opensource\))
|
||||
/Subject (\(unspecified\)) /Title (Panduan Akademik Universitas Contoh) /Trapped /False
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<<
|
||||
/Count 2 /Kids [ 4 0 R 5 0 R ] /Type /Pages
|
||||
>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 1769
|
||||
>>
|
||||
stream
|
||||
Gat=+a`?B3&A@rknjmuO)D+LqZ+$.bp<3YVR%#`TWsj[t1fDj`r:ffgZpI,I33WumA``J<o2WpX(E^tP5NP;*2Y>bm*dKqdT#mH^1YVau?T33/hKc6a4E;37jNV@%$QRb/,4*4QCR;*h*Ai#S;Ja[QcmNbJ$@SPsKt]>Rr$]M[-DAOR'T!D*I8!'RF-Y0<L&+SG[F1eHc\Fh/G40+NVp0SbE4cp7rP7TdDk;1'\$5TYYe_sE[u':p-Q0gm[5Nl9KR^[m8nbf&RE8,SMU%^7k+cmEeU0&hliV(Mg5i?6^VimsWdYOZ<ekN<EiMR9(PSLuYu`[AZ?VY&XG':*>-83tM[Z#BdJA;dVU-Zd6kumlMf`o\@U8Fhdm.!S7T4.:Z/aSDeC^*t';[$SGh?+S6Rns'bH5j!rgM4q&Q+tMM.0)V6ibj_;Q\BHKsXT*<S!&c#"#:;@dfj,J9A'Yo`D@5R0cm?hW,QEK/rY6_g3[[;Zkbo5,X:P'o>V">#hjtjGNA"DDIp91HeLIXmR:gFhmbAqpTh(,L\+S.E.'9mKS)nJo1?VV+AN:7\U#>VJta;/+EeB.qN:9O0_oF9GLbumTuU"8[@MWaa#FF[Sin3)KihQn1#QL-nY&=lu'TqS:tl7eln/0rD1tt\t"U^TK85!**C@@&A4\sBsBHB%6W;mD'2s(]h!p!lGO:/Aed\5n2(a'>#Yt@]jbKsOl<qe7KFq"%]"o@RNPJ,/-'`?F/-:0>Mj;'7\tPAB\mmYc4c0*cq5'X<@c_:(R6oN+H%VeXJ)0)ao0-Rn_J=0B$m*;!>\KUb')\VfZhuD$Xj5>b<s81,W\P8$5X=Ze#.kG2_tO>i-A8OLau?X0P[,PL/;=b*K&Q][]^;D_9rQ@@q6ldpk'NVh,-=;>L`FR95XA,gA0<s^?N4Xj_'PZd,&Y/fh0iTQHklnf)bTYZN,LDBa1E'es<,6V[YGt*VbmW\CL^=crW>R/iPb/Z(_[Dogn_F[-fh+-)"XK?/<sF0bD>=^H#r,OjF.;S-P;tI2N16$_1T:/h!*H6""5tBMQk$op1_qCZmEa*lCl`hqY+P=u`2p/Em6D^68-drWkO=&$A@DDeFt?Se6ZL`fibN(X'nUd4RZ!=>9L3h<$HWa#e+/IURY:ZT/>hY)?eJaf<'aZJFKooCon6"R/Bh)S4!J3(f;t>qW=ZmWNlj;Jf!;nqB*m"U2pnY(:,-+?@JG3VD;"^V</4RbXK>+9&ck,'rJdSI_^>*R5_<i/XGA7uuIlDBs4.9fO(Fi3$9L+c[=Wf.A,<d/ra)"lGB0Yh;p(douTfWUkaMQu1.Wgb=OhTSA"1@n[mjD;1\,b`",BUf&9D%@0NNkVbKT[+(g\dc_5$IXZ.U&[R5#H;,r6kbT4(d!tq+Y`]aUiA1rhrA[)B0b6Bor:BHKNC*bom<H'RSP1^GV3V-63-[Muc^sZ0@pp:H3=Zm'9.!q,</3m8A+&sjH9DX?J-IQ/+0:fJiYJ'@4:ME'I5[b\O!c=3AR"$G:]7!XL11^@$hP)o0fn`XiJl5,0lbc<A$n9:rF<L&n?J]^Ab66)%69(*Q4m+F5'$&O",i_dEM>OQ0nXBKp$L7+&'=4:N8C\(h_;@`gi4@S`R:LEs"\0%'c:d>fgr]@1UrLT7>_g4bt@_fj+g.rqKfOngTs!eNR5%Z\oFU>NFT4.9^u=.)X[NO(qoeI@l.hj%`c9^7PusY%+,3fGWb8*ij:'/_(!D^<E8YeVD5C+goq@IMaX,7k;U*;dj3;rs/qZb$N~>endstream
|
||||
endobj
|
||||
10 0 obj
|
||||
<<
|
||||
/Filter [ /ASCII85Decode /FlateDecode ] /Length 736
|
||||
>>
|
||||
stream
|
||||
Gat='9lJKG&A@7.]OQhVZ/^3GDJ^g8.\NNI[pCD+CJWMsL]&@\^Lq^aP(Gs$OHhenh0JVSc'ndF4@T,GAq5dHDa"j`AuHbIbd0=0:UmDOh^?CHS#^+D[TjShXQHQ7T9W'kPN+bGgL_W8`;t(3ap?o67@=/%,2@:3iEEWgZ_\_NI-rmdr<]J5W`QIZ=*adCEV[%j[UQcc91f^`._/5!5>t48h2\_7%Bkno_f3qV3ij+L.dL&E@i8$a_1jkf]J;bYatFbr(6BTeOMtDVQLpd>e>U)i*B]Cbi*<_mJ#eNK?<Q&Zn'5k?:GNo(4At'%moAQn#s-]U9+9D4aW8`VjL,O>K-(MIG*%#-D98R9NYn$\n6D/H(V8Otaf:qmn5B)dlB]cBo,m7X9tT=M-Z9Sgg8S43@P.Vfg/:X!^\=<6Pj;&V1%KI_fqVS&G%f`h/7("uK^j(GkbtI/'_[AI1ur&01Q9]P&(-;(^^N0ER\:aJ8IJ<S*me]"9c-H2CPTomJb,RICoLp<fo_D*^iMlkR#FKb]+M'[VkY"HWcqpF&No_4gcDhfrrU.UB>o?Mr]8Br11/*`m4eL%4@%#/2=$r3rC;nS4%:`,IXh_g]Yf#ZIItb]1s\s;E=A'\E-Vl9;W.MfO$".=S/1?GDXZ+1-%B\"d@kP;U?%:N^dja32*,>SVEl9cdA]"41.ea[lsI,ZgFlF0;t/1=U%^h\ZB;h*'5PV.S*N9Ba5eVa<H_[~>endstream
|
||||
endobj
|
||||
xref
|
||||
0 11
|
||||
0000000000 65535 f
|
||||
0000000061 00000 n
|
||||
0000000102 00000 n
|
||||
0000000209 00000 n
|
||||
0000000321 00000 n
|
||||
0000000524 00000 n
|
||||
0000000728 00000 n
|
||||
0000000796 00000 n
|
||||
0000001098 00000 n
|
||||
0000001163 00000 n
|
||||
0000003023 00000 n
|
||||
trailer
|
||||
<<
|
||||
/ID
|
||||
[<00caf45b9e105d99b02ef9bd9f5fa72c><00caf45b9e105d99b02ef9bd9f5fa72c>]
|
||||
% ReportLab generated PDF document -- digest (opensource)
|
||||
|
||||
/Info 7 0 R
|
||||
/Root 6 0 R
|
||||
/Size 11
|
||||
>>
|
||||
startxref
|
||||
3850
|
||||
%%EOF
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
Lab 1 - Eksperimen Chunking (15 menit)
|
||||
Hanya butuh library pypdf: pip install pypdf
|
||||
|
||||
Cara jalankan (pilih salah satu):
|
||||
python lab_1.py -> otomatis pakai PDF pertama di folder dokumen/
|
||||
python lab_1.py dokumen/panduan_akademik.pdf -> pakai PDF tertentu
|
||||
python lab_1.py dokumen/panduan_akademik.pdf --overlap 0.2
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import random
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from pypdf import PdfReader
|
||||
|
||||
UKURAN_LIST = [500, 1000, 1500]
|
||||
|
||||
|
||||
def baca_pdf(path):
|
||||
"""Baca seluruh teks dari satu file PDF."""
|
||||
reader = PdfReader(str(path))
|
||||
halaman = [(p.extract_text() or "") for p in reader.pages]
|
||||
teks = "\n".join(halaman)
|
||||
return re.sub(r"[ \t]+", " ", teks).strip()
|
||||
|
||||
|
||||
def potong(teks, ukuran=1000, overlap=0.15):
|
||||
"""Potong teks jadi chunk berukuran `ukuran` karakter dengan overlap (0-1)."""
|
||||
langkah = max(1, int(ukuran * (1 - overlap)))
|
||||
chunks = []
|
||||
for mulai in range(0, len(teks), langkah):
|
||||
potongan = teks[mulai:mulai + ukuran].strip()
|
||||
if potongan:
|
||||
chunks.append(potongan)
|
||||
if mulai + ukuran >= len(teks):
|
||||
break
|
||||
return chunks
|
||||
|
||||
|
||||
def cari_pdf_default():
|
||||
daftar = sorted(Path("dokumen").glob("*.pdf"))
|
||||
if not daftar:
|
||||
raise SystemExit("Tidak ada PDF di folder dokumen/. Taruh PDF di sana atau tulis path-nya.")
|
||||
return daftar[0]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Lab 1: eksperimen chunking")
|
||||
p.add_argument("pdf", nargs="?", help="Path PDF (kosongkan untuk pakai PDF pertama di dokumen/)")
|
||||
p.add_argument("--overlap", type=float, default=0.15, help="Overlap 0.10 - 0.20 (default 0.15)")
|
||||
args = p.parse_args()
|
||||
|
||||
path = Path(args.pdf) if args.pdf else cari_pdf_default()
|
||||
print(f"File: {path}\n")
|
||||
|
||||
# 1. Baca PDF, cetak 500 karakter pertama
|
||||
teks = baca_pdf(path)
|
||||
print(f"=== 1. Hasil baca PDF ({len(teks)} karakter total) ===")
|
||||
print(teks[:500])
|
||||
|
||||
# 2. Potong dengan tiga ukuran, catat jumlah chunk
|
||||
print("\n=== 2. Jumlah chunk per pengaturan ===")
|
||||
hasil = {}
|
||||
for ukuran in UKURAN_LIST:
|
||||
chunks = potong(teks, ukuran, args.overlap)
|
||||
hasil[ukuran] = chunks
|
||||
print(f" ukuran={ukuran:<5} overlap={args.overlap:.0%} -> {len(chunks)} chunk")
|
||||
|
||||
# 3. Lihat 3 chunk acak tiap ukuran
|
||||
print("\n=== 3. Tiga chunk acak per ukuran (terpotong di tengah kalimat/tabel?) ===")
|
||||
for ukuran, chunks in hasil.items():
|
||||
print(f"\n--- ukuran {ukuran} ---")
|
||||
for c in random.sample(chunks, min(3, len(chunks))):
|
||||
awal_rapi = c[0].isupper() or c[0].isdigit()
|
||||
akhir_rapi = c[-1] in ".!?:"
|
||||
print(f"[awal rapi: {awal_rapi} | akhir rapi: {akhir_rapi}]")
|
||||
print(c)
|
||||
print("." * 40)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
import os
|
||||
import chromadb
|
||||
from chromadb.utils import embedding_functions
|
||||
from pypdf import PdfReader
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
def jalankan_lab2():
|
||||
print("--- 🧪 LAB 2: Membangun Indeks & Pencarian Semantik (Multi-PDF) ---")
|
||||
|
||||
folder_dokumen = "dokumen"
|
||||
teks_gabungan = ""
|
||||
|
||||
if not os.path.exists(folder_dokumen):
|
||||
print(f"❌ Folder '{folder_dokumen}' tidak ditemukan!")
|
||||
return
|
||||
|
||||
# Mendeteksi otomatis semua file PDF di dalam folder dokumen/
|
||||
daftar_pdf = [f for f in os.listdir(folder_dokumen) if f.lower().endswith('.pdf')]
|
||||
|
||||
if not daftar_pdf:
|
||||
print(f"❌ Tidak ada file PDF di dalam folder '{folder_dokumen}/'!")
|
||||
return
|
||||
|
||||
print(f"📂 File PDF yang ditemukan: {daftar_pdf}")
|
||||
|
||||
for nama_file in daftar_pdf:
|
||||
file_path = os.path.join(folder_dokumen, nama_file)
|
||||
reader = PdfReader(file_path)
|
||||
teks_file = ""
|
||||
for halaman in reader.pages:
|
||||
t = halaman.extract_text()
|
||||
if t:
|
||||
teks_file += t + "\n"
|
||||
|
||||
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=75)
|
||||
chunks_file = splitter.split_text(teks_file)
|
||||
print(f" - {nama_file}: menghasilkan {len(chunks_file)} chunk.")
|
||||
teks_gabungan += teks_file + "\n"
|
||||
|
||||
# Chunking keseluruhan teks dari semua PDF
|
||||
splitter_utama = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=75)
|
||||
semua_chunks = splitter_utama.split_text(teks_gabungan)
|
||||
print(f"\n📄 Total keseluruhan chunk dari semua PDF: {len(semua_chunks)}")
|
||||
|
||||
# Inisialisasi ChromaDB & Embedding Lokal
|
||||
client = chromadb.Client()
|
||||
ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="paraphrase-multilingual-MiniLM-L12-v2")
|
||||
|
||||
collection = client.get_or_create_collection(name="rag_lab2_multi", embedding_function=ef)
|
||||
|
||||
# Masukkan chunk ke Vector Database
|
||||
collection.add(
|
||||
documents=semua_chunks,
|
||||
ids=[str(i) for i in range(len(semua_chunks))]
|
||||
)
|
||||
print("✅ Berhasil menyimpan seluruh indeks ke Vector Database (ChromaDB)!\n")
|
||||
|
||||
# Mengajukan 5 Pertanyaan (Sesuai ketentuan Lab 2)
|
||||
daftar_pertanyaan = [
|
||||
"Apa itu Retrieval-Augmented Generation?",
|
||||
"Mengapa chunking diperlukan dalam sistem RAG?",
|
||||
"Bagaimana cara kerja vector database?",
|
||||
"Jelaskan teknik memecah dokumen agar tidak kehilangan konteks!",
|
||||
"Bagaimana cara melakukan optimasi sistem RAG?"
|
||||
]
|
||||
|
||||
print("--- 🔍 HASIL PENCARIAN QUERY (5 PERTANYAAN) ---")
|
||||
for i, q in enumerate(daftar_pertanyaan, 1):
|
||||
hasil = collection.query(query_texts=[q], n_results=1)
|
||||
print(f"\nPertanyaan {i}: {q}")
|
||||
print(f"💡 Hasil Chunk Relevan: {hasil['documents'][0][0].strip()}")
|
||||
print("-" * 60)
|
||||
|
||||
if __name__ == "__main__":
|
||||
jalankan_lab2()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,66 @@
|
||||
import os
|
||||
import chromadb
|
||||
from chromadb.utils import embedding_functions
|
||||
from pypdf import PdfReader
|
||||
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
||||
|
||||
def jalankan_lab3():
|
||||
print("--- 🧪 LAB 3: RAG Pipeline & Evaluasi Pencarian ---")
|
||||
|
||||
folder_dokumen = "dokumen"
|
||||
teks_gabungan = ""
|
||||
|
||||
if not os.path.exists(folder_dokumen):
|
||||
print(f"❌ Folder '{folder_dokumen}' tidak ditemukan!")
|
||||
return
|
||||
|
||||
# 1. Otomatis membaca semua file PDF di folder dokumen/
|
||||
daftar_pdf = [f for f in os.listdir(folder_dokumen) if f.lower().endswith('.pdf')]
|
||||
print(f"📂 Memuat dokumen: {daftar_pdf}")
|
||||
|
||||
for nama_file in daftar_pdf:
|
||||
file_path = os.path.join(folder_dokumen, nama_file)
|
||||
reader = PdfReader(file_path)
|
||||
for halaman in reader.pages:
|
||||
t = halaman.extract_text()
|
||||
if t:
|
||||
teks_gabungan += t + "\n"
|
||||
|
||||
# 2. Proses Chunking
|
||||
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=75)
|
||||
semua_chunks = splitter.split_text(teks_gabungan)
|
||||
print(f"📄 Total chunk yang diindeks: {len(semua_chunks)}")
|
||||
|
||||
# 3. Inisialisasi Vector Database (ChromaDB) & Embedding Lokal
|
||||
client = chromadb.Client()
|
||||
ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name="paraphrase-multilingual-MiniLM-L12-v2")
|
||||
|
||||
# Gunakan collection unik untuk Lab 3
|
||||
collection = client.get_or_create_collection(name="rag_lab3_pipeline", embedding_function=ef)
|
||||
|
||||
collection.add(
|
||||
documents=semua_chunks,
|
||||
ids=[str(i) for i in range(len(semua_chunks))]
|
||||
)
|
||||
print("✅ Berhasil membangun indeks untuk RAG Pipeline!\n")
|
||||
|
||||
# 4. Simulasi RAG (Retrieval + Context Generation)
|
||||
pertanyaan_pengguna = "Bagaimana cara kerja vector database dan apa fungsinya?"
|
||||
|
||||
print(f"❓ Pertanyaan Pengguna: \"{pertanyaan_pengguna}\"")
|
||||
|
||||
# Melakukan retrieval (pencarian chunk relevan teratas)
|
||||
hasil_retrieval = collection.query(query_texts=[pertanyaan_pengguna], n_results=2)
|
||||
konteks_terambil = hasil_retrieval['documents'][0]
|
||||
|
||||
print("\n🔍 Konteks Relevan yang Ditemukan dari Dokumen:")
|
||||
for idx, teks_chunk in enumerate(konteks_terambil, 1):
|
||||
print(f" [Chunk {idx}]: {teks_chunk.strip()}")
|
||||
print("-" * 40)
|
||||
|
||||
# 5. Simulasi Jawaban Akhir Berdasarkan Konteks (RAG Generation)
|
||||
print("\n🤖 Simulasi Jawaban Sistem RAG:")
|
||||
print("Berdasarkan dokumen yang tersedia, vector database bekerja dengan menyimpan representasi numerik (embedding) dari teks ke dalam ruang multi-dimensi. Ketika ada pertanyaan, sistem mencari jarak terdekat (cosine similarity) untuk mencocokkan informasi yang paling relevan guna menjawab kebutuhan pengguna.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
jalankan_lab3()
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user