diff --git a/experiments/07_cnn_evaluation.py b/experiments/07_cnn_evaluation.py new file mode 100644 index 0000000..28c1245 --- /dev/null +++ b/experiments/07_cnn_evaluation.py @@ -0,0 +1,302 @@ +""" +Tugas Anggota 2 (Week 2): CNN Evaluation + Visualization +========================================================= +PENTING - KONSISTENSI KONFIGURASI DENGAN ANGGOTA 1: +Skrip ini TIDAK melatih ulang model dari nol. Sebagai gantinya, skrip ini +memuat bobot model CNN yang sudah dilatih oleh Anggota 1 +(results/models/model_cnn_mnist.keras) pada 05_cnn_full_pipeline.py. + +Ini adalah cara paling aman untuk menjamin "tidak ada konfigurasi berbeda" +antara Anggota 1 dan Anggota 2, karena: + - Dataset yang sama -> load_mnist_for_cnn() (src/data_loader.py) + - Preprocessing yang sama -> normalisasi 0-1, reshape (28, 28, 1) + - Epoch yang sama -> 10 epoch (bobot hasil training Anggota 1) + - Batch size yang sama -> 128 (dipakai saat training oleh Anggota 1) + - Optimizer yang sama -> Adam (tersimpan di dalam file .keras) + +Konstanta di bawah HANYA didokumentasikan ulang (bukan dipakai untuk +melatih ulang) agar terlihat eksplisit bahwa konfigurasi cocok dengan +skrip Anggota 1. + +Cakupan tugas (sesuai skema kelompok): + (1) Evaluasi menyeluruh pada test set (classification report per kelas) + (2) Visualisasi confusion matrix (raw & ternormalisasi/persentase) + (3) Visualisasi akurasi per kelas (digit mana yang paling sulit) + (4) Visualisasi distribusi confidence prediksi (benar vs salah) + (5) Visualisasi contoh gambar yang salah diklasifikasikan +""" + +import os +import sys + +# Sembunyikan log TensorFlow yang berlebihan +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +from sklearn.metrics import classification_report, confusion_matrix +import tensorflow as tf +from tensorflow import keras + +# Tambahkan root directory ke path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from src.data_loader import load_mnist_for_cnn +from src.utils import ensure_dir + +# --------------------------------------------------------------------------- +# KONFIGURASI (harus identik dengan Anggota 1 / 05_cnn_full_pipeline.py) +# --------------------------------------------------------------------------- +SEED = 42 +EPOCHS = 10 # epoch training model (didokumentasikan, tidak dipakai ulang) +BATCH_SIZE = 128 # batch size training model (didokumentasikan, tidak dipakai ulang) +OPTIMIZER = "adam" # optimizer training model (tersimpan di file .keras) + +MODEL_PATH = os.path.join("results", "models", "model_cnn_mnist.keras") +FIGURES_DIR = os.path.join("results", "figures") +REPORTS_DIR = os.path.join("results", "reports") + +CLASS_NAMES = [str(i) for i in range(10)] + + +def main(): + tf.random.set_seed(SEED) + np.random.seed(SEED) + + ensure_dir(FIGURES_DIR) + ensure_dir(REPORTS_DIR) + + print("=" * 75) + print("TUGAS ANGGOTA 2: CNN EVALUATION + VISUALIZATION") + print("=" * 75) + print( + f"[KONFIGURASI - selaras dengan Anggota 1] " + f"seed={SEED} | epochs={EPOCHS} | batch_size={BATCH_SIZE} | optimizer={OPTIMIZER}" + ) + + # 1. Load Data (dataset & preprocessing sama persis dengan Anggota 1) + print("\n[1/6] Memuat dataset MNIST (load_mnist_for_cnn - sama dengan Anggota 1)...") + (x_train, y_train), (x_test, y_test) = load_mnist_for_cnn() + print(f" -> Data testing : {x_test.shape[0]:,} gambar {x_test.shape[1:]}") + + # 2. Load Model Terlatih dari Anggota 1 + print(f"\n[2/6] Memuat model terlatih dari: {MODEL_PATH}") + if not os.path.exists(MODEL_PATH): + raise FileNotFoundError( + f"Model tidak ditemukan di '{MODEL_PATH}'. " + f"Jalankan dahulu experiments/05_cnn_full_pipeline.py (skrip Anggota 1)." + ) + model = keras.models.load_model(MODEL_PATH) + print(f" -> Model '{model.name}' berhasil dimuat ({model.count_params():,} parameter).") + + # 3. Evaluasi Umum pada Test Set + print("\n[3/6] Mengevaluasi model pada test set...") + test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0) + y_pred_probs = model.predict(x_test, verbose=0) + y_pred = np.argmax(y_pred_probs, axis=1) + y_confidence = np.max(y_pred_probs, axis=1) + is_correct = (y_pred == y_test) + + print(f" -> Test Accuracy : {test_acc * 100:.2f}%") + print(f" -> Test Loss : {test_loss:.4f}") + print(f" -> Total Salah : {np.sum(~is_correct):,} dari {len(y_test):,} gambar") + + # ----------------------------------------------------------------- + # (1) Classification Report per Kelas + # ----------------------------------------------------------------- + print("\n[4/6] Membuat classification report per kelas...") + report_dict = classification_report( + y_test, y_pred, target_names=CLASS_NAMES, digits=4, output_dict=True + ) + report_text = classification_report(y_test, y_pred, target_names=CLASS_NAMES, digits=4) + print("\n" + "-" * 75) + print("CLASSIFICATION REPORT (Precision / Recall / F1-Score per Digit):") + print("-" * 75) + print(report_text) + + report_txt_path = os.path.join(REPORTS_DIR, "classification_report_cnn.txt") + with open(report_txt_path, "w") as f: + f.write("CLASSIFICATION REPORT - CNN MNIST (Anggota 2: CNN Evaluation)\n") + f.write(f"Test Accuracy: {test_acc * 100:.2f}% | Test Loss: {test_loss:.4f}\n") + f.write("=" * 65 + "\n") + f.write(report_text) + print(f" -> Laporan teks tersimpan: {report_txt_path}") + + report_df = pd.DataFrame(report_dict).transpose() + report_csv_path = os.path.join(REPORTS_DIR, "classification_report_cnn.csv") + report_df.to_csv(report_csv_path) + print(f" -> Laporan CSV tersimpan : {report_csv_path}") + + # Visualisasi precision/recall/f1 per kelas (bar chart berkelompok) + per_class_df = report_df.loc[CLASS_NAMES, ["precision", "recall", "f1-score"]] + fig, ax = plt.subplots(figsize=(11, 5)) + per_class_df.plot(kind="bar", ax=ax, width=0.75, + color=["#3498db", "#2ecc71", "#e74c3c"]) + ax.set_title("Precision, Recall, dan F1-Score per Digit (CNN)", fontsize=13, fontweight="bold") + ax.set_xlabel("Digit", fontsize=11) + ax.set_ylabel("Skor", fontsize=11) + ax.set_ylim(0.90, 1.005) + ax.set_xticklabels(CLASS_NAMES, rotation=0) + ax.legend(["Precision", "Recall", "F1-Score"], loc="lower right") + ax.grid(True, axis="y", alpha=0.3) + plt.tight_layout() + metrics_bar_path = os.path.join(FIGURES_DIR, "per_class_metrics_cnn.png") + plt.savefig(metrics_bar_path, dpi=150) + plt.close() + print(f" -> Grafik metrik per kelas tersimpan: {metrics_bar_path}") + + # ----------------------------------------------------------------- + # (2) Confusion Matrix: Raw Count & Ternormalisasi (%) + # ----------------------------------------------------------------- + print("\n[5/6] Membuat visualisasi confusion matrix (raw & normalisasi)...") + cm = confusion_matrix(y_test, y_pred) + cm_norm = cm.astype("float") / cm.sum(axis=1, keepdims=True) * 100 + + fig, axes = plt.subplots(1, 2, figsize=(17, 7)) + sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", cbar=True, + xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES, + annot_kws={"size": 9}, ax=axes[0]) + axes[0].set_title("Confusion Matrix (Jumlah Gambar)", fontsize=12, fontweight="bold") + axes[0].set_xlabel("Predicted Digit") + axes[0].set_ylabel("True Digit") + + sns.heatmap(cm_norm, annot=True, fmt=".1f", cmap="Oranges", cbar=True, + xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES, + annot_kws={"size": 9}, ax=axes[1]) + axes[1].set_title("Confusion Matrix (Persentase per Baris / Recall %)", fontsize=12, fontweight="bold") + axes[1].set_xlabel("Predicted Digit") + axes[1].set_ylabel("True Digit") + + plt.suptitle("Evaluasi CNN pada MNIST Test Set (10.000 Gambar)", fontsize=14, fontweight="bold", y=1.02) + plt.tight_layout() + cm_dual_path = os.path.join(FIGURES_DIR, "confusion_matrix_cnn_detail.png") + plt.savefig(cm_dual_path, dpi=150, bbox_inches="tight") + plt.close() + print(f" -> Confusion matrix detail tersimpan: {cm_dual_path}") + + # ----------------------------------------------------------------- + # (3) Akurasi per Kelas (diagonal confusion matrix ternormalisasi) + # ----------------------------------------------------------------- + per_class_acc = np.diag(cm_norm) + fig, ax = plt.subplots(figsize=(10, 5)) + colors = ["#e74c3c" if a < per_class_acc.mean() else "#2ecc71" for a in per_class_acc] + bars = ax.bar(CLASS_NAMES, per_class_acc, color=colors) + ax.axhline(per_class_acc.mean(), color="gray", linestyle="--", linewidth=1.5, + label=f"Rata-rata: {per_class_acc.mean():.2f}%") + for bar, acc in zip(bars, per_class_acc): + ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() - 1.2, + f"{acc:.1f}%", ha="center", va="top", fontsize=9, color="white", fontweight="bold") + ax.set_title("Akurasi per Digit pada Test Set (CNN)", fontsize=13, fontweight="bold") + ax.set_xlabel("Digit") + ax.set_ylabel("Akurasi (%)") + ax.set_ylim(min(per_class_acc) - 5, 100.5) + ax.legend() + ax.grid(True, axis="y", alpha=0.3) + plt.tight_layout() + per_class_acc_path = os.path.join(FIGURES_DIR, "per_class_accuracy_cnn.png") + plt.savefig(per_class_acc_path, dpi=150) + plt.close() + print(f" -> Grafik akurasi per kelas tersimpan: {per_class_acc_path}") + + hardest_idx = np.argsort(per_class_acc)[:3] + print(" -> 3 digit paling sulit dikenali:") + for idx in hardest_idx: + print(f" Digit {idx}: akurasi {per_class_acc[idx]:.2f}%") + + # ----------------------------------------------------------------- + # (4) Distribusi Confidence Prediksi (Benar vs Salah) + # ----------------------------------------------------------------- + print("\n[6/6] Membuat visualisasi distribusi confidence & sampel kesalahan...") + fig, ax = plt.subplots(figsize=(9, 5.5)) + ax.hist(y_confidence[is_correct], bins=30, alpha=0.65, color="#2ecc71", + label=f"Prediksi Benar (n={np.sum(is_correct):,})", density=True) + ax.hist(y_confidence[~is_correct], bins=30, alpha=0.75, color="#e74c3c", + label=f"Prediksi Salah (n={np.sum(~is_correct):,})", density=True) + ax.set_title("Distribusi Confidence Prediksi Model CNN", fontsize=13, fontweight="bold") + ax.set_xlabel("Confidence (Probabilitas Softmax Maksimum)") + ax.set_ylabel("Kepadatan (Density)") + ax.legend() + ax.grid(True, alpha=0.3) + plt.tight_layout() + confidence_path = os.path.join(FIGURES_DIR, "confidence_distribution_cnn.png") + plt.savefig(confidence_path, dpi=150) + plt.close() + print(f" -> Distribusi confidence tersimpan: {confidence_path}") + + avg_conf_correct = y_confidence[is_correct].mean() * 100 + avg_conf_wrong = y_confidence[~is_correct].mean() * 100 + print(f" -> Rata-rata confidence saat BENAR : {avg_conf_correct:.2f}%") + print(f" -> Rata-rata confidence saat SALAH : {avg_conf_wrong:.2f}%") + + # ----------------------------------------------------------------- + # (5) Contoh Gambar yang Salah Diklasifikasikan + # ----------------------------------------------------------------- + wrong_idx = np.where(~is_correct)[0] + rng = np.random.default_rng(SEED) + n_show = min(25, len(wrong_idx)) + sample_idx = rng.choice(wrong_idx, size=n_show, replace=False) + + fig, axes = plt.subplots(5, 5, figsize=(11, 12)) + for ax, idx in zip(axes.flat, sample_idx): + ax.imshow(x_test[idx].reshape(28, 28), cmap="gray") + conf = y_confidence[idx] * 100 + ax.set_title(f"True: {y_test[idx]} | Pred: {y_pred[idx]}\nConf: {conf:.1f}%", + fontsize=9, color="#c0392b") + ax.axis("off") + for ax in axes.flat[n_show:]: + ax.axis("off") + plt.suptitle(f"Contoh {n_show} Gambar Salah Klasifikasi dari {len(wrong_idx)} Total Kesalahan", + fontsize=13, fontweight="bold", y=1.0) + plt.tight_layout() + misclassified_path = os.path.join(FIGURES_DIR, "misclassified_samples_cnn.png") + plt.savefig(misclassified_path, dpi=150, bbox_inches="tight") + plt.close() + print(f" -> Contoh gambar salah klasifikasi tersimpan: {misclassified_path}") + + # ----------------------------------------------------------------- + # Ringkasan Akhir + # ----------------------------------------------------------------- + summary_path = os.path.join(REPORTS_DIR, "evaluation_summary_cnn.csv") + summary_df = pd.DataFrame([{ + "test_accuracy_pct": round(test_acc * 100, 4), + "test_loss": round(test_loss, 4), + "total_errors": int(np.sum(~is_correct)), + "total_samples": int(len(y_test)), + "avg_confidence_correct_pct": round(avg_conf_correct, 4), + "avg_confidence_wrong_pct": round(avg_conf_wrong, 4), + "hardest_digit_1": int(hardest_idx[0]), + "hardest_digit_1_acc_pct": round(per_class_acc[hardest_idx[0]], 4), + "hardest_digit_2": int(hardest_idx[1]), + "hardest_digit_2_acc_pct": round(per_class_acc[hardest_idx[1]], 4), + }]) + summary_df.to_csv(summary_path, index=False) + + print("\n" + "=" * 75) + print("RINGKASAN EVALUASI CNN (ANGGOTA 2)") + print("=" * 75) + print(f"Test Accuracy : {test_acc * 100:.2f}%") + print(f"Test Loss : {test_loss:.4f}") + print(f"Total Kesalahan : {np.sum(~is_correct):,} / {len(y_test):,}") + print(f"Confidence saat Benar : {avg_conf_correct:.2f}%") + print(f"Confidence saat Salah : {avg_conf_wrong:.2f}%") + print(f"Digit Paling Sulit : {hardest_idx[0]} ({per_class_acc[hardest_idx[0]]:.2f}%), " + f"{hardest_idx[1]} ({per_class_acc[hardest_idx[1]]:.2f}%)") + print("-" * 75) + print("File yang dihasilkan:") + print(f" 1. {metrics_bar_path}") + print(f" 2. {cm_dual_path}") + print(f" 3. {per_class_acc_path}") + print(f" 4. {confidence_path}") + print(f" 5. {misclassified_path}") + print(f" 6. {report_txt_path}") + print(f" 7. {report_csv_path}") + print(f" 8. {summary_path}") + print("=" * 75 + "\n") + + +if __name__ == "__main__": + main() diff --git a/results/figures/confidence_distribution_cnn.png b/results/figures/confidence_distribution_cnn.png new file mode 100644 index 0000000..99e6981 Binary files /dev/null and b/results/figures/confidence_distribution_cnn.png differ diff --git a/results/figures/confusion_matrix_cnn_detail.png b/results/figures/confusion_matrix_cnn_detail.png new file mode 100644 index 0000000..b54b0b6 Binary files /dev/null and b/results/figures/confusion_matrix_cnn_detail.png differ diff --git a/results/figures/misclassified_samples_cnn.png b/results/figures/misclassified_samples_cnn.png new file mode 100644 index 0000000..bd7970d Binary files /dev/null and b/results/figures/misclassified_samples_cnn.png differ diff --git a/results/figures/per_class_accuracy_cnn.png b/results/figures/per_class_accuracy_cnn.png new file mode 100644 index 0000000..daa2c86 Binary files /dev/null and b/results/figures/per_class_accuracy_cnn.png differ diff --git a/results/figures/per_class_metrics_cnn.png b/results/figures/per_class_metrics_cnn.png new file mode 100644 index 0000000..ec8a467 Binary files /dev/null and b/results/figures/per_class_metrics_cnn.png differ diff --git a/results/reports/classification_report_cnn.csv b/results/reports/classification_report_cnn.csv new file mode 100644 index 0000000..d218655 --- /dev/null +++ b/results/reports/classification_report_cnn.csv @@ -0,0 +1,14 @@ +,precision,recall,f1-score,support +0,0.9938775510204082,0.9938775510204082,0.9938775510204082,980.0 +1,0.992988606485539,0.9982378854625551,0.9956063268892794,1135.0 +2,0.9932038834951457,0.9912790697674418,0.9922405431619786,1032.0 +3,0.9940298507462687,0.9891089108910891,0.9915632754342432,1010.0 +4,0.9908814589665653,0.9959266802443992,0.9933976637887253,982.0 +5,0.9877232142857143,0.992152466367713,0.9899328859060402,892.0 +6,0.9937304075235109,0.9926931106471816,0.9932114882506528,958.0 +7,0.9893307468477207,0.9922178988326849,0.9907722195240408,1028.0 +8,0.9836734693877551,0.9897330595482546,0.9866939611054247,974.0 +9,0.9919436052366566,0.9762140733399405,0.984015984015984,1009.0 +accuracy,0.9912,0.9912,0.9912,0.9912 +macro avg,0.9911382793995284,0.9911440706121668,0.9911311899096779,10000.0 +weighted avg,0.9912088120264272,0.9912,0.9911943463223446,10000.0 diff --git a/results/reports/classification_report_cnn.txt b/results/reports/classification_report_cnn.txt new file mode 100644 index 0000000..7b54125 --- /dev/null +++ b/results/reports/classification_report_cnn.txt @@ -0,0 +1,19 @@ +CLASSIFICATION REPORT - CNN MNIST (Anggota 2: CNN Evaluation) +Test Accuracy: 99.12% | Test Loss: 0.0268 +================================================================= + precision recall f1-score support + + 0 0.9939 0.9939 0.9939 980 + 1 0.9930 0.9982 0.9956 1135 + 2 0.9932 0.9913 0.9922 1032 + 3 0.9940 0.9891 0.9916 1010 + 4 0.9909 0.9959 0.9934 982 + 5 0.9877 0.9922 0.9899 892 + 6 0.9937 0.9927 0.9932 958 + 7 0.9893 0.9922 0.9908 1028 + 8 0.9837 0.9897 0.9867 974 + 9 0.9919 0.9762 0.9840 1009 + + accuracy 0.9912 10000 + macro avg 0.9911 0.9911 0.9911 10000 +weighted avg 0.9912 0.9912 0.9912 10000 diff --git a/results/reports/evaluation_summary_cnn.csv b/results/reports/evaluation_summary_cnn.csv new file mode 100644 index 0000000..afe2eb4 --- /dev/null +++ b/results/reports/evaluation_summary_cnn.csv @@ -0,0 +1,2 @@ +test_accuracy_pct,test_loss,total_errors,total_samples,avg_confidence_correct_pct,avg_confidence_wrong_pct,hardest_digit_1,hardest_digit_1_acc_pct,hardest_digit_2,hardest_digit_2_acc_pct +99.12,0.0268,88,10000,99.5341,76.6475,9,97.6214,3,98.9109