""" Tugas Anggota 2 (Week 2): CNN Evaluation + Visualization ========================================================= PENTING - KONSISTENSI KONFIGURASI DENGAN ANGGOTA 1: Skrip ini TIDAK melatih ulang model dari nol. Sebagai gantinya, skrip ini memuat bobot model CNN yang sudah dilatih oleh Anggota 1 (results/models/model_cnn_mnist.keras) pada 05_cnn_full_pipeline.py. Ini adalah cara paling aman untuk menjamin "tidak ada konfigurasi berbeda" antara Anggota 1 dan Anggota 2, karena: - Dataset yang sama -> load_mnist_for_cnn() (src/data_loader.py) - Preprocessing yang sama -> normalisasi 0-1, reshape (28, 28, 1) - Epoch yang sama -> 10 epoch (bobot hasil training Anggota 1) - Batch size yang sama -> 128 (dipakai saat training oleh Anggota 1) - Optimizer yang sama -> Adam (tersimpan di dalam file .keras) Konstanta di bawah HANYA didokumentasikan ulang (bukan dipakai untuk melatih ulang) agar terlihat eksplisit bahwa konfigurasi cocok dengan skrip Anggota 1. Cakupan tugas (sesuai skema kelompok): (1) Evaluasi menyeluruh pada test set (classification report per kelas) (2) Visualisasi confusion matrix (raw & ternormalisasi/persentase) (3) Visualisasi akurasi per kelas (digit mana yang paling sulit) (4) Visualisasi distribusi confidence prediksi (benar vs salah) (5) Visualisasi contoh gambar yang salah diklasifikasikan """ import os import sys # Sembunyikan log TensorFlow yang berlebihan os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns from sklearn.metrics import classification_report, confusion_matrix import tensorflow as tf from tensorflow import keras # Tambahkan root directory ke path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) from src.data_loader import load_mnist_for_cnn from src.utils import ensure_dir # --------------------------------------------------------------------------- # KONFIGURASI (harus identik dengan Anggota 1 / 05_cnn_full_pipeline.py) # --------------------------------------------------------------------------- SEED = 42 EPOCHS = 10 # epoch training model (didokumentasikan, tidak dipakai ulang) BATCH_SIZE = 128 # batch size training model (didokumentasikan, tidak dipakai ulang) OPTIMIZER = "adam" # optimizer training model (tersimpan di file .keras) MODEL_PATH = os.path.join("results", "02_cnn", "models", "model_cnn_mnist.keras") FIGURES_DIR = os.path.join("results", "02_cnn", "figures") REPORTS_DIR = os.path.join("results", "02_cnn", "reports") CLASS_NAMES = [str(i) for i in range(10)] def main(): tf.random.set_seed(SEED) np.random.seed(SEED) ensure_dir(FIGURES_DIR) ensure_dir(REPORTS_DIR) print("=" * 75) print("TUGAS ANGGOTA 2: CNN EVALUATION + VISUALIZATION") print("=" * 75) print( f"[KONFIGURASI - selaras dengan Anggota 1] " f"seed={SEED} | epochs={EPOCHS} | batch_size={BATCH_SIZE} | optimizer={OPTIMIZER}" ) # 1. Load Data (dataset & preprocessing sama persis dengan Anggota 1) print("\n[1/6] Memuat dataset MNIST (load_mnist_for_cnn - sama dengan Anggota 1)...") (x_train, y_train), (x_test, y_test) = load_mnist_for_cnn() print(f" -> Data testing : {x_test.shape[0]:,} gambar {x_test.shape[1:]}") # 2. Load Model Terlatih dari Anggota 1 print(f"\n[2/6] Memuat model terlatih dari: {MODEL_PATH}") if not os.path.exists(MODEL_PATH): raise FileNotFoundError( f"Model tidak ditemukan di '{MODEL_PATH}'. " f"Jalankan dahulu tasks/cnn/05_cnn_full_pipeline.py (skrip Anggota 1)." ) model = keras.models.load_model(MODEL_PATH) print(f" -> Model '{model.name}' berhasil dimuat ({model.count_params():,} parameter).") # 3. Evaluasi Umum pada Test Set print("\n[3/6] Mengevaluasi model pada test set...") test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0) y_pred_probs = model.predict(x_test, verbose=0) y_pred = np.argmax(y_pred_probs, axis=1) y_confidence = np.max(y_pred_probs, axis=1) is_correct = (y_pred == y_test) print(f" -> Test Accuracy : {test_acc * 100:.2f}%") print(f" -> Test Loss : {test_loss:.4f}") print(f" -> Total Salah : {np.sum(~is_correct):,} dari {len(y_test):,} gambar") # ----------------------------------------------------------------- # (1) Classification Report per Kelas # ----------------------------------------------------------------- print("\n[4/6] Membuat classification report per kelas...") report_dict = classification_report( y_test, y_pred, target_names=CLASS_NAMES, digits=4, output_dict=True ) report_text = classification_report(y_test, y_pred, target_names=CLASS_NAMES, digits=4) print("\n" + "-" * 75) print("CLASSIFICATION REPORT (Precision / Recall / F1-Score per Digit):") print("-" * 75) print(report_text) report_txt_path = os.path.join(REPORTS_DIR, "classification_report_cnn.txt") with open(report_txt_path, "w") as f: f.write("CLASSIFICATION REPORT - CNN MNIST (Anggota 2: CNN Evaluation)\n") f.write(f"Test Accuracy: {test_acc * 100:.2f}% | Test Loss: {test_loss:.4f}\n") f.write("=" * 65 + "\n") f.write(report_text) print(f" -> Laporan teks tersimpan: {report_txt_path}") report_df = pd.DataFrame(report_dict).transpose() report_csv_path = os.path.join(REPORTS_DIR, "classification_report_cnn.csv") report_df.to_csv(report_csv_path) print(f" -> Laporan CSV tersimpan : {report_csv_path}") # Visualisasi precision/recall/f1 per kelas (bar chart berkelompok) per_class_df = report_df.loc[CLASS_NAMES, ["precision", "recall", "f1-score"]] fig, ax = plt.subplots(figsize=(11, 5)) per_class_df.plot(kind="bar", ax=ax, width=0.75, color=["#3498db", "#2ecc71", "#e74c3c"]) ax.set_title("Precision, Recall, dan F1-Score per Digit (CNN)", fontsize=13, fontweight="bold") ax.set_xlabel("Digit", fontsize=11) ax.set_ylabel("Skor", fontsize=11) ax.set_ylim(0.90, 1.005) ax.set_xticklabels(CLASS_NAMES, rotation=0) ax.legend(["Precision", "Recall", "F1-Score"], loc="lower right") ax.grid(True, axis="y", alpha=0.3) plt.tight_layout() metrics_bar_path = os.path.join(FIGURES_DIR, "per_class_metrics_cnn.png") plt.savefig(metrics_bar_path, dpi=150) plt.close() print(f" -> Grafik metrik per kelas tersimpan: {metrics_bar_path}") # ----------------------------------------------------------------- # (2) Confusion Matrix: Raw Count & Ternormalisasi (%) # ----------------------------------------------------------------- print("\n[5/6] Membuat visualisasi confusion matrix (raw & normalisasi)...") cm = confusion_matrix(y_test, y_pred) cm_norm = cm.astype("float") / cm.sum(axis=1, keepdims=True) * 100 fig, axes = plt.subplots(1, 2, figsize=(17, 7)) sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", cbar=True, xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES, annot_kws={"size": 9}, ax=axes[0]) axes[0].set_title("Confusion Matrix (Jumlah Gambar)", fontsize=12, fontweight="bold") axes[0].set_xlabel("Predicted Digit") axes[0].set_ylabel("True Digit") sns.heatmap(cm_norm, annot=True, fmt=".1f", cmap="Oranges", cbar=True, xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES, annot_kws={"size": 9}, ax=axes[1]) axes[1].set_title("Confusion Matrix (Persentase per Baris / Recall %)", fontsize=12, fontweight="bold") axes[1].set_xlabel("Predicted Digit") axes[1].set_ylabel("True Digit") plt.suptitle("Evaluasi CNN pada MNIST Test Set (10.000 Gambar)", fontsize=14, fontweight="bold", y=1.02) plt.tight_layout() cm_dual_path = os.path.join(FIGURES_DIR, "confusion_matrix_cnn_detail.png") plt.savefig(cm_dual_path, dpi=150, bbox_inches="tight") plt.close() print(f" -> Confusion matrix detail tersimpan: {cm_dual_path}") # ----------------------------------------------------------------- # (3) Akurasi per Kelas (diagonal confusion matrix ternormalisasi) # ----------------------------------------------------------------- per_class_acc = np.diag(cm_norm) fig, ax = plt.subplots(figsize=(10, 5)) colors = ["#e74c3c" if a < per_class_acc.mean() else "#2ecc71" for a in per_class_acc] bars = ax.bar(CLASS_NAMES, per_class_acc, color=colors) ax.axhline(per_class_acc.mean(), color="gray", linestyle="--", linewidth=1.5, label=f"Rata-rata: {per_class_acc.mean():.2f}%") for bar, acc in zip(bars, per_class_acc): ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() - 1.2, f"{acc:.1f}%", ha="center", va="top", fontsize=9, color="white", fontweight="bold") ax.set_title("Akurasi per Digit pada Test Set (CNN)", fontsize=13, fontweight="bold") ax.set_xlabel("Digit") ax.set_ylabel("Akurasi (%)") ax.set_ylim(min(per_class_acc) - 5, 100.5) ax.legend() ax.grid(True, axis="y", alpha=0.3) plt.tight_layout() per_class_acc_path = os.path.join(FIGURES_DIR, "per_class_accuracy_cnn.png") plt.savefig(per_class_acc_path, dpi=150) plt.close() print(f" -> Grafik akurasi per kelas tersimpan: {per_class_acc_path}") hardest_idx = np.argsort(per_class_acc)[:3] print(" -> 3 digit paling sulit dikenali:") for idx in hardest_idx: print(f" Digit {idx}: akurasi {per_class_acc[idx]:.2f}%") # ----------------------------------------------------------------- # (4) Distribusi Confidence Prediksi (Benar vs Salah) # ----------------------------------------------------------------- print("\n[6/6] Membuat visualisasi distribusi confidence & sampel kesalahan...") fig, ax = plt.subplots(figsize=(9, 5.5)) ax.hist(y_confidence[is_correct], bins=30, alpha=0.65, color="#2ecc71", label=f"Prediksi Benar (n={np.sum(is_correct):,})", density=True) ax.hist(y_confidence[~is_correct], bins=30, alpha=0.75, color="#e74c3c", label=f"Prediksi Salah (n={np.sum(~is_correct):,})", density=True) ax.set_title("Distribusi Confidence Prediksi Model CNN", fontsize=13, fontweight="bold") ax.set_xlabel("Confidence (Probabilitas Softmax Maksimum)") ax.set_ylabel("Kepadatan (Density)") ax.legend() ax.grid(True, alpha=0.3) plt.tight_layout() confidence_path = os.path.join(FIGURES_DIR, "confidence_distribution_cnn.png") plt.savefig(confidence_path, dpi=150) plt.close() print(f" -> Distribusi confidence tersimpan: {confidence_path}") avg_conf_correct = y_confidence[is_correct].mean() * 100 avg_conf_wrong = y_confidence[~is_correct].mean() * 100 print(f" -> Rata-rata confidence saat BENAR : {avg_conf_correct:.2f}%") print(f" -> Rata-rata confidence saat SALAH : {avg_conf_wrong:.2f}%") # ----------------------------------------------------------------- # (5) Contoh Gambar yang Salah Diklasifikasikan # ----------------------------------------------------------------- wrong_idx = np.where(~is_correct)[0] rng = np.random.default_rng(SEED) n_show = min(25, len(wrong_idx)) sample_idx = rng.choice(wrong_idx, size=n_show, replace=False) fig, axes = plt.subplots(5, 5, figsize=(11, 12)) for ax, idx in zip(axes.flat, sample_idx): ax.imshow(x_test[idx].reshape(28, 28), cmap="gray") conf = y_confidence[idx] * 100 ax.set_title(f"True: {y_test[idx]} | Pred: {y_pred[idx]}\nConf: {conf:.1f}%", fontsize=9, color="#c0392b") ax.axis("off") for ax in axes.flat[n_show:]: ax.axis("off") plt.suptitle(f"Contoh {n_show} Gambar Salah Klasifikasi dari {len(wrong_idx)} Total Kesalahan", fontsize=13, fontweight="bold", y=1.0) plt.tight_layout() misclassified_path = os.path.join(FIGURES_DIR, "misclassified_samples_cnn.png") plt.savefig(misclassified_path, dpi=150, bbox_inches="tight") plt.close() print(f" -> Contoh gambar salah klasifikasi tersimpan: {misclassified_path}") # ----------------------------------------------------------------- # Ringkasan Akhir # ----------------------------------------------------------------- summary_path = os.path.join(REPORTS_DIR, "evaluation_summary_cnn.csv") summary_df = pd.DataFrame([{ "test_accuracy_pct": round(test_acc * 100, 4), "test_loss": round(test_loss, 4), "total_errors": int(np.sum(~is_correct)), "total_samples": int(len(y_test)), "avg_confidence_correct_pct": round(avg_conf_correct, 4), "avg_confidence_wrong_pct": round(avg_conf_wrong, 4), "hardest_digit_1": int(hardest_idx[0]), "hardest_digit_1_acc_pct": round(per_class_acc[hardest_idx[0]], 4), "hardest_digit_2": int(hardest_idx[1]), "hardest_digit_2_acc_pct": round(per_class_acc[hardest_idx[1]], 4), }]) summary_df.to_csv(summary_path, index=False) print("\n" + "=" * 75) print("RINGKASAN EVALUASI CNN (ANGGOTA 2)") print("=" * 75) print(f"Test Accuracy : {test_acc * 100:.2f}%") print(f"Test Loss : {test_loss:.4f}") print(f"Total Kesalahan : {np.sum(~is_correct):,} / {len(y_test):,}") print(f"Confidence saat Benar : {avg_conf_correct:.2f}%") print(f"Confidence saat Salah : {avg_conf_wrong:.2f}%") print(f"Digit Paling Sulit : {hardest_idx[0]} ({per_class_acc[hardest_idx[0]]:.2f}%), " f"{hardest_idx[1]} ({per_class_acc[hardest_idx[1]]:.2f}%)") print("-" * 75) print("File yang dihasilkan:") print(f" 1. {metrics_bar_path}") print(f" 2. {cm_dual_path}") print(f" 3. {per_class_acc_path}") print(f" 4. {confidence_path}") print(f" 5. {misclassified_path}") print(f" 6. {report_txt_path}") print(f" 7. {report_csv_path}") print(f" 8. {summary_path}") print("=" * 75 + "\n") if __name__ == "__main__": main()