303 lines
14 KiB
Python
303 lines
14 KiB
Python
"""
|
|
Tugas Anggota 2 (Week 2): CNN Evaluation + Visualization
|
|
=========================================================
|
|
PENTING - KONSISTENSI KONFIGURASI DENGAN ANGGOTA 1:
|
|
Skrip ini TIDAK melatih ulang model dari nol. Sebagai gantinya, skrip ini
|
|
memuat bobot model CNN yang sudah dilatih oleh Anggota 1
|
|
(results/models/model_cnn_mnist.keras) pada 05_cnn_full_pipeline.py.
|
|
|
|
Ini adalah cara paling aman untuk menjamin "tidak ada konfigurasi berbeda"
|
|
antara Anggota 1 dan Anggota 2, karena:
|
|
- Dataset yang sama -> load_mnist_for_cnn() (src/data_loader.py)
|
|
- Preprocessing yang sama -> normalisasi 0-1, reshape (28, 28, 1)
|
|
- Epoch yang sama -> 10 epoch (bobot hasil training Anggota 1)
|
|
- Batch size yang sama -> 128 (dipakai saat training oleh Anggota 1)
|
|
- Optimizer yang sama -> Adam (tersimpan di dalam file .keras)
|
|
|
|
Konstanta di bawah HANYA didokumentasikan ulang (bukan dipakai untuk
|
|
melatih ulang) agar terlihat eksplisit bahwa konfigurasi cocok dengan
|
|
skrip Anggota 1.
|
|
|
|
Cakupan tugas (sesuai skema kelompok):
|
|
(1) Evaluasi menyeluruh pada test set (classification report per kelas)
|
|
(2) Visualisasi confusion matrix (raw & ternormalisasi/persentase)
|
|
(3) Visualisasi akurasi per kelas (digit mana yang paling sulit)
|
|
(4) Visualisasi distribusi confidence prediksi (benar vs salah)
|
|
(5) Visualisasi contoh gambar yang salah diklasifikasikan
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
# Sembunyikan log TensorFlow yang berlebihan
|
|
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
|
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
import pandas as pd
|
|
import seaborn as sns
|
|
from sklearn.metrics import classification_report, confusion_matrix
|
|
import tensorflow as tf
|
|
from tensorflow import keras
|
|
|
|
# Tambahkan root directory ke path
|
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
|
|
|
from src.data_loader import load_mnist_for_cnn
|
|
from src.utils import ensure_dir
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# KONFIGURASI (harus identik dengan Anggota 1 / 05_cnn_full_pipeline.py)
|
|
# ---------------------------------------------------------------------------
|
|
SEED = 42
|
|
EPOCHS = 10 # epoch training model (didokumentasikan, tidak dipakai ulang)
|
|
BATCH_SIZE = 128 # batch size training model (didokumentasikan, tidak dipakai ulang)
|
|
OPTIMIZER = "adam" # optimizer training model (tersimpan di file .keras)
|
|
|
|
MODEL_PATH = os.path.join("results", "models", "model_cnn_mnist.keras")
|
|
FIGURES_DIR = os.path.join("results", "figures")
|
|
REPORTS_DIR = os.path.join("results", "reports")
|
|
|
|
CLASS_NAMES = [str(i) for i in range(10)]
|
|
|
|
|
|
def main():
|
|
tf.random.set_seed(SEED)
|
|
np.random.seed(SEED)
|
|
|
|
ensure_dir(FIGURES_DIR)
|
|
ensure_dir(REPORTS_DIR)
|
|
|
|
print("=" * 75)
|
|
print("TUGAS ANGGOTA 2: CNN EVALUATION + VISUALIZATION")
|
|
print("=" * 75)
|
|
print(
|
|
f"[KONFIGURASI - selaras dengan Anggota 1] "
|
|
f"seed={SEED} | epochs={EPOCHS} | batch_size={BATCH_SIZE} | optimizer={OPTIMIZER}"
|
|
)
|
|
|
|
# 1. Load Data (dataset & preprocessing sama persis dengan Anggota 1)
|
|
print("\n[1/6] Memuat dataset MNIST (load_mnist_for_cnn - sama dengan Anggota 1)...")
|
|
(x_train, y_train), (x_test, y_test) = load_mnist_for_cnn()
|
|
print(f" -> Data testing : {x_test.shape[0]:,} gambar {x_test.shape[1:]}")
|
|
|
|
# 2. Load Model Terlatih dari Anggota 1
|
|
print(f"\n[2/6] Memuat model terlatih dari: {MODEL_PATH}")
|
|
if not os.path.exists(MODEL_PATH):
|
|
raise FileNotFoundError(
|
|
f"Model tidak ditemukan di '{MODEL_PATH}'. "
|
|
f"Jalankan dahulu experiments/05_cnn_full_pipeline.py (skrip Anggota 1)."
|
|
)
|
|
model = keras.models.load_model(MODEL_PATH)
|
|
print(f" -> Model '{model.name}' berhasil dimuat ({model.count_params():,} parameter).")
|
|
|
|
# 3. Evaluasi Umum pada Test Set
|
|
print("\n[3/6] Mengevaluasi model pada test set...")
|
|
test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0)
|
|
y_pred_probs = model.predict(x_test, verbose=0)
|
|
y_pred = np.argmax(y_pred_probs, axis=1)
|
|
y_confidence = np.max(y_pred_probs, axis=1)
|
|
is_correct = (y_pred == y_test)
|
|
|
|
print(f" -> Test Accuracy : {test_acc * 100:.2f}%")
|
|
print(f" -> Test Loss : {test_loss:.4f}")
|
|
print(f" -> Total Salah : {np.sum(~is_correct):,} dari {len(y_test):,} gambar")
|
|
|
|
# -----------------------------------------------------------------
|
|
# (1) Classification Report per Kelas
|
|
# -----------------------------------------------------------------
|
|
print("\n[4/6] Membuat classification report per kelas...")
|
|
report_dict = classification_report(
|
|
y_test, y_pred, target_names=CLASS_NAMES, digits=4, output_dict=True
|
|
)
|
|
report_text = classification_report(y_test, y_pred, target_names=CLASS_NAMES, digits=4)
|
|
print("\n" + "-" * 75)
|
|
print("CLASSIFICATION REPORT (Precision / Recall / F1-Score per Digit):")
|
|
print("-" * 75)
|
|
print(report_text)
|
|
|
|
report_txt_path = os.path.join(REPORTS_DIR, "classification_report_cnn.txt")
|
|
with open(report_txt_path, "w") as f:
|
|
f.write("CLASSIFICATION REPORT - CNN MNIST (Anggota 2: CNN Evaluation)\n")
|
|
f.write(f"Test Accuracy: {test_acc * 100:.2f}% | Test Loss: {test_loss:.4f}\n")
|
|
f.write("=" * 65 + "\n")
|
|
f.write(report_text)
|
|
print(f" -> Laporan teks tersimpan: {report_txt_path}")
|
|
|
|
report_df = pd.DataFrame(report_dict).transpose()
|
|
report_csv_path = os.path.join(REPORTS_DIR, "classification_report_cnn.csv")
|
|
report_df.to_csv(report_csv_path)
|
|
print(f" -> Laporan CSV tersimpan : {report_csv_path}")
|
|
|
|
# Visualisasi precision/recall/f1 per kelas (bar chart berkelompok)
|
|
per_class_df = report_df.loc[CLASS_NAMES, ["precision", "recall", "f1-score"]]
|
|
fig, ax = plt.subplots(figsize=(11, 5))
|
|
per_class_df.plot(kind="bar", ax=ax, width=0.75,
|
|
color=["#3498db", "#2ecc71", "#e74c3c"])
|
|
ax.set_title("Precision, Recall, dan F1-Score per Digit (CNN)", fontsize=13, fontweight="bold")
|
|
ax.set_xlabel("Digit", fontsize=11)
|
|
ax.set_ylabel("Skor", fontsize=11)
|
|
ax.set_ylim(0.90, 1.005)
|
|
ax.set_xticklabels(CLASS_NAMES, rotation=0)
|
|
ax.legend(["Precision", "Recall", "F1-Score"], loc="lower right")
|
|
ax.grid(True, axis="y", alpha=0.3)
|
|
plt.tight_layout()
|
|
metrics_bar_path = os.path.join(FIGURES_DIR, "per_class_metrics_cnn.png")
|
|
plt.savefig(metrics_bar_path, dpi=150)
|
|
plt.close()
|
|
print(f" -> Grafik metrik per kelas tersimpan: {metrics_bar_path}")
|
|
|
|
# -----------------------------------------------------------------
|
|
# (2) Confusion Matrix: Raw Count & Ternormalisasi (%)
|
|
# -----------------------------------------------------------------
|
|
print("\n[5/6] Membuat visualisasi confusion matrix (raw & normalisasi)...")
|
|
cm = confusion_matrix(y_test, y_pred)
|
|
cm_norm = cm.astype("float") / cm.sum(axis=1, keepdims=True) * 100
|
|
|
|
fig, axes = plt.subplots(1, 2, figsize=(17, 7))
|
|
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", cbar=True,
|
|
xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES,
|
|
annot_kws={"size": 9}, ax=axes[0])
|
|
axes[0].set_title("Confusion Matrix (Jumlah Gambar)", fontsize=12, fontweight="bold")
|
|
axes[0].set_xlabel("Predicted Digit")
|
|
axes[0].set_ylabel("True Digit")
|
|
|
|
sns.heatmap(cm_norm, annot=True, fmt=".1f", cmap="Oranges", cbar=True,
|
|
xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES,
|
|
annot_kws={"size": 9}, ax=axes[1])
|
|
axes[1].set_title("Confusion Matrix (Persentase per Baris / Recall %)", fontsize=12, fontweight="bold")
|
|
axes[1].set_xlabel("Predicted Digit")
|
|
axes[1].set_ylabel("True Digit")
|
|
|
|
plt.suptitle("Evaluasi CNN pada MNIST Test Set (10.000 Gambar)", fontsize=14, fontweight="bold", y=1.02)
|
|
plt.tight_layout()
|
|
cm_dual_path = os.path.join(FIGURES_DIR, "confusion_matrix_cnn_detail.png")
|
|
plt.savefig(cm_dual_path, dpi=150, bbox_inches="tight")
|
|
plt.close()
|
|
print(f" -> Confusion matrix detail tersimpan: {cm_dual_path}")
|
|
|
|
# -----------------------------------------------------------------
|
|
# (3) Akurasi per Kelas (diagonal confusion matrix ternormalisasi)
|
|
# -----------------------------------------------------------------
|
|
per_class_acc = np.diag(cm_norm)
|
|
fig, ax = plt.subplots(figsize=(10, 5))
|
|
colors = ["#e74c3c" if a < per_class_acc.mean() else "#2ecc71" for a in per_class_acc]
|
|
bars = ax.bar(CLASS_NAMES, per_class_acc, color=colors)
|
|
ax.axhline(per_class_acc.mean(), color="gray", linestyle="--", linewidth=1.5,
|
|
label=f"Rata-rata: {per_class_acc.mean():.2f}%")
|
|
for bar, acc in zip(bars, per_class_acc):
|
|
ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() - 1.2,
|
|
f"{acc:.1f}%", ha="center", va="top", fontsize=9, color="white", fontweight="bold")
|
|
ax.set_title("Akurasi per Digit pada Test Set (CNN)", fontsize=13, fontweight="bold")
|
|
ax.set_xlabel("Digit")
|
|
ax.set_ylabel("Akurasi (%)")
|
|
ax.set_ylim(min(per_class_acc) - 5, 100.5)
|
|
ax.legend()
|
|
ax.grid(True, axis="y", alpha=0.3)
|
|
plt.tight_layout()
|
|
per_class_acc_path = os.path.join(FIGURES_DIR, "per_class_accuracy_cnn.png")
|
|
plt.savefig(per_class_acc_path, dpi=150)
|
|
plt.close()
|
|
print(f" -> Grafik akurasi per kelas tersimpan: {per_class_acc_path}")
|
|
|
|
hardest_idx = np.argsort(per_class_acc)[:3]
|
|
print(" -> 3 digit paling sulit dikenali:")
|
|
for idx in hardest_idx:
|
|
print(f" Digit {idx}: akurasi {per_class_acc[idx]:.2f}%")
|
|
|
|
# -----------------------------------------------------------------
|
|
# (4) Distribusi Confidence Prediksi (Benar vs Salah)
|
|
# -----------------------------------------------------------------
|
|
print("\n[6/6] Membuat visualisasi distribusi confidence & sampel kesalahan...")
|
|
fig, ax = plt.subplots(figsize=(9, 5.5))
|
|
ax.hist(y_confidence[is_correct], bins=30, alpha=0.65, color="#2ecc71",
|
|
label=f"Prediksi Benar (n={np.sum(is_correct):,})", density=True)
|
|
ax.hist(y_confidence[~is_correct], bins=30, alpha=0.75, color="#e74c3c",
|
|
label=f"Prediksi Salah (n={np.sum(~is_correct):,})", density=True)
|
|
ax.set_title("Distribusi Confidence Prediksi Model CNN", fontsize=13, fontweight="bold")
|
|
ax.set_xlabel("Confidence (Probabilitas Softmax Maksimum)")
|
|
ax.set_ylabel("Kepadatan (Density)")
|
|
ax.legend()
|
|
ax.grid(True, alpha=0.3)
|
|
plt.tight_layout()
|
|
confidence_path = os.path.join(FIGURES_DIR, "confidence_distribution_cnn.png")
|
|
plt.savefig(confidence_path, dpi=150)
|
|
plt.close()
|
|
print(f" -> Distribusi confidence tersimpan: {confidence_path}")
|
|
|
|
avg_conf_correct = y_confidence[is_correct].mean() * 100
|
|
avg_conf_wrong = y_confidence[~is_correct].mean() * 100
|
|
print(f" -> Rata-rata confidence saat BENAR : {avg_conf_correct:.2f}%")
|
|
print(f" -> Rata-rata confidence saat SALAH : {avg_conf_wrong:.2f}%")
|
|
|
|
# -----------------------------------------------------------------
|
|
# (5) Contoh Gambar yang Salah Diklasifikasikan
|
|
# -----------------------------------------------------------------
|
|
wrong_idx = np.where(~is_correct)[0]
|
|
rng = np.random.default_rng(SEED)
|
|
n_show = min(25, len(wrong_idx))
|
|
sample_idx = rng.choice(wrong_idx, size=n_show, replace=False)
|
|
|
|
fig, axes = plt.subplots(5, 5, figsize=(11, 12))
|
|
for ax, idx in zip(axes.flat, sample_idx):
|
|
ax.imshow(x_test[idx].reshape(28, 28), cmap="gray")
|
|
conf = y_confidence[idx] * 100
|
|
ax.set_title(f"True: {y_test[idx]} | Pred: {y_pred[idx]}\nConf: {conf:.1f}%",
|
|
fontsize=9, color="#c0392b")
|
|
ax.axis("off")
|
|
for ax in axes.flat[n_show:]:
|
|
ax.axis("off")
|
|
plt.suptitle(f"Contoh {n_show} Gambar Salah Klasifikasi dari {len(wrong_idx)} Total Kesalahan",
|
|
fontsize=13, fontweight="bold", y=1.0)
|
|
plt.tight_layout()
|
|
misclassified_path = os.path.join(FIGURES_DIR, "misclassified_samples_cnn.png")
|
|
plt.savefig(misclassified_path, dpi=150, bbox_inches="tight")
|
|
plt.close()
|
|
print(f" -> Contoh gambar salah klasifikasi tersimpan: {misclassified_path}")
|
|
|
|
# -----------------------------------------------------------------
|
|
# Ringkasan Akhir
|
|
# -----------------------------------------------------------------
|
|
summary_path = os.path.join(REPORTS_DIR, "evaluation_summary_cnn.csv")
|
|
summary_df = pd.DataFrame([{
|
|
"test_accuracy_pct": round(test_acc * 100, 4),
|
|
"test_loss": round(test_loss, 4),
|
|
"total_errors": int(np.sum(~is_correct)),
|
|
"total_samples": int(len(y_test)),
|
|
"avg_confidence_correct_pct": round(avg_conf_correct, 4),
|
|
"avg_confidence_wrong_pct": round(avg_conf_wrong, 4),
|
|
"hardest_digit_1": int(hardest_idx[0]),
|
|
"hardest_digit_1_acc_pct": round(per_class_acc[hardest_idx[0]], 4),
|
|
"hardest_digit_2": int(hardest_idx[1]),
|
|
"hardest_digit_2_acc_pct": round(per_class_acc[hardest_idx[1]], 4),
|
|
}])
|
|
summary_df.to_csv(summary_path, index=False)
|
|
|
|
print("\n" + "=" * 75)
|
|
print("RINGKASAN EVALUASI CNN (ANGGOTA 2)")
|
|
print("=" * 75)
|
|
print(f"Test Accuracy : {test_acc * 100:.2f}%")
|
|
print(f"Test Loss : {test_loss:.4f}")
|
|
print(f"Total Kesalahan : {np.sum(~is_correct):,} / {len(y_test):,}")
|
|
print(f"Confidence saat Benar : {avg_conf_correct:.2f}%")
|
|
print(f"Confidence saat Salah : {avg_conf_wrong:.2f}%")
|
|
print(f"Digit Paling Sulit : {hardest_idx[0]} ({per_class_acc[hardest_idx[0]]:.2f}%), "
|
|
f"{hardest_idx[1]} ({per_class_acc[hardest_idx[1]]:.2f}%)")
|
|
print("-" * 75)
|
|
print("File yang dihasilkan:")
|
|
print(f" 1. {metrics_bar_path}")
|
|
print(f" 2. {cm_dual_path}")
|
|
print(f" 3. {per_class_acc_path}")
|
|
print(f" 4. {confidence_path}")
|
|
print(f" 5. {misclassified_path}")
|
|
print(f" 6. {report_txt_path}")
|
|
print(f" 7. {report_csv_path}")
|
|
print(f" 8. {summary_path}")
|
|
print("=" * 75 + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|