classification_general_VGG19.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-09-25T17:31:34.030800Z",
     "iopub.status.busy": "2023-09-25T17:31:34.030396Z",
     "iopub.status.idle": "2023-09-25T17:31:44.427795Z",
     "shell.execute_reply": "2023-09-25T17:31:44.426602Z",
     "shell.execute_reply.started": "2023-09-25T17:31:34.030765Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import cv2\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "import matplotlib as mpl\n",
    "import tensorflow_addons as tfa\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "from collections import Counter\n",
    "from sklearn.manifold import TSNE\n",
    "from matplotlib.patches import Rectangle\n",
    "from matplotlib.colors import ListedColormap\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Prepared Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:31:44.430263Z",
     "iopub.status.busy": "2023-09-25T17:31:44.429767Z",
     "iopub.status.idle": "2023-09-25T17:31:45.981652Z",
     "shell.execute_reply": "2023-09-25T17:31:45.980630Z",
     "shell.execute_reply.started": "2023-09-25T17:31:44.430235Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Load the train, validation, and test datasets\n",
    "x_train = np.load('/kaggle/input/5-class-train-test-val/x_train.npy')\n",
    "y_train = np.load('/kaggle/input/5-class-train-test-val/y_train.npy')\n",
    "x_val = np.load('/kaggle/input/5-class-train-test-val/x_val.npy')\n",
    "y_val = np.load('/kaggle/input/5-class-train-test-val/y_val.npy')\n",
    "x_test = np.load('/kaggle/input/5-class-train-test-val/x_test.npy')\n",
    "y_test = np.load('/kaggle/input/5-class-train-test-val/y_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:31:46.029480Z",
     "iopub.status.busy": "2023-09-25T17:31:46.028295Z",
     "iopub.status.idle": "2023-09-25T17:31:46.036425Z",
     "shell.execute_reply": "2023-09-25T17:31:46.035417Z",
     "shell.execute_reply.started": "2023-09-25T17:31:46.029445Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Dictionary to map short class labels to full class names\n",
    "class_short2full = {\n",
    "    \"G\": \"Glaucoma\",  # \"G\" is mapped to \"Glaucoma\"\n",
    "    \"C\": \"Cataract\",  # \"C\" is mapped to \"Cataract\"\n",
    "    \"A\": \"Age Related Macular Degeneration\",  # \"A\" is mapped to \"Age Related Macular Degeneration\"\n",
    "    \"H\": \"Hypertension\",  # \"H\" is mapped to \"Hypertension\"\n",
    "    \"M\": \"Myopia\"  # \"M\" is mapped to \"Myopia\"\n",
    "}\n",
    "\n",
    "# Dictionary to map short class labels to integer class indices\n",
    "class_dict = {\n",
    "    \"G\": 0,  # \"G\" is mapped to index 0\n",
    "    \"C\": 1,  # \"C\" is mapped to index 1\n",
    "    \"A\": 2,  # \"A\" is mapped to index 2\n",
    "    \"H\": 3,  # \"H\" is mapped to index 3\n",
    "    \"M\": 4   # \"M\" is mapped to index 4\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:31:46.037923Z",
     "iopub.status.busy": "2023-09-25T17:31:46.037642Z",
     "iopub.status.idle": "2023-09-25T17:31:46.048878Z",
     "shell.execute_reply": "2023-09-25T17:31:46.047912Z",
     "shell.execute_reply.started": "2023-09-25T17:31:46.037899Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# List of full class names\n",
    "labels_long = [\"Glaucoma\", \"Cataract\", \"AMD\", \"Hypertension\", \"Myopia\"]\n",
    "\n",
    "# List of short class labels (first letter of each full class name)\n",
    "labels_short = [ll[0] for ll in labels_long]  # Extracts the first letter from each full class name\n",
    "\n",
    "# Dictionary to map short class labels to full class names\n",
    "class_short2full = {\n",
    "    ls: ll  # Creates a mapping from short label (first letter) to full class name\n",
    "    for ls, ll in zip(labels_short, labels_long)  # zips the short and long labels together\n",
    "}\n",
    "\n",
    "# Dictionary to map each class (short label) to a unique index (0-based)\n",
    "class_dict = {class_: i for i, class_ in enumerate(class_short2full.keys())}\n",
    "\n",
    "# Reverse dictionary to map each index back to its corresponding short class label\n",
    "class_dict_rev = {v: k for k, v in class_dict.items()}\n",
    "\n",
    "# Number of unique classes\n",
    "NUM_CLASSES = len(class_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:32:07.488794Z",
     "iopub.status.busy": "2023-09-25T17:32:07.488408Z",
     "iopub.status.idle": "2023-09-25T17:32:13.747601Z",
     "shell.execute_reply": "2023-09-25T17:32:13.746794Z",
     "shell.execute_reply.started": "2023-09-25T17:32:07.488762Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Define the image size\n",
    "image_size = 224\n",
    "\n",
    "# Import necessary layers from TensorFlow Keras\n",
    "from tensorflow.keras.layers import Dropout, GlobalAveragePooling2D  \n",
    "from tensorflow.keras.applications.vgg19 import VGG19  \n",
    "\n",
    "# Load the VGG19 model pre-trained on ImageNet, excluding the top layer\n",
    "vgg = VGG19(weights=\"imagenet\", include_top=False, input_shape=(image_size, image_size, 3))\n",
    "\n",
    "# Set all VGG19 layers as trainable\n",
    "for layer in vgg.layers:\n",
    "    layer.trainable = True  # Allow training of all layers in VGG19\n",
    "\n",
    "# Import the Sequential API for model creation\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Flatten, Dense\n",
    "\n",
    "# Initialize the Sequential model\n",
    "model = Sequential()\n",
    "# Add the VGG19 model as a feature extractor (without the top layers)\n",
    "model.add(vgg)\n",
    "\n",
    "# Add Dropout layer to prevent overfitting (rate of 0.6)\n",
    "model.add(Dropout(0.6))\n",
    "# Add Global Average Pooling layer to reduce dimensionality of the output from VGG19\n",
    "model.add(GlobalAveragePooling2D())\n",
    "\n",
    "# Flatten the pooled features for dense layer processing\n",
    "model.add(Flatten())\n",
    "\n",
    "# Add BatchNormalization to standardize activations and improve training speed\n",
    "model.add(tf.keras.layers.BatchNormalization())\n",
    "\n",
    "# Add a dense layer with 256 neurons and ReLU activation\n",
    "model.add(Dense(256, activation=\"relu\"))\n",
    "# Add another dense layer with 128 neurons and ReLU activation\n",
    "# You can also add a Dropout layer here to further reduce overfitting if desired\n",
    "model.add(Dense(128, activation=\"relu\"))\n",
    "# Add a dense layer with 64 neurons and ReLU activation\n",
    "model.add(Dense(64, activation=\"relu\"))\n",
    "# Add the final output layer with 5 neurons for 5 classes, using softmax for multi-class classification\n",
    "model.add(Dense(5, activation=\"softmax\"))\n",
    "# Display the summary of the model architecture\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:32:13.749103Z",
     "iopub.status.busy": "2023-09-25T17:32:13.748732Z",
     "iopub.status.idle": "2023-09-25T17:32:13.764691Z",
     "shell.execute_reply": "2023-09-25T17:32:13.763449Z",
     "shell.execute_reply.started": "2023-09-25T17:32:13.749068Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Define the number of epochs for training\n",
    "epochs = 35\n",
    "\n",
    "# Import necessary callback modules from TensorFlow Keras\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "\n",
    "# Reduce learning rate on plateau callback: reduces LR when the validation loss plateaus\n",
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n",
    "    factor=0.75,  # Multiplies the learning rate by this factor when activated\n",
    "    patience=10,  # Number of epochs to wait for improvement before reducing LR\n",
    "    verbose=1,  # Print messages when learning rate is reduced\n",
    "    min_delta=0.0001,  # Minimum change to qualify as an improvement\n",
    "    cooldown=0,  # Number of epochs to wait before resuming normal learning rate\n",
    "    min_lr=1e-6,  # Minimum learning rate, prevents LR from going below this value\n",
    ")\n",
    "\n",
    "# EarlyStopping callback: stops training if validation loss doesn't improve for 'patience' epochs\n",
    "early_stopping_cb = tf.keras.callbacks.EarlyStopping(\n",
    "    patience=epochs // 10,  # Number of epochs without improvement before stopping\n",
    "    restore_best_weights=True,  # Restore model weights from the epoch with the best performance\n",
    "    verbose=1,  # Print messages when early stopping is triggered\n",
    ")\n",
    "\n",
    "# ModelCheckpoint callback: saves the best model during training based on validation loss\n",
    "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"/kaggle/working/ocir_model_initial.h5\", save_best_only=True)\n",
    "\n",
    "# List of callbacks to be used during model training\n",
    "# callbacks = [checkpoint_cb, early_stopping_cb, reduce_lr]  # Uncomment this line if you want to use early stopping\n",
    "callbacks = [checkpoint_cb, reduce_lr]  # Currently using only checkpoint and reduce_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:32:13.774651Z",
     "iopub.status.busy": "2023-09-25T17:32:13.773377Z",
     "iopub.status.idle": "2023-09-25T17:32:13.819498Z",
     "shell.execute_reply": "2023-09-25T17:32:13.818624Z",
     "shell.execute_reply.started": "2023-09-25T17:32:13.774617Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Define the metrics to be tracked during training\n",
    "METRICS = [\n",
    "    tf.keras.metrics.AUC(name=\"auc\"),  # Area under the ROC curve metric\n",
    "    tf.keras.metrics.BinaryAccuracy(name=\"acc\"),  # Accuracy for binary classification (if used)\n",
    "    tfa.metrics.F1Score(num_classes=NUM_CLASSES, average=\"weighted\", name=\"f1\"),  # F1 score, weighted average across all classes\n",
    "    tf.keras.metrics.AUC(name=\"prc\", curve=\"PR\"),  # Area under the Precision-Recall curve\n",
    "]\n",
    "\n",
    "# Compile the model with Adam optimizer, categorical crossentropy loss, and the specified metrics\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5),  # Adam optimizer with a small learning rate\n",
    "    loss=\"categorical_crossentropy\",  # Loss function for multi-class classification\n",
    "    metrics=METRICS  # List of metrics to evaluate during training\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-09-25T17:32:13.821242Z",
     "iopub.status.busy": "2023-09-25T17:32:13.820739Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Train the model with callbacks\n",
    "history = model.fit(x_train, y_train, batch_size=128, epochs=epochs, validation_data=(x_val, y_val), callbacks=callbacks)\n",
    "print(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Retrieve training and validation accuracy and loss from history\n",
    "training_accuracy = history.history['acc']  # Training accuracy values from the history object\n",
    "validation_accuracy = history.history['val_acc']  # Validation accuracy values\n",
    "\n",
    "training_loss = history.history['loss']  # Training loss values\n",
    "validation_loss = history.history['val_loss']  # Validation loss values\n",
    "\n",
    "training_auc = history.history['auc']  # Training AUC (Area Under the ROC Curve) values\n",
    "validation_auc = history.history['val_auc']  # Validation AUC values\n",
    "\n",
    "training_f1 = history.history['f1']  # Training F1 score values\n",
    "validation_f1 = history.history['val_f1']  # Validation F1 score values\n",
    "\n",
    "training_prc = history.history['prc']  # Training PRC (Precision-Recall Curve) values\n",
    "validation_prc = history.history['val_prc']  # Validation PRC values\n",
    "\n",
    "# Plot Accuracy\n",
    "plt.plot(training_accuracy, label='Training Accuracy')\n",
    "plt.plot(validation_accuracy, label='Validation Accuracy')\n",
    "plt.title('Training and Validation Accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# Plot Loss\n",
    "plt.plot(training_loss, label='Training Loss')\n",
    "plt.plot(validation_loss, label='Validation Loss')\n",
    "plt.title('Training and Validation Loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# Plot Area Under the Curve\n",
    "plt.plot(training_auc, label='Training AUC')\n",
    "plt.plot(validation_auc, label='Validation AUC')\n",
    "plt.title('Training and Validation AUC')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('AUC')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# Plot F1\n",
    "plt.plot(training_f1, label='Training F1')\n",
    "plt.plot(validation_f1, label='Validation F1')\n",
    "plt.title('Training and Validation F1')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('F1')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "# Plot Precision-Recall Curve\n",
    "plt.plot(training_prc, label='Training PRC')\n",
    "plt.plot(validation_prc, label='Validation PRC')\n",
    "plt.title('Training and Validation PRC')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('PRC')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Evaluate the model on the test data (x_test and y_test)\n",
    "loss, auc, acc, f1, prc = model.evaluate(x_test, y_test)  # Model evaluation returns multiple metrics: loss, auc, accuracy, f1 score, and prc\n",
    "\n",
    "# Print the accuracy of the model on the test data\n",
    "print(\"Accuracy:\", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Post Processes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a random seed for reproducibility\n",
    "SEED = 42\n",
    "\n",
    "# Define color palette for different plot elements\n",
    "COLORS = {\n",
    "    \"fig_bg\": \"#f6f5f5\",              # Background color of the figure\n",
    "    \"plot_neut\": \"#ddbea9\",           # Neutral color for plots\n",
    "    \"plot_text\": \"#343a40\",           # Text color for plots\n",
    "    \n",
    "    # A list of colors for plotting (for different categories or groups)\n",
    "    \"cmap_color_list\": [\"#001219\", \"#005F73\", \"#0A9396\", \"#94D2BD\", \"#E9D8A6\",\n",
    "                        \"#EE9B00\", \"#CA6702\", \"#BB3E03\", \"#AE2012\", \"#9B2226\"],\n",
    "    \n",
    "    # Colors for different splits of data\n",
    "    \"split\": {\n",
    "        \"train\": \"#264653\",          # Training data color\n",
    "        \"val\": \"#2a9d8f\",            # Validation data color\n",
    "        \"test\": \"#e9c46a\"            # Test data color\n",
    "    }\n",
    "}\n",
    "\n",
    "# Assign colors to each class from the 'class_short2full' dictionary\n",
    "# 'class_short2full' contains the mapping from short class labels to full names\n",
    "COLORS[\"class\"] = {ls: c for ls, c in zip(class_short2full.keys(), COLORS[\"cmap_color_list\"][:len(class_short2full.keys())])}\n",
    "\n",
    "# Define color maps for plots, from a list of color codes\n",
    "COLORS[\"cmap\"] = mpl.colors.LinearSegmentedColormap.from_list(\"\", COLORS[\"cmap_color_list\"])\n",
    "COLORS[\"cmap_pos\"] = mpl.colors.LinearSegmentedColormap.from_list(\"\", [\"#F0F3F8\", \"#D1DBE9\", \"#A2B7D2\", \"#7493BC\", \"#6487B4\", \"#3D5A80\"])\n",
    "\n",
    "# List of colors associated with each class for easy access\n",
    "colors_class_list = list(COLORS[\"class\"].values())\n",
    "\n",
    "# Font settings for various plot titles, labels, and texts\n",
    "FONT_KW = {\n",
    "    \"plot_title\" : {                # Font settings for main plot title\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"25\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"plot_title_small\" : {          # Font settings for smaller plot title\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"16\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"plot_subtitle\" : {             # Font settings for plot subtitle\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"12\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"subplot_title\" : {             # Font settings for subplot title\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"18\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"subplot_title_small\" : {       # Font settings for smaller subplot title\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"12\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"plot_label\" : {                # Font settings for plot labels\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"16\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"plot_label_small\" : {          # Font settings for smaller plot labels\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"bold\",           # Bold text\n",
    "        \"size\": \"12\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"plot_text\" : {                 # Font settings for general plot text\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"normal\",         # Normal weight\n",
    "        \"size\": \"12\",               # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "    \"plot_text_small\" : {           # Font settings for smaller plot text\n",
    "        \"fontname\": \"serif\",        # Font type (serif)\n",
    "        \"weight\": \"normal\",         # Normal weight\n",
    "        \"size\": \"8\",                # Font size\n",
    "        \"style\": \"normal\"           # Normal style\n",
    "    },\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Function to plot confusion matrix and its normalized version\n",
    "def plot_confusion_matrix(y_true, y_pred, figsize=(16,6), cmap=\"Blues\", suptitle=None):\n",
    "    # Calculate confusion matrix and its normalized version\n",
    "    cm = confusion_matrix(y_true, y_pred)  # Standard confusion matrix\n",
    "    cm_norm = confusion_matrix(y_true, y_pred, normalize=\"true\")  # Normalized confusion matrix\n",
    "    \n",
    "    # Create subplots for confusion matrix and normalized confusion matrix\n",
    "    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=figsize)\n",
    "    fig.patch.set_facecolor(COLORS[\"fig_bg\"])  # Set the background color for the figure\n",
    "\n",
    "    # Plot the standard confusion matrix on the first axis (ax1)\n",
    "    sns.heatmap(\n",
    "        cm,  # Input confusion matrix\n",
    "        annot=True,  # Annotate with the values\n",
    "        annot_kws=FONT_KW[\"plot_text\"],  # Font size and style for annotations\n",
    "        fmt=\"d\",  # Format as integers\n",
    "        linewidths=3.0,  # Line width between cells\n",
    "        linecolor=COLORS[\"fig_bg\"],  # Line color for separation\n",
    "        cmap=cmap,  # Color map\n",
    "        cbar=False,  # No color bar\n",
    "        square=True,  # Square aspect ratio\n",
    "        xticklabels=[class_short2full[k] for k in class_dict.keys()],  # X-axis labels (class names)\n",
    "        yticklabels=[class_short2full[k] for k in class_dict.keys()],  # Y-axis labels (class names)\n",
    "        ax=ax1  # Plot on first subplot\n",
    "    )\n",
    "\n",
    "    # Customize ax1 title and labels\n",
    "    ax1.set_title(\"Confusion Matrix\", **FONT_KW[\"subplot_title_small\"])\n",
    "    ax1.set_xlabel(\"Predicted Labels\", **FONT_KW[\"plot_label_small\"])\n",
    "    ax1.set_ylabel(\"True Labels\", **FONT_KW[\"plot_label_small\"])\n",
    "    \n",
    "    # Set background color and tick parameters for ax1\n",
    "    ax1.set_facecolor(COLORS[\"fig_bg\"])\n",
    "    ax1.tick_params(axis=\"both\", length=0)\n",
    "    ax1.set_yticks(\n",
    "        np.arange(len(ax1.get_yticklabels()))+0.5,  # Adjust ticks for better positioning\n",
    "        [label.get_text() for label in ax1.get_yticklabels()],\n",
    "        **FONT_KW[\"plot_label_small\"]\n",
    "    )\n",
    "    ax1.set_xticks(\n",
    "        np.arange(len(ax1.get_xticklabels()))+0.5,  # Adjust ticks for better positioning\n",
    "        [label.get_text() for label in ax1.get_xticklabels()],\n",
    "        **FONT_KW[\"plot_label_small\"]\n",
    "    )\n",
    "    \n",
    "    # Color class labels according to class colors\n",
    "    for class_, i in class_dict.items():\n",
    "        ax1.get_xticklabels()[i].set_color(COLORS[\"class\"][class_])\n",
    "        ax1.get_yticklabels()[i].set_color(COLORS[\"class\"][class_])\n",
    "\n",
    "    # Plot the normalized confusion matrix on the second axis (ax2)\n",
    "    sns.heatmap(\n",
    "        cm_norm,  # Input normalized confusion matrix\n",
    "        annot=True,  # Annotate with the values\n",
    "        annot_kws=FONT_KW[\"plot_text\"],  # Font size and style for annotations\n",
    "        fmt= \".0%\" if np.all(np.allclose(cm_norm, cm_norm.astype(int))) else \".1%\",  # Format as percentage\n",
    "        linewidths=3.0,  # Line width between cells\n",
    "        linecolor=COLORS[\"fig_bg\"],  # Line color for separation\n",
    "        cmap=cmap,  # Color map\n",
    "        cbar=False,  # No color bar\n",
    "        square=True,  # Square aspect ratio\n",
    "        xticklabels=[class_short2full[k] for k in class_dict.keys()],  # X-axis labels (class names)\n",
    "        yticklabels=[class_short2full[k] for k in class_dict.keys()],  # Y-axis labels (class names)\n",
    "        ax=ax2  # Plot on second subplot\n",
    "    )\n",
    "\n",
    "    # Customize ax2 title and labels\n",
    "    ax2.set_title(\"Confusion Matrix (Normalized)\", **FONT_KW[\"subplot_title_small\"])\n",
    "    ax2.set_xlabel(\"Predicted Labels\", **FONT_KW[\"plot_label_small\"])\n",
    "    ax2.set_ylabel(\"True Labels\", **FONT_KW[\"plot_label_small\"])\n",
    "    \n",
    "    # Set background color and tick parameters for ax2\n",
    "    ax2.set_facecolor(COLORS[\"fig_bg\"])\n",
    "    ax2.tick_params(axis=\"both\", length=0)\n",
    "    ax2.set_yticks(\n",
    "        np.arange(len(ax2.get_yticklabels()))+0.5,  # Adjust ticks for better positioning\n",
    "        [label.get_text() for label in ax2.get_yticklabels()],\n",
    "        **FONT_KW[\"plot_label_small\"]\n",
    "    )\n",
    "    ax2.set_xticks(\n",
    "        np.arange(len(ax2.get_xticklabels()))+0.5,  # Adjust ticks for better positioning\n",
    "        [label.get_text() for label in ax2.get_xticklabels()],\n",
    "        **FONT_KW[\"plot_label_small\"]\n",
    "    )\n",
    "        \n",
    "    # Color class labels according to class colors for ax2\n",
    "    for class_, i in class_dict.items():\n",
    "        ax2.get_xticklabels()[i].set_color(COLORS[\"class\"][class_])\n",
    "        ax2.get_yticklabels()[i].set_color(COLORS[\"class\"][class_])\n",
    "    \n",
    "    # If a suptitle is provided, display it at the top of the figure\n",
    "    if suptitle is not None:\n",
    "        plt.suptitle(suptitle, y=0.98, **FONT_KW[\"plot_title_small\"])\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Predict class probabilities for the validation set and convert them to class labels\n",
    "y_val_pred = np.argmax(model.predict(x_val), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Plot confusion matrix for the validation set\n",
    "# The true labels are obtained by converting one-hot encoded labels to class labels (using np.argmax).\n",
    "# The predicted labels are compared to the true labels to visualize the model's performance.\n",
    "\n",
    "plot_confusion_matrix(\n",
    "    np.argmax(y_val, axis=1),  # Convert one-hot encoded labels to class labels for true labels\n",
    "    y_val_pred,  # Predicted labels from the model\n",
    "    figsize=(15,4),  # Set the figure size for the plot\n",
    "    cmap=COLORS[\"cmap_pos\"],  # Set the color map for the plot\n",
    "    suptitle=\"Model Performance (Validation)\"  # Set the title of the plot\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Predict class probabilities for the test set\n",
    "y_test_prob = model.predict(x_test)  # Get the predicted probabilities for each class\n",
    "\n",
    "# Convert the predicted probabilities to class labels\n",
    "y_test_pred = np.argmax(y_test_prob, axis=1)  # Get the class with the highest probability for each sample\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Plot the confusion matrix for the test set\n",
    "plot_confusion_matrix(\n",
    "    np.argmax(y_test, axis=1), y_test_pred,\n",
    "    figsize=(15,4),\n",
    "    cmap=COLORS[\"cmap_pos\"],\n",
    "    suptitle=\"Model Performance (Test)\"\n",
    ")\n",
    "\n",
    "# Generate a classification report which includes precision, recall, and F1-score\n",
    "report = classification_report(\n",
    "    np.argmax(y_test, axis=1), y_test_pred,  # Actual labels and predicted labels\n",
    "    target_names=[class_short2full[k] for k in class_dict.keys()]  # Class names mapped from short labels\n",
    ")\n",
    "\n",
    "# Compute the ROC AUC score for the test set (multiclass, one-vs-rest strategy)\n",
    "test_roc_auc = roc_auc_score(np.argmax(y_test, axis=1), y_test_prob, multi_class='ovr')\n",
    "\n",
    "print(report)\n",
    "print(f\"     roc auc       {np.round(test_roc_auc, 2)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import pickle  # Import the pickle module for loading serialized data\n",
    "\n",
    "# Define the folder path where the pickle files are stored\n",
    "folder_path = \"/kaggle/working/\"\n",
    "\n",
    "# Get a list of file paths for all files ending with \".pkl\" in the specified folder\n",
    "filepaths = [os.path.join(folder_path, file) for file in os.listdir(folder_path) if file.endswith(\".pkl\")]\n",
    "\n",
    "# Loop over each file path to load and process the pickle data\n",
    "for filepath in filepaths:\n",
    "    filename = os.path.basename(filepath)  # Extract the file name from the file path\n",
    "    \n",
    "    # Open and load the pickle data from the file\n",
    "    with open(filepath, \"rb\") as f:\n",
    "        data = pickle.load(f)\n",
    "    \n",
    "    # Extract the maximum value for all metrics except for loss/val_loss where minimum value is taken\n",
    "    results = {\n",
    "        key: max(values) if key not in [\"val_loss\", \"loss\"] else min(values)\n",
    "        for key, values in data.items()\n",
    "    }\n",
    "\n",
    "    print(f\"Results for {filename}:\")\n",
    "    for key, value in results.items():  # Loop over the results dictionary to print each metric and its value\n",
    "        print(f\" {key}: {value:.4f}\")\n",
    "    print(\"------\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import scipy.io as sio  # Import the scipy.io module for saving MATLAB .mat files\n",
    "\n",
    "# Function to convert .pkl files in a directory to .mat files\n",
    "def convert_pkl_to_mat(directory):\n",
    "    # Loop through all files in the specified directory\n",
    "    for filename in os.listdir(directory):\n",
    "        # Check if the current file is a .pkl file\n",
    "        if filename.endswith(\".pkl\"):\n",
    "            filepath = os.path.join(directory, filename)  # Get the full path of the .pkl file\n",
    "            \n",
    "            # Open and load the .pkl file\n",
    "            with open(filepath, \"rb\") as file:\n",
    "                data = pickle.load(file)\n",
    "            \n",
    "            # Define the path for the output .mat file by changing the file extension to .mat\n",
    "            mat_filepath = os.path.splitext(filepath)[0] + \".mat\"\n",
    "            \n",
    "            # Save the loaded data to a .mat file\n",
    "            sio.io.savemat(mat_filepath, {\"data\": data})\n",
    "            \n",
    "            # Print a message confirming the conversion\n",
    "            print(f\"Converted {filename} to {os.path.basename(mat_filepath)}\")\n",
    "\n",
    "# Entry point of the script\n",
    "if __name__ == \"__main__\":\n",
    "    dir_path = \"/kaggle/working/\"\n",
    "    convert_pkl_to_mat(dir_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open the 'history.pkl' file and load the training history data using pickle\n",
    "with open('history.pkl', 'rb') as file:\n",
    "    history = pickle.load(file)\n",
    "\n",
    "# Create a dictionary to organize the training history data into a structured format\n",
    "data = {\n",
    "    # Extract training and validation accuracy values from history\n",
    "    'training_accuracy': history.history['acc'],\n",
    "    'validation_accuracy': history.history['val_acc'],\n",
    "\n",
    "    # Extract training and validation loss values from history\n",
    "    'training_loss': history.history['loss'],\n",
    "    'validation_loss': history.history['val_loss'],\n",
    "\n",
    "    # Extract training and validation AUC (Area Under the Curve) values from history\n",
    "    'training_auc': history.history['auc'],\n",
    "    'validation_auc': history.history['val_auc'],  # AUC (Area Under the Curve)\n",
    "\n",
    "    # Extract training and validation F1 score values from history\n",
    "    'training_f1': history.history['f1'],\n",
    "    'validation_f1': history.history['val_f1'],\n",
    "\n",
    "    # Extract training and validation PRC (Precision-Recall Curve) values from history\n",
    "    'training_prc': history.history['prc'],\n",
    "    'validation_prc': history.history['val_prc'],  # PRC (Precision Recall Curve)\n",
    "}\n",
    "\n",
    "# Save the training history data as a .mat file using scipy's savemat function\n",
    "sio.savemat('history.mat', {'data': data})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "trusted": true
   },
   "outputs": [],
   "source": [
    "# Save the model\n",
    "model.save(\"ocular_disease_model.h5\")\n",
    "\n",
    "print(\"Model saved successfully!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}