graph_compare.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare the 3 techniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os \n",
    "sns.set()\n",
    "\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# https://abdalimran.github.io/2019-06-01/Drawing-multiple-ROC-Curves-in-a-single-plot\n",
    "\n",
    "# Define paths\n",
    "base_path = os.path.abspath(\"./\") # Your root directory\n",
    "dataset_path = os.path.join(base_path, \"dataset\") # Your dataset folder\n",
    "model_path = os.path.join(base_path, \"models\")\n",
    "log_path = os.path.join(base_path, \"logs\")\n",
    "print(base_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helpers\n",
    "import pickle \n",
    "\n",
    "def save_obj(obj, name):\n",
    "    with open('{}'.format(log_path+\"/\") + name + '.pkl', 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "\n",
    "def load_obj(name):\n",
    "    with open('{}'.format(log_path+\"/\") + name + '.pkl', 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "\n",
    "def get_unique(arr):\n",
    "    '''\n",
    "    Return unique elements in list\n",
    "    '''\n",
    "    arr = set(arr)\n",
    "    return arr\n",
    "\n",
    "#raw_features = load_obj('raw_features')\n",
    "#target = load_obj('target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sss = load_obj('sss')\n",
    "siz = load_obj('ess')\n",
    "ess = load_obj('siz')\n",
    "siz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lw = 2\n",
    "plt.figure()\n",
    "labels = ['SSS', 'ESS', 'SIZ']\n",
    "plt.plot(sss[\"fpr\"], sss[\"tpr\"], color='b', lw=lw, label='SSS (AUC = {0:.2f}%)'.format(sss[\"roc_auc\"] * 100))\n",
    "plt.plot(ess[\"fpr\"], ess[\"tpr\"], color='g', lw=lw, label='ESS (AUC = {0:.2f}%)'.format(ess[\"roc_auc\"] * 100))\n",
    "plt.plot(siz[\"fpr\"], siz[\"tpr\"], color='r', lw=lw, label='SIZ (AUC = {0:.2f}%)'.format(siz[\"roc_auc\"] * 100))\n",
    "\n",
    "plt.plot([0,1], [0,1], color='orange', linestyle='--')\n",
    "\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.grid(True)\n",
    "\n",
    "plt.xticks(np.arange(0.0, 1.1, step=0.1))\n",
    "plt.xlabel('False Positive Rate', fontsize=15)\n",
    "\n",
    "plt.yticks(np.arange(0.0, 1.1, step=0.1))\n",
    "plt.ylabel('True Positive Rate', fontsize=15)\n",
    "plt.legend(prop={'size':12}, loc=\"lower right\")\n",
    "#plt.title(\"ROC on test set with 44 3D CT scans\", fontsize=15)\n",
    "plt.savefig(\"{}/auc_on_test_set_44scans.pdf\".format(log_path), bbox_inches='tight', pad_inches=0, dpi=1200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['SSS', 'ESS', 'SIZ']\n",
    "\n",
    "# Test set values from ImageCLEF evaluation\n",
    "auc = [64, 67, 73]\n",
    "acc = [59, 61, 67]\n",
    "\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "dummy = np.arange(10)\n",
    "\n",
    "width = 0.4  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "rects1 = ax.bar(x - width/2, auc, width, label='AUC')\n",
    "rects2 = ax.bar(x + width/2, acc, width, label='ACC')\n",
    "#rects3 = ax.bar(x - width/2 * 3.08, sensitivity, width, label='Sensitivity')\n",
    "\n",
    "ax.set_ylabel('Performance Measure (%)', fontsize=14)\n",
    "#ax.set_title('Results')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(labels, rotation=0, fontsize=16)\n",
    "ax.set_ylim([50, 80])\n",
    "#ax.legend(loc='top center')\n",
    "ax.grid(True)\n",
    "\n",
    "def autolabel(rects):\n",
    "    \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",
    "    for rect in rects:\n",
    "        height = rect.get_height()\n",
    "        ax.annotate('{}'.format(height),\n",
    "                    xy=(rect.get_x() + rect.get_width() / 2, height),\n",
    "                    xytext=(0, 3),  # 3 points vertical offset\n",
    "                    textcoords=\"offset points\",\n",
    "                    ha='center', va='bottom')\n",
    "        #ax.set_ylim(ymin=1)\n",
    "        \n",
    "\n",
    "def autolabel_(rects):\n",
    "    \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",
    "    for rect in rects:\n",
    "        height = rect.get_height()\n",
    "        ax.annotate('{}%'.format(height),\n",
    "                    xy=( 0.15 + rect.get_x() + rect.get_width() / 2, height),\n",
    "                    xytext=(0, 3),  # 3 points vertical offset\n",
    "                    textcoords=\"offset points\",\n",
    "                    ha='center', va='bottom')\n",
    "        #ax.set_ylim(ymin=1)\n",
    "\n",
    "\n",
    "autolabel(rects1) # %\n",
    "autolabel(rects2)\n",
    "#autolabel_(rects3) # %\n",
    "\n",
    "fig.tight_layout()\n",
    "fig.set_size_inches(6, 3, forward=True)\n",
    "plt.legend(loc='top center')\n",
    "plt.savefig(\"{}/results_clef_final_test_set.pdf\".format(log_path), bbox_inches='tight', pad_inches=0, dpi=1200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 10-fold Cross val results\n",
    "#SSS: 60.36% (+/- 13.47%)\n",
    "\n",
    "#ESS: 63.82% (+/- 10.66%)\n",
    "\n",
    "#SIZ: 66.73% (+/- 7.65%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create lists for the plot\n",
    "\n",
    "techs = ['SSS', 'ESS', 'SIZ']\n",
    "means = [60.36, 63.82, 66.73]\n",
    "stds = [13.47, 10.66, 7.65]\n",
    "\n",
    "\n",
    "x_pos = np.arange(len(techs))\n",
    "# Build the plot\n",
    "fig, ax = plt.subplots()\n",
    "ax.bar(x_pos, means, yerr=stds, align='center', alpha=0.5, ecolor='black', capsize=10)\n",
    "ax.set_xticks(x_pos)\n",
    "ax.set_xticklabels(techs, fontsize=15)\n",
    "ax.set_ylabel('Accuracy (%)', fontsize=15)\n",
    "#ax.set_title('Mean ACC and STD from 10-fold cross validation', fontsize=12)\n",
    "ax.set_ylim([45, 80])\n",
    "ax.grid(True)\n",
    "\n",
    "# Save the figure and show\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"{}/cross_val_10f_boxplot.pdf\".format(log_path), dpi=1200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}