check_tf_torch_models_on_Affectnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "00a3a5f3-4ea6-4bd1-966e-430bf7fa3fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import  torch\n",
    "from torchvision import transforms\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bc960e9-e4a0-471a-ba1c-3d40996f742e",
   "metadata": {},
   "source": [
    "#### Drawing the confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6a851f85-b2dd-4d24-8d91-d5ef8bfb672d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_conf_matrix_clas_report(y_true, y_pred, name_labels='', name_model=''):\n",
    "\n",
    "    c_m = confusion_matrix(y_true, y_pred)\n",
    "    conf_matrix = pd.DataFrame(c_m, name_labels, name_labels)\n",
    "\n",
    "    plt.figure(figsize = (9,9))\n",
    "\n",
    "    group_counts = ['{0:0.0f}'.format(value) for value in\n",
    "                  conf_matrix.values.flatten()]\n",
    "    group_percentages = ['{0:.1%}'.format(value) for value in\n",
    "                        conf_matrix.div(np.sum(conf_matrix, axis=1), axis=0).values.flatten()]\n",
    "\n",
    "    labels = ['{}\\n{}'.format(v1,v2) for v1,v2 in zip(group_counts, group_percentages)]\n",
    "\n",
    "    labels = np.asarray(labels).reshape(c_m.shape)\n",
    "    sns.set(font_scale=1.8)\n",
    "    chart = sns.heatmap(conf_matrix,\n",
    "              cbar=False ,\n",
    "              annot=labels,\n",
    "              square=True,\n",
    "              fmt='',\n",
    "              annot_kws={ 'size': 18},\n",
    "              cmap=\"Blues\",\n",
    "              )\n",
    "    chart.set_xticklabels(name_labels)\n",
    "    chart.set_yticklabels(name_labels, rotation=360, verticalalignment='center')\n",
    "    plt.savefig('confusion_matrix_{}.png'.format(name_model), bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d878b2d-3074-4e00-bd68-a31a809096f5",
   "metadata": {},
   "source": [
    "#### Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3000fd29-e49b-4237-8010-518c6333d2a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tf_processing(fp):\n",
    "    def preprocess_input(x):\n",
    "        x_temp = np.copy(x)\n",
    "        x_temp = x_temp[..., ::-1]\n",
    "        x_temp[..., 0] -= 91.4953\n",
    "        x_temp[..., 1] -= 103.8827\n",
    "        x_temp[..., 2] -= 131.0912\n",
    "        return x_temp\n",
    "\n",
    "    def get_img_tf(path):\n",
    "        img = tf.keras.utils.load_img(\n",
    "                    path,\n",
    "                    target_size=(224,224),\n",
    "                )\n",
    "        \n",
    "        img = tf.keras.utils.img_to_array(img)\n",
    "        img = preprocess_input(img)\n",
    "        img = np.array([img])\n",
    "        return img\n",
    "\n",
    "    return get_img_tf(fp)\n",
    "\n",
    "\n",
    "def pth_processing(fp):\n",
    "    class PreprocessInput(torch.nn.Module):\n",
    "        def init(self):\n",
    "            super(PreprocessInput, self).init()\n",
    "\n",
    "        def forward(self, x):\n",
    "            x = x.to(torch.float32)\n",
    "            x = torch.flip(x, dims=(0,))\n",
    "            x[0, :, :] -= 91.4953\n",
    "            x[1, :, :] -= 103.8827\n",
    "            x[2, :, :] -= 131.0912\n",
    "            return x\n",
    "\n",
    "    def get_img_torch(path):\n",
    "        img = Image.open(path)\n",
    "        img = img.resize((224, 224), Image.Resampling.NEAREST)\n",
    "        \n",
    "        ttransform = transforms.Compose([\n",
    "            transforms.PILToTensor(),\n",
    "            PreprocessInput()\n",
    "        ])\n",
    "        \n",
    "        img = ttransform(img)\n",
    "        img = torch.unsqueeze(img, 0)\n",
    "        return img\n",
    "    return get_img_torch(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81e827f4-617f-4a90-b7f8-530e2c063d73",
   "metadata": {},
   "source": [
    "#### Loading a model that better predicts Neutral, Happiness, Surprise, and Anger classes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7b9be513-711a-42a7-be74-b3f8d27c88b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = '0_66_49_wo_gl'\n",
    "\n",
    "# tf\n",
    "tf_model = tf.keras.models.load_model('models_EmoAffectnet/weights_{0}.h5'.format(name))\n",
    "\n",
    "# torch\n",
    "pth_model = torch.jit.load('models_EmoAffectnet/torchscript_model_{0}.pth'.format(name))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ed5cae0-23e8-428f-8539-af9075d30a61",
   "metadata": {},
   "source": [
    "#### Test tf_model 0_66_49_wo_gl¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "057b0de4-73ae-4240-8c80-dd2170741abd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 3500/3500 [02:09<00:00, 27.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          NE      0.532     0.694     0.602       500\n",
      "          HA      0.780     0.886     0.830       500\n",
      "          SA      0.680     0.634     0.656       500\n",
      "          SU      0.614     0.632     0.623       500\n",
      "          FE      0.715     0.596     0.650       500\n",
      "          DI      0.749     0.598     0.665       500\n",
      "          AN      0.636     0.614     0.625       500\n",
      "\n",
      "    accuracy                          0.665      3500\n",
      "   macro avg      0.672     0.665     0.664      3500\n",
      "weighted avg      0.672     0.665     0.664      3500\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "names_list = ['subDirectory_filePath','face_x','face_y','face_width','face_height','facial_landmarks','expression','valence','arousal']\n",
    "dict_name_labels = {0: 'Neutral', 1: 'Happiness', 2: 'Sadness', 3: 'Surprise', 4: 'Fear', 5: 'Disgust', 6: 'Anger'}\n",
    "\n",
    "path_df = 'C:/Work/AffectNet/Manually_Annotated_file_lists/validation.csv'\n",
    "path_imgs = 'C:/Work/Databases/Manually_Annotated_Images_valid/'\n",
    "\n",
    "df_valid = pd.read_csv(path_df, names = names_list)\n",
    "df_valid = df_valid[~df_valid.expression.isin([7,8,9,10])] \n",
    "df_valid = df_valid.reset_index(drop=True)\n",
    "\n",
    "labels = df_valid.expression.values.tolist()\n",
    "\n",
    "prob_tf = []\n",
    "\n",
    "for curr_path in tqdm(df_valid.subDirectory_filePath.values.tolist()):\n",
    "    path_img = os.path.join(path_imgs,curr_path)\n",
    "    img = tf_processing(path_img)\n",
    "    prob = tf_model(img, training=False).numpy()\n",
    "    cl = np.argmax(prob)\n",
    "    prob_tf.append(cl)\n",
    "\n",
    "name_labels = ['NE', 'HA', 'SA', 'SU', 'FE', 'DI', 'AN']\n",
    "# draw_conf_matrix_clas_report(labels, prob_tf, name_labels, 'AffectNet_valid_tf_1')\n",
    "print(classification_report(labels, prob_tf, target_names=name_labels, digits=3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "376464b5-ea99-42b0-9931-d4137995dd68",
   "metadata": {},
   "source": [
    "#### Test torch_model 0_66_49_wo_gl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39a3047c-44f2-406f-b1fe-da86d393a7d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 3500/3500 [02:51<00:00, 20.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          NE      0.532     0.694     0.602       500\n",
      "          HA      0.780     0.886     0.830       500\n",
      "          SA      0.683     0.634     0.658       500\n",
      "          SU      0.614     0.632     0.623       500\n",
      "          FE      0.715     0.596     0.650       500\n",
      "          DI      0.749     0.598     0.665       500\n",
      "          AN      0.635     0.616     0.625       500\n",
      "\n",
      "    accuracy                          0.665      3500\n",
      "   macro avg      0.673     0.665     0.665      3500\n",
      "weighted avg      0.673     0.665     0.665      3500\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "prob_torch = []\n",
    "\n",
    "pth_model.eval()\n",
    "with torch.no_grad():\n",
    "    for curr_path in tqdm(df_valid.subDirectory_filePath.values.tolist()):\n",
    "        path_img = os.path.join(path_imgs,curr_path)\n",
    "        img = pth_processing(path_img)\n",
    "        output = torch.nn.functional.softmax(pth_model(img), dim=1).detach().numpy()\n",
    "        cl = np.argmax(output)\n",
    "        prob_torch.append(cl)\n",
    "\n",
    "name_labels = ['NE', 'HA', 'SA', 'SU', 'FE', 'DI', 'AN']\n",
    "# draw_conf_matrix_clas_report(labels, prob_torch, name_labels, 'AffectNet_valid_torch_1')\n",
    "print(classification_report(labels, prob_torch, target_names=name_labels, digits=3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9e9a6dd-3aa4-496d-8145-6d6970419fed",
   "metadata": {},
   "source": [
    "#### Loading a model that better predicts Sadness, Surprise, Fear, and Disgust classes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f1fe0a2f-ecd4-40f4-aaed-020623aaea42",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = '0_66_37_wo_gl'\n",
    "\n",
    "# tf\n",
    "tf_model = tf.keras.models.load_model('models_EmoAffectnet/weights_{0}.h5'.format(name))\n",
    "\n",
    "# torch\n",
    "pth_model = torch.jit.load('models_EmoAffectnet/torchscript_model_{0}.pth'.format(name))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b587c61f-1ea7-43be-a5ca-e04453dd9d03",
   "metadata": {},
   "source": [
    "#### Test tf_model 0_66_37_wo_gl¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3e9746df-a58d-4d8d-87b5-02630bbdcb46",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 3500/3500 [02:06<00:00, 27.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          NE      0.533     0.680     0.598       500\n",
      "          HA      0.779     0.880     0.826       500\n",
      "          SA      0.670     0.642     0.656       500\n",
      "          SU      0.616     0.632     0.624       500\n",
      "          FE      0.714     0.600     0.652       500\n",
      "          DI      0.724     0.620     0.668       500\n",
      "          AN      0.648     0.592     0.619       500\n",
      "\n",
      "    accuracy                          0.664      3500\n",
      "   macro avg      0.669     0.664     0.663      3500\n",
      "weighted avg      0.669     0.664     0.663      3500\n",
      "\n"
     ]
    }
   ],
   "source": [
    "prob_tf = []\n",
    "\n",
    "for curr_path in tqdm(df_valid.subDirectory_filePath.values.tolist()):\n",
    "    path_img = os.path.join(path_imgs,curr_path)\n",
    "    img = tf_processing(path_img)\n",
    "    prob = tf_model(img, training=False).numpy()\n",
    "    cl = np.argmax(prob)\n",
    "    prob_tf.append(cl)\n",
    "\n",
    "name_labels = ['NE', 'HA', 'SA', 'SU', 'FE', 'DI', 'AN']\n",
    "# draw_conf_matrix_clas_report(labels, prob_tf, name_labels, 'AffectNet_valid_tf_2')\n",
    "print(classification_report(labels, prob_tf, target_names=name_labels, digits=3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "252430bd-3de6-4928-a8ec-1e05d5d90ba3",
   "metadata": {},
   "source": [
    "#### Test torch_model 0_66_37_wo_gl¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "86c85c1c-60f6-45b8-ae55-ba2dc48562c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 3500/3500 [02:54<00:00, 20.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          NE      0.533     0.680     0.598       500\n",
      "          HA      0.779     0.880     0.826       500\n",
      "          SA      0.670     0.642     0.656       500\n",
      "          SU      0.616     0.632     0.624       500\n",
      "          FE      0.714     0.600     0.652       500\n",
      "          DI      0.726     0.620     0.669       500\n",
      "          AN      0.648     0.594     0.620       500\n",
      "\n",
      "    accuracy                          0.664      3500\n",
      "   macro avg      0.670     0.664     0.664      3500\n",
      "weighted avg      0.670     0.664     0.664      3500\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "prob_torch = []\n",
    "\n",
    "pth_model.eval()\n",
    "with torch.no_grad():\n",
    "    for curr_path in tqdm(df_valid.subDirectory_filePath.values.tolist()):\n",
    "        path_img = os.path.join(path_imgs,curr_path)\n",
    "        img = pth_processing(path_img)\n",
    "        output = torch.nn.functional.softmax(pth_model(img), dim=1).detach().numpy()\n",
    "        cl = np.argmax(output)\n",
    "        prob_torch.append(cl)\n",
    "\n",
    "name_labels = ['NE', 'HA', 'SA', 'SU', 'FE', 'DI', 'AN']\n",
    "# draw_conf_matrix_clas_report(labels, prob_torch, name_labels, 'AffectNet_valid_torch_2')\n",
    "print(classification_report(labels, prob_torch, target_names=name_labels, digits=3))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}