Gunther_Train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "epGd8bfZQO1l"
   },
   "source": [
    "### 1. Read and explore the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1 Import Python module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "vLgNmD86woYH"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using cuda\n",
      "          code train  val test  total\n",
      "COVID19      0   460   58   58    576\n",
      "NORMAL       1  1266  158  159   1583\n",
      "PNEUMONIA    2  3418  427  428   4273\n",
      "TOTAL           5144  643  645   6432\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import time\n",
    "import os\n",
    "import cv2\n",
    "import shutil\n",
    "import random\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.datasets as datasets\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision import models\n",
    "from torch import optim\n",
    "from sklearn.metrics import *\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "seed = 24\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "num_workers = 4\n",
    "batch_size = 10\n",
    "metrics = ['Accuracy', 'Precision', 'Recall', 'F1-score']\n",
    "categories = ['COVID19', 'NORMAL', 'PNEUMONIA']\n",
    "num_classes = len(categories)\n",
    "splits = ['train', 'val', 'test']\n",
    "DATA_PATH = r\"C:\\Gunther\\Datasets\\Covid19DLH_80_10_10\\Data\"\n",
    "model_path = r\"C:\\Gunther\\OneDrive - University of Illinois - Urbana\\Python\\DLH\\Project\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f'You are using {device}')\n",
    "\n",
    "def show_dataset(data_path=DATA_PATH):\n",
    "    df = pd.DataFrame(columns = ['code'] + splits, index = categories)\n",
    "    for row in categories:\n",
    "        for col in splits:\n",
    "            df.loc[row,col] = int(len(os.listdir(os.path.join(data_path, col+'/'+row))))\n",
    "    df['total'] = df.sum(axis=1).astype(int)\n",
    "    df.loc['TOTAL'] = df.sum(axis=0).astype(int)\n",
    "    df['code'] = ['0', '1', '2','']\n",
    "    print(df)\n",
    "    return df\n",
    "\n",
    "df_dataset = show_dataset()\n",
    "\n",
    "def load_data(data_path=DATA_PATH, num_workers=num_workers):\n",
    "    transform = transforms.Compose([transforms.Resize(224),\n",
    "                                    transforms.CenterCrop(224),\n",
    "                                    transforms.ToTensor(),\n",
    "                                   ])\n",
    "    train_data = datasets.ImageFolder(root=data_path + '/train', transform=transform)\n",
    "    train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
    "    val_data = datasets.ImageFolder(root=data_path + '/val', transform=transform)\n",
    "    val_loader = torch.utils.data.DataLoader(val_data, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n",
    "    test_data = datasets.ImageFolder(root=data_path + '/test', transform=transform)\n",
    "    test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n",
    "    return train_data, train_loader, val_data, val_loader, test_data, test_loader\n",
    "\n",
    "train_data, train_loader, val_data, val_loader, test_data, test_loader = load_data()\n",
    "dataset = torch.utils.data.ConcatDataset([train_data, val_data, test_data])\n",
    "\n",
    "def calc_metrics(predictions, actuals, row):\n",
    "    df = pd.DataFrame(columns =metrics)\n",
    "    Y_pred = np.concatenate(predictions, axis=0)\n",
    "    Y_test = np.concatenate(actuals, axis=0)\n",
    "    df.loc[row, 'Accuracy'] = accuracy_score(Y_test, Y_pred)\n",
    "    df.loc[row, 'Precision'] = precision_score(Y_test, Y_pred, average=\"macro\")\n",
    "    df.loc[row, 'Recall'] = recall_score(Y_test, Y_pred, average=\"macro\")\n",
    "    df.loc[row, 'F1-score'] = f1_score(Y_test, Y_pred, average=\"macro\")\n",
    "    return df\n",
    "\n",
    "def create_confusion_matrix(preds, y_test):\n",
    "    ylist, predlist = [], []\n",
    "    for pred in preds:\n",
    "        for item in pred:\n",
    "            predlist.append(int(item))\n",
    "    for y in y_test:\n",
    "        for item in y:\n",
    "            ylist.append(int(item))\n",
    "    data_dict = {'y_Actual':    ylist, 'y_Predicted': predlist}\n",
    "    df = pd.DataFrame(data_dict, columns=['y_Actual','y_Predicted'])\n",
    "    cm = pd.crosstab(df['y_Actual'], df['y_Predicted'], rownames=['ACTUAL'], colnames=['PREDICTED'])\n",
    "    return cm\n",
    "\n",
    "def train_val_model(model):\n",
    "    t_start = time.time()\n",
    "    df = pd.DataFrame(columns = ['model_name', 'epoch', 'train', 'val', 'test acc'])\n",
    "    print(f\"Training model {model_name} with {df_dataset.loc['TOTAL', 'train']} samples and {n_epochs} epochs, and validating with {df_dataset.loc['TOTAL', 'val']} samples\\n\")\n",
    "    train_size, val_size = len(train_loader), len(val_loader)\n",
    "    for epoch in range(1, n_epochs+1):\n",
    "        # Beginning of training step\n",
    "        t0 = time.time()\n",
    "        model.train()\n",
    "        train_loss, val_loss = 0.0, 0.0\n",
    "        for i, (data, target) in enumerate(train_loader):\n",
    "            target = target.to(device)\n",
    "            data = data.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(data)\n",
    "            loss = criterion(outputs, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            train_loss += loss.detach().cpu().numpy()\n",
    "        # Beginning of evaluation step\n",
    "        model.eval()\n",
    "        for j, (data, target) in enumerate(val_loader):\n",
    "            target = target.to(device)\n",
    "            data = data.to(device)\n",
    "            outputs = model(data)\n",
    "            loss = criterion(outputs, target)\n",
    "            val_loss += loss.detach().cpu().numpy()\n",
    "        predictions, actuals = calc_pred_actuals(model, test_loader)\n",
    "        df_test = calc_metrics(predictions, actuals, 'Test accuracy').astype(float)\n",
    "        print(f\"Epoch {epoch}:\\t train loss={train_loss/train_size:.5f} \\t val loss={val_loss/val_size:.5f} \\t test acc={df_test.iloc[0,0]:.5f} \\t time={(time.time() - t0):.2f}s\")\n",
    "        df.loc[len(df)] = [model_name, epoch, train_loss/train_size, val_loss/val_size, df_test.iloc[0,0]]\n",
    "        if use_scheduler: scheduler.step(val_loss/val_size)   # Optional to use scheduler for dynamic learning rate\n",
    "    print(f\"Total time training and evaluating: {(time.time()-t_start):.2f}s\")\n",
    "    return model, df\n",
    "\n",
    "def calc_pred_actuals(model, loader):\n",
    "    predictions, actuals = [], []\n",
    "    with torch.no_grad():\n",
    "        for data, target in loader:\n",
    "            data = data.to(device)\n",
    "            target = target.to(device)\n",
    "            Y_pred_orig = model(data)\n",
    "            _, Y_pred_tag = torch.max(Y_pred_orig, dim = 1)\n",
    "            Y_pred_tag = Y_pred_tag.detach().cpu().numpy()\n",
    "            Y_pred_tag = Y_pred_tag.reshape(len(Y_pred_tag), 1).astype('int8')\n",
    "            Y_test = target.detach().cpu().numpy()\n",
    "            Y_test = Y_test.reshape(len(Y_test), 1).astype('int8')\n",
    "            predictions.append(Y_pred_tag)\n",
    "            actuals.append(Y_test)\n",
    "    return predictions, actuals\n",
    "\n",
    "def test_model(model):\n",
    "    model.eval()\n",
    "    print(f\"Testing the model {model_name} with {df_dataset.loc['TOTAL', 'test']} samples \\n\")   \n",
    "    predictions, actuals = calc_pred_actuals(model, test_loader)\n",
    "    conf_matrix = create_confusion_matrix(predictions, actuals)\n",
    "    df_test = calc_metrics(predictions, actuals, 'Test Results').astype(float)\n",
    "    print(df_test)\n",
    "    return df_test, conf_matrix\n",
    "\n",
    "def plot_train_val_losses(df):\n",
    "    df2 = pd.melt(df, id_vars=['epoch'], value_vars=['train', 'val'], var_name='process', value_name='loss')\n",
    "    sns.lineplot(x = \"epoch\", y = \"loss\", data = df2, hue = \"process\",\n",
    "                style = \"process\", palette = \"hot\", dashes = False, \n",
    "                markers = [\"o\", \"<\"],  legend=\"brief\").set_title(\"Train and Validation Losses by Epoch\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = [128, 128, 256, 256]\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, w[0], 3, padding=1)\n",
    "        self.bn1 = nn.BatchNorm2d(w[0])\n",
    "        self.pool1 = nn.MaxPool2d(kernel_size=4, stride=4)\n",
    "        self.conv2 = nn.Conv2d(w[0], w[1], 3, padding=1)\n",
    "        self.bn2 = nn.BatchNorm2d(w[1])\n",
    "        self.conv3 = nn.Conv2d(w[1], w[2], 3, padding=1)\n",
    "        self.bn3 = nn.BatchNorm2d(w[2])\n",
    "        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        self.linear1 = nn.Linear(14 * 14 * w[2], w[3])\n",
    "        self.linear2 = nn.Linear(w[3], 3)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout()\n",
    "    def forward(self, x):\n",
    "        x = self.pool1(self.relu(self.bn1(self.conv1(x))))\n",
    "        x = self.pool2(self.relu(self.bn2(self.conv2(x))))\n",
    "        x = self.pool2(self.relu(self.bn3(self.conv3(x))))\n",
    "        x = x.reshape(x.size(0), -1)\n",
    "        x = self.relu(self.linear1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.linear2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model DLH_team with 5144 samples and 100 epochs, and validating with 643 samples\n",
      "\n",
      "Epoch 1:\t train loss=0.34874 \t val loss=0.16678 \t test acc=0.94264 \t time=95.46s\n",
      "Epoch 2:\t train loss=0.20170 \t val loss=0.14297 \t test acc=0.95659 \t time=94.76s\n",
      "Epoch 3:\t train loss=0.16935 \t val loss=0.15792 \t test acc=0.94729 \t time=95.35s\n",
      "Epoch 4:\t train loss=0.15661 \t val loss=0.14228 \t test acc=0.94729 \t time=95.19s\n",
      "Epoch 5:\t train loss=0.14096 \t val loss=0.12323 \t test acc=0.95349 \t time=98.11s\n",
      "Epoch 6:\t train loss=0.12372 \t val loss=0.12225 \t test acc=0.95349 \t time=96.30s\n",
      "Epoch 7:\t train loss=0.11439 \t val loss=0.12023 \t test acc=0.95349 \t time=95.85s\n",
      "Epoch 8:\t train loss=0.11156 \t val loss=0.19400 \t test acc=0.93643 \t time=95.41s\n",
      "Epoch 9:\t train loss=0.10401 \t val loss=0.13085 \t test acc=0.95969 \t time=95.15s\n",
      "Epoch 10:\t train loss=0.09431 \t val loss=0.13129 \t test acc=0.94884 \t time=96.30s\n",
      "Epoch 11:\t train loss=0.09116 \t val loss=0.11182 \t test acc=0.95814 \t time=96.03s\n",
      "Epoch 12:\t train loss=0.08518 \t val loss=0.11569 \t test acc=0.95814 \t time=95.88s\n",
      "Epoch 13:\t train loss=0.07514 \t val loss=0.12566 \t test acc=0.95504 \t time=94.79s\n",
      "Epoch 14:\t train loss=0.06775 \t val loss=0.13309 \t test acc=0.95349 \t time=95.16s\n",
      "Epoch 15:\t train loss=0.06772 \t val loss=0.11499 \t test acc=0.95969 \t time=96.37s\n",
      "Epoch 16:\t train loss=0.05874 \t val loss=0.11126 \t test acc=0.96124 \t time=96.08s\n",
      "Epoch 17:\t train loss=0.05587 \t val loss=0.12773 \t test acc=0.95039 \t time=96.06s\n",
      "Epoch 18:\t train loss=0.04920 \t val loss=0.14253 \t test acc=0.94729 \t time=95.06s\n",
      "Epoch 19:\t train loss=0.04794 \t val loss=0.11850 \t test acc=0.96124 \t time=96.04s\n",
      "Epoch 20:\t train loss=0.04589 \t val loss=0.12436 \t test acc=0.95659 \t time=96.53s\n",
      "Epoch 21:\t train loss=0.03901 \t val loss=0.11595 \t test acc=0.96124 \t time=95.75s\n",
      "Epoch 22:\t train loss=0.04033 \t val loss=0.12255 \t test acc=0.96124 \t time=96.33s\n",
      "Epoch 23:\t train loss=0.03259 \t val loss=0.11836 \t test acc=0.96434 \t time=96.06s\n",
      "Epoch 24:\t train loss=0.03064 \t val loss=0.12767 \t test acc=0.96279 \t time=98.59s\n",
      "Epoch 25:\t train loss=0.02723 \t val loss=0.26241 \t test acc=0.92248 \t time=96.74s\n",
      "Epoch 26:\t train loss=0.02373 \t val loss=0.12535 \t test acc=0.96279 \t time=95.69s\n",
      "Epoch 27:\t train loss=0.02195 \t val loss=0.13619 \t test acc=0.95659 \t time=95.81s\n",
      "Epoch 28:\t train loss=0.02119 \t val loss=0.15352 \t test acc=0.95349 \t time=95.92s\n",
      "Epoch 29:\t train loss=0.01942 \t val loss=0.13622 \t test acc=0.95814 \t time=96.29s\n",
      "Epoch 30:\t train loss=0.01545 \t val loss=0.14855 \t test acc=0.95969 \t time=95.85s\n",
      "Epoch 31:\t train loss=0.01583 \t val loss=0.13295 \t test acc=0.95814 \t time=95.37s\n",
      "Epoch 32:\t train loss=0.01331 \t val loss=0.14390 \t test acc=0.96434 \t time=96.10s\n",
      "Epoch 33:\t train loss=0.01093 \t val loss=0.14147 \t test acc=0.96124 \t time=96.18s\n",
      "Epoch 34:\t train loss=0.01140 \t val loss=0.14571 \t test acc=0.96589 \t time=96.83s\n",
      "Epoch 35:\t train loss=0.01003 \t val loss=0.13946 \t test acc=0.95814 \t time=95.55s\n",
      "Epoch 36:\t train loss=0.01001 \t val loss=0.13594 \t test acc=0.95969 \t time=95.34s\n",
      "Epoch 37:\t train loss=0.00795 \t val loss=0.15328 \t test acc=0.96124 \t time=94.85s\n",
      "Epoch 38:\t train loss=0.00767 \t val loss=0.15814 \t test acc=0.96279 \t time=95.85s\n",
      "Epoch 39:\t train loss=0.00713 \t val loss=0.16198 \t test acc=0.96124 \t time=94.99s\n",
      "Epoch 40:\t train loss=0.00768 \t val loss=0.15584 \t test acc=0.96124 \t time=95.26s\n",
      "Epoch 41:\t train loss=0.00776 \t val loss=0.15404 \t test acc=0.95814 \t time=95.89s\n",
      "Epoch 42:\t train loss=0.00593 \t val loss=0.15897 \t test acc=0.95969 \t time=95.66s\n",
      "Epoch 43:\t train loss=0.00579 \t val loss=0.14861 \t test acc=0.96279 \t time=98.93s\n",
      "Epoch 44:\t train loss=0.00550 \t val loss=0.16493 \t test acc=0.95659 \t time=95.24s\n",
      "Epoch 45:\t train loss=0.00417 \t val loss=0.14944 \t test acc=0.96124 \t time=95.28s\n",
      "Epoch 46:\t train loss=0.00353 \t val loss=0.16505 \t test acc=0.96279 \t time=95.37s\n",
      "Epoch 47:\t train loss=0.00448 \t val loss=0.15867 \t test acc=0.96124 \t time=94.94s\n",
      "Epoch 48:\t train loss=0.00388 \t val loss=0.16235 \t test acc=0.96124 \t time=94.89s\n",
      "Epoch 49:\t train loss=0.00413 \t val loss=0.16791 \t test acc=0.95969 \t time=94.90s\n",
      "Epoch 50:\t train loss=0.00561 \t val loss=0.15881 \t test acc=0.96589 \t time=94.97s\n",
      "Epoch 51:\t train loss=0.00381 \t val loss=0.17582 \t test acc=0.96434 \t time=95.56s\n",
      "Epoch 52:\t train loss=0.00363 \t val loss=0.17454 \t test acc=0.96124 \t time=97.84s\n",
      "Epoch 53:\t train loss=0.00405 \t val loss=0.16786 \t test acc=0.95814 \t time=96.28s\n",
      "Epoch 54:\t train loss=0.00318 \t val loss=0.16752 \t test acc=0.96124 \t time=95.23s\n",
      "Epoch 55:\t train loss=0.00262 \t val loss=0.17745 \t test acc=0.96124 \t time=94.77s\n",
      "Epoch 56:\t train loss=0.00377 \t val loss=0.17788 \t test acc=0.96124 \t time=95.07s\n",
      "Epoch 57:\t train loss=0.00353 \t val loss=0.17000 \t test acc=0.95969 \t time=95.68s\n",
      "Epoch 58:\t train loss=0.00262 \t val loss=0.17018 \t test acc=0.95659 \t time=95.67s\n",
      "Epoch 59:\t train loss=0.00225 \t val loss=0.17185 \t test acc=0.95814 \t time=94.79s\n",
      "Epoch 60:\t train loss=0.00205 \t val loss=0.16973 \t test acc=0.95814 \t time=95.01s\n",
      "Epoch 61:\t train loss=0.00265 \t val loss=0.17471 \t test acc=0.96434 \t time=94.82s\n",
      "Epoch 62:\t train loss=0.00226 \t val loss=0.17797 \t test acc=0.96279 \t time=96.17s\n",
      "Epoch 63:\t train loss=0.00263 \t val loss=0.16905 \t test acc=0.96124 \t time=96.71s\n",
      "Epoch 64:\t train loss=0.00254 \t val loss=0.17296 \t test acc=0.95969 \t time=95.71s\n",
      "Epoch 65:\t train loss=0.00226 \t val loss=0.16962 \t test acc=0.96434 \t time=94.93s\n",
      "Epoch 66:\t train loss=0.00171 \t val loss=0.17859 \t test acc=0.95969 \t time=96.76s\n",
      "Epoch 67:\t train loss=0.00248 \t val loss=0.17751 \t test acc=0.96124 \t time=95.94s\n",
      "Epoch 68:\t train loss=0.00206 \t val loss=0.17964 \t test acc=0.96279 \t time=94.82s\n",
      "Epoch 69:\t train loss=0.00158 \t val loss=0.17346 \t test acc=0.95969 \t time=94.71s\n",
      "Epoch 70:\t train loss=0.00153 \t val loss=0.18441 \t test acc=0.96434 \t time=95.09s\n",
      "Epoch 71:\t train loss=0.00308 \t val loss=0.17695 \t test acc=0.96279 \t time=96.84s\n",
      "Epoch 72:\t train loss=0.00200 \t val loss=0.18112 \t test acc=0.96124 \t time=95.66s\n",
      "Epoch 73:\t train loss=0.00170 \t val loss=0.18089 \t test acc=0.95814 \t time=96.42s\n",
      "Epoch 74:\t train loss=0.00130 \t val loss=0.18302 \t test acc=0.96124 \t time=96.20s\n",
      "Epoch 75:\t train loss=0.00132 \t val loss=0.17657 \t test acc=0.95814 \t time=94.92s\n",
      "Epoch 76:\t train loss=0.00131 \t val loss=0.18359 \t test acc=0.96279 \t time=95.18s\n",
      "Epoch 77:\t train loss=0.00187 \t val loss=0.18359 \t test acc=0.95969 \t time=95.01s\n",
      "Epoch 78:\t train loss=0.00180 \t val loss=0.18825 \t test acc=0.95659 \t time=95.43s\n",
      "Epoch 79:\t train loss=0.00178 \t val loss=0.19559 \t test acc=0.95969 \t time=95.41s\n",
      "Epoch 80:\t train loss=0.00169 \t val loss=0.18160 \t test acc=0.96279 \t time=95.16s\n",
      "Epoch 81:\t train loss=0.00144 \t val loss=0.17997 \t test acc=0.95814 \t time=96.20s\n",
      "Epoch 82:\t train loss=0.00127 \t val loss=0.17848 \t test acc=0.96124 \t time=95.49s\n",
      "Epoch 83:\t train loss=0.00137 \t val loss=0.17390 \t test acc=0.95814 \t time=94.83s\n",
      "Epoch 84:\t train loss=0.00134 \t val loss=0.17684 \t test acc=0.96434 \t time=95.47s\n",
      "Epoch 85:\t train loss=0.00185 \t val loss=0.17779 \t test acc=0.96124 \t time=95.59s\n",
      "Epoch 86:\t train loss=0.00151 \t val loss=0.17741 \t test acc=0.96434 \t time=95.49s\n",
      "Epoch 87:\t train loss=0.00138 \t val loss=0.18286 \t test acc=0.96279 \t time=94.97s\n",
      "Epoch 88:\t train loss=0.00095 \t val loss=0.18595 \t test acc=0.95814 \t time=94.80s\n",
      "Epoch 89:\t train loss=0.00123 \t val loss=0.19233 \t test acc=0.95814 \t time=95.04s\n",
      "Epoch 90:\t train loss=0.00115 \t val loss=0.19856 \t test acc=0.96124 \t time=96.85s\n",
      "Epoch 91:\t train loss=0.00144 \t val loss=0.19092 \t test acc=0.95969 \t time=96.16s\n",
      "Epoch 92:\t train loss=0.00087 \t val loss=0.18991 \t test acc=0.96279 \t time=95.14s\n",
      "Epoch 93:\t train loss=0.00083 \t val loss=0.19503 \t test acc=0.96279 \t time=94.64s\n",
      "Epoch 94:\t train loss=0.00129 \t val loss=0.17988 \t test acc=0.95814 \t time=94.78s\n",
      "Epoch 95:\t train loss=0.00136 \t val loss=0.18300 \t test acc=0.95814 \t time=94.64s\n",
      "Epoch 96:\t train loss=0.00116 \t val loss=0.18453 \t test acc=0.96434 \t time=95.26s\n",
      "Epoch 97:\t train loss=0.00114 \t val loss=0.17783 \t test acc=0.95969 \t time=95.06s\n",
      "Epoch 98:\t train loss=0.00086 \t val loss=0.17861 \t test acc=0.96124 \t time=95.31s\n",
      "Epoch 99:\t train loss=0.00071 \t val loss=0.19004 \t test acc=0.96279 \t time=94.78s\n",
      "Epoch 100:\t train loss=0.00110 \t val loss=0.17970 \t test acc=0.96434 \t time=95.43s\n",
      "Total time training and evaluating: 9565.10s\n"
     ]
    }
   ],
   "source": [
    "test = False\n",
    "\n",
    "seed = 24\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "net = Net().to(device)\n",
    "model_name = 'DLH_team'\n",
    "n_epochs = 100\n",
    "learning_rate = 1e-3\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=learning_rate)\n",
    "\n",
    "\n",
    "use_scheduler = False   # Set True if using scheduler\n",
    "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.8, patience=2, threshold = 0.001, verbose=True, min_lr = 1e-5, threshold_mode = 'abs')\n",
    "our_model, df_epochs = train_val_model(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(our_model.state_dict(), os.path.join(model_path, 'best5.pth'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6.1 Plot epoch train and validation losses of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABHlElEQVR4nO2de3hcVdX/P2smaZr0fr+TVAR6bymlLRYFBLm/FhClCIqI8kNELa8ooOKLKK/6egFRsJSLgmJBkUJfQYqI0BcBaQuVttCWUnpJ2zRp0qRp09xm1u+PfU5yMplJZpKZXNfneebJuZ+9z0z29+y11l5bVBXDMAzDiCXU2QUwDMMwuiYmEIZhGEZcTCAMwzCMuJhAGIZhGHExgTAMwzDiYgJhGIZhxMUEopciIn8VkSu6QDluFZHfZ+C6nxORlwPrh0TkA8kc24Z7dYln2dmIyG9F5AedXY62ICIFIqIiktXZZelKmEB0I7xGzv9EReRIYP2yVK6lqueo6kOZKmt7EZFxIlIvIkfH2bdcRH6ayvVUtb+qbktDuZoJWqaeZXducNuL11gfjvnNf7Ozy9XbMLXsRqhqf39ZRLYDX1DV52OPE5EsVa3vyLKlG1XdLSJ/Bz4D3OpvF5GhwLnAnE4qmtFxzFTVrZ1diN6M9SB6ACJyqogUisiNIlIE/EZEhojIX0SkREQOeMvjA+e8KCJf8JY/JyIvi8hPvWPfF5FzWrjfTSLynohUisjbInJhYF+L1xKRiSLyknfu34DhLVTtIZxABFkEbFTV9S2VI06ZVUQ+6C0PE5EVInJQRF4Hjo459hcissvbv1ZEPuxtPxv4FnCJ90b77zjPMiQi3xGRHSJSLCIPi8ggb59vxrhCRHaKyH4R+XYL9U+IiHxRRLaKSJlXl7HedhGRO7x7V4jIWyIyzdt3rvecKkVkt4jcELje+SKyTkTKReQVEZkR2Hejd3yliGwWkdNbKNpwEfmbd+xLIpLvXeNuEflZTB3+V0QWt6Hut4rI4yLymHefN0RkZmD/ZO87KReRjSLy8cC+XBH5mff9VHi/1dzA5S9r73fTo1BV+3TDD7AdOMNbPhWoB34M5AC5wDDgE0AeMAD4E/Bk4PwXcT0QgM8BdcAXgTDwJWAPIAnu/UlgLO4F4xLgMDAmmWsBrwI/98r5EaAS+H2C++QCFcDJgW2vAouTLMfLgfMU+KC3/CjwR6AfMA3YHXPs5d7zywK+DhQBfb19t8aWN+ZZfh7YCnwA6A88AfzO21fgleM+r24zgRpgcoL6/xb4QZztHwX2A7O95/hLYJW37yxgLTAYEGBy4JnsBT7sLQ8BZnvLs4FiYJ73nV2B+33lAMcBu4CxgToc3UJ5K73vNQf4hf9cgbne7yDkrQ8HqoBRCa7V8H3F2Xcr7jd2MZAN3AC87y1ne8//W0Af71lVAsd5597tfV/jvLp+yCtrSt9Nb/l0egHs08YvrrlA1PqNWILjZwEHAuvBRu1zwNbAvjzvn2V0kmVZByxs7VrAUTgh6xfY/wcSCIS3/35gqbd8jFfPkUmWo5lAeI1CHTApsO+/g8fGue4BnLnDb5xaEoi/A9cG9h3n3S8r0AiND+x/HViU4L6/Jb5APAD8T2C9v3ePAq9B3ALMx2uMA8ftBP4fMDBm+6+B78ds2wyc4j2zYuAMILuV38FvgUdjyhUBJnjr7wAf85avA55p4VoKHATKA5+zAt/Ba4FjQ3ji532KgnUHlnnnhIAj/ncZc7+Uvpve8jETU8+hRFWr/RURyRORe72u9EFgFTBYRMIJzi/yF1S1ylvsH+9AEflswBxRjnsLD5qKEl1rLE6kDgeO3dFKvR4CPiUifXHmpmdVtTjJcsRjBK6x3pWoDCLydRF5xzNBlAODkriuz9iY6+3w7jcqsK0osFxFguec7D1U9RBQCoxT1ReAX+HelPeJyFIRGegd+gmc/2aHZ/45ydueD3zdf45enSfgeg1bgcW4BrZYRB71zVkJaHiuXrnKvPKC+y4v95YvB37XSj1nq+rgwGdlgvtEgULvPmOBXd42nx24HsNwoC/wXgv3bO9306Mwgeg5xKbl/Tru7XWeqg7EdfvBmR3ajGdTvg/3BjhMVQcDG5K87l5giIj0C2w7qqUTVPX/cI3fQlyj8nA7y1GC68VMiFcGz99wI/ApYIh33YrAdVtLf7wH1+AGr10P7GvlvFRocg/veQ7DmcpQ1btU9QRgKnAs8A1v+2pVXQiMBJ7EmdnANba3xzTGeaq6zDvvD6p6sndPxZkyE9HwXEWkPzDUKy/A74GFnr9gsleGthK8TwgY791nDzDB2+ZzFO7Z7AeqifE5GYkxgei5DMB1p8vFRf78V5qu2w/XSJQAiMiVuDf3VlHVHcAa4Hsi0kdETgb+I4lTH8Y1SoOB/21POVQ1gvML3Or1sqbgbO4+A3ANegmQJSLfBQYG9u8DCmIaoCDLgOvFOeP748xXj2nbo8rCItI38OmDM8tdKSKzRCTHu8e/VHW7iJwoIvNEJBvnk6kGIt7zvkxEBqlqHc58E/HucR9wjXeeiEg/ETlPRAaIyHEi8lHvPtW431QktpABzhWRk71yft8r1y4AVS0EVuN6Dn9W1SNtfCYAJ4jIReLGLSzG+QteA/7l1fubIpItIqfifmOPer2KB4Gfi8hYEQmLyEle3Yw4mED0XO7EOdv24/5xnk3HRVX1beBnOGfxPmA68M8ULvFpnDO0DCdaDydxzsO4t8DHVLUmDeW4Dmc6KMLZzX8T2LcS+CvOjr8D1ygGzVF/8v6Wisgbca79IK4BXIVznFYDX0myXPG4Cdco+58XVPXvwC3An3G9sqNx0V3gxOw+nN9kB6735Y8Z+Qyw3TM5XoNn7lHVNbiggl95523F+XDAOXB/hPsdFeF6H99qobx/wH2vZcAJQOz4nIdw31Vr5iWAf0vTcRB3BvY9hQtMOODV6yJVrVPVWuDjwDleme8BPquqm7zzbgDW44SqDPfiYe1gAvzIEsMwjIwjIh/BmZoKYvwEqVzjVlyE0+WtHWu0D1NOwzA6BM/s9TXg/raKg9GxmEAYhpFxRGQyLlR1DM78aXQDzMRkGIZhxMV6EIZhGEZcelSyvuHDh2tBQUFnF8MwDKPbsHbt2v2qOiLevh4lEAUFBaxZs6azi2EYhtFtEJGE2QzMxGQYhmHExQTCMAzDiIsJhGEYhhGXHuWDMAzDSJW6ujoKCwuprq5u/eBuTN++fRk/fjzZ2dlJn5NRgRA3A9cvcDn471fVH8XsX4hL6BXFJUhbrKove/u24yb6iAD1qmpTTBqGkXYKCwsZMGAABQUFiLQr2XGXRVUpLS2lsLCQiRMnJn1exgTCm3fgbuBjuFztq0VkhZdkzefvwApVVXFTHP4RmBTYf5qq7s9UGQE0GiVaXAw1NZCTQ2jkSCRkljfD6C1UV1f3aHEAEBGGDRtGSUlJSudlsiWci5tZbJuXYfFRXE7/BlT1kDYO5fbTN3cYGo0SWb+e8vnzKSsooHz+fCLr16NRSxNjGL2JniwOPm2pYyYFYhxN0yQXetuaICIXisgm4GncfL4+CjwnbtL4qzNRwGhxMRULFxLd4cKAozt2uPXi4kzczjAMo1uRSYGIJ1fNegiqulxVJwEX4PwRPgtUdTYur/uXvTTBzW8icrWIrBGRNal2n6ipaRAHn+iOHc7cZBiG0cvJpEAU0nRaR39KwLio6irgaBEZ7q3v8f4WA8txJqt45y1V1TmqOmfEiLijxROTk0MoP7/JplB+PuTYBFOGYbSNSKSlCfe6F5kUiNXAMd7Ui31wM16tCB4gIh8UzzAmIrOBPriZuvqJyABvez/gTNx8w2klNHIkg556qkEkQvn5bn3kyHTfyjCMHsD27duZNGkSV1xxBTNmzODiiy+mqqqKgoICbrvtNk4++WT+9Kc/sWzZMqZPn860adO48cYbG85/9tlnmT17NjNnzuT0008H4PDhw3z+85/nxBNP5Pjjj+epp54CYOPGjcydO5dZs2YxY8YM3n33XQ4fPsx5553HzJkzmTZtGo899lhG65uxKCZVrReR63BTOIaBB1V1o4hc4+1fAnwC+KyI1OGmU7zEi2gaBSz3tCML+IOqpmXKzCASChGePp0BS5Ygw4cTGj/eopgMw2iRzZs388ADD7BgwQI+//nPc8899wBunMHLL7/Mnj17mD9/PmvXrmXIkCGceeaZPPnkkyxYsIAvfvGLrFq1iokTJ1JWVgbA7bffzkc/+lEefPBBysvLmTt3LmeccQZLlizha1/7Gpdddhm1tbVEIhGeeeYZxo4dy9NPPw1ARUVFRuua0XEQqvoM8EzMtiWB5R/j5oSNPW8bMDOTZfORUIjD3/oWoXHjGPS//9sRtzQMoxszYcIEFixYAMDll1/OXXfdBcAll1wCwOrVqzn11FPxTd6XXXYZq1atIhwO85GPfKRhHMLQoUMBeO6551ixYgU//ambOry6upqdO3dy0kkncfvtt1NYWMhFF13EMcccw/Tp07nhhhu48cYbOf/88/nwhz+c0braqzJAXh5aVdXZpTAMoxsQGy7qr/fr1w9wg9LioapxQ01VlT//+c+sW7eOdevWsXPnTiZPnsynP/1pVqxYQW5uLmeddRYvvPACxx57LGvXrmX69OncfPPN3HbbbWmuXVNMIADJzUWPHOnsYhiG0Q3YuXMnr776KgDLli3j5JNPbrJ/3rx5vPTSS+zfv59IJMKyZcs45ZRTOOmkk3jppZd4//33ARpMTGeddRa//OUvG4TlzTffBGDbtm184AMf4Ktf/Sof//jHeeutt9izZw95eXlcfvnl3HDDDbzxxhsZrasJBJ5AWA/CMIwkmDx5Mg899BAzZsygrKyML33pS032jxkzhh/+8IecdtppzJw5k9mzZ7Nw4UJGjBjB0qVLueiii5g5c2aDSeqWW26hrq6OGTNmMG3aNG655RYAHnvsMaZNm8asWbPYtGkTn/3sZ1m/fn2D4/r222/nO9/5Tkbr2qPmpJ4zZ462ZcKgg4sWUf/mmwzdvDkDpTIMoyvzzjvvMHny5KSO3b59O+effz4bNqQ9qLJDiFdXEVmbKNed9SAAzMRkGIbRDBMIQMxJbRhGEhQUFHTb3kNbMIHAnNSGYRjxMIHACQRVVQnD0wzDMHojJhA4ExNgSfoMwzACmEAA5OYCmB/CMAwjgAkEjT0I80MYhtHRlJeXN+RzSoVzzz2X8vLy9BcogAkEng8CwATCMIxWqH7kEUoLCigJhSgtKKD6kUfadb1EAtFa2vBnnnmGwYMHt+verZHRZH3dhYYehJmYDMNogepHHqHy6qvBayuiO3a4daDvZZe16Zo33XQT7733HrNmzSI7O5v+/fszZswY1q1bx9tvv80FF1zArl27qK6u5mtf+xpXe/crKChgzZo1HDp0iHPOOYeTTz6ZV155hXHjxvHUU0+R67/4tgMTCGj0QVgPwjB6NYcWL6Z+3bqE++tee615MEtVFZVXXUX1fffFPSdr1iz633lnwmv+6Ec/YsOGDaxbt44XX3yR8847jw0bNjRkfX3wwQcZOnQoR44c4cQTT+QTn/gEw4YNa3KNd999l2XLlnHffffxqU99ij//+c9cfvnlSdW5JUwgaDQxWQ/CMIwWSRTpmMYIyLlz5zaIA8Bdd93F8uXLAdi1axfvvvtuM4GYOHEis2bNAuCEE05g+/btaSmLCQTmpDYMw9HSmz5AaUFBs3nswc1GOfjFF9NSBj9tOMCLL77I888/z6uvvkpeXh6nnnoq1dXVzc7JCUyTHA6HOZKmtsyc1ASc1NaDMAyjBfrdfjv446Z88vLc9jYyYMAAKisr4+6rqKhgyJAh5OXlsWnTJl577bU236ctWA8C60EYhpEcviP68Le/TXTnTkJHHUW/229vs4MaYNiwYSxYsIBp06aRm5vLqFGjGvadffbZLFmyhBkzZnDccccxf/78dtchFSzdNxDZvZuy8ePpf++95HoRAoZh9A5SSffd3bF0323AwlwNwzCak1GBEJGzRWSziGwVkZvi7F8oIm+JyDoRWSMiJyd7blrLaQPlDMMwmpExgRCRMHA3cA4wBbhURKbEHPZ3YKaqzgI+D9yfwrnpIycHRKwHYRiGESCTPYi5wFZV3aaqtcCjwMLgAap6SBudIP0ATfbcdCIiNqucYRhGDJkUiHHArsB6obetCSJyoYhsAp7G9SKSPtc7/2rPPLWmpKSkzYWV3FzrQRiGYQTIpEBInG3NQqZUdbmqTgIuAL6fyrne+UtVdY6qzhkxYkRby+qmHbUehGEYRgOZFIhCYEJgfTywJ9HBqroKOFpEhqd6bjqQ3FxzUhuG0eXp379/h90rkwKxGjhGRCaKSB9gEbAieICIfFBExFueDfQBSpM5N91IXp6ZmAzDSI3avbD9Wth4fGeXJCNkbCS1qtaLyHXASiAMPKiqG0XkGm//EuATwGdFpA44AlziOa3jnpupsgLmpDYMI3lq98Ke78P+3wBR0No2X+rGG28kPz+fa6+9FoBbb70VEWHVqlUcOHCAuro6fvCDH7BwYcbidBJiI6k9yk8/Ha2uZsg//5nmUhmG0ZVpMrp452KoWpf44GgN1O6Aun04t2ig/RxwSvxz8mbBUXcmvOSbb77J4sWLeemllwCYMmUKzz77LIMHD2bgwIHs37+f+fPn8+677yIi9O/fn0OHDiVfwQCpjqS2XEwekpeHHjjQ2cUwDKMrU/0ORCrSesnjjz+e4uJi9uzZQ0lJCUOGDGHMmDFcf/31rFq1ilAoxO7du9m3bx+jR49O671bwwTCw8JcDcNo6U0fgLoi2O2bliJNTUuTXmzzbS+++GIef/xxioqKWLRoEY888gglJSWsXbuW7OxsCgoK4qb5zjSWi8nHwlwNw2iN7NFQcDfM3AbDvwCSC9Kn3ZddtGgRjz76KI8//jgXX3wxFRUVjBw5kuzsbP7xj3+wI84cFB2BCYSHmJPaMIxkiRWKvFntutzUqVOprKxk3LhxjBkzhssuu4w1a9YwZ84cHnnkESZNmpSecqeImZg8zMRkGEbK+EKRBtavX9+wPHz4cF599dW4x7XVQd0WrAfhIXl5cOQIPSmqyzAMoz2YQPjk5kI0CrVtj2c2DMPoSZhAeNi0o4bRe+kNloO21NEEwsOfNMgEwjB6F3379qW0tLRHi4SqUlpaSt++fVM6z5zUHn4PAnNUG0avYvz48RQWFtKe6QK6A3379mX8+PEpnWMC4WE9CMPonWRnZzNx4sTOLkaXxExMPr5AWA/CMAwDMIFowJzUKRKpgvrSzi6FYRgZxATCQ6wHkRyRKqgthN3fhcjhzi6NYRgZxATCo8FJbT2I+ESqoPpdJwxvfRD2/QyIdnapDMPIIOak9jEndWIilU4cNp8GkYOdXRrDMDoI60F4NPggzMTUnPAAkL5w7EoYtRgkp7NLZBhGB2AC4WFhrq1Qtwc2nQqDz4MZW2HU17Gfj2H0bOw/3MMXCBsol4BoFWgNHF4HfcbDuNsg3K+zS2UYRgbJqECIyNkisllEtorITXH2XyYib3mfV0RkZmDfdhFZLyLrRKRtE02ngjcE3XoQCYh6whn1IpfCeZA1rPPKYxhGxsmYk1pEwsDdwMeAQmC1iKxQ1bcDh70PnKKqB0TkHGApMC+w/zRV3Z+pMjYpbygEffuaDyIRvjBE7fkYRm8hkz2IucBWVd2mqrXAo8DC4AGq+oqqHvBWXwNSSxSSZsSmHU1MQw/CBMIweguZFIhxwK7AeqG3LRFXAX8NrCvwnIisFZGrE50kIleLyBoRWdPeZFuSm2vjIBJhAmEYvY5MjoOQONvi5tMVkdNwAnFyYPMCVd0jIiOBv4nIJlVd1eyCqktxpinmzJnTrny9kpdnJqZEmInJMHodmexBFAITAuvjgT2xB4nIDOB+YKGqNiT3UdU93t9iYDnOZJVZcnPNxJSIiPUgDKO3kUmBWA0cIyITRaQPsAhYETxARI4CngA+o6pbAtv7icgAfxk4E9iQwbK6++bmWg8iEWZiMoxeR8ZMTKpaLyLXASuBMPCgqm4UkWu8/UuA7wLDgHtEBKBeVecAo4Dl3rYs4A+q+mymyupjTuoWMIEwjF5HRnMxqeozwDMx25YElr8AfCHOeduAmbHbM43k5qIVFR192+6B+SAMo9dhI6mDWA8iMdaDMIxehwlEADEndWJ8YbA5IAyj12ACEcCc1C1gPQjD6HWYQASQvDwbKJcI80EYRq/DBCKI9SAS4wuD1oBGOrcshmF0CCYQASQvDyIRtK6us4vS9YgEhDNqvSzD6A2YQARomDTIehHNiR4G6eMt2/MxjN6ACUQAf9pR80PEoOpEIWuEWzeBMIxegQlEAJt2NAFaC0Qh2wTCMHoTJhBBzMQUH18QsoY3XTcMo0djAhHANzFZDyIGP8TVBMIwehUmEAHMSZ2Ahh6EZ2Ky0dSG0SswgQhgTuoEmInJMHolJhBBrAcRH7/HkG0CYRi9CROIAOaDSID1IAyjV2ICEcDCXBMQ64MwgTCMXoEJRABfIDATU1OsB2EYvRITiABmYkqAH+YaHgySZQJhGL0EE4gg5qSOjy8I4X4QyjOBMIxeQkbnpO6ODFy+nFBBAZGiIkIjRyIh09AGQQjlmUAYRi8io62fiJwtIptFZKuI3BRn/2Ui8pb3eUVEZiZ7brrRaJTI+vUcWryY8uOPp3z+fCLr16PRaKZv3fWJHAYEJMcJhA2UM4xeQcYEQkTCwN3AOcAU4FIRmRJz2PvAKao6A/g+sDSFc9NKtLiYioULie7Y4dZ37HDrxcWZvG33IFrlhEHEehCG0RqRKqgv7exSpIVM9iDmAltVdZuq1gKPAguDB6jqK6p6wFt9DRif7Llpp6amQRx8ojt2QE1NRm/bLYhWQaifWzaBMLoyndU4q0JtMdQWwu7v9phediYFYhywK7Be6G1LxFXAX1M9V0SuFpE1IrKmpKSk7aXNySGUn99kUyg/H3Jy2n5Nn+7+RuH3IMAEwuiaRKo6r3GOVEH127D3e/DWB2Hfz4AkTNPdoF3IpEBInG0a90CR03ACcWOq56rqUlWdo6pzRowY0aaCAoRGjmTQU081iEQoP9+tjxzZ5mt26o82nUQPm0AYXZP6Sjiyxf2PpdI4p4tIJVS/A+98CIrvcXO2t0RdSbdqFzIZxVQITAisjwf2xB4kIjOA+4FzVLU0lXPTiYRChKdPZ9DKlUSLigiNHk34mGPaFsUUqYL6Eii6A0qWuB/NqOvSX+iOwnoQRlckUgk1m2Hz6RA52DllqH7XmV8n/RP2/waK7/ZEwnufjVSBHoGsYXDwBcgaCfsfbBSTdLQLwXukmUz2IFYDx4jIRBHpAywCVgQPEJGjgCeAz6jqllTOzQQSCiEDBlBx6qnU/e1vbRSHSqjeAhtnQPEvWn+j6A5Eq9wYCHD/DCYQRleg8hXY/zvXOI9a7KLsOopIFVS/D/sfgI2z4O05MPo/Yeo6GPlVd0ywl3BkixtoumkB7LsjPe1CB/REMiYQqloPXAesBN4B/qiqG0XkGhG5xjvsu8Aw4B4RWScia1o6N1NlDRIaMwYZPJj6jW28XXgA9D3W/WhHXtuxP9pMETETk9HFqN0DlS/A8Ctg/0Mw/IswbQOM+joZH/8bqYTqzfD2rMaegNaA1kOfo9z/fdFdsH4qaDVoxPUs9j8Ex66EUde3r12IVEHNDth9S8bNahkdKKeqzwDPxGxbElj+AvCFZM/tCESE8JQpRNoqEADhPCj+pUtLMW0DFC+hWw9aNxOT0ZWIHIK6YogecrZ/rYEh/wF774WC+5y5JaMIHFzZ3KwU6uusB5tPcSavggcgd6oTEt8EVnIvTFkNI74AJQ+ScrsQqYTq9xrvkWG6cauVObKmTqV+40ZU4/rFk6Nun3u7OLwaxt3WaKLpjgTDXMOeQLTn2Ri9h1QjdRId72+vP+De3jef0tQpHOoPB/7kfpsZsMU3luOwu/+A02D/w86sNGOr67lEa5z1YNpGt77zeih92JmdRn3d9Rq0xgV9vLcocbsQfAaxzyM8AFDXE+kAC4UJRBzCU6eiZWVoewbJ+V9qXZFrVDP5o800sT0I1HWdDSMekcPuLb8l+3hsw5fInt6w/TaIHoGa7XBkU9NGFyDcH7QOSn+f+B7tqpNXjpr3ofQR2HQK7PuJu2ef8Y2NfTivcX36RpBcIOzWfSEJD4Qj6130U7BdaKjr9yBanfj57VoM710OfafBpJdg1H+SqaY8qauKyNdEZKA4HhCRN0TkzIyUqAuQNXUqQNv9EBAQiB4wEjs2zBXMzGQ0xW+MI4egehMU3hzfPh4rBHHt6eptfx8KvwN7fwpDL4Sin8M7J8H7lwOhpo2u9IV+J0Lp7+KLTVvFIlIFNTuhai3U7k7sZI59CQwKRaxwhIeC9GkUM9/hvftWr64XQdHP4j+/ylegcpWLfhp0Omz6qKu33xNJ89iKZGXn86p6EDgTGAFcCfwobaXoYoQ9gWiXH6K+zPvbzQVCNU4PAvdDNDqHdDUCbblOrPmjbn9jY6z1nvnno1D8q0AjKgEh+I5r+Ip/5RrJ6k0u4m/fzwPmor7uOhtnQd40GHYpbDkzpmHW5o3w+B9B/j1Q+O3APcJtE4tIFdSVeuWYDps/5sxFx650b+zJmnbiCsdoGHQeHFgOtXvdM3h7FuROSlDXQJkiByB7LAz7nLvusEth+9XOvJWBiKZkBcIfuHYu8BtV/TfxB7P1CEKjRyNDhrS9B6EKEe8HWN+O0d1dAT+muyHM1XoQaaEttvlgY9yeRqAt4ZFNTD01Loqo9n3Y+8PGN93oETiwAo57wYV6+o1oqG+jEBTf4xy0k15yPYJ4kT3RI3DoNecEPrIZSv/QcsMcznNiEx7iGteSXwfu8bOWxSK2jsFnHD0EFX/1IhK/DCUPwKZTm/od2mraGX0jHP0Y7P1R4zNoUlfveUgOSHaj6anPRDjuedj7A1eHcbfBcX+DvT/OSERTslFMa0XkOWAicLOIDEhrKboYIkLYc1S3iWile5uC7m9i8icLMhNTeohUQaQMiu6EUV91b4GJBjrVFUHtPsga7Ew3+x9sjJhpywArfwDnvl8kP1CrthjqdrvxBkTdG2vNNtdobTmzaSRN5BDsvQ36zYax34Phn4H9v28Mk570MkQqmp9bci9MfxeGXurs+xpx1yh5CMYsBhSKfuUa5tHXu2cX2zCHB0Df42DqBqjd0XiPaI0Ti2GfcWIRr96RQ65ckXIXWVR8t9s+/PMw8IxGZ7R/b41AzlGucW5LxFSkEkJ9YPOpTZ/BjPcCdf06jLjW+fqKfgFa6579/vsb6zDmG64HEvs9pJFkBeIqYBawTVWrRGQozszUY8maMoWaP/0JVUUkxc5Sw5thqPubmHxTUoNAeD0JE4jUiFS5Br/47sYGaPT17s0wKBY+tcVQuwu0Cuqq298I1JVB7XbYfFrr1/EFS/pA3Q7YfAZMuMOFbDY0ule5N92yPzY2WJEKyDvBNapa7QaGjfgC7Pule6sHrxEMxRl5HIWsoTDii7DvV+54rYHRX4acAhj33UYRTdQwh/MaTTqRCnePSHl8MUPgyFYg4v5HpU9zQQk2xqOu9coRuHc4D8hL/bvwxWzaRvfd++Y4rWusa/QIRIudiASffbAO0WoXNTV1Pey7K8aslx6SFYiTgHWqelhELgdmA79Ia0m6GOGpU9GlS9F9+5DRo1M72ReInImuUejORGMFwnoQKROpdCkZNp8W80b78/g9gkgl1O2ELWckboxbvaffyOdC3V7XSPvhkYmuE9u7qd0MZY/AJG/EctW/vfP/5Mpd8gBMfweGfgpKH3UmnqOXwe7/cm+3kUPN4/X9RjA8CMbc3PhWHurr3oYTxfcHG+PWGmZfKMKDvJDsuqaCBE40oqWB78R7xlqfQFDilKM9+GUcdxuMXty0V+TvC+W6HtG+u5s/+wbTbx6Ej4p/nTSQrED8GpjpTejzTeAB4GHglLSVpIuRNX8+A594gmh5OQqpzS7nO6j7Toaa91wXu7uOg2gQCPNBtJnwAKhaD8c+50wF/ttqojf5umJnl570svtb/CvXGE9d7xrjssdoaASC5qlIldcgVruGYsw3oOYd5zRuYs7Z7DXqy9x1IlXuLXrfXYE35i/Dji+56429Hcbc4F031tQTdseM+ALsvw+KvTf/UV9pHBMQfEtueCZeIwiNb+UtHd+m5x64hy9Io77qBLjoTnf9WMFL2MPJEEGhiO0VNez7rifcLZjZWrpOO0hWaurVjRpbCPxCVX8BDEhbKboYGo0i2dkcuv56DkyenPrscn4PIneyt96NHdXmg2g7viNa66HwBtf45p3g0jFM+mf8lAuRKtfbKP2DS+w25ibnEB15nbNb77weyHVvuTW7nOMycgiqtznfwN7/bnTKRg57ieRebnTw+m+epX909wvleA7kmU0jZyIHYcCH4dhnnEM0WuVFDH3Xve370UOhfi5sc9MC2Hdn4iijlpy6vlko2ePbQjjP1bX+gFfWn0DxXU2dziOvc/6FPmMg5wNOUNJdjpbKl2isVMNziXn28V460zzmKtkeRKWI3Ax8BviwN+NbdtpK0ZWIVKFHDnDwwgubzS43+LXXCCdjbvIFou8k97eu2HWruyNmYkqdWFNN1Qao3w+D/6Op+WP4Vd6b/J9dr6K20OXwGf45mPx/UPQTd37QIXr0H1zK6L0/AsQ5Lmt3xXfKFv/KiRK4N+Pgm+fY77je7Z7/dvv9N+mS+51zOdTfHbdhelMTWKypxzeFtPTmn+rbbYbehhvypCW0/cf4F2J7OJ1NKma2NJGsQFwCfBo3HqLIy8L6k8wVq4Op2eXyJhGBojuR4de2b3a5WIHozo5qXwj8txX/n8bvWfR0EkUYxdseqfLs/b+AkqWNDWv5cvf2PujsxmPDea6HWXIfjLjCM3nEiVQJNsyRCNRXwJaPJXYaJ7KhBxvB6BGo9XwcsVE0Y250+96enbxTvCV7euxxqTRqmWgEW7P9x7tfBzXGXZGk+k2qWgQ8AgwSkfOBalV9OKMl6yjqD0LdHvdGFhjJ2a7Z5erL3Btin7FuvTuHuvaEHkSqYw60Pn6qCK2HuvIEKSEqvQFPs12ixqCp5cByGHSmSwcRJHoY+s11Jo/WBklB0zfgI1saY+aDMfo5EwMmrDjjBsJ5kD2sac4g3/SkdZA9Evp8EKa+1TSVRTLEDlzrynSnsnYiyaba+BTwOvBJ4FPAv0Tk4kwWrEOIVDq77ZYznT3S/6cMh9o3u1x9qXuzzPJmuOvWPYgYH4T09bZ3A4Fo64CwIxuh8FvNUz8c2QR7vhN/QFLNroC9Pzjoq9a9kQ+5qPm9guGOwQa/pZG6QXv0mG9A2ZNts6G3ZO/PHgw5+W33BXSn3GPdqaydQLImpm8DJ6pqMYCIjACeBx7PVME6hODcDSW/dm9hWoMghKdPZ/Brr6ElJUS2boWcnOSjmCKeQIT7uUajO/cgYsdBiLjlrpxqo77SBQb4Yw6SGRAWqWo0vQQHMEFj6ofY7X4Kibp9zlTkx/pPfxeGLnJ5gbQWJA8Gnd/44hAklUiVROfFGx+QrA09qSiaLmKDNzqcZF8LQr44eJSmcG7XJpznnHXQ2OUm5KYgHT2a0PjxVN1zD2RHiR7cRaSoqPVopvpSF9kBrsvenQUiNszVX+6qPYi6fVDzLrx9fNP8PokI9jKih913F5tKOVrtegLHrmxMISE5zjnr59EpvrNx4hgiUPUWbqDkAZj8T2fCbKkXk0qkSux5vugkehtO5i25tSgae8vulSTbg3hWRFYCy7z1S+iEyXwyRv95sPUCyDmm6dtStAbpH2bw8nvRPT9Hiy+l/IzLGPTUU4SnT0/co6gvddcCF6rY7U1MIRdl49MVJw2KVLloHj9qJ9GAMN+5TDbU7XK9xqBDeO8PoL4chn4SJr/izD6IC4s8tNqlcB79n848ufenzWPptYaG6KLq2W5KSq9nmlR6jE6IVDGMRCQlEKr6DRH5BLAAl6Rvqaouz2jJOpLs0c5me+j/XGQDngmlejPs/w1SshTRGiIDL04u5LW+rPGNK3uka7i6K34m12C6kWQFIoOTqTeh/qDrNcQOCJu2wQsj/VMgjPRObwDZtqZRPODs95WrYOwt7vh3PuTMRZIN5U+7NBCo6xUER/z6EUDBUcH+yOkOmPXLMDJF0mYiVf2zqv6nql7fo8TBp/+H4NArXiZWLyJl86lIk4gUR4shr1rv8r/4jWJWDzAxhWLeYlsTiA6YTL2B+oMuOVvsJDJa40KXq7fDiCuh6KeNA8jqDjj/U+xgtehhQGHIJ50PQWtciueDf3OjiIcuahpJFBsB5JuFJKv5MYbRDWlRIESkUkQOxvlUikirr0YicraIbBaRrSJyU5z9k0TkVRGpEZEbYvZtF5H1IrJORNakXrUU6b/A5U2qed81AtXvJZzWr8WQ1/oD7m9WwAdRX0y3naIzWtXcDp5IICJVLmXx7gSRPukkUuWNJP42vH1i/ElkQrkufHTTgqZppvcvdXMNvD0HRlwNk1a5yCGtdWNXcqdCzgTo/2FnYipbBjnHQt4sz7/UQgRQR4wKNowOokUTk6q2OZ2GN9r6buBjQCGwWkRWqOrbgcPKgK8CFyS4zGmqur+tZUiJ/h9yfw+94kL8dl0P/eaiudNg0ivongchlN16yKsfbx/sQTT0KoZkvBppJzibnE8oz9UnSKQSqrfGifSJQ3tNTwknbtfmkTf+m3zN9sRZPTd9BArud47loZe6aCY9AsM+DTsXw8DTYMJPXY8omHU1mSgfiwQyujGZfKWZC2xV1W2qWgs8isvl1ICqFqvqaqAug+VIjtypEBrgBKLyRZcDf9giBIVNJ8OI65FBo+h/992Epk5twUHtJeoL+iCg+5qZEpqYYkxH4QEuvLelydTTZXryzTzHveQGiSWaRCb4Jp83O/4AslCOS4Gx/QtQ9YZLM+GXb/AnXI8DhY2zE/eI2hslZBhdlEwKxDgg6J0t9LYliwLPichaEbk60UEicrWIrBGRNSUl7UiKJ2HoP98JxP7fQXigaziGLUL0CKHav1L7t9c5eP751L/6auLrROL0IKD7RjJFq5qGuEJiE9Pe/4Et57jlaW81mlUiVXDk7capIFM1PcUdCR1xE8f3Pzk5E044L/EAsgk/8YQAWD/ZlS+U414StpyZfIptw+hhZFIg4s2yk4ohfoGqzgbOAb4sIh+Jd5CqLlXVOao6Z8SIEW0pZyP9PwRH1sOBxz0npboIp9zpUPYoOR//OOTmUvPoo4mvEWti6uweRHvnL47EMTGF4whEfTlU/sMloCt7DPb8sDHbZ/VmN9m8P1YgmbJqPdSWJO5xVKyE6EGXziSVlAnhPMge3jQbaX25JwSB1M7ByVhSmYPYMHoQmRSIQmBCYH08sCfZk1V1j/e3GFiOM1lllv4LnJ16+Odg9DcbG6ahi+DQP5HsMnK/8Q36funKxIPm/AYuOFAOOr4HkS5zTlwTU5yBchXPuEZ96MUw+ONegrpsl/enenNi01O0xpvtLFDW2r0uTNRPXR2vx3HgSSfCvu+oLSacRHmJgAZ/Rs5RMO775mg2eiWZ/LWvBo4RkYki0gdYBKxI5kQR6efNe42I9APOBDZkrKQ+eSc0mho2TG1smIYuAslBqzbQ7xtXEO7zW7R4Z/x5IupLXZhjeKBbD3uNVkf1ICJVULMDdt+SnkiiZMNcDzzpelv95sLgC9yUj5UvOYfw9qvdXAHSF6a83tT0VL0J9tzSGIIaynEZUd+ZH7/HEaly6a4r/uJMgJLsWM8WaC3iyBK7Gb2UNPx3xUdV60XkOmAlEAYeVNWNInKNt3+JiIwG1gADgaiILAamAMOB5d5c0FnAH1T12UyVFXCRMbU740e5ZI92o2r3P4iUXNTyoLn6Mtd78AeWhbLdekf0IBJG97SDRD4IrYdonatftBoq/grDLgMJwaCPuWPKn3SJ76KVzkxTsgQ2ne4aYa33xprETMO592fNRycDEG4c6DbiaidAgy9ITx19Wos4spHNRi8jYwIBoKrPEJOSQ1WXBJaLcKanWA4CMzNZtmYkmkwk1Beqt8Dm05A4jW6zQXPxErJ1VD6m8ADIGuwa19LfNaZ4aA+JwlzBE49BcPAFiB5yGUv9ENaBZ7leRSgHBpwKudNg9DfcNI4Hn4ehn3C+imOfc4PMYkNQ/dHJoxY7Mdh3R6OzeNilEB4M/U+J/7zbiwmBYQBmUG1KPFNDtMYJx5Q3kxs0V1/aOEjOxx8s1xGULHXzAgBMW98+u7mqN1CuBYEAZ+4Z+Z9uAhzfjzDkAjeL2qBzoGCp2x7q56K6ypZB1TrY+WWoeD5+CKo/Erq+1Jsi0psjQXKcqWrSiy5vUqZHahtGLyajPYhuS6ypIZwHoQJU+sOk19A99yceNFdf2nx60ayRcCRNLpSWBplp1IXo9v+QSz8dGtK+AVpa7f7GMzGBG1BWXwnDrnQ9lreOaZywftC5zp9T+rum01YO/ZQrW6TSPZeRX3Ll6zOGhonl/ZxGvjhP2whFv2zsPex/oPmMa4ZhpB0TiJYImhokhNTvRd89DyauQeoOxx80V18G/U5oep3skVDpjdEINvDJjihWdY1xcJ7jeOdUvgh1hS6uP1oFh1ZB+Ae02Vziv50nMjFJH6jZ0jzpXaiv8xc08+fgRiqX3Ad9j4Gj7nCRSn594s1h4G8be7NLg5LMSG3DMNKCmZhSYdA5SP0eQlm7qP3rKxw8/3wib77Z9JhIHJt41hgXCVVbCLtvc2/GyYag1hZB1drkBpmVegP8hiyE/ifB4dXOkdxWYqcb9fHXa7c7P8NxLzQNEfXHEDQLHRWXz2jyy6Q8OjlrsJt9zcYlGEaHYQKRCoPOBAQq/kqf88+HcJjaJ59s3B894j7hgA8iUuXmFiDs5g8YemFjZtHWQlDrK12q8M2nJzfIrCwwwG/4lc5EVLWu7fWNFQh/IJsf6llyr/MD1BXFhIhKfH9Og8P/9LaNTrZxCYbRodh/VypkDXNx/hV/JTR0KNmnnELN8kDm89hR1JFK1yC+MxfyZrY+KX0sJfdC6e9h8qswcnHLb83lK2D4FY0D/MJensVDr7SpqoATCMlxkx8Fezy+YJQ+Av3mw4DTPD9CnLECQaFIVxpsG5dgGB2CCUSqDDoHDr8Odfvpe801DFj2W6LlO9yo6lrPz9CQ7TMQOttkUvrrW28cD73uxmWMuApKfusmq5n67/hvzZEql9YaAgP8BPpMgMMJ8kYllYJDnKO57A9NezyhPFf+kdfCB37X1FTW2rSX6UyDbQnwDCOjmECkyqBzQPqgRzaTc84cwrkPo/sLKT/lFPRAhUsbHWy0gnMNj/kGlD3pMpBOeilx41hX7gacEYW357rpLiOlsOfW5m/NkUqXymLTKU1zCUHjJEhBIodb9n/4wlFf6cYnNOvxiIs2mvSSW94wrW2jta0XYBhdHhOIVMmdCpP+Dw48hmyaTOjg/Uj/QQx+9mGk5nE34UzW8OZv6EGh0FrXoA+7rHnjGKmE2q2enT7Q4IdyofwvzkzTRIAGuKieeD2Tfic5H0ZtoRODqvVQeGN8/0ds7qaKp6H0IZj8elNzUKivG/S35czGwYTtwXoBhtFlMYFIhUilSyK35QykZGnDDGVS+yDh0lOQ8vvd23V4QOI39HCeN05C3HSWsY1jtAZKl7m5Dpo0zHlutHLlS82vWfQ/sO0KGPeDpmYbP5Fd5CBUv+PmtWjSyxAvFfcmKPxmo3BovYuayh7vytds4OBxNp2mYfQCTCBSwfcpTN2AHve8yx205UykyXSWD8L641o2u4RyYcBHoCJOeqm9t7vEdqGcmAigfm4Ecflfmh5fX+7mTB78H83NNnmzXEqK0j9CqH+cyXL6unxI78xrFA7JAY3A0X+A+pLGkdTB69p0mobRK7CBcqniD9wKD0XrDsBxL0O0Aok7nWULDDoLdn0dana60E1wA8GK73YO6T75TSOD9AgMPAMq/hf0TpcMMFLlptLUOpdmO1hG8tz+416A/fe7OZjBNejDLoX9v4dorROeSS+7dX+kcsmvAyOVv9r8urHPwqbTNIweib32tRHJ6gc5Y9G6kWj4aDQ8Dj3uZXRkox9AW5ofadBZ7u/BlY3bdn8PRnzZpZuIFxk0+Hyo3eMGqPn+AqIuWqnfvKbX90NsN5/a2NhrjesdSK7r8RT9HDbOhLdPhDFfd2GyqYTh+pgfwTB6JCYQ7UBCIUKDR6EHhbJZp3P4e/dTX/ZxoqNfQUdcD/Uaf1IhgL5TnI2/YqUb7Vz2JIy+wfUMEpmoBn/cmbGK7mz0F0QqYcglXl6koFN8QPwxB6G+rsexaUHj4DutcefbSGXDMAKYQKSDmhqiW7Zw5Cd3cehLN1H/bgVVv8uj9pl/xJ9UCJwQDDoLKle50c45RzXNWhpLpBLq9nnRQ3c1+guyhrmEd/Gc4i1lp526IUYIbAY1wzCaIqqpTBPdtZkzZ46uWbOmw+8bKSqifP58ojt2MPCJJzh0/fVungiPUH5+00mFfCpfc0JRugxQZ//3J8nRGpjxftPMsH7o7L67XC9g2KVuTgXfhBR7fLOCxiQHjFQ1TQDo+0ISHW8YRo9DRNaq6px4+8xJnQZCI0cy6KmnqFi4EBk6tIk4QJxJhcD1CEJ9m87+5k+S46e7jn17b8hs+u22ZTZN1clsE+cYRq/G7AdpQEIhwtOnu15CQYGbRMgja948Bj79NEQiTf0R8XwEWuN6Bq2NME53ZlNzMhuGEQcTiDQhoRDh0aMJTZjgJhHKzydr3jz6/fCHHLr2WsqOPrq5P6Kl8QStNdrmLzAMI8NktEURkbNFZLOIbBWRm+LsnyQir4pIjYjckMq5XZVgb2LgY49ReeWVDSan6I4dVCxcSLQ4ZvrR9uQlspxGhmFkiIwJhIiEgbuBc4ApwKUiMiXmsDLgq8BP23Bul8XvTQDJ+SN82mPqMTORYRhpJpM9iLnAVlXdpqq1wKPAwuABqlqsqquB2GnPWj23W5CT08QfAS6iiRwbY2AYRtcnkwIxDtgVWC/0tmX63C6DH93ki0QoP5+By5YRGjmyk0tmGIbROpkMc5U425IddJH0uSJyNXA1wFFHHRXvkE4j6I+gpoZIaSl64ADR7dshL4/QyJFIyBzLhmF0TTLZOhUCEwLr44E96T5XVZeq6hxVnTNixIg2FTSTBKObRITKq66KH9FkGIbRxcikQKwGjhGRiSLSB1gErOiAc7sk0eJiDl54YZOIpkP/9V9Ed+8msmNH4pxNhmEYnUTGTEyqWi8i1wErgTDwoKpuFJFrvP1LRGQ0sAYYCERFZDEwRVUPxjs3U2XtEGpqmkQ0Zc2bR95XvkL5hz9MdMcOQvn5DHrqKcLTp5vZyTCMLoHlYuoggvmagNRyNhmGYWSIlnIx2atqB9EsomnkyGY9iv533AGHD5u5yTCMLoEl6+sgYiOaCIUI5ecT3bHDpeS4/XYqr7rKzE2GYXQZrPXpQPyIpnB+PqFx4xp6FHk33tggDtBCSg7DMIwOxHoQnUSTHsXhw6ml5DAMw+gArAfRiTTkbOrXz1JyGIbR5TCB6ALES8kx6KmnLCWHYRidipmYugBBc5MeOECkqAiys4nu2uUS/llKDsMwOgFrdboIDSk5jj0WCYWoOPdcygoKLCWHYRidhglEF0NLSqi84gqLaDIMo9MxgehqxKTkAItoMgyjczCB6GrETDKUNW8eA59+GiIR6ktKiBQVWXI/wzA6BBOILkYwoilr3jz6/fCHHLr2Wg5++tNEN2ygfP58800YhtEhWBRTFyM2JUf5KacQ3bGD/nfcQeWVVzbzTVhyP8MwMoX1ILogDQPooEEQZOhQ800YhtGhmEB0ZQL+CC0rs9HWhmF0KCYQXZigP6Lqxz9mwG9+0yAS2QsXMuj5591c1+awNgwjA5gPogsT64/QvDw32joaRffsoeKMMyw9uGEYGcMEoosT9Ef4RIqKqLj44gafRGj0aCK7dyMDBkBenqXmMAwjLVgr0h0JDKbzJxs6dO21lB19tIW/GoaRNkwguiMB57VNNmQYRqbIqECIyNkisllEtorITXH2i4jc5e1/S0RmB/ZtF5H1IrJORNZkspzdjaDzOjb81ea2NgwjXWTMByEiYeBu4GNAIbBaRFao6tuBw84BjvE+84Bfe399TlPV/ZkqY3cl1nmdaG7r7IULGfCzn0FWlqUNNwwjZTLZWswFtqrqNlWtBR4FFsYcsxB4WB2vAYNFZEwGy9RjaEgPPmFC3Lmts+bNI+8rX6H89NMpKyig8ppriL7/vuVxMgwjaTIpEOOAXYH1Qm9bssco8JyIrBWRqxPdRESuFpE1IrKmpKQkDcXuXgR7E1kzZjSYm1oSC3NkG4aRDJkUCImzTVM4ZoGqzsaZob4sIh+JdxNVXaqqc1R1zogRI9pe2m5MvLmtg74Jc2QbhtEWMikQhcCEwPp4YE+yx6iq/7cYWI4zWRktEHReB1NztJTHSaNRSyFuGEZcMikQq4FjRGSiiPQBFgErYo5ZAXzWi2aaD1So6l4R6SciAwBEpB9wJrAhg2XtEQTNTeF58xi4fHkzsYDGOSY0FKL+3/+2FOKGYcQlYwKhqvXAdcBK4B3gj6q6UUSuEZFrvMOeAbYBW4H7gGu97aOAl0Xk38DrwNOq+mymytqT8M1NWWPHkjVzZjOxCM4xEVmzhoMXXmimJ8Mw4pLRVBuq+gxOBILblgSWFfhynPO2ATMzWbbeQDBNh44e3WyOCUshbhhGS1hQfC8h3hwTlkLcMIyWMIHobQTSdFT9+McMeOCBpinEn3vOUogbhgFYNtdehx/pVLFwIfX/+hdVv/wlg//+dzQ3Fy0qouLMMxtTiK9ciQwa5ExONhLbMHodJhC9jNg0HX7DHy0upuKii5qkEI/u2UPlWWfZnBOG0UsxgeiFxJtjIphCHLzBdVdeaXNOGEYvxv7DDUfANwFNB9fZnBOG0TsxgTCApqOwAfTw4YRzTvi9iej27ebMNowejJmYDCD+/Ne+MztebyJeSnHNy0MiEXNqG0YPwf57jQZ830Q4P5+sESMa03YUFMTtTQSzxB685BKiGzY0pO2w9OKG0f0xgTASEm/OiURZYoNObUsvbhg9AxMIo1WaJAEM9CaCYpFsenHLHmsY3QcTCCMp4vUmgqk6Wkov7s+TrXV1lj3WMLoRJhBGSiRKKV714x8z4De/aSYcDSGy11/fLHusRUMZRtfGopiMlEmUJVbz8hj82mtoNMrA5cs5eOGFTcxNFg1lGN0LEwijXcQdlU1AOA4fbpY9tsGpHScaKjR6NP1++MMGh7cJh2F0HiYQRkbwhSNSVNQgCn722Mqrrkro1O5/xx1xo6FaEg5ycpDhw9H9+xt6Mr6QJFo2gTGM1rH/DiOjBEdoB7PHphoN1VIYbeU11xB56y3K589vMh4j0XLsOI36kpKGyKrgcnv8IhatZfQErAdhZJRE2WOBhpHaQdNTcDnZMNrcK67goJeJNtgDSbScqGeSrHmrtWWNRtF9+xoc8rGp01Pt1Wg06qaBtV6Q0cGYQBgZJ5GfwheOoFPbj4aqvPLKpIQDEvdAkhGY9ohIouX+d9zBoeuvj5s6PVVBCopNugSsNZFK1VzX3vu1JnJBgUy2fKmKp4lwAlQ1Yx/gbGAzsBW4Kc5+Ae7y9r8FzE723HifE044QY3uSTQS0fq9e7V++3atKy7W+r17tW73bq194w3dn5+vZfPmac0LL+j+/HytfuIJ3Z+fr8WgxdBkPZnlmhdfbDg30XKq10x0/WTPKZs3T2uef77FuiZzfHuWi0EPLFyotWvXpvW6rd2vfuvWJt97ot9AsuVL5rrJ/M5SvU7scv3evRqpq0vpnPYs1+/dq9FIJOX/PWCNJmrDE+1o7wcIA+8BHwD6AP8GpsQccy7wV08o5gP/SvbceB8TiJ5Ha8LRlkYtXSKSjLi0V5AyIWCJljN13faIXFvKl6p4ZkKEO0Ns9+fna926dSmLRGcJxEnAysD6zcDNMcfcC1waWN8MjEnm3HgfE4jeQ1A4Un1TS/TGmKqIJNOwFINWP/10mwUpEwKWaDlT122PyLWlfOnqUXYnsfU/+/PztX7v3pT+l1oSiEwa1sYBuwLrhd62ZI5J5lwARORqEVkjImtKSkraXWijexDMPBsePZpQVlbTTLQtLGeNHUvWzJkMfu01Bj72GKFp05otJxolnsxyMFpr6PbthE88sSGSK9E5idKW+KHByR7fnuV0XiuZ5WT8RW0pX6o+qXRdp72+sfbeD1zeM2pqSBuJlKO9H+CTwP2B9c8Av4w55mng5MD634ETkjk33sd6EEY6iWfeaqs9uLVrpWoHT6YX1NXNIm3pjSVTvnSZuqwHYSYmw+gyJBKRRM7H9ghYV3CsJiNysQKZTPlSFc9MiHBP8UGI259+RCQL2AKcDuwGVgOfVtWNgWPOA67DOavnAXep6txkzo3HnDlzdM2aNZmojmEYGSBReGl7Q02TuW57xqB015DheIjIWlWdE29fxsZBqGq9iFwHrMRFJT2oqhtF5Bpv/xLgGZw4bAWqgCtbOjdTZTUMo3NINEamq1w3LdfJQP06ioz1IDoD60EYhmGkRks9iF46PNAwDMNoDRMIwzAMIy4mEIZhGEZcTCAMwzCMuPQoJ7WIlAA7Wj2wkeHA/gwVp6vSG+sMvbPevbHO0Dvr3Z4656vqiHg7epRApIqIrEnkve+p9MY6Q++sd2+sM/TOemeqzmZiMgzDMOJiAmEYhmHEpbcLxNLOLkAn0BvrDL2z3r2xztA7652ROvdqH4RhGIaRmN7egzAMwzASYAJhGIZhxKVXCoSInC0im0Vkq4jc1NnlyRQiMkFE/iEi74jIRhH5mrd9qIj8TUTe9f4O6eyyphsRCYvImyLyF2+9N9R5sIg8LiKbvO/8pJ5ebxG53vttbxCRZSLStyfWWUQeFJFiEdkQ2JawniJys9e+bRaRs9p6314nECISBu4GzgGmAJeKyJTOLVXGqAe+rqqTgfnAl7263gT8XVWPwc3i1xNF8mvAO4H13lDnXwDPquokYCau/j223iIyDvgqMEdVp+GmBlhEz6zzb4GzY7bFraf3P74ImOqdc4/X7qVMrxMIYC6wVVW3qWot8CiwsJPLlBFUda+qvuEtV+IajHG4+j7kHfYQcEGnFDBDiMh44Dzg/sDmnl7ngcBHgAcAVLVWVcvp4fXGzWmT600ylgfsoQfWWVVXAWUxmxPVcyHwqKrWqOr7uPl25rblvr1RIMYBuwLrhd62Ho2IFADHA/8CRqnqXnAiAozsxKJlgjuBbwLRwLaeXucPACXAbzzT2v0i0o8eXG9V3Q38FNgJ7AUqVPU5enCdY0hUz7S1cb1RICTOth4d6ysi/YE/A4tV9WBnlyeTiMj5QLGqru3ssnQwWcBs4NeqejxwmJ5hWkmIZ3NfCEwExgL9ROTyzi1VlyBtbVxvFIhCYEJgfTyuW9ojEZFsnDg8oqpPeJv3icgYb/8YoLizypcBFgAfF5HtOPPhR0Xk9/TsOoP7XReq6r+89cdxgtGT630G8L6qlqhqHfAE8CF6dp2DJKpn2tq43igQq4FjRGSiiPTBOXNWdHKZMoKICM4m/Y6q/jywawVwhbd8BfBUR5ctU6jqzao6XlULcN/tC6p6OT24zgCqWgTsEpHjvE2nA2/Ts+u9E5gvInneb/10nJ+tJ9c5SKJ6rgAWiUiOiEwEjgFeb9MdVLXXfYBzgS3Ae8C3O7s8Gaznybiu5VvAOu9zLjAMF/Xwrvd3aGeXNUP1PxX4i7fc4+sMzALWeN/3k8CQnl5v4HvAJmAD8DsgpyfWGViG87PU4XoIV7VUT+DbXvu2GTinrfe1VBuGYRhGXHqjickwDMNIAhMIwzAMIy4mEIZhGEZcTCAMwzCMuJhAGIZhGHExgTCMLoCInOpnnjWMroIJhGEYhhEXEwjDSAERuVxEXheRdSJyrzfvxCER+ZmIvCEifxeREd6xs0TkNRF5S0SW+/n6ReSDIvK8iPzbO+do7/L9A/M5POKNDjaMTsMEwjCSREQmA5cAC1R1FhABLgP6AW+o6mzgJeC/vFMeBm5U1RnA+sD2R4C7VXUmLnfQXm/78cBi3DwlH8DllTKMTiOrswtgGN2I04ETgNXey30uLkFaFHjMO+b3wBMiMggYrKovedsfAv4kIgOAcaq6HEBVqwG8672uqoXe+jqgAHg547UyjASYQBhG8gjwkKre3GSjyC0xx7WUv6Yls1FNYDmC/X8anYyZmAwjef4OXCwiI6FhTuB83P/Rxd4xnwZeVtUK4ICIfNjb/hngJXXzcRSKyAXeNXJEJK8jK2EYyWJvKIaRJKr6toh8B3hOREK4zJpfxk3OM1VE1gIVOD8FuBTMSzwB2AZc6W3/DHCviNzmXeOTHVgNw0gay+ZqGO1ERA6pav/OLodhpBszMRmGYRhxsR6EYRiGERfrQRiGYRhxMYEwDMMw4mICYRiGYcTFBMIwDMOIiwmEYRiGEZf/D44TlmbHXlDZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_train_val_losses(df_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6.2 Test our own model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing the model DLH_team with 645 samples \n",
      "\n",
      "              Accuracy  Precision    Recall  F1-score\n",
      "Test Results  0.967442   0.965444  0.956881   0.96106\n"
     ]
    }
   ],
   "source": [
    "test_results, conf_matrix_test = test_model(our_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 6.3 Show confusion matrix for test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEGCAYAAABFBX+4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAArIElEQVR4nO3de5xVVf3/8dcbUEGxhFCUS+IF7xc0JUsr7+KVtBJQ85Jf8VdYeUmTtNT6Upp381KQFywUKSPJb3nX1FIREVEQFIXkjqCGoCIz8/n9sffoYThz5szM2XPmDO8nj/0456yz91pr9mE+s87aa6+liMDMzCpHu3JXwMzMGseB28yswjhwm5lVGAduM7MK48BtZlZhOpS7AvXZc4v9PNwlY1OXzS53Fdo8lbsC64jVH89v9qlevfTNomPOet22LutH6xa3mVmFabUtbjOzFlVTXe4aFM2B28wMoLqq3DUomgO3mRkQUVPuKhTNgdvMDKDGgdvMrLJUUIvbo0rMzCC5OFnsVgRJ7SW9KOn+9HVXSQ9Lej197JKz73BJsyTNlHRYQ3k7cJuZQdLiLnYrzg+BV3NeXwg8GhF9gUfT10jaCRgM7AwMAG6W1L5Qxg7cZmZAVFcVvTVEUi/gSOD3OckDgdHp89HA13PSx0bEqoiYDcwC+hfK34HbzAySi5NFbpKGSpqUsw2tk9t1wAVAbvO8e0QsBEgfN0vTewJzc/abl6bVyxcnzcygURcnI2IkMDLfe5KOApZExAuS9i8iu3y3zxe8/d6B28wMSnnn5L7AMZKOADoCn5H0R2CxpC0iYqGkLYAl6f7zgN45x/cCFhQqwF0lZmZQsouTETE8InpFRB+Si46PRcRJwATglHS3U4D70ucTgMGSNpC0FdAXmFioDLe4zcygJW55vxwYJ+l04C3gWwARMU3SOGA6UAUMi4iCzX8HbjMzyOTOyYh4Angifb4MOKie/UYAI4rN14HbzAxooJHbqjhwm5lBRd3y7sBtZgaeZMrMrOK4xW1mVmGqV5e7BkVz4DYzA3eVmJlVHHeVmJlVGLe4zcwqjAO3mVllCV+cNDOrMO7jNjOrMO4qMTOrMG5xm5lVGLe4zcwqjFvcZmYVpirzhRRKxoG7ie6f+CdWrviAmuoaqqurOWnA/3Dmed/h2BOP5t1l7wFw469+x78ee7a8FW0jRo28miOPOJglby+l3x5556K3ZurVqwe333Y93TfflJqaGm79/Rh+c+Ot5a5Wy3GLe91w5jd/wHvv/HeNtDEjx/GH395dphq1XXfeOY6bb76d22+/vtxVabOqqqq44ILLeHHKK3TuvBHPPfcAjzz6JK+++nq5q9YyStTHLakj8CSwAUmM/XNEXCLpUuAM4O10159ExN/TY4YDpwPVwA8i4sFCZThwW0V46unn2HLLXuWuRpu2aNESFi1KFh5fsWIlM2a8To8em687gbt0Le5VwIERsULSesDTkv6RvndtRFyVu7OknUgWFd4Z6AE8Imm7QutOepX3JooIbhp7DWMevJXjTjrmk/RB3zmOex69g0uuGc7Gn924jDU0a7ott+xFv913YeLEF8tdlZZTU1P8VkAkVqQv10u3KHDIQGBsRKyKiNnALKB/oTIyCdySOkv6uaRpkv4r6W1Jz0o6NYvyyuG0Y77LiYeezlknnMfxpx7Hnvvszp9Gj+eYfQYx+ODTWLpkGedecla5q2nWaBtttCHj7hnFeT+6hPffX9HwAW1F1BS9SRoqaVLONjQ3K0ntJU0BlgAPR8Rz6VtnSZoq6TZJXdK0nsDcnMPnpWn1yqrFPQZ4EzgMuAy4Afg2cICkX9Z3UO7JWPrBooyqVhpLFy8D4N1l7/H4P55k53478c7Sd6mpqSEi+MsfJ7DzHjuWuZZmjdOhQwfG3TOKu+8ez1//+o+GD2hLqqqK3iJiZETslbONzM0qIqojoh/QC+gvaRfgFmAboB+wELg63V15alOohZ5Z4O4TEXdExLyIuAY4JiJeB04DjqvvoNyT0W3DzTOqWvN17NSRDTfq9Mnzfb62N2/MfJNum33uk30OPOKrvDHjzXJV0axJRo28mhkzZnHd9SMb3rmtiSh+KzrLeA94AhgQEYvTgF4DjOLT7pB5QO+cw3oBCwrlm9XFyZWS9ouIpyUdDbwDEBE1kvL9dakon9u0K1fflnxxaN+hPQ+Mf5h/P/4cv/jNxWy3c1+IYMHcRYy44Moy17Tt+OMfbuJrX/0S3bp1Zc6bk7js51dx+x1jy12tNmXfL+/NSSd9k5dfns6k5x8C4OKfXs4DDzxW5pq1kNKNKtkUWB0R70nqBBwMXCFpi4hYmO52LPBK+nwCcJeka0guTvYFJhYsIxrx16MRFd8N+D2wXVq570TEa+kPNCQibmgojz232K/0FbM1TF02u9xVaPMqvpVSIVZ/PL/Zp/rDMT8tOuZ0OvEX9ZaXxr/RQHuSXo1xEfFzSX8g6SYJYA5wZm0gl3QR8B2gCjg7Igr2U2XS4o6IqeS5KhoRb5P0d5uZtS4lGg6Yxr898qR/u8AxI4ARxZbR4sMBJZ3W0mWamTWourr4rczKMY77sjKUaWZWWInGcbeETLpKJE2t7y2gexZlmpk1SysIyMXKalRJd5Ix3O/WSRfw74zKNDNrOk8yxf1A54iYUvcNSU9kVKaZWZNFTeUMZMtqVMnpBd47IYsyzcyaxV0lCUm7AjukL1+NiFcK7W9mVjatYLRIsbK6OPlZ4D6S2zinkvRt7yrpLWBgRCzPolwzsyZzi5tfAJNI5qStAZDUDricZJD59zMq18ysaRy4ORjYrTZowyfzlPwEeDmjMs3Mmi6D6T+yklXg/jgi1lp5MyKqJK3KqEwzs6Zzi5uOkvZg7Tl2RLIOm5lZ67KuDwcEFgHXFHjPzKx1WddHlUTE/lnka2aWlVjXu0ok1bvKDUBE/CWLcs3MmsxdJRxd4L0AHLjNrHXxXCVcGBGLM8rbzKz0KqjFndV83C9JeljSd9K7KM3MWreq6uK3AiR1lDRR0kuSpkm6LE3vmsbF19PHLjnHDJc0S9JMSYc1VNWsAndP4CrgK8Brkv4qaVC6cKaZWesTNcVvha0iuWt8d5I1JgdI2ge4EHg0IvoCj6avkbQTMBjYGRgA3CypfaECMgnc6RL0D0bEaSTzldwOfB2YLWlMFmWamTVLTRS/FRCJFenL9dItgIEkiwiTPn49fT4QGBsRqyJiNjCLPGv25sp86bKI+BiYDrwKLAd2yrpMM7PGipqaojdJQyVNytmG5uYlqb2kKcAS4OGIeA7oXruqe/q4Wbp7T2BuzuHz0rR6ZTatq6TPA4OAIcBGwFiSmQFfzapMM7Mma8TFyYgYCYws8H410E/SJsB4SbsUyK7uHeaQtNDrldU47n+T/MX4MzA0IiZlUY6ZWclkMKokIt5LV/0aACyWtEVELJS0BUlrHJIWdu+cw3oBCwrlm1VXyXCgT0ScB8yQtFFG5ZiZlUZ1dfFbAZI2TVvapAMyDgZmABOAU9LdTiFZs4A0fbCkDSRtBfQFJhYqI6tb3v8p6buShpN0k0jS+8AVEXFzFmWamTVHCdec3AIYnY4MaQeMi4j7JT0DjJN0OvAW8C2AiJgmaRzJtcAqYFja1VKvrLpKLgL2BfaPiDfTtK2B6yV1jYj/zaJcM7MmK1HgjoipwB550pcBB9VzzAiSRWaKktXFyZOB3SPio9qEiHhT0vHAS4ADt5m1Luv6JFMAuUE7J+1DSZVzdsxs3eFb3pknaa2vBGnawozKNDNruhLdgNMSsmpx/wC4T9LTwAskYxL3Jun3HphRmWZmTRbVldMZkFXgXgWcCmxHcv+9gCeBW4G1ulDymbpsdkZVs1pXbn5AuavQ5g1f8mS5q2DFagUt6WJlFbivA34SEbflJkraK32v0HzdZmYtroTDATOXVeDukw6JWUNETJLUJ6MyzcyazoGbjgXe89SuZtb6VE4Xd2ajSp6XdEbdxPSOoRcyKtPMrMmiqqbordyyanGfTTIj1ol8Gqj3AtYHjs2oTDOzpit/PC5aVnOVLAa+LOkAoHY6w/+LiMeyKM/MrLl8cTIVEY8Dj2dZhplZSazrLW4zs0rjFreZWaVxi9vMrLJEVblrUDwHbjMzINziNjOrMBUUuLO6AcfMrKJETfFbIZJ6S3pc0quSpkn6YZp+qaT5kqak2xE5xwyXNEvSTEmHNVRXt7jNzChpV0kVcF5ETJa0MfCCpIfT966NiKtyd5a0EzCYZCbVHsAjkrYrtO6kA7eZGRDVKk0+EQtJF4yJiPclvQr0LHDIQGBsRKwCZkuaBfQHnqnvAHeVmJnRuK4SSUMlTcrZhubLM50NdQ/guTTpLElTJd0mqUua1hOYm3PYPAoHegduMzOAqFHxW8TIiNgrZxtZNz9JnYF7gbMjYjlwC7AN0I+kRX517a75qlOoru4qMTOjtMMBJa1HErTHRMRf4JM5nGrfHwXcn76cB/TOObwXsKBQ/m5xm5kBESp6K0SSSJZpfDUirslJ3yJnt2OBV9LnE4DBkjaQtBXQF5hYqIwmtbglfSMi7m3KsWZmrVEJW9z7At8GXpY0JU37CTBEUj+SbpA5wJkAETFN0jhgOsmIlGGFRpRA07tKriX5GmBm1ibUlG5UydPk77f+e4FjRgAjii2jqYG7ND+hmVkrETWVE9aaGrgrZ/5DM7MitInALell8gdoAd0zq5GZWRlEBTVHC7W4j2qxWpiZlVmbaHFHxH/ypUvaFzgBGJZVpczMWlpDw/xak6L6uNMhLCcAxwOzgb9kWCczsxZXXaJRJS2hUB/3diQzVg0BlgH3AIqIA1qobmZmLaattLhnAE8BR0fELABJ57RIrczMWlgl9XEXuuX9G8Ai4HFJoyQdhMdvm1kbFVH8Vm71Bu6IGB8Rg4AdgCeAc4Dukm6RdGgL1c/MrEU0ZnbAcmtwkqmIWBkRYyLiKJJZq6YAF2ZdMTOzllRd067ordwKXZzsWicpgHcj4nfA7zKtVYUZNfJqjjziYJa8vZR+exxU7upUtEOuPIOtD+rHB8uW84dDhgOwzznHseuQ/flg2fsA/OvX45jz+EsA7D3saHYZtD811TU8ccmd/OfJl8tW97bg+98/ndNOG0JEMG3aDM4440esWrWq3NVqEa2hC6RYhf50vABMSh9fACYDb0t6RNKWLVG5SnHnneM48qgTy12NNmH6n55k/MlXrpU++fcPMObwixhz+EWfBO2ufXuw/dH7cOfBP2b8yb/mwBGnonbl/xpbqXr06M6wYafx5S8fyRe+cAjt2rXn+OOPLne1WkxNqOit3Ar1cW8VEVunj7VbN+BmmtHiltSjqce2Vk89/RzvvPteuavRJsyfOJOP3ltR1L7bHPoFZv7tWao/rmL53Ld5b85iNu+3TcY1bNs6dOhAp04dad++PRtu2ImFCxc3fFAbUar5uFtCoztr0tUcNmtGmc8241hbR+1+yiGc9OAvOeTKM9jgsxsC0Ll7F95f8M4n+6xY+A6dN+9SXxbWgAULFnPttSN5/fVnmTNnEsuXL+eRR54qd7VaTJsYVVKfdB215vTO1/vnKncBzpqalc0owtqSqX94hNu/ci5/HHARK5e8x1cvTrultPZ/pdbwS1WpNtnksxx99CHssMO+bLXV3my44YYMGXJsuavVYiqpq6TQxclz8yR3AY4BbmxGmfX+aqULbo4E6LB+T/8KGgAfLF3+yfNX7n6cgbefB8CKRe+wcY9Pr6F33qIrKxe/2+L1aysOPHA/5syZy9KlybeY++57gH32+QJ33z2+zDVrGa1htEixCt05uXGd10FyQ85JEVHw0r2k31D/lLCbNKaCZhtttgkrl7wHwDaH7cWymfMAePPhyRx+w/eYPOofbNS9C1222pxFU94oY00r29y58+nff086derIhx9+xAEH7MsLL0wtd7VaTKlaipJ6A3cCmwM1wMiIuD4dqXcP0Idk6bLjI+Ld9JjhwOlANfCDiHiwUBmFAvcGEfGTJtZ9UhPfq0h//MNNfO2rX6Jbt67MeXMSl/38Km6/Y2y5q1WRDv/NMHp/aUc6dunM/zx3A89ccy+9v7Qjm+60JRHB8nlLeXT4bQAse20+r93/HCc/egU1VTU8dvEdRI2/qDXV889PYfz4v/Pss3+nqqqal16axq233lXuarWYEnaBVAHnRcRkSRsDL0h6GDgVeDQiLpd0Icn9MD+WtBPJvFA7Az2ARyRtV2jdSUU9nYKSJkfEnqX6SdI8O5LMffKnhvZ1V0n2rtzc84VlbfiSJ8tdhXXCRx+91eyo+6/Nv1l0zNl30Z+LLk/SfSTdyzcC+0fEwnTF9yciYvu0tU1E/Crd/0Hg0oh4pr48C3XqtJfURVLXfFsjKt1e0uGS7gT+Awwq9lgzs5ZS04gtdyBFug3Nl6ekPsAewHNA94hYCJA+1o7O6wnMzTlsXppWr0JdJTuQ3HiT7y9LAFsXyljSV0nm8D4SmEiyZP1WEfFBoePMzMohGjGHXu5AivqkI/DuBc6OiOXKMwqqdte81SmgUOCeHhF7FDq43lpI84C3gFuA8yPifUmzHbTNrLWqKuEwP0nrkQTtMem9LwCLJW2R01WyJE2fB/TOObwXsKBQ/lmNf7mXpKk/CDha0kZ4ZXgza8UCFb0VoqRpfSvwakRck/PWBOCU9PkpwH056YMlbSBpK6AvSS9FvQoF7lGSNs1Tqc3Si4z1iogfkgx5uQY4AHgN2FTS8enXBzOzVqUxfdwN2Bf4NnCgpCnpdgRwOXCIpNeBQ9LXRMQ0YBwwHXgAGFZoRAkU7irpRzJuu+76kocA+wHfLZRxJMNVHgMeS782HE4y5OVmoFuhY83MWlpj+rgL5hPxNPXfIZ53+tCIGAGMKLaMQi3u/XL6ZnILGAN8tdgC0mNWR8SEiDiBNftyzMxahRK2uDNXqMVd6M9Pwb5xSQ3dbrVbA++bmbWo6gpambFQ4F4iqX9ErNFJLqk/8HYD+daQXIy8C/gb8GGzamlmlrFWsCJZ0QoF7vOBcZLuIBnPDbAXcDJJX3W9IqKfpB2AISTBe3r6+FBEVDW30mZmpVZTQS3uQgspTAS+SNJlciprDmM5uaGMI2JGRFyS3jb/N5JJV85pboXNzLIQjdjKrVCLm4hYDFwiaQ+S1vMpJBcm720oY0k9SVrmxwLvkgTtdWN+SDOrOK3homOxCs3HvR1J4B0CLCOZjlAR0eDMRJL+STIt7DiS1nrtMiXrS+oaEe/Ud6yZWTnU1H9LeqtTqMU9A3iKZDa/WQCSiu3q2JLkG8WZQO7kK6KIeU7MzFpawTteWplCgfsbJC3uxyU9AIyl8BDBT0REn+ZXzcys5VTSqJJCFyfHR8QgklkCnyDpo+4u6RZJhzalMEnbSxrVpJqamWWoBhW9lVuDk0xFxMqIGBMRR5HMWjWFZOWGeknaTdJDkl6R9L+Suku6F3iUZGigmVmrUkmjSho1O2BEvBMRv4uIAxvYdRTJuO1vkNysMxl4E9g2Iq5tUk3NzDJUo+K3cis4HLAZNoiIO9LnMyX9CLiwoRmvzMzKpU0MB2ymjunY79q/TSuA3dJ5aomIyRmVa2bWJNWtoCVdrKwC90KSubhrLcp5HUBDXS1mZi1qnW9xF3OTjplZa7LOB25IVsoBhgE7k7SypwM3RcSSggeamZVBCZeczFwma05K2hd4Pn15J/DH9PnE9D0zs1allAspSLpN0hJJr+SkXSppfp3lzGrfGy5plqSZkg5rKP+sWtxXA1+PiBdz0u6TNB74Hcmsg2ZmrUaJh7zdAdxI0nDNdW1EXJWbIGknkrvUdwZ6AI9I2q7QKLysVnn/TJ2gDUBETCGZfMrMrFUp5TjuiHiSTyfXa8hAYGxErIqI2cAsoH+hA7IK3JLUJU9i1wzLNDNrssZ0lUgaKmlSzja0nmzrOkvS1LQrpTZG9gTm5uwzL02rV1ZB9FrgIUlfk7Rxuu0P/CN9z8ysVWlM4I6IkRGxV842sogibgG2AfqRDJm+Ok3P14YveGd9VsMBR0paAPyCpN8GYBrwvxHxtyzKNDNrjqznIEkXpgEgnWzv/vTlPKB3zq69gAWF8spsOGBE3M+nFTMza9WynoNE0hYRsTB9eSxQO+JkAnCXpGtILk72BSbmyeITmQRuST8r8HZExC+yKNfMrKlKOapE0t3A/kA3SfOAS4D9JfUjadzPIVlohoiYJmkcyb0uVcCwhuZ1UkTpvyBIOi9P8kbA6cDnIqJzQ3l0WL9na5g9sU1br31mX7gstXzu4+WuwjphvW5bN7u9PGLLE4uOORf9Z0xZb9fJqo+7ttMdSRsDPwROI1lF5+r6jjMzKxff8s4nQ//OBU4ERgN7RsS7WZVnZtYclfQVP6s+7iuB44CRwK4RsSKLcszMSsUtbjgPWAVcDFykT5e9F8nFyc9kVK6ZWZNUqXLa3Fn1cfvuSDOrKJUTtjPs4zYzqyTuKjEzqzA1FdTmduA2M8NdJWZmFcddJWZmFaa6gtrcDtxmZrjFbWZWccItbjOzyuIWt5lZhfFwQDOzClM5YduB28wMgKoKCt0O3GZmVNbFSU8GZWZG41Z5b4ik2yQtkfRKTlpXSQ9Lej197JLz3nBJsyTNlHRYQ/k7cJuZkbS4i/1XhDuAAXXSLgQejYi+wKPpayTtBAwGdk6PuVlS+0KZO3CbmVHaFndEPAm8Uyd5IMlqYKSPX89JHxsRqyJiNjAL6F8ofwduMzOgOqLoTdJQSZNytqFFFNE9IhYCpI+bpek9gbk5+81L0+rli5NmZjRuHHdEjCRZmrEU8q0YX7AybnGbmVHyPu58FkvaAiB9XJKmzwN65+zXC1hQKCMHbjMzStvHXY8JwCnp81OA+3LSB0vaQNJWQF9gYqGM3FViZkZpb3mXdDewP9BN0jzgEuByYJyk04G3gG8BRMQ0SeOA6UAVMCwiqgvl78BtZkZpb8CJiCH1vHVQPfuPAEYUm78Dt5kZyaiSSuHAbWaGZwc0M6s4no/bzKzCVNIkUw7cZmZUVleJx3GXwGGH7s+0V55kxvSnueD8YeWuTps1bNhpTJr0EC+88DBnnfWdclenolVXV/PNU4fxvfMvAeDBx55i4Ilnsut+R/DKq699st/q1au5eMQ1HPvt73LcKd9j4uSp5apy5iKi6K3cHLibqV27dtxw/QiOOvokdt39AAYN+jo77ti33NVqc3baaTtOO20IX/nKMfTvP4DDDz+IbbbpU+5qVaw//uk+tu7z+U9eb7v1llz3y5/yhX67rLHfnyc8AMD4P9zCqOt+yVU3jqKmppJ6g4tXTRS9lZsDdzP133sP3nhjDrNnv8Xq1asZN+4+jjm6wel0rZF22GFbJk58kQ8//Ijq6mqeeuo5Bg70eW6KRUve5sl/T+QbOf9Pt+nzebbastda+74x5y2+uFc/AD7XZRM27rwR02a83lJVbVE1RNFbubVo4JbUW9L5LVlm1nr03Jy58z6dVmDe/IX06LF5GWvUNk2b9hr77defrl03oVOnjgwYcAC9evUod7Uq0hXX/45zv3c6UsO//ttvuxWPP/UMVVXVzFuwiOkzZ7Fo8dstUMuWV0ldJZlfnJTUjeTWziEkUxWOz7rMliStPbFXa/hg25qZM2dx9dW/5f77x7By5UqmTp1OVVVVuatVcZ7413N07bIJO+/Qt6j+6mOPPIw358xl0Ok/oMfmm9Fvlx1p36HgHP8VqzW0pIuVSeCWtDFwLHACsB1JsN46Itb+LrbmcUOBoQBq/1natdsoi+qV1Px5C+md0/Lr1XMLFi5cXMYatV2jR9/D6NH3AHDZZeczf/6iMteo8rw4dTpPPP0sTz3zPKs+Xs3KlR/w48t+zRWXXJB3/w4d2vPjH575yesTzzyXLdvoNx0PB0ymK5wIXAw8HREh6diGDsqd47bD+j0r4iw+P2kK2267FX369Gb+/EUcf/xAvn2yR5ZkYdNNP8fbby+jd+8eDBw4gP33b/C/lNVxzndP45zvngbAxMlTuePue+sN2gAffvQREbBhp478e+JkOrRvzzZbbdlS1W1RvuUdfkKyhtotwF2S7smonLKrrq7mh2dfzN//7y7at2vHHaPvYfr01xo+0Brt7rt/S9euXVi9ejVnn/0z3ntvebmr1GY88s9/8atrb+Gd9/7L986/hB36bs3Ia0fwzrv/5cxzLkLt2tF908/xq5/9qNxVzUwldZUoy/5YSVuT9G0PJplj9hJgfEQ0GNkqpcVdydZr7/uvsrZ87uPlrsI6Yb1uW+dbRaZRvtTzgKJjzjPzH292ec2R6aiSiHgzIkZExK7A3sAmwD+yLNPMrCkqaVRJiw0HjIiXI2J4RGzTUmWamRWrksZxZzWq5H3yL3YpICLiM1mUa2bWVOv8qJKI2DiLfM3MslIdpbuVX9Ic4H2gGqiKiL0kdQXuAfoAc4DjI+LdpuTvW97NzMikj/uAiOgXEXulry8EHo2IvsCj6esmybqrJPfKa6TlrR8RHs5gZq1KC/RdDyRZQBhgNPAE8OOmZJRJizsiNo6Iz6SPGwM9SBbCXARcn0WZZmbNEY34J2mopEk529C1soOHJL2Q8173iFgIkD5u1tS6ZtrylbQJcDZwMnAXsHdELMuyTDOzpqhpxDC/3Lu867FvRCyQtBnwsKQZza1frkxa3JK6SfoVMBmoAvaIiIsdtM2stWpMi7vBvCIWpI9LSOZq6g8slrQFQPq4pKl1zarF/R/gbeB24APg9NxZ9CLimozKNTNrklKNKpG0EdAuIt5Pnx8K/ByYAJwCXJ4+3tfUMrIK3Ffy6ThuDw00s1avMV0lDegOjE8bqx2AuyLiAUnPA+MknQ68RTLddZNkNY770izyNTPLSqluwImIN4Hd86QvAw4qRRmZjeOWdLikJyUtlfS2pH9KOiKr8szMmqMmouit3LIax30GcCZwATApTd4LuFxSr/SKrJlZq7HO3/IOnAPsFxHv5KQ9Julw4GkKD6MxM2tx1VFd7ioULavArTpBG0j6ePKt0WhmVm6tYbrWYmXVx71c0lqd82na+xmVaWbWZOv8tK7AecAESbcDL5AMDdybZOziSRmVaWbWZJXU4s5qOODTkr4IfA84lWSyqWnAPhHhpbnNrNVpDaNFipXZXCVpgP5ZVvmbmZXSOj+qRNLU+t4iWQFntyzKNTNrqlIupJC1rFrcNST92ncBfwM+zKgcM7OScB93RD9JOwBDSIL39PTxoYioyqJMM7PmqKQ+7sxueY+IGRFxSUTsSdLqvpPkxhwzs1Yng6XLMpPZxUlJPYHBwLHAuyRBe3xW5ZmZNUdrGJ9drKwuTv6TZDrXcSTDAWvvolxfUtd8d1WamZVTa2hJFyurFveWJBcnzwRy12JTmr51RuWamTXJOj+qJCL6ZJGvmVlWKuniZFZdJXvWSQpgaUTMzaI8M7PmclcJXJ0nrauk9YEhETElo3LNzJqklHdOShoAXA+0B34fEZeXLHOy6yo5IF+6pL2AG4CvZlGumVlTlarFLak9cBNwCDAPeF7ShIiYXpICyHA4YD4RMUlS55Ys08ysGCXs4+4PzErXnkTSWGAgyY2IJdGigVtSdyju+0jVx/MrbsUFSUO9LFu2fI6zt66e48bEHElDWXPE3Micc9YTyL2eNw/4YvNr+KmsLk7+hrUDdFfgy8APsyizlRiKl2XLms9x9nyOG5AG6frOUb4/ACW98plVi3tSndcBLAPOjYglGZVpZtYazAN657zuBSwoZQFZBe7xEbE83xuSPh8Rb2VUrplZuT0P9JW0FTCfZOqPE0pZQFaTTD1R+0TSo3Xe+2tGZbYG/nqZPZ/j7PkcN0M6A+pZwIPAq8C4iJhWyjKUxaBzSS9GxB51n+d7bWZmjZNVizvqeZ7vtZmZNUJWfdybSTqX5Opq7XPS15tmVKaZ2Tohqxb3KJJpXTvnPK99/fuMymwUSZtLGivpDUnTJf1d0naSdpb0mKTXJL0u6adK7C/pmTp5dJC0WNIWku6Q9M00/QlJMyVNlTRD0o2SNsk57jZJSyS9Uie/3SU9I+llSX+T9JkWORkZkhSSrs55/SNJl+a8HpqeoxmSJkraL+e92vP4kqTnJfXLeW+OpKfqlDUlzzm9XtJ8Se1y0k6VdGNpf9JsSKqu/bkk/UnShml6vedV0qXpzzwlZ9sk38+dnuO90ucNnlNJ+6WfU+1nNjTnvUslfSBps5y0Ffmep6/PkfSRpM828zStczIJ3BFxWaEtizIbQ5JIFnV4IiK2iYidgJ8A3YEJwOURsR2wO8nY8+8BTwK9JPXJyepg4JWIWJinmBPTRZF3A1YB9+W8dwcwIM8xvwcujIhd0/qd3+QfsvVYBRwnqVvdNyQdRTL1734RsQPw/4C7JG2es9uJEbE7cDNwZZ0sNpbUO81rxzz5tyNZyGMulTvNwocR0S8idgE+JjlHUOC8pq5Nj6vd3iuyvHrPafq53AX8v/Tz2g84U9KRObstBc4rsqwhJCMwji1yf0tlErgl/azA9tMsymykA4DVEfHb2oR04qvtgH9FxENp2gckV4cvjIga4E/AoJx8BgN3FyooIj4GLgA+L2n3NO1JPl1cItf2JH8gAB4GvtHon6z1qSIZpZBv2bofA+dHxFKAiJgMjAaG5dn3GZI70nKN49PPYwhrfxYHAK8At6TvV7qngG3T54XOa3MUOqfDgDvSz4n0c7sAuDBnn9uAQZK6FipE0jYk38Avpm18Ni0qq66SlXk2gNNJflnLbRfghTzpO9dNj4g3gM5pt8XdJMEaSRsARwD3NlRYRFQDLwE7NLDrK8Ax6fNvseYg/kp2E3Binq/Ea51vkpu3ds6TxwDWHkr6Z+C49PnRJGub5qoNPOOBoySt17hqtx6SOgCHAy/nJNd3XgHOyekmebwRRRU6p8V8XitIgndDd0jXfjZPAdvndq9Yw7LqKrm6diNpFXQCTgPG0rpXv6ldoSefiIjnSYL49iS/RM9GxLuNyLsh3wGGSXqB5JrAx0Xm3aqlN2PdCfygiN3rfgZjJM0j+YP/mzr7vgO8K2kwyXjZDz7JJJlC+Ajgr2n5zwGHNvmHKJ9OkqaQBMi3gFtr32jgvOZ2ldTO1lnv/+2c5/WeU+r//aibdgNwSgPXaAYDY9Nvsn8haahYkbJcLLgrcC5wIsnX3z0bEeSyNg34Zj3pa/SFStoaWBER76dJY0n+0+1IA90kOXm0B3Yl+UWoV0TMIA0ukrYDjiy0f4W5DpgM3J6TNh34AvBYTtqerDmL2okk31YuJ2lhHsea7knTT62TPgD4LPByckmDDUmC0P81/Ucoiw8jol+B969j7fNan2VAlzppXUn6pXPVd06nAXuRXAeq9QXqzHoXEe9Juovk2tBaJO0G9AUeTj+b9YE30zKtCFn1cV9JctHhfWDXiLi0FQVtSALFBpLOqE2QtDfwOrCfpIPTtE4krYdf5xx7N3AScCBr/gfOK/16/itgbkRMbWDfzdLHdiR9f78ttH8lSReIHkfSXVbr18AVkj4HkI4aOZXkQmTusatJzsc+eS5Cjk/zebBO+hDgfyKiT7qU3lbAobWjMtqKes5rfZ4H9q29+JuOJtmANWeyg/rP6U3AqbWje9LP7QrW/P2odQ3Jhed8jcMhwKW1n01E9AB6StqyiJ/ByK6P+zygB8kv2wJJy9PtfUl55zBpSZHcLnoscIiS4YDTgEtJJoIZCFwsaSZJf+LzwI05x04nabk9FhEr6+adY4ykqST91hul+QIg6W6Si23bS5onqfaXboik14AZaV2KaUVVkquBT0ZBRMQEkv7Qf0uaQTJ09KR8o3Qi4sP0+B/VSX8/Iq5ILwIDkAbnw8hpXaef1dMk/baQBKB5OVuvUv2QZbDGeU3l9nFPkdQnIhaT9D3/Pe1+uY5kRao1VsnNd07T9IUkjZZR6ef1b+C2iKh7baH2wuV4kj8MdQ1O38s1Pk23ImRyy7uZmWUnq/m4O5KMN90WmEryV7kqi7LMzNY1WU0ydQ+wmmSoz+HAfyKiLS+gYGbWYrIK3C+nd//Vjj+dGBF7lrwgM7N1UFYXJ1fXPnEXiZlZaWXV4q7m07slRXIDzgfp84iIip88ycysXDyqxJot/UP9MsnF7leBUyLigzrps4Fvpzdn9En3m5mTzTURcaekOSTj/wHak9xV94uIWJUed3864RKS+gNXkUwOFiTD/V4Easfn75SWUQ08QDLM8kqS5aRqnUDSqHg1fb9jWv5NETG62SfHLAMO3NZsklZEROf0+RjghYi4pk76aOC1iBhRNwDXyWsOsFdELJXUmWTKhNURcUrucZK6AxOBwRHxTDrj4zeAp9Lxymvklb4+NX19Vp0y16hPerfsX4DrI6KtjaW3NiCrPm5bd+XOYJcr3+x+BUXECpJhpV/PM9vcMGB0RDyT7hsR8efaoN0cEfEmyXQNxcytYtbiHLitZOqZwa52rpaDWHOKgG3q3Nn3lXx5phMpzSaZ2yJXfTM8NmRQnXI71bPfZBqezdGsLDKbZMrWKbUz2EHS4r61TnofkiD7cM4xbzQweVKuYmZWLNY9ebpKsi7TrKTc4rZSqF2lpV9EfD9njovame22JJkBLt8CCQVJ2pgk8L9W561pJDPTZWUPGpjN0axcHLgtcxHxX5L+4h81ZjGD9OLkzSRzatedXfJGkjmfv5iz/0lac9mzJkkvVl7F2vN/m7UK7iqxFhERL0p6iWQGuKdI+7hzdrktIm5Inz+ejhJpRzJr3C/y5Lc4nez/qnQ63BqSZd/+0kBVBilnQWKSOaMXpPV5kU+HA/7GI0qstfJwQDOzCuOuEjOzCuPAbWZWYRy4zcwqjAO3mVmFceA2M6swDtxmZhXGgdvMrML8fw/GtULEfmyfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = sns.heatmap(conf_matrix_test, annot=True, fmt='g', xticklabels=categories, yticklabels=categories)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Covid19 chest x-ray classification.ipynb",
   "private_outputs": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}