ch-04/multi_input_nn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3fe68682",
   "metadata": {},
   "source": [
    "### Multi-input NN with PyTorch Lightning\n",
    "\n",
    "In this Notebook we see how to create a **multi-input NN**: a NN that has as input an image + tabular data\n",
    "\n",
    "I have started from the example by M. Rosenfelder here: https://rosenfelder.ai/multi-input-neural-network-pytorch/\n",
    "\n",
    "I have made some modifications:\n",
    "* removed the need to use the pickle file. The csv file extracted (tab_data.csv) is in my GotHUb repo, in the same dir as this NoteBook\n",
    "* I have changed the LR\n",
    "* The dataset is only split in train/validation to have more data for training\n",
    "* I have added a LRSCheduler (ReduceLROnPlateau)\n",
    "* I have done some other changes to make the code a little clearer (in my view)\n",
    "\n",
    "Images can be found here: https://1drv.ms/u/s!AqUPqx8G81xZiL1l80RtZbjPj43MhA?e=KagzKc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ba6cdc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader, random_split\n",
    "from torchvision import transforms\n",
    "\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.callbacks import LearningRateMonitor\n",
    "from pytorch_lightning.loggers import CSVLogger\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set()\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "144cd0da",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SEED = 42\n",
    "\n",
    "DATA_PATH = \"/home/datascience/multi-input-data/\"\n",
    "\n",
    "# Seed everything\n",
    "pl.seed_everything(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f31df4ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>zpid</th>\n",
       "      <th>price</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>beds</th>\n",
       "      <th>baths</th>\n",
       "      <th>area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>29776969</td>\n",
       "      <td>525000.0</td>\n",
       "      <td>40.811995</td>\n",
       "      <td>-73.919353</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1904.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>29776973</td>\n",
       "      <td>565000.0</td>\n",
       "      <td>40.812076</td>\n",
       "      <td>-73.919541</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>29777045</td>\n",
       "      <td>805000.0</td>\n",
       "      <td>40.812729</td>\n",
       "      <td>-73.919133</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2280.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>29777854</td>\n",
       "      <td>435000.0</td>\n",
       "      <td>40.826804</td>\n",
       "      <td>-73.917024</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1728.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29777997</td>\n",
       "      <td>665000.0</td>\n",
       "      <td>40.833573</td>\n",
       "      <td>-73.912594</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2871.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       zpid     price   latitude  longitude  beds  baths    area\n",
       "0  29776969  525000.0  40.811995 -73.919353   4.0    3.0  1904.0\n",
       "1  29776973  565000.0  40.812076 -73.919541   6.0    3.0  1800.0\n",
       "2  29777045  805000.0  40.812729 -73.919133   5.0    3.0  2280.0\n",
       "3  29777854  435000.0  40.826804 -73.917024   3.0    2.0  1728.0\n",
       "4  29777997  665000.0  40.833573 -73.912594   6.0    3.0  2871.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# have a look at structured data\n",
    "# to have a look at structured data\n",
    "# now it is in tab_data.csv\n",
    "# remember tab_data.csv must be put in the folder DATA_PATH\n",
    "# \n",
    "data = pd.read_csv(DATA_PATH + \"tab_data.csv\")\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "633f7e7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>zpid</th>\n",
       "      <th>price</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>beds</th>\n",
       "      <th>baths</th>\n",
       "      <th>area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.840000e+03</td>\n",
       "      <td>2.840000e+03</td>\n",
       "      <td>2840.000000</td>\n",
       "      <td>2840.000000</td>\n",
       "      <td>2840.000000</td>\n",
       "      <td>2840.000000</td>\n",
       "      <td>2840.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.026418e+07</td>\n",
       "      <td>7.666985e+05</td>\n",
       "      <td>40.739368</td>\n",
       "      <td>-73.912034</td>\n",
       "      <td>3.672183</td>\n",
       "      <td>2.519877</td>\n",
       "      <td>1796.098592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>6.342499e+05</td>\n",
       "      <td>3.918023e+05</td>\n",
       "      <td>0.115101</td>\n",
       "      <td>0.070550</td>\n",
       "      <td>1.119459</td>\n",
       "      <td>0.644396</td>\n",
       "      <td>565.281025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>5.409160e+06</td>\n",
       "      <td>6.000000e+04</td>\n",
       "      <td>40.573728</td>\n",
       "      <td>-74.039917</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.500000</td>\n",
       "      <td>935.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.983074e+07</td>\n",
       "      <td>4.866252e+05</td>\n",
       "      <td>40.624906</td>\n",
       "      <td>-73.978018</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1364.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.059224e+07</td>\n",
       "      <td>6.250000e+05</td>\n",
       "      <td>40.685717</td>\n",
       "      <td>-73.902310</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1690.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.069626e+07</td>\n",
       "      <td>9.550000e+05</td>\n",
       "      <td>40.851511</td>\n",
       "      <td>-73.848839</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2080.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.074296e+07</td>\n",
       "      <td>2.430000e+06</td>\n",
       "      <td>40.911744</td>\n",
       "      <td>-73.782983</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3585.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               zpid         price     latitude    longitude         beds  \\\n",
       "count  2.840000e+03  2.840000e+03  2840.000000  2840.000000  2840.000000   \n",
       "mean   3.026418e+07  7.666985e+05    40.739368   -73.912034     3.672183   \n",
       "std    6.342499e+05  3.918023e+05     0.115101     0.070550     1.119459   \n",
       "min    5.409160e+06  6.000000e+04    40.573728   -74.039917     1.000000   \n",
       "25%    2.983074e+07  4.866252e+05    40.624906   -73.978018     3.000000   \n",
       "50%    3.059224e+07  6.250000e+05    40.685717   -73.902310     3.000000   \n",
       "75%    3.069626e+07  9.550000e+05    40.851511   -73.848839     4.000000   \n",
       "max    3.074296e+07  2.430000e+06    40.911744   -73.782983     7.000000   \n",
       "\n",
       "             baths         area  \n",
       "count  2840.000000  2840.000000  \n",
       "mean      2.519877  1796.098592  \n",
       "std       0.644396   565.281025  \n",
       "min       1.500000   935.000000  \n",
       "25%       2.000000  1364.000000  \n",
       "50%       2.000000  1690.000000  \n",
       "75%       3.000000  2080.000000  \n",
       "max       4.000000  3585.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compute some quick stats\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10a7b677",
   "metadata": {},
   "source": [
    "as we can see:\n",
    "* **mean** price is: 766698\n",
    "* **std** price is:  391802"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d049e1cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ImageDataset(Dataset):\n",
    "    \"\"\"Tabular and Image dataset.\"\"\"\n",
    "\n",
    "    def __init__(self, csv_file, image_dir):\n",
    "        self.csv_file = csv_file\n",
    "        self.image_dir = image_dir\n",
    "        \n",
    "        # removed pickle, now load from csv file\n",
    "        self.tabular = pd.read_csv(DATA_PATH + self.csv_file)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.tabular)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "        \n",
    "        # all the data for the sample of index idx\n",
    "        all_data = self.tabular.iloc[idx, 0:]\n",
    "\n",
    "        y = all_data[\"price\"]\n",
    "        \n",
    "        # int() to solve problems with data conversion\n",
    "        # get the file name\n",
    "        zpid = int(all_data['zpid'])\n",
    "        image = Image.open(f\"{self.image_dir}/{zpid}.png\")\n",
    "        image = np.array(image)\n",
    "        image = image[..., :3]\n",
    "        \n",
    "        # only transform applied: to_tensor\n",
    "        # since we're not using a pretrained net, we don't need to normalize\n",
    "        image = transforms.functional.to_tensor(image)\n",
    "        \n",
    "        # get all tabular data\n",
    "        tab_data = all_data[[\"latitude\", \"longitude\", \"beds\", \"baths\", \"area\"]]\n",
    "        tab_data = tab_data.tolist()\n",
    "        tab_data = torch.FloatTensor(tab_data)\n",
    "\n",
    "        return image, tab_data, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a98e0cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv_block(input_size, output_size):\n",
    "    block = nn.Sequential(\n",
    "        nn.Conv2d(input_size, output_size, (3, 3)), \n",
    "        nn.ReLU(), \n",
    "        nn.BatchNorm2d(output_size), \n",
    "        nn.MaxPool2d((2, 2)),\n",
    "    )\n",
    "\n",
    "    return block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d57648db",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Here is the NN\n",
    "#\n",
    "class LitClassifier(pl.LightningModule):\n",
    "    def __init__(\n",
    "        self, lr: float = 1e-3, num_workers: int = 4, batch_size: int = 32,\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.lr = lr\n",
    "        self.num_workers = num_workers\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "        # Loss used: MAE\n",
    "        self.criterion = torch.nn.L1Loss()\n",
    "\n",
    "        self.conv1 = conv_block(3, 16)\n",
    "        self.conv2 = conv_block(16, 32)\n",
    "        self.conv3 = conv_block(32, 64)\n",
    "\n",
    "        self.ln1 = nn.Linear(64 * 26 * 26, 16)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.batchnorm = nn.BatchNorm1d(16)\n",
    "        self.dropout = nn.Dropout2d(0.5)\n",
    "        self.ln2 = nn.Linear(16, 5)\n",
    "        \n",
    "        # for tab\n",
    "        self.ln4 = nn.Linear(5, 10)\n",
    "        self.ln5 = nn.Linear(10, 10)\n",
    "        self.ln6 = nn.Linear(10, 5)\n",
    "        \n",
    "        # final\n",
    "        self.ln7 = nn.Linear(10, 1)\n",
    "        \n",
    "    def forward(self, img, tab):\n",
    "        img = self.conv1(img)\n",
    "        img = self.conv2(img)\n",
    "        img = self.conv3(img)\n",
    "        img = img.reshape(img.shape[0], -1)\n",
    "        img = self.ln1(img)\n",
    "        img = self.relu(img)\n",
    "        img = self.batchnorm(img)\n",
    "        img = self.dropout(img)\n",
    "        img = self.ln2(img)\n",
    "        img = self.relu(img)\n",
    "        \n",
    "        # process tabular data: [\"latitude\", \"longitude\", \"beds\", \"baths\", \"area\"]\n",
    "        # so input is 5 values\n",
    "        tab = self.relu(self.ln4(tab))\n",
    "        tab = self.relu(self.ln5(tab))\n",
    "        tab = self.relu(self.ln6(tab))\n",
    "        \n",
    "        # 5 features for img and 5 for tab\n",
    "        x = torch.cat((img, tab), dim=1)\n",
    "        x = self.relu(x)\n",
    "\n",
    "        return self.ln7(x)\n",
    "    \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        image, tabular, y = batch\n",
    "\n",
    "        y_pred = torch.flatten(self(image, tabular))\n",
    "        y_pred = y_pred.double()\n",
    "\n",
    "        loss = self.criterion(y_pred, y)\n",
    "        \n",
    "        self.log(\"train_loss\", loss, on_epoch=True, prog_bar=True)\n",
    "        \n",
    "        return loss\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        image, tabular, y = batch\n",
    "\n",
    "        y_pred = torch.flatten(self(image, tabular))\n",
    "        y_pred = y_pred.double()\n",
    "\n",
    "        val_loss = self.criterion(y_pred, y)\n",
    "        \n",
    "        return {\"val_loss\": val_loss}\n",
    "\n",
    "    def validation_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n",
    "        \n",
    "        # moved here.. to remove spike due to step\n",
    "        self.log(\"val_loss\", avg_loss, on_epoch=True, prog_bar=True)\n",
    "        \n",
    "        return {\"val_loss\": avg_loss}\n",
    "        \n",
    "    def test_step(self, batch, batch_idx):\n",
    "        image, tabular, y = batch\n",
    "\n",
    "        y_pred = torch.flatten(self(image, tabular))\n",
    "        y_pred = y_pred.double()\n",
    "\n",
    "        test_loss = self.criterion(y_pred, y)\n",
    "\n",
    "        return test_loss\n",
    "\n",
    "    def test_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x[\"test_loss\"] for x in outputs]).mean()\n",
    "        logs = {\"test_loss\": avg_loss}\n",
    "        return {\"test_loss\": avg_loss, \"log\": logs, \"progress_bar\": logs}\n",
    "    \n",
    "    def setup(self, stage):\n",
    "        image_data = ImageDataset(csv_file=\"tab_data.csv\", image_dir=f\"{DATA_PATH}images/\")\n",
    "        \n",
    "        # split in train/validation\n",
    "        FRAC_TRAIN = 0.9\n",
    "        train_size = int(FRAC_TRAIN * len(image_data))\n",
    "        val_size = int((len(image_data) - train_size))\n",
    "        # test_size = int((len(image_data) - train_size) / 2)\n",
    "        \n",
    "        # here we split in train,val, test\n",
    "        self.train_set, self.val_set = random_split(image_data, (train_size, val_size), \n",
    "                                                   generator=torch.Generator().manual_seed(SEED))\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr)\n",
    "        \n",
    "        # I have added a LR scheduler... \n",
    "        # it is important in this case the threshold: base on the value of the prediction (z*10**5)\n",
    "        # with lower threshoòds doesn't trigger\n",
    "        scheduler = ReduceLROnPlateau(optimizer, patience=5, factor=0.5, threshold = 0.1, verbose=True)\n",
    "        \n",
    "        lr_scheduler = {\"scheduler\": scheduler, \"monitor\": \"val_loss\", \"interval\": \"epoch\"}\n",
    "        \n",
    "        return [optimizer], [lr_scheduler]\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        return DataLoader(self.train_set, batch_size=self.batch_size, num_workers=self.num_workers)\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        return DataLoader(self.val_set, batch_size=self.batch_size, num_workers=self.num_workers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06468758",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add as a callback if you want to monitor changes of LR\n",
    "# lr_monitor = LearningRateMonitor(logging_interval=\"epoch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4499ce22",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 30\n",
    "\n",
    "# modo corretto di settare il LR iniziale\n",
    "# con questo LR va in overfitting\n",
    "model = LitClassifier(lr=1e-1)\n",
    "\n",
    "\n",
    "trainer = pl.Trainer(max_epochs=EPOCHS, accelerator='gpu', devices=1,\n",
    "                    logger=CSVLogger(save_dir=\"logs_multi/\")) #, callbacks=[lr_monitor])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "06fdfcf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LitClassifier(\n",
       "  (criterion): L1Loss()\n",
       "  (conv1): Sequential(\n",
       "    (0): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (conv2): Sequential(\n",
       "    (0): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (conv3): Sequential(\n",
       "    (0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (ln1): Linear(in_features=43264, out_features=16, bias=True)\n",
       "  (relu): ReLU()\n",
       "  (batchnorm): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (dropout): Dropout2d(p=0.5, inplace=False)\n",
       "  (ln2): Linear(in_features=16, out_features=5, bias=True)\n",
       "  (ln4): Linear(in_features=5, out_features=10, bias=True)\n",
       "  (ln5): Linear(in_features=10, out_features=10, bias=True)\n",
       "  (ln6): Linear(in_features=10, out_features=5, bias=True)\n",
       "  (ln7): Linear(in_features=10, out_features=1, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bcd8b1cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to find a suggestion for the LR\n",
    "# lr_finder = trainer.tuner.lr_find(model)\n",
    "\n",
    "# fig = lr_finder.plot(suggest=True)\n",
    "# fig.show();\n",
    "\n",
    "# new_lr = lr_finder.suggestion()\n",
    "\n",
    "# print(new_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2458c521",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "   | Name      | Type        | Params\n",
      "-------------------------------------------\n",
      "0  | criterion | L1Loss      | 0     \n",
      "1  | conv1     | Sequential  | 480   \n",
      "2  | conv2     | Sequential  | 4.7 K \n",
      "3  | conv3     | Sequential  | 18.6 K\n",
      "4  | ln1       | Linear      | 692 K \n",
      "5  | relu      | ReLU        | 0     \n",
      "6  | batchnorm | BatchNorm1d | 32    \n",
      "7  | dropout   | Dropout2d   | 0     \n",
      "8  | ln2       | Linear      | 85    \n",
      "9  | ln4       | Linear      | 60    \n",
      "10 | ln5       | Linear      | 110   \n",
      "11 | ln6       | Linear      | 55    \n",
      "12 | ln7       | Linear      | 11    \n",
      "-------------------------------------------\n",
      "716 K     Trainable params\n",
      "0         Non-trainable params\n",
      "716 K     Total params\n",
      "2.866     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "925d1ecb2a084958862fe13faba465f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch     7: reducing learning rate of group 0 to 5.0000e-02.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    13: reducing learning rate of group 0 to 2.5000e-02.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    19: reducing learning rate of group 0 to 1.2500e-02.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    25: reducing learning rate of group 0 to 6.2500e-03.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=30` reached.\n"
     ]
    }
   ],
   "source": [
    "trainer.fit(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "073e66bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# trainer.test(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d25789fa",
   "metadata": {},
   "source": [
    "#### Plot training history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "734f42ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAFwCAYAAAA100uIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABINklEQVR4nO3dd5xdVbn/8c8+bVombTLpZULKIgmk0ntv0kEkAgGFe20X5Vouys92VbyoqKiAcFUUEZALkSZgAAEhdNILPOnJpE8mdTL1lN8fZycMYSZzZnLaTL7v1+u85sxauzxnQ84za+9VvEQigYiIiOxfINcBiIiIdAZKmCIiIilQwhQREUmBEqaIiEgKlDBFRERSoIQpIiKSAiVMkTzlnPu+c+4vuY5DRJKUMCVrnHOrnHNn5DoOEZGOUMIUERFJQSjXAYg45wqAnwBX+EX/B9xsZg3OuT7An4ATgDiwCDjZzOLOuZuBLwPdgfXAF83sn/sc+xjgCWCQmcX8skuA/zaz8c65o4C7gdFAHfCgmX21lTjPB34EVACLgc+b2Xy/bhVwL3ANMMA/5xfMrN6v/zfgZqA3MNPfd71fNw64A5gCNAG/MrMf+6eNOOf+DFwCrAGuNbP3/P3a/Pwikj5qYUo++H/AMcBEYAJwFPBtv+5rwFqgHOgH3AIknHMO+A/gSDMrBc4GVu17YDN7C9gNnNas+NPAQ/77X5FMUN2BESST9cc45yYD9wGfA8pIJsen/GS/x1V+HCNIJuBv+/ueBvwPyT8IBgCrgb/6daXAi8A/gIHASKB50rvQ37Yn8BRwp79fSp9fRNJHLUzJB1cBN5rZZgDn3H+TTEjfIdniGgAMM7NlwGv+NjGgABjrnKsys1X7Of7DwFTgBT9BnQd83a9rAkY65/qY2RbgrVaO8W/AvWb2tv/7/c65W0gm+n/5ZXeaWaUf363Ab0gmzauA+8xstl/3LWCbc64COBbYaGY/949RD+w5B8BMM3vW3+8B4Ca/vD2fX0TSQC1MyQcDSba69ljtlwH8DFgGPO+cW+Gc+yaAnzxvAr4PbHbO/dU5N5CWPQRc6rcGLwVmm9me811PsjX4gXPuXf+2a0uGAV9zzm3f8wKGNIsToLKVz/CRz2dmNUA1MMg/xvJWzgmwsdn7WqDQORdq5+cXkTRQwpR8sJ5kQtpjqF+Gme0ys6+Z2SHABcBXnXOn+3UPmdkJ/r4Jks9BP8bMFpNMWOfy0duxmNlSM5sK9PX3f8w5V9LCYSqBW82sZ7NXsZk93GybIS19hn0/n3/8MmCdf9wRrV+a1qX6+UUkPXRLVrIt7JwrbPZ7lOQt0287594l+cX/XeAvsLejzQckW2E7Sd6KjPnP8AYBr5O8jVnH/v8AfIhkB5ljSd4ixT/+1cAMM6vyW43459jX74DHnXMvAu8AxcApwKtmtsvf5kvOub+TbAneAjzS7Nx/dc49BLwP/Bh428xWOeeqgV84524CfgtEgLHNbv22qAOfX0QOkP6BSbY9S/LLfc/r+yR7nr4HzAcWALP9MoBRJDvF1ABvAneb2Sskn9/dBmwheduyL8kk1ZqHSSa4l/xnlXucAyxyztWQ7AB05Z6erc35PVP/jWSnm20kbxNft89mDwHPAyv814/8ff9J8nnsdGADyRbllX7dLuBMkq3njcBS4NT9fI492vv5ReQAeVpAWuTA+cNKbjCzF3Mdi4hkhlqYIiIiKVDCFBERSYFuyYqIiKRALcz2CZGcFk29i0VEDjL64m+fwcDK6uoa4vG2W+a9ehWzbVtt5qPqgHyNLV/jAsXWEfkaF3w0tvLyUi/H4UgnoBZmBoVCwVyH0Kp8jS1f4wLF1hH5Ghfkd2ySn5QwRUREUpC1W7LOuSeA4SSXaKoBbgS2k1wGaY+eQHcz6+3vs4rkLCZ7BpLfbGYz/LrRwP0kpxirBqaZ2dIDqRMREWlNNp9hXmtmOwCccxeRXL1hMsklnfDL72ghpsvNbGELx7sHuMvM/uJPb3YvHy7h1NE6ERGRFmUtYe5Jlr4eJFuaeznnIny4nuB+Oef6ApNJTikGyWnP7nTOlQNeR+rMrKojn0tERA4OWe0l65z7PXAWycR1zj7VFwLr9qwZ2MyDzjmP5Cr1t5jZdpKrQqwzsxiAmcWcc+v9cq+DdSknzLKybil/5vLy0pS3zbZ8jS1f4wLF1hH5Ghfkd2ySf7KaMM3sBgDn3DUk1zk8r1n1Z0muaN/ciWZW6a9jeAfJia+vzkKo+5XqsJLy8lKqqna1uV0u5Gts+RoXKLaOyNe44KOxKXFKKnLSS9bMHgBOdc6VAfgL354MPLjPdpX+zwbgbuB4v6oSGOScC/r7B0ku0lt5AHUiIiKtykrCdM51c84Nafb7BcBW/wXJZZKeMbPqZtuUOOd6+O89ksshzQUws83++6n+5lOBOWZW1dG6NH5cERHpgrJ1S7YEeNRfaT5GMlFeYGZ77mteR3Jx3+b6AdP9VmAQWAx8sVn954H7nXPfJbk+4bQ01ImIiLRIk6+3TwXtmBqvszy/ySf5Ghcoto7I17jgY88wNTWetEkz/YiIiKRACVNERCQFSpgZUvuPX7L56d/kOgwREUkTLe+VKY11RHfECOc6DhERSQu1MDMlXEi8oS7XUYiISJooYWaIFyki0aiEKSLSVShhZogXLiTekJ8rzYuISPspYWZKpIi4WpgiIl2GOv1kSMHE8xlwymVsq297WxERyX9qYWaIV9iNUGnvXIchIiJpooSZIbGNS9n0xC+J1+5oe2MREcl7SpgZEq/dzu5FM0nU78x1KCIikgZKmBniRYqSbxr1EFNEpCtQwswQL1wIQKJJPWVFRLoCJcxMCSdbmAm1MEVEugQlzAzxImphioh0JRqHmSFeYSnlF36Z2uLBuQ5FRETSQC3MDPFCEUoPP5lA9765DkVERNJACTODdi14hej6D3IdhoiIpIESZgZtfekvRJe9keswREQkDZQwMyhQUKResiIiXYQSZgYFIsUkmpQwRUS6AiXMDAoUFIGW+BIR6RKUMDPIKyjWOEwRkS5C4zAzqHjEZKLdBuQ6DBERSQMlzAzqPukMGqp25ToMERFJg6wlTOfcE8BwIA7UADea2Vzn3Cqg3n8B3GxmM/x9RgP3A2VANTDNzJZmqi7dYnW7iG2tJNBrMJ7nZeIUIiKSJdl8hnmtmU0ws0nA7cB9zeouN7OJ/mtGs/J7gLvMbDRwF3BvhuvSatecF6l97DsQa8rUKUREJEuy1sI0sx3Nfu1BsqXZKudcX2AycKZf9DBwp3OuHPDSXWdmVQfw8VoUKNizYkkdXiiS7sOLiEgWZfUZpnPu98BZJBPXOc2qHnTOecBM4BYz2w4MAdaZWQzAzGLOufV+uZeBupQTZllZt5S227UxmTB7lwYI9y5N9fBZU16efzFB/sYFiq0j8jUuyO/YJP9kNWGa2Q0AzrlrgJ8B5wEnmlmlc64AuAO4E7g6m3G1V3V1DfF4os3tiguKk9tvrCYYy69/mOXlpVTlYYekfI0LFFtH5Gtc8NHYlDglFTkZh2lmDwCnOufKzKzSL2sA7gaO9zerBAY554IA/s+Bfnkm6tJu7y1ZjcUUEen0spIwnXPdnHNDmv1+AbAVqHfO9fDLPOBKYC6AmW3230/1d5sKzDGzqkzUpfkjAxAs6k6g10DwND+EiEhnl61bsiXAo865EiBGMlleAPQDpvstvSCwGPhis/0+D9zvnPsusA2YluG6tIr0HUrJJ3+cqcOLiEgWeYlE28/iZK8KYGWqzzA7y/ObfJKvcYFi64h8jQs+9gxTA6WlTbpXmEHxaCM1f76RxnnP5ToUERE5QEqYGeQFwyQaakk07M51KCIicoCUMDPI8zyIFJLQEl8iIp2eEmaGeeFCDSsREekClDAzzIsUQVN92xuKiEheU8LMMC9cpFuyIiJdgNbDzLCic26CoC6ziEhnp2/yDPMKSnIdgoiIpIFuyWZY4/uvUPfSPbkOQ0REDpASZobFt28gump2rsMQEZEDpISZYV6kCKKNJOL7XS9bRETynBJmhnnh5BJfaCymiEinpoSZaZFCABIaiyki0qkpYWaYF/EXkdZYTBGRTk0JM8OC/UZRdPZXCHTrnetQRETkAGgcZoYFSnoRKOmV6zBEROQAqYWZYYmG3TTOf47Y1nW5DkVERA6AEmaGJZrqaXjrEWKbl+U6FBEROQBKmBnmhZO9ZGlUL1kRkc5MCTPT/HGYWhNTRKRzU8LMMC8QgFCBxmGKiHRySphZ4IULQeMwRUQ6NQ0ryYLwuNMJ9Oif6zBEROQAKGFmQcHkC3MdgoiIHCDdks2C+I5NxKrX5DoMERE5AEqYWdDw9iPUv/y7XIchIiIHIGu3ZJ1zTwDDgThQA9wIVAIPACOABmAZ8Dkzq/L3WQXU+y+Am81shl83GrgfKAOqgWlmtvRA6jImXKRhJSIinVw2W5jXmtkEM5sE3A7cBySAn5qZM7PxwHLgtn32u9zMJvqvGc3K7wHuMrPRwF3AvWmoywgvXKjVSkREOrmstTDNbEezX3sAcTPbCrzSrPwt4AttHcs51xeYDJzpFz0M3OmcKwe8jtTtadVmghcpgsZ6EokEnudl6jQiIpJBWe0l65z7PXAWycR1zj51AZLJ8ql9dnvQOecBM4FbzGw7MARYZ2YxADOLOefW++VeB+tSTphlZd1S/szl5aVs79WDrYkYfXoXEghFUt4308rLS3MdQovyNS5QbB2Rr3FBfscm+SerCdPMbgBwzl0D/Aw4r1n1b0g+27yzWdmJZlbpnCsA7vDrrs5OtK2rrq4hHk+0uV15eSlVVbtoCnQn0G8kWzZuxSsoyUKEbdsTW77J17hAsXVEvsYFH41NiVNSkZNesmb2AHCqc64MwDl3OzAK+JSZxZttV+n/bADuBo73qyqBQc65oL9/EBjol3e0LmPCI46m5KJv502yFBGR9stKwnTOdXPODWn2+wXAVmCrc+5WYApwsZ8Y92xT4pzr4b/3gCuBuQBmttl/P9XffCowx8yqOlqX7s+8r0QiQSLRdqtURETyU7ZuyZYAjzrnSoAYyWR5ATAWuAVYArzhnANYaWaXAP2A6X4rMAgsBr7Y7JifB+53zn0X2AZMS0NdRkQ3GHXP/JSi875OaOCYTJ9OREQyICsJ08w2Ace0Ut1it1EzWwFM2s8xPwCOTmddpnihAojHtCamiEgnppl+ssCLJBeR1uQFIiKdlxJmNoT3JEy1MEVEOislzCzwIkUAmu1HRKQTU8LMhmAEvACohSki0mlpPcws8DyPbp+5B4LhXIciIiIdpISZJV4eTYknIiLtp1uyWVI/8880vPNYrsMQEZEOUgszS+LVlRDSLVkRkc5KLcxsiWhNTBGRzkwJM0u8cBEoYYqIdFpKmFniRQo1cYGISCemhJkt4SLdkhUR6cTU6SdLImNOJVQxOddhiIhIBylhZkmgZ38CPfvnOgwREekg3ZLNklh1JQ3v/Y1EfU2uQxERkQ5QwsyS+PYNNM5+injtjlyHIiIiHaCEmSV71sREa2KKiHRKSpjZEtYSXyIinZkSZpbsaWEm1MIUEemUlDCzxPNbmDRq8gIRkc5ICTNLvMJSIpMvIlA2NNehiIhIB2gcZpZ44QIKjrgk12GIiEgHqYWZRdGNS4ltW5frMEREpAOUMLOo/oXf0LTg+VyHISIiHaCEmU2RIq1YIiLSSWXtGaZz7glgOBAHaoAbzWyuc240cD9QBlQD08xsqb9PVusyzdOKJSIinVY2W5jXmtkEM5sE3A7c55ffA9xlZqOBu4B7m+2T7bqM8iJFoBamiEinlLUWppk1n0S1BxB3zvUFJgNn+uUPA3c658oBL5t1ZlaVzs/bEi9cSLxmS6ZPIyIiGZDVZ5jOud8759YAtwLXAkOAdWYWA/B/rvfLs12XcYHy4QTKKrJxKhERSbOsjsM0sxsAnHPXAD8DvpPN86dLWVm3lLctLy/98Jezr8pANB33kdjySL7GBYqtI/I1Lsjv2CT/eIlEIicnds7VARWAAWVmFnPOBUl2xBlF8vbpkmzVpXhLtgJYWV1dQzze9nUrLy+lqmrX3t8TiQTEmvBCkRROlVn7xpYv8jUuUGwdka9xwUdjKy8v9XIcjnQCWbkl65zr5pwb0uz3C4CtwGZgLjDVr5oKzDGzKjPLal1aP3ArGuc9Q819/04i1pSN04mISBpl65ZsCfCoc64EiJFMlheYWcI593ngfufcd4FtwLRm+2W7LqO8sL9iSWMdXlE4W6cVEZE0yErCNLNNwDGt1H0AHJ0PdZm2d8WSpnoo6p6LEEREpIM00082RbSItIhIZ6WEmUXNb8mKiEjnooSZRV6kCAJBiDXmOhQREWknrYeZRYE+FZTe8IdchyEiIh2gFmYWeZ6GeomIdFZKmFmUiEXZ/eSPaPrg1VyHIiIi7aSEmU2BIPHNy4nvyso8CSIikkZKmFnkeR5oTUwRkU5JCTPLvEgRiSYlTBGRzkYJM8u8cCE0ahFpEZHORgkz2yJFJJqUMEVEOhuNw8yywhOvwwvqsouIdDb65s6yYO/BuQ5BREQ6QLdks6xp1Swa3p2e6zBERKSdlDCzLLZhCY0LX8h1GCIi0k5KmFnmhQuhqZ5EIp7rUEREpB2UMLPMiySX+KKpIbeBiIhIuyhhZltYi0iLiHRGSphZ5kX8hKnZfkREOhUlzCwL9hlGwTFX4hWW5joUERFpB43DzLJAj/5Exp+T6zBERKSd1MLMskRTPdFVc4jXVOc6FBERaQclzCxL1O+i7vlfEVu3ONehiIhIOyhhZpmnXrIiIp2SEma2+eMw1UtWRKRzUcLMMi8QgmBELUwRkU4mK71knXNlwAPACKABWAZ8DigBnmi2aU+gu5n19vdbBdT7L4CbzWyGXzcauB8oA6qBaWa29EDqssWLaBFpEZHOJlstzATwUzNzZjYeWA7cZmarzGzinhfJ5PnQPvte3mybGc3K7wHuMrPRwF3AvWmoy4rQ0IkEeg/K9mlFROQAZKWFaWZbgVeaFb0FfKH5Ns65CHAVcHZbx3PO9QUmA2f6RQ8DdzrnygGvI3VmVtX+T9a6+sYoyyq3s6lqF43RGE1NcRqj8eT70rOI1iUosyoGl5dQ3rOIQMBL5+lFRCTNsj5xgXMuQDJZPrVP1YXAOjObvU/5g845D5gJ3GJm24Eh/rYxADOLOefW++VeB+tSTphlZd3a3OaHf3ibdxZvbLEuTJSwF6U2kewAFAkFGNK/lGH9uzOsfylD+3dnxKAe9OpemGpIHVJenp+zDeVrXKDYOiJf44L8jk3yTy5m+vkNUAPcuU/5Z4H79ik70cwqnXMFwB3+PldnPMI2VFfXEI8n9rvNJ08+hLOOHkpdbSPhUICCcJBwKEAkFCDyxj14u6vZcsJ/sa5qN+u21LC2ajezP9jES+9VAhAMeHzx4sOYNLo8I5+hvLyUqqpdGTn2gcjXuECxdUS+xgUfjU2JU1KRUsJ0zv0C+LOZzT2QkznnbgdGAReYWbxZ+UDgZOCa5tubWaX/s8E5dzcftkorgUHOuaDfSgwCA/1yr4N1aVXWo5BDR7b8ZVFXWExsx3qGD+jO8AHdP1JXU9fE+i27eeSlZfz2yUV89YoJHDqsV7rDExGRdkq1008YmOGcW+icu9k5N7i9J3LO3QpMAS42s30Xg7wOeMbMqpttX+Kc6+G/94ArgbkAZrbZfz/V33wqMMfMqjpa197PcyC8SBE0tdxLtltRmNFDevKfV0ygvGchv54+n9Ub8/MvdBGRg0lKCdPMbiTZEvsmMBF43zn3onNumnOuzQd6zrlxwC3+Md5wzs11zj3ebJPr+Pjt2H7AK865+cBCYDTwxWb1nwdudM4tAW70fz/QuqzwwoVtjsPsVhTma5+aSElhiF/831w2bq3NUnQiItISL5HY/7O4lvgJ8CHgcKAW+CvwPTNbl97w8k4FsDKVZ5jQ+vObhjlP0/judLpd/zu8YHi/x9i4tZb/+cssIqEA37p6Cr3T1BEoX58t5WtcoNg6Il/jgo89w+wS3dSdc68At5vZ31uprwDeM7M+2Yyrq0i5049zrjvwSZKdbsYD00m2+NYAXwP+QTKBShu8gm7J9TCbGqCNhNm/dzFfvWIiP3loNj9/ZC7funoK3Yr2v4+IHLgLvvbkNJKdETPhvqd/ftGfM3RsyZBUO/08RnJ85KskB/4/0fw5pHPuq0BjRiLsgiJjTyUy9tSUtx/Wv5SvXD6enz8yj1/+3zy+MXUihREtZSrSlTnnvgP0NrP/9H8vA5YA04BvA4Ukv8NvNbO/dvAc5wD/AwRJDq37nJktc8454E9AsV/3JzO73Tl3EfAjIOaf+z/M7JUOf8hOJtVv3bdIXpgWBxaaWdzvmCMZ4ob24gsXj+Ouvy3krr8t4MuXTyAc0lTAXV0sHudnD80hGk9w2UmHMKaid65DOmj4LcBctgLvB952zn3DzKLAp4EngTeAE/ye/v2AWc65GWa2rT0H9yeAeQA42cwWO+euBx4EjiZ59/BZM/uhv+2ervo/AL5oZq/5owxK0vA5O41UO/3c3lqybGZ3GuI5KMSqVrL70VuIbV7erv0mjSrnM+cdyqJV27jnyYXUN0bTHltdQ5S1VTVpP650zD9nrWPJ2h1s3lbHz/46l188Mpc1m3L/TLCuIcpTM1fytbte5+3Fm3IdTpdkZmuAxcB5ftF1wB+BcuAx59xCYAbQG3AdOMXRwDwz27M47x+Bic65UpJ3Ez/rnPuhc+40YLu/zUvAz51z3wDGmNnODpy301ITJRcSCeLb1pOoa/8X3/GHD+CqM0czd9kWfnj/e6zfkp6/U3bVNvL4qyv4r9++wff+8A7rMpQ0O9LJ7GC1dWc9j7+2gvEjyvj5l47jilNHsnLDTr7/x3f536cXUbU9+yveNEXjvPheJd+8902emLmSeCLBH55ZjK1pV+MmJbF4nN31TWzZUUfl5hqWVG5n7rItvLVoIy/PXssL71bm5Bpk2Z+Aa51zhwE9zOw14Lckpxo93J+Dey3J27Pt5ZGc5/tjzGw6cDzJeb+/SbIlin97+HqSj+Aedc79WwfO22npQVgOeJHkItLxmmoSiTie176/W06fMpgBZcXc+9Qifnj/e3zmvEM5aky/DsWydWc9/3hnDa/OW09jU5xJo/qwYMVWXpqzjmvO6sgfra1bv2U3tz04m9OnDOaC4ysIeLqLvz8P/3Mp8XiCT585mnAoyDlHD+WkCQN47u01vPBuJe++v5lTJw/i/OMqyMx8UB+KJxK8vWgTj7+2gi076jl0aE8uP2Uk/XoX8eMHZnHn3xZwyzVTGFCW2h26eCLBM2+uZu7SLTRFYzRF4zTF4jQ2JX9Go3FiKfRE/+tLS5k0qpwzjxjM6CE98bre/1PTgV8AXyeZPCG5qtMqM0s4584ERnbw2G8Cf3DOHWpmHwDXkhyXvss5NxJYYWZ/cs4tJdn6xDnnzGwBsMAfUngk8LsOnr/TUcLMAa+wFPBoeP0BGt5+hKLTv0ho2ERiW9eR2FVFoNcgvNKy/SbSsRW9+f5njuK3TyzknicXsWztDq44bSShYGrJd+3mXTz47Pu8uSh5p/2Ysf0495hhDOxTwu//vpg3Fm7k8pNHUFSQvv9FXp6zjpq6Jp6cuZLVG3dxw/ljKS5M7fjRWJx/zV1Pj5IIRxzaN20x5av5y7cwy6q49KRD6NuzaG95cWGYy04ewWmTB/PkzJX8c9ZaZs7fwE1TJzN6QPqnd0skEixYUc1jr6xgbVUNQ/t146vnTGBcRe+9yekrn5zArX9+jzsencf/m3YE3Ysj+z1mUzTOH55ZzDvvb2bEoO6U9ywiHAr4U0cG974PBwMURoIUFYT2eQUpLgjRFI3zr3nr+dfc9cxeUsXQvt0444ghHD22L+FQMO3XIhfMrNY59yTwGWC4X/xN4G7n3DeB+f6rI8eucs5dAzzknAuR7PSzZ+rRK4CrnHONJFuhX/HLb3POjQKiJG/TXt+Rc3dWHRqH2RLn3C4z6+oTMlaQhnGYALGtlcQ2ryC+dR2RsacR6NmfhncepXHuM8kNwkWEhk4gNHwKoSHj8cIFLR4nGovz6MvLeeG9SkYM6s4XLjqsxbGaiUSCjVtrWVK5nfnLq5m7bAvhYIATJwzk7KOG0KfHh1/Ky9fv4NY/z+Kas0Zz6uR2T+rUoqZojK/e+Trjhvdm5KAePPLSMvr0LOI/Lj2cQX0+bJW0dM2WVG7ngeeNdVW7KS0O88v/OCHrq7vMsir6lJUwrE9xxs/V0BTjO79/m3AowH9/9qj9/hG0oXo39z37Pis37OLfLxjb4TsNALvrm9i0tY5NW2vZuLWWTdtq/bmOd9O3ZxGXnHQIR47p2+KdgeXrd/DTh+YwtG83vjF1EpFwMmHt+9+zpq6JO6fPZ8naHVx+ygjOPXroAbcKG5tivLV4Ey+8W8m6LbvpXhzmlEmDOHXSIHp0a/nfzb6xdZVxmJJZ6UyYQ/bM/dqFVZCmhNmSRGMtsa3riG9bR3zzcqKr5pBoqKHgxOuIjDmFeO0OvFAYL/LxL+133t/EH5/7gHAwwOcuGseYob32PvdZsnY7Syq3s6u2CYAeJRHOPraC48b2bbE1kEgk+MGf3iMaj/ODzx6Vlttcby3eyP8+tZivXzmRsRW9WVK5nbsfX0BDNM71543Z22psfs121Tby6MvLmblgA2XdCxg/og8vz1nHt66ezKjBPQ84plT9c9ZaHnxhCUDavuT3Z/q/lvPMm6u5+dOTcEPbnke4vjHKXU8sYvHKam74xFiOPax/SueJJxK8PHsdby/exMattdTUNe2t8zzo06OQfr2KmTS6nBPHD2jz7sUs28zdjy9ksivnCxcfRsDzPvLfc/P2On75f/Oo3lHH9Z8Yy9FjO57cW5JIJHh/9TZeeLeSecurKe9ZyG2fO7bV/1ZKmNJeaUuYB4kKMpgw95WIx4htXEKw9xC8wm7Uv/EgTYtfJjhoLOFRxxEacdRHbttuqN7NXY8vZMOW3RQWBKlriAHJL77RQ3oyekhP3JCe9O1VRN++3fcb26vz1vOn5z7gm1dNZvSQnh3+DHv87OE5VG2v47bPH7u3hbJtVwN3Pb6AFet38oljh3HJiYfQr193Nm3eycz5G3j05WXUN8Y466ghXHjccGLxBF/59WucdeQQPnlqRx/btM8L71by8D+XMnFkH7p3K+DVues4ZeJArjprNMFA+vvMrd+ym+/d9w5Hj+3HDeePTXm/0u5FfOee17E12/nMeWM4YfyA/W6/bVcD9z2zmEWrtlHRv5Rh/Uvp16uYfr2L6N+7mD49ijo0bOn5d9bw15eWcc5RQ7nitJF7/w0sX7+DXz82n3g8wY2XjU/L/1P7s2lrLTt2N+73PF05YTrn7gGO2ac4amZH5CKerkLPMPOYFwgSGjhm7+/hUcdDIEh05XvUv3QPgfnPUXDMVEIDDwVgQFkJ3542hSdnrqS+MYbzk2RHptM7emw/HnlpGS/NXnvAX26bt9fx/uptXHLi8I/czutVWsDNn57Mgy8s4Zk3V7N64y4+fe4Y/vDkApav28noIT255qzRDCr/cLpiN7Qnc5dtyUrCnPHOGh55aRmTR5fz+YvG0a9vd0qLQjzz5mq27KznCxcdltIz3qZojPVbahnSt9t+byUnEgkemGEURoJc0c7PV1gQ4iufnMCd0+fzx2ffJ55IcNKEgS1uO8s286fnPqApFmfaOY6TJwxMW4v5zCOHULU92ZGsT89CPnX2GGZZFf/79CJ6dotw0ycnpNwx6ED0611Mv96Zv32er8ws63NkHwyUMDuRYHkFwfIKEkdfQXTZWzS88xh1z/6Mkk//nEBxTwAKIyE+ddqoAz5XQTjICYcP4KXZa9lR07DfZ0FtmTl/PZ6XHBKzr3AowHXnHkrFgFIefH4Jt9z9Ot2Kwlz/iTEcd1j/j32RTxzZh4deXMrGrbX0z+AX4nNvr+bRl5dzhCvn3y8cRygYIBDwuOzkEZT3LOLP/zBue3A2N31yAr1KW742O2sbeWX2Ol6avZadtU0MKCvmguMqOGpMvxYT5xsLN2KV27n2HEf3kv13nGlJQTjIly8fz2/+toA/PfcBsXiCUycN2ltf1xDl4ReXMnPBBir6l/LvF45L+zX0PI+pZ4yiemc9D76whKqdDTz/1mqGD+zOly8b36HPJZIvlDA7Ic8LJG/JDj+C2MalBIp7kohFaZz7dyLjzsArbHMBmZScOnkQL7xXyavzN3DBcRUdOkYsHuf1BRs5/JCy/bZ0T5k4iKF9S1m+cRfHjunb6ny5E0clE+bcpVs45+ihHYqpLc+8uYrp/1rBUWP6csP5Yz/27O6kCQPpXVrA3U8s5Ed/fo+bPjmBIX0/vObrtuzmhXcreWPhRqKxOONHlHH4IWW8Mmcd//v0Yp56fVUycY7tu/e2bk1dE4+8tIwRg7pzYistw1SEQ0FuvHQ8dz++gAdmGPF4gtOnDGbZuh387ulFbNlRz/nHDePC44en3KO6vQIBj3+/cCw/eXAOM95azZTR5fzbBWP3dgQS6az0DLN9KsjiM8z2iK5bTN2zP4NwIQWTLiQ87nS8UOt/zaca28//Oof11bX89AvHduiZ3bxlW/jVY/P50iWHM8W1PVowlbi+d987FEWCfPPqKe2Opy1Pv76Sx19byTFj+3H9+WM+8pn3jW3Npl386rH51DVE+eLFh+F5HjPeXcPCFVsJhwIcf1h/zjxyyN5bkPFEgtlWxVOvr2JtVQ39ehVx/nEVHDOuHw/MWMLM+Rv43meO/EjyTdW+sUVjcX77xELmLN3CpFF9mLesmt7dC7jh/LEZf364x67aRtZtrWf0wNKs92pORVd+himZoYTZPhXkacIEiG1dR8PbjxCrnA+hCMGBY4lMOJfQgI9PQJBqbLOXVHHn3xbwH5cezuTR7R8ef+ffFrBs7XZu/9LxKbVoUonr8VdX8Pc3V3HHjSdQ2saYv+Y2VO8mnoBIKEAkFCAcChIJBwgGPDzP48mZK3ly5kqOHdef6z8x5mNf8i3Ftm1XA3c8Oo/KzcmZkbqXRDh98iBOmTSo1djiiQRzlmzh6ddXsmZzDX16FLJlRz1nHzWkw7fTW4otGotz71OLmGVVHHdYfz59xuiUx72mi5b3kq5Et2S7kGDvQRSf+1Wi6z8guuJdopXzwF+oOrpmHrENRnDoBIL9RqR8zAkjy+hVWsDLs9e2O2HuqGlg3rItnHnkkLTe/ps0ug9Pv7GK+curW3wu2pK5y7bw68daHt/teRAJBWloinH84f35zLkfT5at6VVawDevmsyTM1cyqLyEY8b2b7N3acDzmOLKmTy6D3OXbeGp11cRDHhcdMLw/e7XXqFggC9cdBibttVmpaONpJ9z7vvAj82sXatBOeeOAP7TzK7q4Hn/RHLdzDs7sn++cs4lgFIz69Dcn0qYXVBo4KGEBh5KInE1e6aKjG1ZReP8GTDvWYgUU3jaVSSGHNdm78hgIMApEwfy+Gsr293R5o2FG4nFE5zYxhCH9hrWr5Se3SLMXbol5YT57Jur6dOjkMtOHkGjPxVbY1OcpmiMxmicpmic3t0LOeOIwe2esq+oIMSVp7e/Zeh5HpNGlTNpVDmJRCIjYzsDAU/JsnP7HnA7+yyf6JwL+SuYtMjM3gM6lCyldUqYXVjyCzj5JVww+SIih51FdN0imha/zJZ//I5QxWwKT7lh79y2rTlpwkCeen0Vr8xZl3JiSCQSvDp/A6MG90j7F7bneUwcVc6bCzfSFI21OQ3asrU7WLZuB58+Y1TaB8unSxecA7VLWHHrZa+0VH7I/5t+il9/BzCxhU1uOuT/TZ+74tbLriO5ykiL+++Pc+4u/+0bzrk4sAqoBEaRXLFkinPuQZIrlRQAy4DPmtk259wpwO1mdoRzrgJ4D7iX5MonxcD1ZjazrRj8OLoBvyE5byzAA2b2E7/ue8BUoJ7kX+enkkzu9wPjgCbAzOyK/Rx/gH/8oUAR8LCZ/divWwU8THIi+IHAHXtavc65I4Ffk1xibDfwZTN71687H/g+EAbiwLVmtucW05edc5cAZcA3/InmU6LVSg4iXqSI8PAjKDrva/Q+fRrx3Vsh0HbPxR7dCpjiypk5fwMNTbGUzrV07Q42ba3lxPEd7/G5PxNH9qGhKcb7q7e3ue1zb6+mpDCUsVhEMsHMvuS/Pc5flWQ7cCxwmZnt6fH2FTM7wswOBxYBN7dyuDLgTTObRHJNy5+0I5TvkMwVhwPHAdOcc+f6a2R+HZjkx3cSUAOcDfQys7FmNgH4XBvH/zPwazM7CpgCnOtPKr9HPzM7iWTSvMU5N945FyE5Mf13zGw8yQW1pzvnIs650cDvgan++Y8BVjY73k4zOxK4hmTCTZlamAchzwvQ85iLaBx+Ml4gSHz7BppWzSIy/jy8VnrCnjZ5MO+8v5l3Fm9KadjDa/PWUxgJcmSGJkofM6wXBZEgc5dWMX5EWavbbajezdylWzj/uAoKIhrWIO3TVkvwkP83/aY26v/Eh6uMpMNjZtZ8Tb9pzrmrgAjJltaSVvarMbO/++/fAn7ejnOeQTIxJ4CdzrmH/bLnAQP+4pz7B/B3f6WTecChfgv5FeCZ1g7snCsBTgHKndvbObEUGAO84P/+BwAz2+Sce8bfPgE0mtmLft0//YniHcnE/ayZLfXrGoCGZqf9a7PrMNA5V2hm9alcCLUwD2Ke37psWvoGje88Rt1ztxOv3d7itqMG92BQeQkvzV7X5pqWtfVR3v1gM8eM7ZexJBUOBThseG/mLttCfD/xzHinklAowOlT0jOJvEiO7e2s4pw7EfgCcI7fwvw2ra+L2TxhxGhfY6mldTMTZhYj2Xr7NTAYmOWcG29mK/gw4Z0BzHPOtRZXwD/2kWY20X+NMLPWWn57YmltLc89dftTD+DHD+24FkqYQuSISyk46TPENi6jdvp3iVYu+Ng2nudx2qRBrN60ixUb9r/I+jvvb6IxGj+gAfipmDiyD9trGlm9seVhCztqGnhj4QaOP3yAZpiRzmoX0KOVup7ADqDaOVcAfDZDMbwA3OCc85xzpcCVwIv++3Iz+5eZfQ9YCBzmnBsMxMzsCeA/ST5v7d3Sgc1sF/AaySXLgORCHs655isIXOeXlwPnkmy1fgAUOOdO9etOJfm8cgkwAzjPX4YM51yBH+sBU8IUPM8jcujJFF/yPbzCUuqe+zkN7zz2se2OGdefwkiQ6a8sZ/n6Ha227F6dt57B5d2o6J/Z1d4mjOyD58HcpVtarH9x1lpisQRnHzUko3GIZNDPgZecc3NJJsjmngOWk0wezwGzMxTDD0m22haQXHT6ATP7B8lE/oRzbr5zbiGwEfgbyWedb/q3Zt8B/sfM1u/n+FcBY51zC5xzC4BH+OhnXeOce80/9/+Y2QJ/mM1lwI+dc/OBHwOXm1mjfyv234BH/BjeJDmG/oBp4oL2qSCPJy5oj9ZiS0QbaXjrEQK9BxEZe9rH6v/x9hqm/2s5sXiCXqUFTB5VzmRXzughPQgGAqzZtIvv//Fdpp4xijOPaH+iau81u+3B2dTWR/nB9Ud9pLyuIco37n6DMRW9+NIlh7c7jnTElk35Glu+xgWauKAz8HvJnm9mC3MdC6jTj+zDC0UoPOGavb83vDsdr7gn4bGn4Xke5xw9lJMmDGDesmpmLanitfnr+efstXQrCjNxVB9qapsIBQMcOy61NRkP1MSRffi/l5exZXsdfXp+ODzmtfkbqG2IZmy+WRE5+ChhSqsS8Tix6jXE5jxNtHI+hSdfT6CoO8WFYY49rD/HHtafhsYYC1dWM8uqmGWbqWuIcfTYfq1Onp5uk0YlE+bcZVs4w2/RRmNxnn93DW5IT0YMbO3xj8jBzTk3kZZ78N5pZr9P87nOI3nbdF+3mNmzre1nZhXpjONAZSVhOufKgAeAESR7ay0DPmdmVX6Tu95/AdxsZjP8/UaTHABbBlQD0/Z0Fc5EnXyUFwhQdPZNNC16kYa3H6H2sW9TePINhIaO37tNQSTIFNeXKa4vTdE4y9bt6NDk4R3Vr3cxA8qKmbP0w4T57geb2bqzgWvO+vgcuiKSZGZzaXnShUyc61mg1cTYWWSr008C+KmZOX+Q6XLgtmb1lzfrUjyjWfk9wF1mNhq4i+RMFZmsk314nkfksDMpvuT7eEXdqfvHL2ha/naL24ZDAcYM65W11uUeE0f1YUnldmrrm0gkEjz31hoG9Snh8P2MzxQRaa+sJEwz22pmrzQregsYtr99nHN9gckkp0XC/znZOVeeiboD+oAHgWDvwRRf/F3C488l6K9+El27iPjOqhxHBpNGlhOLJ1iwYiuLVm1lbVUNZx81tN1zwoqI7E/Wn2E65wIkB9s+1az4QeecB8wkeU97OzAEWLdncKmZxZxz6/1yLwN1KX/zl5WlfsuxvDyzQysORIdiu+AGABKJOJX/dz/RHVvodvgp9Dr+UsK90zPJenvj6l3WjR5PLOD9yu3sqGmgd/dCzj95ZJurhmQjtmzK19jyNS7I79gk/+Si089vSM5WsWfZmBPNrNIfeHuHX351DuJKWVceVtIeBZ/4Ft68Z6lZ9Ao1C14mNOIYIpMvINiz4xMWdDSuww8p4435G4jG4nzy1BFs37a77Z2yFFs25Gts+RoXfGxYSY6jkc4gqxMXOOduJznT/qfMLA5gZpX+zwbgbpIT7EJyVv5Bzrmgv2+Q5Gz1lRmqk3YKlPSi8LirKJn6M8KHn0101Swa/vXHnMQyaWQforE4RQVBTp4wKCcxiEjXlrWE6Zy7leRM9Bf7yRHnXIlzrof/3iM55dJcADPb7L+f6h9iKjDHzKoyUZeBj3zQCBT3pPCYKymZejsFJ30GgERjbZtzzqbT2OG9KSkMcfqUwRQXarSUiKRftoaVjANuITnP3xv+rPQrga+RXJIlCASBxcAXm+36eeB+59x3gW3AtAzXyQEIFHWHou4kGmupfeJHBAePo+DYT2dlrceCcJCffP5YCiNKliKSGVn5djGzRbQ+g/yk/ez3AXB0tuokTcJFBAcfRtPC5yEep+D4q7OSNIsLszucRUQOLvpzXNLO8zwKjp0KgQBN8/8BiTgFJ1yD52mufxHpvJQwJSM8z6Pg6E/hBYI0zn0GEjEKTrxOSVNEOi0lTMkYz/OIHHk5eAESTQ20va6riEj+UsKUjPI8j4IjLyORSOB5HrGqlQR6DsALt7YAu4hIflLClKzwPI/4zs3UPnkrBEOEhh9J2J1AsP/orHQIEhE5UEqYkjWB7n0pPv9mmuw1mla8Q3TJa3il5UTGnUZk/Lm5Dk9EZL+UMCWrgv1HEew/ioLjriK6ahZNS2YS37YBgES0gZr3F0Kfw3IcpYjIxylhSk544QLCo44jPOo4EvE4ANHV89j8z7spOGEakbGn5ThCEZGPUh9/yTkvkPzfMDR8CkUjJtPw+gNE18zLcVQiIh+lhCl5wwsE6XfpVwn0Hkrdi3cT27I61yGJiOylhCl5JRApouicm/AKSqj7xy+J11TnOiQREUAJU/JQoKQXRef+J4E+w/BCBbkOR0QEUKcfyVPB3kMoPuc/AYjX7sArLMEL6H9XEckdtTAlryWXCvsBDa/9Oavra4qI7EsJU/KaFykmPOo4muxVGuf+PdfhiMhBTPe4JO9FjriU+K4qGt+djldYSvjQkzWdnohknVqYkvc8z6Pw5OsJDhxDw2t/ou7Z23V7VkSyTi1M6RS8YJii875Ok70GTQ14nkci2kCidgeB7n1zHZ6IHASUMKXT8AJBImNO2ft744IXaJz1OOFxZ1Aw6QK8wm65C05EujwlTOm0wqOPJ7FzE00LnqdpyUwKJl1IeNzpeEH9by0i6adnmNJpBUp6UXjy9RRf9gOCfSpoeOth6l/9Y67DEpEuSglTOr1g2RCKP/ENIpMuACARj+U4IhHpinTvSrqMyBGXariJiGSMWpjSZexJlk2rZlH3/K/3rrMpIpIOSpjS9TTWE101m8Z5z+Y6EhHpQpQwpcsJjTqO0CFH0fje48Q2r8h1OCLSRWTlGaZzrgx4ABgBNADLgM8B8ZbKzazK328VUO+/AG42sxl+3WjgfqAMqAammdnSA6mTrsHzPApPvJbdm5ZR9/K9lFz633jhwlyHJSKdXLZamAngp2bmzGw8sBy4bT/lzV1uZhP914xm5fcAd5nZaOAu4N401EkX4RWUUHjqv5PYsZmGNx/OdTgi0gVkpYVpZluBV5oVvQV8obXyto7nnOsLTAbO9IseBu50zpUDXkfq9rRqpesIDTyUgmOnEuw3ItehiEgXkPVhJc65AMmk+FQq5cCDzjkPmAncYmbbgSHAOjOLAZhZzDm33i/3OliXcsIsK0t9Crby8tKUt822fI0trXGddhmQHJsZb6wnWFhyQIfL12sG+RtbvsYF+R2b5J9cjMP8DVAD3JlC+YlmVumcKwDu8OuuzkaQ+1NdXUM83vZqGeXlpVRV7cpCRO2Xr7FlIq5EIkHdjDsg1kTReV/H8zr2JCJfrxnkb2z5Ghd8NDYlTklFVnvJOuduB0YBnzKzeFvlZlbp/2wA7gaO96sqgUHOuaC/fxAY6Jd3tE66KM/zCA2bRGzdYpo+eDXX4YhIJ5W1hOmcuxWYAlzsJ8C2ykuccz389x5wJTAXwMw2+++n+ptPBeaYWVVH69L9eSW/hA89mWD/0TS+O51Ew+5chyMinVC2hpWMA24BlgBvOOcAVgLfbqnczC4B+gHT/VZgEFgMfLHZYT8P3O+c+y6wDZiWhjrpojzPo+C4q6h9/Ps0zHqCwuOuynVIItLJZKuX7CKSHW5a0mK5ma0AJu3nmB8AR6ezTrq2YJ9hhMecStOifxIecyrBXgNzHZKIdCKafF0OKgVHXEqg10ACPfrlOhQR6WQ0NZ4cVLzCbkTGnYEXCJKINuY6HBHpRJQw5aDUuOB5dj/2bRLRhrY3FhFBCVMOUoGyoSR2bqZxrlY0EZHUKGHKQSk08FBCI46mcd6zxHdpVJGItE0JUw5aBUdfAZ5Hw1uP5DoUEekElDDloBXoVkZk4vlEV75HdN3iXIcjInlOw0rkoBYZfw6J+hoCPQfkOhQRyXNKmHJQ80IRCo/7NACJRLzDE7OLSNenhCkCRNd/QP3L/0uwzzBCo08gNHQCXlD/PETkQ/pGEAECvQYSHnkMTUteJ7p6Dl5hKaGRxxB2JxEsG5Lr8EQkDyhhigCBou4UHH0FkSMvI7Z2IU1LZtK0+GVIJAgefzWJpnrq120ktrMe8MALgOfhFZQQKOmV6/BFJAuUMEWa8QJBQkMnEBo6gUR9DYl4FICmpW+yfub9H9s+fOhJFJ70WeI7NxPbvILwyGOyHbKIZIkSpkgrvMJue5fSCQ0aS88rvsXO7bUkiEMiAYkEgdI+ADTMfpLo8rcJ9j2EQPe+uQtaRDJGCVMkBYEe/SgpH0lt1a4W6wuOvJzoylnUv/EQxefclN3gRCQr1IdeJA0CJb2ITLqQ2Jq5RNfMy3U4IpIBSpgiaRI5/Cy8Hv2pf+MhErGmXIcjImmmhCmSJl4wROFxV5HYXU1s84pchyMiaaZnmCJpFBpyOCVTbydQ3DPXoYhImqmFKZJmgeKeJGJNNK2aletQRCSNlDBFMqBp0T+pf/43RNe/n+tQRCRNlDBFMiA89lS8bmU0vP4giXgs1+GISBooYYpkgBcqoODYqcS3raVp8Uu5DkdE0kAJUyRDQhVTCA4aR8N7fyNetzPX4YjIAVLCFMkQz/MoOP4qaGokuvT1XIcjIgcoK8NKnHNlwAPACKABWAZ8zsyqnHOjgfuBMqAamGZmS/39slonkm7BngMpvuy/CfQaBGiRapHOLFv/chPAT83Mmdl4YDlwm193D3CXmY0G7gLubbZftutE0i7YezCe59G09A3qnr6N+K4tuQ5JRDogKy1MM9sKvNKs6C3gC865vsBk4Ey//GHgTudcOeBls87MqtL3iUVaEAgSq17D7unfofCEa7UUmEgn4yUSiaye0DkXAJ4HngJeB/5sZuOa1S8GriaZ3LJWZ2azUwi/AljZ3s8sskfT9k1sfvJXNKw1uh1+Mn3OvoFAQXGuwxL2ruQm0qpcTI33G6AGuBOYlIPzH7Dq6hri8bb/0CgvL6WqleWgci1fY8vXuCBdsRUTPue/SMx5mprZT1K3fRvF5341T2JLv3yNCz4aW3l5aY6jkc4gq70PnHO3A6OAT5lZHKgEBjnngn59EBjol2e7TiQrvECQgikXU3zBLRQceRmQ7AwkIvktawnTOXcrMAW42MwaAMxsMzAXmOpvNhWYY2ZV2a5L+wcWaUOw/yiCfYYRr9tJ3VP/o7lnRfJctoaVjANuAZYAbzjnAFaa2SXA54H7nXPfBbYB05rtmu06kazzIsUkYo00/OuPBPuO0EonInkq651+OrkKYKWeYWZOvsYFmY0ttn09tdO/T3DAaIrO/Wq7x2rm63XL17jgY88w1elH2qQR1CJ5INhzIAXHXkls7UKaFv0z1+GISAuUMEXyRHjMqQSHTqDh7UeI79iU63BEZB+5GFYiIi3wPI/Ck68nuvxtvO7luQ5HRPahFqZIHgkUdSdy2Jl4XoB4TXWuwxGRZpQwRfJQdM1cdj/8X0TXLsp1KCLiU8IUyUPBgWMI9OhL/Su/I1Ffk+twRAQlTJG85IUKKDztcyTqd1H/2p/Q8C+R3FPCFMlTwT4VRI64lOjK92ic9QSJWFOuQxI5qClhiuSxyPhzCVVMpsleAy85tj5etzPHUYkcnDSsRCSPeYEARWd9mXjdTrxAiPjubex++BsEB48jcthZBAeNxfM0SY1INihhinQCgaLuAHjBMJGJ59G0+GXqnv0ZgV4DCY87g6YJxwBaV1Mkk5QwRToRr7AbBUdcSmTi+URXvEvjwudpmPlndsZ2wuEXE9u8grrnfoFX3CP5KupBoFsZoaETCPYflevwRTo1JUyRTsgLRQiPPp7QqOOIb11L93692R4FL1JE6JAjSdTtIF63k/imZUSXv0OsejXF536NRDxObP37BAc6vID++Yu0h/7FiHRinucRLBtCuFcpVO0i0HMAhSde+5FtEo11JOqTq3LENi2l7tmfQaSY0LBJhIZPITRoHF64IBfhi3QqSpgiXZwXKcKLFAEQLB9O0VlfoWnVe0RXzyW69HUAwmNOofDE60jU19Aw52kC3crwupURKC0j0HsIXiCYy48gkheUMEUOIl4oQqhiEqGKSSTiUWLrPyC2eQWB3oMAiO/eStP7L0O0ce8+wX6jkmt0+klX5GClhClykPICIUKDDyM0+LC9ZcGyoXT7zL3QsJt4zRZiG5fR8ObDNM57loIjL8thtCK5p4QpIh/heR4UdiNY2I1gnwoCfYYSLD8k12GJ5Jxm+hGR/Qr1H40XDBHbspra535BomF3rkMSyQklTBFJSWL3VmLrFlH77O1KmnJQUsIUkZSEhk2i6Mz/IF69RklTDkpKmCKSsmTSvJF4dSW1z/xUa3XKQUUJU0TaJTRsIkVn3Uh823qiGz7IdTgiWaOEKSLtFho6gZIrf0p4+BEkEgnq33qEWNXKXIclklEaViIiHRIo6QVAYlcVTe+/QtP85wgOHENk4icIDhqnZceky1ELU0QOSKB7X7pd9QsKjr6C+PYN1D17O7V/+x7RNXNzHZpIWmWlhemcux24DKgADjezhc65CuCJZpv1BLqbWW9/n1VAvf8CuNnMZvh1o4H7gTKgGphmZksPpE5EOs6LFBGZcB7hw86kaekbNM17jviuLQDEtq0jvnUdwUFjCBSW5jhSkY7L1i3ZJ4BfAa/tKTCzVcDEPb875+5oIZ7LzWxhC8e7B7jLzP7inLsauBc47QDrROQAecEwkUNPJuxOhEQcgOiyt2ic8zQAgbKhBAeNJTRoHMH+o7VKinQqWUmYZjYTwDnXYr1zLgJcBZzd1rGcc32BycCZftHDwJ3OuXLA60idmVV14GOJSCs8LwBe8olPZMrFhIZOILpuMbF1i2la+CJN8/9BwfHXEBl3OrGtlcS3riVYPhyvez89+5S8lS+dfi4E1pnZ7H3KH3TOecBM4BYz2w4M8beNAZhZzDm33i/3OljXroRZVtYt5W3Ly/P3FlS+xpavcYFi64i+/XpCv0lw2CQA4k0N1Fe+T6R8GKHSUrYuns/2mY8CECjsRsGAERQMHEmJO4aCAZmdwzZfr5nkp3xJmJ8F7tun7EQzq3TOFQB3AHcCV2c7sJZUV9cQjyfa3K68vJSqql1ZiKj98jW2fI0LFFtHtBpX6Qh21wP1u0gceg7F/Q4nVrWS+OYVNFStpG7V49R5JURC5cS2riW+ZTWhYRPxCkoyEpsSp6Qi5wnTOTcQOBm4pnm5mVX6Pxucc3cDT/lVlcAg51zQbyUGgYF+udfBOhHJES8QJFg2lGDZUDj0ZAAS0QZIJP8ojS5/O/kMNBAiOHgc4UOOIlQxCS9SnMuw5SCU84QJXAc8Y2bVewqccyVAyMx2+LdkrwTmApjZZufcXGAq8Bf/55w9zyE7Wici+cMLfdgZKHLEJYSGTqBpxbtEV7xL/Zp5EAhRdMaXCFVMIrrBiG1ajhcp8l/FeAXFBHoNVFKVtMrWsJJfA5cC/YEXnXPVZjbOr74O+PI+u/QDpvutwCCwGPhis/rPA/c7574LbAOmpaFORPKQ5wUI9htJsN9IEsd8ivjmFTSteJdAeQUAscoFNM79+8f2Kzr3q4SGjM9ytNKVeYlE28/iZK8KYKWeYWZOvsYFiq0jshFXIpGAaCOJxloSjXXQWEuisZZA+fD9jvvc5xmmuuZKm/LhlqyISId5ngfhguSYTn+6PpFM0NR4IiIiKVDCFBERSYESpoiISAqUMEVERFKghCkiIpICJUwREZEUKGGKiIikQAlTREQkBUqYIiIiKVDCFBERSYGmxmufIEAgkPq0k+3ZNtvyNbZ8jQsUW0fka1zwkdgqgLVANGfBSN7T5OvtcwLwWq6DEJGMGA6synUQkr+UMNunADgS2ADEchyLiKSXWpiyX0qYIiIiKVCnHxERkRQoYYqIiKRACVNERCQFSpgiIiIpUMIUERFJgRKmiIhICpQwRUREUqCEKSIikgLNJZsBzrnRwP1AGVANTDOzpbmNKsk5twqo918AN5vZjBzEcTtwGck5PA83s4V+ec6v3X5iW0UOr51zrgx4ABgBNADLgM+ZWVWur1sbsa0ix//POeeeIDn1XRyoAW40s7m5vm7SuaiFmRn3AHeZ2WjgLuDeHMezr8vNbKL/ynqy9D0BnASs3qc8H67dE7QcG+T22iWAn5qZM7PxwHLgNr8u19dtf7FB7v+fu9bMJpjZJOB24D6/PNfXTToRJcw0c871BSYDD/tFDwOTnXPluYsq/5jZTDOrbF6WL9eupdjygZltNbNXmhW9BQzLh+vWWmzZOn9bzGxHs197APF8uG7SuShhpt8QYJ2ZxQD8n+v98nzxoHNuvnPubudcz1wH04yuXYqccwHgC8BT5Nl12ye2PXJ+3Zxzv3fOrQFuBa4lz66b5D8lzIPPiWY2geSqKx5wZ47j6Uzy6dr9huSzuHz877dvbHlx3czsBjMbCtwC/CwXMUjnpoSZfpXAIOdcEMD/OdAvz7k9txrNrAG4Gzg+txF9hK5dCvxOSaOAT5lZnDy6bi3EljfXbQ8zewA4leRyXnlx3aRzUMJMMzPbDMwFpvpFU4E5ZlaVs6B8zrkS51wP/70HXEky1ryga5dSHLcCU4CL/QSUN9etpdjy4bo557o554Y0+/0CYCuQF9dNOg+th5kBzrlDSXZV7wVsI9lV3XIbFTjnDgGmA0H/tRj4spltyEEsvwYuBfoDW4BqMxuXD9eupdiAC8jxtXPOjQMWAkuAOr94pZldkuvr1lpswNfI/XXrBzwJlJBc+H0r8HUzm53r6yadixKmiIhICnRLVkREJAVKmCIiIilQwhQREUmBEqaIiEgKlDBFRERSoNVKRNLAOVdBchhF2MyiOQ5HRDJALUwREZEUKGGKiIikQBMXSJflnBtIciLwk0hOBv5LM/u1c+77wGEkZ305D1gKfMbM5vn7jQF+C0wE1gHfMrOn/Loi4EfA5UBPYAFwJtCP5C3Z64AfAsX++W7N/CcVkWxQC1O6JH+JqaeBecAg4HTgJufc2f4mFwGPAr2Bh4AnnHNh51zY3+95oC9wI8mlqZy/3+0k50s9zt/3v4B4s1OfADj/fN/1k6+IdAFqYUqX5Jw7GnjUX85pT9m3gNHAauAcMzvGLw+QbEle4W/6KDBwz2obzrmHAQN+AOwGjtnTGm127AqSLcwhZrbWL3sH+IWZ/TVTn1NEske9ZKWrGgYMdM5tb1YWBF4jmTD3LuFkZnHn3FqSSzsBVO5Jlr7VJFupfYBCYPl+zrux2ftaoFtHP4CI5BclTOmqKkmu5DFq3wr/GWbz5Z4CwGBgvV80xDkXaJY0h5JchWMLUA+MIHmrV0QOIkqY0lW9A+x0zt0M/BpoBMYARX79FOfcpcBTwJeBBuAtwCN52/W/nHM/J7nY8QXAkX5L9D7gF865a4BNwFHA7Ox9LBHJFXX6kS7JzGIkE91Eks8WtwC/B3r4mzwJfIrkGojXAJeaWZOZNQIXAuf6+9xNco3ED/z9vk6yZ+y7JNdV/An6dyRyUFCnHzno+LdkR5rZ1bmORUQ6D/1lLCIikgIlTBERkRTolqyIiEgK1MIUERFJgRKmiIhICpQwRUREUqCEKSIikgIlTBERkRT8f3U+Xt8wWaxIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 491.6x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# reading metrics from the logs produced by CSVLogger\n",
    "metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n",
    "\n",
    "del metrics[\"step\"]\n",
    "del metrics[\"train_loss_step\"]\n",
    "# comment if not logging lr changes\n",
    "# del metrics[\"lr-Adam\"]\n",
    "metrics.set_index(\"epoch\", inplace=True)\n",
    "# display(metrics.dropna(axis=1, how=\"all\").head(10))\n",
    "\n",
    "sns.relplot(data=metrics, kind=\"line\").set(title='Loss vs epochs')\n",
    "plt.grid(True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c02efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# con il LR più alto mostra overfitting\n",
    "# loss è MAE, quindi l'errore è intorno a 250000/750000, 30%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b668f910",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:computervision_p37_gpu_v1]",
   "language": "python",
   "name": "conda-env-computervision_p37_gpu_v1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}