ch-04/first_fc_lightning_best_lr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c33a96d0",
   "metadata": {},
   "source": [
    "### A Binary Classifier built using a FC network, with PyTorch\n",
    "\n",
    "We will be working with the Attrition dataset and we want to build a binary classifier: a model enabling to predict if the employee wants to leave the company (target = 1)\n",
    "\n",
    "In this Notebook we show **how-to automatically find the best learning rate**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "768324bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "# adding CSVLogger to the trainer we get at the end\n",
    "# metrics.csv from which we can easily get a plot of loss vs epochs\n",
    "from pytorch_lightning.loggers import CSVLogger\n",
    "\n",
    "from attrition_dataset import AttritionDataset\n",
    "\n",
    "# pyplot to plot the training loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set()\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "deb77adc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check that access to GPU is OK\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b67d18da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f7543554490>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# need to make training reproducible\n",
    "torch.manual_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0e98b3bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# globals\n",
    "\n",
    "EPOCHS = 800"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a72468e2",
   "metadata": {},
   "source": [
    "#### Here you find defined the Network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "febf2b24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# we need to implement a class subclassing nn.Module, writing the forward method\n",
    "\n",
    "#non conviene avere una rete troppo profonda e con troppi parametri... i dati non sono tantissimi\n",
    "class FirstFC(pl.LightningModule):\n",
    "    # here we define the structure\n",
    "    def __init__(self, learning_rate=1e-3):\n",
    "        super(FirstFC, self).__init__()\n",
    "        self.fc1 = nn.Linear(30, 30)\n",
    "        self.fc2 = nn.Linear(30, 30)\n",
    "        self.fc3 = nn.Linear(30, 30)\n",
    "        self.fc4 = nn.Linear(30, 10)\n",
    "        self.fc5 = nn.Linear(10, 1)\n",
    "        # fondamentale memorizzare anche il dropout tra gli attributi \n",
    "        # per disattivarlo in modalità eval\n",
    "        # altrimenti in eval i risultati non sono pienamente riproducibili\n",
    "        self.drop = nn.Dropout(0.1)\n",
    "        \n",
    "        # to find the best LR\n",
    "        self.learning_rate = learning_rate\n",
    "    \n",
    "    # here we implement the forward pass\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.drop(x)\n",
    "        \n",
    "        x = self.fc2(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.drop(x)\n",
    "        \n",
    "        x = F.relu(self.fc3(x))\n",
    "        x = F.relu(self.fc4(x))\n",
    "        \n",
    "        # the last layer must give the prob that it is positive... so we use sigmoid\n",
    "        # so we must use nn.BCELoss\n",
    "        x = torch.sigmoid(self.fc5(x))\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def configure_optimizers(self):\n",
    "        # qui possiamo anche pluggare un LR scheduler !!!\n",
    "        optimizer = optim.Adam(self.parameters(), lr=self.learning_rate)\n",
    "        \n",
    "        return optimizer\n",
    "    \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        inputs, targets = batch\n",
    "        \n",
    "        outputs = self.forward(inputs)\n",
    "            \n",
    "        # we need to add this !\n",
    "        targets = targets.unsqueeze(1)\n",
    "            \n",
    "        loss = nn.BCELoss()(outputs, targets)\n",
    "        \n",
    "        self.log(\"train_loss\", loss, on_epoch=True, prog_bar=True)\n",
    "        \n",
    "        return loss\n",
    "    \n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        inputs, targets = batch\n",
    "        \n",
    "        outputs = self.forward(inputs)\n",
    "            \n",
    "        # we need to add this !\n",
    "        targets = targets.unsqueeze(1)\n",
    "            \n",
    "        loss = nn.BCELoss()(outputs, targets)\n",
    "        \n",
    "        self.log(\"validation_loss\", loss, on_epoch=True, prog_bar=True)\n",
    "        \n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7247c0cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 2219 samples in training set...\n",
      "We have 247 samples in validation set...\n"
     ]
    }
   ],
   "source": [
    "# dataset\n",
    "attrition_path = \"/opt/notebooks/ads-examples/oracle_data/orcl_attrition.csv\"\n",
    "\n",
    "attrition_ds = AttritionDataset(attrition_path, over_sample=True)\n",
    "\n",
    "# split in train and validation\n",
    "FRAC = 0.1\n",
    "N_TRAIN = int(len(attrition_ds)*(1. - FRAC))\n",
    "N_VALID = len(attrition_ds) - N_TRAIN\n",
    "\n",
    "train_set, val_set = torch.utils.data.random_split(attrition_ds, [N_TRAIN, N_VALID], generator=torch.Generator().manual_seed(42))\n",
    "\n",
    "print(f\"We have {len(train_set)} samples in training set...\")\n",
    "print(f\"We have {len(val_set)} samples in validation set...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "41ece34d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# oversampling the size has greatly increased performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "374d4c71",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataloaders\n",
    "\n",
    "# aumentando il btch size le prestazioni peggiorano\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "train_dl = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True, num_workers=6)\n",
    "val_dl = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False, num_workers=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd76549e",
   "metadata": {},
   "source": [
    "#### Do the test to find the best LR (in a given interval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "191e65d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "/home/datascience/conda/pytorch_onoci_v1_0/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1764: PossibleUserWarning: GPU available but not used. Set `accelerator` and `devices` using `Trainer(accelerator='gpu', devices=1)`.\n",
      "  rank_zero_warn(\n",
      "/home/datascience/conda/pytorch_onoci_v1_0/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1892: PossibleUserWarning: The number of training batches (35) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc98dfff18e04d22a1e5b939ae149bbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Finding best initial lr:   0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_steps=100` reached.\n",
      "Restoring states from the checkpoint path at /home/datascience/pytorch-on-oci/ch-04/.lr_find_65e5189f-2545-4050-a6be-b909df0df62c.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAG6CAYAAADqJ1eGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABQWklEQVR4nO3deViVdf7/8ec5h8Mi+6oI4gKI4oK7trikplY61jittmdNZc3aYk01U79maqamzWn5mm2WWdkypak5LpGVS+67IioIKoIsh3055/z+QClSE4HDfTi8HtfVVZxzn5s30Ifz4rOanE6nExEREREPYza6ABERERFXUMgRERERj6SQIyIiIh5JIUdEREQ8kkKOiIiIeCSFHBEREfFICjkiIiLikRRyRERExCMp5IiIiIhH8jK6ACM5nU4cDtds+Gw2m1x2bxE5O7VBEeO5oh2azSZMJlODrm3TIcfhcJKfX9rs9/XyMhMa6o/NVkZNjaPZ7y8iv0xtUMR4rmqHYWH+WCwNCzkarhIRERGPpJAjIiIiHkkhR0RERDySQo6IiIh4JIUcERER8UgKOSIiIuKRFHJERETEIynkiIiIiEdSyBERERGPpJAjIiIiHkkhR0RERDySQo6IiIh4JIUcERER8UgKOXJa1TV20g8X4XQ6jS5FRESkUbyMLkDcT3FZFc9/tIWDR4u5YVx3LhoQa3RJIiIi50w9OVJPvq2Cp+du5ODRYgAWrs6gxu4wuCoREZFzp5AjdXIKynjqvY0cOV5GaKAPwf7eFBRX8v32o0aXJiIics4UcgSAzJxinnpvI8dtFUSF+vHQ9QOYMDQOgEWrM7A71JsjIiKti0KOYCut4pl5m7CVVhEXFcBD1w8kItiPUf1iCPCzcqywnB92HzO6TBERkXOikCMs25BFaUUNsZH+PHBdf4L9vQHw8bZw8aDaScdfrs7AcZqVVtU19rPe3+FwcjS/jPW7j/HpN/uZ+clWFnx3oHm/CBERkZ/R6qo2rqKqhpUbswCYfGFX2vla6z0/emAsi9dmkp1bypZ9efRPjARqV2DNWbKHjWm5jEjpyJWj4k95bVlFDZ9/e4Bvthymsrp+GNqUlkfHCH8GJkW58KsTEZG2TCGnjVu19QilFTVEhfrVBZif8ve1MnpALIvWZLDw+wz6JUSw82ABs7/cSVFJFQCpmw+zOS2PqRd3Z2BS7T3W7Mjhw5X7sJXWXuPtZSYmMoBOUf6UVdSwfk8u7y3dS4/Oofj/LByJiIg0B4WcNszucLB03SEAJgyJw2w2nfa6cYM78b/1hzhwxMbLn21n495cAKLD23HpsM4sXJ1BTn4Zr/x3Oynx4ZRX1rA3qwiA9qF+XDu2O727htXdv7rGTnbeDxw5XsYHy9O47bLkFvhqRUSkrdGcnDbsh93HOG6rILCdlfN7dzjjdUH+3oxI6QhQF3Au6h/DYzcP5oI+0Txx62Amnd8Fi9nElvTj7M0qwttqZsrIbjxx21D6xofXC1BWLwu3XNITE/DdtqNs33/cpV+niIi0TerJaaOcTidL1mYCMHZgLN5Wyy9ef8nQONbuzAHg1kt70i8xou45q5eFK0Z0Y0jPKOZ/nU47Xy+mjIgnPNj3jPdLiA1mzMBYlm3I4p0lu3nitqH4+eh/RxERaT56V2mjdmYUkJlTgrfV3KBjG8KCfHnqt8Pw9rJg9Tp9B2BMZAB/uDKlwTX8emQ3Nu/LI6+ogk9T9zN1XPcGv1ZERORsNFzVRp3sxRnetyMBfg2b+Ovvaz1jwGkMX28vbprQA4DlG7NY8P1BSsqrm+3+IiLStinktBFOp5OKqhryCsvZlJbLjgP5mE0mxg/uZGhdvbqG1c33+eyb/fz55e94a9EuMnOKDa1LRERaPw1Xebiyihqe/2gzGTnF1Njrb+Y3qEckESF+BlX2oxvGdychJphlGw6RmVPCqq1HWLX1CAO7R/Lbyb3wsiiLi4jIuTP83WPVqlVcf/31DBs2jN69ezNmzBieeuopiovr/yWfmprK5ZdfTp8+fbj44ouZO3euQRW3Lkt/yCT9sK0u4HhZzIQG+hDfMYgrhnczuLpaFrOZC/tG89ebB/PQ9QMY0jMKs8nEhr25fLA8zejyRESklTK8J6eoqIj+/ftz0003ERQURFpaGjNnziQtLY0333wTgE2bNnH33XczefJkZsyYwcaNG3nyySfx9vbmyiuvNPgrcF/FZVV89UPtPji3XtqTQT0i8bFaMJlOvx+O0UwmE4mxISTGhrApOZeZn2xjxcZsOncIZHjfjkaXJyIirYzhIWfixIlMnDix7uOhQ4fi7e3No48+Sk5ODu3bt+fll18mOTmZf/zjHwAMGzaMI0eO8OKLLzJlyhTMZsM7pNzS4rWZVFbZiYsK4Pw+HTC7abg5nf6JkUy+sCuff3uAd7/aS2xkAF2jg4wuS0REWhG3TAchISEA1NTUUFVVxZo1a7jsssvqXTNp0iRyc3PZuXOnARW6v8KSSlZsqD2T6ooR3VpVwDlp0gVd6J8YQY3dwX8+3UbRiSMiREREGsJtQo7dbqeyspIdO3bw8ssvc9FFFxETE0NmZibV1dV061Z//khCQgIA6enpRpTr9r78PoOqGgfxHYPoGx9udDmNYjaZmDYxmejwdhQUV/LqZ9uosTuMLktERFoJw4erTrrooovIyandUXf48OE899xzQO2cHYCgoPpDFSc/Pvl8Y3k1474vJ1lOrAayGLQqKK+wnK83ZwNw5UUJWM+ym7E7C/Ty5g9XpfC3N9exN6uIxWszuWKEe0yYFvdldBsUEfdoh24TcmbNmkVZWRn79u3jlVde4c477+Stt96qe/5Mk2WbMonWbDYRGurf6NefTVCQMcuz3126F7vDSd+ECC4YYOw+OM0hNNSfe67sxzPvbWDxmgwuvyiRsKAzHxkhcpJRbVBEfmRkO3SbkNOjR+3OtwMGDCA5OZkpU6bwv//9r25Y6uc9NjabDTi1h+dcOBxObLayRr/+TCwWM0FBfths5dhbeHjlaH4Zy0+sqJp8YRcKCkpb9PO7Su/OIcTHBJGebePtBdu55dKeRpckbszINigitVzVDoOC/BrcO+Q2IeenevbsicViITMzk9GjR2O1Wtm/fz8jRoyou2bfvn0AxMfHN+lz1dS47heg3e5w6f1PcjqdHDhSzHfbjrB2Zw4Op5O+8eF07RDUIp+/pVw5KoGn524kddNhxg6MJTrcdb1w4hlaqg2KyJkZ2Q7dcsB606ZN2O12YmNj8fb2ZtiwYSxevLjeNQsXLiQyMpLk5GSDqjSe0+nk683ZPPrGOp6cs56Vm7Ipq6whMsSXa8YkGl1es+veKYR+CRE4nE4+/loTzkVE5JcZ3pNzzz330Lt3b5KSkvD19WX37t3Mnj2bpKQkxo4dC8D06dO5/vrreeSRR5g0aRIbN25k/vz5PPHEE216j5z1e3KZs2QPAFYvMwO7R3JB32h6dg5tlUvGG+I3o+LZkp7HprQ80rIKSYwNMbokERFxUyan0+k8+2WuM2vWLBYtWkRmZiZOp5OYmBguvvhibrvtNgICAuquS01N5bnnniM9PZ0OHTpwyy23MHXq1CZ9brvdQX5+889Z8fIyExrqT0FBqcu66Kpr7Pzl9bXkFVVw0YAYpoyIp52v4Zm1RbyzZDepmw8THxPEw9cPdNsdnMU4LdEGReSXuaodhoX5N3hOjuEhx0itOeR8ufogn6TuJzTQh3/cPgwf79a7TPxcFZZUMuP/VlNV7WD6Fb0ZmBRldEniZhRyRIznDiGnbfzp3wrk2ypYvycXi9mEt5cZb6sFH6uFhNhgAvys9a4tKq3iy9UZAEwZ2a1NBRyAkAAfxg+OY8H3B/n463T6xkdgdcF+RyIi0rop5LiJ95elsXFv7imPB/hZmX5Fb5LiQuse++yb/VRU2enSIZBhvTq0ZJluY8LQOFK3HCanoJylP2Ry2XldjC5JRETcjP78dROHjhUD0LNzKH3jw+kRF0J4kC8l5dU8+8FmUk/sYJyZU8yqrYcBuHZsosdOMD4bPx8vrr6odg+lBd8f5HhRhcEViYiIu1FPjhuosTvIO/EmPW1iMqGBPgBUVtt588td/LD7GO8s2UN2binZeaU4nTC4R1SbX1k0rFd7UrccZu+hQuYtT+OeX/cxuiQREXEj6slxA3lFFTid4G01ExLgXfe4j9XCnZN7ccXwrgAs25DFrowCvCxmrhzVtE0QPYHJZOL6cd0xm0xs3JvL1vTjRpckIiJuRCHHDRwrqD1aIiqk3SnLoU0mE5Mu6Mr0K3rjba39cY0b3ImIEJ3JAxAbGcDFg2MBeP9/e6musRtckYiIuAsNV7mBnPxyANqHnjm4DEyKomOEP3sPFXJBn+iWKq1V+NUFXVm7M4djheUsXpPJry7sanRJIiLiBtST4waOFdSGnKiwX+6diQ73Z2S/GLwMPLbeHfn5eNUdY/HlmgyOFZYbXJGIiLgDvVu6gZwTw1XtQ9sZXEnrNbhHFD07h1Jd4+DjlfuMLkdERNyAQo4bONmT80vDVfLLTCYT145NxETtmV7ph4uMLklERAymkGOwny4fj1JPTpPERgZwfp/azRHnr0ynDZ9YIiIiKOQY7nhRBQ6nE2+v+svHpXGuGN4NL4uZvYcKtaRcRKSNU8gxWM7JScehfjpNuxmEBfkydlDtkvKPU9NxONSbIyLSVinkGEyTjpvfZed1xt/Xi+zcUr7fftTockRExCAKOQY79pOeHGke/r7WugM7P1u1n6pqbRAoItIWKeQYrK4nJ0w9Oc1pzMAYwoJ8KCiuZPmGLKPLERERAyjkGKyuJ0fHNDQrq5eFK4Z3A2Dh6gyKSioNrkhERFqaQo6BauwO8gprl4+rJ6f5nderA53bB1JeWcP7y9KMLkdERFqYQo6Bjtt+XD4erOXjzc5sNnHzJT0wm0z8sPsYm9JyjS5JRERakEKOgU4ezBkV6odZy8ddonOHQMYP6QTAe0v3Ul5ZY3BFIiLSUhRyDHTsxKRj7XTsWr+6sCtRIX4UFFfycWq60eWIiEgLUcgxUI6Wj7cIH6uFmyYkAfD1xmzSsgqNLUhERFqEQo6BdDBny+nZJYwL+0bjBN5evJvqGofRJYmIiIsp5BhIw1Ut66qLEgjy9+bI8TL+++1+o8sREREXU8gxiN3x4+nj6slpGQF+Vm4YVztstXhNJtv26wBPERFPppBjkONFFdgdTqxeZkICfYwup80YmBTJ6AExALy+YCf5tgqDKxIREVdRyDHITycda/l4y7p6dAKd2wdSUl7Na1/swO7Q/BwREU+kkGMQHedgHKuXhbsu74Wfj4V9WUV89s0Bo0sSEREXUMgxSE6+DuY0UlRoO265pCcAi9ZksDVd83NERDyNQo5BjhVqjxyjDeoRxZgBsQDMXqj5OSIinkYhxyB1PTlaPm6oq34yP2fWgp2anyMi4kEUcgyg5ePuw+pl5s7JvfDxtrD3UCELvjtodEkiItJMFHIMoOXj7qV9WDtuGl+7f86C7w6yK6PA4IpERKQ5KOQYIPdEL05EsK+Wj7uJYb061B37MGvBDmylVUaXJCIiTaSQY4CTE1zDg3wNrkR+aurY7kSHt6OopIrZX+7E4XQaXZKIiDSBQo4BCmyVAIQFaajKnfh4W7jr8t5Yvcxs35/P15uyjS5JRESaQCHHAPnFtT05oYHqyXE3sZEB/GZkPABL1mbicKg3R0SktVLIMUD+yZ4cTTp2SyP6dcTf14u8ogo2peUZXY6IiDSSQo4BCopPDlepJ8cd+VgtjOxXe4jn8g2HDK5GREQaSyHHACeHqzQnx32NHhCD2WRid2Yhh46VGF2OiIg0gkJOCyuvrKG80g5AqIar3FZYkC8DkyIB+N969eaIiLRGCjkt7OTy8XY+Xvh6exlcjfySsYNqz7VasyMHW5n2zRERaW0UclpYfrGWj7cWCTHBdO4QSI3dwTebDxtdjoiInCOFnBamSceth8lk4uITvTkrN2VTY9fhnSIirYlCTgs7OVyl5eOtw+Ae7Qny96aguJINe3KNLkdERM6BQk4LO7lHTqh6cloFq5eZi/rXLidfpgnIIiKtikJOC6tbPq6enFZjVP8YLGYT6YdtZBwtNrocERFpIIWcFqbdjlufYH9vBnSvXU6+aqsmIIuItBYKOS3I6XRq4nErNSKlIwCrd+RQVW03uBoREWkIwzdqWbx4MQsWLGDHjh0UFRXRqVMnrr32Wq655hrM5toMNmPGDD777LNTXvv6668zYsSIli650coqa6is1kaArVHPLqGEB/ly3FbBhj25nNe7g9EliYjIWRgect566y06duzIAw88QHh4OGvXruXvf/87hw4d4sEHH6y7rlOnTjz77LP1XhsfH9/S5TbJyaGqAD8r3laLwdXIuTCbTAxPiea/qw7wzZbDnNerPab8fEylJTj9A3CGhYHJZHSZIiLyE4aHnNdee42wsLC6j4cNG0ZZWRlz587lj3/8I97e3gD4+vrSr18/g6psHlo+3rpd2CeaZcu20f3zBQQ9fzM+mQfrnrN36Ur5tN9ScfV1OINDDKtRRER+ZPicnJ8GnJN69uxJZWUlhYWFLV+QC+VrPk6r1n79t7z9+u1M+/pNrIcy6j1nzjiI/6MPEZ7SE+uKZQZVKCIiP2V4yDmdDRs2EBISQnh4eN1jmZmZDBo0iN69e/PrX/+aZcta3xtJwYnl46E60qHVsa5YRvDUK/GursCME7PTWe95k9OJyemEinKCp16poCMi4gYMH676uW3btvHpp58yffp0LJbaeSs9e/akT58+JCQkUFxczLx585g+fTovvvgiEyZMaNLn8/Jq/pxnsZjr/fukguLaQx4jgv1c8nnFNUxFhQTfdgOcDDK/dK3DgdNsJvi2GyjavkdDVwY5UxsUkZbjDu3Q5HSe5bd2C8rNzeWqq66iffv2vPvuu1it1tNe53A4uOaaaygpKWHRokWN/nxOpxNTC04W/cur37F1Xx5/vm4AowZ2arHPK0304ovwxz/CuTQVkwleeAF+9zuXlSUiIr/MbXpyiouLuf322/H19eXVV189Y8ABMJvNjBs3jmeeeYaKigp8fRs3x8XhcGKzlTW25DOyWMwEBflhs5Vj/8mhjjnHSwHwsZgoKCht9s8rLuB0EvTii5iBc4nDTsDxwgvYrr9Vq64McKY2KCItx1XtMCjIr8G9Q24RciorK7nrrrvIy8vjww8/JDQ09Kyvaa4OqJoa1/0CtNsddfd3Op11E4+D/L1d+nml+ZiOH8dy4MC5v87pxHLgAPbcPJxh4Wd/gbjET9ugiBjDyHZo+IB1TU0Nv//979m9ezezZ88mJibmrK9xOBx89dVXJCYmNroXp6WVlFdTfeKHHBqgicetham0pGmvL2na60VEpPEM78l54oknWLlyJffffz8VFRVs3ry57rmEhASKioqYMWMGEydOJC4ujqKiIubNm8f27duZOXOmcYWfo5MbAQb5e2PVpONWw+kf0LTXBzTt9SIi0niGh5xvv/0WgGeeeeaU5+bMmUNSUhIBAQG8/PLL5OfnY7Va6d27N6+//jrDhw9v6XIbTaePt07OsDDsXbpizjh41pVV9V5nMuHo3AVn6Kn7QImISMswPOSsWLHirNe8+uqrLVCJa9WdPq6NAFsXk4nyab/F/9GHzvml5bffqUnHIiIG0rhJCznZk6ODOVufiquvA792OM0Nay5Osxn82lFx1bUurkxERH6JQk4LKajryVHIaW2cwSEUvfkumExnDTpOsxlMJoreek8bAYqIGEwhp4XUnVsVqOGq1qh69FiK5s4HXz+cJhPOnw1DOTjxmK8fRe9/TPVFYwyqVERETlLIaSF1J5CrJ6fVqh49luNbdlH65NM4Onep99zR4PZ8dfXvOb51twKOiIibMHzicVvgcDopUE+OR3AGh1B++12UT7sTU0E+ppISMsrgr5/tw8vLTHcvPwKNLlJERAD15LSI4tIq7A4nJiA4wNvocqQ5mEw4w8JxxHWmU4/OdO4QRI3dyertR42uTERETlDIaQEn5+MEB3jjpVORPdKIfh0BSN1yuNmOHBERkabRO24L0B45nm9Ycnt8rBaOHC9jZ0aB0eWIiAgKOS1Cux17Pj8fLy7sGw3AkrWZBlcjIiKgkNMiCtST0yaMG9wJkwl2HMgnM6fY6HJERNo8hZwWoN2O24bIED8G94gC4Kt16s0RETGaQk4L0JyctmP8kDgA1u06Vrc3koiIGEMhpwUUaE5Om9E1OogecSHYHU7+t/6Q0eWIiLRpCjku5nA4KSiuAtST01ZMGFrbm5O6+TBlFTUGVyMi0nYp5LhYUWkVDqcTs8lEsL82AmwLencLp2OEPxVVdlK3ZBtdjohIm6WQ42In52WEBHpjNpvOcrV4ArPJxPghnQBYtj6LGrvD4IpERNomhRwX0+njbdOw5A4EB3hTUFzJ2p05RpcjItImKeS4WIFOH2+TrF5mLh5U25uzZF2mjnoQETGAQo6LqSen7RrVryM+3hayc0vZtj/f6HJERNochRwXOzknJ1Q9OW1OO18rI1NqD+5csjbD4GpERNoehRwXU09O23bxoE6YTSZ2ZxZy8KjN6HJERNoUhRwXy9ecnDYtPNiXIcm1Rz3o4E4RkZalkONCNXYHRSUnNgLUbsdt1oQTRz38sPsYuYXlBlcjItJ2KOS4UGFJFU7AYjYRqI0A26y49oH06hKK0wlLf9BRDyIiLUUhx4XqJh0H+mA2aSPAtmzC0M4ArNp6mJLyaoOrERFpGxRyXOjH+TiadNzWJXcJJS4qgKpqBys3ZhldjohIm6CQ40LHbTp9XGqZTCbGnzi4c/mGLKqq7QZXJCLi+RRyXCjfVrt8XHvkCMDgHlGEB/lgK6vm221HjC5HRMTjKeS4UN1wlfbIEcDLYq6bm7N4TaYO7hQRcTGFHBc62ZOjPXLkpOF9owny9+a4rUIHd4qIuJhCjgupJ0d+zttqYfzg2oM7v1ydgcOhgztFRFxFIcdFqmvsFJXWbgSoOTnyU6P6x9DOx4uj+WVs3JtrdDkiIh5LIcdFjhfV9uJYvcwE+lkNrkbciZ+PF2MHxQKwcPVBnE715oiIuIJCjovkndi+PzTQB5M2ApSfGTuoEz5WC5k5JWzbn290OSIiHkkhx0VOhhztkSOnE+BnZVT/joB6c0REXEUhx0VOHsSo3Y7lTMYPicPLYmJfVhF7DxUaXY6IiMdRyHGRnw5XiZxOSIAPF/at7c35ODUdh3pzRESalUKOi+QV6twqObtJ53fBx2ohPdvG6u1HjS5HRMSjKOS4SF6R5uTI2YUG+jDpgi4AzP86nfLKGmMLEhHxIAo5LpKnOTnSQBcP6kT7sHbYSqv4/NsDRpcjIuIxFHJcoKraju3ERoA60kHOxuplZurYRKD2hPLsvFKDKxIR8QwKOS6QX1x7ZpW31Uw7Hy+Dq5HWoHe3cPonRmB3OHn/f3u1pFxEpBko5LhA/ondjsODfLURoDTYNWMS8bKY2ZVRwIY9Ou5BRKSpFHJc4LhNK6vk3EWG+HHpsDgAPliRpknIIiJNpJDjAgUnhqs0H0fO1SXDOhMR7Eu+rZJ3l+7RsJWISBMo5LjAyZ6ccPXkyDnysVq4Y1IvzCYTa3bk8L32zhERaTSFHBfI13CVNEFCbDCTL+wCwHtL93I0v8zYgkREWimFHBfIt2m4SprmsvO60CMuhMpqO699vp3qGofRJYmItDqGh5zFixdz9913M3LkSPr168ekSZN4//33cTjq/1JPTU3l8ssvp0+fPlx88cXMnTvXoIrPThOPpanMZhO3T+pFgJ+VzJwSPklNN7okEZFWx/CQ89Zbb+Ht7c0DDzzAa6+9xtixY/n73//OM888U3fNpk2buPvuu0lOTub111/niiuu4Mknn2T+/PkGVn56FVU1lFXUrorRnBxpitBAH269tCcAS384xNb0PIMrEhFpXQzfqe61114jLCys7uNhw4ZRVlbG3Llz+eMf/4i3tzcvv/wyycnJ/OMf/6i75siRI7z44otMmTIFs9nwrFbn5Mqqdr5e+Pl4UaNhBmmCfokRjBkYy/INWcxeuIvHbx2ik+1FRBrI8HTw04BzUs+ePamsrKSwsJCqqirWrFnDZZddVu+aSZMmkZuby86dO1uq1HMS1z7Q6BLEQ1x1UTxxUQGUlFfz+oIdOBxaVi4i0hCGh5zT2bBhAyEhIYSHh5OZmUl1dTXdunWrd01CQgIA6enuNVehQ1g7HriuP/dfP8joUsRDWL0s/HZyL3ysFnZnFvLl6oNGlyQi0ioYPlz1c9u2bePTTz9l+vTpWCwWioqKAAgKCqp33cmPTz7fWF5ezZ/zUhIjCQryw2Yrb/Z7S9vUqX0gN12SxKwvdvLfbw+Q3DWMpLhQo8tyWxaLud6/RaTluUM7dKuQk5uby+9+9zv69OnD7bffXu+5M50B1ZSzocxmE6Gh/o1+/dkEBfm57N7S9kwamUhato2VG7L4v8938NJ9FxHYztvostya2qCI8Yxsh24TcoqLi7n99tvx9fXl1VdfxWq1AhAcHAyc2mNjs9mAU3t4zoXD4cRma/6N1iwWc11Pjt2uicfSfK4ZncDOA/nk5Jfx7Lvr+f2VfXUI7GmoDYoYz1XtMCjIr8G9Q24RciorK7nrrrvIy8vjww8/JDT0x274uLg4rFYr+/fvZ8SIEXWP79u3D4D4+PgmfW5Xrn6y2x1aXSXNymoxc+evevH3d9ezcW8uS9ZkcvHgTkaX5bbUBkWMZ2Q7NHzAuqamht///vfs3r2b2bNnExMTU+95b29vhg0bxuLFi+s9vnDhQiIjI0lOTm7JckUM17lDIFePTgTgo5X72H/YZnBFIiLuyfCQ88QTT7By5UruvPNOKioq2Lx5c90/JSUlAEyfPp3t27fzyCOPsHbtWl599VXmz5/P73//e7faI0ekpYweEMPApEjsDievfb6d0opqo0sSEXE7JqfTaeimG6NHjyY7O/u0z82ZM4ehQ4cCtcc6PPfcc6Snp9OhQwduueUWpk6d2qTPbbc7yM8vbdI9TsfLy0xoqD8FBaXqKheXKauo4fG315FbWEH/xAju+XUfzc85QW1QxHiuaodhYf4NnpNjeMgxkkKOtHYHj9r4x7sbqLE7uXZMoubnnKA2KGI8dwg5GusRacW6dAjS/BwRkTNQyBFp5UYPiGHQifk5sxbsoLLKbnRJIiJuQSFHpJUzmUzcfEkPwoJ8OFZQzocr9xldkoiIW2h0yNm9ezc//PBD3celpaX87W9/46qrruLFF1+kDU/1EWlx7Xyt3HppTwC+3pTN1vTjBlckImK8Roecp59+mpUrV9Z9/PzzzzN//nyqq6uZNWsW7733XrMUKCINk9wljLGDYgF4a9EuSsq1rFxE2rZGh5y0tDQGDBgAgNPpZMGCBdx777189tlnTJs2jU8++aTZihSRhvnNyHiiw9tRVFrFnK/2qEdVRNq0Roccm81GSEgIUDt0ZbPZuOSSSwA477zzOHToULMUKCIN5221cPukZCxmE+t3H2PtzhyjSxIRMUyjQ05ISAhHjx4FYO3atYSHh9O5c2cAqqur9RekiEG6dAhi0vldAHh36V7ybRXGFiQiYpBGh5xBgwYxc+ZM3n33Xd5++21GjRpV91xGRgbR0dHNUZ+INMJl53ema3QQ5ZU1vPHlLhz6o0NE2qBGh5w//elPmEwm/v73v+Pt7c306dPrnluyZAkpKSnNUqCInDuL2cztk5LxtprZlVHAsh80fCwibU+Tj3UoLCysm5tz0p49e4iMjCQsLKwpt3Y5Hesgnm7lpmze/WoPXhYzj908iNjIAKNLahFqgyLG84hjHX4ecCorK0lKSnL7gCPSFozq15G+8eHU2B28vmAn1XrDF5E2pNEhZ9GiRcydO7fu44yMDC699FL69evHddddR1FRUbMUKCKNZzKZuOWSHgT4WTl0rIT/rtpvdEkiIi2m0SHnjTfeoLy8vO7jf/3rX9hsNm688Ub279/Pa6+91iwFikjTBAf4cPMlPQBYsjaTPZkFBlckItIyGh1ysrKySEysPf24srKSb7/9lvvuu4+HHnqIP/zhDyxfvrzZihSRphnQPZIL+0bjBGYt2KndkEWkTWh0yCkvL6ddu3YAbNmyhaqqKkaMGAFAQkICOTnahEzEnVw7JpH2oX4UFFfy1qJd2stKRDxeo0NOZGQku3btAmDVqlV07dq1brJxUVERvr6+zVOhiDQLPx8v7pzcGy+LiU1peazYmG10SSIiLtXokDNu3Dief/557r33XubMmcOll15a99yePXuIi4trlgJFpPl07hDIlRclAPDhijQyc4oNrkhExHUaHXJ+//vfM2nSJA4ePMjEiROZNm1a3XNff/01559/frMUKCLNa+zAWPolRFBjd/Lq5zuoqKoxuiQREZdo8maArZk2A5S2qqS8mr++uY6C4krO792B2y7riclkMrqsZqM2KGI8j9gMEODAgQNs2rSJgwcPNsftRMTFAvys3DEpGZMJvt9+lMVrM40uSUSk2Xk15cWLFy/mX//6V91p5AAdOnTgwQcfZMKECU0uTkRcJykulKtHJ/LB8jQ+/jodf18vRvaLMbosEZFm0+iQk5qayp/+9CcSEhL485//TFRUFDk5OXzxxRf86U9/ws/Pj5EjRzZnrSLSzMYN7kRxWRVfrs5gzld78Pe1MqhHlNFliYg0i0bPybnmmmsICAhg1qxZmM0/jno5nU6mTZtGaWkpH3zwQbMV6gqakyNS22bfWbKHb7Ycxsti4o9XptCzS+s+e05tUMR4rXpOzu7du7nuuuvqBRyoPSvnuuuuY8+ePY29tYi0IJPJxI3jkxiYFEmN3clLn27jwBGb0WWJiDRZo0OO2Wymuvr0W8PX1NR41EoNEU9nNpu4Y1IvenYOpbLKzqv/3U55pZaWi0jr1uiQ06dPH2bPnk1FRUW9x6uqqnjzzTdJSUlpcnEi0nKsXmbu+XUfIoJ9ySuqYN6yNKNLEhFpkkZPPL733nu5+eabGTt2LBMmTCAiIoLc3FyWLl1KYWEh77zzTnPWKSItwM/Hi2kTk/nn3I18u+0IKQnhDEzSRGQRaZ0a3ZMzaNAg3nzzTWJiYpg7dy4vvPAC8+bNIyYmhjfffJMOHTo0Z50i0kK6dwrhkmGdAXhnyR6KSioNrkhEpHGatBngkCFD+PDDD9m4cSOpqals2LCBDz74gPz8fMaMGdNcNYpIC7t8eFfiogIoKa/mrcW7dWK5iLRKzbLjsZ+fH+3bt8fPz685biciBvOymLl9UjJeFjNb04/z9ebDRpckInLOmiXkiIjniYkM4Dej4oHaE8uzc0sMrkhE5Nwo5IjIGY0dFEuvLqFUVTv4z6fbKKs4/bYRIiLuSCFHRM7IbDJxx696ER7kQ05BOa8v2IlD83NEpJU4pyXkO3bsaNB1hw4dalQxIuJ+Att5c8+v+/KP9zawJf04X3x7gMuHdzO6LBGRszqnkDNlypQG7WTsdDq147GIB+ncIZAbxyfxxpe7+OK7g3TuEEj/xEijyxIR+UXnFHKeeuopV9UhIm7ugj7RHDxSzPKNWcxeuJNHbhxEdLi/0WWJiJzROYWcK664wlV1iEgrcPWYBA4dK2ZvVhEvfbyVh28YSGA7b6PLEhE5LU08FpEG87KYueuKPoQH+ZJTUM7MT7dRXWM3uiwRkdNSyBGRcxLs780frkrBz8eLfVlFzF64SyuuRMQtKeSIyDmLifDnnl/3wWI28cPuY3zydbrRJYmInEIhR0QapWfnUG65tAcAi9dmsnJTtsEViYjUp5AjIo12fu9oLh/eFYD3lu5h495cgysSEfmRQo6INMmk87swvG80Tie89vkOdmUUGF2SiAigkCMiTWQymbhxQhL9EyOosTuY+clWDh61GV2WiIhCjog0ncVs5s7JvegRF0JFlZ3nPtzCkeOlRpclIm2cQo6INAurl4V7p/Slc4dASsqr+feHm8m3VRhdloi0YQo5ItJs/Hy8+ONVKXQIa0e+rZJ/vb9JQUdEDKOQIyLNKqidN3++uh8Rwb4cKyzn6bkbySsqN7osEWmDDA85GRkZPPbYY0yePJnk5GQmTpx4yjUzZswgKSnplH+++eYbAyoWkbMJD/blwesGEBXiR15RBf+cu4ncQgUdEWlZ53RApyukpaWRmppKSkoKDocD5xm2h+/UqRPPPvtsvcfi4+NbokQRaYTwYF8euK4/z8zbRE5BOf98fyMPXNufqNB2RpcmIm2E4T05o0ePJjU1lZdeeolevXqd8TpfX1/69etX75/AwMAWrFREzlVYkC8PXDeA6PDaOTr/fH8Tx9SjIyItxPCQYzYbXoKIuFBooA8PXNufjhH+FBRX8uy8TRQUVxpdloi0Aa0mYWRmZjJo0CB69+7Nr3/9a5YtW2Z0SSLSQMEBPtx3TT+iQmvn6Dz7wSZsZVVGlyUiHs7wOTkN0bNnT/r06UNCQgLFxcXMmzeP6dOn8+KLLzJhwoQm3dvLq/lznsVirvdvEYGIED9mTB3Ak++s58jxMp7/aAsPXT+Qdr7N/2tIbVDEeO7QDk3OM830NcCMGTPYvn07Cxcu/MXrHA4H11xzDSUlJSxatKjRn8/pdGIymRr9ehE5d9m5Jcz4z7cUllTSs0sYT9xxHr4+reLvLRFpZVrlbxaz2cy4ceN45plnqKiowNfXt1H3cTic2GxlzVxdbWoNCvLDZivHbnc0+/1FWrN2Xibuu7Yf/3h3A7sO5vPY/33PH65Kwa8Zg47aoIjxXNUOg4L8Gtw71CpDDnDGpebnqqbGdb8A7XaHS+8v0lp1DPfnj1em8O8PN7Mro4B/zt3IH65MIcDP2qyfR21QxHhGtsNWOWDtcDj46quvSExMbHQvjogYKz4mmPuv7Y+/rxf7D9v41/ubKCrVZGQRaT6G9+SUl5eTmpoKQHZ2NiUlJSxZsgSAIUOGUF5ezowZM5g4cSJxcXEUFRUxb948tm/fzsyZM40sXUSaqGt0EA9OHcC/P9hMVm4JT7+3gfuu6U94sP54EZGmM3zicVZWFmPGjDntc3PmzCEpKYmHHnqIHTt2kJ+fj9VqpXfv3txxxx0MHz68SZ/bbneQn1/apHucjpeXmdBQfwoKStVVLtIAOQVlPDtvE8dtlYQH+fDQ9QMJC2p80FEbFDGeq9phWJh/g+fkGB5yjKSQI+I+8m0VPPPBZnLyy4iLCmDG9QPw9W5cZ7PaoIjx3CHktMo5OSLiecKCfPnz1SkEtbOSeayEWV/sxOFos3+DiUgzUMgREbcREezHvVP64mUxs3lfHvO/3md0SSLSiinkiIhbiY8J5rbLegLw1bpDpG7ONrgiEWmtFHJExO0MTW7P5Rd2BeC9pXvZcTDf4IpEpDVSyBERtzTpgi4MS26P3eHk5U+3cfCozeiSRKSVUcgREbdkMpm45dIe9IgLoaLKznMfbuHI8eZfDSkinkshR0TcltXLwr1T+tK5QyAl5dX8+8PN5NsqjC5LRFoJhRwRcWt+Pl788aoUOoS1I99Wyb8/3ExxmY5/EJGzU8gREbcX1M6bP1/dj9BAH44cL+P5j7ZQVlFjdFki4uYUckSkVQgP9uW+a/oR4Gfl4NFinpm3ST06IvKLFHJEpNWIDvfnvmv6EdjOSkZOMf96fxOFJZVGlyUibkohR0Ralbj2gcyYOoCQAG+y80p5eu5G8orKjS5LRNyQQo6ItDrR4f7MuH4gEcG+HCso5+m5G8nJLzO6LBFxMwo5ItIqRYX4MWPqgLpVV0+/v1H76IhIPQo5ItJqhQX5MmPqAGIj/SkqqeKfczeSnVtidFki4iYUckSkVQvy9+b+a/sTFxWArayaf76/icycYqPLEhE3oJAjIq1eYDtv7ru2f93OyE+9t5F9WYVGlyUiBlPIERGPEOBn5f5r+tGtYxCl5dU88tr3pGcXGV2WiBhIIUdEPEY7Xyt/vrofibHBlJZX88+5G0lTj45Im6WQIyIexc/Hi/uv60+f+Ii608t3ZRQYXZaIGEAhR0Q8jq+3F49NG0rvbmFUVtt5Yf4Wtu0/bnRZItLCFHJExCP5envxh6tS6JcQQXWNg5mfbGX97mNGlyUiLUghR0Q8lreXhbuv6M3ApEhq7E5e+e92Plqxjxq7w+jSRKQFKOSIiEfzspi5c3IvLh7UCYAl6zL51/ubyLdVGFyZiLiaQo6IeDyL2cy1YxOZfkVv/Hy82JddxN/e+oGt6XlGlyYiLuRldAEiIi1lYFIUndoH8up/t5NxtJgX5m+lc/tALujTgaHJ7Qls5210iSLSjExOp9NpdBFGsdsd5Oc3/4F+Xl5mQkP9KSgopaZGY/8iLe1sbbC6xsH8lftYuSkbu6P2V6DFbCIlIYJLhsUR3zG4pUsW8Tiuei8MC/PHYmnYQJRCjkKOiMdpaBssLqti7c4cvtt+lIyjteddWcwmbpyQxPC+HVuqXBGP5A4hR8NVItJmBbbzZuygTowd1Ims3BI+X3WADXtzeWvRbo7klfGbUfGYzSajyxSRRtLEYxERIDYygLuv6M2vLugC1K7C+s+n2yivrDG2MBFpNIUcEZETTCYTlw/vxm9/1Qsvi5nN+/J46r0NHDhiM7o0EWkEhRwRkZ8ZmtyeB6f2J8jfm6zcUv7fO+t5fcFO7a0j0soo5IiInEZ8x2D+evNgzuvVAYDVO47y8Kw1fPbNfiqqNIQl0hoo5IiInEFooA+3T0rm0ZsG0T02mKoaBwu+P8jjb6/nyPHmX5kpIs1LIUdE5Cy6Rgfx4NQBTL+iN6GBPuTkl/HknA1sP6CTzUXcmUKOiEgDmEwmBiZF8debB5MQG0x5ZQ3Pf7SF//1wiDa83ZiIW1PIERE5B0H+3tx/TX8u7BON0wnzlqfx9uLdVFbbjS5NRH5GIUdE5BxZvczccmkPrh6dgMkEq7Ye4dHZa9maruErEXeikCMi0ggmk4nxQ+L441UphAb6kFdUwQvzt/DKf7dTUFxpdHkigkKOiEiT9O4azt9vH8q4wZ0wm0ys332Mv7y+hv+tP4TdobPrRIykkCMi0kS+3l5cMyaRx24eRLeOQVRU2Zm3LI3H31rP3kOFRpcn0mbpFHKdQi7icYxsgw6nk282H+aT1HRKK2o3DTyvV3uuvCiBkACfFq1FxEjucAq5enJERJqR2WRiVP8YnvrteYzs1xETsHpHDg/PWsN3245oublIC1LIERFxgQA/KzdN6MEjNw2ia3TtENYbX+7i9YU7dbK5SAtRyBERcaGu0UH85YaBXDGiG2aTiTU7cvjbW+vYf1gnm4u4mkKOiIiLmc0mJp3fhRlTBxAe5EtuYQVPvbeBL1cfxOHQ8JWIqyjkiIi0kITYYB6/dTCDe0Rhdzj5JHU//3x/I7mF5UaXJuKRFHJERFpQO18rd07uxa2X9sTX20JaVhGPvbmOVVsPa1KySDNTyBERaWEmk4kL+0bz+K1D6B4bTGWVnbcW7eY/n27DVlpldHkiHsPwkJORkcFjjz3G5MmTSU5OZuLEiae9LjU1lcsvv5w+ffpw8cUXM3fu3BauVESkeUWG+PHAdQO4clQ8FrOJTWl5PPbGWjan5RldmohHMDzkpKWlkZqaSufOnYmPjz/tNZs2beLuu+8mOTmZ119/nSuuuIInn3yS+fPnt3C1IiLNy2w2ccmwzjx60yBiIv2xlVXz0idbeXvxLi01F2kiw3c8djgcmM21WWvGjBls376dhQsX1rtm2rRpFBUV1Qs1jz76KCtXruSbb76pe/250o7HIp6ptbbB6ho7n36zn6XrDuEEIoJ9mTYxme6dQowuTeScacdjOGtAqaqqYs2aNVx22WX1Hp80aRK5ubns3LnTleWJiLQYq5eFq0cncv+1/QkPqj3Z/Om5G5mzZDdlFdVGlyfS6hgecs4mMzOT6upqunXrVu/xhIQEANLT040oS0TEZXp0DuXxW4cyvG80AF9vPsxfXl/Lul05WoElcg68jC7gbIqKigAICgqq9/jJj08+31heXs2f8052ozW0O01EmpcntMGgAG9u/1Uvhqd05K1FuzhyvIzXPt/B6h053DQhiYgQP6NLFPlF7tAO3T7knGQymc7p8YYwm02Ehvo3+vVnExSkX0IiRvKENnheqD+Dekczf3ka85fvZcu+PB6eVcD1l/Rk4oXdsJgb/ztQpCUY2Q7dPuQEBwcDp/bY2Gy15778vIfnXDgcTmy2ssYXdwYWi5mgID9stnLs9tYz6VHEU3hiG7xkSCf6dgvjrS93sfdQIbM/387ydZncellPOncINLo8kVO4qh0GBfk1uHfI7UNOXFwcVquV/fv3M2LEiLrH9+3bB3DGZecN5cqVF3a7o1Wt7BDxNJ7WBtuH+PHAdf35Zsth5q9M58ARG399Yx3jhnRi8gVd8fG2GF2iyCmMbIduP2Dt7e3NsGHDWLx4cb3HFy5cSGRkJMnJyQZVJiLS8swmE6P6xfD324cyKCkSh9PJkrWZPDJ7LZv3aRNBkZ8yvCenvLyc1NRUALKzsykpKWHJkiUADBkyhLCwMKZPn87111/PI488wqRJk9i4cSPz58/niSeeaPQeOSIirVlIgA93X9GHzWl5zP3fHo7bKnjp460MTIrkurHdCQ30MbpEEcMZvhlgVlYWY8aMOe1zc+bMYejQoUDtsQ7PPfcc6enpdOjQgVtuuYWpU6c26XNrM0ARz9TW2mBFVQ1ffHuQpT8cwuF04uNt4YoLuzJmUCwW/SEoBnGHzQANDzlGUsgR8UxttQ0eOlbCnCW7ST9cuzAjNtKf68clacdkMYQ7hBxFfBERD9EpKoCHbhjITROS8Pf1Iiu3lKfnbuSNhTt1urm0SerJUU+OiMdRG4Tisio+SU3nmy1HAPC2mhnVL4bxQ+I0X0dahDv05CjkKOSIeBy1wR+lZxfx/rK9HDhSDIDFbOKCPtFcMiyO9qHtDK5OPJlCjsEUckQ8k9pgfU6nkx0H8lm4OoO9hwoBMJlgeN9oLh/ejZAA9exI83OHkGP4EnIREXEtk8lE727h9O4Wzt5DhXy5OoNt+4/zzZYjrNmZw4QhcUwYGoevt94SxLPo/2gRkTake6cQuncKIS2rkI9W7CP9sI0vvjvI15sPc/mFXRmeEq1l5+IxNFyl4SoRj6M22DBOp5MNe3L5+Ot0jhWWA9Axwp+rLkqgT7ewJh2ALOIOw1UKOQo5Ih5HbfDc1NgdrNyUzRffHqC0ogaAXl1CuWp0Ip2iAgyuTlorhRyDKeSIeCa1wcYprajmy+8zWLbhEDV2Jybggr7RXDG8m5adyzlTyDGYQo6IZ1IbbJpjheV88nU6P+w+BtTusTN+cO3kZD8fTeWUhlHIMZhCjohnUhtsHvuyi/hoxT72ZRcBEOTvzeQLuzK8bzReDXyTkbZLIcdgCjkinkltsPk4nU427s1l/tfpHCuonZwcEezLpAu6cH7vDlqJJWekkGMwhRwRz6Q22Pxq7A5SNx9m4fcHKTpxDlb7UD9+dWFXhvZsj9mslVhSn0KOwRRyRDyT2qDrVFbbWbkxm0VrMigprwYgOrwdky7owpAeCjvyI4UcgynkiHgmtUHXK6+sYfmGLL5al1m37Dw6vB2Tzu/CEPXsCAo5hlPIEfFMaoMtp7yyhmUbslj6s7Az+cKuDOoRhVkbCrZZCjkGU8gR8Uxqgy3vdGEnLiqAy0d0IyU+XLsnt0EKOQZTyBHxTGqDximrqGHpD5ks/eEQFVV2AOI7BjHpgi706aaw05Yo5BhMIUfEM6kNGq+kvJrFazJYviGLqhM/g9jIAC4dFsfgnlFaet4GKOQYTCFHxDOpDbqPopJKvlp3iJWbs6k80bMTEezL+CFxDO8bjbfVYnCF4ioKOQZTyBHxTGqD7qe0opoVG7NZtv4QxWW1S8+D2lm5eHAnLuofSztfHRfhaRRyDKaQI+KZ1AbdV1W1nVVbj/DVukzyiioA8POxMKp/DOMGxxHs721whdJcFHIMppAj4pnUBt1fjd3Bul05LFqTyeG82t/DXhYzw1OiuWRIHBEhfgZXKE2lkGMwhRwRz6Q22Ho4nE62pOXx5ZoM9h+2AWA2mRiaHMUlwzoTGxlgcIXSWAo5BlPIEfFMaoOtj9PpZHdmIYtWH2THwYK6x3t3C2P84DiSu4Rq+Xkr4w4hRzO9RETEcCaTiZ6dQ+nZOZQDR2wsXpPBhr25bN+fz/b9+cRG+jNucBxDk6OwemlFljSMenLUkyPicdQGPcOxwnKWrT/Eqi1HqKyuXX4e2M7KiJSOXNQ/hrAgX4MrlF/iDj05CjkKOSIeR23Qs5RVVJO65TDLN2SRb6sEauftDOgewdhBnUiMDdZQlhtSyDGYQo6IZ1Ib9Ex2h4PNaXksW5/FnkOFdY/HRQUwZlAsw5LbayjLjSjkGEwhR8QzqQ16vkPHSli+4RCrd+RQfeJnHOBnZWS/jozqF0N4sIayjKaQYzCFHBHPpDbYdpSUV7Nqy2FWbMzi+ImhLJMJUuIjuGhADL26hmHWUJYhFHIMppAj4pnUBtuek0NZyzdksTuzsO7xyBBfRvaL4YLeHQgO8DGuwDZIIcdgCjkinkltsG07cryUlZuy+W7bUcora4DaicopCeGMSOlI725hOgW9BSjkGEwhR8QzqQ0KQGW1nXU7c/hm62HSs211j4cEeHNh346M6But4yNcSCHHYAo5Ip5JbVB+LjuvlFVbDvP99qOUlNeegm4CkruGMTKlI/0SI/Bq4BunNIxCjsEUckQ8k9qgnEl1jYNNabl8s+UwO39yfERgOyvn9+7A8L4d6Rjhb2CFnkMhx2AKOSKeSW1QGuJYYTmrthzm261HKCqtqns8vmMQw1M6MrhHFH4+Ov2osRRyDKaQI+KZ1AblXNTYHWzbf5xvtx5hy77jOE68LVq9zAzoHskFvTuQ3CUMs1lL0c+FQo7BFHJEPJPaoDRWUUkl328/yrfbjnDkeFnd48EB3pyX3IFhvdrTKSpAx0g0gEKOwRRyRDyT2qA0ldPp5ODRYr7fdpS1u3LqJisDdIzwZ1hye4Ylt9fqrF+gkGMwhRwRz6Q2KM2pxu5ga/px1uw4yuZ9x6mx//j/VGJsMOf16sCgHlEE+FkNrNL9KOQYTCFHxDOpDYqrlFVUs2FPLmt25rA7o4CTb6AWs4m+8eEM69WBvvHh+Fh1UKhCjsEUckQ8k9qgtIR8WwXrdh1j9Y6jHDpWUve4t9VMSnwEg3tE0acNBx6FHIMp5Ih4JrVBaWlZx0pYvfMoP+w6Rl5RRd3jPlYLfeLDGZQUSd/4cHy9286SdIUcgynkiHgmtUExyskJyz/sPsb63fUDj9XLTJ9u4QxMiiQlPpx2vp49h0chx2AKOSKeSW1Q3MHJwLNhTy7rdx/jWGF53XMWs4mkuBD6J0bSPzGCsCBfAyt1DYUcgynkiHgmtUFxN06nk0PHSli/5xib9uaRnVf/vadLh0D6d49kQGIEHSP8PWIfHoUcgynkiHgmtUFxdzn5ZWxKy2NjWi7pWUX89I04KsSP/t0j6J8YSXxMEBZz6zw4VCGngT799FMeeuihUx6//fbbue+++xp9X4UcEc+kNiitSVFpFVv25bFxby47DxbU24cnwM9K3/hw+idGkNwlrFWdpeUOIaf1fLeA2bNnExgYWPdx+/btDaxGRESk6YL9vRmR0pERKR0pr6xh+4F8NqflsjX9OCXl1Xy//Sjfbz+Kl8VEUlwoKfHh9E2IIEq7LZ9Vqwo5vXr1IiwszOgyREREXMLPx4vBPaIY3CMKu8PBvqwiNqXlsTktj2OF5ew4kM+OA/m8vyyN6PB2pMRHkJIQTnxMMF4N7N1oS1pVyBEREWkrLGYzSXGhJMWFcvXoBI7ml7Fl33G27MsjLauII8fLOHI8kyXrMvHz8aJ31zD6dAunV9cwQgN9jC7fLbSqkDNx4kQKCgro2LEjV111FdOmTcNiaZs7SYqISNthMpmIDvcnOtyfCUPjKK2oZseBfLbsy2Pb/nxKyqv5Yfcxfth9DIDYSH96dw2nV7cwuscGY/Vqm++VrWLi8apVq9iyZQspKSmYTCZWrFjBvHnzuPbaa3nssccafV+73YHNVn72C8+RxWImKMgPm60cu12THkVamtqgtCUOh5P0w0Vs2Xec7fuPc+Cwrd5qLauXmaROIfTqGkavbmHEtQ/E3AJL1F3VDoOC/DxrddXp/POf/+Sdd97h66+/JioqqlH3cDqdHrEXgYiIyElFJZVs3pvLxj3H2Lz3GPm2ynrPBwd4k5IYSf/ukfTrHkWEB09gbrUhZ+vWrVx55ZXMmjWLkSNHNuoe6skR8UxqgyK1nE4nh/NK2X5iwvLujAIqquz1rukY4U9yl1B6dQ2jR+dQ/JvpuAl36MlpVXNyXMGVe2jY7Q7t0SFiILVBEWgf2o72oe0YMyCWGruD9OwidhwsYOfBfA4csXE4r5TDeaUsW5+FyVS7+3KPuFB6dA4lISa4yXvzGNkOW23IWbRoERaLheTkZKNLERERaRW8LD+u2Pr1iG6UlFezJ7OAnRkF7M4o4MjxMg4cKebAkWIWr83EbDLRNTqQ7nEhJHUKJTG26aGnJbWKSm+77TaGDRtG9+7dAVi+fDkfffQRN954I5GRkQZXJyIi0joF+FkZmBTFwKTaua0FxZXsyshnd2YhuzMKyCuqIP2wjfTDNhavycRkgs7tA0mKC6F7pxASY0MI8HPf09RbxZycJ598klWrVnH06FEcDgddunThyiuv5IYbbmjSxGEd6yDimdQGRZpHXlE5uzMK2XOogD2ZheQVVZxyTWykP4mdQkg6EXpO7tHjDsc6tIqQ4yoKOSKeSW1QxDXybRXsOVTInsxC0rIKOXK87JRrokL96N4phJ6dQxl/QTfKSioUcoygkCPimdQGRVqGrbSKtKza0LM3q5BDOSX19ugZOziOG8d31wGdIiIi0roE+XvXm9NTVlHDvuwi9h4qJCu3hCG9Ohhan0KOiIiINIt2vl70jQ+nb3x4vR5Vo+jIUhEREfFICjkiIiLikRRyRERExCMp5IiIiIhHUsgRERERj6SQIyIiIh5JIUdEREQ8kkKOiIiIeCSFHBEREfFICjkiIiLikRRyRERExCMp5IiIiIhHUsgRERERj6SQIyIiIh7J5HQ6nUYXYRSn04nD4Zov32IxY7c7XHJvETk7tUER47miHZrNJkwmU4OubdMhR0RERDyXhqtERETEIynkiIiIiEdSyBERERGPpJAjIiIiHkkhR0RERDySQo6IiIh4JIUcERER8UgKOSIiIuKRFHJERETEIynkiIiIiEdSyBERERGPpJAjIiIiHsnL6ALkR3/5y1/4+OOP2bFjB15e+tGItKTf/va3HDlyBJPJREREBE8++STR0dFGlyXSZhQUFHD//feTlZWF1Wqlb9++/PWvf8Xb27vR91RPjptYvXo1drvd6DJE2qxnn32WL774gs8//5yRI0fy7LPPGl2SSJtiMpn47W9/y5IlS/j8888pLy9nzpw5TbqnQs5pZGRk8NhjjzF58mSSk5OZOHHiaa87cOAAt912G/369eO8887jySefpKKi4pw/X1lZGc899xwPPvhgU0sX8Rgt3Q4DAwPr/rukpKTRdYt4kpZshyEhIQwePBgAs9lM7969OXz4cJPq15jIaaSlpZGamkpKSgoOhwOn03nKNTabjZtuuomOHTvy0ksvkZ+fz1NPPUVhYeE5/wX43HPPceONNxIaGtpcX4JIq9fS7RDgz3/+M2vXriU4OJi33nqrOb4MkVbNiHYIUFlZyWeffcb999/fpPoVck5j9OjRjB07FoAZM2awffv2U6754IMPsNls/Pe//yUsLAwAi8XCfffdx1133UV8fDwA1157LTk5Oae8Pj4+ntdff52NGzeSmZnJI4884sKvSKT1acl2eNK///1vnE4ns2fP5pVXXuFvf/ubC74ykdbDiHbocDh48MEHGTZsGCNGjGhS/Qo5p2E2n30U75tvvuG8886r+4ECjB8/nocffpjU1NS6H+q8efN+8T7r169n586djB49uu6xcePG8f7779OhQ4dGfgUirV9LtsOfMplMXH311YwcOVIhR9o8I9rh448/jpeXFw8//HDjiv4JzclppPT09Lof3Ene3t7ExcWRnp7e4PvccccdfPvtt6xYsYIVK1YAsHTpUgUckQZornZYWlrK0aNH6z5esmQJiYmJzVaniCdrrnYI8K9//YucnByefvppTCZTk2tTT04j2Ww2goKCTnk8KCiIoqIiAyoSaXuaqx2Wl5dzzz33UFlZCUB0dDTPPPNMs9Up4smaqx2mpaXxxhtv0K1bN6ZMmQLA+eef36RFOQo5zczpdDYpfe7Zs6cZqxFpm861HUZERPDxxx+7sCKRtudc22FiYmKzvwdquKqRgoKCsNlspzxeXFx82kQrIs1P7VDEeO7cDhVyGik+Pv6UscaqqioyMzNPGZsUEddQOxQxnju3Q4WcRhoxYgRr1qyhoKCg7rH//e9/VFVVMXLkSAMrE2k71A5FjOfO7VBzck6jvLyc1NRUALKzsykpKWHJkiUADBkyhLCwMK655hree+897r77bu6++26OHz/O008/zaRJkwxPriKeQO1QxHitvR2anKfbvrCNy8rKYsyYMad9bs6cOQwdOhSo3cb6ySefZMOGDfj6+jJx4kTuu+8+fH19W7JcEY+kdihivNbeDhVyRERExCNpTo6IiIh4JIUcERER8UgKOSIiIuKRFHJERETEIynkiIiIiEdSyBERERGPpJAjIiIiHkkhR0RERDySQo6IiIh4JIUckTbs008/JSkpiW3bthldyjm74YYbuOGGG4wuo9EWLFjA22+/bXQZIh5NB3SKSKv017/+1egSmmThwoWkpaVx8803G12KiMdSyBERwzmdTiorK8/pML+EhAQXVnTuKioqDD+MUETqU8gRkbM6ePAgM2fO5Pvvv6e4uJhOnTpx/fXXM3Xq1LprKisref7551m9ejVZWVlYLBa6du3K7bffztixY+vdLykpialTp5KYmMicOXM4dOgQf/nLX/Dx8eGhhx7inXfeYfHixSxZsgSAwYMH8+ijj9K+ffu6e5wcqnr33XeBH09LfuCBBzCbzbz77rsUFBTQvXt3HnroIfr161evho8++og33niD7Oxs4uLiuOuuu/juu+9Yt24dK1as+MXvx+jRo0lMTGTKlCm88sorpKenc9NNN3Hfffcxd+5cFi1axP79+ykvL6dTp0786le/4uabb8ZqtdbVvm7durrvxUl79uwBoKqqitmzZ/PFF1+QlZVFQEAAF110Effffz9hYWEN/rmJtHUKOSLyi/bt28c111xDdHQ0Dz74IJGRkXz77bc8+eSTFBQUcM899wC1b8xFRUXceuuttG/fnurqar7//nvuvfdennrqKS6//PJ69122bBnr169n+vTpREREEB4eXjc36JFHHmHUqFH8+9//5siRIzzzzDPcf//9zJkz56z1zp07l27duvHwww8D8OKLL3LHHXewfPlyAgMDAfjwww957LHHGD9+PA899BDFxcX85z//obq6usHflx07dpCens5dd91FbGwsfn5+AGRmZjJx4kRiY2OxWq3s3r2b1157jf379/PUU08BtUNtjz76KIcOHeI///lPvfs6HA7uvvtuNmzYwG233caAAQPIzs5m5syZbN26lU8++UQ9RiINpJAjIr/oqaeewt/fn3nz5hEQEADABRdcQFVVFbNmzeKGG24gODiYwMDAujdxALvdznnnnYfNZuOdd945JeSUlZWxYMECgoOD6x47GXKGDx/OI488Uvd4UVERzzzzDLm5uURGRv5ivf7+/vzf//0fFosFgKioKK688kq++eYbLrvsMhwOBzNnziQlJYWXXnqp7nUDBw5k3LhxREVFNej7kp+fz5dffknXrl3rPf7QQw/V/bfD4WDQoEGEhITw8MMPM2PGDIKDg0lISCAoKAhvb+9TepgWL17MqlWrmDlzJuPGjat7vEePHvzmN7/h008/5brrrmtQjSJtnUKOiJxRZWUla9as4dprr8XX15eampq650aMGMF7773H5s2bGTlyJFD7Bv3OO++wZ88eysrK6q718fE55d7Dhg2rF3B+avTo0fU+Pjmkc/jw4bOGnFGjRtUFHKgNBwDZ2dkAHDhwgNzcXG677bZ6r+vYsSP9+/evu+5skpKSTgk4ADt37uSll15i06ZNFBYW1nvu4MGDpKSk/OJ9V65cSVBQEBdddFG973fPnj2JjIxk3bp1CjkiDaSQIyJnVFhYSE1NDe+++27d3JefKygoAGDp0qX84Q9/YMKECUybNo2IiAgsFgvz5s3jk08+OeV1vxRWQkJC6n3s7e0N1E7uPZszvbaysrJeveHh4ae8NiIiosEh53T1Hz58mKlTp9K1a1cefvhhYmJi8PHxYevWrTzxxBMNqv/48ePYbDZ69+592udP1i8iZ6eQIyJnFBQUhMViYfLkyWfsPYiNjQXgiy++IDY2lhdeeAGTyVT3/DvvvHPa1/30mpYUGhoK1IaJn8vLy2vwfU5X/7JlyygrK2PmzJnExMTUPb579+5zqi8kJITZs2ef9nl/f/8G30ukrVPIEZEz8vPzY+jQoezcuZOkpKS6XpHTMZlMWK3Wem/+ubm5LF++vCVKbbCuXbsSGRnJ4sWLueWWW+oeP3z4MJs2bWrwnJzTOfm1//T75HQ6+eijj0651tvb+7Q9O6NGjeLLL7/E4XCcdWhLRH6ZQo6IsGbNmtMO04wcOZK//OUvXHfddUydOpVrr72WmJgYSktLyczMZMWKFXUrnkaNGsXSpUv529/+xvjx4zl69CivvPIKUVFRHDx4sIW/ojMzm83ce++9PPbYY/zud79jypQp2Gw2/vOf/xAZGdmkHqbzzz8fq9XKn/70J6ZNm0ZVVRXz5s3DZrOdcm337t1ZunQp77//Pr1798ZkMtGnTx8uu+wyFixYwB133MENN9xA3759sVqtHD16lLVr1zJmzBguvvjipnwLRNoMhRwR4dlnnz3t48uXLychIYFPP/2UV155hRdeeIH8/HwCAwPp3Llz3YRjgClTpnD8+HE++OADPvnkEzp16sQdd9zB0aNHT1kmbbSrr74ak8nE7NmzmT59OjExMXXLzI8cOdLo+8bHxzNz5kxeeOEF7r33XkJCQpg4cSI333wzt99+e71rb7zxRtLS0nj++ecpLi7G6XSyZ88eLBYLr776KnPmzOHzzz9n1qxZWCwWOnTowODBg+nevXtTv3yRNsPkdDqdRhchImI0m83G+PHjGTt2LP/v//0/o8sRkWagnhwRaXNyc3N57bXXGDp0KCEhIRw+fJi3336b0tJSbrzxRqPLE5FmopAjIm2Ot7c32dnZPP744xQVFeHr60tKSgqPP/44iYmJRpcnIs1Ew1UiIiLikcxGFyAiIiLiCgo5IiIi4pEUckRERMQjKeSIiIiIR1LIEREREY+kkCMiIiIeSSFHREREPJJCjoiIiHgkhRwRERHxSP8fU41QP9UIsuwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = FirstFC()\n",
    "\n",
    "trainer = pl.Trainer()\n",
    "\n",
    "# Run learning rate finder\n",
    "lr_finder = trainer.tuner.lr_find(model, train_dl, val_dl, \n",
    "                                  min_lr=1e-4, max_lr=1e-2)\n",
    "\n",
    "fig = lr_finder.plot(suggest=True)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7d9c9752",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR suggested is: 0.0003467368504525317\n"
     ]
    }
   ],
   "source": [
    "print(f\"LR suggested is: {lr_finder.suggestion()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c73019ac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch_onoci_v1_0]",
   "language": "python",
   "name": "conda-env-pytorch_onoci_v1_0-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}