Part 1 - Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from jcopdl.callback import Callback, set_config\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset & Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.dataset import AudioFolder\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c8f246436db49b8baa811a199111ba6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Create Data', max=500.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ad28fc81e9349f383a621dd2d1c27c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Create Data', max=200.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "sr = 16000\n",
    "train_set = AudioFolder(\"dataset/train/\", sr=sr, n_data=500, slice_dur=3.3)\n",
    "trainloader = DataLoader(train_set, batch_size=128, shuffle=True, num_workers=2)\n",
    "\n",
    "test_set = AudioFolder(\"dataset/valid/\", sr=sr, n_data=200, slice_dur=2.4)\n",
    "testloader = DataLoader(test_set, batch_size=128, num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Architecture & Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.model import SpeakerDiarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = set_config({\n",
    "    \"n_input\": train_set.n_mel,\n",
    "    \"n_output\": len(train_set.classes),\n",
    "    \"n_hidden\": 256,\n",
    "    \"n_layer\": 5,\n",
    "    \"dropout\": 0.2,\n",
    "    \"audio_params\": train_set.audio_params,\n",
    "    \"classes\": train_set.classes\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Preparation -> MCOC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SpeakerDiarization(config.n_input, config.n_output, config.n_hidden, config.n_layer, config.dropout).to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.AdamW(model.parameters(), lr=0.001)\n",
    "callback = Callback(model, config, outdir=\"model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89bdc712a2a5479f9ab2f44309834e19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5baa95efd0854c548d640c1e7ce24ce3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     1\n",
      "Train_cost  = 0.9493 | Test_cost  = 0.6919 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3042201a330647c0b1b53be02bba1a21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99236ddc27354d63836631ec43d4f9d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     2\n",
      "Train_cost  = 0.4813 | Test_cost  = 0.3747 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c6bcbec9b354cf1b4bede8d34a9b654",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a17753c4e824422cae142205d1f3f71d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     3\n",
      "Train_cost  = 0.2825 | Test_cost  = 0.2378 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72cc76d6df5a4245882676c867190cfb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bc558c9ffd84d65a62285d3d1260abd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     4\n",
      "Train_cost  = 0.1834 | Test_cost  = 0.1420 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33cef353fb6245fca3d3a2f9b574246f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4fe530ce18f47df9cebbda727a0bc2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     5\n",
      "Train_cost  = 0.1092 | Test_cost  = 0.1166 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7dc9aa81002455fabb081dfb0fb0992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75fde157a5ec4b129d4d577e8204a274",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     6\n",
      "Train_cost  = 0.0727 | Test_cost  = 0.0741 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2f4f8c9adca4990a9c8085d719510f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7cab25128bb407d9d3d6e37febceba4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     7\n",
      "Train_cost  = 0.0524 | Test_cost  = 0.0568 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18083b31a1d34e319fd542dd21d838ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0474cc5a16824ff99ef1f4af99c4baa5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     8\n",
      "Train_cost  = 0.0417 | Test_cost  = 0.0514 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61cf730c47b3423185e0dfde5f2d7a00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aa4f322c62b43b9bf6abb36b40fc0f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch     9\n",
      "Train_cost  = 0.0338 | Test_cost  = 0.0400 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2dbd3ee62c7469382190b90e4271095",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec155e016ecb46ffbec8147ca754afd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    10\n",
      "Train_cost  = 0.0315 | Test_cost  = 0.0510 | \n",
      "\u001b[31m==> EarlyStop patience =  1 | Best test_cost: 0.0400\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04ff3fe7a06245f6b65a5701cfefeb3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23c9a18ed29d44bebeba3e5e559d31f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    11\n",
      "Train_cost  = 0.0258 | Test_cost  = 0.0348 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45bd830e7d8149d49e56538f18f7477b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3db4ee9efd494a03a40eba7490692fc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    12\n",
      "Train_cost  = 0.0230 | Test_cost  = 0.0400 | \n",
      "\u001b[31m==> EarlyStop patience =  1 | Best test_cost: 0.0348\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "deb6f71f4ad3450d81cb9d789ca03ee8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1f7a266f8064b56b1c383d23a5dfd25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    13\n",
      "Train_cost  = 0.0206 | Test_cost  = 0.0335 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5eb0f92d128f40cfb14b3f7b9c51c532",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5dec21ee24249be8a6bf9372b0e69c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    14\n",
      "Train_cost  = 0.0183 | Test_cost  = 0.0323 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b010741f935426ca03728bf7159af1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5348369ab604a3fb55f1ccac87daad1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    15\n",
      "Train_cost  = 0.0192 | Test_cost  = 0.0365 | \n",
      "\u001b[31m==> EarlyStop patience =  1 | Best test_cost: 0.0323\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b5f1cc10131400e9fcc101f542b151c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc6ad814721248e3b82a5771dd816a47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    16\n",
      "Train_cost  = 0.0187 | Test_cost  = 0.0300 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34371af1ab724dda941d5f30251e49c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f853d3f443b74b119ea6de2acec8d9a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    17\n",
      "Train_cost  = 0.0177 | Test_cost  = 0.0374 | \n",
      "\u001b[31m==> EarlyStop patience =  1 | Best test_cost: 0.0300\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "456d8f6484c7409aa5648ff56bd02e51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd014e4aaf5f4f2da7f563586cb3ecad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    18\n",
      "Train_cost  = 0.0177 | Test_cost  = 0.0491 | \n",
      "\u001b[31m==> EarlyStop patience =  2 | Best test_cost: 0.0300\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bcd64fe600746f1a9b7e0b296034823",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ecb3c8945ae4a5eb4c7447be455d393",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    19\n",
      "Train_cost  = 0.0175 | Test_cost  = 0.0285 | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59747f46f3e349fa938cb4a00690a7a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee4d97d8eece407384b64b6f221a282a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    20\n",
      "Train_cost  = 0.0158 | Test_cost  = 0.0463 | \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAE9CAYAAADj+KBFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZyN5f/H8ddljF0IRShE9ZWyTUW0FxEp2hcl2uRLO6UU7b9W2hXtaaUSJUrxFdmiRCJJ9iXJPmbm+v3xmWmG5szmnHOfc+b9fDzOY2buc517Pvecmfnc93Vf1+dy3ntEREQkMZUIOgARERGJHCV6ERGRBKZELyIiksCU6EVERBKYEr2IiEgCU6IXERFJYCWDDiASqlWr5uvWrRt0GCIiIlExe/bsDd776rk9l5CJvm7dusyaNSvoMERERKLCOfd7qOfUdS8iIpLAlOhFREQSmBK9iIhIAov5e/TOufLAc0Aq8LX3/q2AQxIRkRixe/duVqxYwc6dO4MOJSrKlClD7dq1SU5OLvBrAkn0zrkRQEdgnfe+cY7tZwJDgCTgZe/9w0AX4APv/Rjn3LuAEr2IiACwYsUKKlasSN26dXHOBR1ORHnv2bhxIytWrKBevXoFfl1QXfevAmfm3OCcSwKeBdoDjYCLnXONgNrAH5nN0qMYo4iIxLidO3dStWrVhE/yAM45qlatWujei0ASvfd+MvDnXpuPBZZ475d671OBd4DOwAos2YPGFIiIyF6KQ5LPUpRjjaXEWYvsK3ewBF8LGAV0dc49D4wJ9WLn3DXOuVnOuVnr16+PbKQiIiLAxo0badq0KU2bNqVGjRrUqlXrn69TU1MLtI/u3buzaNGiiMUYS4PxcjtN8d77bUD3/F7svR8GDANISUnxYY5NRETkX6pWrcrcuXMBuPfee6lQoQK33nrrHm2893jvKVEi92vrV155JaIxxtIV/QqgTo6vawOrAool2+TJMG5c0FGIiEgcWbJkCY0bN+a6666jefPmrF69mmuuuYaUlBSOPPJIBg8e/E/bNm3aMHfuXNLS0qhcuTL9+/enSZMmtGrVinXr1u1zLLGU6GcCDZ1z9ZxzpYCLgE8Cjch7uO02uP562LUr0FBERCS+LFiwgB49evD9999Tq1YtHn74YWbNmsW8efOYMGECCxYs+NdrNm/ezEknncS8efNo1aoVI0aM2Oc4gppeNxI4GajmnFsB3OO9H+6c6w2Mx6bXjfDe/xREfP9wDu6/H9q2hZdegt69Aw1HRETycOONkNmNHjZNm8JTTxXppYceeijHHHPMP1+PHDmS4cOHk5aWxqpVq1iwYAGNGjXa4zVly5alffv2ALRo0YIpU6YUPfZMgSR67/3FIbaPA2Krn/z00+Hkky3hd+8O5csHHZGIiMSB8jnyxeLFixkyZAgzZsygcuXKXHbZZblOkytVqtQ/nyclJZGWlrbPccTSYLzY5Bw88AC0bg3PPAP9+gUdkYiI5KaIV97R8Pfff1OxYkX2228/Vq9ezfjx4znzzDPzf2EYxNI9+n3mnOvknBu2efPm8O74+OOhQwd45BEI975FRCThNW/enEaNGtG4cWOuvvpqWrduHbXv7bxPvJloKSkpPuzr0X//PTRvDnffDTlGS4qISHAWLlzIf/7zn6DDiKrcjtk5N9t7n5Jb+4S6oo+oZs3g/PPhySdBBXlERCROKNEXxuDBsH27deGLiIjEASX6wjjiCOjWzQblrVwZdDQiIiL5UqIvrHvugYwMm24nIiIS45ToC6tuXbj6anj5ZVi6NOhoRERE8pRQiT5i0+v2NmAAlCwJ994b2e8jIiKyjxIq0Xvvx3jvr6lUqVJkv9FBB8F//wtvvgm51CoWEZHiIRzL1AKMGDGCNWvWRCTGhEr0UdWvH1SoAAMHBh2JiIgEJGuZ2rlz53Lddddx0003/fN1znK2+VGij0VVq8LNN8OHH8Ls2UFHIyIiMea1117j2GOPpWnTpvTq1YuMjAzS0tK4/PLLOeqoo2jcuDFDhw7l3XffZe7cuVx44YWF7gkoCCX6fXHzzbD//nDXXUFHIiIiMWT+/PmMHj2ab7/99p+15t955x1mz57Nhg0b+PHHH5k/fz7dunX7J8FnJfzC9AQUhBa12Rf77Qf9+8Ptt8OUKXDCCUFHJCJSbMXSKrUTJ05k5syZpKRYVdodO3ZQp04d2rVrx6JFi+jbty8dOnSgbdu24Q04F7qi31c33AA1athI/ARcN0BERArPe89VV131z/36RYsWcffdd1O1alV++OEH2rRpw9ChQ7n22msjHouu6POxcCH89Re0ahWiQblyttDNDTfAF19Au3ZRjU9EREwsrVJ7+umnc95559G3b1+qVavGxo0b2bZtG2XLlqVMmTKcf/751KtXj+uuuw6AihUrsmXLlojEklCJ3jnXCejUoEGDsOzPe7jwQti5E+bPh5C3TXr2hEcftav6tm1tDXsRESm2jjrqKO655x5OP/10MjIySE5O5oUXXiApKYkePXrgvcc5xyOZa6d0796dnj17UrZsWWbMmBHW+/RapjYfn31mS9E/9hjcckseDV99Fbp3t1H4XbqE5XuLiEjetEyt0TK1+6B9e0v0gwfD2rV5NLzsMjj8cOvGT0+PWnwiIiJ5UaIvgCeesNVp85xFV7Ik3HefVcobOTJqsYmIiORFib4ADj/cKt4OHw7ff59Hw65dbS7GPffA7t1Ri09ERCQUJfoCGjjQiuH17ZvHLLoSJWz52qVLYcSIqMYnIlJcJeJYs1CKcqxK9AVUuTI88IDVxXn//TwadugAxx9v3fg7dkQtPhGR4qhMmTJs3LixWCR77z0bN26kTJkyhXqdRt0XQno6tGgBmzbBzz9D2bIhGn79NZxyCjz+uJXJFRGRiNi9ezcrVqxg586dQYcSFWXKlKF27dokJyfvsT2vUfdK9IWUlcMHD7YB9iGdcYbVYly6FCpWjEgsIiIiUIym1znnOjnnhm3evDli3+Pkk23M3cMPw4oVeTR84AHYsAGGDIlYLCIiIvlJqETvvR/jvb+mUqVKEf0+jz5q3fj9++fR6NhjoXNna/znnxGNR0REJJSESvTRUq8e3HorvPUWTJuWR8P77oMtWyzZi4iIBECJvoj694eaNW26XUZGiEZHHQUXXQRDh8KaNVGNT0REBJToi6xCBXjkEZg5E954I4+GgwbBrl3w4INRi01ERCSLEv0+uPRSOO44u7oPubpgw4a22M2LL8Ly5VGNT0RERIl+H5QoYYPq16yBhx7Ko2HWPLzBg6MSl4iISBYl+n103HFw+eVWG2fp0hCNDj4Yrr/elrL95ZdohiciIsWcEn0YPPQQJCfbSPyQ7rgDSpe2BW9ERESiRIk+DGrVsjw+ejR89VWIRgceaEP033kH5s2LanwiIlJ8KdGHyc03Q926cOONkJYWotFtt0GlSvnUzhUREQmfhEr00SiBG0rZsvDYY/Djj/DSSyEaValiyX7MGJg+ParxiYhI8aRFbcLIe1vwZv58WLzY8vq/bN0K9etbMZ0vv4x6jCIikniKzaI2QXMOnnrKlrEdNChEowoVYMAAu5mf58L2IiIi+06JPsyaNoWrr4ZnnoGFC0M0uuEGW9j+hhtg/fqoxiciIsWLEn0E3HefXbjfdJN15/9LyZI2p/6vv+C//412eCIiUowo0UdA9eo2XX78eBg3LkSjxo2t0bvvwocfRjU+EREpPjQYL0JSU+Hoo21lu/nzoVSpXBrt3g0tW8KKFfDTT1CtWtTjFBGR+KfBeAEoVQqeeMJG3z/zTIhGycnwyis2eq9Pn6jGJyIixYMSfQR16ADt29sI/HXrQjQ6+mgroDNypJXWExERCSMl+gh74gnYvh3uuiuPRv3723D966+HjRujFpuIiCQ+JfoIO+II6N0bXn4Z5s4N0Sg52Ubhb9xo9fBFRETCRIk+CgYOhKpVLYeHHPvYpIld9r/1Fnz8cVTjExGRxKVEHwVVqtjc+smT85lJd8cdlvCvuw7+/DNq8YmISOJSoo+Sq6+2cXe33go7doRoVKqUjcLfsMGWwRMREdlHCZXog1y9Lj9JSVYH//ff4fHH82jYrJld2b/xhq1yJyIisg9UMCfKunaFzz+HX36BWrVCNEpNhZQUu7L/6acQy+CJiIgYFcyJIY8+CunpNqMupKwu/HXrrGC+iIhIESnRR1n9+nDLLfDmm/kMrm/Rws4GXnsNxo6NWnwiIpJY1HUfgJ07oU0bK487axY0bBii4a5dlvA3bbIu/MqVoxqniIjEB3Xdx5gyZWyaXcmSds9+27YQDUuXtkI6a9fCzTdHM0QREUkQSvQBOeQQePttW9nuuuvyKKSTkgK332737D/7LKoxiohI/FOiD1C7dnDvvXa//vnn82h4zz3QqJFNxo/BqYMiIhK7lOgDdtddtsrdjTfC9OkhGpUubVf0q1fbSD4REZECUqIPWIkSdkVfuzacfz6sXx+i4bHHwm23wfDhMH58VGMUEZH4pUQfA6pUscF5GzbARRdBWlqIhvfea8vh9ewJf/8dzRBFRCROKdHHiGbN4Lnn4Kuv4O67QzQqU8a68Fetsqt7ERGRfCjRx5Du3W283cMP51FMp2VLu08/bBhMnBjV+EREJP6oYE6M2bkTTjjBauGHLKazY4d1AezYYfPzKlaMepwiIhI7VDAnjpQpAx98kE8xnbJlrQv/jz9sjr2IiEgISvQx6JBDYOTIfIrptGpl1fJeeAG+/DLqMYqISHxQoo9RbdvCoEH5FNO57z447DDo0QO2bIlqfCIiEh8SKtE75zo554ZtTpDqcQMGwFln5VFMp2xZGDECli/PZ91bEREprhIq0Xvvx3jvr6lUqVLQoYRFiRLwxhtQpw6cd54tT/8vrVvbmcBzz8GkSVGPUUREYltCJfpElFVMZ+NGuPjiEMV07r8fGjSwLvytW6Meo4iIxC4l+jjQtKndpw9ZTKdcORuFv2xZPkvhiYhIcaNEHyeuvBKuuSaPYjpt2tjgvLfegscei3Z4IiISo5To48iQIbY8fbdusHhxLg3uvBMuuAD69YNx46Ien4iIxB4l+jiSVUwnORm6dMmlmI5zNgq/aVO7ob9wYSBxiohI7FCijzOHHAJvvw0//QTXXpvL7fjy5eGjj+ysoHNn2LQpkDhFRCQ2KNHHobZtYfBgux3/3HO5NDj4YBg1ygbnXXhhHuveiohIolOij1N33gkdO8JNN4UoptO6tQ3VnzBB9fBFRIoxJfo4VaIEvP56PsV0evSAPn3gySfh1VejHaKIiMQAJfo4lrOYzkUXheihf/xxOO00u6E/bVrUYxQRkWAp0ce5pk1tAbtJk+Cuu3JpULIkvPuuXfqfey6sWBH1GEVEJDhK9Angiivsgv2RR2DMmFwaVK1qVXa2b4dzzoEdO6Ieo4iIBEOJPkEMGQLNmtlt+Vzv1x95pA3TnzPHGqlMrohIsaBEnyBKl7bBeZs351HuvlMneOABGDnSLv9FRCThKdEnkMaNLY+PHg1vvhmiUf/+NnLvzjvh00+jGp+IiESfEn2CuekmOOEE6N0b/vgjlwbOwfDh1s9/ySWwYEHUYxQRkehRok8wSUk2ZT49Hbp3h4yMXBqVK2dlcsuVg7PPhj//jHaYIiISJUr0Cah+fauR8+WXIUrkgk23Gz3aLvtVJldEJGEp0Seonj2hfXurfvvLLyEatWplk/AnToRbb41qfCIiEh1K9Akq61Z82bK2fn3IC/bu3eHGG21+3ogRUY1RREQiT4k+gdWsaV33332Xz2y6Rx+FM86weXlTp0YtPhERiTwl+gR34YU2m+7ee+H770M0yiqTe8gh0KULLF8ezRBFRCSCEirRO+c6OeeGbd68OehQYsqzz0L16taFv3NniEZVqsAnn1h53HPOsXK5IiIS9xIq0Xvvx3jvr6lUqVLQocSU/fe3+/Xz58PAgXk0/M9/rGre3Llw1VUqkysikgASKtFLaO3b28I3jz0GU6bk0fCss+Chh6wr/6GHohafiIhEhhJ9MfLYY1Cvnq12t2VLHg1vv92q5g0YYN35IiISt5Toi5EKFeC112DZsnymzTsHL78MKSlw6aXw00/RClFERMJMib6YadMGbrsNhg2DcePyaFi2rJXJrVABzj0Xtm2LWowiIhI+SvTF0ODBttJdz56wcWMeDWvVgrffhsWL4Y47ohafiIiEjxJ9MVS6NLzxBmzYADfckE/jU06BPn3g6aeteL6IiMQVJfpiqmlTuOceG1z/zjv5NH7oITjsMCuXqxoFIiJxRYm+GOvXD447Dnr1glWr8mhYrhy8/jqsXGl18UVEJG4o0RdjJUta/t65E3r0yKc+znHH2X36V1/VlDsRkTiiRF/MHXYY/N//weefw0sv5dN44EBo0gSuvtpu8IuISMxTohd69YLTT4ebb4Zff82jYalS1gWwaRNcf71K5IqIxAEleqFECVuKvmRJq5qXnp5H46OPtvl5H3xgdfFFRCSmKdELAHXq2Ay6qVPhiSfyaXzbbdCqlc3NW7kyKvGJiEjRKNHLPy67zJajv+su+PHHPBomJVkt3dRUq7qjLnwRkZilRC//cA5eeAEqV7a161NT82jcsGEhRvGJiEhQlOhlD9WrWx38uXPtVnyerr8eTjvNRvEtXRqV+EREpHCU6OVfOneGK6+0gnjTp+fRsEQJeOUV68q/8sp8RvGJiEgQlOglV089BbVrWxf+9u15NKxTB4YOhSlT7EUiIhJTlOglV5UqWRG8xYutat5ff+XRuFs36wYYMEBr14uIxJgCJXrn3BsF2SaJ5ZRTYNAgW/jm0EPtwj3XAXrOwYsvQsWKNhF/9+6oxyoiIrkr6BX9kTm/cM4lAS3CH47EmoEDYc4caNYM+vaFI4+E0aNzmVF34IE2ZH/2bHjwwUBiFRGRf8sz0Tvn7nDObQGOds79nfnYAqwDPo5KhBK4pk1hwgQYO9aq4HbpAieeCDNm7NWwa1e49FK4/35L+CIiErg8E733/iHvfUXgUe/9fpmPit77qt77O6IUo8QA56BDB5g3z3rpf/nFFrS7+GJYtixHw6eftqv7bt1sWTwREQlUQbvuP3XOlQdwzl3mnHvCOXdIBOOSGFWyJFxzDSxZYhX0Pv4YDj8cbr89c8BelSowfDgsWGANREQkUAVN9M8D251zTYDbgd+B1yMWlcS8ihXhvvvsyv7ii+Gxx6BBA7ug331qO7j2WiuaP2VK0KGKiBRrBU30ad57D3QGhnjvhwAVIxeWxIvatW0a3uzZtlR9nz6ZA/ZOfBJft54V0tm6NegwRUSKrYIm+i3OuTuAy4GxmaPukyMXlsSbZs1g4kT49FPr3u9yaVlOqjiHGUurwa23Bh2eiEixVdBEfyGwC7jKe78GqAU8GrGoJC45B2edBT/8YDPtFq2pxHF8xyUvnsiy174JOjwRkWKpQIk+M7m/BVRyznUEdnrvdY9eclWypN2iX7IEBvRLY7TrwhFXHke/vjvzrrAnIiJhV9DKeBcAM4DzgQuA75xz50UyMIl/FSvC/Q+XZPGnv3CRe49Hh5bKHrCn4nkiIlFR0K77AcAx3vsrvPfdgGOBuyMXliSS2h2O5tV7fmM2LWhScx19+sBRR8GKFUFHJiKS+Aqa6Et479fl+HpjIV4rAnfeSbMWSUxc1Ygxr29i5Uo4//wQtfNFRCRsCpqsP3fOjXfOXemcuxIYC4yLXFiScJKT4fXXcdu20vGDK3n1Fc/06XDzzUEHJiKS2PKrdd/AOdfae38b8CJwNNAEmAYMi0J8kkgaNbIFbz75hK5bX+PWW+HZZ+ENrYMoIhIxzv9rGbIcTzr3KXCn9/6HvbanAPd47ztFOL4iSUlJ8bNmzQo6DMlNerqtfztvHmnTZnLGDYfx3XcwfTocfXTQwYmIxCfn3GzvfUpuz+XXdV937yQP4L2fBdQNQ2xS3CQlweuvQ6lSlDy7A+88s4EqVWxFPE29ExEJv/wSfZk8nisbzkBCcc7Vd84Nd859EI3vJ1FQty6MGQMrV3Jgz068/+Yufv/dFrzLyAg6OBGRxJJfop/pnLt6743OuR5AvguOO+dGOOfWOefm77X9TOfcIufcEudc/7z24b1f6r3vkd/3kjjTsiW8+SZ89x3HP3spTz6RwZgx8NBDQQcmIpJYSubz/I3AaOfcpWQn9hSgFHBuAfb/KvAMOVa6y6yT/yxwBrACO5n4BEgC9v43f9Ve0/okkXTtasve3XILNxxyO9MueYy774ZjjoG2bYMOTkQkMeSZ6L33a4HjnXOnAI0zN4/13n9VkJ177yc75+rutflYYIn3fimAc+4doLP3/iGgYyFil0Rw003w22+4Jx5n2OMN+eGHa7nkElsN75BDgg5ORCT+FbTW/STv/dOZjwIl+TzUAv7I8fWKzG25cs5Vdc69ADTLXEEvVLtrnHOznHOz1q9fv48hStQ4B089BZ06Uf62Xozq/RW7d8N558HOnUEHJyIS/4Kobudy2RZyjp/3fqP3/jrv/aGZV/2h2g3z3qd471OqV68elkAlSpKSYORIaN6chjd34vWBS5g1y9a2FxGRfRNEol8B1MnxdW1gVQBxSCwpX95G4levTudH23BHr8289BKMGBF0YCIi8S2IRD8TaOicq+ecKwVcBHwSQBwSa2rUgHHjYOdO7pvUhtNP3k2vXjBnTtCBiYjEr4gmeufcSKxc7uHOuRXOuR7e+zSgNzAeWAi8573/KZJxSBxp1AhGjyZpySLe3n0BBxzg6doV/vwz6MBEROJTRBO99/5i731N732y976293545vZx3vvDMu+7PxDJGCQOnXIKjBhB9akf8UHjQaxa5bn0UhXTEREpCi01K7Hpsstg8GCO/WwQQ8/4lM8/h8GDgw5KRCT+JFSid851cs4N27x5c9ChSDjcdRd07841Y8/mitaLGTQIxo4NOigRkfiS5+p18Uqr1yWQ3buhQwd2TJrO8XVXsWxjRWbPhvr1gw5MRCR27MvqdSLBSk6GDz6gbKN6fLi2DaSn07Ur7NgRdGAiIvFBiV5iX6VKMHYs9Suu583SPZg7F66/HhKwM0pEJOyU6CU+1KkDY8dy1s4PGVhjGK+9BsOGBR2UiEjsU6KX+NGsGbz3HgPX9ebM6rPo08czY0bQQYmIxDYleokv7duT9NzTvLm+HTVL/cl553m0hpGISGgJleg1va6YuPZaqva7mg+3tmXd6nQuuQTS04MOSkQkNiVUovfej/HeX1OpUqWgQ5FIe/BBWlzYkOfSrmHiRBg4MOiARERiU0IleilGSpSAV1/lqjaL6VliBA8+CB9/HHRQIiKxR4le4leZMvDRRzxd/0laJM2l22Xp/Pxz0EGJiMQWJXqJb1WrUubzj/hwv+6U2v4XLY/LYPTooIMSEYkdSvQS/w49lEPGPc+Mkq1pmLqALl3gxhshNTXowEREgqdEL4mhZUvqffYc/0s6if9WfJUhQ+CEE2DZsqADExEJlhK9JI5TT6X0pM8ZWupW3q/Uk58XpNOsGXzySdCBiYgEJ6ESvebRC8ccA5Mnc16Fz5njUqh/4FY6d4Zbb7WF8EREipuESvSaRy8ANGoEU6dyaI1tTP29Dr06/s7jj8NJJ8EffwQdnIhIdCVUohf5xyGHwJQplDmiLs+Ob8g7fb5l/nxo2hTGjQs6OBGR6FGil8R14IHw9dfQsiUXPt2G2beOpE4dOOss6N9fXfkiUjwo0Utiq1QJPv8cOnSg4T2XMK3Lo1xzteeRR+DUU2HlyqADFBGJLCV6SXzlysHo0XDJJZS953Ze3O823nrT8/331pU/fnzQAYqIRI4SvRQPycnwxhvQuzc8/jiXfNWT2d+lUbMmnHkmDBgAaWlBBykiEn4lgw5AJGpKlIChQ2H//WHwYA7ftInpX79Nn9vL8OCDMHUqvP02HHRQ0IGKiISPruileHEOBg2Cp56C0aMpd/5ZvPzkFl5/HWbOtK78iRODDlJEJHwSKtGrYI4UWN++8Npr8M03cNppXN5hIzNnQvXq0LYt3HMPpKcHHaSIyL5LqESvgjlSKN26wahR8MMPcOKJNKq0khkzbPPgwZbw16wJOkgRkX2TUIlepNDOPtum3/3xB7RuTflVi3n1VXjlFZg2zbryP/4Ydu0KOlARkaJRohc5+WSYNAm2bYM2bWDuXK68EmbMgCpV4JxzoGpV6NQJnnkGliwJOmARkYJTohcBaNECpkyBUqUs8f/vfzRuDN9/b6vfXXEFLFwI//0vNGwIhx4KvXrZc1u2BB28iEhoznsfdAxhl5KS4mfNmhV0GBKPli+HM86wrvwPPoAOHfZ4eskSK7Azfjx89ZV1AiQnw/HH23z8du2gSRObySciEi3Oudne+5Rcn1OiF9nLunXQvr0N0nv9dbj44lybpabCt9/aLf7x42HuXNt+wAGW8Nu1s3OGAw6IYuwiUiwp0YsU1t9/20C9yZPhySehTx+bg5+HNWvgiy8s6X/xBWzYYNtbtMhO/K1aWQ+AiEg4KdGLFMWOHXY1//HHcNppMGwY1K9foJdmZMCcOdnd/N9+a/PyK1a0XXXoYNP4SpeO8DGISLGQV6LXnUSRUMqWtXn2L7xgQ/AbN7ar+wJU0ilRAlJSrIb+5MmwcaPt6pJLrIv/mmvguONgwYIoHEcRpaXBgw/CRRfB1q1BRyMiRaVEL5KXEiXg2mstI592Gtx8s428mz+/ULupVAnOPdfOGZYutdH6q1ZZt/4zz0CsdawtXGiHOWAAvPuuxa5aAiLxKaESvUrgSsTUrm3ZeeRIy9TNm8O99xYp+zlnc/J//BFOOcWm7HXsCGvXhj/swsrIsE6L5s3tMN9914oHTZwIl16qssASu8aOhffeCzqKGOW9T7hHixYtvEjErF/v/aWXeg/eN2rk/bRpRd5VRob3zzzjfZky3lev7v2YMWGMs5CWLvX+xBPtsDp29H716uznnnjCtvfoYTGLxIpdu7zv29d+P0uU8H7mzKAjCgYwy4fIiQl1RS8SFdWqwZtv2iXEli3Wx33TTTapvpCcgxtugNmzbXncTp2sEM/27RGIOwTv4aWX4OijrWNInyQAABp8SURBVEDQiBHWeVGjRnabm26Cu+6C4cPh9ttj71aDFE8rVlh9qyFD7O+oRg246iqb+io5hDoDiOeHruglajZv9r5XL7ucqFvX+wkTiryrnTu9v/VW29URR3g/e3YY4wxh5Urv27e373nqqd4vWxa6bUZG9qE+9FDkYxPJyxdfeF+tmvcVKnj/3nu27eOP7fdz8OBgY8vPH3+Ev2eMPK7oA0/KkXgo0UvUTZ7s/WGH2Z9U9+7e//lnkXc1caL3tWp5n5zs/SOPeJ+WFsY4M2VkeP/WW95XqeJ92bLeP/209+np+b8uPd37iy+2w3zxxfDHJZKf9HTvBw3y3jnvjzzS+59/3vP5iy6yv52ffgomvvysX+/9QQfZ7YZwUqIXiYYdO7y/4w7vk5K8r1HD+w8+KPKuNm70vmtX+ws9+WTvly8PX5jr1mXvu2VL7xctKtzrU1O979DB/tG++2744hLJz/r13rdrZ7+7l13m/dat/26zdq33Vat6f9xxkTlJ3hcZGd536uR9qVLez5kT3n0r0YtE05w53jdrZn9eXbp4v2pVkXaTkeH9iBHWNVm5svfvvLPvoX38sfcHHGBXPA89VPR/hNu2ed+mje3n88/3PS6R/Hz3nfd16liSfOGFvLu+33zT/vyefDJ68RXE009bXE89Ff59K9GLRNvu3d4//LD3pUtblh4+vMg35ZYssasT8P7yy21YQGH99Zf3V1xh+2jSxPt584oUyh42bbJ9lSvn/dSp+74/kdxkzUxJTrZhMLNmFew1Z51lv5u//hr5GAti7lz7d3DWWZGZuaJELxKURYuy56ydfnqR/+ukpno/cKBNH6pXr3CJdcIEuxIqUcL7AQNsOlK4rFnjfYMGdi4TjpMHkZy2bLF77mAJcuPGgr92+XLvK1b0/rTTgp8SunWrDbCtWdNunUVCXole0+tEIumww2DSJHj+efjuOzjqqAKX0c0pORkGDYIpU+zrE06AgQNh9+7Qr9m2DXr3thX0ypWzevv33w+lSu3D8ezlwANhwgQoX94W7fn11/DtW4q3hQvh2GOtCM6DD9qUz/33L/jr69SB//s/+PJLmzIapL59YdEim5VbvXoAAYQ6A4jnh67oJSYtX26XJeD9sccWeTTO5s3ed+tmuznuOO8XL/53m6lT7UobvL/xRu+3b9/H2PPx00/e77+/9TasXBnZ7yWJ7+23vS9f3saTfPll0feTnm4dapUqBfd7+c479nd4552R/T7oil4kBtSpA2PGwNtvw2+/WaH766+3FW8KYb/94LXXrDztokXQtKldsXhvFXn79bMr/t27rTPhySdtfZ5IatQIPvsM1q+3K/s//4zs95PEtGuXFb655BJo1swKOJ16atH3V6IEvPyy7bdXr+gXevrtN1vAqmVLq5gdmFBnAPH80BW9xLxNm7zv08em4u2/v/fPPVekIfDLl9v0O/C+c2fvGze2z3v29P7vvyMQdz4mTrRR0S1b2v3VaNq82WY4Snxatsw6usAKR6Wmhm/f//d/tt9oTgdNTbW/g/328/633yL//dBgPJEY9eOP2Zm6aVPvp0wp9C7S0qywTnKyDfYZOzYCcRbCqFE28O+MM6zaXySlp1uFtAsusOOvWtUmO0T7JEP2zWef2fnufvvZ70+47d7tfYsWtp7Ehg3h339u7rwzuicXxSbRA52AYQ0aNAjbD08k4jIy7L9B7dr2J3nppUW6obhsmU2jiwUjRtihnHdeZIqW/PGH9/fdZ9OtwJJEnz7Z5XyrVbOTn9wKqkjsSEvz/u67rfjS0Ud7/8svkfte8+Z5X7KkTVGNtC+/tGPq2TPy3ytLsUn0WQ9d0Utc2rrV5r+VKmVVch55JLxz4aLs8cf9P7cRwjG9KTXV+9GjbTxjiRL+nxmL77yzZ8/BtGnen3mmPV+9unXbKuHHnnXrrNcnq2p0pAeMem8nFeD9uHGR+x7r1lnP2hFHRPf3ToleJJ4sXmzrxILVz4/j0nMDBthh3H570fexeLH3/ftbVWGwOuEDBuRfkuDbb7PLpVav7v2jjyrhx4pvv7X1HMqUsVpS0bJzp60sXadO0QpP5SerUE/p0lYgJ5qU6EXi0dix2XPkOneOnRJfhZCR4f1119khPPxwwV+3Y4ctupM1fCEpyfuzz/Z+zBi731oYU6d637at7eeAA7x/7DEr4SvRt3Wr97fcYu/noYd6//330Y9h2jTrVu/VK/z7fuop+z17+unw7zs/SvQi8WrnTitKX768XSbcfXfcZam0tOzqZsOG5d123jzv//tfW1UPvK9f3/sHHwzPHOj//S+7q/iAA+zWQpz9KHOVkWElGe64w06mYrXX4tNPvT/4YPv5X3ttsONJbrzR4pg8OXz7nDPH7rqdfXYwlfiU6EXi3R9/ZK8Pe/DB3r//fvB1PQth1y4bKOdc9trhWf7+204AsqZWlSplh/rllwVbOrewpkyxe/vg/YEHev/EE9G5PxxuCxZYWeSs1ZGTkuxjzZrWHR4rK7etXOn9+edbbI0a2QlX0LZuteJODRuG573fssXeh1q1ojeqf29K9CKJ4ptvbHgyWBHvWF10OxfbtnnfurVNgxs/3u7TXnWVdVaA1QAYMiR6/ygnT7YfIdj9/yefjP2Ev3SpdfA0aWJxO+f9Kad4/+KLtoTr1Kk2dxu8P+oo+zkHJT3d+2eftSlzpUt7/8ADsTW2dOJE+zn167fv++re3d6LSZP2fV9FpUQvkkh277blvCpXtsu4G2+MnXl1+cha8c5qlFmS79nT++nTg+ug+OYb7089NTvhP/VUbCX8FSvsJCRrBUPwvlUrOynKbQXkjAzrNalXz9qeeaaVa4imH37IPuE4/fTcyzTHgh497E+oICvihfL223acd98dvriKQoleJBGtW+f91VfbpcQBB3j/yiuR6esOszVr7AropZeCqd4Xytdf29VxVvf3kCHBVdpbt87755/3/qST7O0F75s1sxmXBa2ytnOnjUOoXNmmI159tferV0cyauu16dfP5qtXr27rwsfyHaZNm+y9btKkaJX4fv3VVshr3brwg0TDTYleJJHNnJl9+VSlivVH33abTTD/5Ze4SP6xZNIkS7BZU/keeMD7Dz6wbvHffotc8t+0yc7V2rXLvt9+xBHeDxrk/c8/F32/GzZYp09ysvWgDBoUmQF7n3+e3Ytw1VXB3asurNGjLeb77y/c61JTbVxJ5cpWrCpoeSV6Z88nlpSUFD9r1qygwxCJnowM+OADmDgR5syBH3+E1FR7br/9bIWQFi2geXP72LAhJCUFG3OM+/pruOcemDz538/tvz/UrGmPgw7K/WPNmvkvJrRtm61z9M47tihQairUqwcXXWSPo44C58JzPEuWQP/+8OGHFuP990O3bvv+a7B2Ldx0E4wcCYcfDi++CCedFJ6Yo+XCC+Gjj2DuXPjPfwr2mv794ZFH7M+ua9fIxlcQzrnZ3vuUXJ9TohdJQKmp8NNPMHu2Jf7Zs2HePFvGC2wB+WbNshN/8+ZwxBFQsmSwcceg9eth1Sp7rF79749Zj927//3aypVzPwmoVAm++MLWWN++3bZfeKEl92OOCV9yz83UqXDLLfDdd9CkCTz2GJx+euH3k5EBw4fD7bfbMdx5pyW/0qXDH3OkrV1rKzAefjhMmZL/yc+ECdC2LVx7LbzwQnRizI8SvYhYJlq4MDvxz5ljlzDbt9vzZcvaf/6cV/6NGkFycrBxx4GMDFuaN7eTgL23ZXW0VKsG551nyf2EE2xJ1WjxHt57zxLzsmXQvj08+igceWTBXr9ggSW5//0PTj7Zkt3hh0cy4sh74w3r4RgyBPr0Cd1u3To4+mh7/2bMgHLlohdjXpToRSR36em2qH3O5P/997Bliz1fujS0bg0PPGCLass+8d5OCDZsgPr1gz+H2rkTnnnGuvG3bIGePWHQIKhRI3T7Bx6wLuuKFeHxx+GKKyLbAxEt3sNZZ8E331hnWN26/26TkWFtvv4aZs6Exo2jHWVoeSX6wAfOReKhwXgi+yA93ftFi2ze0C23ZBeZv/hi73//PejoJALWr7fV/0qWtPWU7rvv31UDJ07MrsjcrZvNDEg0v/9ux3/GGbnPFshaqOm556IfW37QqHsRKbItW7y/6y5bgaRMGVtRRgu+J6RffvG+SxfLDLVq2SyANWsssYMl+okTg44ysp591o71lVf23D5rls1cOPfc2JwymFeiV9e9iBTM8uVwxx3w9tvWt/vAA9Zvq9H7CWfKFBuwN3Omvb0lSkC/fjBgAJQpE3R0kZWRYeMOfvzRxiLUrGm3NZo3t1sX8+bZrItYk1fXfRSHf0Sec66Tc27Y5s2bgw5FJPEcfDC89RZMm2Y3MHv0gJQUu2EpCeWEE2D6dDun69nTxmzed1/iJ3mwk5qXX4YdO6B3b9vWuzcsXWo/j1hM8vnRFb2IFJ738O67dpm3fDmcc44N227QIOjIRMLikUdsVsLll9uI/HvvtboKsarYXNGLSJQ4Z/PCfv7ZuvAnTrSpeLfcAn/9FXR0Ivvsllusu/6NN6yHY8CAoCMqOiV6ESm6smWtUsrixTYJ+ckn7ar+2WchLS3o6ESKrGRJeP11q3Xw1lvxXUtKiV5E9l2NGnZjc84cqybSu7d9/OyzoCMTKbIjj4T334c6dYKOZN8o0YtI+DRtCl9+aYXDd++GDh3gzDOtAomIBEKJXkTCyzno3NmS+xNPZBdV79XLCseLSFQp0YtIZJQqZcuaLV4M118Pw4bZqnmPPZa9uI6IRJwSvYhEVrVq8PTTVoGkdWu47TYbof/QQ1ZnX0QiSoleRKLjP/+BsWNh/Hg48EAbrX/EETbiaeBAq8qSgHU9RIKmRC8i0dW2LXz7LfzxBwwdCgccYHPxmzWzqXm33WbV9zIygo5UJCGoMp6IBG/9evjkExg1CiZMsBH7NWvCuedC165w4onxPZFZJMK0Hr2IxI/Nm62Lf9Qom4e/fbsVGO/cGbp0gTPOgNKlg45SJKYo0YtIfNq+3e7pjxoFY8bYSUDFinDWWZb027eHChWCjlIkcEr0IhL/UlNh0iRL+h99BOvW2XJq7dpZ0u/UCapUCTpKkUAo0YtIYklPh6lTLemPGmUD+0qWhFNOsfv655xj9/hFigklehFJXN7DrFnw4YcwejT88otV52vZ0q70zz0XDj006ChFIkqJXkSKB+9h4UK7yh892hbZATjqqOykf/TRdiIgkkCU6EWkeFq2zO7njx4NU6bYiUD9+pbwzz0XWrWCEionIvFPiV5EZN06m6s/ejRMnGiD+2rUyJ62d/LJVp9fJA4p0YuI5PT33zBunHXxjxsH27ZBpUrQsaMl/XbtoHz5oKMUKTAlehGRUHbutCv8UaPsin/jRihb1pL9ueda8t9//6CjFMlTXoleNSVFpHgrU8aSeceOkJZm9/JHj7bHRx/ZtL0zzoALLrBpe5UrBx2xSKFoFIqISJasufhDh8Ly5TBjBtx8s43k797dFuDp1AneeMOq9InEgYTqunfOdQI6NWjQ4OrFixcHHY6IJIqsufrvvWeP5ctt4F779nal36mTleYVCYju0YuIhIv38N132Ul/5Urr/u/QwZJ+x47BDOTz3h6aLlgsKdGLiERCRgZMm2YJ//33YfVqG8jXsaMl/Q4doFy58H0/722w4C+/2GPx4j0/T02FqlWhWjWoXr1gH8uUCV98EhglehGRSMvIgP/9z5L+Bx/A2rWW5Dt1ggsvhDPPtJOAgti6dc8knjOZb9qU3a5kSSsAdNhh0LChfb8NG+yxfr09Nmywk4OMjNy/V/nyeZ8MHHig7bt+fdUZiGFK9CIi0ZSeDpMnW9L/8ENLuBUqwNlnW9Jv187aLV2a+9X56tV77u/ggy3ZHnZY9qNhQ6hbF5KT848nI8NOELISf0E+bt++5z6SkqBevezvf/jh2R8POkhlhQOmRC8iEpS0NPj6a0v6o0Zlz9PftWvPq+zq1f+dyA87zBbkCWf3f0Ft325Jf/Xq7BOQRYuyP9+xI7tt+fJ7xp51EnDYYVaISCJOiV5EJBbs3g2TJlk1vkqV9kzq8TQ/PyPDBiHmTP5ZH5ct2/ME5sAD90z8hx9ux7vfftZLkJRkAwizPt/7aw0uLBAlehERiY5du+DXX3M/CVi/vmj7zO9kIOvrUqWstyTno1y5f28LtT23bZUr28lKjFNlPBERiY7SpaFRI3vsbdOm7PEI27fbWIb0dOsByPq8IF+HapOaarcUsh5//22DInNu27Hj3+MP8tOgAbRtaxUSTzkl7m5H6IpeRESKF++t5yG3E4C9t61dC199Zbdctm2z3oOWLS3pt20Lxxxjsx8Cpq57ERGRfZGaajUTJkyAL76wSone29X9qadmX/Efemgg4SnRi4iIhNPGjXal/8UX9li+3LbXr599tX/qqVEbZKlELyIiEine27iDL76wK/6vvrKiRyVKwLHHWtJv29Y+L0jdgyJQohcREYmW3bth+vTsbv6ZM22wYMWKdpV/xhm2IFL9+mH7lkr0IiIiQdm0ac9u/mXLoG9feOqpsH0LTa8TEREJSpUq0LWrPby3OgNRHKmvRC8iIhItztm8/ChSbUEREZEEpkQvIiKSwJToRUREEpgSvYiISAJTohcREUlgSvQiIiIJTIleREQkgSnRi4iIJDAlehERkQSmRC8iIpLAEnJRG+fceuD3oOMIo2rAhqCDCLNEPCZIzONKxGOCxDwuHVP8CPdxHeK9r57bEwmZ6BONc25WqFWJ4lUiHhMk5nEl4jFBYh6Xjil+RPO41HUvIiKSwJToRUREEpgSfXwYFnQAEZCIxwSJeVyJeEyQmMelY4ofUTsu3aMXERFJYLqiFxERSWBK9DHCOVfHOTfJObfQOfeTc65vLm1Ods5tds7NzXwMDCLWwnDOLXPO/ZgZ76xcnnfOuaHOuSXOuR+cc82DiLMwnHOH53gP5jrn/nbO3bhXm5h/r5xzI5xz65xz83Ns2985N8E5tzjzY5UQr70is81i59wV0Ys6fyGO61Hn3M+Zv2OjnXOVQ7w2z9/XoIQ4pnudcytz/I51CPHaM51zizL/xvpHL+q8hTimd3MczzLn3NwQr43V9ynX/+OB/1157/WIgQdQE2ie+XlF4Beg0V5tTgY+DTrWQh7XMqBaHs93AD4DHNAS+C7omAt5fEnAGmwOa1y9V8CJQHNgfo5t/wf0z/y8P/BILq/bH1ia+bFK5udVgj6efI6rLVAy8/NHcjuuzOfy/H2NsWO6F7g1n9clAb8C9YFSwLy9/6/E0jHt9fzjwMA4e59y/T8e9N+VruhjhPd+tfd+TubnW4CFQK1go4qKzsDr3kwHKjvnagYdVCGcBvzqvY+7Ak3e+8nAn3tt7gy8lvn5a8A5uby0HTDBe/+n934TMAE4M2KBFlJux+W9/8J7n5b55XSgdtQD2wch3quCOBZY4r1f6r1PBd7B3uPA5XVMzjkHXACMjGpQ+yiP/+OB/l0p0ccg51xdoBnwXS5Pt3LOzXPOfeacOzKqgRWNB75wzs12zl2Ty/O1gD9yfL2C+DrBuYjQ/4zi7b0CONB7vxrsnxZwQC5t4v09uwrrRcpNfr+vsaZ35u2IESG6g+P1vToBWOu9Xxzi+Zh/n/b6Px7o35USfYxxzlUAPgRu9N7/vdfTc7Au4ibA08BH0Y6vCFp775sD7YEbnHMn7vW8y+U1cTEVxDlXCjgbeD+Xp+PxvSqoeH7PBgBpwFshmuT3+xpLngcOBZoCq7Gu7r3F63t1MXlfzcf0+5TP//GQL8tlW1jeKyX6GOKcS8Z+Od7y3o/a+3nv/d/e+62Zn48Dkp1z1aIcZqF471dlflwHjMa6EnNaAdTJ8XVtYFV0ottn7YE53vu1ez8Rj+9VprVZt04yP67LpU1cvmeZg5s6Apf6zJuieyvA72vM8N6v9d6ne+8zgJfIPda4e6+ccyWBLsC7odrE8vsU4v94oH9XSvQxIvOe1HBgoff+iRBtamS2wzl3LPb+bYxelIXjnCvvnKuY9Tk2IGr+Xs0+Abpljr5vCWzO6uKKAyGvOuLtvcrhEyBrtO8VwMe5tBkPtHXOVcnsLm6buS1mOefOBPoBZ3vvt4doU5Df15ix11iWc8k91plAQ+dcvcweqIuw9ziWnQ787L1fkduTsfw+5fF/PNi/q6BHKerxz4jLNlg3zQ/A3MxHB+A64LrMNr2Bn7CRs9OB44OOO59jqp8Z67zMuAdkbs95TA54FhsZ/COQEnTcBTy2cljirpRjW1y9V9hJympgN3Y10QOoCnwJLM78uH9m2xTg5RyvvQpYkvnoHvSxFOC4lmD3P7P+tl7IbHsQMC6v39dYeIQ4pjcy/2Z+wBJJzb2PKfPrDtjo719j/Zgyt7+a9XeUo228vE+h/o8H+nelyngiIiIJTF33IiIiCUyJXkREJIEp0YuIiCQwJXoREZEEpkQvIiKSwJToReRfnHPpbs8V+sK26plzrm7OFctEJLJKBh2AiMSkHd77pkEHISL7Tlf0IlJgmeuAP+Kcm5H5aJC5/RDn3JeZC6x86Zw7OHP7gc7Wf5+X+Tg+c1dJzrmXMtfs/sI5VzawgxJJcEr0IpKbsnt13V+Y47m/vffHAs8AT2VuewZbbvhobMGYoZnbhwLfeFvcpzlWyQygIfCs9/5I4C+ga4SPR6TYUmU8EfkX59xW732FXLYvA0713i/NXLxjjfe+qnNuA1aCdXfm9tXe+2rOufVAbe/9rhz7qIutu90w8+t+QLL3/v7IH5lI8aMrehEpLB/i81BtcrMrx+fpaLyQSMQo0YtIYV2Y4+O0zM+/xVZGA7gU+F/m518C1wM455Kcc/tFK0gRMTqLFpHclHXOzc3x9efe+6wpdqWdc99hFwoXZ27rA4xwzt0GrAe6Z27vCwxzzvXArtyvx1YsE5Eo0T16ESmwzHv0Kd77DUHHIiIFo657ERGRBKYrehERkQSmK3oREZEEpkQvIiKSwJToRUREEpgSvYiISAJTohcREUlgSvQiIiIJ7P8BB1f3KH6InH4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m==> EarlyStop patience =  1 | Best test_cost: 0.0285\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ca5c63d74c045e0865a7a5c4b250954",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d26c8e57b0f461f987e1004c84f3b0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    21\n",
      "Train_cost  = 0.0134 | Test_cost  = 0.0335 | \n",
      "\u001b[31m==> EarlyStop patience =  2 | Best test_cost: 0.0285\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1d622ecbd6b4000a166ef0d716d45d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2da0f6d584864728871cdbfeb083151b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    22\n",
      "Train_cost  = 0.0117 | Test_cost  = 0.0357 | \n",
      "\u001b[31m==> EarlyStop patience =  3 | Best test_cost: 0.0285\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c66d67878c043b4b3f663c7b72c9300",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c89185f8983a4021b142f51a39a54226",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    23\n",
      "Train_cost  = 0.0129 | Test_cost  = 0.0394 | \n",
      "\u001b[31m==> EarlyStop patience =  4 | Best test_cost: 0.0285\u001b[0m\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c5a3493d6e14c1b88a807b79064829f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Train', max=4.0, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fe51bac62c547bd9121631bd5f12557",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Test', max=2.0, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Epoch    24\n",
      "Train_cost  = 0.0113 | Test_cost  = 0.0353 | \n",
      "\u001b[31m==> EarlyStop patience =  5 | Best test_cost: 0.0285\u001b[0m\n",
      "\u001b[31m==> Execute Early Stopping at epoch: 24 | Best test_cost: 0.0285\u001b[0m\n",
      "\u001b[31m==> Best model is saved at model\u001b[0m\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAE9CAYAAADqEwA/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZyVc//H8de3aV+0S4rSYiklNYU7t+1OCgm3ZElJmiJbtmxJITuJbgrZd79QCNmyR5FEIuQuUqm7fZ3m8/vjM2maZmpmmnOuM+e8n4/HeczMdV3nuj5zdZrP9/quwcwQERGR1FAq6gBEREQkfpT4RUREUogSv4iISApR4hcREUkhSvwiIiIpRIlfREQkhZSOOoBYqFWrljVs2DDqMEREROJi2rRpf5lZ7YIcm5SJv2HDhkydOjXqMEREROIihPBbQY9VVb+IiEgKUeIXERFJIUr8IiIiKSQp2/hFRCQ1bNy4kfnz57Nu3bqoQ4mL8uXLU79+fcqUKVPkcyR84g8hVAL+A2wAPjCzpyMOSUREEsT8+fOpUqUKDRs2JIQQdTgxZWYsWbKE+fPns9deexX5PJFU9YcQxoYQFoUQZuba3imEMDuEMCeEcFX25pOBl8ysL3BC3IMVEZGEtW7dOmrWrJn0SR8ghEDNmjV3unYjqjb+x4BOOTeEENKAUUBnoBlwegihGVAfmJd92KY4xigiIiVAKiT9zYrjd40k8ZvZh8DSXJvbAXPM7Bcz2wA8B3QF5uPJH7YTbwghI4QwNYQwdfHixbEIW0REZCtLliyhVatWtGrVit1224169er9/fOGDRsKdI7evXsze/bsGEe6RSK18ddjy5M9eMI/CBgJ3B9COA6YkN+bzWwMMAYgPT3dYhiniIgIADVr1mT69OkA3HDDDVSuXJnLL798q2PMDDOjVKm8n10fffTRmMeZUyIN58ur/sLMbLWZ9Taz8+Lesc8MPvgAXn89rpcVEZGSbc6cOey///7079+f1q1bs2DBAjIyMkhPT6d58+YMGzbs72MPPfRQpk+fTmZmJtWqVeOqq67igAMO4JBDDmHRokXFHlsiJf75wB45fq4P/BFRLFtcdRVceCFkZkYdiYiIlCDff/89ffr04euvv6ZevXrceuutTJ06lW+++YZJkybx/fffb/Oe5cuXc/jhh/PNN99wyCGHMHbs2GKPK5Gq+r8EmoYQ9gJ+B04Dzog0ohBg0CA4+WR46SU47bRIwxERke245BLIrnYvNq1awYgRRXpr48aNadu27d8/P/vsszzyyCNkZmbyxx9/8P3339OsWbOt3lOhQgU6d+4MQJs2bfjoo4+KHns+ohrO9yzwGbBPCGF+CKGPmWUCFwBvAbOAF8zsuyji20rXrrDPPnDbbV71LyIiUgCVKlX6+/uffvqJe++9l/fee48ZM2bQqVOnPIfllS1b9u/v09LSyIxBbXMkT/xmdno+298A3ijqeUMIXYAuTZo0KeoptlWqFFxxBZx7LkyaBB07Ft+5RUSk+BTxyTweVqxYQZUqVdhll11YsGABb731Fp06ddrxG2Mgkdr4d5qZTTCzjKpVqxbviXv0gN13h1tvLd7ziohISmjdujXNmjVj//33p2/fvrRv3z6yWIIlYfV1enq6TZ06tXhPeued/uT/xReQo81GRESiM2vWLPbbb7+ow4irvH7nEMI0M0svyPuT6ok/pjIyoGpVb+sXEREpoZT4C2qXXeD882HcOPjxx6ijERERKRIl/sK4+GIoW9ar/UVEREogJf7CqFMHeveGxx+HBQuijkZERKTQkirxhxC6hBDGLF++PHYXufxyn8UvgYeNiIiI5CepEn/MhvPl1LgxnHIKPPggxLKAISIiEgNJlfjjZtAgWLHCk7+IiKSs4liWF2Ds2LH8+eefMYx0CyX+omjdGo4+2qv785hyUUREUsPmZXmnT59O//79GThw4N8/55x+d0eU+EuCQYPgzz/hiSeijkRERBLQ448/Trt27WjVqhXnn38+WVlZZGZmctZZZ9GiRQv2339/Ro4cyfPPP8/06dPp3r17oWsKiiKRVucrWY46Ctq0gTvugD59IC0t6ohERCRBzJw5k5dffplPP/2U0qVLk5GRwXPPPUfjxo3566+/+PbbbwFYtmwZ1apV47777uP++++nVatWMY9Nib8AzHyF3q1sXrL31FPh5Ze9w5+IiEQmkVblfeedd/jyyy9JT/dZdNeuXcsee+zBMcccw+zZs7n44os59thj6RjBwm9JVdVf3MP5zGDAAJ+iP08nnwxNmmjJXhER2YqZcc455/zd3j979mwGDx5MzZo1mTFjBoceeigjR46kX79+cY8tqZ74zWwCMCE9Pb1vcZwvBNiwAR5+2Cft22OPXAekpXmpoF8/eP99r/4XEZFIJNL0Kh06dOCUU07h4osvplatWixZsoTVq1dToUIFypcvT7du3dhrr73o378/AFWqVGHlypVxiS2pnvhjYfBg/3rTTfkc0LOnz+inJXtFRCRbixYtGDJkCB06dKBly5Z07NiRhQsXMm/ePA477DBatWpF3759GT58OAC9e/fm3HPPjUvnPi3LWwAXXuhD9n/4wefv2catt8LVV8O0aT7UT0RE4kLL8joty1vMrrkGSpeGYcPyOeC883z1vttvj2tcIiIihaXEXwB168IFF8BTT8GsWXkcULUq9O8PL74IP/8c9/hEREQKSom/gAYNgooV4YYb8jngkku8WkBL9oqISAJT4i+gWrVg4EB44QX45ps8Dqhb1zv6PfooLFwY9/hERFJVMvZVy09x/K5K/IVw6aVQrRpcf30+B1xxhY//GzkyrnGJiKSq8uXLs2TJkpRI/mbGkiVLKF++/E6dJ6l69YcQugBdmjRp0venn36KyTWGD4drr4UpU6BduzwOOOUUePdd+O9/oUqVmMQgIiJu48aNzJ8/n3UpsmBa+fLlqV+/PmXKlNlqe2F69SdV4t+suIfz5bRqFey1l4/ae+utPA748ksvEdx5J1x2WUxiEBERyUnD+WKocmUfsv/22/Dhh3kc0LYtHHkk3H03rF8f9/hERES2R4m/CM47z/vyXXddPlP0DxoEf/wBTz8d99hERES2R4m/CCpU8KT/0UcwaVIeB3Ts6Es63X47ZGXFPT4REZH8KPEXUZ8+0KBBPk/9m5fsnT0bxo+PJD4REZG8KPEXUblyPqzvyy9hwoQ8DjjlFO8FeOutWrJXREQShhL/TujZE5o08RX8tqnRL10aLr/cx/3l2QtQREQk/pT4d0Lp0jB0KMyYAS+9lMcBvXtD7dpw221xj01ERCQvSvw7qXt3aN7cq/0zM3PtrFABLr4YJk700oGIiEjEkirxhxC6hBDGLF++PG7XTEvz5Xpnz4ZnnsnjgPPP98H/WrJXREQSQFIlfjObYGYZVatWjet1TzoJDjzQV+7buDHXzurVISMDnnsO5s6Na1wiIiK5JVXij0oIcNNN8OuvvjjfNgYOhDJl4Jpr4h6biIhITkr8xaRzZzjkELjxRthmrYj69X1c/7PPwuTJkcQnIiICSvzFZvNT//z5MGZMHgcMGgQNG8IFF+TRC1BERCQ+lPiL0VFH+fo8w4fD6tW5dlaoAPfcAzNnwn/+E0l8IiIiSvzF7MYbYeFCGDUqj51du8Ixx/iMPwsXxj02ERERJf5i1r69t/ffdhusWJFrZwhw772wdq2v7SsiIhJnSvwxcOONsHQpjBiRx8599oFLL/Xu/59/HvfYREQktSnxx0CbNj62/667vACwjeuug913945+mzbFPT4REUldSvwxMmwYrFwJd96Zx87Klb1UMG0aPPJI3GMTEZHUpcQfI/vvD6ef7k36ixblcUD37nD44d7Wv2RJ3OMTEZHUpMQfQ0OGwPr1cOuteewMAe67D5Yv917+IiIicZBUiT+KRXq2Z++9oVcvH7b/++95HNCihbfzP/ggfPVV3OMTEZHUk1SJP6pFerZn8GDIyoKbb87ngBtugNq1vQCQlRXP0EREJAUlVeJPRA0bQt++8PDD8MsveRxQrZoP+v/sM3jyyXiHJyIiKUaJPw6uvRbKlvUCQJ4P9T17wsEHw5VXepu/iIhIjCjxx8Huu/tkPu+9ByNH5nFAqVJw//2weLFX/YuIiMSIEn+c9OkDXbrAVVfBd9/lcUCbNpCR4T39Z86Me3wiIpIalPjjJAR46CHYZRfo0QM2bMjjoJtvhqpV4cILwSzuMYqISPJT4o+jOnW8k9/06fnU6Nes6cn/gw/ghRfiHJ2IiKQCJf44O+EEOPdc78j/8cd5HNC3Lxx4IFx2GaxaFff4REQkuSnxR+Duu32YX8+eeSzdm5bmHf1+/307g/9FRESKRok/AlWqwBNPwG+/wcCBeRzwj3/4lH933QU//hj3+EREJHkp8UekfXvv4T92LLzySh4H3HorVKgAF12kjn4iIlJslPgjNGSIN+f37QsLF+baudtuMHQovPUWjB8fSXwiIpJ8lPgjVLYsPPWU9+E799w8HuwHDIDmzeGSS2Dt2khiFBGR5KLEH7FmzbyH/2uv+VC/rZQp4xP6zJ0Lt98eRXgiIpJklPgTwAUXQIcO3tFvzpxcO488Erp39zb/X3+NJD4REUkeSZX4QwhdQghjlpewhW5KlYJHH/UH/LPOgszMXAfceacfdOmlkcQnIiLJI6kSv5lNMLOMqlWrRh1KodWvDw88AJ9/7g/32+wcPNi7/7/5ZiTxiYhIcgiWhEPF0tPTberUqVGHUSRnnAEvvgiffQbp6Tl2rF8PLVr499OnQ8WKkcQnIiKJJ4QwzczSd3xkkj3xJ4NRo3xO/7POgjVrcuwoV86rBObMyWfWHxERkR1T4k8w1avD44/DDz/4BD9b+de/YNAgGDPGqwVEREQKSYk/Af3rXz50/7774O23c+0cNgwOPthn/VEvfxERKSQl/gQ1fLiP8e/dG5YuzbGjTBl45hn//vTTYePGSOITEZGSSYk/QVWo4LP6LV4M/fvnmtVvr73goYdgyhS4/vrIYhQRkZJHiT+BHXigT9f/4otbHvL/1q0bZGT42L9JkyKJT0RESh4N50twmzbBYYfBd9/BjBmw5545dq5ZA+3awV9/wTff+HAAERFJORrOl0TS0uDJJ70AcPbZkJWVY2fFivDcc7B8OfTsmWuniIjItpT4S4BGjWDECHj/fbj33lw799/fd779Ntx1VyTxiYhIyaHEX0Kccw507QpXXw0//ZRrZ0YGnHIKXHONd/gTERHJhxJ/CRGCT9xXrhwMGJCrl38I3su/Xj0f4lfCFikSEZH4UeIvQerWhZtu8k78L7yQa2e1avDss/Df/0K/frlKBiIiIk6Jv4Q5/3xo3dqn61+xItfOQw6BG2+E55+HsWMjiU9ERBKbEn8Jk5YGDz4If/7pK/VuY9Ag6NABLrwQvv8+7vGJiEhiU+Ivgdq29dn87r8fvvoq185SpeCJJ6ByZTjtNFi7NpIYRUQkMSnxl1DDh0OtWnDeeT7Gfyt163ry//ZbuOyySOITEZHEpMRfQlWrBnffDV984R36t9GpE1x+uQ8FGDcu7vGJiEhi0pS9JZiZN+d/9RX88EMeM/Zu2ACHHuoD/6dPhwYNIolTRERiS1P2pogQYNQoWL0arrgijwPKlvUpfbOy4IwzIDMz7jGKiEhiSarEH0LoEkIYszyFJrDZd1+48kqfz/+DD/I4oFEjGD0aPv0UbrghztGJiEiiUVV/Eli7Fpo3h/LlvUa/bNk8Djr3XB/b/847cNRRcY9RRERiR1X9KaZCBR/aN2vWdtbpufde2Gcf6NEDFi+Oa3wiIpI4lPiTxLHHwskn+8R9v/6axwGVKvmMfkuXQq9eWsJXRCRFKfEnkREjfP6eiy7KZ6r+li19DODEiTByZNzjExGR6CnxJ5E99oChQ+G11+DVV/M56LzzoEsXuPZaX9BHRERSihJ/krnoImjRwr+uWpXHASHAfff595dcEtfYREQkekr8SaZMGZ+sb948GDYsn4MaNIDrr4eXX4bXX49rfCIiEi0l/iTUvj306QP33AMzZ+Zz0MCB0KwZXHABrFkT1/hERCQ6SvxJ6rbboGpVb9LPswN/2bLwn//A3Llwyy3xDk9ERCKixJ+kataE22+Hjz+Gxx/P56DDD4ezzvJSwuzZcY1PRESiocSfxM4+26v9r7gClizJ56A77oCKFWHAgHzGAIqISDJR4k9ipUp5R79ly+Cqq/I5qE4dGD4c3n3XJ/gREZGkpsSf5Fq08H58Dz/s6/TkqV8/SE/3A1NogSMRkVSkxJ8ChgzxyX3OOy+flXnT0rxqYOFCH+YnIiJJS4k/BVSu7Gv0zJixnZl609O9ZHD//fD113GNT0RE4keJP0WceCIcf7w//c+fn89BN98MtWptZwygiIiUdEr8KWLzTL2bNm1npt5q1Xxd3ylTvFOAiIgkHSX+FNKwIQweDP/3f/DKK/kcdOaZPr7/qqtg0aJ4hiciInGgxJ9iLrvMe/r/+9++kM+KFbkOCMFn9Fu5EgYNiiRGERGJHSX+FFO2LHz0EZx/vvfj228/eOmlXHP3NGsGl18Ojz3mB4uISNJQ4k9BVat6e/+UKT5/T7du3vFv7twcB113Hey5p5cQNm6MKlQRESlmSvwprG1b+OILX8Vv8mR/0L/ttuw8X6mSj/2bOdPHAoqISFJQ4k9xpUt7L/9Zs+CYY7xPX+vW2bP8de0KXbrADTfAvHlRhyoiIsVAiV8An9nv5Zfh1Vd91t727SEjA5YOvc/H9Oc7BlBEREoSJX7ZygknwPffe+//sWNh32Ma8NTxz2LjxsEbb0QdnoiI7CQlftlG5cpw550wbRo0agRnvdiVoyt+wo/97oK1a6MOT0REdoISv+TrgAPgk098WP/U0JaW819nWOfPWL8+6shERKSoCpT4QwhPFmSbJJ+0NJ+6f9ZPZThxz68ZMvkoDthvA++/H3VkIiJSFAV94m+e84cQQhrQpvjDkURVty4893lD3qx4Mhv//IujjoJevWDVqqgjExGRwthu4g8hXB1CWAm0DCGsyH6tBBYBr8YlQkkcdetyzK1HMnNtY6458Xueesqn/RURkZJju4nfzG4xsyrAHWa2S/aripnVNLOr4xSjJJLzz6fCgftx85QOXH3peh59FMaNizooEREpqIJW9b8WQqgEEELoEUK4O4TQIIZxSaJKS4MHHoA//+T69dfSurWP9//zz6gDExGRgiho4n8AWBNCOAC4EvgNeCJmUUliO+ggyMig7Kh7eGrQt6xeDeeck2uhHxERSUgFTfyZZmZAV+BeM7sXqBK7sCTh3XIL1KvHfld24fYhq5k4EUaPjjooERHZkYIm/pUhhKuBs4DXs3v1l4ldWFuEEBqFEB4JIbwUj+tJAVWv7uv5LljAgPdPoePRxmWXwY8/Rh2YiIhsT0ETf3dgPXCOmf0J1APu2NGbQghjQwiLQggzc23vFEKYHUKYE0K4anvnMLNfzKxPAeOUeGrXDkaOpNTbb/Lo/ndRrhz06KFVfEVEElmBEn92sn8aqBpCOB5YZ2YFaeN/DOiUc0N2bcEooDPQDDg9hNAshNAihPBarteuhfllJAIZGdCrF7vfcwWj+33Fl1/CzTdHHZSIiOSnoDP3nQp8AXQDTgWmhBBO2dH7zOxDYGmuze2AOdlP8huA54CuZvatmR2f67WoUL+NxF8IPqfvAQfQbXQHzjppJTfdBFOmRB2YiIjkpaBV/dcCbc2sl5n1xJP34CJesx6Qc3H3+dnb8hRCqBlCeBA4MLufQX7HZYQQpoYQpi5evLiIoUmRVKwI//d/kJXFfT8fR73ds+jRA1avjjowERHJraCJv1Sup+8lhXhvbiGPbfkOBDOzJWbW38wam9kt2zlujJmlm1l67dq1ixiaFFnjxvDUU1Sd8RFP7H8HP//sS/uKiEhiKWjyfjOE8FYI4ewQwtnA60BRF2efD+yR4+f6wB9FPJckkuOPh2uv5fCJV3HZ0TMYPRpefz3qoEREJKcdzdXfJITQ3syuAEYDLYEDgM+AMUW85pdA0xDCXiGEssBpwPginksSzdChcPTR3PTBobRsuoY+fUAtLyIiiWNHT/wjgJUAZjbOzC41s4H40/6IHZ08hPAsXkjYJ4QwP4TQx8wygQuAt4BZwAtm9t3O/BKSQNLS4JlnKFenGk+tOon//c/IyNCsfiIiiWJHib+hmc3IvdHMpgINd3RyMzvdzOqaWRkzq29mj2Rvf8PM9s5ut9fgr2RTqxa89BItlnzA8L0e5pVX4NFHow5KRERgx4m//Hb2VSjOQIpDCKFLCGHM8uXLow5Fsif3GTi7H0c2/JWLL4Zffok6KBER2VHi/zKE0Df3xhBCH2BabEIqOjObYGYZVatWjToUAcjIoFSvnjw29wjSbCM9e8KmTVEHJSKS2naU+C8BeocQPggh3JX9mgycC1wc+/CkRMue3GfPA6ozygbwySdw++1RByUiktqCFaDXVQjhSGD/7B+/M7P3YhrVTkpPT7epU6dGHYZs9vPPWOs2nFbqBcatOpopUwKtW0cdlIhI8gghTDOz9AIdW5DEX9Io8Seg115jaZeetKw4h10a1GDaNKiQcL1ERERKpsIk/qLOvidSOMcfT41rz+fRNd2ZNQuu2u6ajCIiEitJlfjVqz/BDR3K0UcHLkq7n5EjYdKkqAMSEUk9quqX+PrrL9a2OoQ2i95gec3GfPtdKWrUiDooEZGSTVX9krhq1aLCuKd5ynqwaGEW5/U3zeonIhJHSvwSf+3a0fq+3gyzwbzwYuCZZ6IOSEQkdSjxSzT69ePKs/6kPR+T0SeT8VqmSUQkLpT4JRohkPbgKF5qNoTmmd9w4onGnXdqMR8RkVhT4pfoVKzIbm+MZfJup9GtzKtccQX07QsbNkQdmIhI8lLil2g1aECFDybybI0BXF/pLh55BDp2hCVLog5MRCQ5JVXi1zj+EqpJE0q9O4mhFW7lqRoX8vlnWRx8MMyeHXVgIiLJJ6kSv1bnK8GaNYN33uFMe5r3a3ZjxbJNHHwwvPtu1IGJiCSXpEr8UsIdcAC89RaHrH6HKVWOpv5uGznmGBg9OurARESShxK/JJa2beGNN2i46As+4VCOOXID/fvDwIGwaVPUwYmIlHxK/JJ42reH8ePZZe4Mxv/1Dy7pv44RI6BrV1ixIurgRERKNiV+SUxHHQUvv0zadzO45+sjeOCedbz5ppcJfvst6uBEREouJX5JXJ06wQsvwNSp9B/XkTdfWcu8edCuHXz+edTBiYiUTEr8kthOPBGefho++YQO957A55PXU6UKHHEEPPts1MGJiJQ8SZX4NY4/SXXvDmPHwjvvsO+1/2bKRxs46CA44wwYMkTT/IqIFEZSJX6N409ivXrBAw/A669T88IzmDQxk3POgWHD4PTTYe3aqAMUESkZSkcdgEiB9e8P69bBwIGULdeLhx9/gn33TWPQIPj1V28RaNIk6iBFRBKbEr+ULJdc4o/311xDKF+eKx56iKZNS3HmmdC0KbRs6d0CunaFAw+EEKIOWEQksSRVVb+kiKuvhsGDvd3/oos4savxww9w991QrRrcdBO0aQMNG8JFF8F778HGjVEHLSKSGIIlYc+o9PR0mzp1atRhSCyZwRVXwF13weWXw+23//14v3gxvPYavPIKvP22tw5Urw7HHee1AcccA5UrRxy/iEgxCiFMM7P0ghyrqn4pmUKAO+7wrH7nnVChgvf0A2rXht69/bV6NUya5IWACRPgqaegXDk4+mgvBHTpArvuGvHvIiISR0r8UnKFACNHevK/8Ub/evPNUKbM34dUquQJ/sQTITMTPv4YXn3VCwKvvean+Mc/tvQLaNo0wt9HRCQO1MYvJVupUr58X//+XgPwz3/CL7/keWjp0j7xzz33+CHTp/s8AGvWeKvB3nt758BPPonvr5Cfn37SMEURKX5K/FLypaX5GP/nn4cffoBWrXY4rV8IvgrwkCHw1Vcwdy7ce683DRxxBIwaFd3EQFlZ3mqxzz4+a/GGDdHEISLJSYlfksepp8I330CLFj6t39lnw8qVBXprgwY+AmDaNO/8d8EF3kcg3k/cixdD585eIDnsMPjwQxgwQLMTSsEsW+aFV5HtSarEryl7hQYNYPJkuP56ePJJaN3as3kBVasG48d74n38cTj00PitBvjppz73wOTJMGYMvP8+XHMNPPywd2UQ2Z4PPoBGjbzgqoKibE9SJX5N2SuAN+YPHeqZc906OOQQ7/mflVWgt5cqBTfc4AWAOXN8ToB3341duGY+B8Hhh/uIg08/hb59vTnixhu94+Gll8Jbb8UuBinZnngCOnb0z+4nn/gsliL5SarEL7KVww7zqv8uXbz3XufO8OefBX57ly7w5ZdQp47/Ub3jjuJ/klq2DP79b7jsMjj+eK+caN16y/5SpbziokULX6vohx+K9/pSspl57VSvXt6v9ccfoW1bGDQIVq2KOjpJVEr8ktxq1ICXXoIHH/QG8wMOgIkTC/z2vfeGzz+Hk0+GK6+E004rvj+oX3/ttQkTJvg8ROPGeVNDbpUr+xDEcuW8MLJ0afFcX0q29euhRw/vCHrOOf6xrlHDO6n+8QcMHx5tfA8+6ENlf/892jgkD2aWdK82bdqYyDZmzjRr0cIMzAYONFu3rsBvzcoyu+02s1KlzJo3N/vxx6KHkZVlNnq0WblyZvXqmX38ccHe98knZmXLmh11lNmGDUW/vpR8ixebHXqof5SHD/fPVE49evhn5eefo4nvhx/88w1me+9t9scf0cSRSoCpVsAcGXmSjsVLiV/ytXat2QUX+Ef/wAP9L1QhvP22WY0aZlWrmk2YUPjLr1pldtZZfvmOHc0WLSrc+x97zN97/vmFv7Ykh9mzzZo08cT63HN5H/P772aVKpmdeGJ8YzMz27TJrH17s+rVzV56yePYbz+zhQvjH0sqUeJX4pcdefVVs5o1zSpWNHvkkW0fmbbj11+9zABmN9zgf+gK4vvvvbYgBLOhQ80yM4sW+hVX+LVHjSra+6XkmjzZC561ankN0PYMH+6fk0mT4hPbZuPdoEsAABsSSURBVPfd59d97DH/+f33zSpU8Mq2xYvjG0sqUeJX4peCmD/f7Mgj/b/Bqaea/e9/BX7rmjVmPXv6W7t0MVu2bPvHP/OMP/nUru21BjsjM9Ps+OPN0tLM3nln584lJceTT5qVKWO2zz5mc+bs+Pi1a80aNTJr1ix+TUO//uqf806dti5LT5rkNRStWpktWRKfWFKNEr8SvxRUZqY/GqWlmTVosOPHqByysvzppnRps6ZNvQtBbuvWmZ13nv9Pa9/eyxrFYflyrz2oXn3n+htI4svK8polMDviCLOlSwv+3pdf9veNHBm7+DbLyvLmq8qVzebO3Xb/xIne7yA9fccFZSk8JX4lfimszz8322svLwBcfbU/LhXQhx+a1anjTzovvLBl+y+/+B85MLv88uJ/6vrlF2+t2GefQlVWSAmybp131AOzXr3M1q8v3Puzssw6dDCrVi321eyPPupx3n9//sdMmOC1Fgcf7IVXKT5K/Er8UhTLl5v17u3/LfbbzwsDBfT772aHHOJvvfJKf9KqVs07Ab7ySuxCnjzZ/5B27Gi2cWPsriPxt2SJ2WGH+WfqxhsL1Q1lKzNnenm2f//ijS+nP/7wz/s//7njPi/jxnk8hx5qtnJl7GJKNUr8SvyyMyZONNtjDx+7d9ll3qBfAOvX+x9Xn1bFrHXr+Aynevhhv97FF8fm/HPmmF16qdlFF3kBR2Lvp598GFzZsmZPP73z57vwQv84T5++8+fKy8knm5Uv7yMOCuL55z2eI44wW706NjGlmpRN/EAXYEyTJk2K5UZKClu+3KxfP/8v0rSp2UcfFfitTz5pdt11hWot2GkDB3qoY8YUz/mysrw3dteuPgqhTBl/VazoIxJWrSqe68i2Pv7Ym3Bq1izUx267li718x1+eNFrDvLz4ov+2bvttsK976mn/LPVoUN8/68UVlaW2Xffmd10kzfdNW7stXpff13893JnpGzi3/zSE78Um3feMWvY0P9CXXRRwma8jRu9J3Xp0mYffFD086xd6221Bxzgfx1q1TIbPNircufMMTvlFN9er57Z448XfCijFMwzz/hTftOmxd9p84EH/N8uZz+UnfXXX2a77mrWpk3Rmpoee8z/a3XuXKj5tGJu0yazTz/1obNNm9rftXgHH+yxli5tf7cIDhvmNTRRU+JX4pfitHLllkl/GjXyR+EEtGyZ2b77+pNdYZsY/vzTbMgQ/yMOZvvv700IebVyfPSRWdu29ndzRoLejhIlK8vb8cHb9f/6q/ivkZlp1rKl2Z57Fl/1+llneRLcmSaEMWPs72Gxhe28WJzWrfNWvn79zHbbzWMqXdr7zzzwwNbNXH/9Zfbgg1v6YIBZu3Zm99wT3SyFSvxK/BILkyd7PR/4GL0VK6KOaBs//eQTvDRrVrBe019/7b3Fy5b1X+v4472SY0dVmJs2eVXtHnv4+048UcMKi2rxYm8jB+/BH8sn3w8+8OsMHbrz53rjDT/X4ME7f65Ro/xcJ50U3+moly/32Q9PO82sShWPoXJls27dvG9FQUbL/Pe/ZnfcsWVSrxB8Wu2HHy7c0MudpcSvxC+xsnq1N6iH4OP+d3Y2nhh47z1/Ujn22LxnB8zM9FEHhx/ufwEqVfIKjYJ2zMppzRqzm2/2P5alS3sHw5IwQcuPP3rCuuuuaKuY33zTny7LlPHkEY82427dfCa9334r+jmWL/dCX7NmxXf/Rozwz2P37rEdobJgga+V0bnzlgJv7dpm555r9tprO9ffYNYss+uv9ymVwc/ftat3Zox1J0YlfiV+ibVPPvEB9OB/MRJsRpLN7bmXX75l2/Ll/se1USPf16CB2Z13Fs8cAAsWmGVkeE/t6tW9yjPKatu8rFrlbcr//Kf//qVK+dd99925fhFFsWbNltaj5s1j19s+L3Pneg/87t2Lfo7zzvOy72efFV9cZl742VzzUdQprXNbudLn2rjtNrN//MPjBp+249JLvemquK61WVaW2Zdf+jNC3br2d01Cjx5mr78em1oNJX4lfomHNWu8e2+pUmb163vdZwLZnFhuv93skku2VGW2b++Lp8TiqWrGDLOjj/brNGniY7aj7PmcleXJqW/fLb9/06Zmt9zibbYTJ3oCALOzz47PXPLTpnmnMPB/lyh6tF9/vV9/8uTCv3fyZPt7gctYuPlmP3/v3oXvPLpqlSfyESO8/8F++21J9JvX5Ro61Oybb+L3uczM9Fq4c8/1uQ7A++GMGFG811HiV+KXeJoyxes8N0+vFs+Gve3YuNGHSm3upHTmmWZffBH762ZleRlo8y057DCzqVNjf92cFi702ozNMVSs6In9ww+3/YO/erXZVVf5PapZ02zs2NgkhcxML3CUKWO2++7xXzwnp9Wrvaq+VavCPe2uWeMFp0aNYjvAZcgQ/3fLyMg/+a9a5UMf773X181o1mxLLQ54E8rxx/u5JkxIjKWB163z9cFOO80/Z8VJiV+JX+Jt3Tqza6/1Kcnq1vUxcf/9b+QDfZct27ZHcrxs3OjXrl3b/9KcdZbZvHmxvd6ECd5BbPNwq0MOMXvooYJ1dPz2W68NAe//MGtW8cX2669bmhi6dUuMfhDPPefxjB5d8PdsXhny3XdjF5eZ/7e5+mq/1oABnuQ/+cTXHOjZ05tHcif5447zJD9+fGpONFWYxB/8+OSSnp5uU6dOjToMSUVffQW9e8OMGf7z7rvDIYfAwQf719atoUKFaGOMs+XL4ZZbYMQIKFUK0tOhfn3YY49tv9au7ccUxk8/wdix8PjjsGCBn6NnTzjnHGjWrHDnysqCRx6BQYNg1Sr/es01Rf8nM4OnnoIBA/znUaOgRw8IoWjnK05mcPjhMGsW/PgjVK++/eOnToWDDoI+fWDMmPjEd+WVcOedfr82p6o6dfwz1KbNltfuuyfGPY1SCGGamaUX6FglfpFitmkTfP01fPYZfP65f/31V99XujQceOCWgsDBB0PDhinxV2vuXLjtNvj+e5g3D+bPh40btz6mbFmoVy//gkH9+p7Y166Fl17yJP3RR15YOPZYT0rHHQdlyuxcrIsWwWWXedJu0gQeeAA6dCjcOZYuhf794cUX4Z//hCee8H/qRPL11544L7rIC2b52bDBk+2SJf7vV7VqfOIzg9Gj4Y8/PM70dCX5/CjxK/FLolm4EKZM2VIY+OILWLPG99Wps3VBID0dKlWKNt44yMqCxYu9ALC5IJD7a36Fg7Q0T/5Nm/qTfc+enhCK27vvwnnnea3CGWfA3Xf7P9eOTJoEZ5/tv9+wYXDFFR5zIurXzwtQM2bkX0MybBgMGQLjx0OXLvGNTwpGiV+JXxJdZibMnLmlRuDzz72+FTxDtGzphYA+ffxRJ0XlVzhYtw5OOQUOPTT2T3/r1nlTxa23QsWK/rVv37ybJNauhauvhnvvhf328xqD1q1jG9/OWrzYC1AHHQRvvrnt/Zw503+HU06BZ56JJkbZMSV+JX4piZYs8VqBnIWBNWvg4ov9katy5agjTGmzZ3vV/QcfeOXM6NHQosWW/d98A2eeCd99Bxde6M0aJaU7x4gRMHAgvPoqnHDClu2bNsE//gG//OJV/LVrRxejbF9hEn8hu9GISMzUrOkN1cOGeV3x/PleD3vPPbD//vDGG1FHmNL22Qfee887Ef70kz8FDxoEK1fCHXdA27ZednvzTRg5suQkffDOh/vtB5deCuvXb9l+773eKnXffUr6yUSJXyRRVa0K//kPfPyx1zEfdxycfrr3F5BIhOD9CX74wb/efru3+V95pbd9f/stHHNM1FEWXpkyXr78+ectnfzmzIHrrvPfq3v3aOOT4pVUiT+E0CWEMGb58uVRhyJSfNq39+7Xw4bBuHGw777eGysJm+lKipo1/Z/gww+9x/7YsT7KoFatqCMrumOO8SR/003ei75vXy8QPPCAetEnG7Xxi5Qks2dDRoZnnMMP9wHVe+8ddVSSJObMgebNfejkzz/DQw/BuedGHZUUhNr4RZLVPvvA++/Dww97b7KWLf0RbcOGqCOTJNCkiXfy+/lnOOooH1QiyUeJX6SkKVXK/yLPmgUnngiDB/ukQJ9+GnVkkgSuu85nK3zsMVXxJyslfpGSarfd4Lnn4LXXfH7Z9u3h/PN9jlyRIqpcGW6+2av7JTkp8YuUdMcd54PHBw70weX77eedAEVE8qDEL5IMKlf2+WSnTPHxZf/+N5x0ks8FICKSgxK/SDJJT/cZV26/Hd56yydfv+++LesCiEjKU+IXSTZlyviqMDNn+nz/F13ktQC9evmMgJs2RR2hiERIiV8kWTVq5E/9H3wAp53mE7F37Oi9ti67zCcFSsJ5PERk+5T4RZJZCD7Rz0MPwZ9/+vRyBx3k1f+tW/saALfcAr/9FnWkIhInSvwiqaJ8ee/09/LLXgh48EGoUcMHbTdsuKWA8L//RR2piMSQEr9IKqpRw1f+++gjX3P1pptg0SKfDni33eDkk31IYM6l2kQkKSjxi6S6vfaCa6/1BdenTvVJgD791GsHdttty9oAWVlRRyoixUCJX0RcCNCmja/POn++Lyx//PHwzDPeDLDXXnD11T5aQERKLCV+EdlW6dK+TuuTT8LChfDUU75s2x13QIsWcMABPlfAvHlRRyoihaTELyLbV6kSnHkmvPGGL9R+331QsSIMGgQNGsARR6hToEgJosQvIgW3665wwQXw2We+ePvQoT5CYHOnwJNO8iGD69ZFHamI5EOJX0SKpnFjXxJ41izvFDhgAHz+OXTr5jMFnnMOvPuuZgoUSTBK/CKyczZ3Crz7bu8UOGmSDwd86SXo0AH23NNnCvzqK80UKJIAlPhFpPikpXmyf/RR7xT4wgvQtq33C2jTxjsI3nIL/P571JGKpCwlfhGJjQoVvNr/lVe8H8Do0VCrls8UuOee0KkTPP+8+gOIxJkSv4jEXo0aWyYCmjPHJwyaNcsXD6pb1ycN+uILNQWIxEFSJf4QQpcQwpjly5dHHYqI5KdxYxg2DH79Fd55B447zpsGDjrIFw264w6vIRCRmAiWhCXs9PR0mzp1atRhiEhBLV/u/QEee8ynC05L86aA3r199sBy5Xbu/H/9Bd9959MS53ytW+cjEHK+dt1122116vjcBSIJKoQwzczSC3SsEr+IJJQff/QCwBNPeCfAGjV8AqGzz4YDD/RRBHkx84WGNif1nIl+8eItx1WpAs2a+atSJe+EmPO1bFne569cOe8CQZ06sPfe3nGxTp384xOJISV+JX6Rkm/TJm8KePRR7yC4fj20bOkFgOOOg//+d9skv3TplvdXrerJvXnzLYm+WTOoX3/7yXn9ei8o5C4Q5PVasmTrfgk1avj1cr923TVmt0kElPiV+EWSzf/+B8895zUBX3yx9b7q1bck95xJvm7d2D99Z2Z6AeCHH7zwkfOVs+agVq0t8eV81a4d2/gkZSjxK/GLJK/vv/d+AI0abXmaTrTqdTNYsGDbwsB338GKFVuOq117SyGgcWMoW9b7N+T1Kl06/30595cr5/0RKlb0poyKFaF8+cS7R1KslPiV+EUkEZl5v4W8CgSrVsXuuiFsKQzkLhTk9XOVKvDPf8Jhh3lhRBJeYRJ/6VgHIyIi2ULwPgb16/uyx5uZeXNGZqb3bcjrtb19OfevXw9r1mz9Wr06/21Ll/pUyzm3rV4NWVmwyy7QuTOccIJ/rV49unsnxUaJX0QkaiF4x8BEsWaNL7A0fjxMmOAzLKaleQ3ACSdAly7eNCElkqr6RUQkf1lZ8OWXWwoB337r25s180LACSdAu3ZeMJDIqI1fiV9EJDZ+/dULAOPHw+TJ3sSw664+0dIJJ/giTZUqRR1lylHiV+IXEYm9ZcvgzTe9EDBxov9cvjz8619bagN22y3qKFNCYRJ/Us3VLyIicVStmi+09MwzPmvie+9B//4+5LJfP1+F8eKLfZ8kDCV+ERHZeWXKwJFHwj33wM8/e1+A3r1h1CjvCHjDDVvPYSCRUeIXEZHiFYKvtDh6tM9R0LkzDB3qBYB77/UhhxIZJX4REYmdffbxlRe/+AJatYJLLvFFjR5/3OcdkLhT4hcRkdhr2xYmTfJX7dq+2NIBB3jHwCTsZJ7IlPhFRCR+OnTweQFefBE2boSuXeHQQ+Gjj6KOLGUo8YuISHyFAKec4u3/Y8bA3Lk+K+Bxx8GMGVFHl/SU+EVEJBqlS0PfvjBnDtx2m6+62KoV9OgBv/wSdXRJS4lfRESiVaECXHmlJ/urroJx42DffeGCC2DhwqijSzpK/CIikhiqV4fhw70GoE8fePBBHwJ42WW+TYqFEr+IiCSW3XeHBx6AWbO889/IkdC0KXTq5OsEaBjgTlHiFxGRxNS0KTz9NPz2m08A9O23Pv9/48Zwyy3xnQp42TIfifDYY7ByZfyuGwNK/CIikth23x2uv957/7/0kif+a66BPfbwjoCfflr8cwGYwTffwK23+oiDWrXg1FN9GuLdd/c1Cb75pnivGSdanU9EREqeWbO8OeDxx30NgFat4Pzz4Ywzir4s8IoV8M478MYbvurg77/79lat4NhjferhtDQfgvjcc7BuHRx8sBcCTj3VOylGRMvyKvGLiKSGVau8OWDUKG8KqFrVZwU87zyfLnh7zHwugYkTPdl//DFkZsIuu0DHjp7oO3XyJ/zc/vc/L3Q8+CDMnu0dE3v18kLAjq4bA0r8SvwiIqnFDD75BP7zH28O2LgR/vUvGDAAunTxOQPACwrvvuuJfuJEmDfPt7ds6Yn+2GPhkEN8tcGCXnfyZC8AjBvn1z3ySC8AnHgilC0bm983FyV+JX4RkdS1cCE8/LCvDjhvHtSrB926eY3Ahx96cq5cGY4+2pN9585Qv37xXPfRR/26c+fCrrv6sMSMDGjYcOfPvx1K/Er8IiKSmQmvv+7NAJMmQfPmW57q27eP3dP4pk3w9tteC/Daa14r0Lmz1wIce6z3EyhmSvxK/CIiktP69VCuXPyvO2+e1z489BAsWOAjETIyvCagbt1iu0xhEr+G84mISPKLIumDJ/qhQ30ugs1TEQ8eDDfeGE08QOnIriwiIpIqypSBk07y15w5cev0lxclfhERkXhq0iTSyyd8VX8I4cQQwkMhhFdDCB2jjkdERKQki2niDyGMDSEsCiHMzLW9UwhhdghhTgjhqu2dw8xeMbO+wNlA9xiGKyIikvRiXdX/GHA/8MTmDSGENGAUcDQwH/gyhDAeSANuyfX+c8xs8yoM12W/T0RERIooponfzD4MITTMtbkdMMfMfgEIITwHdDWzW4Djc58jhBCAW4GJZvZVLOMVERFJdlG08dcD5uX4eX72tvxcCHQATgkh9M/voBBCRghhaghh6uLFi4snUhERkSQTRa/+kMe2fGcRMrORwMgdndTMxgBjwCfwKXJ0IiIiSSyKJ/75wB45fq4P/BFBHCIiIiknisT/JdA0hLBXCKEscBowPoI4REREUk6sh/M9C3wG7BNCmB9C6GNmmcAFwFvALOAFM/sulnGIiIiIi3Wv/tPz2f4G8EYsry0iIiLbSsrV+UIIi4HfgFrAXxGHk+x0j2NL9ze2dH9jS/c3tnLe3wZmVrsgb0rKxL9ZCGFqQZcplKLRPY4t3d/Y0v2NLd3f2Crq/U34ufpFRESk+Cjxi4iIpJBkT/xjog4gBegex5bub2zp/saW7m9sFen+JnUbv4iIiGwt2Z/4RUREJIekTfwhhE4hhNkhhDkhhKuijifZhBDmhhC+DSFMDyFMjTqeZBBCGBtCWBRCmJljW40QwqQQwk/ZX6tHGWNJls/9vSGE8Hv253h6COHYKGMsyUIIe4QQ3g8hzAohfBdCuDh7uz7DxWA797fQn+GkrOoPIaQBPwJH42sDfAmcbmbfRxpYEgkhzAXSzUxjdItJCOEwYBXwhJntn73tdmCpmd2aXYCtbmaDooyzpMrn/t4ArDKzO6OMLRmEEOoCdc3sqxBCFWAacCJwNvoM77Tt3N9TKeRnOFmf+NsBc8zsFzPbADwHdI04JpHtMrMPgaW5NncFHs/+/nH8P7oUQT73V4qJmS0ws6+yv1+JT8leD32Gi8V27m+hJWvirwfMy/HzfIp4gyRfBrwdQpgWQsiIOpgkVsfMFoD/xwd2jTieZHRBCGFGdlOAqqGLQQihIXAgMAV9hotdrvsLhfwMJ2viD3lsS742jWi1N7PWQGdgQHY1qkhJ8wDQGGgFLADuijacki+EUBn4P+ASM1sRdTzJJo/7W+jPcLIm/vnAHjl+rg/8EVEsScnM/sj+ugh4GW9ekeK3MLttb3Mb36KI40kqZrbQzDaZWRbwEPoc75QQQhk8KT1tZuOyN+szXEzyur9F+Qwna+L/EmgaQtgrhFAWOA0YH3FMSSOEUCm7cwkhhEpAR2Dm9t8lRTQe6JX9fS/g1QhjSTqbE1K2k9DnuMhCCAF4BJhlZnfn2KXPcDHI7/4W5TOclL36AbKHNIwA0oCxZnZzxCEljRBCI/wpH3xp52d0f3deCOFZ4Ah8xa2FwBDgFeAFYE/gv0A3M1MHtSLI5/4egVeRGjAX6Le5PVoKJ4RwKPAR8C2Qlb35GrwdWp/hnbSd+3s6hfwMJ23iFxERkW0la1W/iIiI5EGJX0REJIUo8YuIiKQQJX4REZEUosQvIiKSQpT4RWQbIYRNOVb7ml6cK1yGEBrmXCFPROKrdNQBiEhCWmtmraIOQkSKn574RaTAQghzQwi3hRC+yH41yd7eIITwbvZCIe+GEPbM3l4nhPByCOGb7Nc/sk+VFkJ4KHtd8bdDCBUi+6VEUowSv4jkpUKuqv7uOfatMLN2wP347Jhkf/+EmbUEngZGZm8fCUw2swOA1sB32dubAqPMrDmwDPh3jH8fEcmmmftEZBshhFVmVjmP7XOBo8zsl+wFQ/40s5ohhL+Auma2MXv7AjOrFUJYDNQ3s/U5ztEQmGRmTbN/HgSUMbObYv+biYie+EWksCyf7/M7Ji/rc3y/CfU3EokbJX4RKazuOb5+lv39p/gqmABnAh9nf/8ucB5ACCEthLBLvIIUkbyplC0ieakQQpie4+c3zWzzkL5yIYQp+IPD6dnbLgLGhhCuABYDvbO3XwyMCSH0wZ/szwO0+p1IhNTGLyIFlt3Gn25mf0Udi4gUjar6RUREUoie+EVERFKInvhFRERSiBK/iIhIClHiFxERSSFK/CIiIilEiV9ERCSFKPGLiIikkP8HdtkfzjQ14M0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "while True:\n",
    "    model.train()\n",
    "    cost = 0\n",
    "    for specs, labels in tqdm(trainloader, desc=\"Train\"):\n",
    "        specs, labels = specs.to(device), labels.to(device)\n",
    "        \n",
    "        output, _ = model(specs, None)\n",
    "        loss = criterion(output, labels)\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        cost += loss.item()*specs.shape[0]\n",
    "    train_cost = cost/len(train_set)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        cost = 0\n",
    "        for specs, labels in tqdm(testloader, desc=\"Test\"):\n",
    "            specs, labels = specs.to(device), labels.to(device)\n",
    "        \n",
    "            output, _ = model(specs, None)\n",
    "            loss = criterion(output, labels)\n",
    "            \n",
    "            cost += loss.item()*specs.shape[0]\n",
    "        test_cost = cost/len(test_set)\n",
    "\n",
    "    # Logging\n",
    "    callback.log(train_cost, test_cost)\n",
    "        \n",
    "    # Runtime Plotting\n",
    "    callback.cost_runtime_plotting()\n",
    "    \n",
    "    # Early Stopping\n",
    "    if callback.early_stopping(model, monitor=\"test_cost\"):\n",
    "        callback.plot_cost()\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:speech_diarization]",
   "language": "python",
   "name": "conda-env-speech_diarization-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}