IAM_HTR_CTC08.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from IPython.display import display\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from warpctc_pytorch import CTCLoss\n",
    "\n",
    "from lib.utilities import HYPERPARAMETERS, Metric, torch_weight_init, print_model_summary, DelayedKeyboardInterrupt, \\\n",
    "    plot_learning_curves, visualize_data, SequenceWise\n",
    "from lib.vocabulary import Vocabulary\n",
    "from lib.dataloader import DataArgumentation, FromNumpyToTensor, IAMHandwritingDataset, alignCollate, ResizeAndPad\n",
    "from lib.checkpoint import Checkpoint\n",
    "from lib.model_crnn import CRNN\n",
    "from lib.decoder import CTCGreedyDecoder\n",
    "from lib.stopping import Stopping\n",
    "from lib.logger import PytorchLogger, TensorboardLogger\n",
    "from lib.spelling import Spelling\n",
    "from lib.trainer import Trainer\n",
    "from lib.evaluator import Evaluator\n",
    "from lib.recognizer import Recognizer\n",
    "from lib.scorer import Scorer, SpellingScorer, ScorerSM, ScorerLEV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.cuda.is_available = lambda : False\n",
    "#torch.backends.cudnn.enabled=False\n",
    "torch.backends.cudnn.deterministic = True\n",
    "#torch.backends.cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 0\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(seed)\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "H = HYPERPARAMETERS({\n",
    "    'MODEL_PATH': './chkpt/IAM_HTR_CTC08.tar',\n",
    "    'ROOT_DIR': './data',\n",
    "    'EXPERIMENT': 'IAM_HTR_CTC08',\n",
    "    'BATCH_SIZE': 16,\n",
    "    'HEIGHT': 64,\n",
    "    'PADDING': 10,\n",
    "    'NUM_WORKERS': 8,\n",
    "    'HIDDEN_SIZE': 256,\n",
    "    'NUM_LAYERS': 2,\n",
    "    'RNN_DROPOUT': 0.5,\n",
    "    'LR': 0.0003,\n",
    "    'LR_LAMBDA': lambda epoch: max(math.pow(0.78, math.floor((1 + epoch) / 11.0)), 0.4),\n",
    "    'WEIGHT_DECAY': 0,\n",
    "    'MAX_GRAD_NORM': 5.,\n",
    "    'ARGUMENTATION': 0.9,\n",
    "    'STOPPING_PATIENCE': 10,\n",
    "    'NUM_EPOCHS': 20,\n",
    "    \n",
    "    'CHECKPOINT_FILE': 'IAM_HTR_CTC08',\n",
    "    'CHECKPOINT_INTERVAL': 5,\n",
    "    'CHECKPOINT_RESTORE': False,\n",
    "    \n",
    "    'USE_CUDA': torch.cuda.is_available(),\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdYAAAGDCAYAAACWWTEeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucFPWd7//XZ249jYwYcbzAjIIyksVVjCZojNklcVcwF4gbL2C8xOiSzdETL+eXlcSsGg2PR8zF24lu1ijeFgPGNQkxrLfoxKwJF/V4Q0IcReOERAUUZoBhbp/fH1UDTds9F+juqup5Px+Pedj9rapvfapp5z1V9a0qc3dERESkMCqiLkBERKScKFhFREQKSMEqIiJSQApWERGRAlKwioiIFJCCVUREpIAUrCKSCGb2cTNbHXUdIgNRsIrsAjN73cz+IUf7VDPrNbN2M2szs9Vmdm4//YwzMw/nbzezt8zsQTP7x+JuwdCY2VVm1pVRZ7uZ/WuR1+lmNqHvvbv/1t0nFnOdIoWgYBUpvLXuPhLYE7gE+LGZDRQIe4XLTAYeBX5mZl8sbplDtsjdR2b8fDfqgkTiSMEqUiQeWAJsAI4Y5DJ/dfcbgauAa82sAsDMxpjZf5nZO2a2xsy+2reMmVWa2TfM7NVwL/kZM2sMp91oZm+a2aaw/eNh+/5mtsXMRmf0c3TYf/VQtjN77z3cu/3P8HXfHvk5ZvYnM1tnZpcPVLuZPRnO8ny4d3x6eDSgNWPZvzGzZjN7z8xWmtmMjGl3mtnNZvarsN9lZnZIOM3M7Hoze9vMNprZC2b2t0PZZpH+KFhFisTMKsJf9vsALUNc/AFgX2BiGK6/BJ4HxgInABeb2bRw3kuB2cCnCPaSvwRsCaetAI4E9gbuBX5qZrXu/legGTgtY51nAgvdvWuItQ7G8cDEsPYrzOxv+qvd3f8unD453DtelNlZGP6/BB4h+Jz+N7Ag68jAbOBbwAcIPv95YfuJwN8BhwJ7AacD6wu3qTLcKVhFCm+Mmb0HbAV+Blzq7v9viH2sDf+7N/ARoN7dr3b3Tnd/DfgxMCuc53zgm+6+OtxLft7d1wO4+3+6+3p373b3HwApgoADuIsgTDGzSoIguqefmk4L9w77fsYMYXu+5e5b3f15gj8QJg9U+wCOBUYC3wk/k8eBB8Nt6POAuy93925gAcEfGABdQB3wQcDcfZW7/2UI2yLSLwWrSOGtdfe9CPbAbgI+uQt9jA3/uwE4iDCs+36AbwD7hfM0Aq/m6sTM/o+ZrQoPeb4HjCLYgwb4BTDJzA4G/hHY6O7L+6npPnffK+NnbT/zZvtrxustBKHYb+0DGAO86e69GW1vsONzy7vOMIR/CNwMvGVmt5rZnrtQg0hOClaRInH3bcBlwOFm9rkhLn4y8DawGngTWJMVanXu/qlw3jeBQ7I7CM+nXkZwuPcDYdhvBCysrwO4D/gCcBb97632ZzMwIuP9/kNYNmftg7AWaOw7Bx06EPjzYBZ295vc/WjgMIJDwl/bhRpEclKwiuy6ajOrzfipyp7B3TuBHwBXDKZDM9vPzC4ErgS+Hu6RLQc2mdllZpYOB/z8rZl9JFzsNuAaM2sKB+YcEQ5KqgO6gXeAKjO7gmAvOtPdwBeBGcB/DvUDCD0HzDKzajP7MHDKEJbNVzvAW8DBeZZbRhDo/xqudyrwWWDhQCs0s4+Y2THhedrNQAfQM4SaRfqlYBXZdUsIzqP2/VyVZ775wIFm9tl++nrPzDYDLxIM5DnV3ecDuHsPQWgcCawB1hEE0qhw2esI9jwfATYBtwNp4GHgv4E/Ehwm7SDYQ9zO3Z8CeoFn3f31wW32+/wbwV7nuwSDhe4dwrL5aofg87wrPPydOciq7w+WGcBJBJ/HLcDZ7v6HQaxzT4Jz1O8SfC7rge8PoWaRfpkedC4yvJnZ48C97n5b1LWIlAMFq8gwFh5OfhRodPe2qOsRKQc6FCwyTJnZXcBjwMUKVZHC0R6riIhIAWmPVUREpIAUrCIiIgX0vuvuBPbaay+fMGHCwDPGyObNm9ljjz2iLmPQVG/xJa3mpNULyatZ9e6eZ555Zp271w80n4I1h/3224+nn3466jKGpLm5malTp0ZdxqCp3uJLWs1JqxeSV7Pq3T1m9sZg5tOhYBERkQJSsIqIiBSQglVERKSAdI5VRKSAurq6aG1tpaOjI+pS3mfUqFGsWrUq6jIGLap6a2traWhooLq6epeWV7CKiBRQa2srdXV1jBs3DjOLupydtLW1UVdXF3UZgxZFve7O+vXraW1tZfz48bvUhw4Fi4gUUEdHB6NHj45dqMrgmBmjR4/erSMOClYRkQJTqCbb7v77KVhFREQKSMEqIjIM3HDDDWzZsqWo6xg5cmRR+y+k119/nXvvvbcofRc1WM1supmtNrMWM5ubY3rKzBaF05eZ2biMaV8P21eb2bSB+jSz283seTN7wczuN7ORA61DRGS4uOGGG9i6dWvUZcRGMYO1aKOCzawSuBn4R6AVWGFmi9395YzZzgPedfcJZjYLuBY43cwmAbOAw4AxwGNmdmi4TL4+L3H3TeG6rwMuBL6Tbx3F2m4RkT6vvHIx7e3PFbTPkSOPpKnphn7n2bx5M6eddhqtra309PRw6qmnsnbtWj796U+z77778sQTT/DII49w5ZVXsm3bNg455BDuuOMORo4cybhx4zj99NN54oknALj33nvJd+/0NWvWcMYZZ9Dd3c306dN3mva9732P++67j23btnHyySfzrW99C4C7776b73//+5gZRxxxBPfccw+//OUv+fa3v01nZyejR49mwYIF1NfXc+SRR7J06VLq6+vp7e3l0EMPZenSpeyzzz7vq+WNN97gS1/6Eu+88w719fXccccdHHjggXzxi1/kM5/5DKecckr4+Y2kvb2duXPnsmrVKo488kjOOeccLrnkkiH/W+RTzMttpgAt7v4agJktBGYCmcE6E7gqfH0/8EMLzhrPBBa6+zZgjZm1hP2Rr8+MUDUgDXh/6/AiP4h248an6O7eVMxVZHmB9euT9Neo6t0VVVV7sueex2lwjPTroYceYsyYMfzqV78CYOPGjdxxxx386le/Yty4caxbt45vf/vbPPbYY+yxxx5ce+21XHfddVxxxRUA7Lnnnixfvpy7776biy++mAcffDDnei666CK+8pWvcPbZZ3PzzTdvb3/kkUd45ZVXWL58Oe7OjBkzePLJJxk9ejTz5s3jqaeeYp999mHDhg0AHH/88SxduhQz47bbbuO73/0uP/jBDzj99NNZsGABF198MY899hiTJ0/OGaoAF154IWeffTbnnHMO8+fP56tf/So///nP835G3/nOd/j+97+fd9t2RzGDdSzwZsb7VuCYfPO4e7eZbQRGh+1Ls5YdG77O26eZ3QF8iiC8/88A61iXWYiZzQHmANTX19Pc3Dz4Lc3py8Afd7OPoXnxxZKubrep3l31H8ChA87V3t5egO9x6SStXshd86hRo2hrawNg//2vKcp6+/rPZ/z48Tz66KNccsklTJ8+neOOOw53p6enh7a2Nh5//HFWrlzJRz/6UQA6OzuZMmUKbW1tuDuf/exnaWtr4zOf+QwXX3xx3vX9z//8D3feeSdtbW187nOf47LLLqOtrY0HH3yQhx9+mMmTJwPB5/Tiiy+yZcsWZsyYQSqVoq2tjerqatra2li9ejXf+MY3eOutt+js7OSggw6ira2NM844gzPPPJPzzjuP//iP/+D000/PW8vvfvc77rrrru21fO1rX6OtrY2uri62bt2603JtbW1s2bKF7u7uvP11dHTs8vexmMGa60/q7L3EfPPka891Tnh7n+5+bngI+v8SHO69Y5B14O63ArcCTJw40Xf3iQrt7f9Fb+/m3epjKJ599hmOOurokq1vd6neodu6dQ2rVs1m0qS92HffqQPOH7cngwwkafVC7ppXrVoV+U0YjjrqKJ599lmWLFnCNddcw4knnoiZUVlZSV1dHel0mhNPPJGf/OQn71vWzKirq6Ouro6uri4qKirybo+Zseeee1JVVUXfQcC6ujqqq6v5xje+wZe//OWd5r/ppptIpVLv62/u3LlceumlzJgxg+bmZq666irq6uo46KCDOOCAA1ixYgXPPvss9913H5WVlXlr6Vt3Zt3pdHr7Ot2dzs5O6urqGDFiBFVVVXm3rba2lg996EMDfta5FDNYW4HGjPcNwNo887SaWRUwCtgwwLL99unuPWa2CPgaQbDmW0dRjRz5t8VeRZat7Lln9gGBOFO9Q5VOTwRg27Y3B5hThru1a9ey9957c+aZZzJy5EjuvPNO6urqtu+dHXvssVxwwQW0tLQwYcIEtmzZQmtrK4ceGhwJWbRoEXPnzmXRokXb92pz+djHPsbChQs588wzWbBgwfb2adOm8W//9m984QtfYOTIkfz5z3+murqaE044gZNPPplLLrmE0aNHs2HDBvbee282btzI2LHBQcm77rprp3Wcf/75nHnmmZx11ll5QxXguOOOY+HChZx11lksWLCA448/HoBx48bxzDPPcNppp/GLX/yCrq4ugJ0+j0Ir5qjgFUCTmY03sxqCwUiLs+ZZDJwTvj4FeDw897kYmBWO6B0PNAHL8/VpgQmw/RzrZ4E/DLAOkUSpqhpFZeVItm1rjboUibkXX3yRKVOmcOSRRzJv3jy++c1vMmfOHD7/+c/ziU98gvr6eu68805mz57NEUccwbHHHssf/vCH7ctv27aNY445hhtvvJHrr78+73puvPFGbr75Zj7ykY+wcePG7e0nnngiZ5xxBh/96Ec5/PDDOeWUU2hra+Owww7j8ssv5+///u+ZPHkyl156KQBXXXUVp556Kh//+Mffdw51xowZtLe3c+655/a7zTfddBN33HHH9gFRN954IwD//M//zG9+8xumTJnCsmXLtj84/YgjjqCqqorJkyf3u427xN2L9kNwvvOPwKvA5WHb1cCM8HUt8FOghSA4D85Y9vJwudXASQP0WQE8BbwIvAQsAPYcaB35fg499FBPmieeeCLqEoZE9e6aZcs+6C+++PlBzRuXmgcrafW656755ZdfLn0hg7Rp06YB5znooIP8nXfeKUE1A9u0aZOvWLHCjz/++JKvO9e/I/C0DyL7inoTfndfAizJarsi43UHcGqeZecB8wbZZy/wsTz95F2HSNKkUo06FCzDxnXXXcf8+fN3OsycBHq6jUiCpFKNbN78UtRlSBl7/fXX39c2b948fvrTn+7Uduqpp3L55ZcXtZZLL72UK6+8Mha1DIWCVSRBUqlGOjv/Sm9vFxUVu/asSJGhuvzyy2MTXHGqJR/dK1gkQVKpBsDp7MweYC9x4hofmWi7+++nYBVJkNra4Gqzjg6dZ42r2tpa1q9fr3BNKA8fdF5bW7vLfehQsEiCpFJBsOqSm/hqaGigtbWVd955J+pS3qejo2O3AqPUoqq3traWhoaGXV5ewSqSIDuCVXuscVVdXc348eOjLiOn5ubmXb6bUBSSVm8fHQoWSZCqqjoqK/dUsIrEmIJVJGF0LatIvClYRRKmtrZR51hFYkzBKpIwqVSDRgWLxJiCVSRhUqlGurreord3W9SliEgOClaRhNkxMlg3iRCJIwWrSMLokhuReFOwiiRMcFtDBatIXClYRRJGtzUUiTcFq0jCVFbuQVXVB3TJjUhMKVhFEiiVatChYJGYUrCKJJDuviQSXwpWkQRSsIrEl4JVJIFqaxvp6lpHT09H1KWISBYFq0gC7bjkRgOYROJGwSqSQLpJhEh8KVhFEmhHsGqPVSRuFKwiCZRKjQW0xyoSRwpWkQSqrBxBVdVoBatIDClYRRKqtrZRtzUUiSEFq0hCBdey6hyrSNwoWEUSSrc1FIknBatIQqVSjXR3b6CnZ0vUpYhIBgWrSELpkhuReFKwiiRU33NZdThYJF4UrCIJ1XdbQ40MFokXBatIQu24X7CCVSROFKwiCVVRkaK6el+dYxWJGQWrSILpkhuR+FGwiiSYHnguEj8KVpEEq63V3ZdE4kbBKpJgwU0i3qO7uz3qUkQkpGAVSTCNDBaJHwWrSILtuPuSglUkLhSsIgmm2xqKxI+CVSTBUqkxgGmPVSRGihqsZjbdzFabWYuZzc0xPWVmi8Lpy8xsXMa0r4ftq81s2kB9mtmCsP0lM5tvZtVh+ygz+6WZPW9mK83s3GJus0gpVVTUUFOzn25rKBIjRQtWM6sEbgZOAiYBs81sUtZs5wHvuvsE4Hrg2nDZScAs4DBgOnCLmVUO0OcC4IPA4UAaOD9svwB42d0nA1OBH5hZTeG3WCQaupZVJF6Kucc6BWhx99fcvRNYCMzMmmcmcFf4+n7gBDOzsH2hu29z9zVAS9hf3j7dfYmHgOVAQ9ivA3VhvyOBDUB3cTZZpPSCYNU5VpG4qCpi32OBzD+jW4Fj8s3j7t1mthEYHbYvzVp2bPi63z7DQ8BnAReFTT8EFgNrgTrgdHfvzS7WzOYAcwDq6+tpbm4ezDbGRnt7e6JqVr2F9vr76ot/zTtLWr2QvJpVb2kUM1gtR5sPcp587bn2sLP7vAV40t1/G76fBjwHfBI4BHjUzH7r7pt26sT9VuBWgIkTJ/rUqVNzrCq+mpubSVLNqrdw/vSnp3nttQc4/vgPUVU1ant7nGvOJWn1QvJqVr2lUcxDwa1AY8b7BoK9xpzzmFkVMIrgUG2+Zfvt08yuBOqBSzPmORd4IDxK3AKsITgXK1IWdjzwXIeDReKgmMG6Amgys/HhYKFZBIdkMy0GzglfnwI8Hp4jXQzMCkcNjweaCM6b5u3TzM4n2DudnXWo90/ACeE8+wETgdcKvrUiEdEDz0XipWiHgsNzphcCDwOVwHx3X2lmVwNPu/ti4HbgHjNrIdhTnRUuu9LM7gNeJhhodIG79wDk6jNc5Y+AN4DfB+OUeMDdrwauAe40sxcJDjFf5u7rirXdIqWmuy+JxEsxz7Hi7kuAJVltV2S87gBOzbPsPGDeYPoM23Nui7uvBU4cUuEiCVJTMwaoULCKxITuvCSScBUVVdTUHKBzrCIxoWAVKQOpVIP2WEViQsEqUgZqaxs1eEkkJhSsImWg7+5LwaB6EYmSglWkDKRSjfT2bqa7+72oSxEZ9hSsImWg71pWnWcViZ6CVaQM6FpWkfhQsIqUAd3WUCQ+FKwiZaCmZn+gUiODRWJAwSpSBswqSaXG6FCwSAwoWEXKhB54LhIPClaRMhEEq/ZYRaKmYBUpE323NdRNIkSipWAVKRO1tY309nbQ1bU+6lJEhjUFq0iZ2HEtq86zikRJwSpSJnT3JZF4ULCKlAndfUkkHhSsImWipmY/zKoVrCIRU7CKlAmzClKpsTrHKhIxBatIGUmlGnRbQ5GIKVhFyohuEiESPQWrSBnpu62hbhIhEh0Fq0gZSaUacO+kq+udqEsRGbYUrCJlRJfciERPwSpSRvoeeK4BTCLRUbCKlBHd1lAkegpWkTJSXb0PZjU6FCwSIQWrSBkJbhLRoGAViZCCVaTM9F1yIyLRULCKlJnaWt0kQiRKClaRMhMcCv4z0Bt1KSLDkoJVpMykUo24dwHvRl2KyLCkYBUpM32X3IDuviQSBQWrSJlJpRrCV29HWofIcKVgFSkzO/ZYFawiUVCwipSZ6urRVFTUAuuiLkVkWFKwipQZMwv3WrXHKhIFBatIGQrOsypYRaKgYBUpQ8Eeq0YFi0RBwSpShoJgXYd7T9SliAw7ClaRMhQcCu6ls/OvUZciMuwUNVjNbLqZrTazFjObm2N6yswWhdOXmdm4jGlfD9tXm9m0gfo0swVh+0tmNt/MqjOmTTWz58xspZn9pnhbLBIPeuC5SHSKFqxmVgncDJwETAJmm9mkrNnOA9519wnA9cC14bKTgFnAYcB04BYzqxygzwXAB4HDgTRwftjXXsAtwAx3Pww4tThbLBIfOx54rmAVKbVi7rFOAVrc/TV37wQWAjOz5pkJ3BW+vh84wcwsbF/o7tvcfQ3QEvaXt093X+IhYDnQd/uZM4AH3P1P4XwaKillb0ew6vFxIqVWVcS+xwKZfy63Asfkm8fdu81sIzA6bF+atezY8HW/fYaHgM8CLgqbDgWqzawZqANudPe7s4s1sznAHID6+nqam5sHs42x0d7enqiaVW+xOe4pXn31d7z66oeiLmZQkvcZJ69m1VsaxQxWy9Hmg5wnX3uuPezsPm8BnnT334bvq4CjgRMIDhH/3syWuvsfd+rE/VbgVoCJEyf61KlTc6wqvpqbm0lSzaq3+Jqb96W+Hg47bGrUpQxKMj/jZNWsekujmMHaCjRmvG8A1uaZp9XMqoBRwIYBls3bp5ldCdQDX85axzp33wxsNrMngcnATsEqUn721aFgkQgU8xzrCqDJzMabWQ3BYKTFWfMsBs4JX58CPB6eI10MzApHDY8HmgjOm+bt08zOB6YBs9098wnPvwA+bmZVZjaC4NDxqiJsr0jM1GtUsEgEirbHGp4zvRB4GKgE5rv7SjO7Gnja3RcDtwP3mFkLwZ7qrHDZlWZ2H/Ay0A1c4OGV7rn6DFf5I+ANgkO9EAxYutrdV5nZQ8ALQC9wm7u/VKztFomPejo7/0JvbzcVFcU8OCUimYr6f5u7LwGWZLVdkfG6gzyXv7j7PGDeYPoM2/Nui7t/D/jeoAsXKQv7EtwkYi21tQdGXYzIsKE7L4mUrX0BXXIjUmoKVpGyVQ/oJhEipaZgFSlbQbBqAJNIaSlYRcrWSCor63QoWKTEFKwiZSyVatChYJESU7CKlLFUqlHBKlJiClaRMqZgFSk9BatIGautbaSz8y16ezujLkVk2FCwipSxVKoBcLZty75Nt4gUi4JVpIzpgecipadgFSljClaR0lOwipSx4FCwbmsoUkoKVpEyVlVVR2XlKO2xipSQglWkzNXWNuq2hiIlpGAVKXPBtaw6FCxSKgpWkTKn2xqKlJaCVaTMpVKNdHW9TW/vtqhLERkWFKwiZW7HJTc6HCxSCgpWkTJXW6tgFSklBatImeu7llUjg0VKQ8EqUuZ09yWR0lKwipS5ysoRVFXtrUPBIiWiYBUZBnTJjUjpKFhFhgE98FykdBSsIsOAbmsoUjoKVpFhIJVqpLt7PT09W6MuRaTsKVhFhgE9Pk6kdBSsIsOALrkRKR0Fq8gwoNsaipSOglVkGEilxgLaYxUpBQWryDBQWZmmunofjQwWKQEFq8gwoWtZRUpDwSoyTATBqnOsIsWmYBUZJnRbQ5HSULCKDBPBTSLepadnc9SliJQ1BavIMNH3wHMNYBIpLgWryDCha1lFSkPBKjJM7LitofZYRYppwGA1s0oz2yfjfY2ZzTGzVcUtTUQKSTeJECmNfoPVzGYBG4AXzOw3ZvYJ4DXgJOALJahPRAqkoiJFdfV+OhQsUmRVA0z/JnC0u7eY2VHA74FZ7v6z4pcmIoWmS25Eim+gQ8Gd7t4C4O7PAmuGEqpmNt3MVptZi5nNzTE9ZWaLwunLzGxcxrSvh+2rzWzaQH2a2YKw/SUzm29m1Vnr+oiZ9ZjZKYOtX6Tc6IHnIsU30B7rvmZ2acb7kZnv3f26fAuaWSVwM/CPQCuwwswWu/vLGbOdB7zr7hPCw87XAqeb2SRgFnAYMAZ4zMwODZfJ1+cC4MxwnnuB84F/z6jlWuDhAbZXpKylUo28++7jUZchUtYG2mP9MVCX8ZP5fuQAy04BWtz9NXfvBBYCM7PmmQncFb6+HzjBzCxsX+ju29x9DdAS9pe3T3df4iFgOdCQsZ7/DfwX8PYANYuUtVSqkZ6eTXR3b4q6FJGy1e8eq7t/K980M7t4gL7HApnHnFqBY/LN4+7dZrYRGB22L81admz4ut8+w0PAZwEXhe/HAicDnwQ+MkDNImVtxyU3rVRVTYq4GpHyNNCh4P5cCtzQz3TL0eaDnCdfe6497Ow+bwGedPffhu9vAC5z955gZzhPsWZzgDkA9fX1NDc35503jtrb2xNVs+otvtw1rwNgxYoHidsBnPL5jONL9ZbG7gRr/pQKtAKNGe8bgLV55mk1sypgFMHlPf0tm7dPM7sSqAe+nDHPh4GFYajuA3zKzLrd/eeZhbj7rcCtABMnTvSpU6cOsHnx0tzcTJJqVr3Fl6vmjo7xLF36VSZOHM0BB0zNuVxUyuUzjjPVWxq7c+el7D3FbCuAJjMbb2Y1BIORFmfNsxg4J3x9CvB4eI50MTArHDU8HmgiOG+at08zOx+YBsx2997tRbqPd/dx7j6O4Dzu/8oOVZHhoqZmDGAaGSxSRP3usZpZG7kD1IB0f8uG50wvJBiJWwnMd/eVZnY18LS7LwZuB+4xsxaCPdVZ4bIrzew+4GWgG7jA3XvCmt7XZ7jKHwFvAL8P904fcPerB/oARIaTiopqamr217WsIkU00OClut3p3N2XAEuy2q7IeN0BnJpn2XnAvMH0GbYPeFjb3b84YNEiZS544LmCVaRYdBN+kWEmCFbd1lCkWBSsIsNM320Ng+EMIlJoClaRYaa2tpGenna6uzdGXYpIWVKwigwzeuC5SHEpWEWGGT3wXKS4FKwiw8yOPVYFq0gxKFhFhpmamgOACgWrSJEoWEWGmYqKKlKpMTrHKlIkClaRYSiVatBtDUWKZHduwi8iCZVKNbJhw0O88MKnoy4lw3peeGF01EUMUdJqTk69NTX7A2dEXcYuUbCKDEP77juLjo7X6eqK06Pj2ujq6om6iCFKWs3JqLe7eyMbNiwBjou6lF2iYBUZhurr/4n6+n+KuoydNDc3c/TRU6MuY0iSVnNS6m1re5Znnjka+HPUpewSnWMVEZFYSaebwlcKVhERkd1WVVVHdfV+QDJHritYRUQkdkaMaELBKiIiUiDp9AR0KFhERKRAgvOs6+nubo+6lCFTsIqISOz0DWDq6Hg14kqGTsEqIiKx0xesW7a8EnElQ6dgFRGR2AnOscLWrQpWERGR3VZVNRIYrWAVEREpnLEKVhERkcIZq3OsIiIihdNAV9dbdHe3RV3IkChYRUQkpsYCsHVrS8R1DI2CVUREYqoBSN7IYAXwLccXAAAT+UlEQVSriIjE1BhAwSoiIlIgaWpqxiRuAJOCVUREYiudbtI5VhERkUIZMaJJh4JFREQKJZ1uoqvrbbq7N0VdyqApWEVEJLb6bsafpL1WBauIiMRW3834kzSAScEqIiKxlU4fAmiPVUREpCAqK0eQSjUkamSwglVERGItuORGe6wiIiIFoWAVEREpoHR6Al1d6+jqei/qUgZFwSoiIrGWtEtuFKwiIhJrI0YoWEVERAqmtvYQwBIzMljBKiIisVZZWUsq1ag9VgAzm25mq82sxczm5pieMrNF4fRlZjYuY9rXw/bVZjZtoD7NbEHY/pKZzTez6rD9C2b2QvjzOzObXMxtFhGRwkunmxJz96WiBauZVQI3AycBk4DZZjYpa7bzgHfdfQJwPXBtuOwkYBZwGDAduMXMKgfocwHwQeBwIA2cH7avAf7e3Y8ArgFuLcLmiohIEaXTE7THCkwBWtz9NXfvBBYCM7PmmQncFb6+HzjBzCxsX+ju29x9DdAS9pe3T3df4iFgOdAQtv/O3d8N17G0r11ERJJjxIgmurs30NW1IepSBlRVxL7HAm9mvG8Fjsk3j7t3m9lGYHTYvjRr2bHh6377DA8BnwVclKOm84D/zlWsmc0B5gDU19fT3NycZ7Piqb29PVE1q97iS1rNSasXkldzsuvtAOCppxYSHLCMr2IGq+Vo80HOk6891x52dp+3AE+6+293WpHZJwiC9fhcxbr7rYSHiSdOnOhTp07NNVtsNTc3k6SaVW/xJa3mpNULyas5yfVu3rwfK1Z8k7/5m1Hst9/USOsaSDGDtRVozHjfAKzNM0+rmVUBo4ANAyybt08zuxKoB76cuRIzOwK4DTjJ3dfv4vaIiEhE0umDgYpEDGAq5jnWFUCTmY03sxqCwUiLs+ZZDJwTvj4FeDw8R7oYmBWOGh4PNBGcN83bp5mdD0wDZrt7b98KzOxA4AHgLHf/Y5G2VUREiqiiIkVt7YGJGMBUtD3W8JzphcDDQCUw391XmtnVwNPuvhi4HbjHzFoI9lRnhcuuNLP7gJeBbuACd+8ByNVnuMofAW8Avw/GP/GAu18NXEFw3vaWsL3b3T9crO0WEZHiSMrI4GIeCsbdlwBLstquyHjdAZyaZ9l5wLzB9Bm259wWdz+fHZfeiIhIQqXTTbz99k9wd8IdpVjSnZdERCQR0ukmurvfo7s73pfcKFhFRCQR+p5yE/cBTApWERFJhKQ85UbBKiIiiVBbOx6oULCKiIgUQkVFDbW1BylYRURECiUJT7lRsIqISGKk001s3dpCcC+heFKwiohIYowY0URPz0a6utZFXUpeClYREUmMvktu4nyeVcEqIiKJkU5PABSsIiIiBRFcclMZ6wFMClYREUmMiopqamvHaY9VRESkUEaMCEYGx5WCVUREEiW45OaV2F5yo2AVEZFESaeb6Olpo6vr7ahLyUnBKiIiidI3MjiuA5gUrCIikihxv5ZVwSoiIolSWzsOs6rYDmBSsIqISKJUVFRRWztee6wiIiKF0jcyOI4UrCIikjjp9AS2bInnJTcKVhERSZx0uone3s10dv416lLeR8EqIiKJM2JEfEcGK1hFRCRxdlxyE7+RwQpWERFJnFTqQMyqtccqIiJSCMElNwfH8u5LClYREUmkdHqC9lhFREQKpe/xcXG75EbBKiIiiRRccrOFzs61UZeyEwWriIgkUlxHBitYRUQkkfqCNW4DmBSsIiKSSLW1jZjVxG4Ak4JVREQSyaySdPpgBauIiEihxPEpNwpWERFJrCBYW3DvjbqU7RSsIiKSWMElNx1s2xafS24UrCIiklhxfMqNglVERBIrnZ4AKFhFREQKIpVqxCylYBURESkEswrS6UNidZMIBauIiCRa38jguChqsJrZdDNbbWYtZjY3x/SUmS0Kpy8zs3EZ074etq82s2kD9WlmC8L2l8xsvplVh+1mZjeF879gZkcVc5tFRKS0RoxooqPj1dhcclO0YDWzSuBm4CRgEjDbzCZlzXYe8K67TwCuB64Nl50EzAIOA6YDt5hZ5QB9LgA+CBwOpIHzw/aTgKbwZw7w74XfWhERicqOS25aoy4FKO4e6xSgxd1fc/dOYCEwM2uemcBd4ev7gRPMzML2he6+zd3XAC1hf3n7dPclHgKWAw0Z67g7nLQU2MvMDijWRouISGnFbWRwMYN1LPBmxvvWsC3nPO7eDWwERvez7IB9hoeAzwIeGkIdIiKSUHF7yk1VEfu2HG3Zj3nPN0++9lx/CGT3eQvwpLv/dgh1YGZzCA4VU19fT3Nzc47F4qu9vT1RNave4ktazUmrF5JXc/nW2wvU8Morj/PKKx8sclUDK2awtgKNGe8bgOx7TvXN02pmVcAoYMMAy+bt08yuBOqBLw+xDtz9VuBWgIkTJ/rUqVP73bi4aW5uJkk1q97iS1rNSasXkldzOde7YsWh1NZu4/DDBzd/MRXzUPAKoMnMxptZDcFgpMVZ8ywGzglfnwI8Hp4jXQzMCkcNjycYeLS8vz7N7HxgGjDbdx4athg4OxwdfCyw0d3/UowNFhGRaMTpKTdF22N1924zuxB4GKgE5rv7SjO7Gnja3RcDtwP3mFkLwZ7qrHDZlWZ2H/Ay0A1c4O49ALn6DFf5I+AN4PfB+CcecPergSXApwgGQG0Bzi3WNouISDTS6QmsX/8r3HsILiCJTjEPBePuSwiCLbPtiozXHcCpeZadB8wbTJ9he85tCfeALxhS4SIikijpdBPunXR0vEk6PS7SWnTnJRERSby+kcFxOBysYBURkcSL0+PjFKwiIpJ4NTVjqKgYEYt7BitYRUQk8cyMdHqC9lhFREQKJZ2eEIu7LylYRUSkLKTTTXR0vEZvb3ekdShYRUSkLIwY0YR7F9u2/SnSOhSsIiJSFnZcchPtACYFq4iIlIW4XMuqYBURkbJQU7M/FRV7RD6AScEqIiJlIS6X3ChYRUSkbIwYEf1TbhSsIiJSNoJLbtZEesmNglVERMpG8JSbbrZteyOyGhSsIiJSNvpGBkc5gEnBKiIiZSOdngBEe8mNglVERMpGTc1+VFaOVLCKiIgUQnDJTbQjgxWsIiJSVtLpJp1jFRERKZTgkpvX6e3timT9ClYRESkrI0Y0AT10dLweyfoVrCIiUlaiHhmsYBURkbIS9VNuFKwiIlJWqqvrqazcM7IBTApWEREpK1FfcqNgFRGRshM85aYlknUrWEVEpOyk0xPCS246S75uBauIiJSdYABTLx0da0q+bgWriIiUnSifcqNgFRGRshPlJTcKVhERKTvV1aOpqtorkgFMClYRESk7UV5yo2AVEZGylE5PULCKiIgUSvCUmz/R27utpOtVsIqISFnqu+Rm69bXSrpeBauIiJSl4PFxpR8ZrGAVEZGytOOSm9KODFawiohIWaqu3puqqr21xyoiIlIo6fSEkt99qaqkaxMRESmhvfc+ka6ud0u6TgWriIiUrfHjryn5OnUoWEREpICKGqxmNt3MVptZi5nNzTE9ZWaLwunLzGxcxrSvh+2rzWzaQH2a2YVhm5vZPhnto8zsl2b2vJmtNLNzi7fFIiIy3BUtWM2sErgZOAmYBMw2s0lZs50HvOvuE4DrgWvDZScBs4DDgOnALWZWOUCfTwH/ALyRtY4LgJfdfTIwFfiBmdUUcltFRET6FHOPdQrQ4u6vuXsnsBCYmTXPTOCu8PX9wAlmZmH7Qnff5u5rgJawv7x9uvv/c/fXc9ThQF3Y70hgA9BdwO0UERHZrpiDl8YCb2a8bwWOyTePu3eb2UZgdNi+NGvZseHrgfrM9kNgMbAWqANOd/fe7JnMbA4wB6C+vp7m5uYBuo2X9vb2RNWseosvaTUnrV5IXs2qtzSKGayWo80HOU++9lx72Nl9ZpsGPAd8EjgEeNTMfuvum3bqxP1W4FaAiRMn+tSpUwfoNl6am5tJUs2qt/iSVnPS6oXk1ax6S6OYh4JbgcaM9w0Ee4055zGzKmAUwaHafMsOps9s5wIPeKAFWAN8cEhbIiIiMkjFDNYVQJOZjQ8HC80iOCSbaTFwTvj6FOBxd/ewfVY4ang80AQsH2Sf2f4EnABgZvsBE4HSPupARESGjaIFq7t3AxcCDwOrgPvcfaWZXW1mM8LZbgdGm1kLcCkwN1x2JXAf8DLwEHCBu/fk6xPAzL5qZq0Ee7EvmNlt4TquAY4zsxeBXwOXufu6Ym23iIgMb0W985K7LwGWZLVdkfG6Azg1z7LzgHmD6TNsvwm4KUf7WuDEodYuIiKyK3TnJRERkQJSsIqIiBSQglVERKSALBiEK5nMrA1YHXUdQ7QPkKRBWaq3+JJWc9LqheTVrHp3z0HuXj/QTHpsXG6r3f3DURcxFGb2dJJqVr3Fl7Sak1YvJK9m1VsaOhQsIiJSQApWERGRAlKw5nZr1AXsgqTVrHqLL2k1J61eSF7NqrcENHhJRESkgLTHKiIiUkDDOljNbLqZrTazFjObm2N6yswWhdOXmdm40le5vZZGM3vCzFaZ2UozuyjHPFPNbKOZPRf+XJGrr1Iys9fN7MWwnqdzTDczuyn8jF8ws6OiqDOsZWLGZ/ecmW0ys4uz5on8Mzaz+Wb2tpm9lNG2t5k9amavhP/9QJ5lzwnnecXMzsk1T4nq/Z6Z/SH8N/+Zme2VZ9l+vz8lrvkqM/tzxr/9p/Is2+/vlRLWuyij1tfN7Lk8y5b8M873+yzO3+Mhcfdh+QNUAq8CBwM1wPPApKx5/hfwo/D1LGBRhPUeABwVvq4D/pij3qnAg1F/tlk1vQ7s08/0TwH/TfAM3mOBZVHXnPH9+CvBdWux+oyBvwOOAl7KaPsuMDd8PRe4NsdyexM82Wlv4APh6w9EVO+JQFX4+tpc9Q7m+1Pimq8C/r9BfG/6/b1Sqnqzpv8AuCIun3G+32dx/h4P5Wc477FOAVrc/TV37wQWAjOz5pkJ3BW+vh84wcxyPYS96Nz9L+7+bPi6jeDpPmOjqKXAZgJ3e2ApsJeZHRB1UQSPGnzV3d+IupBs7v4kwXOLM2V+V+8CPpdj0WnAo+6+wd3fBR4Fphet0FCuet39EQ+eVgWwlOCpVLGR5zMejMH8Xim4/uoNf2edBvyk2HUMVj+/z2L7PR6K4RysY4E3M9638v6g2j5P+EtgIzC6JNX1Izwk/SFgWY7JHzWz583sv83ssJIWlpsDj5jZM2Y2J8f0wfw7RGEW+X8Rxe0zBtjP3f8CwS8tYN8c88T1s/4SwVGLXAb6/pTaheHh6/l5DlPG8TP+OPCWu7+SZ3qkn3HW77Mkf4+3G87BmmvPM3uI9GDmKSkzGwn8F3Cxu2/KmvwswaHLycD/BX5e6vpy+Ji7HwWcBFxgZn+XNT2On3ENMAP4aY7JcfyMByuOn/XlQDewIM8sA31/SunfgUOAI4G/EBxezRa7zxiYTf97q5F9xgP8Psu7WI62qD/jnQznYG0FGjPeNwBr881jZlXAKHbt8FBBmFk1wZdwgbs/kD3d3Te5e3v4eglQbWb7lLjM7JrWhv99G/gZwaGyTIP5dyi1k4Bn3f2t7Alx/IxDb/UdQg//+3aOeWL1WYeDTj4DfMHDk2fZBvH9KRl3f8vde9y9F/hxnlri9hlXAf8ELMo3T1SfcZ7fZ4n7HucynIN1BdBkZuPDPZRZwOKseRYDfSPOTgEez/cLoNjC8yS3A6vc/bo88+zfdw7YzKYQ/PuuL12V76tnDzOr63tNMGDlpazZFgNnW+BYYGPfoaAI5f0LP26fcYbM7+o5wC9yzPMwcKKZfSA8jHli2FZyZjYduAyY4e5b8swzmO9PyWSd+z85Ty2D+b1SSv8A/MHdW3NNjOoz7uf3WaK+x3lFPXoqyh+CEal/JBjFd3nYdjXB/+wAtQSHA1uA5cDBEdZ6PMHhjheA58KfTwH/AvxLOM+FwEqCkYhLgeMi/nwPDmt5Pqyr7zPOrNmAm8N/gxeBD0dc8wiCoByV0Rarz5gg9P8CdBH89X4ewbn/XwOvhP/dO5z3w8BtGct+Kfw+twDnRlhvC8F5sr7vct/o+zHAkv6+PxHWfE/4HX2BIAAOyK45fP++3ytR1Bu239n33c2YN/LPuJ/fZ7H9Hg/lR3deEhERKaDhfChYRESk4BSsIiIiBaRgFRERKSAFq4iISAEpWEVERApIwSpSxsysx3Z+Yk/BnrZiZuMyn6YiIoGqqAsQkaLa6u5HRl2EyHCiPVaRYSh8Bue1ZrY8/JkQth9kZr8ObzT/azM7MGzfz4Lnpj4f/hwXdlVpZj8On6n5iJmlw/m/amYvh/0sjGgzRSKhYBUpb+msQ8GnZ0zb5O5TgB8CN4RtPyR4jN8RBDfGvylsvwn4jQcPHziK4C49AE3Aze5+GPAe8PmwfS7wobCffynWxonEke68JFLGzKzd3UfmaH8d+KS7vxbeDP2v7j7azNYR3KqvK2z/i7vvY2bvAA3uvi2jj3EEz8VsCt9fBlS7+7fN7CGgneDpPz/38MEFIsOB9lhFhi/P8zrfPLlsy3jdw45xG58muAf00cAz4VNWRIYFBavI8HV6xn9/H77+HcETWQC+APxP+PrXwFcAzKzSzPbM16mZVQCN7v4E8K/AXsD79ppFypX+ihQpb2kzey7j/UPu3nfJTcrMlhH8gT07bPsqMN/Mvga8A5wbtl8E3Gpm5xHsmX6F4GkquVQC/2lmowieXnS9u79XsC0SiTmdYxUZhsJzrB9293VR1yJSbnQoWEREpIC0xyoiIlJA2mMVEREpIAWriIhIASlYRURECkjBKiIiUkAKVhERkQJSsIqIiBTQ/w9f5PUwUG6wSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bb397940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#https://research.fb.com/wp-content/uploads/2017/06/imagenet1kin1h5.pdf?\n",
    "\n",
    "max_epochs = H.NUM_EPOCHS\n",
    "epts = np.linspace(1, max_epochs+1, max_epochs)\n",
    "step_decay_out = np.vectorize(H.LR_LAMBDA)(epts) * H.LR\n",
    "\n",
    "plt.figure(figsize=(7,6))\n",
    "plt.plot(epts,step_decay_out, label='step_decay_out', color='y')\n",
    "plt.title(\"LR Decay Functions\")\n",
    "plt.xlabel(\"Epochs\");\n",
    "plt.ylabel(\"LR\");\n",
    "plt.grid(True)\n",
    "plt.legend(loc='upper right');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # IAMHandwritingDataset.create('./data/IAM/')\n",
    "# vocab = Vocabulary(\"./data/IAM/\")\n",
    "# train_dataset = IAMHandwritingDataset('./data/IAM/', vocab, dataset=\"train\")\n",
    "# len(train_dataset.annotations[0]['transcript'])\n",
    "# train_dataset.annotations[0]['transcript']\n",
    "# d = {}\n",
    "# for ann in train_dataset.annotations:\n",
    "#     for c in ann['transcript']:\n",
    "#         d[c] = 1 if not c in d else d[c] + 1\n",
    "# vocab.create(sorted(list(d.keys())))\n",
    "# vocab.dump()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "class Vocabulary(object):\n",
    "    def __init__(self, root_dir, structure='lines', architecture=\"ctc\"):\n",
    "        self.root_dir = root_dir\n",
    "        self.structure = structure         # 'lines', 'words'\n",
    "        self.architecture = architecture   # 'ctc', 'seq'\n",
    "\n",
    "        self.token2idx = {}\n",
    "        self.idx2token = {}\n",
    "        self.idx = 0\n",
    "        self.path = os.path.join(self.root_dir, 'vocab_'+ self.structure + '.json')\n",
    "\n",
    "        if self.architecture == 'ctc':\n",
    "            self.add(chr(1)) # extra output dimension for the BLANK label required for CTC loss\n",
    "        else:\n",
    "            self.add(chr(0)) \n",
    "            \n",
    "    def add(self, token):\n",
    "        if not token in self.token2idx:\n",
    "            self.token2idx[token] = self.idx\n",
    "            self.idx2token[self.idx] = token\n",
    "            self.idx += 1\n",
    "\n",
    "    def __call__(self, val):\n",
    "        res = None\n",
    "        if isinstance(val, str):\n",
    "            res = self.token2idx[val] if val in self.token2idx else self.token2idx['']\n",
    "        elif isinstance(val, int):\n",
    "            res = self.idx2token[val] if val <= self.__len__() else self.token2idx['']\n",
    "        else:\n",
    "            raise RuntimeError\n",
    "        return res\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.token2idx)\n",
    "\n",
    "    def create(self, alphabet):\n",
    "        for c in alphabet:\n",
    "            self.add(c)\n",
    "\n",
    "    def dump(self):\n",
    "        data = {'idx': self.idx, 'token2idx':self.token2idx, 'idx2token':self.idx2token}\n",
    "        with open(self.path, \"w\") as fd:\n",
    "            json.dump(data, fd)\n",
    "\n",
    "    def load(self):\n",
    "        with open(self.path, \"r\") as fd:\n",
    "            data = json.load(fd)\n",
    "            self.idx = int(data['idx'])\n",
    "            self.token2idx = data['token2idx']\n",
    "            self.idx2token = {int(k):v for k,v in data['idx2token'].items()}\n",
    "            \n",
    "        if self.architecture == 'seq':\n",
    "            self.add('<SOS>')\n",
    "            self.add('<EOS>')\n",
    "            self.add('<UNK>')\n",
    "\n",
    "    def __repr__(self):\n",
    "        return ''.join(list(self.token2idx.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\u0000 !\"#&'()*+,-./0123456789:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz,\n",
       " 80)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = Vocabulary(\"./data/IAM/\")\n",
    "vocab.load()\n",
    "vocab, len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class DataArgumentation(object):\n",
    "#     def __init__(self, threshold=0.5):\n",
    "#         self.threshold = threshold\n",
    "\n",
    "#     def __call__(self, img):\n",
    "#         if random.random() < self.threshold:\n",
    "#             img = np.array(img)\n",
    "#             img = img.reshape(1, img.shape[0], img.shape[1])\n",
    "\n",
    "# #             img = prepro.rotation(img, 5, is_random=True, row_index=1, col_index=2, channel_index=0)\n",
    "#             img = prepro.shift(img, 0.01, 0.04, is_random=True, row_index=1, col_index=2, channel_index=0)\n",
    "# #             img = prepro.shear(img, intensity=0.1, is_random=True, row_index=1, col_index=2, channel_index=0)\n",
    "# #             img = prepro.zoom(img, (1.0, 1.2), is_random=True, row_index=1, col_index=2, channel_index=0)\n",
    "\n",
    "#             img = prepro.elastic_transform(img.squeeze(0), is_random=True, alpha=5.5, sigma=35, cval=0.0)\n",
    "\n",
    "#             img = Image.fromarray(img)\n",
    "#         return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from glob import glob\n",
    "from os.path import basename\n",
    "\n",
    "import json\n",
    "import random\n",
    "import numpy as np\n",
    "import re\n",
    "import math\n",
    "\n",
    "import PIL\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import tensorlayer.prepro as prepro\n",
    "\n",
    "class IAMHandwritingDataset(data.Dataset):\n",
    "    \"\"\"IAM Handwriting Database ( http://www.fki.inf.unibe.ch/databases/iam-handwriting-database )\"\"\"\n",
    "\n",
    "    def __init__(self, root, vocab, dataset, structure, \n",
    "                 transform=None, target_transform=None, max_size=None):\n",
    "        self.root = root\n",
    "        self.vocab = vocab\n",
    "        self.dataset = dataset\n",
    "        self.structure = structure         # 'lines', 'words'\n",
    "        self.transform = transform\n",
    "        self.target_transform = target_transform\n",
    "        self.max_size = max_size\n",
    "\n",
    "\n",
    "        self.annotations = []\n",
    "        self.max_seq_length = 0\n",
    "        self.index = []\n",
    "\n",
    "        self.annotations, self.max_seq_length = self.load_annotations(self.root, self.dataset, \n",
    "                                                                      self.structure)\n",
    "        self.index = self.load_index(self.root, structure)\n",
    "\n",
    "        if isinstance(self.max_size, int):\n",
    "\n",
    "            assert self.max_size < len(self.annotations) # max_train_size needs to select a subset\n",
    "\n",
    "            self.annotations = random.sample(self.annotations, self.max_size)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        annotation = self.annotations[index]\n",
    "        anid = annotation['anid']\n",
    "        path = self.index[anid]\n",
    "\n",
    "        image = Image.open(path).convert('L')\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "\n",
    "        target = annotation['transcript']\n",
    "        if vocab.architecture == 'seq':\n",
    "            target = [self.vocab(\"<SOS>\")] + [self.vocab(token) for token in target] + [self.vocab('<EOS>')]\n",
    "        else:\n",
    "            target = [self.vocab(token) for token in target]\n",
    "        target = np.array(target)\n",
    "\n",
    "        if self.target_transform is not None:\n",
    "            target = self.target_transform(target)\n",
    "\n",
    "        return image, target\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.annotations)\n",
    "\n",
    "    @classmethod\n",
    "    def create(cls, root, structure='lines', partition='aachen'):\n",
    "        path = os.path.join(root, 'part', structure, partition, 'tr.txt')\n",
    "        with open(path, 'r') as infile:\n",
    "            trainset = {v.strip() for v in infile.readlines()}\n",
    "\n",
    "        try:\n",
    "            path = os.path.join(root, 'part', structure, partition, 'va.txt')\n",
    "            with open(path , 'r') as infile:\n",
    "                validationset1 = {v.strip() for v in infile.readlines()}\n",
    "                validationset2 = {}\n",
    "        except:\n",
    "            try:\n",
    "                path = os.path.join(root, 'part', structure, partition, 'va1.txt')\n",
    "                with open(path , 'r') as infile:\n",
    "                    validationset1 = {v.strip() for v in infile.readlines()}\n",
    "                \n",
    "                path = os.path.join(root, 'part', structure, partition, 'va2.txt')\n",
    "                with open(path, 'r') as infile:\n",
    "                    validationset2 = {v.strip() for v in infile.readlines()}\n",
    "            except:\n",
    "                raise RuntimeError('Dataset files not found.')\n",
    "\n",
    "        path = os.path.join(root, 'part', structure, partition, 'te.txt')\n",
    "        with open(path, 'r') as infile:\n",
    "            testset = {v.strip() for v in infile.readlines()}\n",
    "            \n",
    "        file_index = cls.load_index(root, structure)\n",
    "\n",
    "        path = os.path.join(root, 'ascii', structure + '.txt')\n",
    "\n",
    "        annotations = []\n",
    "        max_seq_length = 0\n",
    "        with open(path, \"r\") as fd:\n",
    "            for idx, line in enumerate(fd.readlines()):\n",
    "                if '#' in line[0]:\n",
    "                    continue\n",
    "                fields = line.rstrip().split(\" \")\n",
    "\n",
    "                transcript = fields[8]\n",
    "                # transcript field has whitespaces in it - need to fix the erroneous splits\n",
    "                if len(fields) > 9:\n",
    "                    for i in range(9, len(fields)):\n",
    "                        transcript += fields[i]\n",
    "\n",
    "                if structure == 'lines':\n",
    "                    # replace | seperator with whitespace\n",
    "                    transcript = transcript.replace(\"|\", \" \")\n",
    "\n",
    "                l = len(transcript)\n",
    "                if not l:\n",
    "                    continue\n",
    "\n",
    "                if l > max_seq_length:\n",
    "                    max_seq_length = l\n",
    "\n",
    "                if structure == 'lines':\n",
    "                    annotation={\n",
    "                        'anid': fields[0],\n",
    "                        'seg_result': fields[1],\n",
    "                        'graylevel': fields[2],\n",
    "                        'components': fields[3],\n",
    "                        'bounding_box':[fields[4], fields[5], fields[6], fields[7]],\n",
    "                        'transcript': transcript\n",
    "                    }\n",
    "                else:\n",
    "                    annotation={\n",
    "                        'anid': fields[0],\n",
    "                        'seg_result': fields[1],\n",
    "                        'graylevel': fields[2],\n",
    "                        'bounding_box':[fields[3], fields[4], fields[5], fields[6]],\n",
    "                        'tag': fields[7],\n",
    "                        'transcript': transcript\n",
    "                    }\n",
    "                    \n",
    "                anid = annotation['anid']\n",
    "                \n",
    "                try:\n",
    "                    path = file_index[anid]\n",
    "                    _ = Image.open(path)\n",
    "                except (IOError, KeyError):\n",
    "                    print('Corrupted image for %s ' % path)\n",
    "                    continue\n",
    "\n",
    "                if not anid in file_index:\n",
    "                    print('File missing %s' % path)\n",
    "                    continue\n",
    "\n",
    "                if structure == 'words':                                \n",
    "                    anid = annotation['anid'][:-3]\n",
    "                    \n",
    "                if anid in trainset:\n",
    "                    dataset = \"train\"\n",
    "                elif anid in validationset1:\n",
    "                    dataset = \"valid1\"\n",
    "                elif anid in validationset2:\n",
    "                    dataset = \"valid2\"\n",
    "                elif anid in testset:\n",
    "                    dataset = \"test\"\n",
    "                else:\n",
    "                    dataset = \"train\"\n",
    "                \n",
    "                annotation['dataset'] = dataset\n",
    "\n",
    "                annotations.append(annotation)\n",
    "\n",
    "        path = os.path.join(root, 'annotations_'+ structure + '.json')\n",
    "\n",
    "        with open(path, 'w') as outfile:\n",
    "            json.dump({'annotations': annotations,\n",
    "                       'max_seq_length': max_seq_length},\n",
    "                      outfile)\n",
    "\n",
    "    @classmethod\n",
    "    def load_annotations(cls, root, dataset, structure):\n",
    "        path = os.path.join(root, 'annotations_'+ structure + '.json')\n",
    "\n",
    "        with open(path, \"r\") as fd:\n",
    "            content = json.load(fd)\n",
    "\n",
    "        if not len(content):\n",
    "            raise RuntimeError('Dataset empty.')\n",
    "\n",
    "        annotations = content['annotations']\n",
    "        max_seq_length = content['max_seq_length']\n",
    "\n",
    "        if \"train\" in dataset:\n",
    "            annotations = [a for a in annotations if 'train' in a['dataset']]\n",
    "        elif \"valid\" in dataset:\n",
    "            annotations = [a for a in annotations if 'valid1' in a['dataset'] or 'valid2' in a['dataset']]\n",
    "        elif \"test\" in dataset:\n",
    "            annotations = [a for a in annotations if 'test' in a['dataset']]\n",
    "        else:\n",
    "            raise RuntimeError('Dataset parameter can be: \"train\", \"valid\" or \"test\".')\n",
    "\n",
    "        return annotations, max_seq_length\n",
    "\n",
    "    @classmethod\n",
    "    def load_index(cls, root, structure):\n",
    "        extension = 'jpg' #if 'lines' in structure else '.png'\n",
    "        path = os.path.join(root, 'imgs', structure)\n",
    "                \n",
    "        files = []\n",
    "        for root, dirs, flst in os.walk(path):\n",
    "            for file in flst:\n",
    "                if file.endswith(extension):\n",
    "                    files.append(os.path.join(root, file))\n",
    "\n",
    "        index = dict([(basename(path).rstrip(\".\" + extension) , path) for path in files])\n",
    "\n",
    "        if not len(index):\n",
    "            raise RuntimeError('Image files empty.')\n",
    "        return index\n",
    "\n",
    "    def __repr__(self):\n",
    "        fmt_str = self.__class__.__name__ + '\\n'\n",
    "        fmt_str += '    Number of images found: {}\\n'.format(len(self.index))\n",
    "        fmt_str += '    Number of datapoints: {}\\n'.format(self.__len__())\n",
    "        fmt_str += '    Root Location: {}\\n'.format(self.root)\n",
    "        tmp = '    Transforms (if any): '\n",
    "        fmt_str += '{0}{1}\\n'.format(tmp, self.transform.__repr__().replace('\\n', '\\n' + ' ' * len(tmp)))\n",
    "        tmp = '    Target Transforms (if any): '\n",
    "        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\\n', '\\n' + ' ' * len(tmp)))\n",
    "        return fmt_str\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_transform_train = transforms.Compose([\n",
    "    transforms.Pad((H.PADDING, H.PADDING, H.PADDING, H.PADDING), fill=255),\n",
    "#     DataArgumentation(threshold=H.ARGUMENTATION),\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "image_transform_test = transforms.Compose([\n",
    "    transforms.Pad((H.PADDING, H.PADDING, H.PADDING, H.PADDING), fill=255),\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "target_transform = transforms.Compose([\n",
    "    FromNumpyToTensor()\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(IAMHandwritingDataset\n",
       "     Number of images found: 13353\n",
       "     Number of datapoints: 9472\n",
       "     Root Location: ./data/IAM/\n",
       "     Transforms (if any): <torchvision.transforms.transforms.Compose object at 0x1bb416c18>\n",
       "     Target Transforms (if any): <torchvision.transforms.transforms.Compose object at 0x1bb4160f0>,\n",
       " 592)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = IAMHandwritingDataset('./data/IAM/', vocab, dataset=\"train\", structure=\"lines\", #max_size=10000,\n",
    "                                     transform=image_transform_train, target_transform=target_transform)\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    train_dataset, batch_size=H.BATCH_SIZE, num_workers=H.NUM_WORKERS, shuffle=True,\n",
    "    collate_fn=alignCollate(img_height=H.HEIGHT), pin_memory=True)\n",
    "\n",
    "train_dataset, len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(IAMHandwritingDataset\n",
       "     Number of images found: 13353\n",
       "     Number of datapoints: 966\n",
       "     Root Location: ./data/IAM/\n",
       "     Transforms (if any): <torchvision.transforms.transforms.Compose object at 0x1bb4167f0>\n",
       "     Target Transforms (if any): <torchvision.transforms.transforms.Compose object at 0x1bb4160f0>,\n",
       " 61)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_dataset = IAMHandwritingDataset('./data/IAM/', vocab, dataset=\"valid\", structure=\"lines\", #max_size=200,\n",
    "                                     transform=image_transform_test, target_transform=target_transform)\n",
    "\n",
    "valid_loader = torch.utils.data.DataLoader(\n",
    "    valid_dataset, batch_size=H.BATCH_SIZE, shuffle=False, num_workers=H.NUM_WORKERS, \n",
    "    collate_fn=alignCollate(img_height=H.HEIGHT), pin_memory=True)\n",
    "\n",
    "valid_dataset, len(valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(IAMHandwritingDataset\n",
       "     Number of images found: 13353\n",
       "     Number of datapoints: 2915\n",
       "     Root Location: ./data/IAM/\n",
       "     Transforms (if any): <torchvision.transforms.transforms.Compose object at 0x1bb4167f0>\n",
       "     Target Transforms (if any): <torchvision.transforms.transforms.Compose object at 0x1bb4160f0>,\n",
       " 183)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dataset = IAMHandwritingDataset('./data/IAM/', vocab, dataset=\"test\", structure=\"lines\",\n",
    "                                     transform=image_transform_test, target_transform=target_transform)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    test_dataset, batch_size=H.BATCH_SIZE, shuffle=False, num_workers=H.NUM_WORKERS,\n",
    "    collate_fn=alignCollate(img_height=H.HEIGHT))\n",
    "\n",
    "test_dataset, len(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 1, 64, 1259]) torch.Size([16, 63])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bac125157bf437daeeb3bd10e7201c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>interactive</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "interactive(children=(IntSlider(value=7, description='idx', max=15), Output()), _dom_classes=('widget-interact',))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "import ipywidgets as widgets\n",
    "\n",
    "input_vars, target_vars, input_sizes, target_len = next(train_loader.__iter__())\n",
    "\n",
    "print(input_vars.shape, target_vars.shape)\n",
    "\n",
    "@interact(idx=(0, H['BATCH_SIZE']-1))\n",
    "def show(idx):\n",
    "    print(input_vars[idx].shape)\n",
    "    visualize_data(input_vars[idx], target_vars[idx], vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[55, 58,  1,  ...,  0,  0,  0],\n",
       "        [36, 59,  1,  ..., 54, 67, 57],\n",
       "        [66, 58, 54,  ...,  0,  0,  0],\n",
       "        ...,\n",
       "        [48, 67, 62,  ...,  0,  0,  0],\n",
       "        [31, 42, 41,  ...,  0,  0,  0],\n",
       "        [72, 74, 57,  ...,  0,  0,  0]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Metric([('train_loss', np.inf), ('train_score', np.inf), ('valid_loss', np.inf), ('valid_score', 0), \n",
    "            ('train_lr', 0), ('valid_cer', np.inf)])\n",
    "\n",
    "crnn = CRNN(num_classes=len(vocab), hidden_size=H.HIDDEN_SIZE, num_layers=H.NUM_LAYERS, rnn_dropout=H.RNN_DROPOUT,\n",
    "            initialize=torch_weight_init)\n",
    "if H.USE_CUDA:\n",
    "    crnn.cuda()\n",
    "\n",
    "optimizer = optim.Adam(list(filter(lambda p:p.requires_grad, crnn.parameters())),\n",
    "                       amsgrad = False,\n",
    "                       betas = (0.9, 0.999),\n",
    "                       eps = 1e-08,\n",
    "                       lr = H.LR,\n",
    "                       weight_decay = H.WEIGHT_DECAY)\n",
    "\n",
    "criterion = CTCLoss()\n",
    "\n",
    "stopping = Stopping(crnn, patience= H.STOPPING_PATIENCE)\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=[H.LR_LAMBDA])\n",
    "\n",
    "decoder = CTCGreedyDecoder(vocab)\n",
    "\n",
    "logger = TensorboardLogger(experiment_dir=H.EXPERIMENT) #PytorchLogger()\n",
    "\n",
    "checkpoint = Checkpoint(crnn, optimizer, stopping, m, experiment_dir=H.EXPERIMENT, checkpoint_file=H.CHECKPOINT_FILE,\n",
    "                         restore_from=-1, interval=H.CHECKPOINT_INTERVAL, verbose=0)\n",
    "\n",
    "scorer = ScorerSM()\n",
    "\n",
    "trainer = Trainer(crnn, train_loader, optimizer, scheduler, criterion, decoder)\n",
    "\n",
    "train_evaluator = Evaluator(crnn, valid_loader, criterion, decoder, scorer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['\\x00', ' ', '!', '\"', '#', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab.token2idx.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 38min 16s, sys: 1min 5s, total: 1h 39min 21s\n",
      "Wall time: 1h 38min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "epoch_start = checkpoint.restore()+1 if H.CHECKPOINT_RESTORE else 1\n",
    "\n",
    "epoch_itr = logger.set_itr(range(epoch_start, H.NUM_EPOCHS+1))\n",
    "\n",
    "epoch = 0\n",
    "for epoch in epoch_itr:\n",
    "\n",
    "    with DelayedKeyboardInterrupt():\n",
    "\n",
    "        m.train_loss, m.train_lr = trainer.train(epoch, H.MAX_GRAD_NORM)\n",
    "        \n",
    "        m.valid_loss, m.valid_score, m.valid_cer = train_evaluator.evaluate()                               \n",
    "            \n",
    "        epoch_itr.log_values( m.train_loss, m.train_score, m.train_lr, m.valid_loss, m.valid_score,\n",
    "                              stopping.best_score_epoch, stopping.best_score)\n",
    "        \n",
    "        if checkpoint:\n",
    "            checkpoint.step(epoch)\n",
    "            \n",
    "        if stopping.step(epoch, 0, m.valid_score):\n",
    "            print(\"Early stopping at epoch: %d, score %f\" % (stopping.best_score_epoch, stopping.best_score))\n",
    "            break\n",
    "            \n",
    "checkpoint.create(epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ARGUMENTATION': 0.9,\n",
       " 'BATCH_SIZE': 16,\n",
       " 'CHECKPOINT_FILE': 'IAM_HTR_CTC08',\n",
       " 'CHECKPOINT_INTERVAL': 5,\n",
       " 'CHECKPOINT_RESTORE': False,\n",
       " 'EXPERIMENT': 'IAM_HTR_CTC08',\n",
       " 'HEIGHT': 64,\n",
       " 'HIDDEN_SIZE': 256,\n",
       " 'LR': 0.0003,\n",
       " 'LR_LAMBDA': <function __main__.<lambda>>,\n",
       " 'MAX_GRAD_NORM': 5.0,\n",
       " 'MODEL_PATH': './chkpt/IAM_HTR_CTC08.tar',\n",
       " 'NUM_EPOCHS': 20,\n",
       " 'NUM_LAYERS': 2,\n",
       " 'NUM_WORKERS': 8,\n",
       " 'PADDING': 10,\n",
       " 'RNN_DROPOUT': 0.5,\n",
       " 'ROOT_DIR': './data',\n",
       " 'STOPPING_PATIENCE': 10,\n",
       " 'USE_CUDA': True,\n",
       " 'WEIGHT_DECAY': 0}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorboardLogger\n",
      "    Last Epoch/LR:    20 / 0.000234\n",
      "    Train Loss/Score: 8.890155523612693 / inf\n",
      "    Valid Loss/Score: 9.35405035683366 / 0.9572857934714726\n",
      "    Best Epoch/Score: 18 / 0.9566100093689708\n",
      "\n",
      "Stopping\n",
      "    Patience: 10\n",
      "    Best Score: 0.9573\n",
      "    Epoch of Best Score: 20\n",
      "\n",
      "Checkpoint\n",
      "    Timestamp: 2019_02_05_16_54_46\n",
      "    Last Checkpoint: chkpt/IAM_HTR_CTC08/2019_02_05_16_54_46/state.tar\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+gAAAGoCAYAAADVZM+hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xl8VOX5///XlW0SlhkQWTKgBRGQPSKKC37AalX0o1iqFSp1r1Wxrm21dal1+X20tS4oilRBS7GUuvKtKLYVRVwqiWIQlEUBCSBBlLAGSHL//jgzYQiTPZkzSd7Px2MeM3PmnHvuE3jkznXf17mOOecQEREREREREX+l+N0BEREREREREVGALiIiIiIiIpIUFKCLiIiIiIiIJAEF6CIiIiIiIiJJQAG6iIiIiIiISBJQgC4iIiIiIiKSBBSgi4iIiIiIiCQBBegiScbMVpvZKX73Q0RERKpnZsPN7D0zKzKzb83sXTM72u9+iUjTlOZ3B0REREREmiIzCwL/BK4CZgEZwInA7gb8jlTnXGlDtSciyU0r6CJNhJn9zMxWRmbnZ5tZOLLdzOwhMyuMzN7nm9mAyGdnmNlSM9tmZuvM7Jf+noWIiEiz0hvAOfc351ypc26Xc+4N51w+lI/dn0XG4aVmNiSyva+ZvWVmW8xsiZmdHW3QzJ4xsyfMbI6Z7QBOMrOAmT1gZl+Z2UYzm2xmWb6csYg0KgXoIk2AmX0f+D/gx0A2sAaYGfn4VOB/8P5IaAecD2yOfPY08HPnXFtgAPBmArstIiLS3C0HSs3sWTMbZWbtox+Y2XnAncCFQBA4G9hsZunA/wPeADoBvwBmmFmfmHZ/AtwLtAUWAPfjjfM5wOFAV+COxj01EfGDAnSRpuECYKpz7iPn3G7gN8BxZtYd2Is3gB8BmHPuM+fchshxe4F+ZhZ0zn3nnPvIh76LiIg0S865rcBwwAF/BjZFstw6A5cDf3DOLXSelc65NcCxQBvgPufcHufcm3hp8uNimn7FOfeuc64ML13+Z8ANzrlvnXPbgP8PGJuwExWRhFGALtI0hPFWzQFwzm3HWyXvGhnYHwMmARvNbErkmjiAHwFnAGvM7G0zOy7B/RYREWnWIhPjFzvnuuFlq4WBh4FDgC/iHBIG1kaC76g1eKviUWtjXncEWgF5kZT4LcDrke0i0swoQBdpGtYD34u+MbPWQAdgHYBzbqJz7iigP14K3K8i2xc650bjpdC9jFfARkRERBqBc+5z4Bm8QH0t0DPObuuBQ8ws9u/wQ4mM6dGmYl5/A+wC+jvn2kUeIedcmwbtvIgkBQXoIskp3cwyow+8wPoSM8sxswBeatt/nXOrzexoMxsWuaZtB1CMdz1chpldYGYh59xeYCugKrAiIiINxMyOMLObzKxb5P0heKnqHwBPAb80s6MiBV0PN7PvAf/FG69/bWbpZjYSOIt9tWX2E1lp/zPwkJl1inxPVzM7rbHPT0QSTwG6SHKagzdbHn2cCNwOvABswJuRj157FsQbuL/DS5HbDDwQ+eynwGoz2wpcCYxPUP9FRERagm3AMOC/kYrrHwCfAjc55/6BV+jtuch+LwMHOef24BWMG4W3Ov44cGFk9b0yNwMrgQ8iY/q/gT5V7C8iTZQ556rfS0REREREREQalVbQRURERERERJKAAnQRERERERGRJKAAXURERERERCQJKEAXERGRSpnZVDMrNLNPK/nczGyima00s3wzG5LoPoqIiDQXaX53oD5SUlJcVlaW390QERGf7dy50znnNOncOJ4BHgP+Usnno4Bekccw4InIc5U0houICGgMr6hJB+hZWVns2LHD726IiIjPzGyX331orpxz882sexW7jAb+4rzbwnxgZu3MLNs5t6GqdjWGi4gIaAyvSDMVIiIiUh9dgbUx7wsi2w5gZleYWa6Z5ZaUlCSkcyIiIk2JAnQRERGpD4uzzcXb0Tk3xTk31Dk3NC2tSSfxiYiINAoF6CIiIlIfBcAhMe+7Aet96ouIiEiTpulrkRZm7969FBQUUFxc7HdXRGotMzOTbt26kZ6e7ndXZJ/ZwDVmNhOvOFxRddefi4iISHwK0EVamIKCAtq2bUv37t0xi5eZKpKcnHNs3ryZgoICevTo4Xd3Wgwz+xswEjjYzAqA3wHpAM65ycAc4AxgJbATuMSfnoqIiDR9CtBFWpji4mIF59IkmRkdOnRg06ZNfnelRXHOjavmcwdMSFB3REREmjVdgy7SAik4l6ZK/3dFRESkOVOALiIiIiIiIpIEFKCLSEJt3ryZnJwccnJy6NKlC127di1/v2fPnhq1cckll7Bs2bIq95k0aRIzZsxoiC7Hddttt/Hwww83Sts7duxg5MiRlJWV1fiYupzvTTfdRP/+/enbty833HADXqby/j766COGDRtGTk4ORx99NLm5uQB8++23nH322QwaNIhhw4axdOnS8mPmzJlDnz59OPzww/njH/9Yvn3s2LH06dOHAQMGcPnllxO9D/bLL7/M3XffXau+i4iIiDRHCtBFJKE6dOjAokWLWLRoEVdeeSU33HBD+fuMjAzAKwZWVXA6bdo0+vTpU+X3TJgwgQsuuKBB+54oTz31FOeddx4pKTX/FV3b850/fz4LFy5k8eLFLF68mHfffZd33333gP1++ctfcs8997Bo0SJuv/12brnlFgDuvvtuhg0bRn5+PlOnTuW6664DvLsEXHPNNbzxxhssWbKEv/zlLyxfvhyACy+8kM8//5z8/HyKioqYNm0aAKNHj+aFF17QnQVERESkxVOALiJJYeXKlQwYMIArr7ySIUOGsGHDBq644gqGDh1K//79ueuuu8r3HT58OIsWLaKkpIR27dpxyy23MHjwYI477jgKCwuB/Ve4hw8fzi233MIxxxxDnz59eO+99wBvpfpHP/oRgwcPZty4cQwdOrS83Z/+9KcMHDiQAQMGMHHixCr7Hl1lHjRoED/60Y8oKioC4KGHHqJfv34MHjyY8ePHA/Dmm28yePBgcnJyGDJkCDt27DigvRkzZjB69GgA/v3vf3PSSSdx7rnn0qtXL2677Tb+8pe/cPTRRzNo0CBWr15d4/ONZWYUFxezZ88edu/eTUlJCZ06dYq739atWwEoKioiHA4DsHTpUk4++WQA+vfvz/Lly9m8eTMffPABffv25Xvf+x6BQIAf//jHvPLKKwCcccYZmBkpKSkcc8wxFBQUlH/HiSeeyJw5c6r8OYuIiIg0d6riLtKSXX89LFrUsG3m5EAdU7+XLl3KtGnTmDx5MgD33XcfBx10ECUlJeVBar9+/fY7pqioiBEjRnDfffdx4403MnXq1PJV3ljOOT788ENmz57NXXfdxeuvv86jjz5Kly5deOGFF/jkk08YMmQIAHl5eXzzzTcsXrwYgC1btlTZ7/HjxzNlyhSGDx/Ob3/7W+6++24eeOAB/vCHP7BmzRoyMjLK2/jjH//IlClTGDZsGNu3byczM3O/toqLiykoKKBbt27l2z755BM+++wzQqEQ3bt35+qrr2bhwoX86U9/4rHHHuOBBx6o0fnGOvHEEzn++OPp0qULzjmuv/56evfufUA7EydO5LTTTuP666/HOcf7778PwODBg3nhhRc49thjef/99ykoKKCgoIB169ZxyCGHlB/frVs3Pvnkk/3a3LNnDzNmzOCJJ54o3zZ06FDeeecdxowZU+XPWkRERKQ50wq6iCSNnj17cvTRR5e//9vf/saQIUMYMmQIn3322X7XOUdlZWUxatQoAI466qjyFeWKooFf7D4LFixg7NixgBdw9u/fH4DDDz+cZcuWcd111zF37lxCoVClfd68eTPFxcUMHz4cgIsuuoj58+cD3sry+PHjmTFjBunp6QCccMIJXH/99Tz66KNs3bqV1NTU/dorLCzkoIMO2m/bsGHD6Ny5M5mZmRx22GGcdtppAAwcOLBW5xtr2bJlfPHFF6xbt461a9fy2muvxV1pnzRpEpMmTWLt2rX84Q9/4Gc/+xkAt956K4WFheTk5DB58mQGDx5MWlpa3OvYK1Zev/LKKznllFM47rjjyrd16tSJ9evXxz2XhmBmp5vZMjNbaWYHzOCYWcDM/h75/L9m1j3ms99Eti8zs9Oqa9PMnjazT8ws38yeN7M21X2HiIiIxNfSxnCtoIu0ZI1U5KyuWrduXf56xYoVPPLII3z44Ye0a9eO8ePHx71GOXrdOkBqamp54bGKAoHAAfvECybBu04+Pz+f1157jYkTJ/LCCy8wZcqUuPtW1gbA3Llzefvtt3nllVe45557+PTTT7nttts4++yzefXVVzn66KN566236NWrV/kxWVlZB5xntO8AKSkp5e9TUlJqdb6xXnzxRY4//vjyn/npp5/OBx98wPHHH7/ffn/96195/PHHATj//PO5+uqrAQiFQjz77LMAlJWV0b17d7p37863337L2rVry48vKCgoT4sHuP322ykqKuKpp57a73uKi4vJysqKey71ZWapwCTgB0ABsNDMZjvnYmd8LgO+c84dbmZjgfuB882sHzAW6A+EgX+bWTTVoLI2b3DObY1894PANcB9lX1Ho5y0iIhIM9ASx/AWHaAXrfuS4x/sBwcfDFWskInUxqU5l3LT8Tf53Y0mb+vWrbRt25ZgMMiGDRuYO3cup59+eoN+x/Dhw5k1axYnnngiixcvLl+h37RpE5mZmZx33nn06NGDK6+8stI2Dj74YLKysnjvvfc4/vjjmT59OiNGjKC0tJSCggK+//3vM3z4cGbMmMHOnTspLCxk0KBBDBo0iHfffZdly5btF6B37Nix/Nrw2MmHhnbooYcyffp0fvWrX+Gc4+233457aUDnzp1ZsGABw4cP59///nd5cb4tW7bQunVr0tPTefLJJznllFNo3bo1xx57LEuXLmXNmjV06dKFWbNm8fzzzwMwefJk3nrrLf71r38dUABv+fLlDBgwoLFO9xhgpXPuSwAzmwmMBmIH99HAnZHXzwOPmbf0PxqY6ZzbDawys5WR9qiszZiB3YAswFX1Ha6qWR6p0pAnh7C7dLff3ZBm5H97/S/3/+B+v7shtbR3LxQXQ0lJ5Y/S0pp/7hykpIBZ/Z6jfYt97NlTt/fRkcKsdo+Kx8CBP4u9eyv/uVT12dFHw9Spjf7P2+LG8BYdoKeG2tFv7W4IhqBjv+oPEKnG/DXzeenzlxSgN4AhQ4bQr18/BgwYwGGHHcYJJ5zQ4N/xi1/8ggsvvJBBgwYxZMgQBgwYQCgUYu3atVx22WU45zAz7r+/6j/Wpk+fzlVXXcWuXbs4/PDDmTZtGiUlJfzkJz9h27ZtlJWVcfPNN9O2bVt+/etf884775CSksKgQYM49dRTD2jv5JNP5r333mPkyJENfs5RY8eO5a233mLQoEEAnHnmmeWXClxyySVcd9115OTk8PTTT3PttddSWlpKVlYWTz75JACLFy/m0ksvJTU1lQEDBpSviKenpzNx4kR+8IMfUFpayhVXXEGfPn0oLS3lmmuuoXv37hx77LEAnHfeedx6660AzJs3jwcffLA+p5RmZrkx76c456JpD12BtTGfFQDDKhxfvo9zrsTMioAOke0fVDi2a+R1pW2a2TTgDLw/IKK/ECr7jm9qfpoSq2/HvuwprdntGUWqs3DdQv6x9B8K0BuQc7BzJ+zY4T22bz/w9c6dsGtX/Z5LS/0+04ZlBunp3iMjw3tOSfF+njV9QOXb09LiP9LTK9+ekQGtWu2//Xvfa7BT1hgew5ryxH3r1q1dvArItdKpE4wZA5GiVCL18cO//5Avvv2C/Kvy/e5KpT777DP69u3rdzeSQklJCSUlJWRmZrJixQpOPfVUVqxYQVqav3OXCxcu5PHHHy+/DVlzt379ei6++GLeeOONGu0f7/+wme10zrWOt7+ZnQec5py7PPL+p8AxzrlfxOyzJLJPQeT9F3iz7HcB7zvn/hrZ/jQwB6+GS3VtpgKPAgudc9Mq+w7n3OYanXgz0yBjuEgDuvrVq5m1ZBbf/LppzZmVlcHWrbBly77Hd98d+L6oyAtkK670xlv9rW5bWVnlAXfF4Lu2oUZWlvdo1erA15U9Z2VBZmb8ADM1tfKANN4+qaneOTrnnWd9np3bF2jHBtvVva9QnqbZ0xi+vxa9gg5AOAyNWJhIWpZQIMTW3Vv97obU0Pbt2zn55JMpKSnBOceTTz7pe3AOcPTRRzN8+HDKyspqdS/0pmrt2rVxK9E3oALgkJj33YCKv/ij+xSYWRoQAr6t5tgq23TOlZrZ34FfAdOq+A4RSQLRMTyaPVUbZWXeam50Rbe42EtNrixNOPZ9da937ao6+C4qqjoINvOu5AyFvMCvskCyNttSUqB1a2jTZt9z27bQpcu+bRU/r7gt9hENtjMz96Vhi0S0uDHc/79E/ZadDRs2+N0LaSaCgSBFu4v87obUULt27cjLy/O7G3FddtllfnchYYYNq5ip1uAWAr3MrAewDq9gzE8q7DMbuAh4HzgXeNM558xsNvBcpFBMGOgFfAhYvDYj16z1dM6tjLw+C/i8qu9orJMWkco55wW6GzfueyxeHWRv2V5++Zti9u7M2i/gru717kYuhdCmDbRrt+9xyCEwcOC+9+3b7/957PtgcN/10CJNUIsbwxWgh8OQn7zpyNK0BAPBOs++i0jjiFwrdg0wF0gFpjrnlpjZXUCuc2428DQwPVJA5lu8wZrIfrPwrkMrASY450oBKmkzBXjWzIJ4fwB8AlwV6Urc7xCRhlFWti/o/vrr/YPv2MfXX0NhobdCvZ+jg3AmPDZlK61cVtw06w4dDkytjrdfZqaXrlzxut7KXlf1WSDgvRdpiVriGK4APTvb+01dWtryLviQBhcKhChzZezcu5PWGXEvpRERHzjn5uBddxa77Y6Y18XAeZUcey9wbw3bLAPiVjSs6jtEpHo7dsCaNbBq1b7H6tXetg0bvKA73p0n09Kgc+d9j4EDvVTs2G2dO8P8ohBX/ws+XbGVXh06J/z8RCS+ljaGK0APh70p102bvN/WIvUQDAQBKNpdpABdRESkFvbs8YLt1asPDMJXrfIC8FiZmdC9u/cYNCh+0N2li5fuXZOkti+W7RvDRUT8ogA9HPae169XgC71Fg3Qt+7eSrht2OfeiIiI+GfPHq+6eGWPDRv2D8LXrdu/2Fn0Nk7du8PZZ0OPHt6je3fvuXPnhi0oFjuGi4j4RQF6drb3vH49DBnib1+kyQtlhgAoKtbsu4iINH3FxV4gvX6997xxo1c1vKrAO/qornCaGXTr5gXb3//+gQF4166JvfowFNAYLiL+U4AeXUFXJXdpAJp9r97IkSP5zW9+w2mnnVa+7eGHH2b58uU8/vjjlR7Xpk0btm/fzvr167n22mt5/vnn47b9wAMPMHTo0Erbefjhh7niiito1aoVAGeccQbPPfcc7dq1q8dZwZ133kmbNm345S9/Wa924tm1axenn346b775Jqk1/Gt18uTJtGrVigsvvLDG33PzzTfz6quvAnD77bdz/vnnx2130qRJpKam0qZNG6ZMmUK/fv1YvXo1ffv2pU+fPgAce+yxTJ48eb9jzz77bL788ks+/fRTAP7xj39w55138tlnn/Hhhx+W/7stXryYP/3pTzzzzDM17ruI1M727d6fPrHBd7z3W7bEPz493bttVzC479G1K/Ttu/+2qh4dOniF1JKFxnARSQYK0KNp7boXujSA6Oy7BvfKjRs3jpkzZ+4XoM+cOZM//vGPNTo+HA7HDc5r6uGHH2b8+PHlAfqcOXOqOcJ/U6dOZcyYMTUOzgGuvPLKWn3Hq6++ykcffcSiRYvYvXs3I0aMYNSoUQSDwf32+8lPflLe9uzZs7nxxht5/fXXAejZsyeLFi2K2/6LL75ImzZt9ts2YMAAXnzxRX7+85/vt33gwIEUFBTw1Vdfceihh9bqPERkf99+C08/DR99tH/wvX37gftmZHjrFtnZXqD9/e/vex99dOni3borEEj8uTS2aBacxnAR8ZMC9PR06NhRK+jSIGKLxDUF118PlcRTdZaTAw8/XPnn5557Lrfddhu7d+8mEAiwevVq1q9fz/Dhw9m+fTujR4/mu+++Y+/evdxzzz2MHj16v+NXr17N//7v//Lpp5+ya9cuLrnkEpYuXUrfvn3ZtWtX+X5XXXUVCxcuZNeuXZx77rn8/ve/Z+LEiaxfv56TTjqJgw8+mHnz5tG9e3dyc3M5+OCDefDBB5k6dSoAl19+Oddffz2rV69m1KhRDB8+nPfee4+uXbvyyiuvkJWVVek5Llq0iCuvvJKdO3fSs2dPpk6dSvv27Zk4cSKTJ08mLS2Nfv36MXPmTN5++22uu+46AMyM+fPn07Zt2/3amzFjBs899xwAb731Fr/73e/o3LkzixYtYsyYMQwcOJBHHnmEXbt28fLLL9OzZ8/9VvRHjhzJsGHDmDdvHlu2bOHpp5/mxBNP3O87li5dyogRI0hLSyMtLY3Bgwfz+uuv8+Mf/3i//WID9h07dtTodoLbt2/nwQcfZMqUKfu117dv30qPOeuss5g5cya//vWvq21fRA60fDk88gg884x3v+7DDvNWuI88Es44wwu2KwbfNS2m1ly1zfB+9zaVMVxEmqcUvzuQFMJhraBLg1B6XPU6dOjAMcccU77qOnPmTM4//3zMjMzMTF566SU++ugj5s2bx0033YSLrRhUwRNPPEGrVq3Iz8/n1ltvJS8vr/yze++9l9zcXPLz83n77bfJz8/n2muvJRwOM2/ePObNm7dfW3l5eUybNo3//ve/fPDBB/z5z3/m448/BmDFihVMmDCBJUuW0K5dO1544YUqz/HCCy/k/vvvJz8/n4EDB/L73/8egPvuu4+PP/6Y/Pz88vTvBx54gEmTJrFo0SLeeeedAwL/PXv28OWXX9K9e/fybZ988gmPPPIIixcvZvr06SxfvpwPP/yQyy+/nEcffTRun0pKSvjwww95+OGHy/sTa/Dgwbz22mvs3LmTb775hnnz5rF27dq4bU2aNImePXvy61//mokTJ5ZvX7VqFUceeSQjRozgnXfeKd9+++23c9NNN5VnLdTE0KFD92tDRKrnHLz1lldQ7Ygj4Kmn4PzzIT8fvvgC5s+Hv//dm0S9+Wb46U/hlFOgf3846KCWHZwDBNICBFIDGsNFxFdaQQdv2lgBujSAphagV7XS3Ziiae6jR49m5syZ5avWzjl++9vfMn/+fFJSUli3bh0bN26kSyV3WJg/fz7XXnstAIMGDWLQoEHln82aNYspU6ZQUlLChg0bWLp06X6fV7RgwQJ++MMf0rq1d3u8MWPG8M4773D22WfTo0cPcnJyADjqqKNYvXp1pe0UFRWxZcsWRowYAcBFF13EeeedV97HCy64gHPOOYdzzjkHgBNOOIEbb7yRCy64gDFjxtCtW7f92vvmm28OuD7+6KOPJjtS4LJnz56ceuqpgJcaXnHiIWrMmDFV9v/UU09l4cKFHH/88XTs2JHjjjuOtLT4Q8SECROYMGECzz33HPfccw/PPvss2dnZfPXVV3To0IG8vDzOOecclixZwpdffsnKlSt56KGHqvy5VdSpUyfW6/eySI3s2eMF3g89BB9/DAcfDLffDldf7VU6l5oLZYaazBguIs2TVtDBW0FXirs0gNSUVFqnt1YF2Gqcc845/Oc//+Gjjz5i165dDIncQWHGjBls2rSJvLw8Fi1aROfOnSkuLq6yrXgp1qtWreKBBx7gP//5D/n5+Zx55pnVtlPVSn0g5mLL1NRUSkpKqmyrMq+++ioTJkwgLy+Po446ipKSEm655Raeeuopdu3axbHHHsvnn3++3zFZWVkH9D22PykpKeXvU1JSKu1bdJ+q+n/rrbeyaNEi/vWvf+Gco1evXlWez9ixY3n55ZfL2+/QoQPgTQL07NmT5cuX8/7775OXl0f37t0ZPnw4y5cvZ+TIkVW2C1BcXFzlZQQi4l1f/n//51U9v/BCr+L6n/8MX30Fv/+9gvO6CAaCSnEXEV8pQAcvQP/6aygt9bsn0gwEA0HNvlejTZs2jBw5kksvvZRx48aVby8qKqJTp06kp6czb9481qxZU2U7//M//8OMGTMA+PTTT8nPzwdg69attG7dmlAoxMaNG3nttdfKj2nbti3btm2L29bLL7/Mzp072bFjBy+99NIB12nXRCgUon379uXp2dOnT2fEiBGUlZWxdu1aTjrpJP7whz+wZcsWtm/fzhdffMHAgQO5+eabGTp06AEBevv27SktLa12gqG+SktL2bx5MwD5+fnk5+eXr8zHWrFiRfnrV199tTyI37RpE6WR36FffvklK1as4LDDDuOqq65i/fr1rF69mgULFtC7d2/eeuutavuzfPlyBgwY0ABnJtL8LF8OEybAIYfAb38LAwfCa6/BkiVw+eWgua260xguIn5Tijt4Ke5lZbBp076q7iJ1FMoMsXWPBvfqjBs3jjFjxjBz5szybRdccAFnnXUWQ4cOJScnhyOOOKLKNq666iouueQSBg0aRE5ODscccwzgXU995JFH0r9/fw477DBOOOGE8mOuuOIKRo0aRXZ29n7p4EOGDOHiiy8ub+Pyyy/nyCOPrFVadtSzzz5bXiTusMMOY9q0aZSWljJ+/HiKiopwznHDDTfQrl07br/9dubNm0dqair9+vVj1KhRB7R36qmnsmDBAk455ZRa96Wm9u7dWz4hEQwG+etf/1qe4n7HHXcwdOhQzj77bB577DH+/e9/k56eTvv27Xn22WcB73KDO+64g7S0NFJTU5k8eTIHHXRQld/50ksv8Ytf/IJNmzZx5plnkpOTw9y5cwGYN28eZ555ZqOdr0hT4xy8/TY8+CD8859ejdvx471inwMH+t275iMUUIq7iPjLqkrrTHatW7d2O3bsqH9DL78MP/wh5OVBJNVWpK6GPTWM9pnteX386353Ja7PPvusyurZknw+/vhjHnzwQaZPn+53VxIiepu3BQsWxL0OPt7/YTPb6Zxrnag+Sv012BjezFW8vrxjR+/a8quuUgp7Yzhn5jl8+d2X5F+V73dSyStDAAAgAElEQVRXRFoMjeH70wo6eCvo4BWKU4Au9aT0OGloRx55JCeddBKlpaW1uhd6U/XVV19x3333VVqkTqQl+OYb73ryRx/1yuT06+e9v+ACpbA3JhWJExG/6a8f8K5BBxWKkwYRCoRYvy25q08752p0/2pJHpdeeqnfXUiYXr16VVqgrilnfYlUxzn48EOYNAlmzYLdu+HUU2HaNO9Zv7YbXzBDReJExF8K0GHfdee6pY80gGRfQc/MzGTz5s106NBBQbo0Kc45Nm/eTGZmpt9dEWlQO3fC3/4Gjz8OH30EbdvCz37mpbH36+d371qW6BiuiWwR8YsCdPAqrXTsqBV0aRDJXmCmW7duFBQUsGnTJr+7IlJrmZmZB9wrXqSpWr4cnngCnnkGtmzxir1Nnuylsbdp43fvWqZQZogyV8bOvTtpnaFLYkUk8RSgR4XDWkGXBhEMBNm2extlrowUS747Gaanp9OjRw+/uyEi0iKVlHhV2B9/HP71L2+N4NxzvcJvJ5ygNHa/BQNBAIp2FylAFxFfKECPUoAuDSQYCOJwbN+zvXygFxGRlu3rr+Gpp+DJJ6GgwLuH+b33wmWXqRp7MomO21t3byXcNuxzb0SkJVKAHpWdDZ984ncvpBkIZYYAb3BXgC4i0nI5BwsWeKvlL7wAe/d6xd4eewzOPBN0o4LkEwrsG8NFRPygoSEqHPamt0tLoQXcxkgaT3l6XHER3YK6VlZEpKXZtg3++lcvMP/0U2jXDq65Bq68Enr39rt3UpXYMVxExA8K0KOys6GsDAoL990XXaQOYtPjRESk5Sgthdtv91bIt22DIUPg6adh7Fho1crv3klNaAwXEb8pQI+KvRe6AnSpB6XHiYi0PLt3w/jx8PzzXkB+/fVwzDEq+tbUxF6mJiLih+QrMe2XaICuQnFST7EVYEVEpPnbvh3OOssLzh96yLun+bBhCs6bIo3hIuI3raBHRVfNdS90qSelx4mItBybN3sF33JzvfuZX3SR3z2S+tAYLiJ+U4Ae1aWL96wVdKmnaHqcCsyIiDRv69Z5Vdm/+MKr0j56tN89kvpKS0mjVXorjeEi4hsF6FHp6dCxowJ0qbc2GW0wTLPvIiLN2IoVXnC+eTO8/jqMHOl3j6ShBANBjeEi4hsF6LHCYaW4S72lWAptA201uIuINFOLFsFpp3k3f5k3D446yu8eSUMKBUJs3aMxXET8oSJxscJhraBLgwgGgiowIyLSDC1Y4K2WZ2TAO+8oOG+OgoGgUtxFxDcK0GNlZ2sFXRqE0uNERJqfOXO8tPbOneHdd+GII/zukTQGjeEi4qdGC9DNbKqZFZrZpzHbDjKzf5nZishz+8h2M7OJZrbSzPLNbEhj9atK4TB8/TWUlvry9dJ8hAIhDe4iIs3Ic895ReD69vVW0Q891O8eSWMJZWoMFxH/NOYK+jPA6RW23QL8xznXC/hP5D3AKKBX5HEF8EQj9qty2dneBWWFhb58vTQfSnEXEWk+Jk2C8ePhhBO8a847dvS7R9KYNIaLiJ8aLUB3zs0Hvq2weTTwbOT1s8A5Mdv/4jwfAO3MLLux+lapcNh7Vpq71JPS40REmj7n4O674Zpr4Kyz4LXXIBj0u1fS2IIZGsNFxD+Jvga9s3NuA0DkuVNke1dgbcx+BZFtBzCzK8ws18xyS0pKGrZ30QBdheKknpTiLiLStJWVwQ03wB13wIUXevc5z8ryu1eSCKHMENt2b6PMlfndFRFpgZKlSJzF2ebi7eicm+KcG+qcG5qW1sB3icuOLNprBV3qSRVgRUSarr174eKL4ZFH4PrrYdo0aOg/OSR5BQNBHI7te7b73RURaYESHaBvjKauR56jF3sXAIfE7NcNSPwydpcu3rNW0KWegoEgO/buoLRMBQdFRJqSXbvgRz+C6dO99PYHH4SUZFnOkIQIBUIAyoQTEV8kesiZDVwUeX0R8ErM9gsj1dyPBYqiqfAJlZ4OnTopQJd6C2VqcBcRaWqKimDUKPjnP73CcLfdBhYvx0+atWDAKzSgTDgR8UOjJWyZ2d+AkcDBZlYA/A64D5hlZpcBXwHnRXafA5wBrAR2Apc0Vr+qpXuhSwOIDu5bd2+lfVZ7n3sjIiLVKSyE00+HxYthxgwYN87vHolfYsdwEZFEa7QA3TlX2dB2cpx9HTChsfpSK+GwVtCl3pQeJyLSdGzYACNHwldfwSuvwBln+N0j8ZOy4ETET7qqqqLsbAXoUm/l6XG6j6qISNL7859hxQp44w0F56IxXET8pQC9onAYNm6EUhX3krpTepyISNORmwt9+sCJJ/rdE0kGGsNFxE8K0CsKh72bnxYWVr+vSCWUHici0nTk5cFRR/ndC0kWukxNRPykAL0i3QtdGoAqwIokFzM73cyWmdlKM7slzucBM/t75PP/mln3mM9+E9m+zMxOq65NM5sR2f6pmU01s/TI9vZm9pKZ5ZvZh2Y2oHHPWmri66+9K9sUoEtUm4w2gMZwkWTR0sZwBegVhcPes65Dl3pQepxI8jCzVGASMAroB4wzs34VdrsM+M45dzjwEHB/5Nh+wFigP3A68LiZpVbT5gzgCGAgkAVcHtn+W2CRc24QcCHwSCOcrtRSXp73rABdolJTUmmT0UZjuEgSaIljuAL0iqIr6ArQpR5ap7cmxVI0uIskh2OAlc65L51ze4CZwOgK+4wGno28fh442cwssn2mc263c24V3u1Aj6mqTefcHBcBfAh0i7TbD/hPZJ/Pge5m1rlxTllqKjfXu9f5kUf63RNJJqFASGO4SHJocWO4AvSKunTxnpXiLvVgZgQDQVWAFUmcNDPLjXlcEfNZV2BtzPuCyDbi7eOcKwGKgA5VHFttm5G0uJ8Cr0c2fQKMiXx2DPA99g384pO8POjdG9q29bsnkkw0hosklMbwGI12H/QmKz0dOnXSCrrUWzAQ1Oy7SOKUOOeGVvKZxdnmarhPZdvjTXBXbPNxYL5z7p3I+/uAR8xsEbAY+BgoqaTPScXMTsdL50sFnnLO3Vfh80PxVi/aRfa5xTk3J+EdrYO8PDjpJL97IckmlKkVdJEE0hgeQwF6PNnZWkGXegsFQpp9F0kOBcAhMe+7ARVnYaP7FJhZGhACvq3m2ErbNLPfAR2Bn0e3Oee2ApdEPjdgVeSR1GKu1fsB3s9joZnNds4tjdntNmCWc+6JyHV8c4DuCe9sLalAnFQmGAiypXiL390QkRY4hivFPZ5wWCvoUm9aQRdJGguBXmbWw8wy8ArGzK6wz2zgosjrc4E3I9efzQbGRirE9gB64V2TVmmbZnY5cBowzjlXFv0CM2sX2Re8ojPzIwN+sqvJ9X8OCEZehzjwj6ekpAJxUhmN4SJJo8WN4VpBjycchkWL/O6FNHGhzBCFOwr97oZIi+ecKzGza4C5eOnXU51zS8zsLiDXOTcbeBqYbmYr8Wbdx0aOXWJms4CleKlsE5xzpQDx2ox85WRgDfC+N8nOi865u4C+wF/MrDTS3mUJOP2GEO9avWEV9rkTeMPMfgG0Bk6J11DkusIrADIyMuLtklB5eSoQJ/GpSJxIcmiJY7gC9Hiys2HjRigthdRUv3sjTVQwEGTF5hV+d0NE8Kqy4qVdx267I+Z1MXBeJcfeC9xbkzYj2+OOrc659/Fm75uamlz/Nw54xjn3JzM7Du8PpQGxqw8AzrkpwBSA1q1bV2wj4VQgTioTDAR1H3SRJNHSxnCluMcTDkNZGRRq9VPqLpih9DgRaRZqcv3fZcAsKP8jJhM4OCG9q4fcXKW3S3zBQJAde3dQWlbqd1dEpIVRgB6P7oUuDUAVYEWkmajJ9X9fAScDmFlfvAB9U0J7WUvRAnFDK6sbLC1aKBACYNuebT73RERaGgXo8YTD3rMquUs9BANBdpXsYm/pXr+7IiJSZ5F7ykav1fsMr1r7EjO7y8zOjux2E/AzM/sE+BtwcaRAT9JSgTipSjDg1TxUmruIJJquQY8nGqBrBV3qITq4b929lQ6tOvjcGxGRuqvB9X9LgRMS3a/6UIE4qUoo01tBVyaciCSaVtDj6dzZG7W1gi71EE2P0+AuIpJ8VCBOqhI7yS4ikkgK0ONJT4eOHbWCLvVSnh63W+lxIiLJJi9P6e1SOY3hIuIXBeiVyc5WgC71otl3EZHk9PXXsG6dAnSpnLLgRMQvCtArEw4rxV3qJXr9mgrMiIgkl2iBOFVwl8qoSJyI+EUBemXCYa2gS71oBV1EJDmpQJxUR2O4iPhFAXplsrNh40YoLfW7J9JEKT1ORCQ5qUCcVKdNRhsM0xguIgmnAL0y4TCUlUFhod89kSZKBWZERJKTCsRJdcyMYCCoMVxEEk4BemWys71npblLHWWmZZKWkqbZdxGRJKICcVJTwUBQY7iIJJwC9MqEw96zCsVJHZkZoUBIg7uISBKJFohTgC7VCWVqDBeRxFOAXplogK4VdKkHpceJiCQXFYiTmtIYLiJ+UIBemc6dvRFcK+hSD0qPExFJLtECccGg3z2RZKcsOBHxgwL0yqSnQ8eOWkGXelF6nIhIclGBOKkpTbKLiB8UoFdF90KXegoGghQVKz1ORCQZbNyoAnFScxrDRcQPCtCrkp2tFHepF82+i4gkDxWIk9pQiruI+EEBelW0gi71FAqEVGBGRCRJRAN0FYiTmggGguwq2cXe0r1+d0VEWhAF6FXJzvby4UpL/e6JNFFaQRcRSR65udCnjwrESc0EA95/FI3jIpJICtCrEg5DWRkUFvrdE2miQoEQe0r3sLtkt99dERFp8VQgTmojlBkCFKCLSGIpQK+K7oUu9RSdfVeau4iIv1QgTmpLY7iI+EEBelWys71nFYqTOlJ6nIhIclCBOKktjeEi4gcF6FXRCrrUk9LjRESSgwrESW2FAhrDRSTxFKBXpXNnMFOALnVWnh6n+6iKiPgqLw9691aBOKk5jeEi4gcF6FVJT4eOHZXiLnWm9DgRkeSQmwtDh/rdC2lKlAUnIn5QgF4d3Qtd6kHpcSIi/lOBOKkLTbKLiB8UoFcnO1sr6FJnqgArIuI/FYiTushKyyLVUjWGi0hCKUCvjlbQpR40+y4i4j8ViJO6MDNCmSGN4SKSUArQqxMOe7lxpaV+90SaoEBagEBqQAVmRER8pAJxUlfBQFAr6CKSUArQq5OdDWVlUFjod0+kiQoGgpp9FxHxUV6e0tulbjSGi0iiKUCvju6FLvUUygyxdY8GdxERP2zcCAUFquAudRMKKMVdRBJLAXp1srO9ZxWKkzoKBoJKcRcR8YkKxEl9aAwXkURTgF4draBLPSk9TkTEPyoQJ/WhInEikmgK0KvTuTOYKUCXOlN6nIiIf1QgTuojmKFJdhFJLAXo1UlPh44dleIudaYKsCIi/lGBOKkPjeEikmgK0GtC90KXelCKu4iIP6IF4hSgS12FMkPsKd3D7pLdfndFRFoIBeg1kZ2tAF3qLJri7pzzuysiIi1K9PpzVXCXugoGvGsjNNEuIomiAL0mwmGluEudBQNBSspK2FWyy++uiIi0KCoQJ/UVDdCV5i4iiaIAvSbCYS9PrrTU755IE6TZdxERf6hAnNRXKBACNIaLSOIoQK+J7GwoK4PCQr97Ik1QKNMb3HUfVRGRxFKBOKmv8hV0jeEikiAK0GtC90KXetAKuoj/zOx0M1tmZivN7JY4nwfM7O+Rz/9rZt1jPvtNZPsyMzutujbNbEZk+6dmNtXM0iPbQ2b2/8zsEzNbYmaXNO5Zt2yFhSoQJ/WnMVzEfy1tDFeAXhPZ2d6zAnSpA6XHifjLzFKBScAooB8wzsz6VdjtMuA759zhwEPA/ZFj+wFjgf7A6cDjZpZaTZszgCOAgUAWcHlk+wRgqXNuMDAS+JOZZTT8GQvsu/5cAbrURzQLTmO4iD9a4hjuS4BuZjdEZh4+NbO/mVmmmfWIzHisiMyAJM8fLdEVdBWKkzpQgRkR3x0DrHTOfemc2wPMBEZX2Gc08Gzk9fPAyWZmke0znXO7nXOrgJWR9ipt0zk3x0UAHwLdIu06oG2k3TbAt0BJ45yyqECcNASN4SK+a3FjeMIDdDPrClwLDHXODQBS8WY27gcecs71Ar7DmwlJDp07g5lW0KVOlB4nkhBpZpYb87gi5rOuwNqY9wWRbcTbxzlXAhQBHao4tto2I2lxPwVej2x6DOgLrAcWA9c558pqeZ5SQ7m5XoG4UMjvnkhTpiw4kYTQGB4jrTEareH3ZpnZXqAVsAH4PvCTyOfPAncCT/jSu4rS06FjR62gS50oPU4kIUqcc5Xd7dribHM13Key7fEmuCu2+Tgw3zn3TuT9acAivPGuJ/AvM3vHOadfDo0gLw9OPNHvXkhTF0gLkJGaoTFcpHFpDI+R8BV059w64AHgK7zAvAjIA7ZEZjwg/swIAGZ2RXR2paQkgZmB4bBW0KVO2ma0BVQBVsRHBcAhMe+74c2Ax93HzNKAEF76WmXHVtmmmf0O6AjcGLPPJcCLkcy5lcAqvOvcpIGpQJw0pGAgqDFcxD8tbgz3I8W9PV6Ofw8gDLTGu0C/ooqzGN5G56Y454Y654ampSUwAUAButRRemo6WWlZmn0X8c9CoFek1kkG3mVVsyvsMxu4KPL6XODNyPVns4GxkQqxPYBeeNekVdqmmV2ON9M+rkL621fAyZF9OgN9gC8b/GxFBeKkQYUCIbbu0Rgu4pMWN4b7keJ+CrDKObcJwMxeBI4H2plZWmQVPd7MiL+ys+Hjj/3uhTRRocyQAnQRnzjnSszsGmAuXt2Tqc65JWZ2F5DrnJsNPA1MN7OVeLPuYyPHLjGzWcBSvGIwE5xzpQDx2ox85WRgDfC+V0uGF51zdwF3A8+Y2WK8tLubnXPfJOBH0OKoQJw0pGAgqDFcxCctcQz3I0D/CjjWzFoBu/BmInKBeXgzHjPxZkBe8aFvlQuHYeNGKCmBRK7cS7MQDARVAVbER865OcCcCtvuiHldDJxXybH3AvfWpM3I9riDhHNuPXBqrToudZKXB716qUCcNAyluIv4q6WN4X5cg/5fvPL3H+FVwEsBpgA3AzdGZj464M2EJI/sbCgr8y5sE6klzb6LiCRObi4MrazckEgtKQtORBLJl6Vg59zvgN9V2Pwl3j3pklPsvdCjr0VqKBQIaQVdRCQBVCBOGpqy4EQkkRK+gt5kRYNyFYqTOtAKuohIYqhAnDS0YIbGcBFJHAXoNZWd7T3rXuhSB0qPExFJDBWIk4YWHcO9otAiIo1LAXpNde4MZlpBlzoJZqjAjIhIIqhAnDS0YCBISVkJu0p2+d0VEWkBFKDXVHo6dOyoAF3qJJrirtl3EZHGlZen9HZpWKGAN9ujTDgRSQQF6LURDivFXeoklBnC4dixd4ffXRERabYKC2HtWlVwl4YVDAQBBegikhgK0GsjHNYKutRJdHBXmruISONRgThpDBrDRSSRFKDXRna2VtClTjT7LiLS+FQgThpDKFMp7iKSOArQayMcho0boaTE755IE6Pr10REGp8KxElj0CS7iCSSAvTaCIehrMy7yE2kFsrT43YrPU5EpLGoQJw0Bo3hIpJICtBrQ/dClzrS7LuISOPatMkrEKcAXRqasuBEJJEUoNdGOOw9q1Cc1FL0+jUVmBERaRzR689VwV0amorEiUgiKUCvDa2gSx1pBV1EpHHl5nrPKhAnDS09NZ2stCyN4SKSEArQa6NzZzDTCrrUWtuMtoACdBFpmszsdDNbZmYrzeyWSvb5sZktNbMlZvZcovuoAnHSmIKBoMZwEUmINL870KSkp0OnTgrQpdZSU1Jpk9FGBWZEpMkxs1RgEvADoABYaGaznXNLY/bpBfwGOME5952ZdUp0P/Py4IQTEv2t0lIEA0GN4SKSEFpBry3dC13qSLPvItJEHQOsdM596ZzbA8wERlfY52fAJOfcdwDOuYTe7kQF4qSxhTJDGsNFJCEUoNdWOKwVdKmTUECDu4g0SV2BtTHvCyLbYvUGepvZu2b2gZmdHq8hM7vCzHLNLLekpKTBOhgtEKcAXRqLJtlFJFFadIC+cSN07w5Tp9bioOxsBehSJ0qPE5EmyuJscxXepwG9gJHAOOApM2t3wEHOTXHODXXODU1La7ir7KIB+pAhDdakyH40hotIorToAP3gg71s9c8/r8VB4TAUFkIDzvxLy6DZdxFpogqAQ2LedwMqzlQXAK845/Y651YBy/AC9oTIzVWBOGlcyoITkURp0QF6aiocfjgsX16Lg8JhKCvzgnSRWtD1ayLSRC0EeplZDzPLAMYCsyvs8zJwEoCZHYyX8v5lojqYl6f0dmlcmmQXkURp0QE6QO/etQzQdS90qaNgRpCiYqXHiUjT4pwrAa4B5gKfAbOcc0vM7C4zOzuy21xgs5ktBeYBv3LObU5E/1QgThIhGqA7V/HqDhGRhtXib7PWpw+8+iqUlnor6tUKh73n9ev114DUimbfRaSpcs7NAeZU2HZHzGsH3Bh5JJQKxEkihAIhylwZO/buoE1GG7+7IyLNmFbQe8PevbBmTQ0PiK6gq1Cc1FIoM8S2PdsoLSv1uysiIs2GCsRJIgQDQQBlwolIo1OA3tt7Xrashgd07gxmSnGXWosO7tv3bPe5JyIizUdengrESeMLZXr/wZQJJyKNTQF6JECv8XXo6enQqZNW0KXWQgEN7iIiDS03V+nt0viik+waw0WksbX4AL1jR2/WvdaF4rSCLrVUnh6n+6iKiDQIFYiTRNEYLiKJ0uIDdDOvUFytb7WmFXSpJc2+i4g0LBWIk0RRFpyIJEqLD9ChDrdaU4AudaDr10SkLsystd99SFYqECeJokl2EamLuozhCtDxAvSvvoKdO2t4QHY2FBZCSUmj9kuaF1WAFZHaMLPjI/cV/yzyfrCZPe5zt5JKXh4cfrgKxEnj0xguIrVRnzFcATr7CsWtXFnDA8JhKCvzgnSRGtLsu4jU0kPAacBmAOfcJ8D/+NqjJJOXB0OH+t0LaQk0hotILdV5DFeATh0quete6FIHun5NRGrLObe2wqZSXzqShHbvhiOOgJEj/e6JtASpKam0Tm+tMVxEaqyuY3haI/SlyenVy3uucYAeDnvPquQutdA6ozWGqQKsiNTUWjM7HnBmlgFcSyRVTiAQgLlz/e6FtCTBQFBjuIjUVJ3HcK2gA23aQNeudQjQtYIutZBiKbQNtNXsu4jU1JXABKArUADkAFf72iORFiyUGdIYLiI1VecxXCvoEb17w7JlNdy5c2fv/mxaQZdaCgVCmn0XkZrq45y7IHaDmZ0AvOtTf0RaNK2gi0gt1HkM1wp6RK1utZaWBp06aQVdai0YCGr2XURq6tEabhORBAgFtIIuIjVW5zFcK+gRvXvDt9/C5s3QoUMNDsjOVoAutab0OBGpjpkdBxwPdDSzG2M+CgKp/vRKRIKBIOu2rfO7GyKSxBpiDNcKekStK7mHw0pxl1oLBoK6h6qIVCcDaIM3id425rEVONfHfom0aBrDRaQG6j2GawU9ok8f73n5cjjuuBocEA7DRx81ap+k+QkGgqz6bpXf3RCRJOacext428yecc6t8bs/IuJRiruIVKchxnAF6BHdu3uXltfqXuiFhVBS4h0oUgMa3EWkFnaa2R+B/kBmdKNz7vv+dUmk5QoGgmzbs40yV0aKKQlVRKpU5zFcv10i0tPhsMNqUck9HIayMi9IF6khVYAVkVqYAXwO9AB+D6wGFvrZIZGWLBgIArBt9zafeyIiTUCdx3AF6DFqVcld90KXOggGguzcu5OSshK/uyLSopjZ6Wa2zMxWmtktcT4PmNnfI5//18y6x3z2m8j2ZWZ2WnVtmtmMyPZPzWyqmaVHtv/KzBZFHp+aWamZHVRFtzs4554G9jrn3nbOXQoc2wA/DhGpg1BmCECZcCIJ1tLGcAXoMXr3hhUrvIXxamVne88qFCe1EAp4g7tm30USx8xSgUnAKKAfMM7M+lXY7TLgO+fc4cBDwP2RY/sBY/FS1E4HHjez1GranAEcAQwEsoDLAZxzf3TO5TjncoDfAG87576tout7I88bzOxMMzsS6FbXn4OI1E90BV0BukjitMQxXBdPx+jTB4qLoaAADj20mp21gi51EB3ci3YX0T6rvc+9EWkxjgFWOue+BDCzmcBoYGnMPqOBOyOvnwceMzOLbJ/pnNsNrDKzlZH2qKxN59ycaKNm9iHxB+RxwN+q6fc9ZhYCbsK7d2oQuKFGZywiDS46yd4Yl6rt3buXgoICiouLG7xtkWSRmZlJt27dSE9Pr81hLW4MV4AeI/ZWa9UG6J07g5lW0KVWNPsu0mjSzCw35v0U59yUyOuuwNqYzwqAYRWOL9/HOVdiZkVAh8j2Dyoc2zXyuso2I2lxPwWuq7C9Fd5M/jWVnUxkdr+Xc+6fQBFwUmX7ikhiNOYYXlBQQNu2benevTteXCHSvDjn2Lx5MwUFBfTo0aPixxrDYyjFPUY0QK9Robi0NOjUSSvoUivR69d0H1WRBlfinBsa85gS81m8v3ZdhfeV7VPb7bEeB+Y7596psP0s4N2qUuOcc6XA2ZV9LiKJV54F1whjeHFxMR06dFBwLs2WmdGhQ4fKskQ0hsfQCnqM7Gxo3bqWheIUoEstaAVdxBcFwCEx77sBFX95R/cpMLM0IAR8W82xlbZpZr8DOgI/j9OfsVSfGgfwnpk9Bvwd2BHd6Jz7qAbHikgDa+wicQrOpbmr4//xFjeGK0CPYVbLSu7Z2Upxl1qJXr+mAF0koRYCvcysB7AOb3D9SYV9ZgMXAe8D5wJvOuecmc0GnjOzB4Ew0Av4EG/2PW6bZnY5cBpwsnNuv7KjkevRRgDja9Dv4yPPd8Vsc4Dugy7iA02yi/iixY3hSnGvoE8fraBL44ktEiciieGcK8G7Vmwu8Bkwyzm3xMzuMvgQ7OQAACAASURBVLNoCtrTQIdIAZkbgVsixy4BZuEVo3kdmOCcK62szUhbk4HOwPuR27HcEdOdHwJvOOd2UA3n3ElxHvUOzs1suJldEnndMfIHiohUo01GGwxrlmP4yJEjmTt37n7bHn74Ya6++uoqj2vTpg0A69ev59xzz6207dzc3LifxX7Xzp07y9+fccYZbNmypSZdT1ovv/wyd911V/U7NpBVq1YxbNgwevXqxfnnn8+ePXsO2GfPnj1ccsklDBw4kMGDB/PWW2+VfzZy5Ej69OlDTk4OOTk5FBYWAnDDDTeUb+vduzft2rUDYM2aNRx11FHk5OTQv39/Jk+eXN7WKaecwnfffdcg59Uix3DnXJN9tGrVyjW0O+5wLiXFueLiGux8++3OmTm3d2+D90Oap+27tzvuxN2/4H6/uyLSrAA7XBKMS8n+AH4H/D9geeR9GO9aumYxhos0tuD/Bd11r13X4O0uXbq0wdusjcmTJ7uLL754v23Dhg1z8+fPr/K41q1bV9v2iBEj3MKFC6vc53vf+57btGlT9R39/9m78/ioqvv/469PFiYQmAFBKJsGgSAEwhbAFaGooK0BFVncoKjUre79urTV709t61ZUXKuioqWgYBWkil8XFKTKLihQBAUVUFSQHQIJ5/fHnYkJZJkkN5lM8n4+HvOYzJ07556JyIfPOed+Tozl5uZGfe7xxx9fpd/pvPPOc5MnT3bOOffb3/7WPf7444ed8+ijj+b/d968ebPr0aOHy8vLc85F999p/Pjx7je/+Y1zzrmcnBy3L5ww7dy50x199NFu48aNzjnnnn/+eXf33XcX2UZRf9YVwws/NIN+iPR0bx/0L76I4uQWLcA5CI8wiZSmXnI9Ei1Ry+NEJFbOxitcsxvAObcJaBDTHonEkWAgWOkx/LrroF8/fx/XXVfyNYcOHcrMmTPJyckBYP369WzatImTTjqJXbt2MWDAAHr06EGXLl2YPn36YZ9fv349nTt3BmDv3r2MGDGCzMxMhg8fzt69e/PPu+KKK8jKyiIjI4M77rgDgPHjx7Np0yb69+9P//5eseu0tDR+/PFHAMaNG0fnzp3p3LkzDz30UP71OnbsyGWXXUZGRgann356oetETJ06lc6dO9O1a1f69u0LQF5eHjfddBNdunQhMzOTRx55BIB3332X7t2706VLF8aMGZP/u0hLS+POO+/kpJNOYurUqXzxxRcMGjSInj17cvLJJ/Pf//73sOt+/vnnBAIBmjRpAsDo0aO54oor6N+/P8cccwwffPABY8aMoWPHjowePbrE38/27dvp0KEDq8NVrEeOHMnTTz9d6HrOOd577738VQyjRo3itddeO6xfK1euZMCAAQA0bdqUhg0blrq6oaDJkyczcuRIAOrUqUMgEAAgJyeHgwd/Xg2enZ3N5MnR3KYtRVGCfoiCW62VSnuhSxmZGcFAUFXcRSRW9odnKxyAmaXGuD8icSUYCNbIJe6NGzemd+/ezJo1C4ApU6YwfPhwzIyUlBReffVVlixZwuzZs7nxxhsjK3KK9MQTT1CvXj2WL1/OH/7wBxYvXpz/3p///GcWLVrE8uXL+eCDD1i+fDnXXHMNLVq0YPbs2cyePbtQW4sXL+a5555j/vz5fPzxxzz99NMsXboUgDVr1nDVVVexYsUKGjZsyCuvvHJYX+68807eeustli1bxowZMwB46qmnWLduHUuXLmX58uVccMEF7Nu3j9GjR/PSSy/x6aefkpubyxNPPJHfTkpKCh9++CEjRoxg7NixPPLIIyxevJgHHnigyNsA5s2bR48ePQod++mnn3jvvfd48MEHOeuss7j++utZsWIFn376KZ988kmxv59QKMSjjz7K6NGjmTJlCj/99BOXXXZZoba3bNlCw4YNSUryyou1atWKjRs3Htavrl27Mn36dHJzc1m3bh2LFy/mm29+3m3sN7/5Dd26deOuu+467L/xV199xbp16/jlL39epf3NN9+QmZlJ69atufnmm2kRzo0aNWpETk4OW7ZsOawPUjoViTtE+/bec1QJevPm3rMKxUkZBANBduzXDLqIlMzMzini8HbgU+dceZduvWxmfwcamtllwBjg6VI+IyJhoUCo0mfQw5PEVW7kyJFMmTKFwYMHM2XKFJ599lnAm5297bbbmDNnDgkJCWzcuJHNmzfzi1/8osh25syZwzXXXANAZmYmmZmZ+e+9/PLLPPXUU+Tm5vLtt9+ycuXKQu8f6sMPP+Tss88mNdUbSzznnHOYO3cu2dnZtGnThm7dugHQs2dP1q9ff9jnTzzxREaPHs2wYcM45xzvr9R33nmHyy+/PD+ZPeKII1i2bBlt2rQhPTxTN2rUKB577DGuCy89GD58OAC7du3iP//5D+edd17+NSIz7QV9++23HHnkkYWOnXXWWZgZXbp0oVmzZnTp0gWAjIwM1q9fT7du3Yr9/Zx22mlMnTqVq666imXLlh12vaIGTIqqmD5mzBhWrVpFVlYWRx99NCeccEL+72HSpEm0bNmSnTt3cu655/Liiy9y8cUX5392ypQpDB06lMTExPxjrVu3Zvny5WzatIkhQ4YwdOhQmjVrBngz9Js2baJx48aH9aM2qEgMjypBN7NrgeeAncAzQHfgFufc/5Wxr9Vew4be9uaaQZfKEkqp/OAuIjXCJcDxQGRKqR/wMZBuZnc6514sa4POuQfM7DRgB9ABuN0597ZP/RWp8YKBID/t86f4VXUzZMgQbrjhBpYsWcLevXvzZ4AnTZrEDz/8wOLFi0lOTiYtLa24vazzFZUcrlu3jgceeICFCxfSqFEjRo8eXWo7Jc3UR5ZXAyQmJha5xP3JJ59k/vz5/Pvf/6Zbt2588sknOOcO619J1wHyBwgOHjxIw4YN82e8i1O3bl22by+80iLS34SEhEJ9T0hIyJ/RLu73c/DgQVatWkXdunXZunUrrVq1KtR2kyZN2LZtG7m5uSQlJbFhw4b82eyCkpKSePDBB/Nfn3DCCbQPz062bNkSgAYNGnD++eezYMGCwxL0xx57rMjv26JFCzIyMpg7d27+Mvt9+/ZRt27dEn9PNVy5Y3i0S9zHOOd2AKfj7Qn3G+Cecne3mou6knuzZt7ebErQpQy0xF1EonQQ6OicO9c5dy7QCcgB+gA3l7UxM0s0s3ecc287537vnLtJyblI2YRSQjU2htevX59+/foxZsyY/PuMwbsHumnTpiQnJzN79my++uqrEtvp27cvkyZNAuCzzz5j+fLlAOzYsYPU1FRCoRCbN2/mzTffzP9MgwYN2LlzZ5Ftvfbaa+zZs4fdu3fz6quvcvLJJ0f9nb744gv69OnDnXfeSZMmTfjmm284/fTTefLJJ8nNzQVg69atHHvssaxfv561a9cC8OKLL3LKKacc1l4wGKRNmzZMnToV8BL7oma0O3bsmN9WtEr6/Tz44IN07NiRyZMnM2bMGA4cOFDos2ZG//79mTZtGgATJ05k8ODBh10j8nsEePvtt0lKSqJTp07k5ubm3/N/4MABZs6cmV9TAGD16tX89NNPHH/88fnHNmzYkD8o8tNPPzFv3jw6dOiQ/3v57rvvSEtLK9PvoIYpdwyPNkGPDDOdCTznnFtW4FiNE/Ve6ElJ3nS7lrhLGVRFgRkRqRHSnHObC7z+Hkh3zm0FDhTzmWI55/KAPeF9XEWkHIJ1anYMHzlyJMuWLWPEiBH5xy644AIWLVpEVlYWkyZN4thjjy2xjSuuuIJdu3aRmZnJfffdR+/evQHv/ufu3buTkZHBmDFjOPHEE/M/M3bsWM4444z8InERPXr0YPTo0fTu3Zs+ffpw6aWX0r1796i/z+9//3u6dOlC586d6du3L127duXSSy/lqKOOIjMzk65du/LPf/6TlJQUnnvuOc477zy6dOlCQkICl19+eZFtTpo0iQkTJtC1a1cyMjKKLJrXt29fli5dWurMfEHF/X4+//xznnnmGf72t79x8skn07dvX+6+++7DPn/vvfcybtw42rVrx5YtW7jkkksAmDFjBrff7u0U9v3339OjRw86duzIvffey4svepO4OTk5DBw4kMzMTLp160bLli0L3ec+efJkRowYUWjlwapVq+jTpw9du3bllFNOyS+8B17tgOOOOy5/+XwtVe4YbtH8wTGz54CWQBugK5AIvO+c61nuLvsgNTXVRUaB/HTffXDzzbBtG4RK+2dMjx7eUveZM33vh9RM579yPgs3LWTN79bEuisiNYaZ7XHO1aiCZ2b2OHAUMDV86FxgA/B7YKZzrn9xny2hzZeB44C3CVdyB3DOXVPhDpdRZcVwkcp041s38uTiJ9l9m79/dletWkXHjh19bVNi69prr+Wss87i1FNPjXVXqty1115LdnZ2fsX4gor6s64YXli0wxqXAN2AL51ze8zsCLxl7uXtcEO8e9k741WSHQOsBl4C0oD1wDDnXExu8ilYyb1Xr1JObt5cM+hSJppBF5EoXYUX0E/EW7X2AvBKuAp7mZPzsH+HHyJSDqGUEHsO7CH3YC5JCbV6dlBKcdtttzF//vxYdyMmOnfuXGRyXsuUO4ZH+zfL8cAnzrndZnYh0AN4uPz95WFglnNuqJnVAeoBtwHvOufuMbNbgFsoxz12fihTgt6iBSxZUul9kpqjKirAikj8CwfxaeGHX21ODMfdcKRjtXOuzMvlRWqrYCAIwM6cnTSq2yjGvZHqrFmzZmRnZ8e6GzFx6DZwtVFFYni096A/gXffWlfgf4Cv8EYByszMgkBfYAKAc26/c24bMBiYGD5tIjCkPO37oW1bSEgoQyX3zZshXGhCpDTBQJB9ufvYn7c/1l0RkWrMzM4xszVmtt3MdpjZTjOr0OiemfUD1gCPAY8Dn5tZXx+6K1IrRBL0mrgXuoj4pyIxPNoEPTc8CjAYeNg59zDQoJz9PQb4AXjOzJaa2TNmlgo0c859CxB+blrUh81srJktMrNFuZWUFAcCkJZWhr3QnYPvy7slrdQ2keCuWXQRKcV9QLZzLuScCzrnGjjnghVs82/A6c65U5xzfYGBwIOlfEZEwkIBrziRYriIlKLcMTzaBH2nmd0KXAT828wSgeRydjYJb4n8E8657nhFam6J9sPOuaecc1nOuazKrAyYng6rV0dxovZClzIKpSi4i0hUNjvnVvncZrJzLj+6Oec+p/zxXKTW0SC7iESp3DE82gx3OHA+3n7o35nZUcD95bkgXvW6Dc65SNWEaXgJ+mYza+6c+9bMmuOVoo+Z9HSYO9ebHLeSNpRr3tx7VqE4iVL+8rgauo+qiPhmkZm9BLyGt3cqAM65f1WwzQnAi+HXFwCLK9CeSK2iGC4iUSp3DI9qBt059x0wCQiZ2a+Bfc65ct2DHm7rGzPrED40AFgJzABGhY+NAg7fVLAKpafD7t1R5N2aQZcy0ui7iEQpCOwBTgfOCj9+XcE2rwBWANcA1+LF36I3+xWRw9TUVXD9+vXjrbfeKnTsoYce4sorryzxc/Xr1wdg06ZNDB06tNi2Fy1aVGI7Dz30EHv27Ml/feaZZ7Jt27Zoul6lrrvuOubMmVNl15s1axYdOnSgXbt23HPPPUWek5OTw/Dhw2nXrh19+vRh/fr1+e/99a9/pV27dnTo0KHQf9/i2l23bh19+vShffv2DB8+nP37vXpJjz76KM8991zlfMmaq9wxPKoE3cyGAQuA84BhwHwzK/r/wuj8DphkZsvxtm/7C3APcJqZrQFOC7+OmQ7h4YNS70Nv1sybYleCLlHS/WsiEg3n3G+KeIypYLNJeLVkznHOnQ2MBxIr3luR2qGmDrKPHDmSKVOmFDo2ZcoURo4cGdXnW7RowbRp5d9w4tAE/Y033qBhw4blbs8PeXl5hV5v3bqVjz/+mL59q6auZl5eHldddRVvvvkmK1euZPLkyaxcufKw8yZMmECjRo1Yu3Yt119/PTff7G2CtXLlSqZMmcKKFSuYNWsWV155JXl5eSW2e/PNN3P99dezZs0aGjVqxIQJEwAYM2YM48ePr5LvXVNUJIZHew/6H4BezrlRzrmLgd7AnyrQ4U/C95FnOueGOOd+cs5tcc4NcM61Dz9vLW/7fii41VqJkpKgaVMtcZeoqQKsiJTEzP4n/PyImY0/9FHB5t8F6hZ4XRd4p4JtitQakUH2yozh110H/fr5+7juupKvOXToUGbOnElOjrcSd/369WzatImTTjqJXbt2MWDAAHr06EGXLl2YPv3wRa7r16+nc+fOAOzdu5cRI0aQmZnJ8OHD2bt3b/55V1xxBVlZWWRkZHDHHXcAMH78eDZt2kT//v3p39/bHjotLY0ff/wRgHHjxtG5c2c6d+7MQw89lH+9jh07ctlll5GRkcHpp59e6DoRmzdv5uyzz6Zr16507dqV//znPwD84x//oHfv3nTr1o3f/va3+cl4/fr1uf322+nTpw8fffRRobamTZvGoEGD8l+npaVx2223cfzxx5OVlcWSJUsYOHAgbdu25cknnwQo9ne3cOFCMjMz2bdvH7t37yYjI4PPPvus0PUWLFhAu3btOOaYY6hTpw4jRowo8nc/ffp0Ro0alf/f8d1338U5x/Tp0xkxYgSBQIA2bdrQrl07FixYUGy7zjnee++9/JUQo0aN4rXXXgOgXr16pKWlsWDBgsOuL4X5EcOjvQc9wTlX8J7wLUSf3MelVq0gJaUMW61pBl2iVFNH30XEN5GiMiWvCS2fFOfcrsgL59wuM6tXCdcRqZHqJdcjwRJqXAxv3LgxvXv3ZtasWQwePJgpU6YwfPhwzIyUlBReffVVgsEgP/74I8cddxzZ2dlYMUWannjiCerVq8fy5ctZvnw5PXr0yH/vz3/+M0cccQR5eXkMGDCA5cuXc8011zBu3Dhmz55NkyZNCrW1ePFinnvuOebPn49zjj59+nDKKafQqFEj1qxZw+TJk3n66acZNmwYr7zyChdeeGGhz19zzTWccsopvPrqq+Tl5bFr1y5WrVrFSy+9xLx580hOTubKK69k0qRJXHzxxezevZvOnTtz5513Hva95s2bd9gy/tatW/PRRx9x/fXXM3r0aObNm8e+ffvIyMjg8ssvL/Z316tXL7Kzs/njH//I3r17ufDCC/MHOCI2btxI69at81+3atWK+fPnc6iC5yUlJREKhdiyZQsbN27kuOOOK/T5jRs35vf70Ha3bNlCw4YNiRThLng+QFZWFnPnzqV3796H9UEKqXAMjzZBn2VmbwGTw6+HA2+U96LxICEB2rePspJ78+ZK0CVqkfvXVGBGRIrinHs9/DyxEprfbWY9nHNLAMwsCzh82klEimRmBAPBSo3h4UniKhdZ5h5J0J999lkAnHPcdtttzJkzh4SEBDZu3MjmzZv5xS9+UWQ7c+bM4ZprrgEgMzOTzMzM/PdefvllnnrqKXJzc/n2229ZuXJlofcP9eGHH3L22WeTmpoKwDnnnMPcuXPJzs6mTZs2dOvWDYCePXsWuvc64r333uOFF7yyWYmJiYRCIV588UUWL15Mr169AG/Gv2nTpvnnnHvuuUX25dtvv+XII48sdCw7OxuALl26sGvXLho0aECDBg1ISUlh27ZtpKamFvu7u/322+nVqxcpKSlFLh/3drgurKhBkeLOK+74wYMHy3R+RNOmTfnvf/972DlSmB8xPKoE3Tn3ezM7FzgRMOAp59yr5b1ovEhPh08/jeLEFi1gyZJK74/UDIHEAMkJyTVu9F1E/GVm6cBNQBoF4rVz7pcVaPZaYKqZbQIc0AJv0F1EohQKhNixv+bF8CFDhnDDDTewZMkS9u7dmz/zPWnSJH744QcWL15McnIyaWlp7Nu3r8S2ikok161bxwMPPMDChQtp1KgRo0ePLrWdopLGiEAgkP9zYmJikUvci2tz1KhR/PWvfz3svZSUFBITiy7LUbdu3cP6G+lDQkJCof4kJCSQm5tb4u9u69at7Nq1iwMHDrBv3778QYiIVq1a8c033+S/3rBhAy0ixamLOK9Vq1bk5uayfft2jjjiiBI/X9TxJk2asG3bNnJzc0lKSjrsevv27aNu3YJ3SElJKhLDo16m7px7xTl3g3Pu+tqQnINXKO7LL+HAgVJObNECNm+G3Nwq6ZfENzMjlBJSgi4ipZkKLAX+CPy+wKMi2gDd8aq5vw2sxkvURSRKwUCwRsbw+vXr069fP8aMGVOoONz27dtp2rQpycnJzJ49m6+++qrEdvr27cukSZMA+Oyzz1i+fDkAO3bsIDU1lVAoxObNm3nzzTfzP9OgQQN27txZZFuvvfYae/bsYffu3bz66qucfPLJUX+nAQMG8MQTTwBe0bUdO3YwYMAApk2bxvffe3fvbt26tdTvBNCxY0fWrl0b9bWh5N/d2LFjueuuu7jgggvyC7sV1KtXL9asWcO6devYv38/U6ZMyZ+xLyg7O5uJE73J2mnTpvHLX/4SMyM7O5spU6aQk5PDunXrWLNmDb179y62XTOjf//++cX+Jk6cyODBg/Ov8/nnnx+2DF9KVO4YXmKCbmY7zWxHEY+dZlbz/mY6RHq6l3MXsWKmsObNvQ3Tv4/p1u0SR4KBoIrEiUhpcp1zTzjnFjjnFkceFWzzT865HUBDvB1TngKeqHBPRWqRyl7iHksjR45k2bJljBgxIv/YBRdcwKJFi8jKymLSpEkce+yxJbZxxRVXsGvXLjIzM7nvvvvy71nu2rUr3bt3JyMjgzFjxnDiiSfmf2bs2LGcccYZ+UXiInr06MHo0aPp3bs3ffr04dJLL6V79+5Rf5+HH36Y2bNn06VLF3r27MmKFSvo1KkTd999N6effjqZmZmcdtppfBtFsedf/epXvP/++1FfG4r/3b3wwgskJSVx/vnnc8stt7Bw4ULee++9Qp9NSkri0UcfZeDAgXTs2JFhw4aRkZEBwO23386MGTMAuOSSS9iyZQvt2rVj3Lhx+dumZWRkMGzYMDp16sSgQYN47LHHSExMLLHde++9l3HjxtGuXTu2bNnCJZdckt+fefPmceqpp5bp+9dy5Y7hVtLSkeouNTXV7d69u9La/+gjOOEEmDkTfvWrEk6cMQMGD4aFCyErq9L6IzVH9793p3WwNTNGzoh1V0RqBDPb45xLLf3M+GFm/wt8D7wK5ESOV2SXEzNb6pzrbmZ/BT51zv0zcqzCHS6jyo7hIpXlV//8FZt3bWbRWP/qOK5atYqOHTv61p5UjpNOOomZM2fGfAu4qrZ06VLGjRvHiy++WOG2ivqzrhheWLRF4mqlyFZrq1eXkqBH7s9QoTiJUiigJe4iUqpR4eeCS+IccEwF2txoZn8HTgXuNbMANXxXFhG/BQNB1mxZE+tuSAz87W9/4+uvv651CfqPP/7IXXfdFetuxJtyx3Al6CVo3BiOOCKKrdaaN/eetRe6RCkYCPLNjm9KP1FEaiUzSwAudM7N87npYcAg4AHn3DYza07F72sXqVVCgZBuU6ul+vTpE+suxMRpp50W6y7ElYrGcI2alyI9PYoEvVkzMNMMukStphaYERF/OOcOAg9UQrt7nHP/cs6tCb/+1jn3f35fR6Qmq6wYHs+3nYpEo7b8Ga9oDFeCXooOHaJI0JOSoGlTzaBL1LTEXUSi8H9mdq4VtV+RiMRMMBBkX+4+9uft963NlJQUtmzZUmsSGKl9nHNs2bKFlJSUWHelqpQ7hmuJeynS02HiRNi1C+rXL+HEFi00gy5Ri1SAdc4VuVeoiAhwA5AK5JrZPsAA55wLxrZbIrVbKBACYEfODprUa+JLm61atWLDhg388MMPvrQnUh2lpKTQqlWrWHejqpQ7hitBL0WkUNyaNVDirg5K0KUMgoEgBw4eICcvh5SkWjOSKCJl4JxrEOs+iMjhggHv39d+JujJycm0adPGl7ZEJPYqEsOVoJcikqB//nkpCXrz5rC4otvTSm0RSvFG37fv205KfSXoIlI0M2sEtAfy/6Jwzs2JXY9EJJKg19S90EXEH+WN4UrQS9Gunfdc6n3o7drBd995s+iRbddEilFw9L1Z/WYx7o2IVEdmdilwLdAK+AQ4DvgI+GUs+yVS20UG2VVLRkSKU5EYriJxpahXD446KooEPbJR+syZld4niX8F718TESnGtUAv4CvnXH+gO6AbVEVirOAgu4hIMcodw5WgRyGqrdYyMuCYY2D69Crpk8S3/OVx2kdVRIq3zzm3D8DMAs65/wIdYtwnkVpPMVxEolDuGK4EPQqRBL3EnS/MIDsb3n3XK/kuUgKNvotIFDaYWUPgNeBtM5sOqBqpSIxpFZyIRKHcMVwJehTS02HbNih154vsbMjJgbffrpJ+SfzS/WsiUhrn3NnOuW3Ouf8F/gRMAIbEoi9mNsjMVpvZWjO7pYTzhpqZM7OsquyfSFXSILuIlKYiMVwJehQKVnIv0UknQaNGWuYupVIFWBGJhpmdZGa/cc59gFdcpmUM+pAIPAacAXQCRppZpyLOawBcA8yv2h6KVK2UpBSSE5IVw0WkROWN4UrQo9AhfLdAqQl6cjKceaZXKC4vr9L7JfFLo+8iUhozuwO4Gbg1fCgZ+EcMutIbWOuc+9I5tx+YAgwu4ry7gPuAfVXZOZGqZmYEA0HFcBEpVkViuBL0KBx9tJd7l5qgg7fMfcsW+OijSu+XxK86iXVISUpRcBeRkpwNZAO7AZxzm4AGMehHS+CbAq83cMgsgJl1B1o750rcysTMxprZIjNblJub639PRapIMBBkx37FcBEpVrljuBL0KCQmetucR5WgDxrkZfNa5i6lCAaCqgArIiXZ75xzgAMws9QY9cOKOJZfNtXMEoAHgRtLa8g595RzLss5l5WUlORjF0WqViglpCXuIlKScsdwJehRSk+H1aujODEYhP79YcaMSu+TxDctjxORUrxsZn8HGprZZcA7Jr/XCAAAIABJREFUwNMx6McGoHWB160oXIm2AdAZeN/M1gPHATNUKE5qMsVwESlFuWO4EvQopafD2rVR3lqene1Nt0eV0UttFQqENIMuIsVyzj0ATANewds79Xbn3CMx6MpCoL2ZtTGzOsAIIH8U2jm33TnXxDmX5pxLAz4Gsp1zi2LQV5EqoVVwIlKSisRwJehRSk+H/fvh66+jODk723vWMncpgUbfRaQ0zrm3nXO/d87d5JyLyR6ezrlc4GrgLWAV8LJzboWZ3Wlm2bHok0ishQIhxXARKVF5Y7gS9ChFXckdoHVr6N5dy9ylRKEUBXeRqlLaPt5mFjCzl8LvzzeztALv3Ro+vtrMBpbWpplNCh//zMyeNbPkAu/1M7NPzGyFmX1QTF93mtmOIh47zSwmf2k4595wzqU759o65/4cPna7c+6wQOec66fZc6npNMguUnVqWwxXgh6lqPdCj8jOhv/8B374odL6JPEtGAiqwIxIFYhyH+9LgJ+cc+3wCp7dG/5sJ7wl3RnAIOBxM0sspc1JwLFAF6AucGm4rYbA43jLvzOA84rqr3OugXMuWMSjgXMu6MfvREQqJhLDvRpQIlJZamMMV4IepaZNvfpvUd9WPngwOOftiS5ShGAdjb6LVJFo9vEeDEwM/zwNGGBmFj4+xTmX45xbB6wNt1dsm+HZZheu3roAr6gawPnAv5xzX4fP+76Svq+IVLJQIMSBgwfIycuJdVdEarpaF8OVoEfJzJtFj3oGvVs3aNVKy9ylWJEl7hp9F/FFUmR/7fBjbIH3St3Hu+A54XuutwONS/hsNHuDJwMXAbPCh9KBRmb2vpktNrOLy/41RaQ6CAa8iTANtIv4QjG8AG1CWgbp6TBvXpQnm3nL3J9/Hvbuhbp1K7NrEoeCgSB5Lo89B/aQWidW2xuL1Bi5zrnitvUqcR/vUs4p7nhRA9yHtvk4MMc5Nzf8OgnoCQzAWzb3kZl97JyLduhXRKqJUEoIgO37ttM0tWmMeyMS9xTDC9AMehl06OBVcd+7N8oPDB4Me/bAu+9War8kPmn0XaTKlLaPd6FzzCwJCAFbS/hsiW2a2R3AkcANh1xjlnNut3PuR2AO0LXc30pEYkYxXKTK1LoYrgS9DNLTvdvKv/giyg+ccgo0aKBl7lKkUMAbfVdwF6l0Je7jHTYDGBX+eSjwXvj+sxnAiHCF2DZAe7x70opt08wuBQYCI51zBwtcYzpwspklmVk9oA/etmUiEmeUoItUmVoXw7XEvQwKVnLv3DmKDwQCMGgQvP46HDwICRoPkZ9Fgvv2HFVyF6lMzrlcM4vs450IPBvZxxtYFN4qbALwopmtxRt1HxH+7AozexlYCeQCVznn8gCKajN8ySeBr/CWv4FXVOZO59wqM5sFLAcOAs845z6rit+BiPgrMsiuGC5SuWpjDLd4LlCVmprqdu/eXWXX27nTq+T+l7/ArbdG+aFJk+DCC+Hjj6FPn0rtn8SXuV/Npe/zfXn7orc59ZhTY90dkbhmZnuccyrmEEeqOoaL+OmLrV/Q7pF2TBwykYu7qt6jSEUohhemKd0yaNAAmjcvQyV3gDPOgMRELXOXwxQsMCMiIiLxI38VnGK4iPhMCXoZdehQxgT9iCOgb1+YPr3S+iTxSfeviYiIxCfFcBGpLErQy6hMe6FHZGfDihVlqC4ntYGKxImIiMSnQFKAQGJAMVxEfKcEvYzS0+HHH2Hr1jJ8KDvbe3799Urpk8SnBoEGgArMiIiIxKNgIKgYLiK+U4JeRgUruUftmGO8su9a5i4FJCUkUS+5nkbfRURE4lAoJaQYLiK+U4JeRuVK0MGbRZ87t4xT71LThQIK7iIiIvEoGAgqhouI75Sgl9Exx3hF2cuVoOflwZtvVkq/JD5peZyIiEh8CgVCiuEi4jsl6GWUnOwl6WVO0Hv1gl/8QsvcpRCNvouIiMQnxXARqQxK0MuhXJXcExLgrLNg1izIyamUfkn80f1rIiIi8UkJuohUBiXo5RBJ0A8eLOMHs7Nh50744INK6ZfEn2AgyPZ9Wh4nIiISb0KBkGK4iPhOCXo5pKfD3r2wcWMZPzhgANSrp2Xukk+j7yIiIvEpEsOdc7HuiojUIErQy6Hcldzr1oXTT4cZM0B/mQsqMCMiIhKvgoEgeS6PPQf2xLorIlKDKEEvhw4dvOcyJ+jgLXPfsAE++cTXPkl8CgaC7MzZyUFX1vslREREJJZCKSEArYQTEV8pQS+HFi28lerlStB//Wsw0zJ3AbwZdIdj9/7dse6KiIiIlEEwEASUoIuIv5Sgl4NZOSu5Axx5JJxwgrfMXWq9SHDXMncREZH4Egp4M+iK4SLiJyXo5ZSeDqtXl/PD2dmwdCl8/bWvfZL4o9F3ERGR+KQYLiKVQQl6OaWnw7p1sH9/OT48eLD3/PrrvvZJ4o/uXxMREYlPStBFpDIoQS+nDh28fdC//LKcH05P1zJ3+XmJu/ZRFRERiSuRQXbFcBHxkxL0cir3VmsR2dkwezZs11/qtZlG30VEROKTYriIVAYl6OXUvr33XO4EffBgOHAA3nrLtz5J/IkUmFFwFxERiS8N6jQAFMNFxF9K0MupUSOvIHu5C8Udfzw0aaJl7rWcqriLiIjEp+TEZOol11MMFxFfxSxBN7NEM1tqZjPDr9uY2XwzW2NmL5lZnVj1LVrl3moNIDHR2xP93//2ZtKlVqpfpz6g0XcREZF4FAwEFcNFxFexnEG/FlhV4PW9wIPOufbAT8AlMelVGXToUIEEHbz70Ldtgw8/9K1PEl8SExJpUKeBCsyIiIjEoWAgqBl0EfFVTBJ0M2sF/Ap4JvzagF8C08KnTASGxKJvZZGeDt99BzvKO3B62mkQCGiZey2n0XcREZH4FAqEFMNFxFexmkF/CPgf4GD4dWNgm3MuN/x6A9CyqA+a2VgzW2Rmi3Jzc4s6pcpEKrmvWVPOBurXh1NPhenTwTnf+iXxJRgIsmO/gruIiEi80SC7iPityhN0M/s18L1zbnHBw0WcWmTG6px7yjmX5ZzLSkpKqpQ+RqvCW62Bt8x93TpYscKXPkn8CaWEtMRdREQkDimGi4jfYjGDfiKQbWbrgSl4S9sfAhqaWSTjbgVsikHfyqRtWzCrQCV38ArFgZa512IafRcREYlPiuEi4rcqT9Cdc7c651o559KAEcB7zrkLgNnA0PBpo4DpVd23skpJgbS0Cs6gt2gBvXt7y9ylVtL9ayIiIvEpWEcJuoj4qzrtg34zcIOZrcW7J31CjPsTlQpttRaRnQ0LFsC33/rSJ4kvqgArIiISn0Ip3iD7QXew9JNFRKIQ0wTdOfe+c+7X4Z+/dM71ds61c86d55zLiWXfohVJ0CtU4y0723ueOdOXPkl80fI4ERGR+BQMBHE4du/fHeuuiEgNUZ1m0ONSejrs3AmbN1egkc6doU0bLXOvpUKBELv27yLvYF6suyIiIiJlEAwEATTQLiK+UYJeQZFK7hUqFGfmzaK/8w7s1ghsbRMJ7jv374xxT0RERKQsQoEQgG5VExHfKEGvIF+2WgMvQc/JgbffrnCfJL5o9F1ERCQ+KYaLiN+UoFfQUUdBIOBDgn7yydCwoZa510KhlPDou/ZRFRERiSuRBF0xXET8ogS9ghISoH17HxL05GQ480yvUFye7kWuTTT6LiIiEp8ig+yK4SLiFyXoPvBlqzXwlrn/+CN8/LEPjUm8UIIuIiISnxTDRcRvStB9kJ4OX3wBubkVbGjQIG8mXcvcaxUVmBEREYlPiuEi4jcl6D5IT4cDB2D9+go2FApBv34wY4YPvZJ4odF3ERGR+FS/Tn1AMVxE/KME3QcdOnjPvi1zX726gvu2STzR/WsiIiLxKTEhkfp16iuGi4hvlKD7wLet1sBL0AGefdaHxiQepCanYpgqwIqIiMShUCCkGC4ivlGC7oPGjaFRI58S9KOOgosugvvv157otYSZEQwENfouIiISh4KBIDv2K4aLiD+UoPvAzMdK7gBPPAGdOsHIkfD11z41KtVZKCWk4C5SicxskJmtNrO1ZnZLEe8HzOyl8PvzzSytwHu3ho+vNrOBpbVpZpPCxz8zs2fNLDl8vJ+ZbTezT8KP2yv3W4tIVdAgu0jlqm0xXAm6T9LTYdUqOHjQh8ZSU+GVV2D/fjjvPMjJ8aFRqc6CgaCWx4lUEjNLBB4DzgA6ASPNrNMhp10C/OScawc8CNwb/mwnYASQAQwCHjezxFLanAQcC3QB6gKXFrjOXOdct/DjTv+/rYhUtVCKlriLVJbaGMOVoPtkwADYtAnuucenBjt0gOeegwUL4MYbfWpUqiuNvotUqt7AWufcl865/cAUYPAh5wwGJoZ/ngYMMDMLH5/inMtxzq0D1obbK7ZN59wbLgxYALSq5O8nIjGkGC5SqWpdDFeC7pOLL/ZWpP/xjzBrlk+Nnnsu3HADPPYY/POfPjUq1VEoENIeqiIVk2Rmiwo8xhZ4ryXwTYHXG8LHKOoc51wusB1oXMJnS20zvCzuIqBgVDjezJaZ2ZtmllHG7ygi1ZBiuEiFKYYXkFRZDdc2ZvD007BiBZx/PixaBMcc40PD99zjzaJfdhl07QoZ+vdcTRQMBPnipy9i3Q2ReJbrnMsq5j0r4piL8pzijhc1wH1om48Dc5xzc8OvlwBHO+d2mdmZwGtA+2L6LCJxQjPoIhWmGF6AZtB9lJoK//oXOAdnnw27d/vQaHIyvPQSNGgA55wDOxQAaiIFd5FKtQFoXeB1K2BTceeYWRIQAraW8NkS2zSzO4AjgRsix5xzO5xzu8I/vwEkm1mTinwxEYm9YCDIrv27yDuYF+uuiNREtS6GK0H3Wdu2MHkyfPopjB3rJesV1qKFl6R/8QVccolPjUp1oj1URSrVQqC9mbUxszp4BWNmHHLODGBU+OehwHvh+89mACPCFWLb4I2WLyipTTO7FBgIjHTO5ZcONbNfhO+Jw8x648XgLZXyjUWkyoQCIQB27t8Z456I1Ei1LoYrQa8EgwbBXXd5t40//LBPjZ5yCvzlLzBtmo+NSnURDATZm7uXA3kHYt0VkRonfD/a1cBbwCrgZefcCjO708yyw6dNABqb2Vq8EfNbwp9dAbwMrMS7D+0q51xecW2G23oSaAZ8dMhWLEOBz8xsGTAeGBH+B4SIxLFgIAiglXAilaA2xnCL538bpKamut2+rCP338GDXo2311+Hd96Bfv18aNQ5b5n7zJkwezacdJIPjUp1MH7+eK6ddS1b/mcLR9Q9ItbdEYk7ZrbHOZca635I9KpzDBcpi6krpjJs2jA+veJTOjftHOvuiMQdxfDCNINeSRISYOJEaNcOhg2Db74p/TOlMoPnn4e0NK/RzZt9aFSqg8jou5a5i4iIxJdQirfEXTFcRPygBL0SBYPw6quwd683m75vnw+NhkLwyiuwbRuMGAG5uT40KrGm5XEiIiLxSTFcRPykBL2SdewIL7wACxfC737nU6OZmfDEE/D++/CnP/nUqMRSpMCMgruIVEdmNsjMVpvZWjO7pYj3bzCzlWa23MzeNbOjY9FPkVhQgi4iflKCXgXOPhtuuw2eeQaeesqnRkeN8srE33MPTJ/uU6MSK/lL3HO0PE5EqhczSwQeA84AOgEjzazTIactBbKcc5nANOC+qu2lSOxEBtkVw0XED0rQq8idd8LAgXD11fDxxz41+vDD0LOnl6yvXetToxILGn0XkWqsN7DWOfelc24/MAUYXPAE59xs59ye8MuP8faUFakVFMNFxE9K0KtIYqK37VqrVt796N9950OjKSnetmsJCTB0qHezu8QlFZgRkWqsJVCw1OmG8LHiXAK8WdQbZjbWzBaZ2aJc1VCRGqJ+nfoYphguIr5Qgl6FjjjCKxr3009eEfYDfmx5nZYG//gHLFsGV17pbcUmcUej7yJSjVkRx4oMNmZ2IZAF3F/U+865p5xzWc65rKSkJB+7KBI7ZkYwEFQMFxFfKEGvYl27eveiz50LN93kU6NnnukVi3v+eZgwwadGpSrVTapLoiUquItIdbQBaF3gdStg06EnmdmpwB+AbOdcThX1TaRaCAaC7NivGC4iFacEPQbOPx+uuw7Gj/cmv31xxx1w2mneTe5LlvjUqFQVMyOUElKBGRGpjhYC7c2sjZnVAUYAMwqeYGbdgb/jJeffx6CPIjEVSglpibuI+EIJeozcdx+ccgpcdhksXepDg5Gb3Js29W5y37rVh0alKml5nIhUR865XOBq4C1gFfCyc26Fmd1pZtnh0+4H6gNTzewTM5tRTHMiNZJiuIj4RTeAxUhyMrz8MvToAeecA4sWQePGFWy0SROYOhVOPhkuvhhmzPAKyElcCAVCCu4iUi05594A3jjk2O0Ffj61yjslUo0EA0G27NkS626ISA2g7C2GmjaFf/0LNm2CkSMhL8+HRvv0gQcfhH//G/76Vx8alKoSDAS1xF1ERCQOhQK6TU1E/KEEPcZ694bHHoO334Y//tGnRq+80rvR/U9/gnfe8alRqWxaHiciIhKfFMNFxC9K0KuBSy+FsWPhnnvglVd8aNAMnnoKOnWC4cNh1iwfGpXKFkrREncREZF4pARdRPyiBL2aGD8ejjsORo+GlSt9aDA1FaZPhxYt4Iwz4MYbIUe73lRnwTpBVYAVERGJQ6FAiD0H9nAg70CsuyIicU4JejURCMC0aVCvHpx9Nmz3I09r2xYWLICrroJx4+D442H1ah8alsqg0XcREZH4FAwEAdi5f2eMeyIi8U4JejXSsqVXhP3LL+GCC3wqGle3Ljz6qDeb/vXXXtn4CRPAOR8aFz+FUkLk5OWQk6uVDiIiIvEklBIC0Eo4EakwJejVTN++8PDDXhH2P/zBx4azs2HZMq/K+6WXwogRsG2bjxeQioqMvmsWXUREJL4ohouIX5SgV0NXXgmXXw733guTJvnYcMuWXrn4v/zFq0bXtSvMm+fjBaQiFNxFRETik2K4iPhFCXo1NX48nHIKXHKJdxu5bxIT4dZbvcQ8MdGbsv9//w9yc328iJRHKBBeHqd9VEVEROKKYriI+EUJejWVnOwVjWveHIYMgU2bfL5Anz7wyScwciT87/9C//7ePeoSMxp9FxERiU+K4SLiFyXo1ViTJjBjBuzc6SXpe/f6fIFgEP7xD3jhBS9Z79rVGxWQmIgUmFFwFxERiS9K0EXEL0rQq7kuXbwcetEiuOyySiq+ftFFsHQptG8P550HY8fC7t2VcCEpSSS4qwKsiIhIfFEVdxHxixL0ODB4MNx9t1cw7r77Kuki7drBhx/CzTfDM89AVpY3qy5VRqPvIiIi8aluUl0SLVExXEQqTAl6nLj1Vm9ntFtvhZkzK+kiderAPfd4ld63b/fuU3/oIe2ZXkUiBWYU3EVEROKLmRFKCSmGi0iFKUGPE2YwYQL06AHnnw8rVlTixQYMgOXLYeBAuP56+PWv4fvvK/GCAhBIClAnsY4qwIqIiMShYCCoGC4iFaYEPY7UqwevvQapqZCdDVu2VOLFmjSB6dPhkUfg3XchMxNeegny8irxohIMBDX6LiIiEocUw0XED0rQ40yrVvDqq7BhAwwbBgcOVOLFzODqq72N2Js29dbYd+oEzz0H+/dX4oVrr1AgpNF3ERGROKQYLiJ+UIIeh447Dp5+Gt57D264oQoumJnpVXmfOtWbvh8zxisq9+ijlbD3W+2m0XcREZH4pBguIn5Qgh6nLr4YbrrJy5GfeqoKLpiYCEOHwuLF8MYbcNRR8LvfQVqaV1huhwKSHxTcRURE4pNiuIj4QQl6HLvnHjjjDLjqKpgzp4ouauZd9MMPvYt27+6Vlj/qKPjTn+DHH6uoIzVTKCWkPVRFRETiUCigGC4iFacEPY4lJsLkydC2LZx7LqxfX8UdOPlkmDULFi3yKr/ffTccfbS37n7jxiruTM2g0XcREZH4pBguIn5Qgh7nQiF4/XXIzfUqu+/aFYNO9OwJr7zi7f02dCiMHw/HHAO//S188UUMOhS/QgHtoSoiIhKPgoEgOXk55OTmxLorIhLHlKDXAO3bw8sve/nxRRfBwYMx6kinTjBxIqxZA5dc4v2cng4XXgiffRajTsWXyB6qzrlYd0VERETKIJQSAtBAu4hUiBL0GuK002DcOG+f9P/93xh3pk0bePxxWLfOW+7+2mvQpQucfTYsXBjjzlVvwUCQ3IO57MvdF+uuiIiISBkEA0FACbqIVEyVJ+hm1trMZpvZKjNbYWbXho8fYWZvm9ma8HOjqu5bvLvmGm/i+q67vBn1mGveHO6/H776Cu64Az74AHr3hlNP9TqYoyVghwoFNPouIiISjxTDRcQPsZhBzwVudM51BI4DrjKzTsAtwLvOufbAu+HXUgZm3sT1SSfB6NGwZEmsexTWuLE3rf/VV17C/vnnMHy4l8BffbW3dZuWdAM/j75vz1EVWBERkXiiGC4ifqjyBN05961zbkn4553AKqAlMBiYGD5tIjCkqvtWE9Sp49VrO/JIGDwYvvsu1j0qoEEDb/P2devg//4PBg2CCRMgKwu6dvXW6H//fax7GVNaHiciIhKfFMNFxA8xvQfdzNKA7sB8oJlz7lvwknigaex6Ft+aNoXp02HrVhgypBrmvImJ3k3z//wnfPstPPEE1K0LN94ILVt6nZ4+HQ4ciHVPq1ykwIz2URUREYkviuEi4oeYJehmVh94BbjOORf1UKOZjTWzRWa2KDc3t/I6GOe6dYNJk+CTT7z6bG++GeseFaNhQ7j8cpg/3ytDf/313s9DhnjJ+g03wKefxrqXVUaj7yIiIvFJMVxE/BCTBN3MkvGS80nOuX+FD282s+bh95sDRc77Oueecs5lOeeykpKSqqbDcWrIEFi0yJtRP/NMr4jcvupcHLxTJ7jvPvjmG29z95NPhkcfhcxMbxn8Y495ywJqMAV3ERGR+KQYLiJ+iEUVdwMmAKucc+MKvDUDGBX+eRQwvar7VhN17uztbHbNNfDII9CrVxxMSCclwa9/7d1Mv2kTPPww5OV5BeWaN/cKzL35pnesholUgFWBGRF/mdkgM1ttZmvN7LAipGYWMLOXwu/PD9+CFXnv1vDx1WY2sLQ2zWxS+PhnZvZseFC64LV6mVmemQ2tnG8rIrGQkpRCncQ6iuEiPqttMTwWM+gnAhcBvzSzT8KPM4F7gNPMbA1wWvi1+CAlxctx33gDfvjBS9LHj4+TwulNmnijC0uXeo/LL4d33/WWBBx1FFx3HcyaBXv3xrqnvtDou4j/zCwReAw4A+gEjAzvHlLQJcBPzrl2wIPAveHPdgJGABnAIOBxM0sspc1JwLFAF6AucOkhfbkXeKsSvqqIxFgwEFQMF/FRbYzhsaji/qFzzpxzmc65buHHG865Lc65Ac659uHnmr2WOQbOOAOWL/e2Ib/2Wi/H3bw51r0qg27dvJGGjRth2jTo2RP+/nfvizVqBAMHepXgV66Mk9GHwyUnJlM3qa6Cu4i/egNrnXNfOuf2A1Pwdg4pqOBOItOAAeEVX4OBKc65HOfcOmBtuL1i2wzHNOecc8ACoFWB6/wO7xav6la+U0R8oARdxHe1LobHtIq7VL2mTb3bux99FN5/3ysg9+9/x7pXZRQIwLnnwowZ3j3pb70FV14JGzZ4leAzMuDoo+Gyy7xl8tu2xbrHZRIMBFUBVqTskiIFRMOPsQXeawl8U+D1hvAxijrHOZcLbAcal/DZUtsML4u7CJgVft0SOBt4sjxfUESqv1AgpCXuImWnGF6AqqzVQmZw1VXQrx+cf753u/dVV8H993u7ncWVunXh9NO9B8DXX3sJ+1tvwcsvwzPPeNu6HXecN8M+aJA3855QfcemgoEgO/Zr9F2kjHKdc1nFvGdFHDt0mU1x5xR3vKi/RA5t83FgjnNubvj1Q8DNzrk8b2BfRGoazaCLlItieAHVN0uRSpeR4e1odv31XoH0Xr28JfBx7aijvJnzadPgxx9h7ly49VbIyYE77oDevaFZM29k4oUX4LvvYt3jw4RSQgruIv7aALQu8LoVsKm4c8wsCQgBW0v4bIltmtkdwJHADQXOyQKmmNl6YCjevXBDyvulRKT6UQwX8V2ti+FK0Gu5lBTvtu1Zs2DLFi9/ffhhOHgw1j3zQXIynHQS3HWXV8p+82Zvc/gzzoD33oNRo7yq8N27w803w/Tp1SJh1xJ3Ed8tBNqbWRszq4NXMGbGIecU3ElkKPBe+P6zGcCIcIXYNkB7vHvSim3TzC4FBgIjnXP5f5s659o459Kcc2l498hd6Zx7rXK+sojEgmK4iO9qXQxXgi6At/p7+XJvpfh113kF5KpBruqvI4/8eeZ80yZYsgT++lcIhbxRiiFDvIT96KNh2DB44AFvBn7PnirtppbHifgrfD/a1XhVV1cBLzvnVpjZnWaWHT5tAtDYzNbijZjfEv7sCuBlYCXefWhXOefyimsz3NaTQDPgo/BOJbdXyRcVkZgL1lEMF/FTbYzh5uK02jVAamqq2717d6y7UaM4B08+CTfcAPXrw3PPefeo13h793oJ+4IF3rr/+fNh/XrvvcREr5penz7eEoM+faBjx0q7j330a6OZvX42X133VaW0L1ITmdke51xqrPsh0VMMl5rotndv4/7/3M/+P+5HtSZEoqMYXpiKxEkhZnDFFV4BuZEj4ayzvALp998P9erFuneVqG5dOPFE7xGxefPPCfuCBTBliretG0CDBt5N+wWT9ubNfemKZtBFRETiUzAQJPdgLvty91E3Od4q74pIdaAEXYrUsaOXl/7hD/C3v8Hs2XD77ZCVBW3beol8jdesmTdCcdZZ3uuDB+Hzz3+eYZ8/3xu5yM313m/d2kvUTz7Zu0egXbtyXTaSoDvnNPouIiISR4KBIAA7cnYoQReRclGCLsUKBLzbsAf5lWnyAAAVY0lEQVQOhNGjvRl1gIYNvZ3Kevb0EvaePaFNm1qQtCckwLHHeo9R4ToUe/fC0qWFk/Zp0+Daa6F9ey9RP/NM6NvXq8gXhVAgxEF3kN0HdlO/Tv1K/EIiIiLip1AgBMD2nO00q98sxr0RkXikBF1Kddpp3u3Yn30GixfDokXe48EH4cAB75xGjQon7FlZXq21Gp+0160LJ5zgPSK++ALefBPeeMNbEv/ww979AQMGeMn6GWd4v5xiFBx9V4IuIiISPwrGcBGR8lCCLlFJTvZ2I+veHS691DuWk+Ml7YsW/Zy4P/DAzyu+Gzc+PGlv3boWJO1t28LVV3uPPXvg/fe9ZP2NN+D1171zOnX6eXb9xBOhTp38j4dSvNH3HTk7aNGgRQy+gIiIiJRHwRguIlIeStCl3AKBn5e6R+zbB59+Wjhpv/deyMvz3m/SxEvy27aFtDRvaXzkuUmTGpi816v3cyLunHcPeyRZf/hhb0SjQQNvmUJ4dj0y+q59VEVEROKLYriIVJQSdPFVSopX3LxXr5+P7d3r7bEeSdiXLfN+3rq18GdTU71k/dDEPfLcsGGcJ/Bm0KGD97j+eti1C9599+eE/V//AiD4y7bQF3Ys+hDqd/C+uIiIiFR7WuIuIhWlBF0qXd26XnHzPn0KH9++Hb76Ctat8+5xX7/+55/nzoUdh8S2YPDwxL1TJ2/pfKNGVfJV/FW/Pgwe7D2cgxUr4I03CH3wMgA7brsJVt0ERx7pFZxLT/cekZ/btavhe9+JiIjEl0iROCXoIlJeStAlZkIhyMz0HodyDrZtKzp5X7sW3nkHdu/++fy2bX+eue/Vy1tGXz+e6quZQefO0LkzwbHD4eE0tv/+d/Bja1izxlsa/9Zb8PzzhT/XqlXhpD3yc5s2he5rFxERkcrXINAA8Kq4i4iUhxJ0qZbMvFnxRo2gR4/D33cOfvzRWy6/aBEsXAjz5sGUKd77CQneXu69enkz7L16Qdeu3n3z1V3+8riOx8Bx1xV+c+dOb4QikrRHnqdOLXzPQGKit8QgkrS3bfvzsoO0tDgbvRAREYkPdRLrkJKUohl0ESk3JegSl8y8ld+nnuo9IjZv/jlhX7TIu7U7MumcnOzN1kcS9l69vCXySdXs/4ISC8w0aPBzOf1DbdniJeyRpD2SwM+ZU3i5AXgV+QreL1Dw56OPjnrPdhERESksFAipSJyIlFs1S01EKqZZM/jVr7wHeDPt33zzc9K+cKE3y/73v3vv163r5bpZWZCRAcce6z2OPDJ2BekSExJJTU4t++h748be47jjCh93Dr7/vvB9ApHnTz6B6dNh//7Cn2nevPiKfa1aKYEXEfn/7d19jBx3ecDx77O3d3vx2XfxxY7zSkhClCZUJQSU8hIQKjQNUUUgoiW8RoCEkEBqKlUiEhWNUP+AAhV9IeUtiASiAiVNsSIopLSiqkRC3pwX4tDYITR2TBLbyV1s5+zb21//mFnfem/3Xuy7nVnf9yP9NLMzv917bm68j5+Z38xIXYzWRpk85Bl0SUfHAl3HtQh4yUuydtVV2bJGA7Zvnz3LfvfdcOONR55kHh/PCvULLpgt2i+4IKtPBwZWPu7R2ujyDY+LyI5cbNo09059kG2QXbvmFu+//jXceSd873uzz8lrGh+H008/sp1xxpGvTzqpz2+7L0nS0i1rDpe06liga9WpVLL7qJ13HrznPdmyRgN27IBHH4WtW2ent9+eFe9NtVp2SXd78X7++ct7Q/Wx4bHe3WCmUpktqi+9dO76eh127pwt3HfsyF432/33Z2foUzryfbUanHba3EK+2U47DU45xTvRS5KOK2PDDnGXdPQs0CWyGrV5pv2yy45ct3cv/OpXRxbu998Pt96aFfZNZ52VFe1nnpndoX4xrdv176U6+l6tZr/cWWd17zM9nZ2Fby3cW9u998LmzfDii3PfOzqaFert7dRTj3y9cWNvhi9IknQMRmujbN+/vegwJPUpC3RpAePj8NrXZq3V1FR2Q/XWwn3r1uyy7uefz9YvZM2azoX7znPHeLI6yXXXZQcPBgay1ml+ofUDAzAycmStu+yXkA8Ozh7h6Kb57Lxm0f7UU9ld/Xbtgt/+NmtbtmTTyQ4HJyqVrEjvVsRv2jQ7Pzbm8HpJUiHGamPlOcguqe9YoEtHaXj48KPLOzp0CCYmltaefx5+8xvYPTzKobGdfPFr2eXfMzNzR5Afi7GxuSeoO7UNG5bxpHXrs/O6bbSmAwey4r1ZuLcW8c32yCPZdHp67vtrtblFe6dC3iH2kqRlNlob9Tnoko6aBbq0QoaGshO+Gzcu/b0f+sEodzw+yZMtZ+FTyobUz8zMThcz32hkJ6Rb693Wdu+92fSFF+bGUanAyScfWduuXz97pv/EE7sP3z+mZ86vWTP7+Lf5pJRdg9D6y7X/ok88AXfd1fk6ecgeXdcs2jdtyo5KzNdGRjw7L0nqqnmZWkqJMF9IWiILdKmEOg2Pi5gdsr4S9u/vXMC3tocfzs7y79u38OcND88t2lsL+nXrsiK+Vsv6tk4XvywYaD5e7sIL5w+oXofduzsX8c1ljzySPU9+z565d65vqtUWLuKbMY2PZ9O1ay3qJWmVGKuN0UgNDkwfYGRopOhwJPUZC3SphEZro7xw8AUaqUElKj35mSMjcO65WVvIzEx2Vr51aH6n4frty3bunF1+4MDyxF2tZnVwc7TChg2z80e2Khs3nsKG3z2F6kULfGijkQW5e/fC7f77s+nevQsH2SzYm/MLvV63zsJekvrMaG0UgImDExbokpbMAl0qodHaKInEvkP7Dif6MhkYmL2c/Gg1Gtl1+lNTcPBg1przi1128GB2Y/i9e+HZZ7P20EPZdL56ef36uQX8hg1ZTZydoa9Qq61neHg9tdp52bKzofY785zRH6hTfeE5Ys/u2YJ9z55s2mzN108+CQ88kL3ev3/+Dd0s2tvPys+3bNnvAphdHTA9PXvJ/5o1HjuQpE6aeXvy4CSnrTut4Ggk9RsLdKmExobHAJiYmihlgb4cKpWsjlyBWhLIRrTv2TNbuLe23btn57dtg5//PFvWbVT74lSJ2Mjw8EZqteyk+bx32R+BgVEYqDSoNGYYSHUG0jSVmToDjUNUZqYZmDlEpX6I2DsNTx8i8go5pg8efsZfkICDBDuBndnrygAMDRKDg0dMZ6o1pis1pmMoawwynQaZTgNMN6pMNyrU63G4EG9t7dumUsmekNftPgSdlrcvs8jvHxFxOfB3wADw9ZTSZ9rW14CbgVcBe4B3pZSe6HWcUhm05nBJWioLdKmEWo++6+hUq9k93zZtWlz/RiO7tr55Zn4xZ+/nW1+vz71ZX+eb+VWYmakwMzNIo3HCEeunZw7X4Yfvb3f4PneNBml6Gup10vRMPq3DTP3IZVMzsL9Oqs8wUJ9kkEMMMs1IVp4fblXq2XwVBmvBYG2AwXUDDJ5QpXrCIINr8rY2GzIwOTPCRH2EiUPDTEwNM/HiIP+3Z5CJfRUmJoOJiTgc+3x/ow98AG688aj+xOqRiBgAvgT8IbADuDsiNqeUHmnp9mHguZTSyyLiauCzwLt6H61UPHO4pGNhgS6VUDO533D3DZw5dmbB0QiAwbyt7by6mrdSX23YHKc+9WJ2bcCLU0ubnzpIej77qHV5O6P52UN5Ww9UKqShIQ4NrWNqcC1T1XVMVUeYqowwVVnDVGWYKU5gKtVYv+EC4NrebwstxSXAtpTS4wAR8R3gSqC1QL8SuD6f/z7wjxERKS3nAyLnuvZa2LJlJX+CtHT7RkbhEnjf529mZP+9RYcjrZjzN53Fjz737qLDOO5YoEsldN74eQxXh7nhnhuKDkWrzXDejkkDmMrbs/P2PP2Fs7FAL73TgSdbXu8Afr9bn5RSPSImgJOA3a2dIuIjwEcAhoaGVipeqVC1g2dSnT6RZ075dtGhSCtqevoPAAv05WaBLpXQuePnMnndJDPpmC6KlsqtXqcys8A4eJVBpzsFtJ8ZX0wfUkpfBb4KMDIycsxn17/4xWP9BGklrKfeeJZ6o150INKK6tWThlYbC3SppAYHBhlksOgwpJVjBuoXO4DWa23OAJ7q0mdHRFSBMWCeZylIx7dqpUq14pecpKXzsIckSZrP3cB5EXF2RAwBVwOb2/psBq7J598J/OdKX38uSdLxyEN7kiSpq/ya8o8DPyZ7zNo3Ukq/jIhPA/eklDYDNwLfiohtZGfOry4uYkmS+lf08wHukZGRtH///qLDkCQVLCIOpJRKfRN9HckcLkkCc3g7h7hLkiRJklQCFuiSJEmSJJWABbokSZIkSSVggS5JkiRJUglYoEuSJEmSVAIW6JIkSZIklYAFuiRJkiRJJWCBLkmSJElSCVigS5IkSZJUAhbokiRJkiSVgAW6JEmSJEklECmlomM4ahHRAF4sOo4lqgL1ooM4Sv0ae7/GDf0bu3H3Xr/Gvlxxn5BS8qBzH+nTHA7+W+u1fo0b+jd24+69fo3dHL4C+rpA70cRcU9K6dVFx3E0+jX2fo0b+jd24+69fo29X+PW6tWv+6xx916/xm7cvdevsfdr3GXnkQpJkiRJkkrAAl2SJEmSpBKwQO+9rxYdwDHo19j7NW7o39iNu/f6NfZ+jVurV7/us8bde/0au3H3Xr/G3q9xl5rXoEuSJEmSVAKeQZckSZIkqQQs0CVJkiRJKgEL9BUQEWdGxH9FxNaI+GVE/FmHPm+KiImI2JK3TxURaycR8UREPJTHdU+H9RERfx8R2yLiwYi4uIg422I6v2VbbomIyYi4tq1PabZ5RHwjIp6JiIdblo1HxB0R8Vg+Xd/lvdfkfR6LiGt6F3XXuD8XEY/m+8JtEXFil/fOu1+tpC5xXx8RO1v2hyu6vPfyiPhVvr9f17uou8b93ZaYn4iILV3eW9j2zn9+x+/BftjPtbqZw3vPHN4b5vDe5vD85/dlHjeHFyylZFvmBpwKXJzPrwP+F7iwrc+bgNuLjrVL/E8AG+ZZfwXwIyCA1wB3FR1zW3wDwG+Bs8q6zYE3AhcDD7cs+xvgunz+OuCzHd43DjyeT9fn8+sLjvsyoJrPf7ZT3IvZrwqI+3rgLxaxL20HzgGGgAfa/y33Ou629V8APlW27Z3//I7fg/2wn9tWdzOHFx6/Oby3cZvDexx72/pS5nFzeLHNM+grIKW0K6V0Xz7/ArAVOL3YqJbVlcDNKXMncGJEnFp0UC3eDGxPKf2m6EC6SSn9N7C3bfGVwE35/E3A2zu89Y+AO1JKe1NKzwF3AJevWKBtOsWdUvpJSqmev7wTOKNX8SxWl+29GJcA21JKj6eUDgHfIfs79cR8cUdEAH8K/HOv4lmKeb4HS7+fa3UzhxfOHL5CzOG9zeHQv3ncHF4sC/QVFhEvBV4J3NVh9Wsj4oGI+FFEvLyngc0vAT+JiHsj4iMd1p8OPNnyegfl+s/L1XT/sivrNgfYlFLaBdkXI3Byhz5l3/YfIjsz08lC+1URPp4P6/tGl2FaZd7ebwCeTik91mV9abZ32/fg8bCfa5UwhxfCHF4cc3hv9UUeN4f3ngX6CoqItcCtwLUppcm21feRDd96BfAPwL/1Or55vD6ldDHwVuBjEfHGtvXR4T2leF5fRAwBbwP+pcPqMm/zxSrztv8kUAdu6dJlof2q1/4JOBe4CNhFNsysXWm3N/Bu5j/qXortvcD3YNe3dVhWlu2uVcIc3nvm8OKYwwtR+jxuDi+GBfoKiYhBsh36lpTSv7avTylNppT25fM/BAYjYkOPw+wopfRUPn0GuI1siFCrHcCZLa/PAJ7qTXQLeitwX0rp6fYVZd7muaebwwzz6TMd+pRy2+c3APlj4L0ppY5fwovYr3oqpfR0SmkmpdQAvtYlnrJu7ypwFfDdbn3KsL27fA/27X6u1cMcXhhzeAHM4b3XD3ncHF4cC/QVkF9TciOwNaX0t136nJL3IyIuIftb7OldlJ1FxEhErGvOk9085OG2bpuBD0TmNcBEc7hLCXQ9GlnWbd5iM9C80+U1wA869PkxcFlErM+Hc12WLytMRFwOfAJ4W0rpQJc+i9mveqrtmst30Dmeu4HzIuLs/MzO1WR/p6K9BXg0pbSj08oybO95vgf7cj/X6mEOL5Q5vMfM4YUpdR43hxcsFXR3uuO5AZeSDeV4ENiStyuAjwIfzft8HPgl2R0l7wReV3TceVzn5DE9kMf3yXx5a+wBfInszpgPAa8uOu48rjVkyXqsZVkptznZf0B2AdNkRxo/DJwE/BR4LJ+O531fDXy95b0fArbl7YMliHsb2bVGzX39y3nf04AfzrdfFRz3t/L990GyhHNqe9z56yvI7l66vQxx58u/2dyvW/qWZnvnMXT7Hiz9fm5b3W2efbeU+aQtdnN4b2I1hxcfd+lzeLfY8+XfpMR5fJ7vwdLv58dDi3wjSpIkSZKkAjnEXZIkSZKkErBAlyRJkiSpBCzQJUmSJEkqAQt0SZIkSZJKwAJdkiRJkqQSsECXjlMR8aaIuL3oOCRJ0tKYw6XVywJdkiRJkqQSsECXChYR74uIX0TEloj4SkQMRMS+iPhCRNwXET+NiI1534si4s6IeDAibouI9fnyl0XEf0TEA/l7zs0/fm1EfD8iHo2IWyIi8v6fiYhH8s/5fEG/uiRJfc0cLmm5WaBLBYqIC4B3Aa9PKV0EzADvBUaA+1JKFwM/A/4qf8vNwCdSSr8HPNSy/BbgSymlVwCvA3bly18JXAtcCJwDvD4ixoF3AC/PP+evV/a3lCTp+GMOl7QSLNClYr0ZeBVwd0RsyV+fAzSA7+Z9vg1cGhFjwIkppZ/ly28C3hgR64DTU0q3AaSUplJKB/I+v0gp7UgpNYAtwEuBSWAK+HpEXAU0+0qSpMUzh0tadhboUrECuCmldFHezk8pXd+hX1rgM7o52DI/A1RTSnXgEuBW4O3Avy8xZkmSZA6XtAIs0KVi/RR4Z0ScDBAR4xFxFtm/zXfmfd4D/E9KaQJ4LiLekC9/P/CzlNIksCMi3p5/Ri0i1nT7gRGxFhhLKf2QbOjcRSvxi0mSdJwzh0tadtWiA5BWs5TSIxHxl8BPIqICTAMfA/YDL4+Ie4EJsmvcAK4Bvpwn78eBD+bL3w98JSI+nX/Gn8zzY9cBP4iIYbIj93++zL+WJEnHPXO4pJUQKc036kZSESJiX0ppbdFxSJKkpTGHSzoWDnGXJEmSJKkEPIMuSZIkSVIJeAZdkiRJkqQSsECXJEmSJKkELNAlSZIkSSoBC3RJkiRJkkrAAl2SJEmSpBL4f+p1lHuTtg/mAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bc2ef048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(H)\n",
    "print(logger)\n",
    "print(stopping)\n",
    "print(checkpoint)\n",
    "plot_learning_curves(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "crnn.load_state_dict(stopping.best_score_state)\n",
    "torch.save(crnn.state_dict(), H.MODEL_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "import Levenshtein as lev\n",
    "\n",
    "class Scorer(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    @staticmethod\n",
    "    def wer(s1, s2):\n",
    "        b = set(s1.split() + s2.split())\n",
    "        word2char = dict(zip(b, range(len(b))))\n",
    "\n",
    "        w1 = [chr(word2char[w]) for w in s1.split()]\n",
    "        w2 = [chr(word2char[w]) for w in s2.split()]\n",
    "        return lev.distance(''.join(w1), ''.join(w2)) \n",
    "\n",
    "    @staticmethod\n",
    "    def cer(s1, s2):\n",
    "        s1, s2, = s1.replace(' ', ''), s2.replace(' ', '')\n",
    "        return lev.distance(s1, s2) \n",
    "    \n",
    "    def score(self,predicted_vars, target_vars):\n",
    "        idx, acc, cer = 0, 0.0, 0.0\n",
    "        for idx, (dseq, tseq) in enumerate(zip(predicted_vars, target_vars)):\n",
    "#           acc = acc + 1 if dseq == tseq else acc\n",
    "#             acc += fuzz.ratio(tseq, dseq)\n",
    "#             acc += SequenceMatcher(None, dseq, tseq).ratio()\n",
    "#             acc += sentence_bleu([tseq], dseq, emulate_multibleu=True)\n",
    "            acc += self.wer(dseq, tseq) / float(len(tseq.split()))\n",
    "            cer += self.cer(dseq, tseq) / float(len(tseq))\n",
    "\n",
    "        return (acc / (idx + 1)), cer / (idx + 1)\n",
    "    \n",
    "scorer = Scorer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "crnn_pred = CRNN(num_classes=len(vocab), hidden_size=H.HIDDEN_SIZE, num_layers=H.NUM_LAYERS, rnn_dropout=H.RNN_DROPOUT,\n",
    "                 initialize=torch_weight_init)\n",
    "if H.USE_CUDA:\n",
    "    crnn_pred.cuda()\n",
    "\n",
    "state = torch.load(H.MODEL_PATH)\n",
    "crnn_pred.load_state_dict(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Results:\n",
      "     Loss:           16.3392\n",
      "     Accuracy:       0.2523\n",
      "     CER:            0.0703\n",
      "CPU times: user 24.3 s, sys: 538 ms, total: 24.9 s\n",
      "Wall time: 25 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "decoder = CTCGreedyDecoder(vocab)\n",
    "\n",
    "test_evaluator = Evaluator(crnn_pred, test_loader, criterion, decoder, scorer)\n",
    "\n",
    "avg_loss, avg_score, avg_cer = test_evaluator.evaluate()\n",
    "\n",
    "print('Test Results:')\n",
    "print('     Loss:           {:.4f}'.format(avg_loss))\n",
    "print('     Accuracy:       {:.4f}'.format(avg_score))\n",
    "print('     CER:            {:.4f}'.format(avg_cer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2335"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crnn_pred.eval()\n",
    "with torch.no_grad():\n",
    "    idx_batch = 0\n",
    "    bad_lines = []\n",
    "    for idx_batch,(input_vars, target_vars, input_size, target_size) in enumerate(test_loader):\n",
    "        input_vars = Variable(input_vars)\n",
    "        if H.USE_CUDA:\n",
    "            input_vars = input_vars.cuda()\n",
    "\n",
    "        preds = crnn_pred(input_vars)\n",
    "        preds = preds.transpose(1,0)\n",
    "        preds_size = torch.IntTensor(preds.size(1)).fill_(preds.size(0))\n",
    "\n",
    "        decoder_seq, target_seq = decoder(preds, target_vars, preds_size, target_size)\n",
    "\n",
    "        for idx, (dseq, tseq) in enumerate(zip(decoder_seq, target_seq)):\n",
    "            if dseq != tseq:\n",
    "                bad_lines.append((input_vars[idx], dseq, tseq))\n",
    "                    \n",
    "len(bad_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92 5\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA34AAAGGCAYAAAAkd1NJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xm8JGV18PHfgQGFGWTVAXEZVFyIqHFG1IivM6JmxChq5I1LFIhKjEY0wSiuqK/EJe5rgguDio6KCwgYJchITEQFBAZBlCCyCio4OEpU8Lx/1HOlp7m3u+feXm499/f9fOrT3VVPnzpdt6r6nnqqqiMzkSRJkiTVa4tJJyBJkiRJGi0LP0mSJEmqnIWfJEmSJFXOwk+SJEmSKmfhJ0mSJEmVs/CTJEmSpMpZ+EmSZi0iLouIx8zifRsj4h6jyGk+iIh1EfG8Iccc2jKLiFdFxEfK82URkRGxaEix71Zy3XIY8SRJw2HhJ0kLSCnUbir/mN8QESdHxF1HNK81EfGm6aZl5pLMvHQE8xx6wTXAPF8fEZ+cw/tXRsQfyt9kY0RcGRGfjYiHdLYbZJmVWFf2m2dm/nNmDmU5dRf/mXl5yfWWYcSXJA2HhZ8kLTxPzMwlwG7AtcD7JpyP4OryN9kOeBjwA+A/I2K/Yc9oWD17kqR2sfCTpAUqM/8XOB7Ya2pcRDwhIr4XETdGxBUR8frO90TEsyPiJxHxi4h49WznXU4tvFd5viYiPlB6H38VEd+OiHt2tL1vRJwaEddHxMUR8X9nOc+HRcR/R8QvI+K8iFjZMW1dRPy/iPivksPXImKXjunP6fjcr53q5YqI1cCrgL8qvXXndczy7jPFm0k2rszM1wEfAd46wzLbPyIuLLGvioiXRcRi4CvAnTt6D+9ceiSPj4hPRsSNwMEz9FL+TURcHRHXRMThHfPdpOe2s1cxIj4B3A34cpnfy7tPHS05nFj+fpdExPM7Yr2+9G5+vHyW70fEin7LSZK0+Sz8JGmBiohtgb8CzuwY/WvgOcAOwBOAv4uIJ5f2ewEfAp4N3BnYGbjLkNJ5BvAGYEfgEuCoMs/FwKnAp4A7lXYfjIg/2ZzgEbE7cDLwJmAn4GXA5yPijh3NngkcUuazdWkz9bk/CDyLppd0e2B3gMz8d+Cfgc+U0xsf2C/eZvgC8OCyDLp9FPjbzNwOuD/w9cz8NfB4Su9hGa4u7Q+gKfJ3AI6bYX6rgD2BxwFHxADXbmbms4HLKb3Imfm2aZp9GriSZp15GvDPXT2ZTwLWltxOBN7fb76SpM1n4SdJC8+XIuKXwI3AY4F/mZqQmesyc31m/iEzz6f5p/1RZfLTgJMy84zM/C3wWuAPQ8rpC5n5ncy8maYweVAZ/xfAZZl5TGbenJnnAJ8vuWyOvwZOycxTymc7FTgL2L+jzTGZ+cPMvAn4bEcOTwO+nJnfzMzfAa8DcoB5zhRvUFcDQVMQdfs9sFdE3CEzbyjLpZdvZeaXyme/aYY2b8jMX2fmeuAYmiJ7Tsr1o/sCr8jM/83Mc2l6Mp/d0eyb5e9yC/AJ4IHThJIkzZGFnyQtPE/OzB2A2wF/D3wjInYFiIiHRsTpEfGziNgAvACYOkXxzsAVU0FKD9MvhpTTTzue/wZYUp7fHXhoOT3zl6VgfRaw62bGvztwYFecfWl68Prl0P25f8Ngn3umeIPanabA/OU00/6Spmj9SUR8IyIe3ifWFX2md7f5Cc3nnqs7A9dn5q+6Yu/e8bp7Od3e6xAlafgs/CRpgcrMWzLzC8AtNEUQNKdUngjcNTO3B/6VptcJ4Brgj3cALaeK7jziNK8AvpGZO3QMSzLz72YR5xNdcRZn5lsGeO81dJzSGhHbsOnnHqT3bzaeApxTCuxNZOZ3M/MAmtNIv0TTo9grl0Fy7Ly7691oehyhOf13245p3UV3r9hXAztFxHZdsa8aIB9J0hBZ+EnSAhWNA2iuq7uojN6OpofmfyNiH5rr1KYcD/xFROwbEVsDb6T/98iWEXH7jmHrzUzzJODe0dxUZqsyPCQi7tfjPYu65rkV8EngiRHx5xExldPKiBjkGsXjy3v/rOT/Bm4thqG5M+qyiJjzd2r5m+weEUcCz6O5cUx3m60j4lkRsX1m/p7mlN2pn064Ftg5IrafxexfGxHblusnDwE+U8afC+wfETuVnuGXdr3vWmDa3xfMzCuA/wbeXJb5A4DnMvN1hpKkEbHwk6SF58sRsZGmYDgKOCgzv1+mvRB4Y0T8iuZatqmeJEqbF9H0Cl4D3EBz045ejgBu6hi+vjmJllMEHwc8nab36Kc0d7q8XY+3fahrnseUAuQAmkLqZzQ9gP/EAN+D5XO/mOYGJNcAvwKuA35bmnyuPP4iIvpdazeTO5e/yUbgu8DewMrM/NoM7Z8NXFbu0vkCmmsYycwf0FyXeWk5pXVzTtf8Bs2NdU4D3t4x708A5wGXAV/j1oJwypuB15T5TXcDm2cAy2j+fl8EjizXWEqSxigyR3WGiiRJ9YmIJTTX3e2ZmT+edD6SJA3CHj9JkvqIiCeW0yAXA28H1tP0gEmS1AoWfpIk9XcAzamKV9P81t3T01NmJEkt4qmekiRJklQ5e/wkSZIkqXIWfpIkSZJUuUWTTmAudtlll1y2bNmk07iNX//61yxevHhObdoSoy151hSjLXnWFKMtedYUoy151hSjLXnWFKMtedYUoy151hSjLXkOK8YknH322T/PzDv2bZiZrR2WL1+e89Hpp58+5zZtidGWPGuK0ZY8a4rRljxritGWPGuK0ZY8a4rRljxritGWPGuK0ZY8hxVjEoCzcoDayVM9JUmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOwk+SJEmSKmfhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOwk+SJEmSKrdo0glIktRt/VUbOPiIk3u2WbN68ZiykSSp/ezxkyRJkqTKWfhJkiRJUuUs/CRJkiSpchZ+kiRJklQ5Cz9JkiRJqpyFnyRJkiRVzsJPkiRJkipn4SdJkiRJlbPwkyRJkqTKWfhJkiRJUuUs/CRJkiSpchZ+kiRJklQ5Cz9JkiRJqpyFnyRJkiRVbmSFX0R8LCKui4gLppn2sojIiNilvI6IeG9EXBIR50fEg0eVlyRJkiQtNKPs8VsDrO4eGRF3BR4LXN4x+vHAnmU4FPjQCPOSJEmSpAVlZIVfZp4BXD/NpHcBLweyY9wBwMezcSawQ0TsNqrcJEmSJGkhGes1fhHxJOCqzDyva9LuwBUdr68s4yRJkiRJcxSZ2b/VbINHLANOysz7R8S2wOnA4zJzQ0RcBqzIzJ9HxMnAmzPzm+V9pwEvz8yzp4l5KM3poCxdunT52rVrR5b/bG3cuJElS5bMqU1bYrQlz5pitCXPmmK0Jc+aYlx3/QauvannLNhj+y0nnmdNMdqSZ00x2pJnTTHakmdNMdqS57BiTMKqVavOzswVfRtm5sgGYBlwQXm+N3AdcFkZbqa5zm9X4N+AZ3S872Jgt37xly9fnvPR6aefPuc2bYnRljxritGWPGuK0ZY8a4rx3k9+Ke/+ipN6DvMhz5pitCXPmmK0Jc+aYrQlz5pitCXPYcWYBOCsHKA2G9upnpm5PjPvlJnLMnMZzemcD87MnwInAs8pd/d8GLAhM68ZV26SJEmSVLNR/pzDp4FvAfeJiCsj4rk9mp8CXApcAnwYeOGo8pIkSZKkhWbRqAJn5jP6TF/W8TyBF40qF0mSJElayMZ6V09JkiRJ0vhZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOwk+SJEmSKmfhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOwk+SJEmSKmfhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOwk+SJEmSKmfhJ0mSJEmVWzTpBCRJmo31V23g4CNOnnH6mtWLx5iNJEnzmz1+kiRJklQ5Cz9JkiRJqpyFnyRJkiRVzsJPkiRJkipn4SdJkiRJlbPwkyRJkqTKWfhJkiRJUuUs/CRJkiSpciMr/CLiYxFxXURc0DHuXyLiBxFxfkR8MSJ26Jj2yoi4JCIujog/H1VekiRJkrTQjLLHbw2wumvcqcD9M/MBwA+BVwJExF7A04E/Ke/5YERsOcLcJEmSJGnBGFnhl5lnANd3jftaZt5cXp4J3KU8PwBYm5m/zcwfA5cA+4wqN0mSJElaSCZ5jd/fAF8pz3cHruiYdmUZJ0mSJEmao8jM0QWPWAaclJn37xr/amAF8NTMzIj4APCtzPxkmf5R4JTM/Pw0MQ8FDgVYunTp8rVr144s/9nauHEjS5YsmVObtsRoS541xWhLnjXFaEueNcW47voNXHtTz1mwdBt6ttlj+y1b8VnnS4y25FlTjLbkWVOMtuRZU4y25DmsGJOwatWqszNzRd+GmTmyAVgGXNA17iDgW8C2HeNeCbyy4/VXgYf3i798+fKcj04//fQ5t2lLjLbkWVOMtuRZU4y25FlTjPd+8kt591ec1HPo16Ytn3W+xGhLnjXFaEueNcVoS541xWhLnsOKMQnAWTlAbTbWUz0jYjXwCuBJmfmbjkknAk+PiNtFxB7AnsB3xpmbJEmSJNVq0agCR8SngZXALhFxJXAkTc/e7YBTIwLgzMx8QWZ+PyI+C1wI3Ay8KDNvGVVukiRJkrSQjKzwy8xnTDP6oz3aHwUcNap8JEmSJGmhmuRdPSVJkiRJY2DhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyI7urpyRJk7T+qg0cfMTJPdusWb14TNlIkjRZ9vhJkiRJUuUs/CRJkiSpchZ+kiRJklQ5r/GTJI1dv+vvDt97jMlIkrQA2OMnSZIkSZWz8JMkSZKkyln4SZIkSVLlLPwkSZIkqXIWfpIkSZJUOQs/SZIkSaqchZ8kSZIkVc7CT5IkSZIqZ+EnSZIkSZWz8JMkSZKkyi2adAKSpHZZf9UGDj7i5J5t1qxePKZsJEnSIOzxkyRJkqTKWfhJkiRJUuUs/CRJkiSpcl7jJ0lasPpdr+i1ipKkWtjjJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5bzGT5I0dP2unTt87zEmI0mS7PGTJEmSpNpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVLmRFX4R8bGIuC4iLugYt1NEnBoRPyqPO5bxERHvjYhLIuL8iHjwqPKSJEmSpIVmlD1+a4DVXeOOAE7LzD2B08prgMcDe5bhUOBDI8xLkiRJkhaUkRV+mXkGcH3X6AOAY8vzY4End4z/eDbOBHaIiN1GlZskSZIkLSTjvsZvaWZeA1Ae71TG7w5c0dHuyjJOkiRJkjRHkZmjCx6xDDgpM+9fXv8yM3fomH5DZu4YEScDb87Mb5bxpwEvz8yzp4l5KM3poCxdunT52rVrR5b/bG3cuJElS5bMqU1bYrQlz5pitCXPmmK0Jc9xxbju+g1ce1PPECzdhp5t5jp9XDH22H7LebHMXc/rjNGWPGuK0ZY8a4rRljyHFWMSVq1adXZmrujbMDNHNgDLgAs6Xl8M7Fae7wZcXJ7/G/CM6dr1GpYvX57z0emnnz7nNm2J0ZY8a4rRljxritGWPMcV472f/FLe/RUn9Rz6tZnr9HHFmC/L3PW8zhhtybOmGG3Js6YYbclzWDEmATgrB6jNxn2q54nAQeX5QcAJHeOfU+7u+TBgQ5ZTQiVJkiRJc7NoVIEj4tPASmCXiLgSOBJ4C/DZiHgucDlwYGl+CrA/cAnwG+CQUeUlSZIkSQvNyAq/zHzGDJP2m6ZtAi8aVS6SJEmStJCN+1RPSZIkSdKYWfhJkiRJUuUs/CRJkiSpchZ+kiRJklQ5Cz9JkiRJqpyFnyRJkiRVzsJPkiRJkio3st/xkyRpIVh/1QYOPuLkGaevWb14jNlIkjQ9e/wkSZIkqXL2+EmSNIN+vXkAh+89pmQkSZoDe/wkSZIkqXIWfpIkSZJUOQs/SZIkSaqchZ8kSZIkVc7CT5IkSZIqZ+EnSZIkSZUbqPCLiPuPOhFJkiRJ0mgM2uP3rxHxnYh4YUTsMNKMJEmSJElDNVDhl5n7As8C7gqcFRGfiojHjjQzSZIkSdJQDHyNX2b+CHgN8ArgUcB7I+IHEfHUUSUnSZIkSZq7Qa/xe0BEvAu4CHg08MTMvF95/q4R5idJkiRJmqNFA7Z7P/Bh4FWZedPUyMy8OiJeM5LMJEmSJElDMWjhtz9wU2beAhARWwC3z8zfZOYnRpadJEmSJGnOBr3G7z+AbTpeb1vGSZIkSZLmuUELv9tn5sapF+X5tqNJSZIkSZI0TIMWfr+OiAdPvYiI5cBNPdpLkiRJkuaJQa/xeynwuYi4urzeDfir0aQkSZIkSRqmgQq/zPxuRNwXuA8QwA8y8/cjzUySJEmSNBSD9vgBPARYVt7zpxFBZn58JFlJkiRJkoZmoMIvIj4B3BM4F7iljE7Awk+SJEmS5rlBe/xWAHtlZo4yGUmSJEnS8A16V88LgF1HmYgkSZIkaTQG7fHbBbgwIr4D/HZqZGY+aSRZSZIkSZKGZtDC7/XDnGlE/APwPJrrBNcDh9D8RMRaYCfgHODZmfm7Yc5XkiRJkhaigU71zMxvAJcBW5Xn36UpzjZbROwOHAasyMz7A1sCTwfeCrwrM/cEbgCeO5v4kiRJkqRNDVT4RcTzgeOBfyujdge+NIf5LgK2iYhFwLbANcCjyzwAjgWePIf4kiRJkqQiBrlRZ0ScC+wDfDsz/7SMW5+Ze89qphEvAY4CbgK+BrwEODMz71Wm3xX4SukR7H7vocChAEuXLl2+du3a2aQwUhs3bmTJkiVzatOWGG3Js6YYbcmzphhtyXNcMa67fgPX3tQzBEu3oWebuU6vLcYe22858b/9fFm/FlKMtuRZU4y25FlTjLbkOawYk7Bq1aqzM3NFv3aDXuP328z8XUQAUHrqZvXTDhGxI3AAsAfwS+BzwOOnaTpt/Mw8GjgaYMWKFbly5crZpDFS69ato19e/dq0JUZb8qwpRlvyrClGW/IcV4z3HXcC71jf++vj8L1v7tlmrtNri7Fm9eKJ/+3ny/q1kGK0Jc+aYrQlz5pitCXPYcWYzwb9OYdvRMSraE7PfCxNsfblWc7zMcCPM/Nnmfl74AvAnwE7lIIS4C7A1bOML0mSJEnqMGjhdwTwM5o7cP4tcArwmlnO83LgYRGxbTRdiPsBFwKnA08rbQ4CTphlfEmSJElSh4FO9czMPwAfLsOcZOa3I+J4mruC3gx8j+bUzZOBtRHxpjLuo3OdlyRJkiRpwMIvIn7MNNfcZeY9ZjPTzDwSOLJr9KU0N5CRJEmSJA3RoDd36bxLzO2BA2l+aF2SJEmSNM8NeqrnL7pGvTsivgm8bvgpSZImaf1VGzj4iJNnnH74rH7IR5IkTdKgp3o+uOPlFjQ9gNuNJCNJkiRJ0lANeqrnOzqe3wxcBvzfoWcjSZIkSRq6QU/1XDXqRCRJkiRJozHoqZ7/2Gt6Zr5zOOlIkiRJkoZtc+7q+RDgxPL6icAZwBWjSEqSJEmSNDyDFn67AA/OzF8BRMTrgc9l5vNGlZgkSRpcv7uxrlm9eIzZSJLmmy0GbHc34Hcdr38HLBt6NpIkSZKkoRu0x+8TwHci4otAAk8BPj6yrCRJkiRJQzPoXT2PioivAI8sow7JzO+NLi1JkiRJ0rAM2uMHsC1wY2YeExF3jIg9MvPHo0pMkiQ1+l2/B3D43mNKRpLUSgNd4xcRRwKvAF5ZRm0FfHJUSUmSJEmShmfQm7s8BXgS8GuAzLwa2G5USUmSJEmShmfQwu93mZk0N3YhIrwntCRJkiS1xKCF32cj4t+AHSLi+cB/AB8eXVqSJEmSpGEZ9K6eb4+IxwI3AvcBXpeZp440M0mSJEnSUPQt/CJiS+CrmfkYwGJPkiRJklqm76memXkL8JuI2H4M+UiSJEmShmzQ3/H7X2B9RJxKubMnQGYeNpKsJEmSJElDM2jhd3IZJEmSJEkt07Pwi4i7ZeblmXnsuBKSJEmSJA1Xv2v8vjT1JCI+P+JcJEmSJEkj0K/wi47n9xhlIpIkSZKk0ehX+OUMzyVJkiRJLdHv5i4PjIgbaXr+tinPKa8zM+8w0uwkSZIkSXPWs/DLzC3HlYgkafTWX7WBg4/ofZPmw/ceUzL6o35/F/8mkqS56vsD7pIkSZKkdrPwkyRJkqTKWfhJkiRJUuUs/CRJkiSpchZ+kiRJklS5iRR+EbFDRBwfET+IiIsi4uERsVNEnBoRPyqPO04iN0mSJEmqzaR6/N4D/Htm3hd4IHARcARwWmbuCZxWXkuSJEmS5qjfD7gPXUTcAfg/wMEAmfk74HcRcQCwsjQ7FlgHvGLc+UlSm/l7cJIkaTqT6PG7B/Az4JiI+F5EfCQiFgNLM/MagPJ4pwnkJkmSJEnVicwc7wwjVgBnAo/IzG9HxHuAG4EXZ+YOHe1uyMzbXOcXEYcChwIsXbp0+dq1a8eU+eA2btzIkiVL5tSmLTHakmdNMdqSZ00x2pInwHXXb+Dam2aevnQbek4fpM04YrQlzzbF2GP7LefFOmqM9uVZU4y25FlTjLbkOawYk7Bq1aqzM3NFv3aTKPx2Bc7MzGXl9SNprue7F7AyM6+JiN2AdZl5n16xVqxYkWedddaoU95s69atY+XKlXNq05YYbcmzphhtybOmGG3JE+B9x53AO9bPfBb/4Xvf3HP6IG3GEaMtebYpxprVi+fFOmqM9uVZU4y25FlTjLbkOawYkxARAxV+Yz/VMzN/ClwREVNF3X7AhcCJwEFl3EHACePOTZIkSZJqNPabuxQvBo6LiK2BS4FDaIrQz0bEc4HLgQMnlJskSZIkVWUihV9mngtM1x2537hzkSRpIeh3x9c1qxePMRtJ0rhN6nf8JEmSJEljMqlTPSVJUmX69SqCPYuSNCn2+EmSJElS5ezxkyRJ9tZJUuXs8ZMkSZKkyln4SZIkSVLlLPwkSZIkqXIWfpIkSZJUOQs/SZIkSaqchZ8kSZIkVc7CT5IkSZIqZ+EnSZIkSZXzB9wlqSUG+YHtw/ceUzJakPqtg65/kjR/2eMnSZIkSZWz8JMkSZKkyln4SZIkSVLlLPwkSZIkqXIWfpIkSZJUOQs/SZIkSaqchZ8kSZIkVc7CT5IkSZIqZ+EnSZIkSZWz8JMkSZKkyln4SZIkSVLlLPwkSZIkqXIWfpIkSZJUuUWTTkCS1Fh/1QYOPuLkGacfvvcYk5EkSVWxx0+SJEmSKmfhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVLmJFX4RsWVEfC8iTiqv94iIb0fEjyLiMxGx9aRykyRJkqSaTLLH7yXARR2v3wq8KzP3BG4AnjuRrCRJkiSpMhP5AfeIuAvwBOAo4B8jIoBHA88sTY4FXg98aBL5SZKkyVh/1QYOPuLknm3WrF48pmwkqR6T6vF7N/By4A/l9c7ALzPz5vL6SmD3SSQmSZIkSbWJzBzvDCP+Atg/M18YESuBlwGHAN/KzHuVNncFTsnMvad5/6HAoQBLly5dvnbt2rHlPqiNGzeyZMmSObVpS4y25FlTjLbkWVOMceV53fUbuPammacv3Yae0wdp05YYbcmzphjjynOP7bfsuS302w4GieG+xRhtz7OmGG3Jc1gxJmHVqlVnZ+aKfu0mcarnI4AnRcT+wO2BO9D0AO4QEYtKr99dgKune3NmHg0cDbBixYpcuXLlWJLeHOvWraNfXv3atCVGW/KsKUZb8qwpxrjyfN9xJ/CO9TPvlg/f++ae0wdp05YYbcmzphjjynPN6sU9t4V+28EgMdy3GKPtedYUoy15DivGfDb2Uz0z85WZeZfMXAY8Hfh6Zj4LOB14Wml2EHDCuHOTJEmSpBrNp9/xewXNjV4uobnm76MTzkeSJEmSqjCRu3pOycx1wLry/FJgn0nmI0mSJEk1mk89fpIkSZKkEbDwkyRJkqTKWfhJkiRJUuUmeo2fJC0U66/awMFHnNyzzeG3+eVSSZKk4bDHT5IkSZIqZ+EnSZIkSZWz8JMkSZKkyln4SZIkSVLlLPwkSZIkqXLe1VOShqDfXTu9Y6fUcFuRpMmwx0+SJEmSKmfhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOH3CXtOD1+0HpNasXjzEbSZKk4bPHT5IkSZIqZ+EnSZIkSZWz8JMkSZKkynmNnyRJahWvy5WkzWePnyRJkiRVzsJPkiRJkipn4SdJkiRJlbPwkyRJkqTKWfhJkiRJUuW8q6ck9dHvDoIAh+89pmQkSZJmwR4/SZIkSaqchZ8kSZIkVc7CT5IkSZIq5zV+kiSpKoNdl3tzzzZrVi8edlqSNFFj7/GLiLtGxOkRcVFEfD8iXlLG7xQRp0bEj8rjjuPOTZIkSZJqNIlTPW8GDs/M+wEPA14UEXsBRwCnZeaewGnltSRJkiRpjsZe+GXmNZl5Tnn+K+AiYHfgAODY0uxY4Mnjzk2SJEmSahSZObmZRywDzgDuD1yemTt0TLshM29zumdEHAocCrB06dLla9euHU+ym2Hjxo0sWbJkTm3aEqMtedYUoy15tinGdddv4NqbZn7/0m3oOX2QNsaYf/MwRjvzHFeMPbbf0v15hTHakmdNMdqS57BiTMKqVavOzswV/dpN7OYuEbEE+Dzw0sy8MSIGel9mHg0cDbBixYpcuXLlyHKcrXXr1tEvr35t2hKjLXnWFKMtebYpxvuOO4F3rJ95d3j43jf3nD5IG2PMv3kYo515jivGmtWL3Z9XGKMtedYUoy15DivGfDaRn3OIiK1oir7jMvMLZfS1EbFbmb4bcN0kcpMkSZKk2kzirp4BfBS4KDPf2THpROCg8vwg4IRx5yZJkiRJNZrEqZ7XVG8zAAAUs0lEQVSPAJ4NrI+Ic8u4VwFvAT4bEc8FLgcOnEBukiRJklSdsRd+mflNYKYL+vYbZy6SJEnTGeRH4P2Rd0ltMpFr/CRJkiRJ42PhJ0mSJEmVs/CTJEmSpMpZ+EmSJElS5Sz8JEmSJKlyFn6SJEmSVDkLP0mSJEmq3CR+wF2Sxqrf73EdvvcYk5FUjX77ln6/8+dvBUoaJ3v8JEmSJKlyFn6SJEmSVDkLP0mSJEmqnIWfJEmSJFXOwk+SJEmSKuddPSW12iB3xfOunZIkaaGzx0+SJEmSKmfhJ0mSJEmVs/CTJEmSpMp5jZ9UoX7Xva1Zvbg1MSRpIXM/KmlY7PGTJEmSpMrZ4ydpVrybpiT11m8/OYx95CD7YnsFJYE9fpIkSZJUPXv8NHI1XZ/Q/+jtzR55HbJxHDGXJEmqnT1+kiRJklQ5Cz9JkiRJqpyneqoVajpdVLfyBjGS1A5+D0vtZ4+fJEmSJFXOHj9N3ELr9ZnrDWI8qipJ2hz21kkCe/wkSZIkqXr2+KmnhXSUcKH1PEqSNCzj+iF5/y+5VU2fVeNhj58kSZIkVc4ev4oN1oPV+3qyfj1c86WXbFxHGiVJkqQ2ssdPkiRJkio373r8ImI18B5gS+AjmfmWCackVWe+9NRKkhaOud7Vumkz7Kwmw+/h+an26yrnVY9fRGwJfAB4PLAX8IyI2GuyWUmSJElSu823Hr99gEsy81KAiFgLHABcONGsVIX+RxrHmMwceJRQkqTpjaPHZhz3UFhoau9pmy/mVY8fsDtwRcfrK8s4SZIkSdIsRWZOOoc/iogDgT/PzOeV188G9snMF3e0ORQ4tLy8D3Dx2BPtbxfg53Ns05YYbcmzphhtybOmGG3Js6YYbcmzphhtybOmGG3Js6YYbcmzphhtyXNYMSbh7pl5x76tMnPeDMDDga92vH4l8MpJ5zWLz3HWXNu0JUZb8qwpRlvyrClGW/KsKUZb8qwpRlvyrClGW/KsKUZb8qwpRlvyHFaM+TzMt1M9vwvsGRF7RMTWwNOBEyeckyRJkiS12ry6uUtm3hwRfw98lebnHD6Wmd+fcFqSJEmS1GrzqvADyMxTgFMmncccHT2ENm2J0ZY8a4rRljxritGWPGuK0ZY8a4rRljxritGWPGuK0ZY8a4rRljyHFWPemlc3d5EkSZIkDd98u8ZPkiRJkjRkFn6SJEmSVDkLP0mSJEmqnIXfGETEfSNiv4hY0jV+dcfzfSLiIeX5XhHxjxGxf4+YH+8zz31LjMeV1w+NiDuU59tExBsi4ssR8daI2L6MPywi7toj5tYR8ZyIeEx5/cyIeH9EvCgitupod8+IeFlEvCci3hERL5iahxaOiLjTEGLsPIxcJI2H2/3wuUyHy+WpBW3SPyRY8wAcAhwGXAx8CbgMOKBj+jnl8UjgTOAs4M3A14HXAWcAr6b5LcPO4cvAxqnXJcZ3OuI+Hzi3xP0v4Ajg+8CiMv1o4N3AvqXNF8r4DcDVwH8CLwTu2PV5jgM+U+b/CeCLwLOBNcCxpc1hwKnAa4D/Bj4IHAVcCKyc9N+kz9/rTkOIsfME8t4eeAvwA+AXZbiojNuhz3u/Uh7vUNa9TwDP7GrzQWBX4EPAB4CdgdcD64HPArsBO3UNO5f1fUdgpxJndVfOHwXOBz4FLC357lKmrwAuBS4BfgI8amqbKevWPWf4PCuA04FPAnct6+IGmt8I/dPSZgnwxrJNbAB+RrP9HVymLwL+Fvj3kt95wFeAFwBbDfD3OJrm52j+Fvh/wCO6pr+mPG4LvBz4J+D2wME02/TbgCUzxP5hx/MHdDzfqiyXE4F/BrYt4/++Y5nei2af8kvg28DewBeAv55pfuV99wA+BrypLLsPAxcAnwOW0RxA/Bvg5LKszgbWUrb3tizPQZZpv+VZxvdcpv2WZ2nTb5nOepuf2u7ps82Xxyq2e/ps8/NlP1ra9Vym/ZZnGd9zmc51eQ64H3UdHf4ynfW+FDi6PPbclzKL/Wj3vpQxfDcxT/aj/drMx2HiCdQ8AJeXHdCS8noZTXH3kvL6e+VxfdkYtwVuBO5Qxm9TNu5zys5iJc1OeyVwTXn+qM5Y5fl3KUUbsLjEv6hj+jldeZ47FaNsKI+j2fH9rOxgDgK2A84v7RYB1wJbltfRMW19x/htgXXl+d1KfL8MNs13GF8GXwVeAezaEXfXMu5U4MEzDMuBa0r7z5fP82SaHfTngdt1fIZ/B15McxDh/BL7bmXcCcAfgB93Db8vj5d2r3fAR2h22HcH/oHmwMj6jumnAw8pz+8NnFWe/xh4O8229Z3y3jt3vO87wOOBZwBXAE8r4/cDvlWen0DzZXYX4B+B1wJ7AsfSfDF9mmbdeVhpc5fy/EPAZ0qM7vWncz26sny+TwEvpfmyeWf39kezLr6D5h/C04D3A/8H+BeadftXNPuDG8vzXwG3dIzvXJ7voDkA8yjgXcDHy/jvd7Q5GXhKeb6S5qDQVcDxwPUln6cAW3eto2cAf1f+9hcAh9Osq8+lOUh1DM32tS/NAaU3Ao8F/oNm/WjF8pxmHb3NMu23PMvznsu03/Isbfot057bfHndc7unzzZfHqvY7umzzc+X/Whp13OZ9lueU9/DvZbpXJfngPtR19ExfzfRZz/a8flm3JfSZz9a2k38u4l5sh+d6X/V+TxMPIG2DzQ7mumG9cBvgQu72i+h2VG9k46Cq2P697ran0tTjP0DzZfPg8r4S7vanUdTtOxM2RF1xqQ5CnJIeX0MsKI8vzfw3fK8uyDcCngSzc7mZ2Xj2rrM51fcWiDdnlJYls89tWPeETi7I94Fw9jQ8Mug+8vg4h7r58U0O+Svlxy7h5um1rOu972aZue7M82XQec6evk06+jLyjLfu2P8j7vadS7P7vmdS3MwYKpX+syu6eunifFImi+nn5bPcmifPKcOtJzXNX5q/d+i5NBref6wPN5C809Y5/oz9fp3lAMhpe0iml6rLwC368hjavuP8hmi4/X5wPtoio2l0y3Trs96LuWIL5seiLm4+3N27bumctmOpvf+FJpt/RjgcdPM5zbLtPOzdv7tyme9qC3Lc5Bl2m95dq1n0y7TfsuzM9Ysl+nFHct0xu2ePtv8AH/71mz39Nnmu7eV6ZZpv+U5w+fbrP1oeey5TPstz/LYc5nOdXmWx7nsR11Hh79Mf0if/Whp13NfSp/9aHk+8e+mAZfnyPejM71/Pg8TT6DtA03P14NoCoLOYRnNaZNfpxRrHe9ZVDaaW8rrb3Nr9/cWHe2279qZ3IWmgHv/NCv6ZR0b+KWUwoqm0Dy3xFoD/E+Z3+9Lu28ADyxtv9fjc25DU7RcStOjdRjN0aAP03yRHFnavaRstEfT7Kimis070hyh8ctg+F8GX6M5NaNzJ7yUptj9D5qCe88ZlvkV5fGiznWvjDuIppfxJ505AG+a4bNOrZ/vpNlZdx+cuJKmcD28rEfRMe18mqL8a8CjaY7SvZvmSOMbuPVI4znTfIYtgdU0XwjfovlSOLDk/eTS5lHcWoz/N7Bvef5E4Kud6yBNb+qBbLotbgH8FfDt8vpHwN1mWqaUfya7xh9Js57+qHudAj7W1fa88ricZls4rORwaUebS4GnAn9JR49+1/uPotnu7wG8iuYo791oTkM/aYbluRPNqUNTR03PpjmYsQ/wc249aHSv8nc7m9KjTXOA5oyOWBe2ZXl2LNOnzLRM+y3PHuvoH5dpx/J8yHTLs2OZ91qmPbf58rrndk+fbb5738Ntt/upXOf9dk+fbb48Tno/2vkP+YzLtN/yLI89lynTfNdvzvIsz/vtRxfaOjqOZdpzX0qf/Wh57LkvZYD9aHk+0e8mZv5e2pMx7kenGz/fh4kn0PaB5vS/fWeY9qmyw9l1humPKI+3m2H6LnQUJh3jn0A5PWWA/LYF9uh4vR3wwLLRLu1qe+8B4t2Z0mMF7AA8Ddinq82flPH3neb98+HLoLZCZUfgrTRF4A00p0ZcVMbtVP4W95lheU7N723AY6aZvprmy+CNTHOuPc0/q8d3jXsizRfUT7vGH9k1TJ2OvCu3nv6xkuaUle/RHFA4haZAnjpiuLbP+vlAml7lrwD3Bd5Dc93A94E/62jznTL+m1PLhubgxGE0B20+A1xHcxT1h+X5ZyjbEvAiygGTaXJ4Mc2pu6unmfY84Pfl+UdmWKb3BL7Z8XqLktd/Ald3jD+ma1jasTxP62h3MM0/BT+n6am/kKaneHs6vgh7LNP9ynp2Ec0pM58v68R1wAE06//lZTn9GHhox/J8W8fy/FlpM/XeebU8y7Q1/ZYpzT8m0y7PMr3nMu2zPKe2x6ll+qOyTB/WtUx7bvOlbc/tnj7bfHnelu3+Qdx2u7+BZrt/BLfd5u/duc2X5/NqPzrTMh1kefZbpgMsz+n2o39cnqXNA3ot037Ls4Xr6KqZlueA6+gg301Ty3TDDMt0GT2+m+izHy2PPfelDLgfLeN6fTd9jBF+N9Hne6m06ffdNLWOXkSzfm72Otorx/k6TDwBh4U1sOmXQfeGtmNp06Yvg5VM/2Uw1RM4zC+DXv+03Bd4TPdnpuzgy/T9Zprep83jB4nROZ2mh/j+mzGPYeQ5FeN+A8S4X5/l9VCaI4k703ypvAzYv6vtPtx6eu9eNAcJ9h90eo82T+DW02s6pz+S5qZPnfN46GbO409oDmRsbp4P7Yrxsq4YD+8Xo0zbmeaA1icH2Fd8fC7TO9vQcdCmY9puwC+GkMcnhhDjJG57MCso1xD3i1HWjcMpp+fO0Gbf8neZtk2/6QPGeCTNNc1zjTHXz9IzxnTvL+v4VPG+Lc13yEk0303bl+md19+/keZGZ2/teF9njNu0mSbGG2aIcYeOPN5G0+PYGaM7z34xNuuzzPA5TppmHp15vKErxmHAXfus9z3bjCIGHd9Nk8xjljG2pjnI/Viafelf05xF9CKagv52wHMo/x8Bz6Q5Q+xF3Fqg9mqz9YAxtu5q82yag+Mv7MjjoM2M8SyaezIM9Fk6lkXn+z84zTwOAQ7skce9aG5k816aToAXUNbrjuV+T5rvvPfQXLN4mzZtGqb+uZAmLiIOycxj5tJmtjEiYhuaUwIumGQes4lB01v5IpoC+kE0Nw86oUw/h6YXY8bpmfngiHgxzV22ZopxTJ/pw5jHsGK8kObAwkwxDuvVhuaaysfTnJJ9Kk1h9A2aQvGrmXlURBzZ1eahwLqpNmX8jNNnGWOTPPpNn2Ee/WIMkudm5VFidns0zek6ZOaTIuLErulBc4T969O89zbTZxljkDw2aTPLGKPIY9fM3AcgIp5Hs918iebsgS9n5lsi4jsdbZ5f2nxxqg3w1F7TZxnjhX3yeB7N9jtTjGF9lk3ymGYem+RQ5vF9mt6SmyPiaODXNL0I+9EcoLtP1/Tf0Nx8Yr8y/qnTxNikzSxjDCOPzYoxzfRN3j/gZ92vvO9/aM5++lxm/pwOEbGho82nS5ufDTp9ljE+O8c8xvVZPkVzkLozxnE0+9ltaHoFF9Osx/vR7GcWlWFbmgPFS2iu39uP5uDXQR0xpmsDTVE14/TMPHiaGJuTxzBidE6fdlkM+FnPAf6C5jKk/Wku4bmB5nT/F2bmuvL/Qs82tM2kK08Hh6mBrmvcZtNmIcagz51j+00vj3OKMY55jDnGjHfZ7YjR6068I4/RojwHuitxrzb9pg8YYxh5zJfP0vMuzp3r+0xtFlKMAefR887X/abXFGNI8+h5l/CO9bzXncSNsRl3Vu83vTyvIsaQ5tHzLvTZ8f3Xq03bhkVIYxQR5880ieZav75tjHGbNhszcyNAZl4WESuB4yPi7qXNln2mM0CbuU4fxjzGFePmzLwF+E1E/E9m3lja3hQRfygx+rXJMcQYxzyGEWMFzU2fXg38U2aeGxE3ZeY3uNXyXm0iouf0AWMMI4/58lm2iIgdaf5JjCy9Apn564i4ucTo12YhxRhkHp1ne5wXESsy86yIuDfNtU8/6jO9phjDmMeWmfkHmuvgvxYRW3HrHazfTnOpQvZpc4UxNmlzbURsTXPAYluaU2qvpzktcivglj7TodkWaogxjHncTFMU3lLGbUfzR7i8LPspg7Rpj5wH1afDwhnocxfUQdoYY/PuHNtvenk9pxjjmMcYY/S9y26/NuOI0ZY8O17PeFfiQdsY44/jL6PHXZwHabOQYgw4j553vu43vaYYQ5pHz7uEl8d+dxI3xqYxet5Zvd/0EqeKGEOaR8+70Jfnfdu0bZh4Ag4La6DPXVAHaWOM28ToeefYftPL45xijGMeY4zR9y67/dqMI0Zb8pxmfN+7EvdrY4wZ221yF+fZtFlIMaabTo87Xw8yvaYYc5kHg90lvGcbY0zbpued1ftNrynGkOYx413oN6dNmwZv7iJJkiRJldti0glIkiRJkkbLwk+SJEmSKmfhJ0lasCJi44jjHxwRd+54fVlE7DLKeUqSNB0LP0mSRudgmhsMSJI0Uf6OnyRJHSLijsC/0vxIL8BLM/O/IuL1Zdw9yuO7M/O95T2vBZ4FXAH8HDib5mcEVgDHRcRNwMNLvBdHxBNpfkvqwMz8wTg+lyRpYbPHT5KkTb0HeFdmPgT4S+AjHdPuC/w5sA9wZERsVX7Y/S+BPwWeSlPskZnHA2cBz8rMB2XmTSXGzzPzwcCHgJeN4wNJkmSPnyRJm3oMsFdETL2+Q0RsV56fnJm/BX4bEdcBS4F9gROmCruI+HKf+F8oj2fTFIqSJI2chZ8kSZvaAnh4Rw8dAKUQ/G3HqFtovkeDzTMVY+r9kiSNnKd6SpK0qa8Bfz/1IiIe1Kf9N4EnRsTtI2IJ8ISOab8Ctpv+bZIkjY9HGiVJC9m2EXFlx+t3AocBH4iI82m+J88AXjBTgMz8bkScCJwH/ITmur4NZfIa4F+7bu4iSdLYRWZOOgdJklotIpZk5saI2JamUDw0M8+ZdF6SJE2xx0+SpLk7OiL2Am4PHGvRJ0mab+zxkyRJkqTKeXMXSZIkSaqchZ8kSZIkVc7CT5IkSZIqZ+EnSZIkSZWz8JMkSZKkyln4SZIkSVLl/j/HBhWqOF8VZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1bca8a940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.mlab as mlab\n",
    "\n",
    "line_len = sorted([len(l[1]) for l in bad_lines], reverse=True)\n",
    "print(max(line_len), min(line_len))\n",
    "\n",
    "if max(line_len):\n",
    "    fig = plt.figure(figsize=(15, 6))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.set_xticks(range(1, max(line_len)))\n",
    "\n",
    "    ax = plt.hist(line_len, bins=range(1, max(line_len)))\n",
    "\n",
    "    plt.xlabel('Length')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title(\"Bad Line Length Distribution\")\n",
    "    plt.xticks(rotation=90)\n",
    "    plt.grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fb32afd61874afe8a1d83b3f474cbea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>interactive</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "interactive(children=(IntSlider(value=1167, description='idx', max=2335), Output()), _dom_classes=('widget-interact',))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from ipywidgets import interact, interactive, fixed, interact_manual\n",
    "import ipywidgets as widgets\n",
    "\n",
    "@interact(idx=(0, len(bad_lines)))\n",
    "def show(idx):\n",
    "    img = bad_lines[idx][0]\n",
    "    dseq = bad_lines[idx][1]\n",
    "    tseq = bad_lines[idx][2]\n",
    "    visualize_data(img, tseq + \"==\" + dseq + \"=\" + str(tseq==dseq), vocab, figsize=(18,8))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}