06-自然语言处理/30-Transformer模型及源代码(PyTorch).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "原始链接：http://nlp.seas.harvard.edu/2018/04/03/attention.html   \n",
    "参考链接：http://jalammar.github.io/illustrated-transformer/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-05T02:13:03.921352Z",
     "start_time": "2020-05-05T02:13:03.503310Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import math, copy, time\n",
    "from torch.autograd import Variable\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "seaborn.set_context(context='talk')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型架构\n",
    "通用的 **编码器解码器** 架构：\n",
    "       \n",
    "<img src=\"../images/Transformer.jpg\" width=\"50%\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.813950Z",
     "start_time": "2020-05-04T03:29:55.810154Z"
    }
   },
   "outputs": [],
   "source": [
    "class EncoderDecoder(nn.Module):\n",
    "    def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):\n",
    "        \"\"\"\n",
    "        编码器、解码器、输入嵌入层、目标嵌入层、输出层\n",
    "        \"\"\"\n",
    "        super(EncoderDecoder, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.src_embed = src_embed\n",
    "        self.tgt_embed = tgt_embed\n",
    "        self.generator = generator\n",
    "\n",
    "    def forward(self, src, tgt, src_mask, tgt_mask):\n",
    "        \"\"\"\n",
    "        src --> memory\n",
    "        memory + tgt --> output\n",
    "        \"\"\"\n",
    "        memory = self.encode(src, src_mask)\n",
    "        return self.decode(memory, src_mask, tgt, tgt_mask)\n",
    "\n",
    "    def encode(self, src, src_mask):\n",
    "        \"\"\"\n",
    "        src --> memory\n",
    "        \"\"\"\n",
    "        return self.encoder(self.src_embed(src), src_mask)\n",
    "\n",
    "    def decode(self, memory, src_mask, tgt, tgt_mask):\n",
    "        \"\"\"\n",
    "        memory + tgt --> output\n",
    "        \"\"\"\n",
    "        return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.817552Z",
     "start_time": "2020-05-04T03:29:55.815030Z"
    }
   },
   "outputs": [],
   "source": [
    "class Generator(nn.Module):\n",
    "    \"Define standard linear + softmax generation step.\"\n",
    "\n",
    "    def __init__(self, d_model, vocab):\n",
    "        super(Generator, self).__init__()\n",
    "        self.proj = nn.Linear(d_model, vocab)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.log_softmax(self.proj(x), dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 编码器\n",
    "编码器由多层 `N=6` 完全相同的层堆叠而成\n",
    "<img src=\"../images/Transformer-encoder.png\" width=\"60%\">\n",
    "其层次结构如上图中所示：\n",
    "```\n",
    "1. Encoder\n",
    "    2. EncoderLayer\n",
    "        3. SublayerConnection\n",
    "            4. sublayer --> self_attn\n",
    "            \n",
    "        3. SublayerConnection\n",
    "            4. sublayer --> feed_forward\n",
    "            \n",
    "    2. EncoderLayer\n",
    "    .\n",
    "    .\n",
    "    .\n",
    "    .\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.820463Z",
     "start_time": "2020-05-04T03:29:55.818510Z"
    }
   },
   "outputs": [],
   "source": [
    "def clone(module, N):\n",
    "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.824245Z",
     "start_time": "2020-05-04T03:29:55.821286Z"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, layer, N):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.layers = clone(layer, N)\n",
    "        self.norm = LayerNorm(layer.size)\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        \"\"\"\n",
    "        需要自主生成 mask \n",
    "        \"\"\"\n",
    "        for layer in self.layers:\n",
    "            x = layer(x, mask)\n",
    "        return self.norm(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.832685Z",
     "start_time": "2020-05-04T03:29:55.829070Z"
    }
   },
   "outputs": [],
   "source": [
    "class LayerNorm(nn.Module):\n",
    "    \"\"\"\n",
    "    inputs: batch, seq_len, features\n",
    "    沿输入数据的特征维度归一化\n",
    "    \"\"\"\n",
    "    def __init__(self, features, eps=1e-6):\n",
    "        # 需要指定特征数量 features\n",
    "        super(LayerNorm, self).__init__()\n",
    "        self.a_2 = nn.Parameter(torch.ones(features))\n",
    "        self.b_2 = nn.Parameter(torch.ones(features))\n",
    "        self.eps = eps\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        x --> (x - x.mean) / x.std \n",
    "        \"\"\"\n",
    "        mean = x.mean(-1, keepdim=True)\n",
    "        std = x.std(-1, keepdim=True)\n",
    "        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.837426Z",
     "start_time": "2020-05-04T03:29:55.833561Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before Norm: \n",
      " [[[1. 2. 3.]\n",
      "  [2. 4. 5.]]]\n",
      "After Norm: \n",
      " [[[ 9.99999000e-07  1.00000000e+00  1.99999900e+00]\n",
      "  [-9.10887369e-02  1.21821775e+00  1.87287099e+00]]]\n"
     ]
    }
   ],
   "source": [
    "def test_layernorm():\n",
    "    x = np.array([[[1, 2, 3], [2, 4, 5]],],\n",
    "                 dtype=np.float)\n",
    "    print(\"Before Norm: \\n\", x)\n",
    "    x = torch.from_numpy(x)  # batch, seq_len, features\n",
    "    norm = LayerNorm(x.shape[-1])\n",
    "    x = norm(x)\n",
    "    print(\"After Norm: \\n\", x.detach().numpy())\n",
    "\n",
    "\n",
    "test_layernorm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.841658Z",
     "start_time": "2020-05-04T03:29:55.838867Z"
    }
   },
   "outputs": [],
   "source": [
    "class SublayerConnection(nn.Module):\n",
    "    def __init__(self, size, dropout):\n",
    "        super(SublayerConnection, self).__init__()\n",
    "        self.norm = LayerNorm(size)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        \n",
    "    def forward(self, x, sublayer):\n",
    "        \"\"\"\n",
    "        指定内部的结构 sublayer，是 attention 层，还是 feed_forward 层\n",
    "        \"\"\"\n",
    "        return x + self.dropout(sublayer(self.norm(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.845956Z",
     "start_time": "2020-05-04T03:29:55.842657Z"
    }
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    \"\"\"size: d_model\"\"\"\n",
    "    def __init__(self, size, self_attn, feed_forward, dropout):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        self.self_attn = self_attn\n",
    "        self.feed_forward = feed_forward\n",
    "        self.sublayer = clone(SublayerConnection(size, dropout), 2)\n",
    "        self.size = size\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n",
    "        return self.sublayer[1](x, self.feed_forward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 解码器\n",
    "<img src=\"../images/Transformer-decoder.png\" width=\"60%\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.849691Z",
     "start_time": "2020-05-04T03:29:55.846713Z"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, layer, N):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.layers = clone(layer, N)\n",
    "        self.norm = LayerNorm(layer.size)\n",
    "\n",
    "    def forward(self, x, memory, src_mask, tgt_mask):\n",
    "        for layer in self.layers:\n",
    "            x = layer(x, memory, src_mask, tgt_mask)\n",
    "        return self.norm(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.854449Z",
     "start_time": "2020-05-04T03:29:55.850519Z"
    }
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        self.size = size  # 作为参数用于 layernorm 层\n",
    "        self.self_attn = self_attn\n",
    "        self.src_attn = src_attn\n",
    "        self.feed_forward = feed_forward\n",
    "        self.sublayer = clone(SublayerConnection(size, dropout), 3)\n",
    "\n",
    "    def forward(self, x, memory, src_mask, tgt_mask):\n",
    "        m = memory\n",
    "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n",
    "        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n",
    "        return self.sublayer[2](x, self.feed_forward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.934256Z",
     "start_time": "2020-05-04T03:29:55.855299Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f83a0e5ec90>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAE5CAYAAAAQtqIuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAXRElEQVR4nO3de5CldX3n8fcHlq14Cc3NES3+IOAqKKQ2S8hivDBeMK5RHAUrjosCWqsR3ex6NyFZ8UbMoiumkniPgxRVa0kcQYQAMUwQdFJaDkEujivoKAJBZ+ghXIQZ57t/nDNJczjdffrXz+kzM/1+VZ16un/P83vO95w685nfczm/TlUhSVqYvSZdgCTtjgxPSWpgeEpSA8NTkhrs0eGZZFOSTZOuQ9LuZ778yJ58tT3JDiDA1knXImm3MwVUVQ0dZC6L8Jzad+ED7Pvu2bv7giTtNrazDeYIz3/X9RMmeSxwNvAKYD/gRuB9VXXxCH0PBz4CPIfeKYWvA2+vqpsay7lnat+9prZsPGzBHX/nif+x8Skl7QnW1UVsZ9s9s60fxznPtcB/Bf4Y+F3gJmBtkhfN1SnJCnpheShwKrAaOAD4hySHjKFOSWrW6cizH5DPB15eVWv7bVcBh9EbUV46R/e3A/sDv1lVt/f7fhP4IXAm8MYua5Wkxeh65PkyehdnLtrZUL2TqucBRyR56jx9r9wZnP2+m4GvAC/vuE5JWpSuw/Mo4Kaq2jHQfv2M9Y+Q5FHA4cANQ1ZfD6zoH9YP9pue60Hvapkkda7r8DwQ2DKkfcuM9cPsT++Wopa+krTkOr/aDsx179N890UtqG9V7TfXzhx9ShqXrkeemxk+Qjygvxw2sgS4m144tvSVpCXXdXjeCByZZHC/R/eXw85pUlUPALcy/Jzo0cDPququzqqUpEXqOjzX0rsx/iUD7a8BNs5zs/ta4IQkB+9sSHJAf19f6rhOSVqUrsPzUuAq4LNJXpvkOUnWAM8E3rFzoyTrkgyew/wwvducLk3y0iS/C3wV2E7vG0uStMvoNDz793SuAv4vvcC7DPh1ejfNf2Wevv8MPAv4CXA+8AVgGnh2Vf24yzolabH29IlBplu/297C78NLe47+d9u3znZXzx49n6ckjYvhKUkNDE9JamB4SlIDw1OSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUoNx/BmOZevy269r6ueEItLux5GnJDUwPCWpgeEpSQ0MT0lqYHhKUgPDU5IaGJ6S1MDwlKQGnYZnkuclWZNkY5L7k9yW5EtJjh6h71lJasjjzi5rlKQudP0No98HDgQ+CtwMPB54J/CtJCurav0I+zgBuHfG7w91XKMkLVrX4fmmqrprZkOSK4AfAu8AThphH9+uqumO65KkTnV62D4YnP22aeD/AYd0+VySNEljv2CU5HHAUcANI3a5Ockvk9yR5NNJVsyx7+m5HsBUF69BkgaNdValJAE+RS+kPzzP5rcAfwRsoHee8xn0zpc+L8kxVXX3OGudpJbZmJyJSZqscU9Jdw6wCji9qm6ea8OqOn+g6e+TrAeuAN4EfGBIn/3m2qejT0njMrbD9iQfBN4G/I+qWtOyj6q6ErgDeHqHpUnSoo0lPJO8j94h+Dur6s8Xubu9gB2Lr0qSutN5eCZ5D/AnwJ9U1TmL3NcL6N0rOsr9oZK0ZDo955nkbcBZwCXA3yU5bsbqB6tqQ3+7dcDxVZUZfTcAnwc2AtuA3wbeDvwA+Msu65Skxer6gtFL+ssX9x8zbQIOnaPv94AzgCcC+wA/AT4DvN+b5iXtajoNz6pa2bpdVa3ushZJGidnVZKkBoanJDUwPCWpgeEpSQ0MT0lqMO7vtmtMWiYTAScUkbriyFOSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUgPDU5IaGJ6S1MDwlKQGhqckNTA8JamBsyotM87GJHXDkackNeg0PJOsTFKzPI4Yof/hSb6cZGuSf0lyaZKndlmjJHVhXIft7wKuHmj70VwdkqwAvg7cBZwKbAf+GPiHJL9RVbeNoU5JajKu8Px+Va1fYJ+3A/sDv1lVtwMk+SbwQ+BM4I3dlihJ7Xalc54vA67cGZwAVbUZ+Arw8olVJUlDjCs8P5lke//c5SVJjplr4ySPAg4Hbhiy+npgRf+wfrDf9FwPYKqTVyNJA7oOz63AucDrgecA7wCeClyb5D/P0W9/IMCWIet2th3YYZ2StCidnvOsqg3AhhlNX09yMb0R5QeB58+3i4Wsq6r95tqZo09J4zL2c55VdSdwBXDcHJvdTS8ch40uD+gvh41KJWkiluqC0V7MMaqsqgeAW4Gjhqw+GvhZVd01ptokacHGHp5JDgZOAOa7dWktcEJ/+519DwBeAnxpfBVK0sJ1es4zyQX0RpDfoXcofgS9G+YfBfzhjO3WAcdXVWZ0/zDwauDSJO/l326S3w6c3WWdkrRYXd8k/13glcB/Bx4DbAbWAR+oqmG3If2rqvrnJM+iF6Ln0xsVfx14dlX9uOM6JWlRUjXXBe7dW5LpqX33mtqy8bBJl7IsOROTdmfr6iK2s23rbHf17ErfMJKk3YbhKUkNDE9JamB4SlIDw1OSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUoNx/elhictvv66pnxOKaHfgyFOSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUoNOwzPJmiQ1x+PgOfqeNUufO7usUZK60PU3jN4PfGKgbR/gcuD6qholCE8A7p3x+0Md1SZJnek0PKvqFuCWmW1JXg48CvjsiLv5dlVNd1mXJHVtKc55vha4H/jCEjyXJC2JsU4MkuQJwAuBC6rqnhG73ZxkBXAXcAlwZlXdNcv+5xuhTo1crCQtwLhnVToV2JvRDtlvAf4I2EDvPOczgHcCz0tyTFXdPbYqtUtpmY3JmZi01MYdnqcBP6iqq+fbsKrOH2j6+yTrgSuANwEfGNJnv7n22R+ZOvqU1LmxnfNM8kzgKcDnWvdRVVcCdwBP76ouSerCOC8YvRb4JXDeIvezF7Bj8eVIUnfGEp5JHgO8Ari8qn66iP28AHg8sL6r2iSpC+M65/l7wGOBvx62Msk64Piqyoy2DcDngY3ANuC3gbcDPwD+ckx1SlKTcYXn6cDPgYsX0Od7wBnAE+l9K+knwGeA93vTvKRdzVjCs6qeNc/6lUPaVo+jFkkaB2dVkqQGhqckNTA8JamB4SlJDQxPSWow7u+2S0uiZTIRcEIRtXPkKUkNDE9JamB4SlIDw1OSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUgPDU5IaGJ6S1MBZlbSsORuTWjnylKQGI4VnkkOSfCzJNUnuTVJJVs6y7auS/FOSXyS5LcmHkvzKiM+zT5L3JtmU5MEkNyZ53QJejyQtiVFHnk8CVgP3Al+bbaMkpwAXANcC/wU4G3gTsGbE5/k48A7gXOB3gMuAzyT5/RH7S9KSGPWc59VVtQIgySrgxMENkuwNnANcXFVn9JuvSrIN+FSSj1bVP872BEmeBrwOeGtVfbTfvC7JE4Czk6ypql+MWK8kjdVII8+q2jHCZscBBwPnDbRfAGwDTpqn/yqggPMH2tcA+wPPHaEGSVoSXV5tP6q/vGFmY1Xdn+SWGevn6n9nVf18oP36GesvnbkiyfQ8+5yaZ70kNenyavuB/eWWIeu2zFg/V//Z+s7cvyRN3Dju86wFts+3Tc22rqr2m2tn/ZGpo09Jnety5Lm5vxw2QjyA4aPKwf7D+s41opWkiegyPG/sLx92bjPJo4HDGTgXOkv/g5MMBujR/eV8/SVpyXQZnuuBO4FXD7SvBvYBvjRP/y8DAU4ZaD8VmAau6qBGSerEyOc8k5zc//HY/vL4JAcB91XVZVW1Pcm7gTVJ/gK4EDgS+DPgwqpaP2NfpwGfA06vqjUAVXVDkjXAnyYJsAF4Mb0wfXNVPdD+MiWpWwu5YPTFgd/P6i83AYcCVNV5SX4JvAv4b8DPgU8A7xnxOd4A3Aa8FXg8cCvw+qr69ALqlKSxS9UoF8F3T0mmp/bda2rLxsMmXYrkTEy7mXV1EdvZtnW2u3qcVUmSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUgPDU5IaGJ6S1MDwlKQGhqckNRjHn+GQNMTlt1/X1M8JRXZNjjwlqYHhKUkNDE9JamB4SlIDw1OSGhiektTA8JSkBoanJDUYKTyTHJLkY0muSXJvkkqycmCbJyT5YJL1STYn2ZrkW0lOTTLv8yQ5tL/fYY8XNr4+SRqLUUeeTwJWA/cCX5tlm2OAVwN/B5wCvAJYD6wB/s8CajoXePrA45sL6C9JYzfq1zOvrqoVAElWAScO2eZa4PCq2jaj7YokjwXenOSsqpoe4bk2VdX6EeuSpIkYaeRZVTtG2ObugeDc6VvA3sATFlibJO2yluKC0XOB+4Afjbj9mUkeSnJfkquSPG+2DZNMz/UApjqoX5IeYayzKiV5GXAS8L6qemCezR8EPg1cAdwJ/BrwNuDKJCdV1dpx1irtqlpmY3ImpvEbW3gmOQ44n94FpPfPt31V3QG8fkbTNUn+BrgOOAd4RHhW1X7z1ODoU9JYjOWwPcmxwN8CG4CXVtX2lv1U1f3AhcDhSR7XYYmStCidh2eSY+gdet8MvKgfgIuxs8Z5L1pJ0lLpNDyT/AZwJXAL8MKq+pdF7u/R9M6Z/qCqNndQoiR1YuRznklO7v94bH95fJKDgPuq6rIkT6EXnDuA/wUcmWTmLm6qqnv6+zoN+BxwelWt6bd9hF6YfwP4GXAo8BbgMGBVw2uTpLFZyAWjLw78flZ/uYle0D0dOLDf9tUh/Z8DrJtj/zcCbwBOBX4V2Ervm0VnVNW1C6hTksYuVTXpGsYmyfTUvntNbdl42KRLkZaUtyot3rq6iO1s2zrbXT3OqiRJDQxPSWpgeEpSA8NTkhoYnpLUYKwTg0iajJbJRMCr9AvhyFOSGhiektTA8JSkBoanJDUwPCWpgeEpSQ0MT0lqYHhKUgPDU5IaGJ6S1MDwlKQGhqckNTA8JamBsypJ+lfOxjQ6R56S1GCk8ExySJKPJbkmyb1JKsnKIdv9qL9u8PGhEZ9nnyTvTbIpyYNJbkzyugW+Jkkau1EP258ErAa+A3wNOHGOba8G3jXQ9tMRn+fjwKuAM4ENwIuBzyTZp6o+MeI+JGnsRg3Pq6tqBUCSVcwdnndX1fqFFpLkacDrgLdW1Uf7zeuSPAE4O8maqvrFQvcrSeMw0mF7Ve0YdyHAKqCA8wfa1wD7A89dghokaSTjuGD03P550YeSfDfJG5NkhH5HAXdW1c8H2q+fsf5hkkzP9QCmFvlaJGmorm9VugT4NnArcCBwCvBXwJOBt8zT90Bgy5D2LTPWS9IuodPwrKo3DzStTXIB8AdJzq2qTfPtYo62R6yrqv3m2pmjT0njshT3eZ7Xf57fmme7zQwfXe5sGzYqlaSJWIrw3Pkc8110uhE4OMlggB7dX97QaVWStAhLEZ6voRec35pnuy8DoXeedKZTgWngqu5Lk6Q2I5/zTHJy/8dj+8vjkxwE3FdVlyVZDbwU+CpwG3AAvSBcBZxTVT+esa/TgM8Bp1fVGoCquiHJGuBP+1fnd94kfwrw5qp6oPVFSlLXFnLB6IsDv5/VX24CDgV+CBwE/G965ykfBL4LnFZV5434HG+gF7xvBR5P76r966vq0wuoU5LGLlXDLnDvGZJMT+2719SWjYdNuhRJA3b1mZjW1UVsZ9vW2e7qcVYlSWpgeEpSA8NTkhoYnpLUwPCUpAaGpyQ1MDwlqYHhKUkNDE9JamB4SlIDw1OSGhiektSg679hJEkjufz265r67SoTijjylKQGhqckNTA8JamB4SlJDQxPSWpgeEpSA8NTkhoYnpLUYKTwTHJIko8luSbJvUkqycqBbVb222d7vHue5zh0jr4vXMRrlKTOjfoNoycBq4HvAF8DThyyzXeApw9pfzfwUuDLIz7XucAXBtpuHrGvJC2JUcPz6qpaAZBkFUPCs6ruAdbPbEvy74FnAN+oqu+N+Fybqmr9/JtJ0uSMdNheVTsa938icBDw1439JWmXNO4LRq8F7uORh+FzOTPJQ0nuS3JVkufNtmGS6bkewNRiX4AkDTO2WZWSPBF4AfD5qrp3hC4PAp8GrgDuBH4NeBtwZZKTqmrtuGqVtPtomY1pHDMxjXNKutOAvRnxkL2q7gBeP6PpmiR/A1wHnAM8Ijyrar+59unoU9K4jPOw/TTg+1V1TesOqup+4ELg8CSP66owSVqssYRnkmcD/4FuLhTtrLH1opUkdW5cI8/XAr8EPr+YnSR5NHAS8IOq2txFYZLUhZHPeSY5uf/jsf3l8UkOAu6rqstmbPdY4GTgsv55zGH7Og34HHB6Va3pt32EXph/A/gZcCjwFuAwYNXIr0iSlsBCLhh9ceD3s/rLTfSCbqffAx7Dwg/ZbwTeAJwK/CqwFfgmcEZVXbvAfUnSWKWqJl3D2CSZntp3r6ktGw+bdCmSJqjlVqV1dRHb2bZ1trt6nFVJkhoYnpLUwPCUpAaGpyQ1MDwlqcE4v9suSbuElslEDnjKL9l6z+zrHXlKUgPDU5IaGJ6S1MDwlKQGhqckNTA8JamB4SlJDQxPSWpgeEpSA8NTkhoYnpLUwPCUpAZ7+p/h2AFkal//j5C0MFvv2QFQVTU0QPb08NxOb3Q9bG6Uqf5y69JVtEvz/Xg434+HW47vx77AjqoaOvvcHh2ec0kyDTDbH3dabnw/Hs734+F8Px7J41lJamB4SlIDw1OSGhiektTA8JSkBoanJDUwPCWpwbK9z1OSFsORpyQ1MDwlqYHhKUkNDE9JarDswjPJY5P8eZI7kjyQ5NtJTpx0XZOQZGWSmuVxxKTrG6ckhyT5WJJrktzbf80rZ9n2VUn+KckvktyW5ENJfmWJSx6rUd+PJD+a5fPyoQmUPVFDp1raw60F/hPwTuCHwGnA2iQvqapLJ1nYBL0LuHqg7UcTqGMpPQlYDXwH+Bow9D/QJKcA5wMfB/4ncCTwZ8ChwCuXotAlMtL70Xc1vc/MTD8dU127rGUVnkleBDwfeHlVre23XQUcBnwEWK7h+f2qWj/pIpbY1VW1AiDJKoaERZK9gXOAi6vqjH7zVUm2AZ9K8tGq+sclq3i85n0/Zrh7GX5eHmG5Hba/jN5krhftbKjeja7nAUckeeqkCtPSqqodI2x2HHAwvc/HTBcA24CTuq5rUkZ8PzTDcgvPo4CbhnxQrp+xfjn6ZJLtSbYmuSTJMZMuaBex8/Nww8zGqrofuIXl+3l5bv+86ENJvpvkjUky6aKW2rI6bAcOBL4/pH3LjPXLyVbgXGAdvffgSODdwLVJjt+DDklb7fw8bBmybgvL7/MCcAnwbeBWeq//FOCvgCcDb5lgXUtuuYUnwFzfR11W31Wtqg3AhhlNX09yMb2R1gfpnR/W7J+LZfV5AaiqNw80rU1yAfAHSc6tqk2TqGsSltth+2aGjxYO6C+HjTCWlaq6E7iC3vm+5W5zfznbZ2bZf176zqOXJb816UKW0nILzxuBI5MMvu6j+8sbEPQ+F8tuVDXEjf3lw85tJnk0cDh+Xnba+e9pWV10Wm7huRbYD3jJQPtrgI1VddPSl7RrSXIwcAKw7G9Fofce3Am8eqB9NbAP8KUlr2jX9Bp6wfmtSReylJbbOc9LgauAzyY5kN5N8qcCzwReOsnCJqF/rupWejdG3w0cQe/m50cBfzjB0pZEkpP7Px7bXx6f5CDgvqq6rKq2J3k3sCbJXwAX8m83yV+4p93rON/7kWQ1vX8nXwVuo3fq4hRgFXBOVf14qWuepGU3n2eSfYGzgZPpjUJvAt5XVV+eaGET0A+GV9L7tsxj6J3jWwd8oKr2+EPSJLN9+DdV1aEztjuF3n8qTwZ+Tu8+z/dU1QNjL3IJzfd+JDkO+ADwNHrngR8Evgt8sqoG74Xd4y278JSkLiy3c56S1AnDU5IaGJ6S1MDwlKQGhqckNTA8JamB4SlJDQxPSWrw/wH/cjx8XGILWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 解码器一次输入序列中向量，当前步后面的序列需要被遮盖\n",
    "# 需要被遮盖的单词被标记为 False \n",
    "\n",
    "def subsequent_mask(size):\n",
    "    attn_shape = (1, size, size)\n",
    "    subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')\n",
    "    return torch.from_numpy(subsequent_mask) == 0\n",
    "\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "plt.imshow(subsequent_mask(20)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T01:30:55.533041Z",
     "start_time": "2020-05-04T01:30:55.527340Z"
    }
   },
   "source": [
    "`np.triu(m,k=0)`：第 k 对角线以下的元素归零，中心对角线索引为 0 ，索引向右上角增加 1、2、3 ，向左下角-1、-2、-3\n",
    "```\n",
    ">>> np.triu([[1,2,3],[4,5,6],[7,8,9],[10,11,12]], -1)\n",
    "array([[ 1,  2,  3],\n",
    "       [ 4,  5,  6],\n",
    "       [ 0,  8,  9],\n",
    "       [ 0,  0, 12]])\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 注意力\n",
    "### 点积注意力\n",
    "<img src=\"../images/scaled_attention.png\" width=\"50%\">\n",
    "$$\\mathrm{Attention}(Q, K, V) = \\mathrm{softmax}(\\frac{QK^T}{\\sqrt{d_k}})V$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.937875Z",
     "start_time": "2020-05-04T03:29:55.934987Z"
    }
   },
   "outputs": [],
   "source": [
    "def attention(query, key, value, mask=None, dropout=None):\n",
    "    \"\"\"\n",
    "    query : batch, target_len, feats\n",
    "    key   : batch, seq_len,    feats\n",
    "    value : batch, seq_len,    val_feats\n",
    "    \n",
    "    return: batch, target_len, val_feats\n",
    "    \"\"\"\n",
    "    d_k = query.size(-1)\n",
    "    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n",
    "\n",
    "    if mask is not None:\n",
    "        scores = scores.masked_fill(mask == 0, -1e9)\n",
    "    p_attn = F.softmax(scores, dim=-1)\n",
    "\n",
    "    if dropout is not None:\n",
    "        p_attn = dropout(p_attn)\n",
    "    return torch.matmul(p_attn, value), p_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.943707Z",
     "start_time": "2020-05-04T03:29:55.938525Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 5, 8])\n",
      "Test passed\n"
     ]
    }
   ],
   "source": [
    "def test_attention():\n",
    "    query = torch.randn(3, 5, 4)  # batch, target_len, feats\n",
    "    key = torch.randn(3, 6, 4)  # batch, seq_len, feats\n",
    "    value = torch.randn(3, 6, 8)  # batch, seq_len, val_feats\n",
    "    attn, _ = attention(query, key, value)\n",
    "    print(attn.shape)\n",
    "    assert attn.shape == (3, 5, 8)\n",
    "    print(\"Test passed\")\n",
    "\n",
    "\n",
    "test_attention()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 多头注意力\n",
    "<img src=\"../images/multi_head_attention.png\" width=\"50%\">\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.949204Z",
     "start_time": "2020-05-04T03:29:55.944451Z"
    }
   },
   "outputs": [],
   "source": [
    "class MultiHeadedAttention(nn.Module):\n",
    "    def __init__(self, h, d_model, dropout=0.1):\n",
    "        \"\"\"\n",
    "        h, num_heads\n",
    "        d_model, features\n",
    "        \"\"\"\n",
    "        super(MultiHeadedAttention, self).__init__()\n",
    "        assert d_model % h == 0\n",
    "        self.d_k = d_model // h\n",
    "        self.h = h\n",
    "        self.linears = clone(nn.Linear(d_model, d_model), 4)\n",
    "        self.attn = None\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, query, key, value, mask=None):\n",
    "        # query,key,value: batch,seq_len,d_model\n",
    "        \n",
    "        if mask is not None:\n",
    "            mask = mask.unsqueeze(1)\n",
    "        nbatches = query.size(0)\n",
    "\n",
    "        query, key, value = [\n",
    "            l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
    "            for l, x in zip(self.linears, (query, key, value))\n",
    "        ]\n",
    "\n",
    "        x, self.attn = attention(\n",
    "            query,  # batch,num_head,seq_len,feats\n",
    "            key,\n",
    "            value,\n",
    "            mask=mask,\n",
    "            dropout=self.dropout)\n",
    "\n",
    "        x = x.transpose(1, 2).contiguous().view(nbatches, -1,\n",
    "                                                self.h * self.d_k)\n",
    "        # batch,seq_len,num_head*feats\n",
    "        return self.linears[-1](x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.957096Z",
     "start_time": "2020-05-04T03:29:55.950055Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test passed!\n"
     ]
    }
   ],
   "source": [
    "def test_multi_head():\n",
    "    x = torch.randn(2, 4, 12)\n",
    "    d_model = x.shape[-1]\n",
    "    model = MultiHeadedAttention(2, d_model)\n",
    "    attn = model(x, x, x)\n",
    "    assert attn.shape == (2, 4, 12)\n",
    "    print(\"Test passed!\")\n",
    "\n",
    "test_multi_head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 前向层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.960554Z",
     "start_time": "2020-05-04T03:29:55.957872Z"
    }
   },
   "outputs": [],
   "source": [
    "class PositionwiseFeedForward(nn.Module):\n",
    "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
    "        super(PositionwiseFeedForward, self).__init__()\n",
    "        self.w_1 = nn.Linear(d_model, d_ff)\n",
    "        self.w_2 = nn.Linear(d_ff, d_model)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.w_2(self.dropout(F.relu(self.w_1(x))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 嵌入层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.963755Z",
     "start_time": "2020-05-04T03:29:55.961282Z"
    }
   },
   "outputs": [],
   "source": [
    "class Embeddings(nn.Module):\n",
    "    def __init__(self, d_model, vocab):\n",
    "        super(Embeddings, self).__init__()\n",
    "        self.lut = nn.Embedding(vocab, d_model)\n",
    "        self.d_model = d_model\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.lut(x) * math.sqrt(self.d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 位置编码\n",
    "\n",
    "$$\n",
    "\\text{PE}(i,\\delta) = \n",
    "\\begin{cases}\n",
    "\\sin(\\frac{i}{10000^{2\\delta'/d}}) & \\text{if } \\delta = 2\\delta'\\\\\n",
    "\\cos(\\frac{i}{10000^{2\\delta'/d}}) & \\text{if } \\delta = 2\\delta' + 1\\\\\n",
    "\\end{cases}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:55.968633Z",
     "start_time": "2020-05-04T03:29:55.964479Z"
    }
   },
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "    \"Implement the PE function.\"\n",
    "\n",
    "    def __init__(self, d_model, dropout, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "        # Compute the positional encodings once in log space.\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len).unsqueeze(1)\n",
    "        div_term = torch.exp(\n",
    "            torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + Variable(self.pe[:, :x.size(1)], requires_grad=False)\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:56.078421Z",
     "start_time": "2020-05-04T03:29:55.969378Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f83a03f5f10>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4gAAAE5CAYAAADSuQ43AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3QU1RfA8e9sdje9N0IKIXRCld57ld5BlKKiiGIF/dkLir0hggqKoEiPKFVQ6YTeW4AUkgDpPdlky/z+mBBAWkKy2U3yPufkZLI7O3OTk2zmznvvXkmWZQRBEARBEARBEARBZekABEEQBEEQBEEQBOsgEkRBEARBEARBEAQBEAmiIAiCIAiCIAiCUEgkiIIgCIIgCIIgCAJQxRJESZJiJEmKsXQcgiAIgiAIgiAIlnCvnEiqSlVMJUkyARKQYelYBEEQBEEQBEEQLMAVkGVZvu1gYZVMEF1dXS0diiAIgiAIgiAIQrnLyMiAuySI6vINx+IyXV1dXdPT0y0dhyAIgiAIgiAIQrlzc3MjIyMj807PV6k1iIIgCIIgCIIgCMKdiQRREARBEARBEARBAESCKAiCIAiCIAiCIBQSCaIgCIIgCIIgCIIAiARREARBEARBEARBKGTWBFGSpABJkr6SJGmXJEnZkiTJkiR1LcHrW0iS9LckSTmSJKVJkrRMkiR/M4YsCIIgCIIgCIJQZZl7BLE2MBbIBv4uyQslSWoAbENpbD8CeBxoDmyTJMmpbMMUBEEQBEEQBEEQzN0HcYcsyz4AkiQNAQaV4LXvAFnAQFmWcwqPcRI4BUwDPirjWAVBEARBEARBEKo0syaIsiyb7ud1kiRpgAHAgmvJYeHxzkqSFA4Mp5IkiMbkBPL+/A7Jrz4q39pIdvZItrZIWltUtlpl284OlVZr6VCtRnpuAVHJOVRztcPP1d7S4Qj3YDAZyCrIosBYgN6kp8BUgN6oV7YLH5OQsFPbYae2w97GvmjbTm2HRqWx9LcgFJdeB8igEX+XgiBYuYIcyEsHRy9Q21o6mgohNjUXvdFEdTd77DQ2lg7Hqpl0OowZGRgzMpD1euxDQy0dUomYewTxfoUA9sDJ2zx3HJhwuxdJkpR+j+O6ljKuMqfbt4XYj367534qJyfUXl6ovb1ReyufbQq/1vj6og0JQe3jgyRJ5RB1+UjNKeB8QhYRidlcSMgiIiGb84nZJGfnF+0T4G5P62APWgZ70LqmO7W8nSrVz8DapevSic6MJjozmuS8ZFLyUkjRpZCal0qKLoWUvBTS89ORke/7HGpJjbudO94O3vg4+OBj74O3gze+Dr5Fn4NcgrC1Ef/gze7SPjj2G+QkgS4D8jMhPwt0mcq2sQAkFdTsDI2GQ4OBYO9u6agtJiU7n2UHYtl6JoG8AiN6owm9US78fH3bzV7DwKbVGdkygNo+zpYOWxAqr+wkOLcBzq6DyG3KexaArauSKDp6gaP39c+1ukON9hYN2ZJyCwyER6aw/VwS2yOSiE7JLXrOy8kWf3d7Atzs8Xe3x9/NnkAPe9qGeOKgtdb0ovSMWVnoY2MpiI1DHxdLQWwshsQkjJkZmDIyMKZnYMzMRM6/fq2qqV6d2v+UaKWdxUmyfP8XbiU6kTLFNAzoJsvytnvs2x7YDYyUZXnVf557H3gVcJBlOe8/z90zQXR1dSU9/V67lZ+cFV8R+848ZGPpkxqVoyPakBBsQ0LQ1qqFba0QtCEhaAMDkdQV549127lEPtx4lrNXs0r8WncHDS2DPWhT04ORLQNxtRejT6UlyzKxWbGcTz9PdIaSDF77nJ5vHX9LKklFkHMQIa4h1HKrVfRR07WmSBxLy2SCiI2w+yuI3Vey16o0ULunkizW6we2VWP5+PG4dBbtiWbdsSsUGEs2kaZZoBsjWwYwsGl1XOzE+5cglFr6JTizTkkKL+2Fkk5uC2oPnV9SksVKfgNalmXOJ2YXJYT7o1JL/B7m5aRlatfaPNQmqEKPMhqSk9GdPq18nDuHPjYOfWwsxoyMEh9L5ehIvUMHzRDl/XNzcyMjIyNDlmW32z1v7QniCFmWV//nuWsJor0sy7oSxpDu6urqak0JIgCyjJwWjRxzEPnSYeT445jiTyLnZSMbwWRQYdCpMDjWxVCtK8Y8MCQmYUhOxpCkfMZ05z9gycEB+0aNsG/WrPCjKWoPj3L8Bovn3NUs3t9whh0RSTc97u9mT11fJ+r4OlPbx4m6vs7U9HQkOiWHA9GphR9ppOYU3PQ6H2db3hkUSt9G1cSoYglk5GdwIvkEJ5JOcDz5OCeTT94zEfSy98LXwRdPe0887Txv+exq64qtjS1aGy0alabos8ZGg1pSIyOTb8xHZ9ChM+jIM+Zd3zbkkaJLITE3kaTcJJLykkjITSjaNpgMd4xLJamo4VKDZt7NaOajfNR0qSl+H4pDr4Pjy2HPHEg5f/1xv2YQ2BpsXcDO5ebPti7K6OKpMOUuvf763WbU9lC3DzzwCNTuUf7fj5nlG4xsPHGVRXuiORp7/e/FxU7N8BYBBLo7oFGr0KgkNDYqNGoVWhsJtUrF6SuZrDoUx6XU6z8vW7WKfo2qMbJlIO1CPFGpxO+sIJTIqTDY+TlcPX7z41onqNML6g+Aao0hN1V538pJgpxkyE1WtpMvQMKJ66+r/gB0nqHc7KqE/0Mik7KZueo4B2PSbnrcRiXxQJAbXep606WuDx5OWuLT8ohLyyU+LY/49MKPtDxi03LRG5W8ws/Vjuk96jCiRQAaG+vuqqdPSER36iS6U6fRnTqF7vRpDImJd32NysEBTWAgmsAANNX8sHF1VT7cXFG5uBR+7YaNqws2Li5IGuu64VdRE8R6wFngCVmWv//Pc/OACbIsO9xHDNaZIN6OLENaFFw+CidWwbn1yuMqNbR5Erq+ArbKVCRTfj4F0TEURF4k/2Kk8jkyioKoqJuGuG+kqRGEfdOm2DdrhmPbdmhrBlvsojk5O58vtkTw2/5LmAp/HduFePJczzo08nfF0fbeo5+yLHMxqTBhjEpl48mr5OmNAPRs4Mt7Q0LFesU7uJpzlT2X93Dw6kFOJJ8gOjP6tvvZ2dhRw6UGwa7BBLsEE+waTE2XmtRwqYGT1jIjQybZRFJuEpEZkVxMv8jFjItEpkdyIf0CmQWZt32Nq63r9YTRuxmNvRuLUcYb5aXBwR8hfD7k3PAPsk5vaD8dgjsW7+KoIAciNsHJNXD+r+tTuQBaTIK+syvFWkWd3sj87Rf5JTyG5Ozr32P9as5MaB/MkGb+2GvvfRfdZJLZF5XKykOxbDxx/f0LILS6C3PHPUCwl6NZvgdBqFR0mbBxpjId/hoHT6jXX5n2XrMLaOzufRxZhqgdsOMTiN55/XHfRsqIYoNBoKq4I2TXmEwyi/ZE8/Hms+j0ymBDNRc7utT1pms9b9rX9ir2bKyETB3f/HOBZQcuFSWKNTwdeL5nXQY2rY6NldzoMqSkkLt/Pznh+8gND6cgJub2O6rV2NaujV39+miDa6AJCEQbGIAmMBAbd/cKfbO5oiaIGiADpUjN9P88twvQyrLc+j5iqDgJ4n+d3wobZ0BqpPK1UzXo874ydesOv6Cy0Yg+Ph7dyZPkHj1K3tFj6M6cAb3+ln01AQE4duqIU6fOOLZtg8qhxPl3ien0RhbtiWbuPxfIyldGgUK8HHm1fwN6NCjdesrY1Fxe//0k2wtHI51s1czoU4/xbWtYzRuUpegMOg4lHGL35d3sid/DxYyLt+xjI9lQ170ujb0a09i7MU28mhDsGoxKsu67gNfIskyKLoUL6Rc4lXyKo4lHOZp09LYjofZqe9r4taGTfyc6+XfCz8nPAhFbicjtsHKCkiSCckOq8Sho/wz4Nrz/4+oy4Ox6OPQzxIYrj/mEwsifwLte6eO2kIRMHU8sOVQ0YmijkugbWo0J7YNpFXz/Fw9ZOj0bTlxhxcE4DhXezXe2U/PZyKb0Dq1WZvELQqUTux9WPwbphRf8tXtCx+chsC3YlGKpzaVwJVG8sPX6Y151YfBcZUZFBRWbmsuMVccIj0wFlBlbs4c1plMdr1Jfg3259TxhR+KKbvzX9XXixd716N3Qt9wTK2NWFrkHDpATHk5u+D7yIyJu2UfSaLCtVw+7hg2Vj9BQbOvWQWVbOW8gV8gEsXD/lUB7oI4sy7mFj9UFzgCvybL84X3EUHETRABDPuz5GnZ8BobC5Zc1OsKDn4JPg2IdwqTToTt9mryjx8g7epTcw4cxJifftI+k0eDQqiWOHTvh1KUztrVqlfV3wu4Lybyy5jixqcr34Wqv4dkedRjftgZaddkkIbIs8+fxK7z756miO/vNg9yYPawx9au5lMk5Kor47Hj+jvmb3Zd3cyjhEPnGm0eWnTXOtPZrXTSi1tCzIfbqij+6cyNZlonOjC5KFo8mHiUyI/KW/eq41ylKFpv5NEOtqjjrd0vl0M+w/gUwGUDrDC0nQpup4OpfducwmWDPV/D3eyAbQeMA/T6G5uMr3JStI5fSeGLJIRKz8pEkeLxTCJM71KSaazFGJkpg78UUnvntSFFxrie71OKl3nVRW/mULUEoV0YD7PwMtn+kvLfY2ELvWdD68bJ9b4k/DDs+vT6ry8YWhv8ADQeX3TnKgSzL/LY/lvfXnyanQJmtMLplIK8PaIBzGa59vpCYxRdbzrP+xJWixya2D+bNAQ3NPm3ekJJC1t9/k7VlKznh4bcMjki2tji0eACHNm1xbNMau9BQq5sGak4WTxAlSRpRuNkKmAm8jdLLMEeW5Y2F+0QDyLIcfMPrGgL7gXDgU8AReB/QAM1kWS5xBZMKnyBek34JNv1PWXANyl3+od9B4xF3f91tyCYT+WfPkr1zF9k7d5B35CgYjTftY1unDi4P9self3+0QUGlDn/DiSs8u+wIeqOMWiXxSLtgpveojZuDeVp5pOcW8MGGM6w4GAeAWiXxVNdaPNuzbqUeTbyac5XN0ZvZHL2ZE8knbnpOQqKxV2Pa+7enQ/UONPJqVHUSoRuk6lLZHb+bnfE72R2/+5Zpqc5aZ3oE9aB/zf60rtYam0ownegWJhNsfUu5+QTKyN64ZeBW+r/1O4rdD6smQ0as8nXjkTDgi6Jp89Zu1aE4Xl1zggKjCWdbNV+NbUb3+r5mO19ipo6nlx5hf7Ryl79tiAdzxj6At3PlvLMtCCWSFg1rplwvouUTCsMXlG7Ww71cOQ4rJ0LqRUBSZnS1fapC3Oi6kpHHy6tPFNV78HG25cPhjc36HnYyPoOPNp1l53llQGJAEz8+G9UUW3XZ/k/VX75M1tatZP21hdzDh2+uz6FWY9+0KY5t2uDQtg32zZpV6RZy1pAg3ukEMdcSwtsliIWPt0Lpd9gG0AN/AS/Kshx7n7FUjgTxmgtbYUPhtFNJBUPmQ9PRpTqkMTOTnD17yd65g5ydu25ZpGvXuDEu/fvj0q8vmmoln+q06lAcM1cdwyRDPV9n5o1/gBDv8lm/tudiMq+FnSQqWWmtOaplAB8Oa1Kpij8k5iayJWYLm6I2cTTp6E3Pedh50DmgMx2qd6CtX1vc7G77nlBlGUwGTiSfYGfcTnbG7+Rs6tmbnvey96JvcF/61+xPI69GFXrtQZGCHOXC6trNpjq9YfhCpeiMueWlwdqnr5/bIwRG/ATVm5n/3PfJYDQxe+NZFu6KApQp8d8/0pLaPuZ/D9MbTXyy+Rzf71BGvX2cbZn70AO0Cra+gmOCUG6Or4D1LyptdkBJ0nq8Vbw1hqWVkwLLxl5PTFs/oayttuIbidsjknh66WGydMqynsHNqvPOoFCz3aC/kckk89760/y0OxqAjrW9mP9wC5yKUWfibgwpKWT88SeZ69ejO3lzdzyVoyNOXbrg3LsXjh07YeMk1nFfY/EE0ZpUugQRlCavvwyD+EOApMyHb/5QmRxalmV0J06QuX49mRs2Yki6obqoJOHQogUuAwfi8uCDxfqjW7w3mjfXngKgaYArP09uXS5vSjfS6Y3MWn+aX8IvATC2dSDvD2lcoZPEfGM+W2O2sub8Gg5cPXBTz0E3Wzd61uhJ3+C+tPRtWTlHwMwkISeBf2L/YWPURo4kHrnpuUDnQPrV7MeDIQ8S4hpioQhLKfMy/DYGrhxTvm7zJPR+v3TrdEpKluHAAtj8qlLExkarjCQ2H19+MRRTem4Bz/x2pOgOeJe63nw9tnm5t9LZdPIKL608Tna+ARuVxP/61efRjqIyr1AF7fgU/nlP2XbyhSHfKmsOy5M+T7nJduYP5et6Dyqjl1rz13EoqfDIFCb8uJ98gwkPRy3vD2lEv8blu+ZelmXmbb/Ix5vOAdDI34VFk1rj5VSy2RCyXk/2jh2krwkje/t2MFyvZm7j7o5Tj+649OqFQ7t2VXqU8G5EgniDSpkgglIA4pcRELcfkGDgV9BiQpmeQjYayT10iMwNG8jatBnjDT9DlYMDLgMH4j56FHYNbz+lY962i3y0SRmRaV3Tg4UTWpbpPPeSkGWZd/48zaI90QCMbxvEe4Mr3ohQZHokq86v4o+Lf5CRf70vj4vWhZ41etKnRh9a+bVCo6o6c+rNJT47no1RG9kQtYHzaedveq6lb0tG1xtNj6AeaGwqyM/6yjFYOgayLiuzD/p9rKzVsVg8x2HVJEi5AEgw6merWtNzPiGLxxYfJKawSfQTnUOY2be+xaaoRyZlM/WXw5xLUFZaPNKuBu8MCq1w72GCcN8O/gjrnle2a/dUltk4elkmFpMJtrwBe79RvvZvAWOXg5O3ZeK5jaOx6Tz0Qzg5BUaCPR1Y/kQ7fF3KYZT1DlYcjOV/a05gNMkEezqweHIbgjzvnVTrIiLIWBNGxp9/YkxJKXpc5eKCS/9+uPTrj0OLBypU729LEQniDSptggiQnwW/jlSawAI8+Dm0etQsp5L1enLCw8lct47MzX8h6663o7Rr0gT30aNw6dcPlYMDsizz6V/nmPuvUimzc11vvhvfolhl381JlmXeWHuyaCRxYvtg3hrY0OovsHQGHVtitrAqYhWHEw8XPa6W1HQL6saQ2kNo59eu4iQqFVBEWoSSLEZu4HLO5aLHPe08GVZnGCPqjqC6U3ULRngPEZuVtTP6XKUYzchFUKec77rfTn4WLBkKcQeUkcSHw5SWGhYWlZzD0G93k56rx1at4qPhTRjSvAwL99yn3AIDr645we9Hld/B//WrzxNdyr6gmCBYnVNhsHISIEOtHjB2GaitYJRo33ew8WVABvdgeGgVeNWxdFScvZrJ6O/CycjTU93VjhVPtiPA3fIjnH+fSWDa0sPo9Ca8nW1ZNKkVodVdb9nPVFBA5oYNpP26FN2JG+opSBKO7dvjOmwozj17Vtpqo+YiEsQbVOoEESA/G5aOhphdytf9PoE2U8x6SmNGBhlr/yBtxXIKLlxvl6BydsZl4ECWVm/DnAtK5ai+odX4amyzMl+UfL9MJpnXfj/Jb/uVJHFyh5q8MaCBVSaJV3Ou8svpXwi7EHZTIZUApwCG1x3OkNpD8LK30N3TKsokmwi/HM7yc8vZFrcNk6wshldJKjr5d2JUvVF0qN7Buqb1xh+Cn/qDQQeuQTBuuXkLOZRUbios7A0p58HWFSZvBN9Qi4WTnlvAsG/3EJmcg6ejlp8mtaJJgPWs3TWZZJ5dfpQ/jylJ4pyxzRnY1IpvTghCaV38V7kZbtJDQCt4ZC1orWhd2dn1sOpRpdK8vTs8usWiSWJUcg4j5+8lOTsfLydbVjzRttzqPhTHoZhUJi86SEaeHmdbNd8/0pJ2tTwBZW1h2rJlpP227KZq+5qgINyGDcV18GA0flW4LVUpiQTxBpU+QQSl6MRvY5TmrgB9PoB208x+WlmWyTt8mLRly8natAm5sJywCYlwv1CSHhzJS8+PsLrS7CaTzCtrjhdVOJ3SOYT/9atvNUniudRzLDq1iE1RmzDIyhx7taSme1B3RtQdQRu/NhWmN2FldjXnKqvPr2Z1xGqS8q6v1Q1yDmJSo0kMqjUIrY2F73BnXoEfukHWFaUgzOTN4ORj2ZhuJ/0SLOgF2VfB2U+5wHILLPcw9EYTjyzcz97IFLRqFb893pYWNdzLPY57yTcYeXjBfvZHp6K1UfHLY21oXVMUrhEqofhDsGgg6HPAuz5M2ggOVvi7HncIlo6C3GTwrAOP/w12t46MmVt8eh6j5u8lPj0PV3sNy6a0pYGf9bX4ikjI4pGF+7maqcNWrWJ1b288NoeR+ec65AKlRRk2Nrj06Y372LHYt2xpNddoFZlIEG9QJRJEUBZNLxsHF/9Rvu49S2l0XU4MaWmEzf4Oz63r8Mu9PkfcvkULPB97FKcuXZBU1pPUmEwyM1YdZ/VhJUl8skstXu5bz2JvQLIsE34lnEWnFrHn8p6ix73svRhXfxxD6wwVo4VWSm/S8++lf1lxbgX7ru4retzb3pvxDcczqu4onLQWuHur18Gi/soFlq0LPLbVuhvUXz0JP/VTKhN61VWS2XK8EJRlmf+tOcGyA0rB7K/GNGNwM8tPK72T9NwChs3bQ2RSDq72GtY81Z5aVjRKIAillhQBP/aBvFRwDVTeE8qyR2tZiz2gvOcaC6BuPxizFMrxuicxS8fo78KJSs7BUWvDr4+3pVmg9cx++K+4tFw+eHMBnY5soVnyhaLHVa6uuI8aifu4cWK0sIyJBPEGVSZBBOWCcPl4uLBF+XrcCqjbp1xO/depq0xZcgiVbOJV56v0OLIZ3alTRc/b1qmNx+RHcX2wP5KVVJcymmReWnmMsCPxADzXsw7P9axbrjEYTAY2R29m0alFN7VYCHENYWLoRB4MedDyo1BCsV1Mv8hPJ39ifeT6otFfZ40zo+uP5qEGD5Vfki/LEPYEHF8OSIXvBb3L59ylEbVTqdBsLICA1oVTycpn3cyCnZHMWn8GgGd71OH5XuX7XnA/YlNzGfrtbpKzCwj0sGfN1A6iT6JQOWTEwcI+kBkHDp4w+S/wqm3pqO7t8GL4o/DmfOeZ0P21cjltem4BY74P5+zVLGzVKhZNal00bdPayLJM9j//kDz3W3SnTxc9nuzhR8OnH8d9yGBUDpZfL1kZiQTxBlUqQQQw5MPiwUrhGnsPeHKX2e+4XUrJ5cE5O8nSGWhd04Olj7XBRiWRu28/KQsWkLNrV9G+al9fPJ+YgvuIEVaRKBpNMs8vP8ofhet5fprUim71zD8Fz2gysjF6I/OPzScmM6bo8Za+LZkYOpFOAZ3ENNIK7Er2FRafXszq86vJM+QBoFVpGVpnKJMbTTZ/QZtdX8LWt5TtXu9Ch2fNe76ydGMxinr9YdQSs7fh2Ho6gceXHESWYWDT6nw9plmFmc50LDad0d/vRac30TTAld+mtMVBK6r5CRVYTgr81BeSI0DrBBPXQfXmlo6q+Na/BAd+ULZHLYGGg8x6urwCI2N+COdYbDoaG4nvH25Jt/rWt5RANpnI2rqV5G/nkX/2+g3xnMYtmO3UnMM+dXm6R11e7G3FM10qOJEg3qDKJYgAGfEwv6MyLSOoHUxYZ7YLLJ3eyIj5ezgZn4mXky0bpnfE5z9llHVnz5KyYCGZGzeC0QiAxt8fr2nTcB000OKliQ1GE2O+D+dgTBqejlo2PtcJH2fzlII2ySa2xmzl26PfcjFDKfAjIdGrRi8mhk6ksXdjs5xXsIx0XTq/nfuNpWeWkp6vvAdpVBpG1RvFY40fM8+IYsRmpXAVMjQZA0PnQwVJdors+w42zlS2H5igtPEx0/dw+nImI+bvIbfASLNAN5ZNaYudxoqKDBXDltMJPLHkICYZejbw5buHW1isHYcglIohXymqFX9QqWz80CoI6WLpqErGqFdu1MfsBo2jMr3fjIXBXgs7wa/7LqGSYM7YB3iwiXVNy5RNJrL+2kLyvHnknztX9Lhj5054T5uGfdOmvP3HqaI2ZN8/3ILeodUsFG3lJhLEG1TJBBEg4i9YOlLZ7vgC9HzLLKd5NewESwvfmH55rA3ta935grcgLp6U7+aTviasKFHU1qyJ9zNP49y3r0XXKMal5dL/q51k6gx0rO3F4smtUZXhBZYsy2yL3cbco3M5l3b9DbJPcB+eavoUIW4VtPG6UCy5+lzCLoTx44kfScxLBMBebc/4BuOZEDoBV9syKmaQeBYW9ISCLPBvCRPXg8Zyfa9KZes7sOtzZbvPbGj3VJmfIjFTx5C5u7mcocPfzZ6wae3NdnPI3BbvjebNtcq0/gntavC26JEoVER/vQF7vlZ6tY782eyjb2aTnQTfd1WmyLoHw+P/mmVN9ZbTCTy++CAAL/etz9Su1tP2RpZlJTH8Zg7556+vMXTq2hWvaU9h3/j6DXG90cS4H8I5EJ2Gs62atU93sKrKq5WFSBBvUGUTRLj+RgswfrXSWLYM/X4knueWHwVgRp96TOtWvPUBBTExJH0zl8x165S1UoBtvXp4Pzsdp27dLHZRs/HEFab+qvQZLKs3WlmW2XN5D98c+YaTKSeLHu8W2I1pzaZRz0NMpahKdAYdy88tZ8GJBUUjis5aZyY3msy4+uNw0JRi3UVuKvzQHdKilEqgU7aBcwW+CyvLsGYKnFgBNrbw5M4yLbKj0xsZ/d1ejsVl4Ki1YdXU9lZZ7a8kPthwhu93RALw2cimDG8RYOGIBKEEonfBogGAXK7r98zm8hH4sa/SYqhWd2U0tAxbICVm6ej75U5ScwpoF+LJr4+1KdMb26WRe/gIiR9/TN7Ro0WPOXXvjtdTT2Hf6PZtjBIzdQyYs4vErHzq+joR9lQHHG3FdPmyJBLEG1TpBNGoV6ZqxO0HBy9lPaJL2Uw9OJ+QxaBvdpOnN9KtnjcLJ7Qq8RuTLiKC5DnfkLVlS9Fjdk2b4PvKKzg0t8x6g2sjomqVxMon29E86P5L3J9PO88nBz5h75W9RY919O/I082eJtTLcn3eBMvLLshmyekl/Hz6Z3L0OQB42nnyeJPHGVVvFBqVpmQHNOrhl+EQtR3UdkopeP8HzBB5OcvPgnkdID1GWYP06BawKeHP5g5mrDzGykNxqCRYMKEl3ev7lslxLclkknnil0NsOZ2As52aLc93oZprxVIJTVwAACAASURBVBwRFaoYXYbyt54RC37NlGmZZfS3blHHV8Cax5Xt9tOh93tlcliTSWbiogPsiEjCxU7Npuc6U93NvkyOXRr5UVEkff7FTdd1Tl274j39Gewa3nua7aGYVEZ/F47BJDOgiR9zxjYXMyHK0L0SRFH5oqqw0cCIH8HOTenNs/oxMBpKfdicfANTfz1Mnt6Iv5s9n49qdl93rezq1iVgztcEr1qFY6dOAOiOHSdm7DjiX3wJ/ZUrpY61pN4c0JC6vk4YTDLTlx0hU6cv8TFS8lJ4d++7jPhzRFFy2Lpaa5b0W8K8nvNEcijgpHViarOpbBy2kQkNJ6BVaUnRpfDh/g8Z/sdwdsXvuvdBbrRttpIcAgyeWzmSQwBbZxgyD5CUu/E7Py+Tw247l8jKQ0qLm//1a1ApkkMAlUpi9rDGeDhqydIZeGXNcarSDWGhAtv4ipIcqu1g2A+VIzkEaDIK2j2tbO/5Go6vLJPD/rw3mh0RSv/d2cOaWDw5NKSmcvXd94gcOKgoObRr1Iign38mcP68YiWHAC1qePDGAGXfdcevsHBXlNliFm4lEsSqxC2w8AILiNkF2z8s1eGu9Qq7kJiNxkZi7kMP4O5Yumqk9o1CCfrhe2osWYxdqJI8Za5fz8V+/Un6eg6m3NxSHb8k7DQ2zBn7ALZqFbGpebwWdrLYF1gFxgJ+PPkjA8IGsDJiJSbZRLBLMN90/4YFvRfQzKeZmaMXKhp3O3deavUS64etZ0TdEagkFVEZUUzdOpWntj5FVEYx/jleOa5ULQWl92njEeYNurwFd4B205TtHR8riWIpZOn0vLrmBABtanrwaMeapY3Qqng52fLe4EYAbDuXxMqDcRaOSBDu4fRaOLZU2e71Hnhbf4uZEun5DoR0VbbXPae08CiFs1czmb1RqQI6okWARYvSmHQ6kud/x8VevUlbuhQMBjT+/lT/9FOCVyzHsU3rEh/zkXY1GNZcqb4/e+NZ9l5MuccrhLIipphWRZtehfC5gAQPh0Gtbvd1mCXhMbzxu7KW7p1BoUxoH1x2MaJUu8r4fS2JX3yOMSkZUFpj+Lz4Ai4DBpRbIZtfwmN4vfD7/Hh4E0a1CrxzzLLMlpgtfH7oc+KzlZ6KLloXnmr21P1NFxSqrHOp5/j4wMfsv7ofALWkZlyDcTzR9AlctLdZH2c0wILucOUYeNZRppFX1KI0d6PXwfddIOkseNeHKdvv+/u8VvHPTqNi07OdCfZyLONgrcPTSw+z7vgVnG3VbH7eOqafCcItsq7Ct+2Uquu1usNDq8u1uXy5yU2FuW0gJxHq9oOxv91XZWad3sjgb3ZzLiGLIA8HNjzbCScLrdPL+vdfEt7/AH2ckvCqXF3xevJJ3B8ah6qUbczyCowMm7eHM1cyqe5qx5YXuoj1iGVATDEVbtXzbfBvAcjKfPisqyU+xMWkbN77U2lq+mATPx5pV6NMQwSQVCrchg2l1sZNeD7xBJJWiyEhgcszXyZ6zFjyjh0r83PezkNtguhbWGb5rT9OcSEx67b7RaZHMnnzZF7c/iLx2fGoJTXjG4xnw7ANPNTgIZEcCiVSz6MeC3ov4IuuX+Dv5I9BNrD49GIGhg1kVcQqjCbjzS8I/1ZJDgEGzamcySEo39fQ+aBSK0niP/e3jmfPxWR+3XcJgJd616u0ySHAu4Mb4eWkJSvfwMurxVRTwQrJMqx9WkkO7dyU6fGVMTkEpYJpv8IZXBEblVHT+/DRprOcS8jCRiXx5ZhmFkkOC+LiiJ36FHFTn1KSQ40Gj4kTqb15E56TJpY6OQSw19rw7UPKbK7LGTq+2BJRBpEL91JJ//qEu1JrlfWItq6QkwR/PFNUQbQ4ZFnmzbUnKTCaCPSw56PhTcy6cNjGyRGf558jZMN6nPv2BUB3/DjRY8Zy9d13MWbdPmErK5Ik8eHwxlR3tSNPb+SZ346i01+/ONcZdHx9+GuG/zmcgwlKiemuAV1ZM3gNL7d+uezaFghVjiRJ9KzRk7VD1vLsA89ir7YnVZfKO3vfYez6sZxKUVoZkBoJ/36gbLd6DGq0s1zQ5aF6c6WyIcDeuUrFwxLILTDwymplammzQDcmdahcU0v/y8NRy6whShn5neeTWXYg1sIRCcJ/HPwRLhQWMxnwBbhUt2w85hY6DOr0UbY3zoS8ks1s2x6RxE+7owGY3r0OD5SiiN79MOXnk/Ttt0Q+OIDsf/8FwKFdW0LW/o7vKy9j43bbQan7VtPLkek96gDw055oTsZnlOnxhVuJBLGqcg+GQV8p2+f/gnMbiv3SP45dZvcFZR74u4MaldtdK21AAAFffkGNX5ZgW78+yDJpS3/jYv/+ZG7aZNa74m4OWr4a2xyVBGeuZPLFVuUO1q74XQxdO5QfTvyAwWQgwCmAeT3nMafHHGq6Vu6LTqH82NrY8ljjx1g3dB2Daim9wM6knmHc+nF8tP8jcv98Bgx54OIPPczT59TqdHpBSRSR4fepSpXTYvrsrwgupeaitVHxyYgmVaKRfN9G1RjSTLnonrXuNHFp5beeWxDuKvkC/PW6st14FDQaZtl4yoMkwYOfgcYRshNga/Hft1Oy83lppTJbpEUNd6Z1K99+h9k7dhA5cBDJX89Bzs9H7eOD/xefE/Tjj9iGmK+P8+OdQqjj44TRJPNa2AmMJjETwpxEgliVNRxyvR/ixleg4N4XDJk6PbPWnwGgb2g1utX3MWeEt+XQsiU1V63EZ+ZMJHt7jEnJxD/3PLFPPEFBnPmKMLQK9uDp7sodrIV7jvHk5ueYunUqcdlxqFVqpjSZQtjgMDr6dzRbDELV5uPgw/sd32dJvyXUdquNSTbxy5lfGGyIZJu9PTz4OdhV7P59xWajgaHfKX0R0y/B5uL1STsUk8aPu5WCP8/2rEMdX2dzRmlV3h4UirezLTkFRmauOo5JXGAJlmY0QNgU0OcqN7j6f2LpiMqPWyD0eEPZPrQIYvYU62X/W3OCpKx8nGzVfDm6GWqb8rmU1yckEvfMM8ROeQL9pUugVuPx6GRCNmzApV8/s7eg0KpVvD9UmQlxLC6DX8JjzHq+qk4kiFWZJEG/j8FGCxmXYNe9y8Z//lcESVn5OGhteHNg8UoVm4OkVuM5eRK11v2JU9euAOTs2EnkgIGkLFiArC95S4rieKJzMD7+B7Gr+Sm7r/4NQKtqrVg9aDXPNH8GO3UlXfclWJVmPs1YMWAF0xtOQivLXFWreaaaNy9c+YvE3ERLh1d+vOspa6oBDv8MEZvvurtOb2TmqmPIMoRWd2FKZ/Pd7bZGbg5aZhdeYO25mMKv+y9ZOCKhytv1OcQfUraHzAP7sp2aaPVaT4Hqha2I/nwWDPl33f3fs4n8dToBUIoDBno4mDtCZFkmfdUqIgcMIGvLVgAc2rQh5PcwfGfMwMap/NZvt67pwZjCQoGfbD5HQqau3M5d1YgEsarzrKU0bAXY/RWkXLzjrifjM1i8NxqAZ3vUsYpKeBp/fwLmfYv/11+h9vFB1ulI/PQzooaPIO/EiTI9V2RGJFP+nkSeyyokm3xMBkfGBM9kYe+FhLhWrQtNwfI0NhoejzxCWNwV2uQrPU23xGxh8O+DWX52OSbZZOEIy0mbJyFY6Z3KH8/cdS3P13+f52JSDmqVxMcjmqAppzvv1qRnQ1+GPVBYNn7DGWJTxVRTwULSYmDHp8p226cgpItl47EElQ0M+hokG0iOuGt/1wKDiffWKcUB29fyLPo7NqeCuDhiH32UK6+/gSkrCxt3d6p/8jFBi37CtnZts5//dl7pVx9PRy3Z+Qbe+fOURWKoCqref0fhVp1eBNdAMBbAxpdvW7DGZJJ57feTmGSo6+vEZCvqFyZJEi69exOyYT3u48eDJJEfEUH0mLEkfvElpoKCUh3faDKy6OQiRv4xkuNJxwHwkjuRc/FFNob7k2+oIhfignU58yec+YMgg4EfWr/JBx0/wN3WnWx9NrP2zWLSpknEZlaBYiQqlVLxUOukrOXZ+eltdzsRl8F3OyIBeKprLUKrV93iUW8NCMXXxZbcAiMzVh0TVU0Fy9j6Fhjzwbk6dH/d0tFYTrXGSt9agJ2fQdK52+62eG80kck5qCR4c2BDs07plE0mUpf8QuSgweTs2QuAS//+hKxfh+vAgWafTno3bg5aXh/QAIANJ67yz9kEi8VSmYkEUQCtA/QtLLl8YQucXX/LLssOxHIsVrkzP2tIY6u8827j5ES1118jeMVybOvUBqORlO++I3rESPJO3d9dpqiMKCZsmsBnhz6jwFSAv5M/C3ovYOGDH6ORHIlLy2P+9juPugqCWeSlw/qXlO1aPZCajmFgrYGsHbKWwbUGA3A48TDD/xzOinMrKn8C4F4DOj6vbIfPV6q63qDAYGLGqmMYTTJ1fJyY1t0yd76thauDhg+HNQEgPDKVjSdL3upIEEolZi+cClO2e74N2srbZqZYur4C7jXBpIc/poPp5hvPSVn5fLX1PADj29agfjXzrTXPj4wiZvzDJLz/PnJuLmpvbwK+nYv/55+h9vAw23lLYkgzfzrU9gTgjd9PkVtgsHBElY/1XeULllH/wesFazbdXLAmOTufjzadBWD4AwG0rmkdbxB3Yt+4McGrV+M5ZQqoVMpo4ugxJM35BrmYo4lGk5GfT/3MyD9HcixJqRY2ut5oVg9aTRu/NoR4O/FoR2Va6bxtF8U0LaF8bXkTsq8qFfAGfFHUZNndzp1ZHWcxr+c8fBx8yDPk8V74ezyx5Qmu5lTyJKDdNGUmhEkPW26uCLh4bzRnr2ahkuDjEU2wVdtYJkYr0q2+D70a+gLw4caz5BuM93iFIJQRk0m5zgClJ3PjkZaNxxpo7GHgl8p2bDgcXnTT05/9dY6sfAOu9hqe71nXLCHIRiMpCxcSNWQIeYcPA+A6Yjgh69fh3L27Wc55vyRJYtaQxmjVKuLT84qSZ6HsiARRUNxUsCb2poI1H248S0aeHhc7Nf/rX9+CQRafSqvF54XnCV72G9qQEDAYSJ47l6jRY9Cdu/30jWuiMqKYuGkinx78lHxjPtUdq7Og9wJeb/s6jprrdzmf6V4bXxdb8g0mZq0/be5vSRAUsfuVgiygVMBzr3HLLh39O7Jm0BoGhgwEYO+VvQxdO5TfL/xeeUcTNfbXC9ac+QOidwOQkatnzj8XAHi4bQ2al3O/MGv2Sr/6qFUSl1JzWbJXVAQUysnxZXDlqLLd90NlmrgAIV2h6Thle8tbkHkFUOo/LD+oLBd4sXdd3B1L33z+v/Tx8VyaMJHETz5FLihA4+9P0I8LqT5rFjYu1lkZu6aXI093U2aDLNgVxenLmRaOqHIRf5XCdZ61oMOzynZhwZr9UamsOqS0jpjZtz5eTrYWDLDk7Js0oWbYGjwenaysTTxzhqgRI0meNw/ZcPOUBFmWWXZ2GSP/HMnRJOWf16i6o1gzeA1t/NrccmxHWzWv9lfmwW8+lcDO80nm/4aEqk2Wr4+O+TVVKuDdgautKx90+oAvu32Jh50H2fps3tj9BtP/mU5yXnI5BVzOGg2HgFbK9uZXwWRi7rYLZOTpcbJVFzVaFhS1vJ14qE0QAHP+uUB6bunWawvCPeVnw9Z3lO1GIyCwtWXjsTZ93gcHT8jPhC1vIssyb/9xClmGer7OjGsdVKank2WZjLVriRw8hNyDBwFwGzuGkD/W4ti+fZmeyxye6BJCLW9HjCaZV8NOiNY9ZUgkiMLNOr4ArkFgLMC0YQZvhCmVQJsGujG2jN+YyovK1hbfGTOo8euvaGvUAL2epK++JmbCRPTx8QCk6lJ55p9neH/f++Qb8/Fz9OOH3j/wRrs3bho1/K9BTasXTbl9+49TFIiCNYI5RWyGS4W9snq9p1TAu4ceQT0IGxxGrxq9ANgWt40ha4fwd8zf5ozUMiQJ+nygbF85SsrexSzaHQ3A1K618KxgN7jKw/QedXC2VZORp+frvy9YOhyhstv1hTI9Xn3DiL9wnYMH9HpX2T6xku07/+VgTBqgFKYpy56HxvR04l94gcsvv4IpOxsbLy8Cv5uP31tvoXKsGGtCbdU2Rb0Rj8amF420CqUnEkThZloH6DsbANXFvwlO/heVBLMGN8JGZbmqVWXB4YHm1Pw9DPeHHwYg79AhIocM5cCvXzJs7TC2x20HYEDIANYMWkNbv7b3PKYkSbwzKBSVBBeTcli0J8qs34NQhZmMsPVtZbtWjxKVhPew8+CzLp/xceePcdG6kJGfwXPbnuO9ve+hM1SyPlKBrZWRRMDm3/ewMebi62LL5A7WU3nZmng62fJU4TStJeHRRCfnWDgiodJKvwR75ijbHaYrjeKFWzUdC971ARnttvcA6BPqS4faXmV2ipw9e4gcPISsjZsAcOrRg5A/1uLUpeK1Gmkb4smw5krLjy+3RqDTi/XUZUEkiMKt6j+IPqQHAG9oljC5tS+NAypHSXiVvT3VXnuVwO+/w8bTA1NWFk7vfcfI1Yl4yo582OlDZneajZPWqdjHbODnwsNtlXVgX209T6Jo3CqYw7HfIOkMIN3XnXdJkuhXsx9hg8OKpkyviFjB2PVjiUiLKNNQLa7n25hsbHEzJDPFZj0v9qqHvVYUprmTSR2C8XezR2+UiwqSCUKZ23KtrYXf9eUswq1UNtDjTQDamw7TQX2W1/o3LJNDm/LzSZg9m0uTH8WQkIDk4IDfrPcI+GaO1VQovR8v9K6L1kZFQmZ+Ub9uoXREgijcSpL40WUq+bKaACmZF523WDqiMpfQxJ9ZT3lxtKYyKtr9uMz8pU50zw2+r+O90KseHo5acgqMzN4oLrCEMqbPg38Lp042GQV+Te77UD4OPnzf63uee+A51JKaC+kXGLtuLL+d/a3SFLCRXQP502EIAFM16xheR/yruxs7jQ0z+9YDYOPJqxyMTrVwREKlcykcTq1Rtnu+Ldpa3EOcT1cOy0q10k/cVhPkYV/qY+ZHRRE9egypPy8GwL5pU0J+D8NtxAiL9jUsCwHuDowrXE/97baLZOr0Fo6o4hP/NYVbJGbp+PKQkZ+MfQGwPzhP6btWCciyzPKzyxm9bjRHDJF8NEbNhUc6IWk0yJfiiR47lpSFC5FNJVtL6OqgYWYf5QIr7Eg8Ry6lmSN8oara/z1kxitVhru9VurDqSQVjzZ+lMX9FhPgFECBqYAP9n3A9H+nk6ar+L+7O84n81pSL5JkF+zIx+bfWZYOyeoNbFKdpoUzRWatP1NpbhYIVsBkgo0vK9vVH4DGoywbTwUwe+M5ZheMAaB69qnb9qcuiYw/1xE9fAT5Z8+CSoXXM09T49df0AZVzNoSt/N099o4aG1Iz9WzYEfkvV8g3JVIEIVbfPvvRfL0RpZrhiJrHUGXAXvnWjqsUsvR5zBzx0xm7ZtFvjEffyd/FvVbzMBXvyd4+TKlHYZeT+InnxL7+BQMqSW7iz6qZSCh1ZVy0J9vqWRT9gTLyUuDnZ8p260eu21bi/vV2LsxKweuZEDIAAC2xW5jxB8j2H9lf5mdo7wZTTKzN5whGwd+d5ukPHhsKVw+YtnArJxKJfHag8o0tqOx6aw7fsXCEQmVhmhrUSLhkSmsP3GFA3J9rvoWrgn8+10wlrwZvCkvjytvvMHlGTMw5eai9vWlxuKf8Z42DUmtLuPILcvLyZZHOyprzRfsiiI5O9/CEVVs4q9UuEl8eh5L910CYGy35khtn1KeCP8WclIsGFnpnEs9x5h1Y9gUrSzI7hvcl5UDV9LMpxkAdg0bUnPVStxGKXc2c3bvJmroMHILm8UWh0ol8UIvZUrIzvPJYpqWUDZ2fq7cpNE6Q6eXyvzwTlonZneazQcdP8BB7UBiXiKPb3mc749/j0mueFV5w47Ec/ZqFgBthz8HPqHKE5teVdqECHfUuqYHfUJ9Afho01lR7EEovZvaWgyHoFtbRgnXyfL1dcDNg9zwGfI+IEHyOWUdegnkX7xI9KjRpK9cBYBj507U/D0Mh5Ytyzpsq/F45xDcHDTkFhiZ+6+oylwaIkEUbjLn7/MUGE34ONvySLtgaPc02LlCQTbs+crS4d2XsPNhPLThIaIzo9GoNLze5nU+7vwxzlrnm/ZTOTjg9+47+H/xOSoHBwwJCcQ8MoGUnxYVe7pV9/o+RdO0vtgqRhGFUsqIg33fKdsdnwVHT7OdamCtgawauIpQz1BMsok5R+Yw7e9ppOsqzvRynd7IZ3+dA2Bws+o0DvJQ+oqB0h7kzJ8WjK5ieLlvfdQqibi0PFHsQSi9vXML21rYQc93LB2N1dtxPpkjl5T33Ff61kfl11hZdw6wbbayHr0Y0sN+J2rESPLPnwcbG3xmvETg/Pmo3d3NFbpVcLHT8FTXWgD8Gn6JuLRcC0dUcYkEUSgSlZzDykNxADzTvTZ2Ghuwd4N2zyg77PseshIsGGHJ5BnyeH3X67y5582iKaVL+i9hdP3Rd12Q7dKvH8GrVmFbpw4YDCR+9BHx06djzMy85zklSeL5wlHE3RdSCI+suKOughX4d7ZS9c+pGlwbzTejQJdAFvdbzOh6owHYFb+LUetGcSLphNnPXRZ+2h3NlQwdWhsVL/VW1gRTqxvU6aNsb337vqZpVSUh3k6ML6zKPOefC6TmFFg4IqHC0mVAeOHylDZPirYW9yDLMl8W3lhuX8uTNiGFNwS7vQoqjbIOff8Pdz2GKS+Py6/8jyv/+x9yXh7q6n7U+GUJno8+ilRFpvY+0i6Yai52FBhNfLn1vKXDqbCqxm+LUCxfbo3AaJIJcLdndKsbFi63fRLsPcCQpzS5rQAiMyIZt34cay+uBaB7YHdWDFxBqGdosV5vG1KT4BXLcR2iVELM2rKVqOEj0J0+fc/XdqnrzQNBboCyFlEUexDuS8JpZe0cQNdXyq3qn9ZGy+ttX2d2p9nYq+25knOFRzY9wrKzy6z6dzk1p4BvC6cUPdKuBoEeDtef7PUuIEHqxeuVFIU7mt6jDs52arJ0Bub8Iy6whPu073slSdQ4Qvvplo7G6t04evhsjzrXn3APhlaPKts7P7tj0cCC2Fiix44j4/ffAXDq3p2QNWtwaN7cnGFbHTuNDdMLf35rDsdxPiHLwhFVTCJBFAA4dzWLP45dBpQ3Jq36hl8NW2fo+JyyfXAhZMRbIMLi2xS9iTHrxnAh/QJqSc2MljP4stuXuGhdSnQclb09frM/wG/We0haLfrYWKLHjCVtxYq7XihLksQLvZTRi/1Rqey9KEYRhfvw97sgm8CzNjR/uNxPPyBkAEv7LyXYJRiDycD7+97nlZ2vkKu3zik73/xzgax8Ay52ap7uXvvmJ33qQ6hys4cdn4BJrK27Gw9HLdO6KT/DpfsukZglersKJaTLhL3fKNutHzPr9PjK4I6jh9d0egm0TqBLh923LvfJ3r6dqGtVSm1s8Jk5k4C532Dj5lYe4VudkS0DCPZ0wCTDZ3+J5T73QySIAgCf/XUOWYYQb0eGNve/dYdWj4OjDxgLYOen5R9gMRhMBj4/+Dkzts8gz5CHr4MvP/X9iUdCH7nvHj+SJOE2YgTBy5ehCQpCLijg6ptvceW11zHl37lCVofanrQOVprOilFEocRi9kDERmW7x1tgY5lqc7Xda7NswDL6BistbzZEbWDs+rFEpltXCfHL6XksCY8GYFq32rg5aG/dqfMM5XNyBJz+vfyCq6AeaVcDdwcN+QYTC3ZGWTocoaI58IOSzGgcri9TEe7ojqOH1zh5Q/vCn2P4PMhUqgzLJhNJc+cS++RUTJmZ2Hh6EvTTj3hOnlThexuWhsZGxQuFyww2nbrKsdiKs5beWogEUeBYbDp/nVbWFr7Qqy5qm9v8WmgdoNOLyvbhJZAWXX4BFkO6Lp2pW6fy06mfAGjj1+amKqWlZdegATVXr8K5Vy8AMtasIebhR9An3H5N5o1rEQ/GpLHzfHKZxCFUAbIMW95StgNaQYOBFg3HUePIx50/5pXWr6BWqZXp2xvG8e+lfy0a142+3xGJ3ijj7WzLhPbBt9/JNxTqK+082PGp0ptNuCMHrZrHOoUA8Et4jFiLKBRffjbsKRw9bDlZSW6EO7rn6OE17aaBg5ey3Gf7RxgzM4l7ahrJc74BWca+aVNqrl6FY+vW5Ri99RrQ2I+GfsrMsU82n7NwNBWPSBAFPi2s+tfAz4X+jfzuvGOLieDiDyY9bP+kfIIrhnOp5xizfgzhV8IBmBg6kfk95+NuV7bVumycnfH/+iu8X3gBJAnd8eNEDR9xx1YY7Wp50jZEGUX8YqsYRRSKKWoHxBX2Iez5NljBXWBJkniowUMs6rsIHwcfcvQ5TP93OvOPzbd4K4ykrHx+26+05pnSKUQprnUnXWYqnxNPw9l15RBdxfZwuxq42KnJLTDy4y4xiigU04EfIC9VqVza4VlLR2P1tkckFY0ePtez7p13tHUumgmh+2cpUUOHkL1tGwBuY0YTtGQxmmrVzB1uhaFSSczoo4wi7rqQzJ4L4kZ9SYgEsYrbF5lSNLr1Uu+6qFR3uRjV2EHnwj5sx36DZMv3mNkYtZHxG8YTnx2PnY0dH3X6iBdbvohaZZ4peZIk4TXlcQK/m4/K2RljcjIxEyaStmz5bfd/vvDN/sildLZFJJklJqGS2fW58jmoHQR3tGws/9HUuynLByynmbcyMj/36Fxe3PYiOfoci8X04+4o8g0m3Bw0jGsTdPed/ZpCXWW6LDs+Fn0R78HFTsPEDkrj6Z/3RJORp7dwRILVK8iBPXOU7ZaTwcnHsvFYOWX0UCkE1aG2J61retz9BS0nkZEUQPRf7ujjryBptfh98AF+b7+NSnubqfVVXNd63rQKVgYLPtp8TtyoLwGRIFZhsiwXjR42D3Kje/1ivJE3Gw9uQSAbYfuHZo7wzgwmA58d/IyZO2aiM+qKWlj0D+lfLud36tyZXC93bgAAIABJREFUmitXoK1dC/R6rr79NlfefAtTwc3TsNqEeNKxthcAX4i1iMK9xB2CyG3K9rUp3VbGy96LhX0WMrzOcAC2XtrK+A3jic2MLfdYMnL1LNkbA8Ck9jVxtC3GjaHOhaOIV09AxCYzRlc5TGofjKPWhqx8A4v3RFs6HMHaHVgIuSli9LCYtkckcTT22trDu4weoqw3TJwzj8t/m5CNKjSORmr88BVuw4aWR6gVkiRJzOxbH1CWU20XN+qLTSSIVdiO88kciE4DYEbvesVb0KzWQpeXle0TqyDxjBkjvL2M/Aymbp3KolOLAGW94bIHl1Hfo365xqENDiZ42XKcevYAIH3FCi5NmIg+MfGm/Z7vpSw4Px6Xwd9nEm85jiAUuTZ6WK0x1O5p2VjuQmuj5e32b/NG2zdQS2oupF9gzPox7Lm8p1zj+HlvNNn5Bpxs1Uy809rD/wpocf1nu/0jMYp4D+6OWsa3U/oiLtwdRXa+6CMp3EFBLuz5WtluMRGcxXTHuynJ6KEx+//snXd8VGX2/993SnrvkEIg9J7QQYqAIJa1ohTpomJb3XXXXX/63V1XsawdBJSOIM2yllWwI0rvvZMe0nubdn9/PDMJImlkJndmct+vF695XiRz72eX8Zl7nnPO55ST/tjj5L/3HgA+bSzEj8vFu+jbFtHqygyID2FYR9HXufin8wqrcR3UALEV8651ZtiQDqEMtWa5GkXvSRCSAMjw43zHiKuDi8UXmfK/Kb/rNwzyUsbKWevnS8w77xD2uHAXqzx4kOS7J1J59FjN7/RrF8LIzqJJX3U0VamTnFO1fXHX/ckpeg8b4p4u97Bs/DJCvEIoMZQw77t5rD6+ukU+4+XVJlb8Kvri7hvcjkAffePfbMsiZh6Ec985QJ17MXd4B7z0GooqjKzblaK0HBVnZf9KKM8FrYeaPWwEjc0eGtIzSJkyhbLvvwcgeMoU4v7vfnSeFti3EioKWkSvK/PQyAQAdl8s4EBqocJqXAM1QGylHEgtZM9Fsan8bmZYQ2h1MOrvYn3y8xbLIu7M3MnU/00ltTQVD40HLw9/2aH9ho1F0mgIf/hhYhYtQuPriyknh5Rp0yjZsrXmd2yOpieySth6/OrOpyqtnF/eFK8hCdD9NmW1NIF+kf3YeMtGuod2xyJbeG3fazz363MYzY7tV/twdypFFUY8dRrmXNe+aW+OGwTtR4q1mkVskDA/TyYPFP2dS7dfoMqozpFUuQJjZe18vqQZENBWWT1OTmOzhxV795I8cSLVZ86ATkfUP/9J1P89hzTkAfAMAGM57FnaktJdkus6htEzWjiaLlGziI1CDRBbKe9tE/+B9IwOYGjCNQyw7XmnNYtIbUO6A9l4aiPzvptHqbGUUK9QVt64kps73Ozw+zYF/9HXE79pI/rYWOSqKjKeeIK8JUuQZZm+sUGMsfZ4vqU6mqpcSWEKHN0s1tc9AZp6nDidkCjfKFbfuJpbOogxEp+d/4y5386lsMoxJ7VVRjNLt4tZjJMGxBLu79n0i9hK5dP31vZ9qtTJgyMS8NBqyCsz1LjGqqjUsH8VlGWL7OF1TyqtxulpTPawcPNmUmbPwVxYiDYwkLjlywmedK/4oVcgDJgj1ruXCHMglTqRJIl5I0Uy5JsT2ZzLKVVYkfOjBoitkPO5ZTVzDx8ckXBtw1Q1Whj6qFgf2QQlmXZUWIvJYmL+7vm8sPsFzLKZLsFd2HDLBnqH93bI/ZqLZ0IC8Zs24t2/HwC5b71N5l+fxlJdXWNffepSqdoorfJbdrwjjJ8CokUJtwvipfNi/nXzeTzxcQD2Z+9n6ldTuVB8we73+mh/Ojml1eg0Eg9YS4eaTPwwaDdMrLe9aj9xbkpUoBcT+8cA8N62C1Sb1CyiihVjFfzyllgnToPAaGX1ODkNZQ9lk4lL8+dz6bn/A6MRz04dif9oM76DrphvOGgeaD3FSJEDa1pKvstyY88o4kN9ALGHqdSPGiC2Qpb+fAFZhrgQHyb0bEYTeZ/J4Bsu5iLuWmw/gVZKDCU88v0jrD+1HoDRsaNZM2ENUb7O3fiuCw4mbsUKAu8QzmIlX3xB6sxZdPM21TRK27IfKiqUZsOBD8R66GPCCMpFkSSJub3n8vrI1/HUepJWmsZ9X91X0zNsD4xmC0usFRB3JkUTHeR97RezzUVM3QHJv9hBnXvz0MgEtBqJSyVVfLw/Q2k5Ks7CgTVQdgk0ejV72Ai2n82rM3toLisjbd7DFK4R3wl+o0bRbv16PGJjf38h/0hIvE+sdywAk+H3v6NSg1Yj8cAIcaD430MZZBZVKqzIuVEDxFZGTkkVnxwQX+xzh7dHp23GR0DvDQMfFOv9q6CquPkCraSViAdLmyvinJ5zePP6N/HR+9jtHo5E4+FBm/kvEvHUn0GShHnNxHt4yDqm7ddz+RzLsN//XyouzK5FYK4G7xBImq60GrswLn4cq25cRZh3GKWGUh769iE2n9lsl2t/fiiT9MJKNBLMG9XE/ukraT8SYgeJ9bZXmi/OzYkN8eGORJEdWvTTOYxmi8KKVBTHVF3bP504FYKuEsio/AbbAfHgDiG/yR4as7JImXof5du3AxB6/xxi3l2I1s+v7osNexwkLZRkwNFNDtXtDtyZFE2YnydGs8zyXy4qLcepcWiAKEmSnyRJ70iSlCVJUqUkSfskSfpDI973T0mS5Kv8ueRIva2BlTuSMZgthPp6MLG/HTbyAXNA7wPVJSJItAOHcg4x5aspXCy+iF6j58XrXuSJfk+gkVzrPEOSJELvv5+YBe8geXtjzMwk4m+PcHt1MqBmEVWAyiIxNwxg8MPg4ausHjvSM6wn629eT5fgLphlM8/vfJ5X976K2XLtpYkWi8yin4T78s2929I+rJn/f0lSbRbx4s+Qar9Mp7vy8KgENBKkF1by2SHHtBaouBBHN0NpJmh0wn1ZpV5OZpWw/WweAA+M6FDz91UnTpB87ySqT58GnY42L/ybiKeeQtI20I8eHA89xUxafnkLLOqhTX146bU1pmbr96RSVKFmXevC0U/cnwJTgWeBm4ETwKeSJDV2mvkNwJDL/rTMFHQ3pbTKyFqrRfmMofF46e1ghOFzWdZj15JmlzhsTd7KnK1zKKouItgzmGXjlvGHhAbPFJwa/7FjiV+3Fl1kJJaKCh7YsoibLu7gyyNZZKglDq2bvUvBUAoefjDwfqXV2J0o3yjWTFjDqNhRAHxw4gP++OMfqTBWXNP1th6/xPlcYcbw8Khr7D28koQxEC16hmv6qFTqpEO4H7f0Fg6Vi348h9miGm61WmQZdr4r1j3uhOB2yupxAZZtF1mrhHBfRnUWxnWlP/5I8n3TMOXkoPHzI+799wi6++7GX/S6J8Rr/tnaUUkqdTJ1cBz+njoqDGbW7FTH9tSFwwJEaxA4FrhfluXlsiz/AMwAdgKvN/Iy+2RZ3nXZnwOO0tsaWL8nldIqE956LdMG23EjH/ywKHEozYRjH13TJWRZZtWxVTy17SkMFgPtAtqx7qZ1JEUm2U+ngnh170785k149eiBJFt47PAnTD/6JSt+Vu2WWy2Gitre3QFzwDtYWT0Owkfvw1uj3mJmj5kAbEvfxqyts8irzGvSdWRZZqF1duvYbhF0axNgH4GSVDuz7cwWyDtnn+u6MY9cL0p7L+SV89XRLIXVqCjGhR8h54RYD3lYWS0uQHZJFZ8fFi0+9w/vgEYjUfDBWtIfeRS5ogJ927bEr/8Q36FDm3bhyB7Q+Uax/uUNdWxPAwR46ZlqfQZetSOZSoNquHU1HJlBvAMoBj6z/YUsvP1XA10lSeruwHurXEG1yVxTbz1pYCzBvnY0wghuBz1uF+tf32ny5mS2mJm/ez6v7xfnBokRiaydsJbYAPfqZdBHRNBuzWr8Ror5a/ec/ZGoBfMpKlbtqVslB9ZARb5woRv8iNJqHIpWo+XP/f/Mc4OfQyNpOJF/gvu+uq9JDqc/ncnleGYJUBug2I2ut0BQHCDDbvsbbrkbXaL8Gdc9EkDt42nN7FgoXttdB20TldXiAqzakYzRLBPq68HtvaO4NH8+2S++CBYLXr16Eb9xA56dOl3bxW3lvZkH4eI2+4l2U2YPi8dDp6Gg3MCmfWlKy3FKHBkg9gROyLJ8ZUH0kct+3hAnJUkyW3sYl0qSFGFfia2Hzw5lkl1SjVYjNX2odGMYKqztyT0JZ79t9NsqjBU88eMTbDi9AYDx8eNZOm4pQV5B9tfoBGh8fYl5dyE+d08E4Lq0g5y8bybmYtWwplVhMojRFiBc6PwjldXTQtzT5R4WjF6At86bjLIMpn01jQPZjSsMWWwdbjysYyiJcXbOtmq0wjIe4NCHUFFg3+u7IbbvkUNpRexPccy8SxUnJvsEnP9erIe49wGXPSivNrHO2uIzKymKvD89WetUOnYM7dasRhcefu03iBtUO7Zn+xvNlev2RAR4cXc/Mbbn/Z8vqIZbV8GRAWIocLVv2YLLfl4X54FngFmIPsRFwCRglyRJdT4ZSJJUVN8fIPCa/pe4OBaLzPs/i5P6W3u3ISbYAU6gbfsKR0CoffBtgLzKPGZvnc1P6T8BMKvnLF4d8Sqe2msYeu1CSDodcf/+F0dungZA0NljXJw8BUO6ahvfajiyUbjOSVrhQteKGBEzgpXjVxLiFUKJoYS538xla/LWet9zLKOYPRfFV8dD1zr3sCES7wPPADBW2M1wy50Z2D6EntGizHeFmkVsfexaJF5DEmrLG1XqZPO+NEqqTISbKxi/8t+U/fADACEzZxLz9ttovJsxrseGbcTIxW2Qsb/513NzHhjeAY0EGUWVfHlENdy6Ekeb1NRXa1jnz2RZ/kCW5ZdkWd4iy/IPsiz/G7gTaA+oR1VN5PtTOZzLKQPgQUc9XEHtg27y9gY3pwtFF5j6v6kczz+ORtLw7KBn+VO/P7mcU+m1IkkSw559glcHTMWo0WK8cIHkSZOoPHZcaWkqjuZyY4eedwkXulZGj7AerLtpHfEB8RgsBp7a9hSrj69GrqM8fcWvIgDpHOnHdR3DHCPKK6DWcGvP++pMsQaQpNpqlK+PZZFeeG3GQyouSFmOOOQC0XuoaR3f29eK2SKz/NeLRJbns3DHYkzHjoIkEfnss0T+7emGnUobS8exENVLrNUsYoPEh/kyoVcbAJb8dKHO75/WiiP/q87n6llC29CXJtXwyLL8LZCFcDOt63eC6vuD6IlsdbxnHSo9snO4/YwdrkbCGIi0Vg7/WncW8VDOIaZ9PY3M8ky8dd4sGL2Ae7ve6zhdTkpMsA8BN9/MM0MfoMLDB3NeHinTplG2Te0fcGsubhOl2NCqS7Ni/GP4YMIHJEaI3qXX9r3GK3tf+d0YjJzSKr44LE53Zw9rjyRJjhM16EGQNFCaBSf+67j7uAk392pLhL8nFhnVDbA1sXcZmA3CWKvPFKXVOD1bj1/C48JZ3vh5IQH5WUgeHkS//RYh9021740kqTaLeOp/UKCO0mqIedakyensUn48naOwGufCkQHicaCbJP0uJWQ93uDYNVxTA6iFwk1gX3IB+6z9IQ+O7NDAbzcTSartRTz5ORT8vuzop7SfmPvNXEoMJYR4hbDyxpWMiBnhWF1OzNzhHTgWlsATwx/BHBGFXFlJ2sOPUPSp+nDqtuxaIl7jhojS7FZMkFcQS8ctZVy7cQCsO7mOp7Y9RbW5uuZ31u1KxWiWCfbRc7t1SLvjBMVB99vEeudC1Q2wATx0GqYPEW6A6/ekUl5tUliRisMxVooAEaD/HPBwQMuKm/Hjui949ZfFhFSXogkIIG7lCgLGjXPMzbrdBoGxgAx7ljrmHm5Ez+hAhncSVSlLflID6stxZID4KRAE3HrF308HTsuyfKIpF5MkaRwQCaiTjJvAkm3iA98nJpAhHepr+7QTPe+EgBiQLbVldFY+OfsJT/z4BFXmKmL9Y1k7YS09Qns4XpMT0zM6kKEJoaT5R/LOXX/Ds3s3MJvJ+vvfyV+2TC15cDcKLohRCgCD5ymrxUnw1Hryn5H/YXp3Ud75Xep3PPTtQ5QaSqkymlm3W2Smpg5qZ5/ZrQ1hc5TNOgwpvzr+fi7OlEHt8NRpKK0ysVl1A3R/jmwU7ssaPQycq7Qap+fQivVM/+xtfEzVWMIjiP9wHT79+jnuhlodDLDO1D24FqpLHXcvN+HBESKLuCe5gGMZrbLQ8Ko4MkD8CvgRWC5J0mxJkq6XJGkVcB3wF9svSZL0kyRJv3kKliTpoCRJT0qSdJMkSTdIkvQP4GPgHPDbqEOlTs7nlvHdyWxA9B46tDTLhlZfOw/p4Fooz0eWZd4/8j7/2PEPzLKZ7qHd+WDCB243xuJaeWCEyOx+k2Wi4uWF+AweDEDOa6+T8/IryBY1ae427H4fkMUJb5eblVbjNGgkDX8Z8Bee6v8UAPuy9zFryyw+3HeMvDIDOo3EtCEtNIQ7dgDEDBTrnYta5p4uTIivB3cmCTfAlTuSMVvUQy23xXLZwW+vieAfpaweJ0aWZfKXLcPz1efRyRayQqPptHkjnh3tPKLnaiRNB503VJfAofWOv5+LM6xjKJ0i/ABYvSNZWTFOhMMCROvMw9uBDcB84GugN3CnLMtfNPD2U8DDwGbgf4is4zJgkCzLRY7S7G7YPuixId6M79GCG3nSdPAMBFMl5j3vMX/3fBYcXADAkDZDWDF+BaHeLZDNdBFGdg6nS6Q/AEsPZBP7/nv43yhc4QpWrybzr08jG1TDDJenqkQcmoA4edfqlNXjhMzoMYP5181HJ+k4XXiad07+EUmfx8292xAZ4NVyQmy9oae/gvzzLXdfF2XOdfEApORX8MMptY/HbTn3HeSdEetW3D/dELLFQvZLL5HzmpjtfDgsgYr/LMIjqoWew3xCoI/V12H3EhHYq9SJJEnMHBYPwGeHM8kvq67/Da0Eh1pPybJcIsvyo7IsR8my7CXLcpIsy/+94ndGybIsXfF3k2VZ7iTLsq8syx6yLCfIsvykLMvqcKpGUlpl5OP96QBMHxyPVtMC2UMbnv4wYDYG4C9n1tbMOJzQfgLvjnkXX71vy2lxASRJ4v7hwg3wyyNZZFWYiX79NYKniOb/ki+/JG3ew1jKy5WUqdJcDq0DQynofWrdMlV+x60Jt/LO6Hfw0Hhh1ubjE7+YUb1a+ICk6y0QGAfIsGtxy97bBekY4c/IzmKG2/Jf1D4et2XnQvHaYRRENWaUdetDNhrJ/OvTNTMOt0X3YfGEx7hxUAtkDi9n0EPiteB87bxKlTq5IzGaAC8dBpOFDXvVUnlw/JgLFYX4aH865QYz3not9/Rv+VLOsr5TmBcVybdeomdoWvdpvDz8ZfRafYtrcQVu6xtNZIAnZovMyl8uImm1RD73LOF/FKY/5b/+SsrMWZgK1DMSl8Riht3viXWfScL9T6VOhscMp5PlKSwmHzS6cl4+9Di7slqw/Vyrg8HWB6xD66BSHQTfELOtIy92XSjgeKbax+N2ZB0RDswAQx5VVouTYqmsJO2RRyj58ksAvup4Ha/0n8q0EZ3QaVv4cTuiW+1savWQq0F8PHRMGhgHwAc7UzCa1ayrGiC6IRaLXGM5fntiNIE+LRuUFVQVMHvns+zxFgPv/2Ty5S/9/9JqZhxeC8INMB6AjfvSqDCYkCSJsHnziPr386DRUHX0KCmTp2BIz1BWrErTOfsNFFpdfW0nuyp1kpJfzu5TflSmPESgPpwKUwUPf/cwW5K3tJyIxGng4Q/GCti/quXu66KM6BRW08ez/JffO1iruDi7rP244V3FvD2V32AuLiZ1zv2U/7wdgAu3TGFBj9vw8/Lg3gEK+S3YjNDOfw+5Z5TR4EJMG9wOSYJLJVVsPX5JaTmKoz6xuyE/n83lYp4oR5wxtIWMHaxklWUx4+sZnCw4iRYNL+TmMyvtJFLmgRbV4YpMGhCLh9UN8LNDmTV/HzxxIjEL3kHy9MSQkkLK1KlUn1f7olwK2wluwmgI76KsFhdg9Y4UZBmivOPYeMs6EgITMFqM/HXbX9l4amPLiPAKqC0F3v0+mI0tc18XRZKkmiziF4czySmpUliRit0oyYKjH4n14IfFSCuVGow5OaRMm07lgQMgSYQ9+yz/Ch4CksSkgbH4eylUOdVpHATHi/We95TR4ELEhvgwtlskoJrVgBoguiW2D/bgDiF0jQposfteLL7I9C3TSS5JxkPjwZvXv8ltPvHih+o8ngYJ9fPk1t5tAfFvePmIC/8xY4hbvgyNnx+m7GxSpt5H5dFrGSWq0uJkn6gtzRqkjrZoiNIqI5us4xJmDI0nOqANqyespm94X2RkXtj9AsuOttAImEEPgqSB0kw4rs4mbYg7EqMJ8fXAaJZZuytFaTkq9mLvUrAYwScMet+rtBqnwpCaSsqUqVSfOQM6HW1f+w87e11PTmk1GglmDmuvnDiNFgY+KNaH1kOl6vHYELOGxgOwN7mw1Y+8UANENyM5r5yfzuQCMNP6QW8JTuSfYMbXM7hUfgkfnQ+Lxy7m+rjRMOgB8QvHPobyvBbT46rYMr6nLpWyN/m3fU8+/fsTt3oV2pAQzEVFpM6cSfnuPUrIVGkKu5eI19COamlWI9i8L52yahPeei2TBoiekEDPQN4f9z7DoocB8PaBt3nzwJuODxKD20G3P4j1zgWgziWtFy+9lqmDxL/Z2t2pVBnNCitSaTbGSti3QqwH3A/6FnQTdnKqTp0iecpUjOnpSN7exC5eRODNN/PBzmQAbugeSXSQt6IaSZwKHn5gLK910VapkyEJoXSOFKXyq1p5FlENEN2MNTtFaVbbQK+aVLmj2XdpH7O3zqawupAgzyBWjF/BwDbWOWK9JoJXIJgNah9PI+gdE0Sf2CAAVu9M/t3PvXv0oN3atejatMFSXk7a3LmU/vBjy4pUaTwVBWKwNIiTXI265daH2SLXfCnf3S/mN/3T3jpvFly/gHHtxgGw8thKnt/1PGaLg4MQm51/1mFIbUGjHBdl2uB26LUSBeUG/ntQ7Zd2eY5/KkyaNHroP1tpNU5DxYEDpEybjjkvD01gIHErluM3fDgns0pqDndtvgKK4hUIfYUjOnveE4ZpKnUiSRIzh4qs7+eHMslrxSMv1KcVN6K82sRma2nWtCHxLeKatS1tGw999xDlxnIifSJZfeNqeoT1qP0FD19h9gDiFNJscrgmV2eGdSD41mOXyL5KH49nh/bEr1uLR3w8ssFA+mOPUfxFQ6NFVRRh/yowVYFnAPSdrLQap+f7k9mkFlQA1Myluhy9Vs+rI17lzk53AvDRmY/42/a/YXRkf2DsQGibJNZ7lznuPm5CRIBXTan88l8utkwpsIrjsLWHdP8D+LfMobOzU/bLr6TOnoOltBRdeDjtPliDT2IiQI1BYEK4L0MTnGTes63MtCgVzrSg0ZeLcntiWwK99RjMFjbsSVVajmKoAaIb8cnBDEqrTXjqNExqAdesLy98yR9//CPV5mraBbRjzYQ1dAjq8PtfHDAHkKAkQwyeVqmXm3q1IdTXA5NF5sPdV9+c9G3b0m7dWjy7dQOzmcy//JWCdetaWKlKvZiNtQ9XSdPFfFCVelnxq3C/vL5LOAnhflf9Ha1Gyz+H/JMZ3WcAsCV5C4//+DiVpkrHCRs4V7ye+AzK1EHwDWEzqzmbU8bOC/kKq1G5ZjL2g81gbsBcZbU4CSXffkv6vHnIVVXoY2Npt/5DvDp3BqC40liTNReOmE5i5hPWETreINbqyIsG8fHQ1TxDr92V2mpHXqgBopsgyzJrrKVZt/VtS7Cvh0Pvt+n0Jp7Z/gxm2UzXkK6sunEVbf3aXv2XQzoINy2APe87VJc74KXXMmmg2Jw+3JOKwXT1zUkXGkq7Navx7tcPgOx/v0DekiXqib2zcPJzYW4iaWoDDJU6OZFZwq4LYs6nLcCoC0mS+HP/P/NY4mMA/JLxCw99+xClhlLHiOtxh5hdaTHCgTWOuYcb0TM6kH7txKxP1azGhdm7XLxG9IC4wcpqcQKKP/+cjCeeRDYa8eiYQLu1a/GIian5+Uf706k0mvHx0HJnv5h6rqQAtrmuydsh+7iyWlyA+wa3Q9PKR16oAaKbsON8PmdzygDh/OdIVh9fzb93/RsZmcSIRJaPX06Yd1j9b7KZ1SRvF66OKvUydZDYnHJLq9lSz+ak9fcnbtlSfEcMByD3rbfJfaMFzDtUGmaX1Zymy021VuMqdbJmZzIAnSL8uK5jA/sJIkh8oPcDPDPoGQAO5BxgztY5FFQV2F+c3hsS7xPrfSvVUvlGMG2wKJX/5nj2VUvlVZycigJhLgcw8P5WP9qicP16Mv/6NJjNePXoQbsPPkAfGVHzc4ul1rn3jsRoApQabVEXHUZDaCexthmnqdTJ5SMvVv2arKwYhVADRDfBZuwwID6YHm0DHXIPWZZZfHgxr+17DYDBbQazZOwSAjwaMUqjw2gISRBrNYvYIG2DvLmhu9ic1jTgpKXx9iZ24UL8J9wIQP7SpWS/OB/Z0jrLIpyCjAOQbnWYHfSQslpcgJIqY83sz2lDmlaaNbnrZOZfNx+tpOVkwUlmb5lNbkWu/UXaDDpK0uHsVvtf382Y0CuKEGup/IY9aUrLUWkqB9fW9k/3ukdpNYqSv2wZl/71PADe/foRt2oluuDg3/zOL+fyauZPO4U5zZVoNGJsD8CRTeIAQKVebH3w+1IKOZre+kZeqAGiG5BWUMH3J7MBx2UPZVnmzf1vsujQIgBGxYxi4ZiF+Oh9GncBzWVldkc2qvN4GsEM65fMvpRCjmfWvzlJHh5Ev/YagXfcAUDh2rVkPfccsll1LFOEy0uz4q9TVosL8MllpVl3JEY3+f23JtzK66NeR6/Rc774PDO3zCSrLMu+IkM61I4pUc1qGsRTp+We/rZS+ZRW28fjklhm+6b4AAAgAElEQVQssM+6h/WZDJ5X7wd2d2RZJuett8h57XUAfIcNI27p+2j9f99PbjOnGdg+hC5RTtpv3mcyeAaKwF91lW+QIR1C6RIp/i1b48gLNUB0A9buSsEiQ1SAF+N7RNn9+hbZwou7X2Tl8ZUA3Bh/I29c/waeWs+mXajvFND7grECDn1od53uxpCEUDpGiC/mD3Y23McjabW0efEFgqcIt8zijz8h8y9/RTY60OFR5fdUFtaWZg2Y3epLsxpClmXWWs2Ybusbjf81lmaNiRvDgtEL8NR6klqayowtM0gtsbMD3YD7xev5HyD/vH2v7YZMHRSHJEF2SXXNIaaKC3D+eyhMFmvbZ76VIVssZM9/ifwl7wHgf8NYYhYvQuPz+0Px9MIKfjhlPaR3xuyhDU8/SLK5yq9UR140gCRJNVnELw63vpEXaoDo4lQazGzYK8p3pg6KQ2/n0RYmi4nnfn2OjafFLLfbO97Oy8NfRq+5hoc4r0DoM0ms9y4Vp5QqdSJJUs3Ii/8eyqCowtDwezQaIp97jpA5ohyu5KuvSH/iSSyGht+rYicObwBTpRhO3PtepdU4PbsvFnDO2j993+C4Zl1rWPQwFo9djLfOm6zyLGZumcn5IjsGcp3GQaBVo214uEqdxIb4cH0X0ae1dlfrtYt3OWzuy+1HQHhnZbUogGw2c+kf/6Dwgw8ACLztD0S/+SYaj6ub/63bnYpFhsgAT8b1cPJRILZS+eJUOPe9slpcgNv7RrfakRdqgOjifHYog+JKIx5aDZMHNe/h6kqMZiNP//w0n5//HBC9Pv8a+i+0Gu21X3Sg1aym4II4pVSplzuSYvDz1FFltLB5X3qj3iNJEhFPPUXYY48CUPb996TPexhLpQPHAKgIZLk2cOg1UR1t0Qhsxg6JcUF26Z8eEDWApeOW4q/3J7cyl1lbZnGq4FSzrwuARgv9Z4n1wbVgqLDPdd0YW9D/y7k8zueWKaxGpUEKk+HsN2LdCkdbyCYTWc88Q9HmjwAImnQvbV56CUmnu+rvVxnNbLQe0k8eaP9DersTmgAdRom1rYxYpU68PbTcax15sX5PGmZL6zEAdPJPskp9yLLMamvp4S292xDm18SSz3qoNlfz5E9P8k2K+KKY03MOfx/4dzRSMz8yEV3FqSTA7veaqdL98fPUcVeS6Mn6YFcKlkZuTpIkEf7II0T85S8AlP/6K6lz52IuUx/QHEryL5B3RqwHzFFWiwuQU1rFlmPCpfe+Qe3sdt0+4X1YPn45wZ7BFFYXMnvrbI7kHrHPxROngdYDqopqS4lV6mRk5whigr0BWKdmEZ2ffSsAGfzbCgfmVoRsNJLx1F8o/kwciofMmEHUP/6BpKn7ueero1kUlBvQaSSmDLTvIb3DsGURz2yFIvW/yYaw/btmFFWy7UzrmYOrBoguzMG0Ik5mlQDC+c9eVJoqefyHx9mWvg2AR/s+yhP9nrDf0FdbFvHct2ofTyOYZu1pSC2oYNuZprkzhs6ZTdQ//g+Ayn37SZ09B3NJib0lqtiwncjGDISoXspqcQE27U3DZJEJ8tFzc+82dr12t9BurLxxJWHeYZQaSpn7zVz2XdrX/Av7hUP328V671KRNVapE61GYqo1+P9ofxqVBrXvyWkxVsEBUVZJ/1mgvXrWzB2xGAykP/EkpVu2ABD6wANE/O3pBp97bOY043tGERHg5XCddqHLTeAXBcjqXNdGEB/my/BOYvRSazrkUgNEF+ZDq7FDj7YB9I0Nsss1K4wVPPr9o+zI3AHAU/2f4sE+D9rl2jV0ngCBImWv9vE0TMfL5sKt3pnc5PcHT55Mm5deAo2GqiNHSJ05C3OR6iJrd0qz4eQXYm07oVWpE7NFZr11/MHEfjF46ZtRul4HCUEJrL5xNW1821BhqmDed/PYlbWr+Re2GXdkHRYjTVTq5Z7+MXhoNZRUmfjicKbSclTq4vinUFkAGh0kzVBaTYthqaoi/ZFHKftetL2EPf4Y4U82fCh+JL2IQ2niu3T6YPsd0jscrR6Spov1gTVgVo3sGmKqtYXrh9M5pBe2jtYCNUB0UYorjDVftFMGxdklu1duLGfed/PYc0nMb/v7wL8zo4cDviS0Ouhnve6hdeLUUqVeplszxD+dziXZOmupKQTdcTttX30VtFqqTpwgZcZMTAXqHCS7cnANWEzgHQw97lBajdPz46kcMopEX+wUO5aXXklcQByrblxFrH8sVeYqHv3+UX7J+KV5F40dCJHWDLE68qJBQv08uamXcNhesysZWc26Oid7reY03f4A/k5utmInLBUVpD00j/Lt2wGIeOrPhD/8cKOeqWzZwy6R/gxsH+JQnXan3wyQNFCWDaf+p7Qap2dMt0jC/T2RZWp6Tt0dNUB0UT45mE61yYKvh5bb+jZ9btiVlBhKeODbBziQcwAJif8b8n9M6TbFDkrrIHGaOKWsLISTnzvuPm7CmG6RRAeJPp4Pr9FJK/CWm4l+4w3Q6ag+fZqU6dMx5TpgoHhrxGKG/avFuu9U0LtIqZGCrN0tHq6GdwqjfZivQ+/V1q8tK8evJD4gnmpztSihT9t27ReUpNoe02Mfq0OnG4GtDeJYRgmHW+HQaacn4wBk7Bfrga3DnMZcVkbq3Aeo2CWqCiKfeYbQ+xs31qOw3FBzSD99aDv7teC0FIEx0PlGsVYruRpEr9UwyWpWs2FvWquY66oGiC6ILMuss80NS4zGz7N5fQLF1cXM/WYuR3KPICHx/LDnmdh5oj2k1o1/VG0D/L6Vjr2XG6DVSDWb00f706k2XVsfT8D4ccS88zbo9RjOnSdl+gyM2ep8smZz9lsotp4qquWlDZJ2WT/tfS1UmhXpG8mK8SvoENgBo8XIEz89wfcpzXBS7jURPAPAXC0cTVXqJSkumG5tAoBa51oVJ2KvtX86ojvEDVFWSwtgLikhdc4cKvfvB0ki6l//ImT6tEa/f9O+NKpNFvw9ddxuh0N6RbB9V13cBnnnlNXiAkwaGIdGgtzSar474f7PTWqA6ILsTS6smRvWXNesgqoC5mydw4n8E2glLS8Nf4nbO95uD5kNY7OLT90BOXayoXdj7hkQi1YjUVBuYOvxa9+c/EePJvbdhUgeHhguXiRl2nSMmWpfULOwmdN0GCVsxFXqZd3uVGQZ2gR6MaZrRIvdN9wnnBXjV9ApuBMmi4k/b/szW5K3XNvFPP2gr7XKYt9yda5rA0iSVDPy4ovDmRSWq7NZnYaKAjgmxjow4H6RIXdjzEVFpM6aTdXhI6DR0Gb+fILvvafR77dY5JpKnrv6xeDbzEN6xUgYDUHWZ8j96kF9Q0QHedfMdb3WSi5XQg0QXZB11tKsPrFB9Iy+9rlheZV5zNk6h9OFp9FJOl4d8So3d7jZXjIbpv0oCI4X6/2rWu6+LkpkQO3D9Ie7m3cC7zdiBLFLFiN5eWFMTSVl2nQM6Y2bs6hyBYUpIoMI0F8dbdEQ1SYzm/aJbOukAXHoWnhuWKh3KMvHLadrSFfMspmnf36aLy98eW0Xs/17Fyarc10bwe19RcVLtcnCR/vV/cZpOPQhmKrAwx9636u0GodiKiwkZfZsqo4fB42Gtq+8QtAdTTsU33khn5R8YVQy1c7zp1sUjRb6WQ/qD60DozoruSGmWg+5tp/NuyY/CFdCDRBdjIJyA18fFXPDmrMx5VbkMnvrbM4VnUOn0fH6qNcZFz/OXjIbh0YD/WaK9eEP1c2pEUyx/pvvulDQ7KHTvkOHEvvee0g+PhgzMkSQmKKWfjWZ/asQc8PaQJcJSqtxer4+eomCcoMomx4Yq4iGYK9glo1bRo/QHlhkC89sf4bPzn3W9AuFd66d67pnqX1FuiG+l811Xbe78XNdVRyILNdmj/pMEplxN8VUUEDqzFlUnzgJWi3Rr/2HwFtvafJ1bA7yA+KD6RTpb2+ZLUvifaDRCz+I4/9VWo3TM7JzRI0fxHo3zyKqAaKL8dH+NAxmC/5eOm7t3faarpFTkcPsrbO5WHwRD40Hb1//NqPjRttZaSPpa92cqoqFxbZKvYzoFF4zdHr97uZvTr6DBhK3bCkaX19MWVmkTJ+BITm52ddtNZgMcNA6NyxpurAPV6kXW//ZuO6RRCo4NyzQM5Cl45bSO7w3MjLP/vosH5+5hsH3tpEX576FotbhbtccbD2nyfkV/HIuT2E1KqT8CvnW/jNb24cbYsrPJ3XGTKpPnwadjug33iDgppuafJ3c0mq2HheH9JOb2eLjFPhFQLdbxVo1q2kQrUZisvVgU/Shuu9cVzVAdCEsl80NuzMxGm+Pps8Nyy7PZvbW2SSXJOOh8eCd0e8wImaEvaU2Hr/wyzYntQa+ITQaqeZL6aMD6VQZm785+SQlEbdiORo/P0zZ2aRMn0H1xYvNvm6r4NQXUJ4LkrZVzQ27Vk5dKmFfSiHQcuY09eHv4c/7N7xPUkQSAP/c+U82n9nctIt0uQl8I0C2qGY1jaBTpD+DrCMBPlDNapTH9r0bMwAieyirxUGYcnNJmTGD6rNnQa8n5q03CRh/bRVTH+1Px2SRCfTWc1OvNnZWqhA2R+b0PXDpqLJaXIB7+sei00gUVhjZcuyS0nIchhoguhA7L+Rz0VrzfC1zwy6VX2LW1lmklKTgqfVkwegFDIseZm+ZTcd2apm+B7KPK6vFBZjYPwadRqLIjpuTd58+Ikj098eUk0Pq9BlUX1CDxAbZaz1x7TIBAl3Uya4FsWUPO4T5MjQhVGE1Al+9L4vHLqZfZD8Ant/5PJtOb2r8BbR6UaYFIptsNjlApXthOxz44VQO2SXqHFzFKM+vHTPVzz2zh8bsHFEZc+48kl5PzNtv4z927DVdy2KR2bBXVO7cmRSNl77ph/ROSbthENZZrNUsYoNEBHgxroeYE7pul/uWmaoBogthq3vv3y6YLlFNq3vPKsti1pZZpJWm4aX1YsHoBQyNHuoImU0nfjiEdhRrNYvYIBH+XtzQXWxO9nTS8u7dm7gVK9AEBFhPXKdTfeGC3a7vduScghTrwHU3Ls2yF2XVJj49kAGIXlpnmhvmo/dh0ZhFDIgaAMC/d/2bDac2NP4CSdPFa0kGnPvOAQrdi3E9Ignx9cBskdm8Ty3LVYzD68FsAM9A6HGH0mrsjjE7m9Tp0zFcvIjk4UHMwgX4j77+mq+343ytOU1zHeSdCkmqHXlxZBNUlyqrxwWYak3S7Eku4Ey2e/7/pQaILkJOaVVN3bvNRamxZJRlMGvrLNLL0vHSerFwzEKGtHWiOUeSVGtWc2QjGNzbGcoe2Mxq9lws4FyO/TYn7149RZAYGIg5N0+Um55T5yNdFdtJa3B76KBQD68L8cXhTMoNZjx1Gu7uF6O0nN/ho/dh4eiFDIwaCMCLu1/kw5MfNu7NIe2hg/XBU3VkbhBPnbbmM7B+T5pqVqMEslz7We19D3j4KCrH3hgvXSJlujBekzw8iHn3XfxGjmzWNW2mJG5hTnMlfSaBzhsMZXC0iWX2rZAhHUJpH+YL1CZv3A01QHQRNu8Tde9BPnom9Gx83Xt6aTqzt8wmoywDb503i8YuYlCbQQ5Ueo30mQJaT6gugWPXYBTRyhiWEEZciPhC/3C3fU/gvXv2EOWmgYGY8/JImTFT9G6o1GKogMPWDFP/WcKRV6VeNlgfrm7u1YYgHw+F1VwdH70PC8csZFCU2CNf2vMS606ua9ybbYdcZ7dCcYZjBLoRkwYIo4eMokq2q2Y1LU/KDsi37uv93Kt/WgSHMzCmpCJ5ehKzeBF+w69r1jUvN6eZ4sqjLerCOxh63iXWe1eIAwSVOtFopJos8scH0qkwuF9rgfpU4wIIcxrxcHV3Ukyj697TStOYvXU2meWZIji8rITK6fANhe63ibVaZtogmstGBHxsJ7Oay/Hu0YN2K1egDQzEnJ9PyoyZVJ05Y9d7uDQn/gvVxcKBt+9UpdU4PccyijmcXgzAJCcvzfLWebNgzIKag7SX97zMByc+aPiNXW4C33DVrKaRdAj3Y3AHYVZjD0dmlSZiG20R3R+ieimrxY7UBIepIjiMXbIYv2HN91q43JymKYf0LoWtzDT7KKTvVVaLC3BXvxg8dBpKq0x8eThLaTl2Rw0QXYCfz+aSXihmBE5u5MlVWmkac7bOIas8Cx+dD0vGLqF/VH9Hymw+tj6uzAOQdVhZLS7AxH7CSau40shXR+2/OXl1707c6lVog4IwFxSQOmMmVafVIBGA/avFa7dbwTdMWS0ugM3YISHclwHxwQqraRhvnTcLRy9kSBtRiv/q3ldZc3xN/W/SedQeFhxYAxb3tT+3FzZH5u9OZpNTqprVtBgVBXDCOvfTlvl2A4zWUU3G1FQkLy9ilyzGd0jz22kuP6R3K3OaK4lOgqjeYm37jlOpkxBfD262Otmu2+1+jsxqgOgCrLOerg7pEEpCeMNDbNNL038THC4eu5ikyCRHy2w+cUMgrItYq1nEBgn392R8jyjAcTXwXl27iiAxOBhzYSGpM9VMIjknIW2XWLtZaZYjqDCY+OxgJiACAmcyp6kPL50X74x+h6FthZnXf/b9p+EgscasJh3Ofe9gha7P+B5RBPnoMVlkPtqfrrSc1kONOU0A9LxTaTV2wZiVRcqMmbXB4eJFdgkOQZjTpBa4oTnNlVzuB3H8E6gqUVSOKzDVmrQ5nF7MsYxihdXYFzVAdHKyiiv54VQO0Li69/TSdGZvne16wSFYnbSsWcSjm1UnrUZg+0zsSyl0mJOWV5cuxK1aWRsktvZy0wPWICE4HuIVnCHqInx5JIvSahMeWg13JjmfOU192IJE2zigBoPE0ARobzXCUM1qGsRLr+Uu62dig2pW0zL8zpzGV1E59sBRmUMbH+4R2SG3NKe5kl4TQe8DxgrVrKYR9GsXTOdIkbhZb0dXeWdADRCdnI170zBbZEJ9PWqyRXXh0sGhjT6TQOdlddL6SGk1Ts+QDqHEh9rMahy3OdUEiUFB1kzirNYZJBqrxOk7QNIM1ZymEdjMacb3jCLE1znNaerDU+vJ29e/zbC2tUFivT2JthP4M1ugJNPxAl2cydZe6tSCCnacz1dYTSsgdSfkWfduNygvrQkO09Jqg8PBg+12/dzSar45ng24qTnNlXgFQA9rVvmAWmbaEJIkMWmA+Fx8dijTrcxq1KcbJ8Zskdm0VzhU3t1fNMPWRUZZRk1ZqbfO2zWDQxBOWrbNaZ/qpNUQGo1U08fziQPMai7Hq0uX3/YktsYg8eQXUFkIGp1qTtMITl8q5UBqEVAbCLginlpP3h79dk256at7X2XtiTqMaLreAj5hIJvhYCMdUFsxHSP8GRhvNavZ614n8E6JrX0jup/Lm9M4OjgE2Lw/zf3Naa7E1jqRdRgyDyqrxQW4MykaD52GsmoTXx5xH7MaNUB0Yn4+m0tmsWjcnzyg7pOrjLIMZm+pdStdMnaJawaHNmxlppeOqJtTI7i7Xwx6rURJleM3p99kEq1BYqsagWE7Ue18I/hHKqvFBbCV3MSH+jCkQ6jCapqHLZNoM655Ze8rVx+BofOAvlPEWjWraRQ2R+Zvjl8ir6xaYTVujBuZ09S4ldYEh0vsHhxaLDIb9ohD+rua4CDv8sQMgPBuYq2a1TRIkI8HE3qKCr8NblRmqgaIToztgzakQyjxYVfvE3C74BDE5hTRXazVEocGCfW73KzG8U5aXl27/iZITGktQWL+eUjeLtb9ZimrxQWoMpr55IAwHpnkQuY09WHrSbQFiS/vefnqQWKS9QS+OBXO/9iCCl2Tm3q1IcBLh9Es87FqVuM4Dm8AczV4+NdW6rggxuzsqwSH9p/v/Ov5vFpzmkGuWwHRZC43qzn6EVSXKSrHFbCVmR5ILXKYH0RLowaITkpOaRXfnxTmNJPqKM3KLMt0v+AQxOZke8BSN6dGYeuNOJBaxOlLjt+caoJE25zE1hAk2g4rAmMh4XpltbgAXx/LoqTKhE4j1RiRuAO2IHFwG5GteHnPy3x48sPf/lJYR4gfLtb7VUfmhvDSa2sMjDbsTUNWWwvsz5XmNJ4NO6I7I8bsbFKvmHPoiOAQaisgBsaH0DHCzc1prqT3PaD1BEMpHP9UaTVOz+AOIbS3JnLcxaxGDRCdFNtQ1iAf/VXNabLKspi9tTY4dNmew7qo2ZzK1M2pEVxuVrOhhfp4rhoknjvXIvducUyG2n6yxGmgaSWlRs1g/W5RmjWuRyTh/p4Kq7EvXjovFoxeUBMkvrTnJdafWv/bX7KdwJ/+GkovtaxAF8TWS30xr5xdFwoUVuOGpO6CvNNi7aLlpcacHFJnzMSQkiKCw8WL7F5WaiOntKrGnGZya8oe2vAJge63ibXqyNwgkiRx7wDxOfn0YIZD/SBaCjVAdEIsFpmNVnOaOxJ/P5T1UvklZm+dTUZZBt46bxaNWUS/yH5KSHUcl29Oaplpg4jNSTxgteTm5NWt2++DxAsXWuTeLcrpr6AiDyQNJN6ntBqn51xOGXuSxUP+ZDedG2bLJA5qI7IX83fPZ+OpjbW/0O1W8A6xmtXUYWijUkOXKH/6tQsG3OcE3qmwZbLbJkGb3spquQZMubkiOExORvLwIGbRu/gOHeqw+328P6P1mdNcic2sJmMfZB9XVosLcFdSDDqNRFGFka3HXf9QUA0QnZBdF/JJyRd171c+XGWXZzN762zSy9Lx1nnz7ph36R/VXwmZjsc2dDp9L2SfUFaLC3B3P2U2J69u3YhdsRxNYCDmvDxSZ8yk+uLFFrt/i2A7pOg0DgKjldXiAtj6p2NDvBmWEKawGsfhrfNmwegFDIoSQeILu19g8xnr7DCd52VmNavBYlFIpeswyXoCv+XYJQrKDQqrcSMqCuD4f8XaBbOHprw8UmbMxHDxoggO330Xv2HDHHY/WZbZaK3EuTPp94f0rYZ2wyC0o1irZjUNEu7vyQ3dhXmdOxxyqQGiE7Lemj1Migui82VDWXMqcpjzzRzSStPw0nqxcPRCBkQNUEqm44m/DkISxFrNIjZIuL8nY7uJzcnmvNZSePfoQdzy5WgCAn5z0usWFCbD+R/E2tYbq1In1SYzH9vMaQbEodG4vjlNfXjrvFkwZkHNXvz8zuf5+MzH4oe2h/GiVLigmtU0xC292+LvpcNgttQYHKnYgSObrOY0ftDzLqXVNAlTXh4pM2diuHABSa8n5t2F+A2/zqH33Hkhn2TrIf2kehzk3Z7L/SCObABjpbJ6XIBJ1qTOrgsFXMh1bf8MNUB0MgrLDWw9JrI/ky7LHuZW5DJn6xxSSlLw1HqyYMwCBrYZqJTMlkGSarOIhzeIIeUq9WIzNNp5IZ/kvPIWvbd3zx7ELV+Gxt8fU06OOPFNcbyrqsM5YB2K7t9GZBBV6mXr8WwKK4xoNRIT+7mPOU19eOu8WTh6YU2p/792/otPz34KYZ2gnfVhVu3jaRBvDy13JIoM/fo9qapZjT2Q5doD1l53u5Q5jSk/n9RZszCcOy+Cw4UL8Bs+3OH3tbX4JMYF0SWqlZnTXEnfKaDRQ1UxnPhcaTVOz/COYUQHeQO1nyNXRQ0QnYxPDmZgMFvw99RxS29R955XmcfsrbNJLknGQ+PxGwc9t6fvFDGUvKpIDClXqZfhncJrNqcNCmxO3r16EbdsKRo/P0zZ2SJITHXhUguzqbZ/LPE+0OqU1eMC2MpLx3SNICLAS2E1LYeP3odFYxaRFJGEjMw/dvyDz859VtvHc/orKMtVVqQLYMvYnM8tZ89F1aym2WTshxxri4YLVUCYCgutc3bPgV5P9IJ38Bs50uH3Laow8LX1kL6++dOtBt8w6HqzWKuHXA2i0dSa1Xy0Px2DyXVbC9QA0YmQZbnm4eoPfdvi46EjrzKPOVvn/CY4HNrWcY3ZTodfBHSZINZqmWmDaDUSE/uLrM1H+9Mxmlt+c/Lu00cEib6+mC5dEkFiuouWi53dCmWXAEm4l6rUS3JeOTvO5wPua05THz56HxaNXURiRCIyMs/9+hxfeOnAKwgsJjj8YcMXaeV0bxtAn9ggwPVP4J0C20N9ZC9om6iolMZSGxyeBb2emLffxn/UqBa59ycHMjCYLPh6aLm5dys1p7kS2yFX6g7IPaOsFhdgYv8YNBLklxv47mS20nKuGTVAdCIOpBZyNkfULE8eGEd+ZT73b72fC8UX0Gv0vD36bYZFO64x22lJmilek7eLYeUq9XJP/1gkCfLKqmtmabY03n37Ert0KRofH0xZWaROn4EhPUMRLc3C1pifMBqC2ymrxQWwZa3bBnoxonO4wmqUwVfvy6Ixi+gT3gcZmWd3/Yv/dbGWxR1YI0r+VOplsvUE/n9HsyiuMCqsxoWpLoVjn4h1vxmibcPJMRcVkTp7DtWnT4NOR8xbb+I/umXmzgpzGrGH/aFvNL6easUIAO1HQZD1+089qG+QNoHeXN8lAnBtsxo1QHQi1luNRXpGBxAdamHut3M5X3wenUbHW9e/xXXRjm3MdloSrhfDyUHdnBpB2yBvRlofzltqJuLV8ElKJHbp+0g+PhgzM0mdMQNjZqZieppMcTqc+1as+7lOaZZSGM0WPtovMsX3DIhF6+bmNPXh5+HHkrFL6B3WG4ts4ZmSI3zt6wP55yBlh9LynJ5b+7TF10NLtcnCfw+54MGSs3DsYzCWg85L9B86OebiYhEcnjwJWi3Rb7yO/5gxLXb/g2lFnM4uBWDywFY4+7AuNJrL/CDWg6laWT0ugM1D5JdzeaQVVCis5tpQA0QnoaTKyJdHxMPzbUlBzP1mLmcLz6LT6Hhz1JuMiBmhsEIF0Whry/sOfSiGlqvUi62PZ9uZXDKKlHMe8+nXj7j330Py9saYkUHKjJkYL7nIfKCDa0G2gG84dJ6gtBqn5/uT2eSVVaORRBa7tePn4ceSG5bQK6wXFmT+Hh7GVh9v9ZCrEZ5J9JUAACAASURBVPh66ri1T1tANatpFrYKiO63g3ewsloawFxSQuqc+6k6cUIEh6+/TsC4ljUF22g9pO/eJoBe0YEtem+nJ/E+kLRQkQ+n/qe0Gqfn+i7hRAZ4IsuwaZ9rlso7NECUJMlPkqR3JEnKkiSpUpKkfZIk/aGR702QJOm/kiQVS5JUKknSV5IkdXekXiX57FAmVUYL3p7VbMl7ntOFp9FJOl4b+RqjYkcpLU95EqeKIeXluXBmi9JqnJ4x3SII8xOb02aFNyef/v2JXbIEycsLY1oaKTNmYMx28rp8y2XDzftOAZ2HsnpcAFsFxMjO4bS1GiW1dvw9/FlywxK6h3bHLMHTEWF8d3ELVBYqLc3psZ3An7pUypH0YoXVuCCXjkLmAbF28goIc1kZqXPnUnXsGGg0RP/nVQJuHN+iGsqqTXxhPaSfNDAWyQXKcVsU/yjofKNYq2Y1DaLTapjYTxyUbtqXhkkBP4jm4ugM4qfAVOBZ4GbgBPCpJEk31fcmSZIigO1APDADmAyEANskSXJL3/QNe1JBU0lIx1WcKTqFVtLy6shXGRPXcuUVTk1gDHQcK9bqCXyD6LWaGrOaTXvTMFuUPYH3HTSQ2MWLkDw9MaakkjpjJsYcZfojG8X5H6HYGli7kPOfUmQUVfLzWeHQea/q/PcbAjwCeP+G9+kW3BmzJPGX0AB++PUVpWU5PX1iAulqHTGghCOzy3NgjXgN7QRxQ5TVUg/msnLS7p9L1eEjoNHQ9pVXCLip3kdEh/D5oUwqDGa89Bpu6xvd4vd3CWwHDRe3QcEFZbW4ADY30+ySan467XoO1g4LEK1B4FjgflmWl8uy/AMi2NsJvN7A258CgoGbZFn+ryzLXyICTE/g/zlKs1IcTS/m+KUcfOKWU2K5iFbS8sqIV7ih3Q1KS3MubA/q574Xg6dV6uVea5lfZnFVzcO7kvgOGULMoneRPDwwJCeTOnMWplzldV2VA6vEa/xwCE1QVIorsGlvGrIMYX6ejOkWobQcpyPQM5D3xy2ni9YXkyTx5/T/8VPqj0rLcmokSWKS9QHr80MZlFebFFbkQhgr4chGsU6a7rTmNOayctIeeIDKQ4dAkmj78ksE3nqLIlo2Wvv1b+rVhkBvvSIanJ6OYyHAGjzbKmxU6iQ2xIfhncIAZf0grhVHZhDvAIqBz2x/IYtGgtVA1wbKRe8AvpVlucbRQpblfOAL4E7HyFWONXtO4RO7Aq13OhpJw0vDX2J8fMuWV7gEnceDXyQgq5tTI4gP82VIh1Cgdjad0vgNG0bMuwuR9HoMFy6QMmsWpvx8pWX9lrIcOP21WKvZwwYxW+SaMuaJ/WPQa9XW9qsR5BXE0qEv08lgwCTBn376Ez+n/6y0LKfm9sRoPHQayg3mmh59lUZw4nMx2Fyjhz6TlVZzVSwVFaQ99CCVBw6AJNHmxRcJ/EOjOpDszonMEg5by5gnqRUQdaPRil5EgIPrxJxglXqxfZ5+PJ1LXplrmfs48pu8J3BCluUrC2+PXPbz3yFJkjeQABy7yo+PABHWEtSrvbeovj+A03Ud55QV83XuC2h9UgGJF697kQntVUOMq6LVQ9+pYn1wregTU6mXSVYntu9P5pBTWqWwGoHf8OHELFwAej2Gc+dFJrHAiQZiH/pQzKzzCoJutyqtxun5+WwumcXis3Wvak5TL8HtR7LUFEKCwYBRNvHkj0/ya8avSstyWoJ8PLipZxSglpk2CVt5adebwM/5xs1YKitJe2gelfv2A9DmhX8TdOcdiumxZQ8Twn0ZEO/cZj6Kk3gfIIn5wGe3Kq3G6bmheyTzRiXw9R+HE+bnqbScJuHIADEUuNpTX8FlP78awYB0je91OV7cvhzJKxlZlnhmwD+5pYMy5RUuQ5LVzbQkQ5SaqtTL+B5RBPnoMVlkPt7vPHbxfiNHEvP226DXU332LKmzZmMqdALjDlmufbjqMwn0XsrqcQFszn9DOoQSH+arsBonR5IITZrJsqwc2hvNGCwGHv/hcXZm7lRamdNiM6s5mFrE6UulCqtxAfLOQcovYu2EFRCWqirSHn6Yij17AIh6/l8E3XWXYnqqjGY+PSi+GycNiFPNaRoiKE7MBYZal1yVOvHQaXj6xq50jvRXWkqTcXQtUH3OGA25ZjT5vbIsB9X3B1Hy6lT8bdiDdPMby+CAh5jc3e2qZ+1PSAdobx35oZrVNIiXXssdiaJnYONe57KL9x99PTFvvQk6HdWnT5M6ew7moiJlRaX8CgXnxdo290mlTnJLq/nupHCknaTODWscve8hTNKzPCuLeI9gDBYDj/3wGLuzdiutzCkZ1D6E9taDB1fs42lxbN+LgXHQoWUGzDcWS3U16Q8/QsXOXQBE/fMfBN9zj6KavjqaRUmVCb1W4s4k1ZymUdjMas59C8XOc/CsYl8cGSDmc/VMX4j1ta6askJEAHgt73U52gT4sumuN1l258NKS3EdbKeip7+GUheZqacgthr45PwKdl5wrn4//zFjiH7jddBqqT55UgSJxQqe49hORKP7Q2QP5XS4CB/tT8dkkQny0TO+R5TSclwD72DofhvhZgvLyjTE+cdRba7m0e8fZe+lvUqrczokSapxA/z0YAZVRrW1oE5MBjHIHES1jcZ5+oEt1dWkP/oY5Tt2ABD53LMET5qksKra0uVx3aMIdbESQMXoPEHMB5YtcGid0mpUHIQjd4/jQDdJkq68Ry/r69V6DJFluRK4wNV7FHsBubIsO7E/vorD6XYreIeAbFY3p0bQJcqfxLggADbscb4+noBx44h+/TXQaqk6cYLUOfdjLilpeSGVhXDC6qnl5HPDnAFZlmt6d+5IjMZLr1VYkQth/XxFZhxkeb+nifWPpcpcxSPfP8L+7P0Ki3M+7kqKQaeRKKowsvW4eihYJ2e+FrOCJU1tv74TYDEYyHj8j5Rv3w5A5DN/J2Sq8vrO55ax56LIN6gVEE1A51FrfnTgA7C43oy/lmRL8hbyKvOUltFkHBkgfgoEAVe6PEwHTsuyfKKB994gSVLNkbQkSSHWa31ib6EqLobO87LNaY26OTWCydYs4pbjlygsNyis5vcE3Hgj0f95FTQaqo4dI3XuXMxlZS0r4sgmMFeDhx/0UMu9G2LXhQKS8ysA1fmvybQbBiFifErUif+xYvwKov2iqTRVMu+7eRzMOaiwQOci3N+Tsd0iAec85HIabP3THW+AQOcol5QNBv4/e+cd3lTZ/vHPSdqme6S7tLTsvfdWEHDgi4u9lyLIEJQhojhef6ivL+orKgKlgCwVQVFQZO+9NwKdtJS2dM8k5/fHaVpktiXJSdLzua5ceWiS83xb2nPO/Tz3/b0TJr9O9s6dAARMn452qHWk7/9QvHsY6uNChxp+MquxMYyZXBmxcHWbvFqsmHWX1/HmzjcZ+edImwsSzRkgbgS2A4sFQRgpCMLjgiBEAR2BN41vEgRhhyAIdxZG/QepXnCjIAi9BUF4Bvgd0AEfmVGzgq1g3OG5FQ3RilX8w+jVJBh3jQOFOgM/H7fOmgHPp58m5OOPpSDx5CniRo9Bn51jmclFsTS9tOGLoHG3zLw2jHH3sFlVb+oE2V4BvqwIQmmN68nVBDl5E9kzkhC3EPJ0eYz9aywnkk/Iq9HK6Fe8w7P/airRKRY6L9gS6bGlxm1WkgEhFhURP2UK2dukACLgzTfwHTFcXlHFFOoMrD0WD0juyyqVYk5TLvxqQnhHaWxcmFD4B7/8/Qvv7nsXAB+ND64OrjIrKh9mCxCLex4+B6xGCuo2AY2BF0RR3PCQz94AOgFxwHJgDZAOdBZFUalSVwD/OhDWVhorTloPxdXJgd5NQwCpJ6I1mdXcjtezvQj5v49AEMg7cYK4V17BkGOBm8GEY5B8Vhpbyc2VNZOeW8jGM1Kqn7GZuUI5aToQVA6Qnw7nNxDiHsLinosJdgsmV5fL2C1jOXXz1MOPU0noXMufEC/JVXjNEWUX8S6OrwBEcA+CWvL3URaLikiYMpXsLVLQ6v/66/iOGiWzqlK2nL9BSnYhKgH6KO15KoZxkevCRsi+Ka8WK2PDlQ3M3jsbEZGm/k35+omvcXVUAsQSRFHMFEXxNVEUg0RRdBZFsbkoiuvveM9joijetXQjiuJlURR7i6LoKYqiuyiKT4mieNacehVsDOON/PkNkGNbW/dyMKDYLv5ycjbHYq2gpcR98Ordm+B//1sKEo8eJe6VsRhyc8076bEo6TmwEYQ0N+9cdsC64wkU6gy4Oanp1ThEbjm2iXsA1CnueVvsPBnqEcriHosJdA0kpyiHsX+N5WyKctkDUKuEkhv5H4/EU6RXSgtKMOil3sAgLTyoHWSVI+p0JLw5jay//gLAf9JE/F55WVZNd7LqkLTX0LVuAEFeSjujClH/X+DsBYYiOLlSbjVWw+9Xf+ftvW8jItLYvzHfPPENbo621wLKeiyuFBTKS/3nQGM8Oa2SW43V07CKFw2reAKwysrreLxfeJ7gD94HIPfIEeLGvoohL888kxVkwem10rj5UCn9T+G+iKJYUgf2r6ZVcNPIezNq0zQfLj1H74ZUqb1KmGcYkT0jCXAJIKsoizF/jeFc6oNK9isPfVuFIQiQkl3A1vOKV10Jf2+BTCldsqRXsEyIOh3Xp00j648/APB77TX8Xn1VVk13EpeWy+7L0qKyUj/9CDi6QONiJ9pjy6RSjUrOpmubeGvPWxhEA438GvHtE9/i7mSbJStKgKhguzi5QuM+0vjoUuXkVAaMF8PfTl0nM79IZjUPxvullwh67z0Acg8dIu7VceYJEs/8DEU54OBc+vukcF9OxKVz8YbUsFxJL31EajwOXsU/w9v6ulb1rMrinovxd/EnqzCLMZvHcD71vEwirYcq3i50ruUPlNbAKlBaZlH9MalXsEyIOh3Xp88gc+MmAPzGvYr/a+Nl03M/1hSb0wR5OvNYHX+Z1dg4xjTT1L8hZp+8WmTmz+g/mbl7JgbRQAPfBnzb/Vs8nGy3Pl8JEBVsG6OTVupliN0vrxYboHfTEFwc1eQXGfjFSs1qbsenX1+C5khF3rkHDhA/fjyG/HzTTmK8Ma//nNSjTuGBGG+u6gV70jjUS2Y1No5KDc2Kd3yOr5D62BUT4RXBop6L8HX2JbMwkzF/jeFi2kWZhFoPA4rNanZeuklCupmyCmyJzES4JO3WlVwPZUDU67k+8y0yf/8dAN9XXsFvwgTZ9NwPnd7AD8U1rH1bhuKgVm6DH4mghlClhTQ+Vnn9IP6K+Yvpu6ajF/XU09ZjQfcFeDp5yi3rkVD+MhRsm+DGENJMGitmNQ/Fw9mRXo2DASnN1FrNam7Hp39/Ame/DUDOvv3EjzNhkJh0BhKK+841tw7rdWsmu0DHryevA9KNuqCk4z46zQZLfetyU+Dixn+8VN2rOpE9I9E6a8koyGD05tGVPkjsVi8QP3cNBrG0TUGl5sT3Uk9gVz+o20sWCaJeT+Jbs8jcIPkP+o4Zjf/kSVZ5fth2IZnkrAIEQUpZVjABxoWJc79I/YQrGVtjtzJt5zT0op662ros7LEQL43tL54qAaKC7VNyclpfKU9O5WVAGynN9FxiJqcTMmRWUza0gwYROGsWADn79hH/2gQMBQWPfmCjPbdvLQhv/+jHs3M2nLxObqEejYOK3k2so8+azeNVpdR18mjUXS9X9y4NEtML0hmzeQyXbl2yrEYrwlGtok/LUAB+OBKH3mD9i1xmw2CAo8XnsKYDpQbmFkY0GEic/Q4Zv/wCgHbkSPynTLHK4BBKzWk61/In1Me2XCWtloYvgKMb6PLh1I9yq7Eo22K38caON9CJOur41GFhd/sIDkEJEBXsgUYvVdqTU0VoFuZNnUApL97azWpuRztkMIFvzQQgZ8+eRw8Si/Lg1GpprJjTlAnjzdUzjYLxcnWUWY0dYXRkvrod0q7d9XIN7xos6rEIrbOWWwW3GP3naC7fumxhkdaDsfY1MSOfnZcqsVnN1W1So3KQJb1UNBhIfHs2GT//DIB22DAC3nzDaoPD6+l57LwktWMwpiormACNBzR6URofqzx+ENtjtzN151R0oo5aPrVY2GMh3s7ecssyGUqAqGD7aDykFSyoVCeniiIIAv2LL46/nkggp0Ans6Kyox06lIAZ0wHI2b2b+IkTMRQWPuRT9+Hcr5CfASpHaDLAhCrtkzMJGZyKl3acjbvQCiaiZnfwKG4Xcp+m08YbEB+NjxQkbh7N37f+tqBI6yHc142ONf0AWHnQdha5TI6xrCKik9S43IJIO4elwaHP0CEEzJhutcEhSDvOBhH83DV0qxcotxz7wujIfOOM1FfYztkRt4MpO6egM+io6V2TRT0W4eNsXx4GSoCoYB+0GC49V5KT06PyfLMqODmoyCnU89up63LLKRe+w4cTMG0aADk7d5EwoYJBojGdr+7T4K442T0M4+5hrQB3Wobb14VQdtQOpe0JTqwA/b0dhmv71JZWqTXepOWnMWrzKK6kX7GgUOvB2Nd124UbJGWY2LjKFshOLq1ZtfDuoWgwkPjOO2SsLQ0OA2fOtOrgUG8QS2pW+7QMxVExpzEtVZpDQANpbOwrbKfsjNvJ6zte/0dwqHXWyi3L5Ch/IQr2QZUWlebkZAq8XZ14umEQACttKM3UiO/IEQS8+QYA2Tt3kjBpMmJ5gsTkCxBbbMndYoQZFNoXOQU6fjlhNKepatU3gjZLsyGAANk3Sl0p70EdbZ0SE4S0/DRG/TmKq+lXLafTSuhePxBfNyfJrOaI7Z3DHpkTK8Cgk5yX6z1rsWlFg4Gkd98l4yepd6zP4MFWHxwC7Lp0k+vFCwlKex4zIAilqfKn10J+prx6zMSu+F13BYe+Lr5yyzILSoCoYB/ceXIqyJJXjw3Qv3gF/mRcOucTbe9k7jtqFP5TpwCQvX078eUJEo123D7VoFoXMym0HzacvE52gQ4nBxUvNFfMacyCdxjUfEIa38Os5nbqauuysPtCPJ08Sc1PZeSfI7maUbmCRCcHFS8Vm9WsOVzJzGoMhtL00iYDwNHZItNKweEc0n/8CQCfQYMInPWW1QeHACuLMyA61PQl3NdNZjV2SuN+Uj/hohw4bX9+ELvjdzN5+2SKDEXU8Kph18EhKAGigj3RuG/pyenMWrnVWD1tqmmp7iddKFcfss2m035jxuA/pZxBYlEenFgpjVsMA5VyGnwYxvTSXo2C8Xa1vFNipcGYKv/3Vkh/8N9kPd96LOxxW5D4R+ULEvu3kha5EtLz2HX5psxqLEj0brhVbGZkofRS0WAg6b33Sf9RuvH3GTiQwLdn2URwmJyZz7YLkpmR8XdGwQy4eEODYj+Io0vsyg9iT8KekuCwuld1qUetHQeHoASICvaEi4/U7ByUnohl4HazmnXHE8gv0susqGL4vVzOIPHcL5CfLpnTNB1sIZW2y5mEDE4q5jSWoXZPcA8CRDi2/KFvr+9bv1IHidX83GhfQ7pJW3XQNhe5KoRxhzmsLQTUNft0osFA0vvvk75mDQA+AwcQOPttmwgOAX48Go/eIKJ1c6JHA8Wcxqy0LC7ZSDoN1+3DD2Jvwl4mbZtEoaGQal7VWNxzMX4ufnLLMjtKgKhgXxjTTK8fk05QCg/kxeahOKoFMvN1bDydKLecClOuILHEnOYZxZymDBh3D2sq5jTmR+0IzYoXLY4vB/3DHYbvGSRWoppEo1nN1gvJ3MisBGY1OSlwXmpIX7LjbEZEg4GkDz4gfbUUHHoP6E/g7Nk2ExwaDCKrD0vnsBebV0HjoJZZkZ0T2qrUD+LIEnm1mIA9CXuYuG0ihYZCIjwjWNyjcgSHoASICvZG1XbgV1saP6SORwF83TX0qC+Z1ay2QbOa2ylTkJh8HmL3S+OWijnNw1DMaWTA6GaalQiXN5fpI3cFiX9WniCxR4NAtG5O6A0iP1YGs5qTq8BQBBovqN/brFOV7ByukvrFevfvR5ANBYcAe6+kEJeWB0A/Jb3U/AhC6bX1zFqplZSNsjt+d8nOYYRnBJE9I/F3rTyLykqAqGBfCEJpTcbJNVCQLa8eG8CYZnooOo2/k23b3OehQaIx9VhbHSI6y6DQtvjtVKk5zYuKOY1l8ImAGl2l8bGyp8pX1iBR46DmpRaSWc2qQ3EY7NmsRhRvM6fpB06u5pvKYCBpznulO4f9+xH0zjsINlazbVz4bF1NS80Ad5nVVBIa9wUHFyjKhVM/yK2mQuyK38Wk7aVppZUtOAQlQFSwR5oOBLUGCrMUs5oy0KGGH2FaF0C6wbJ17hUkGgoLJXOak8XmNM0Vc5qyYGyB8oxiTmNZjKmDlzdDRnyZP1ZZg0Rj24KE9Dx2/50isxozErMPUi9LYzOa0xhbWaT/IN3c+wwcQNC779pccJiSXcDmc0kADGittLawGM5e0PBFaXw0yubManbF7/qHIU1lDA5BCRAV7BFXLTR4XhofiZRXiw2gUgkldTw/HY23WbOa27kzSEyYMBHDyeJ0F5UjNB0ks0Lr5+z1DE7GpQOldV4KFqLO0+AWAKIBjn9fro/eK0i8kn7FTEKtg+r+7rStLjWqtmuzGmPZRJWWENTQLFOIBgOJ77zzz1YWNpZWamTt0XiK9CKezg481TBYbjmVC2Oa6Y0zkHBUXi3lYGfcTiZtn1QSHFYWQ5p7oQSICvZJy5HSc+IJSLAPJy1z0rdlGI5qgYy8In47ZbtmNbfj9/KY0j6JO3cSP/szDHqgXi/FnKYM3G5O0ypCMaexKGpHKRMCJDdTQ/kWbe4VJF6+ddkMQq0H4yLGlvM3SM6yQ7Oa3DTJgRlKzdhMjGgwkPj2bDJ+kjJvfAYPtplWFndiMIgl57AXmofi7KiY01iUKi0gsJE0thGzmh1xO5i8YzI6g44aXjUqdXAISoCoYK+Etb7NSUvZRXwYfu4anixeYV1xMEZmNabDb8wYAt58E4Cca/nE79ZiaKjsHj6M3EId648r5jSy0nyo9JwZL/VFLCf1feuzqMcivDRepOWnMerPUVxMu2hikdZDzwZB+Lg6ojOI/HS07Gm5NsOpNaAvACeP0l5zJkTU60mc9TYZP/8MgM/QIQTOestm//b3XkkhOjUXgIFKex7LIwjQcrg0PrMW8tJllfMwtsVu4/Udr6Mz6KjpXbPSB4egBIgK9oodOWlZikHFF9HjsemcvW4/Py/fUSMJ6C0tFuQkORP/6WoMeXkyq7JuNpwsNad5oZliTiMLvjWgWhdpXEFH5nq+9VjcYzHeGm9uFdxi1OZRXEi7YDqNVoSzo5oXm0tmNavtzaxGFEsXOhu9BBrTmq2Iej2Jb80iY906ALTDhhI4c6bNBocA3x+QFjpbV9NSO9BDZjWVlEZ9wNEVdHlw+ke51dyXrbFbmbpzaklwuKjHInxdfOWWJTtKgKhgvzTuK52cbNhJy5K0qaalhr8bACvsqY6nKA9fn0MENpOC3px9+4h7dZwSJD4AoznN0w2D8HFTzGlkw2hWc2kTZCRU6BB1tHVY3HMxWmctGQUZjPpzFOdSz5lOoxXRvzjNNDYtl31XUmVWY0Kid0PKJWncapRJDy3qdFyfOZOMX6T0Ve3w4QTMmGHTwWFSRj5bzicDMLhtuMxqKjG3m9UcWWKVZjWbozfzxo43/rFzqASHEg5yC7A2RFEkJSWF/Px8DAaD3HIUbkOlUuHs7Iyfn1/ZLl7OXtJq67Fl0uprq9HSzqLCPREEgUFtwnn/t3P8cjyBt56uh7vGDk4RZ9dDfgbaeo7wxCRufPoFuQcOEPfKWMK+/QaVq/ms4m2R281pBrZRbq5kpW4vcPOHnJtSy4vH36rQYWr71CayZySj/hxFan4qozeP5rvu39HQzzxGJ3JRM8Cd1tW0HLqWxspDMXSsZScpYocXS89hbSCokckOKxYVcX36DDI3bgRAO2IEAdPetOngEGD14Vj0BhFfNyd6NgiUW07lpuUIOL4cks9C/GGp/MdK2HRtEzN3z0Qv6qnjU4eFPRbi46zU2xtRdhBvQxRFEhISSElJoaioSG45CndQVFRESkoKCQkJiGVdiTKa1SSfg7iD5hNnJ7zYPBRnRxU5hXrWH6/YjoXVcbS4QL7es2hHjSXo3XcAyD10iNiXX8aQkyOjOOvD2Deshr+bYk4jNw5OpbWIR5eCvuLXpRreNYh8MhI/Fz+yCrN4efPLnLp5ykRCrYeBxbuIm8/eIDnTDsxqspLgwm/SuKXpdg/FwkISpr5REhz6jhljF8GhTm8oOYf1bRWGxkExp5GVkOYQ1FgaVzBV3hxsuLKBGbtnoBf11NPWY3HPxUpweAd2sD1gOlJSUsjKyiIwMBCtViu3HIV7kJaWxo0bN0hJScHfvwxOlCHNpMf149IuYtW25hdpw3i5OvJs4xB+PBrPioOxDGpj4wYlN25bGCiuSfUZMABUapLefZe8I0eJHfMyYd8tQO2uNFGWzGmkhQHFnMZKaDEc9syD7CS48Ds0eK7ChzL29Br952iS85J5+a+X+faJb2ka0NR0emXmyYZBaH9zIi2nkNWH45jYrZbckh6NY8vAoAMXLdTvbZJDGgoLSXh9CtlbJfMjv3Hj8Jvwml38vW85n0xSZj6CULpYoCAjgiCdw36fAmd+hp4fgYu3rJLWXV7Hu/veRUSkoW9Dvu3+LV4aL1k1WSPKDuJt5Ofno9FolODQitFqtWg0GvLzy7EybNxFPLsecuyoLsVMDCqu2TifmMmxWOt2HnsoxhVLbQ2I6FTyZZ9+fQn64H0QBPKOHSN25Cj0GfZjzFNRfjlxnaxicxqj4YeCzHhXhVo9pfHhRY98uGpe1Yh8MpIA1wByinJ45a9XOJJ05JGPay04O6rp10pqir7yYCw6vQ2Xiuh1peewZoPB0fmRD2koKCBhwsTS4HDiBPwnTrCL4BBKXbgfq+1PmFYpURhU/gAAIABJREFUH7AKGvUBRzfJrObUGlml/HTpJ97Z9w4iIk38m/Bdj++U4PA+KAHibRgMBtRqJR3B2lGr1eWrD234Img8JYvwkyvNJ8xOaBLqRYMQT8DGW14U5sKp1dK4xfC76k99+vQh+KOPQKUi/9QpYkaMQHfrluV1WgmiKLJsv/T/3atxsGJOY020Gi09R++Gm4/eqiLcM5yonlEEuQWRq8tl3NZxHEg88MjHtRYGtq6KIEBSZj5bzt+QW07FufwnZCYAt7lyPwKG/Hzix40ne+dOAPynTsF/3LhHPq61EJ2Sw+7LKQAMUuqnrQdnT8kPAmQ1q1l9YTXv7X8PgOYBzVnQfQEeTorD7f1QAkQF+8fJDRr3k8ZW6qRlTRjNagB+O5VIem6hzIoqyOkfpfYmaqfSpuN34P38c4R88gmo1RScO0/ssOHoUivnLvOx2FucT8wEYGi7CHnFKPyTGl3BJ0Iam6iva5hnGFFPRlHFvQp5ujzGbxnP7vjdJjm23IRpXelaJwCA5QdseJHLaE5Tsxtoqz/SoQx5ecS9+io5e/cCEDB9On5jxjyqQqti5SHJfbuKtwuP1w2QWY3CPzAucNw8L4sfxIrzK/j3wX8D0CqoFd888Q1ujm4W12FLKAGiQuXAeHJKuwLXdsmrxQbo3TQEd40DhTqDbTadFkU4vFAaN3gB3O7vZujV6xmq/Pe/4OBAwaVLxAwdRlFysoWEWg/G3cPGoV40DZO3RkThDlSq0lT5E6ug0DTGSlXcqxD1ZBQRnhEUGgqZtH0S22K3meTYcjO4nbTItffvVP5OzpZZTQVIuwpXpDTQRzWn0WfnEPfyK+Tul3aJA2fNwnfE8EcUaF3kF+n58YhkTjOgdRhqlX2kzNoNIc0guLjW2QSp8uUh8kwkcw/NBaBNcBvmd5uPq6OSfvwwlABRoXIQ2ADCig1qTLQCb8+4aRx4vrhB+oqDsWV3jbUW4g5B0mlp3Prhq+SePXsQ+uWXCI6OFF65QsyQIRQlJppZpPVwM6uAjael71fpG2alNB0Mag0UZMDpn0x22CC3IJY8uYQaXjUoMhQxdcdU/oz+02THl4sutfwJ07oANpoqb7xOeYZC7Z4VPow+M5O40aPJPXwYgKA576IdMtgUCq2KjacTuZVbhINKoG9xDaqClWG8Fp9dD9nmX4QVRZGvT3zNvKPzAOgQ0oGvun6Fi4OL2ee2B5QAsZIzZ86cu4rTBUFgzpw5FtVx8eJFXFxcEASBEydOmGcS4wr8hd8gy4brUizEwDaSA9y1lBz221rT6UPfSc8hzaBKizJ9xKPr44R+/TWCRkNRTCwxQ4ZSGG8nrT4ewg9H4ijSi3i7OvKvJiFyy1G4F26+0OB5aXx4kUlT5f1c/Ih8MpI6PnXQiTqm7ZrGhisbTHZ8OVCpBAYXp8r/dDSe3EKdzIrKQVE+HF8hjVsMB1XFvBF0t24RO3wEeSdOgCAQ/OEH+PTvbzqdVsSKg1J6ac8GQQR4PLqZj4IZaPgiuPiAoUhq22NGRFFk3rF5fHPyGwAeD3ucL7t+ibOD8rtRVpQAUeEu9u/fz+jRoy02nyiKjB49Gh8fM/egqd+7+OSkkxq3KjyQesGetAiX/k+MF1+bIDsZzv0ijVuNucuc5kG4d+pI2IJvEVxcKIqPJ2bIEApjbHD3oRzo9AZWFNdp9W0ZhrOjYtRltRjNapJOQcJRkx5a66xlcc/FNPBtgEE0MGvPLNZdXmfSOSxNn5ZhODmoyMrX8euJ63LLKTvn1kNeGqgcSvtglhPdzZvEDh1K/rlzoFYT8umneL/0komFWgfnEzM5GiMZjA1qq7S2sFocXaDZEGl8JFJy6TUDBtHA/x36P5ackXogPxnxJJ899hlOasV4rTwoAaLCXbRt25bQUMtZ3M+fP5+rV68yY8YM807k6AxNB0njo0vBoDfvfHbA4OKL7Z9nk2yn6fTRpdIKpYsPNHyh3B93a9uWqgu/Q+Xqii4xkejBg8m/dMkMQq2DrReSuZ4h9Q0b1Ea5ubJqQltCUCNpbDQwMSFeGi8W9lhIE/8miIi8s+8dVl9YbfJ5LIXWzYlejYMBqcbWZlLljf+39Z4Fj8Byf7woMZGYwUMouPw3ODpS5fN5ePV6xsQirYfvixe4qvu70a66r8xqFB5Iq1GAAFnX4eLvJj+83qDn/f3vs+rCKgB61+jN3E5zcVQ5mnwue0cJECsRGzZsoEmTJmg0GiIiIpg7d+49L5h3ppga01BPnTpF79698fDwwN/fn5kzZ2IwGDh69ChdunTBzc2NWrVqsXx52XfnYmJimDlzJv/73//w9PQ0xbf5YFoUm9VkxMLlzeafz8Z5qmEw3q6O6AwiPxQbAFg1el1p7U6zIdKKZQVwbdmSqksiUXl6or+ZQuyQoeSdOWtCodaD8eaqS21/wn0VVzerRhBKdxHPrIXcNJNP4eHkwYLuC2gZ2BKAfx/8N5FnbLdue0hxTe25xEyOx9lAX9fEUxB/SBpXwJymMC6OmMFS5oOg0RA2/ys8u3c3sUjrIbtAx/rjUinAoDbhdtPP0W7xiSitqT200KSH1hl0zNo7i7WX1wLQr04/3u/wPuoKpmhXdpQAsQzo9Abi0nKt5lGRxr+bN2/mueeew9vbm9WrV/Ppp5/y008/sWTJkjIfo2/fvrRr145169YxYMAA5s6dyxtvvMGAAQMYPHgw69ato169egwbNqzMdYSvvPIK3bp144UXyr/TUyH8akL1x6XxwQWWmdOGcXZU06eFtJu86lAceoOVr8Bf/F1amUQoXqmsOC5NmhC+NAq1Vos+I4PY4cPJPWratD65uXozu6Rv2NB2ijmNTdCoT2lf1xMrzDKFm6MbXz/xNR1COgAw7+g8vjz2pe3swN1G0zBvGlaRFh+/328D6eJHincP/epARMdyfbTg6lViBg+hKCEBwdWVsAXf4t65sxlEWg/rjyeQU6jH2VHFS80tl/mk8AgYzWqid0PyeZMcskhfxLRd0/j9qrQrOaT+EGa1mYVKUMKciuIgtwBbIDEjn06fbJdbRgm7pz1OmLZ8Fr2zZ88mJCSEzZs3o9FoAOjRowfVqlUr8zHGjx/PhAkTAOjWrRu//fYb8+bNY/fu3XTsKF3IWrZsSUBAAKtWraJp06YPPN7SpUvZt28f586dK9f38si0GQtXt0uPmxfBv45l57cxBrYJZ+HuaySk57Hl/A16NgiSW9L9Ma5I1u5Z2jfuEXCuV4/w75cTO2Ikuhs3iB09hrD5X+HWvv0jH9saMPaIC/VxoUttpW+YTeDkBk0GwKEFUipi2/FSGwwT4+Lgwpddv2Tarmlsjd3KwtMLydPlMa3VNJvapREEgSFtw5m+9jS/nUrk7V710bpZaS1Sfiac+lEatxpVrvrp/IuXiB05En1qKip3d8K++w7X5s3MJNQ6EEWxJAPi2cYheLkqaYQ2QfWuoK0htR07tBB6/feRDpevy2fqzqnsipdamI1pNIYJzSbY1HnKGlFC60pATk4Ohw8f5qWXXioJDgG8vLx49tlny3ycZ54prWEQBIG6devi4eFREhwCaLVaAgICiHmIsceNGzeYMmUKH330kUXrHQGo1QN8igNjo9ulwn2p5ufGY3X8AYjaGy2vmAeRfEFakQTJnMZEaKpXJ3zF9ziGhiLm5RH3yliyttl+r7jcQl1Jj8vBbcOVvmG2hNGR+dY1aaHLTDipnfhPl//Qq3ovAL4//z1z9s9Bb2P12/9qUgUPZwcK9QbrTpU/tQaKcsDRFZqU3W007+RJYoYORZ+aitrLi6pRUXYfHAIci73FhaQsQGnPY1OoVKWp8idXQ35GhQ+VXZjNq1teLQkOJzSbwMTmE5Xg0AQoO4hlINjLmd3THpdbRgnBXuWz6b116xaiKBIUdPfOT3BwcJmPo9Vq//FvJyenu75m/Hp+/oMNTaZMmUJYWBgDBw4kPV2qC8nNzQUgKyuLzMxM89UkqlRSisOfb0lNp7vOBhelMfiDGN4+gh0Xb7L/aioXkjKpG2SBetHyYmy+q60ONbqa9NBOoaGEr/ie2BEjKbx6lfgJEwn55GO8nrFd44dfTlwnK1+Hk4OKvi2VvmE2RUBdiOgkLYgcXgw1u5ltKgeVA//u+G9cHVz54dIP/Hz5Z3KLcvmo00c2Y/zg4qSmT4swIvdeY8XBGMZ0qm59CyKiWGpO0+glcPYq08dyDhwgbtx4xNxc1L6+VI1cjHOdypEVE7VPWohuWMWTxqFl+3kpWAlNB8K2D6QFkZOroc0r5T5Een46Y7eM5Wyq5A8wo/UMBtUbZGqllRYlQCwDDmpVuVM6rQkfHx8EQSApKemu1xJlagZ+9uxZTp48ia/v3Y5jnTt3JjAw8J56TUbTQbDt39LJ6cQKaDfefHPZAZ1r+VPd342rN3OI2hvN3Bcbyy3pn+RnwknJtYyWo8yScucYGEj48mXEjh5DwfnzXH/jTcS8PJu0jhdFkWXF9Vi9Ggdbb8qdwv1pOVIKEC9tgvQ48DZfkK8SVLzd9m1cHV2JOhvFH9F/kKfL47PHPkOj1jz8AFbAoLZVidx7jbi0PHZdusnjda0spTpmL9wsrscqozlN1tatJLw+BbGwEIeQYMIjI3GKiDCfRisiMSOPTael+5fh7aspO0a2hos3NO4HR5dImVytxpTrup2cm8zLm1/mSsYVVIKK99q/x3M1nzOj4MqHkmJaCXBzc6N169asXbuWgoKCkq9nZmayYYM8zZAXLVrE9u3b//GYPn06AAsXLuTnn382rwAXb2g6QBof+k5pefEQVCqB4e0jAFh3PIFbOYXyCrqTU2ugMBscXKCZ+VYQHXx9CY9agkuTJiCKJL49m9QlUWabz1wci73F+cRMAIa2i5BXjELFqNsL3ANBNMDRKLNPJwgCU1pMYXxTaTFtZ/xOxm8ZT25RrtnnNgU1/N3pWNMPKK29tSoOSA29CW0FIQ+u3wfI+PVX4idOQiwsxKlaNSJWrKg0wSHA8v0x6Awifu5OPNuk7JlQClaE0awm9W+4tqPMH4vPimfYpmFcybiCg8qBz7p8pgSHZkAJECsJH3zwAQkJCfTo0YP169fz008/0bVrV9zd3WXR07JlSx577LF/POrWrVvyWntLmIC0fll6vhUNl/8y/3w2zgvNQ/HQOFCgM7D6sBXV8YhiqTlN4z5S/0MzovbyImzxYlzbtAEg+eOPSf78c5tyeDTuHjYO9aJpmJJebZM4OEHzYdL46BIoyjP7lIIgMLbJWN5s+SYAB5MOMnrzaNLzbaB9BKV1atsvJhOXZkWBbdo1uFDcE67tqw9/+8qVXJ82HfR6NMVGWo7lKBexdfKL9Kw6FAtIrS00DkobA5sksAGES07JHFpUpo9cSb/CsE3DiM+Ox1ntzPyu83ki/Akziqy8KAFiJaF79+6sX7+e9PR0+vXrx9SpU+nTpw8jR46UW5p8+Ne5reXFt/JqsQHcNQ70bSWlsS3fH12hditmIXo3pFyUxiY0p3kQanc3wr5bgHs3qfYr9dsFJL33HqLe+neib2YVsLE4NUsxdrBxWo0ClSPkpsLpHy027dAGQ5nTbg4qQcXplNMM/2M4N3JuWGz+ivJEvQCCvZwRRSvbRTy0EBDBswrU+9d93yaKIikLvuPG+x8A4NK8OeFLo3C4R6mGPbP+eAK3cotwVAsMaltVbjkKj4JxF/HSJrj14L/JsylnGf7HcJLzknF3dOe7Ht/Rvop9OIpbI4ItrXo/KoIgpHt5eXkZTVHuxOi8GR6u3DRZMyb9f7r4B6zqJ43HH1JaXjyE2NRcuvxnO6IIXw9qztONrGDVes0QOP8rhLWBUZstOrWo05E4+x0y1q0DwPPppwiZOxfByXpr+uZv/5tP/7yIt6sjB2Z2w9lRWX23aX5+BU6tBv96MG5/uVojPCqbozczY/cMigxFhLiFsKD7AiK8Iiw2f0Uw/v57ODuwf2Y33DUyWzHkZ8J/60NhFjzxHnScfM+3iaLIzc8+I3WRZGTj1rEjof/7EpWLiyXVyo4oivT8fBeXbmTzQvMq/Lfvw9NxFawYfRF83giyEqHDZOj+3j3fdjjpMBO2TSCnKAcfjQ8Lui+gnm89C4u1L7y9vcnIyMgQRfGeaUTKDqJC5UZpeVEuqvq60q1uIGAlLS8yEkpTs4wpwxZEcHAg+N8foh0mpfplbtxE3PjXMOSZP92vIhTpDSV9w/q2DFOCQ3ug3Tjp+eZ5s7a8uBc9Inowv9t8XBxcuJ5znWF/DON8qmkaX5uLga2r4uyoIitfx4/W0PLixEopOHR0heZD7/kWUacj6Z13S4JDj549Cft6fqULDgH2XUnl0o1sAEZ2KHsfZwUrRe0ILUZI42PLoOhuB/ytsVsZ+9dYcopyCHANIOqpKCU4tABKgKhQuTG2vACp5UWebdTSyMmIDhEAHIpO40xCxfsXmYSjS0DUg1vAA1OzzImgUhEwYzr+kycBkLN7N7EjR6HPkPlncw82nk4kMSMflQBDlPRS+yC4CYQX96I1Gp1YkHYh7VjUYxFeGi/S8tMY+edIjiQdsbiOsuLj5sSLzaXeu0v2RqM3yJhFZdCXljc0GQCud7eNMhQUED95Muk/SinEXi++QJX/fmbVWQrmZMneawC0ivChYRWltYVd0GK4lCqflwZn/2lQ+PPln5myYwqFhkIiPCNY/tRyqntVl0dnJUMJEBUUmg4CR7fSlhcKD6R9DV9qB0rmRlH7ouUTUpQHRyKlcYthkmmHTAiCgN/YsQS9+w4IAnnHjxMzdBi6mzdl03QnoiiycPdVAJ5sGGTTrXsU7sBobHJ5M9y8ZPHpG/s3JqpnFAEuAWQXZTN2y1h2xO2wuI6yMrKjtPMUm5bLX+dkrJ289CfckgIe2oy962V9VhZxo8eQvWUrAL5jRhP84YcI6sq58x+dksPWC8mAsntoV3gEQv3e0rg4k0sURRadXsS7+97FIBpo4NuApU8tJcQ9REahlQslQFRQUFpelAtBEBjeXro4/3riOinZBQ/5hJk4uUoy51A7QavR8mi4A58BAwj59FNwcKDg4kWiBw6iMMY6zDAOXkvjTILU2mJUR2UF1q6o8xT4REjjg5bfRQSo6VOTZU8vI9wznAJ9AZO3T2bDFXnaKD2MGv7udC3ugxi555p8Qg58LT3X7A7+tf/xUlFyMjFDhpJ7+DAAAdOnEzB1aqXu9xe1LxpRhCreLnSvHyi3HAVTYszkun4cQ8w+Pj3yKV8c+wKAtsFtWdxzMVrnu3fYFcyHEiAqKIDS8qKcPN+sCl4ujhTqDaw6GGt5AQYD7J8vjRv1BY8gy2u4D169niFs/lcIzs4UxcURPWAgeafPyC2LRbulG+HmVb1pEW7eViAKFkalhjbFu4gnVkFumiwyqrhXIerJKOpq66IX9by15y2Wnl0qi5aHMap4F/FQdBqn4mUoLUg6LTkww12tLQpjYogZOIiCCxfAwYGQj+fiO2K45TVaEVn5Rfx0NB6Aoe3CcVArt692RVgbCGlOEfD2ruksP7ccgJ4RPZnfbT5ujm7y6quEKH9hCgqgtLwoJy5Oavq3Lm55cSCGIku3vLj0h9RcF6DdeMvOXQbcu3Sh6pJI1F5e6NPSiBk2jOw9e2XTc/VmNlsvSKl0ozspu4d2SbNBoPEEXR4cjZJNhp+LH5E9I2kZ2BKA/xz5D58c/gSDaCVtcYppX8OXukEeACyWYxfxQPF1xq8O1Oha8uW8s2eJHjiIovh4BGdnwr6ej1fv3pbXZ2X8eCSe7AIdLo5q+rdSWlvYHYJAXttXmBTozwaDtGDTr04/Pu70MU7qyllvKzdKgKigYMRYA3J1O9y8KK8WG2BI23BUAiRnFbDpTJJlJ9//lfRc8wkIrG/ZucuIa7NmhK9aiUNIMGJuLnFjx5KxQZ6Uu8i91xBFCNO60LOB9ey2KpgQjUepC+ahhZJ9vEx4OHnwbfdv6R7eHYDl55YzY9cMCvWFsmm6E0EQSnYRfz+VSGKGBZ2Hs2/C6R+kcdtXS1qT5Bw4SOzQYehTU1F5eVF1SSTunTtbTpeVojeILN0fDcCLLarg5eooqx4F05Oen87L8b+x21Vy5h3nUp1ZbWahVlXOeltrwKwBoiAIgYIgLBUEIUUQhBxBEHYLglCmrpaCIEQJgiDe43HAnJoVKjFKy4tyEerjSo/6UrBhdJazCAlHIaZ4N67da5abtwJoqlcnYtVqNLVrg07H9TenkRq5xKIabuUUlqRmjWhfDbWq8tYw2T2tXwZBBVnX4dwvskrRqDV82vlTBtSV6rs3RW9i3JZxZBdmy6rrdv7VNAQ/dw06g8jSfRasFT4SCfpCcPGBxlIf3sxNm4gbMwZDTg4OQUFErPge12bNLKfJitl2IZmY1FyAkvp3BfshPiueIZuGcOLmSQRgVkoar146gJCbKre0So3ZAkRBEJyBrUAXYALwPJAFbBUEoaxnvWyg3R2PUaZXq6DAHS0vVspWx2NLGFteHI9N50Schep49hXvHgY2guqPWWbOR8AxMIDw75fj2qoVAMmffMKNuR8jGiyTcrfiYAz5RQY8nB3o2yrMInMqyIRPONTtJY33zwdRxhYOgFqlZmbrmUxqLrWAOZh0kOF/DOdmrnW4+2oc1AxtJ7V7WXkwhpwCnfkn1RXA4UXSuMUIREcXUpdEkfD6FMSiIpyqVydi5Qo0NWuaX4uNYFyA7FLbn5oB7jKrUTAlZ1PPMnjjYKIzo3FSOfFp+w/pX+QAuvzSvxMFWTDnDuJIoAHwgiiKq0RR3IwUJCYCH5XxGHpRFA/c8ThrLsEKCjQbAhovKMpVdhHLQOtqWuoFewIW2kW8FQPn1kvj9q+VpGZZO2pPT8IWLcSjZ08A0qKiuP7mNMRC86bcFej0LN0v7YwMbF0Vd42DWedTsAKMNbnXj0HcIXm1IKVyjm40mg87fIhaUHPx1kUGbxzMtQwZ3UNvY1Cbqjg5qMjM17H2WLz5JzzzM+Qkg8oBsfkIbnz0fyR//DEALs2bE7FyBY4hipW/kQtJmey7Iu0kGRckFeyDPQl7GPHHCFLzU/F08uS7Ht/Rs1ZvaFW8D3ToO6mdlYIsmDNAfB44LYriMeMXRFEsAFYB3QVB8DDj3AoKFcPZE1qNlMYHF0Bhjrx6rBxBEBhZfNH+7VQicWm55p3w4LcgGsAjGBq8YN65TIxKo6HKfz/DZ+BAADJ//53Yl19Bn5lptjl/PXGdm1kFOKgEhis3V5WDsDYQUpykc2C+vFpuo3fN3vyv6/9wcXDhes51KaUs+YTcsvB11/Bi8yqA1PLCYDDjrqsolrS2MNR6loR3/8Ot5ZJbo0fPnpKxlbe3+ea3QaL2RgNQ3d+NzrX85RWjYDLWXV7Ha1tfI0+XR7BbMMufWk6LwBbSi61fltpX5aZK7awUZMGcAWJD4F7e7qcANVCvDMdwFwThhiAIekEQYgRB+EwQhPvmFwiCkP6gB+BVsW/FfpkzZ85dfZUEQWDOnDlmnzsiIgJBEO56zJgxw+xzP5A2r4JaA3lpcPx7ebXYAL2bViHYyxm9obQRu1nIS4djy6Rxm7HgYHvOZoJaTeDst/GfPBmA3AMHiB44kKKEBJPPJYpiiTvjM42DCfZyMfkcClaIIEDb4l3E8xukXXcroVNoJyJ7RqJ11pJRkMHozaP5K0b+tkLGpuvRqbkljdjNQsw+SDqFrkAg9qc0sv6SvnftsGFUmfdfVBqN+ea2QVKyC1h3XDo3jmgfgUqpn7Z5RFHkm5Pf8M6+d9CLeur41OH7p7+nuvdt7toeQdC4rzTe95XSm1omzBkg+gL3KuJKu+31B3ESeAMYBDwN/Ai8BmwTBEGxsDIj+/fvZ/RoyzQe79y5M/v37//HY/x4mdsWeARCU2mXh33/k9UN0BZwclCVtE5YcziOlOwC80x0NAoKs8HJHVoMN88cFkAQBPzGvkLIJx+DoyOFf1/hWv/+Ju+VuPtyCheSsgAY3VFpbVGpaPCctMsuGqwuVb6hX0OWPbWMqh5VKdAXMHXHVKLORCHKWC9ZK9CDLrWl3anFe8y4yHXgawqz1cTsCCPv7GUQBAJmTCdw5gwElWIqfydL9l6jQGfAy8WRF5qHyi1H4RHRGXS8t/89vj4h7aK3C25H1JNRBLgG3P1mowFd2hW4uMmCKhWMlOmMJAjCY/dxFL3Xw++2jz7ojP/Aq4EoivNEUfxcFMUtoij+KYriG0gBYiug330+4/2gB5BRlu+3stO2bVtCQy1zMvbx8aFt27b/eISFWYGRRvsJkhtgRpxUM6LwQPq3CsPb1ZECncE8tYi6QinlF6Q6URfbT8Py+te/qLpoESpPT/Q3U4gZOpSsbdtNdvxFxbuHbappaRSqJE9UKtSOpYZbx5ZBQZa8eu4g3DOc75/+nqb+TRER+ezoZ3x44EN0BguYxNwHY8uLA1fTOJNghluFm5fI27eZ6L/8KLylQ3Byosq8/+I7fLjp57IDMvOLWFbsLDuiQwRuSv20TZNTlMOEbRNYe3ktAP+q8S/md5uPu9N9kgID6knO8iAt1CtYnLIuWV0ARpTxYbwSpXLvXUJt8XNFLCK/BwxIbqaWQ6+T0nSs5aGv2EV0w4YNNGnSBI1GQ0REBHPnzr3nqu2dKabGNNRTp07Ru3dvPDw88Pf3Z+bMmRgMBo4ePUqXLl1wc3OjVq1aLC+uqbBpfGtA/eLmxHs/l90N0Npx0zgwvH0EAMv2x5CVb+Jd17M/S9b9gkrqG2YnuLVpTcSqlThWqYKYl0f8a6+RtmLFIx/3YlIWuy5JTpFjOim7h5WSFiPAwQUKMqXddyvDx9mHRT0X8WTEkwD8cOkHJmybQE6RPHXfnWr5UTtQulmN3GP6Ra6sBW8Rs1WLvkCNytOTqksi8XzySZPPYy8s3x9DVoEOVyd1ybVFwTZJzE5kyKavDrKgAAAgAElEQVQh7EnYA8CYRmP4sMOHOKofkgzYfoL0HHfAKgy3KhtlWpIRRTEJiCrnsc8i1SHeSSNAjxR0lhdjArpl/OGNZCbAF40tOuUDmXRKsjMvB5s3b+a5556jY8eOrF69Gp1Ox8cff0xyctnrLfr27cvw4cOZMGECv/76K3PnzqWgoIDffvuNN998k1mzZvHVV18xbNgwGjVqRNOmTR96zG3btuHu7k5hYSF16tRh3LhxjB079q66SFnoMBnOroPkc3B5M9TuKbciq2ZYuwi+23WVrHwdKw7GMrZLDdMcWBRLW1vU713u331rR1OjBhFrVhP36jjyT5/mxgcfUhQXT8C0NyucdmZMk6vu50bXuvdI31Gwf1y10HwoHFogrcC3GgOOznKr+gcatYaPO39MFfcqLD6zmD0Jexi2aRjzu80n0C3QoloEQWBUx2pMX3uaX09eZ9qTdQnyevSflyiKpH31GcnfnwFUOPp7EbZ0JZrqysLN/cgr1JcE6YPbhuPtanv15goSZ1LO8NrW10jNT8VBcGB2u9m8UKuMBnMRnSC4CSSelM5h/exg88GGMGfS+zqgkSAIJVGCIAhOwABgiyiKFbHuG4yk+YBpJFYeZs+eTUhICJs3b+b555+nT58+bN26lezssjctHj9+PDNmzOCJJ57giy++oFq1asybN4/IyEjGjBlDjx49iIqKQqVSsWrVw52nevXqxZdffsnGjRtZs2YNtWvXZty4cUyZMuVRvlXTEdIUqj8ujfd8Lq8WG8DHzYkBrasCsHjPNfKLTFRYfnUH3DgtjdtNMM0xrQwHPz/Cly3FvVs3QGqDkTBpMoa88lt8J2fls/74dQBGdqymGDtUZjpMBJUjZN+A49Z5c6USVExuMZl3271b0gZj4MaBXEy7aHEtvZtWwc9dg84gsmDXlUc+nlhYSOLs2STPXwwIuARCxE/rlODwIaw5HEtqTiFOahWji1N/FWyPLTFbStpYeDh58G33b8seHIJkuNV+ojQ+vwFSH/1vUqHsmDOpezEwHvhZEISZSCmlk4AQoO/tbxQEIRpAFMWI4n+HA8uRWmJcQXI9fQKpBnE/sMaMuu/Gs4q0a2cteFYp19tzcnI4fPgwkyZNQnObS5qXlxfPPvssy5YtK9NxnnnmmZKxIAjUrVuXlJQUOnbsWPJ1rVZLQEAAMTEPd8776quv/vHv559/nkGDBvHll18yefJkwsOtYKeo42S4uh1i90HsQajaRm5FVs3oTtVYtj+am1kFrD0Wz6A2Jvg/3F/8e1K1PYS2ePTjWSkqFxdCv/yCGx9/zK1ly8n66y9iBl8n9Ov5OAaWfTdl6b5oCvUGfFwdeVExdqjceIVKhlvHlsLeL6D5MKt1/32p9ksEuwUzdedUknOTGbppKJ92+ZTOoZ0tpsHZUc2YTtX4v00XWHkwllcfq0GAR8V2EfUZGcRPnETuwYMAeFbNJfjtN1EFBptSst1RqDPw3S4pA6JPy1ACPK1r11vh4YiiyJKzS5h3dB4Aoe6hzH9iPtW9KrAwUv852DJH8oM48DU885lpxSrcF7PtIIqimA90BfYC3wC/AN5Ad1EUjz7k45lACjC9+HPrgKeAuUA3URQtW8mudpDS2qzloS5fXH/r1i1EUSQoKOiu14KDy36x0mq1//i3k5PTXV8zfj0/P79cGo0MGzYMg8HAoUNWkm9erQsEF2+C71V2ER9GsJcLzzeTFjAW7LyKTv+I2eA3zsHfW6Rx+9ceUZ31I6jVBL31FoGzZoFKRf7Zs0S/1Ie806fL9Pn03EKWFhs7DGkXgYuT2pxyFWyBjq+DoJZusE5Zdm21vHSo0oGlTy4l0DWQXF0ur219zeIOp4PbhuNTbLi1aHfFahELY2KI7j+gJDj0a5hJSDcHVG1HmlKqXbL+RALXM/JRqwRe6WyiMgUFi1FkKGLO/jklwWGzgGasfGZlxYJDkO53246TxsdXQE6qiZQqPAyz+iqLopgkiuIQURS1oii6iqLYURTFPfd4X4Rx97D437dEUXyh+Osuoig6i6JYXxTFd0VRLH/OVSXHx8cHQRBISkq667XExEQZFN0fg0EKKFTWYvktCNINFsDFjZBckdLZysUrXWogCBCblsvGM3f/zpWL3cWrhdoaUPupRxdnI2iHDCZswbeo3N3R3bxJzOAhZG7c+NDPRe65RnaBDg+NA6M6KKlZCoC2GjTqI433/LfCJmeWoo62DiufWUlD34YlDqfv7HuHQn2hReZ30ziUtO35/kAMaTnlmzf38GGi+/aj8No1BEdHQjrm4t8wG6H9eHByNYdku0FvEPl2h5RG+K8mIVT1VX5etkRGQQav/vUqP1+WnN+frvY0C3ssxMfZ59EO3HwIaLxAlweHF5lAqUJZsJK7cAVz4ubmRuvWrVm7di0FBaU96jIzM9mwYYOMyu5m2bJlqFQqWrVqJbeUUuo9KwUoAPu+lFeLDVDD350nG0i71d/suFLx1f/kC3BGssSm8xtgLYsGFsK9Uyci1qzGsWpVxIICEqZM5eb/vkI03HtXNiO3iCV7owEY3iECL1elXaxCMZ2mAgKkXZWMt6ycANcAljy5hKerPQ3A+r/XM3rzaFLzLLN7MLRdOJ7ODuQW6svVFzH953XEjByFPiMDtVZL1YmP4xWaDs5e0MoyvYVtmT/OJHE1RXKxffUxZffQlriafpWBvw/kYJK0az6uyTjmdpqLRq15yCfLgMYDWo6Qxge/gfyKWJgolJfKdcdVifnggw9ISEigR48erF+/np9++omuXbvi7n6fHjRmZtWqVfTv35/ly5ezfft21q5dy/PPP8+qVauYOnUqVatWlUXXPVGpJbMHkFK0MuLl1WMDGC/u5xMz2VHcbqHc7PoEEEFbHRr1fejb7RGjw6lr69YApMyfT8KUqfc0r4nce42sAh3uGoeSnm4KCgD414YGz0nj3f+B+ywyWBPODs7M7TSXic2kc+/x5OMM/N0y5jUezo6MLP4bWrovhvTcB+8iijodSR99ROJbb0FREU41axCxfBGuKcXBeOtXwNnT3LJtGlEU+XrH3wB0rx9I7UAPmRUplJWdcTsZuHEgsVmxOKmcmNtpLq82fdW0bvTtxktte/JuSc7MCmZHCRArCd27d2f9+vWkp6fTr18/pk6dSp8+fRg5Up6aiGrVqpGSksK0adPo2bMnw4cP58aNG0RFRfHJJ5/IoumBNO4P7oFg0MH+r+VWY/U0DvWmY00/QNpFLDfJ5+GMlKZC52nlrru1Jxx8fKi6aCHefaUgOeuPP4gZPISiGzdK3pORV0TkXqlealh7xRZe4R50ekN6vnkBLlhX5sj9EASBMY3H8Pljn+Pi4ML1nOsM2TSEbbHbzD73iPbVcNc4kF2gK9mZvxe6W7eIHTOGW8skl1i3Lp2JWLUKp+sboSADHN3sqnerudh56SZnr0s7Q+OU3UObQBRFFp1eVNK/NMA1gGVPLeOZ6s88/MPlxT0AWo+Rxvu+gvwM08+h8A8ESxZ/y40gCOleXl5e6enp93zd6LxpFe6ZCvdFtv+nPfMkNy1HN3j9jNRnTOG+7P07hUGLpHSTta+2o0V4OX5eP46Asz9Lqb3jD1XqANGIKIrcWv49N+bOBYMBtb8foV98iWvzZnyx5TLztlzCzUnNnuld8XFTAkSFe7BqgFRLHdQIXtkt1VjbCBfTLjJh2wQScxIREJjYfCKjGo4ya8/cT/+8wPztV/B0dmDPjK54Ov8zbTv/4iXix4+nKF7KKvF9+WX8J01E0BfA540gN0Vq9t3jQ7NptBf6frufQ9FpdKjpy4rRbeWWo/AQ8nR5vLv3XTZFbwKgiX8T5j02D39Xf/NNmpMCnzeGohx47C14bLr55qoEeHt7k5GRkSGKove9Xld2EBUUykrLkaDxlE5Oh76TW43V076GL01CvYBy7iImny+tk+pSuXcPb0cQBLRDh5SY1+hvphAzbBiJy1eweLf08x3aPkIJDhXuj3EXMek0XN4sr5ZyUkdbh1XPrKJZQDNERL449gVTd04lpyjHbHOO6lgdVyc1mfk6lu2L/sdrmZs3Ez1gAEXx8QjOzoR89h8CpryOoFZLbUVyU0CtgXb27778qBy6lsah6DQAxj9WU2Y1Cg8jMTuRYZuGlQSHz9V8jsiekeYNDgHc/KDNy9J4/3zIu/dmj4JpUAJEBYWycrvRwP75kJsmrx4rRxCEklrELeeTuZBUxsLynR8DIvjWhIYvmU+gjeLeqRMRP/6AU40aUFRE+r8/ZOT+VXipDYzppDTgVngAoS2gRldpvPMTsLEMIl8XXxb1WMRzNaV6yr9i/mLQ74OIzog2y3xaNyeGtJUyVRbvuUZOgQ7RYODm/74iYeIkxNxcHEKCiVi5Ai9jn2BdAewtNjNrPgQ87m4vpfBPjLWHTcO8aVfDV2Y1Cg/i2I1j9P+9P+fTzqMW1ExvNZ3327+Pk9pCC5PtJ4KTu5S+fUAp9zEnSoCooFAeOkyU7JYLMqWUU4UH0qN+EDX83QCY99elh3/gxjk4u14aV/LawwehqVaNiDVrcOnWDYCesYf45vB3eGQqPaIUHkLnN6XnhCNwbae8WiqAk9qJ99u/z9tt3sZB5cCVjCsM+H0A22O3m2W+0Z2q4+yo4lZuEau2nyd+4kRS5s8HwLVlS6r9+CPO9euXfuDkKsi6DioH6DDJLJrsiTMJGey4KBmZjX+8pllThhUqjiiKrDy/klGbR5GWn4ankyffPPENg+sPtuz/masW2oyVxge+URbqzYgSICoolAcXn1JH00PfQeZ1efVYOSqVwOQnagPw59kbHI+99eAP3L572EjZPXwQanc3fn9xEkvqP4UBAZ/Yv7n2Uh9yDx+WW5qCNRPeHsI7SONd/5FXSwURBIF+dfuxpOcS/F38yS7KZuL2icw/MR+DaFqHVn8PDQNbhxOemUj12a+RvWUrAD4DB1B1SSQOvrfteOl1pQuHjfuBtxW5cVsp87dLu4d1Aj3oVjdAZjUK9yK3KJfpu6fzf4f+D51BR03vmqx+ZjXtQtrJI6jdeKncpyBTyuZSMAtKgKigUF7ajAU3f9DlS2laCg/kmUbBNAiRLN4//uPC/fsi3jgH54p3D7tMl9qLKNyX7AIdC/dc44fa3dg/ZhYqLy/0qanEjBhJ2rLlFe8/qWD/dC6uRYzeDTH75dXyCDQNaMqaXmtoFtAMgG9PfstrW18jo8C0DofDc8/z+c7/EZyVjMHh/9m77/Aoqi6Aw7+72c2m90ZCIIQi0osU6UgvoVdRQUUFlU9QUbACNrBiVxRFUEFBOgiE3kTpvYQQICQkIaT3LfP9MSGhBEhCkt0k932ePDs7M7t7NjvZzJl777la/GZMx++ttxC6m+YaPb4UEs8DAtpNKtEYKqJDkUn8fSwGgGc710Sjka2H1iYiOYJRa0fxd4Q63rBXjV781vs3Al0CLReUg0d+ZeB/v5OtiKVEJoiSVFR6p/xuWgcXwNViTONQiWg0gld71gVgz7kEtofFF7zjtpnqrWdtaDC4jKIrv+b/c56kDAN2Og39nx5CjSWL0d93HxiNxL7/PtGTX8GcXnoFPKRyLLgzBDRXl7d/ZNlY7pG3gzdzu89lZN2RAOyI2sHINSNLZL5Ec04Ol6dPJ/3tN7Az5RDj4M707pOwH1jA95MxB7a8ry7XHwBete/59SsyRVGY+fdJAOpVcSGkkb+FI5JuFnohlJFrRnI26SxaoWVKyynMaj8LB52DpUOD1s+qw31y0mD3F5aOpkKSCaIkFUfzMeBaTZ0XcesHlo7G6rWv7cWDwWpXrA/XncJsvql1K/Y4nFihLsvWw7tKzzbyw/ZzADzSqjreznpsAwMJWvg7Lr17A5CyejURQ4aSdboQYz+lykWI/Itc4Zvg/E7LxnOPdDY6Xmv1Gu+2fRe9jZ7I1EhGrR3F0rClxW5JN0RFcWHUIyQtXASAtk07XuzyIv/Z+rJ4X+StD9j/MyRGgLBRS/BLd7T1zBX2nFNbfqb0qitbD62I0Wzk470f8+LWF9X5De19+Lnnz4y6f5T1jBG1d1O7mgL8O0edAkMqUTJBlKTi0Oqh0xR1+egSiDlm2XisnBCCV3uprYjHo1NYc/TyjTtszW099KoDDQaVcXTlz4I9F0jMMKDXani6Y37lUo2DA/6ffIzvG2+ATkdORATnhw8n6a+lFoxWskp1ekLVFuryhjfAXLJj9yyhf63+zO81nwCnALJN2by9+21e2/kaGYaMIj1P2o4dRAwaTNbRoyAE3hNfoNaP39PzQXU89ddbwsnMMeU/ICs5/zus+WjwrlNSb6lCMpkVZv19CoC2tTxpX9vLwhFJ18RnxjN2w1h+OfELAC39WvJnyJ808Wli4cgK0Hoc2LmpU4/JVsQSJxNESSquxiPA6z5Agc1yIuS7aRLoRs/6asn3TzacxmDKPSGNOQYnV6rLsvXwrjJy8lsPR7Wqjo+z3Q3bhRB4PDKKoN9/R1e1KkpWFpdff53oKVMxZxTtRFmqwISA7u+py9EH4dhflo2nhNTzrMefIX/SpZpa4Xf1udWMWDOCM4l3b0lXTCaufPkVkU8/gyk5GRt3dwJ//AGvceMQGg3PdqqFrVZDTEoWP+w4l//AnbMhMwF0jtBxSmm9tQpj2cEoTsWkAjCl5/3W0ypVye2O3s2QlUPYH7sfgCcaPMH33b7H095Kpx6xc4U2ufOM/vcDpF2xbDwVjEwQJam4NDbw0Bvq8pm/4eK/lo2nHHi5Rx00As5fzeCPvbndtLbNUm+97oP6Ay0XXDnx3bZzXE3PQa/VMK7j7ec9tG/YgBpL/8Kpq3qinLx8ORHDhpF99mxZhSpZu2qtoF5/dXnTdDBkWTaeEuJi68JnnT5jSsspaDVaIpIjeHjNw3fscmq4fJmLo8eoU1goCvaNG6t/P23b5u0T6OHAE21rAPDt1nBiU7Ig+VL+fGxtXwBn31J/f+VZlsHEpxvU8aH9GvvTsKqrhSOSDGYDs/fPZlzoOK5mXcVJ58TszrOZ1HwSWo2VTzXVapxaXd6QAbtmWzqaCkUmiJXctGnTbrl6J4Rg2rRpZfL6cXFxPPfccwQGBqLX6/H392fgwHKUJNwfAv5qBT02zSh3E0+XtVo+zgxtrlY/+3xTGFkRe65rPXxFth7eRVRSJt9vU4siPdU+GB8Xuzvub+PiQtUvv8R36hTQask5G07E0GEkLV9eFuFK5UGXt0Gjg+RItSJgBSGEYNT9o1jQa8Fdu5ymbtxIxICBZOzbB4D7Y49SfcF8dFWq3PK8z3auiaejLZkGE59sOK0WpjFmgZNvfmuGdFvz/zlPdHIWOhvBy93vs3Q4lV5UWhRj1o1h7rG5KCg08mrE4pDFeS3wVk/vDG1ypx7bOxdSYy0bTwUiE0TpFv/88w9jx44t9de5dOkSLVu25L///mPmzJmEhoby2Wef4e7uXuqvXWKEgC5vqcsXdkL4ZsvGUw680LU2tloN8amZJC99UV3p11C2HhbCrL9PkW004+OsZ3ynmoV6jBACj9GjCfp1AVr/KiiZmVyeMpWoV17BlJpayhFLVs+zJrR8Sl3e8QmkX7VsPCWsgVcD/gz5k67VugJql9Phq4dzOuE05qwsYmbM4NLzE/K6lFb99hv8XnsNYWtb4PO52OmY1E0dY3jswC6UQ7+rGzq/BraOZfKeyqvkDANfb1EvcI1qVZ1qnlZQDbMS23B+A0NXDuXIlSOA2qV0Xq95VHWuauHIiqjl0+DgCcZM2Pq+paOpMGSCKN2idevWVK1a+l8Q48ePx93dnZ07dzJq1Cg6dOjA8OHD+emnn0r9tUtUcGcIaq8uy1bEu/J3s2dMmyAGaXbim3pcXdnrQ9l6eBf7LySw8nA0AK/0rIujvmhdf+ybNCF46VKcOncGIGXlKrXV5MCBEo9VKmc6TFbH82Sn5Hf5rkBcbF34tNOneV1Oz6ec5+V5wznYrzuJvy8EwKF1a2osX45z7t/HnYxoEUhtHyem2PyOQEHxrgtNHintt1HufbPtLMmZBpz0WiY8VMvS4VRaWcYsZvwzg5e2vUSqIRUPOw++7/o9k5pPQqfR3f0JrI3eSa1fALD/F3VMtXTPZIJYCEazkai0KKv5MZqNxXofq1atonHjxuj1eoKCgpg5c2aB4zFu7mJ6rRvqkSNH6N+/P87Oznh7ezN16lTMZjP79++nY8eOODo6Urt2bRYsWHDXWCIiIlizZg0TJ05Er9cX6/1YDSHUbloAlw/lT9cg3db4B32ZolPLx5/w6AbV21g4IutmNitMX3UCgMZVXRnUNKBYz2Pj5kbVb77G9603EXq9Wsr/kUe58sUXKAZDSYYslScOHvnTXuybC/EVb5zqtS6nv/ZcwNBT7rzzUzYOF69g1oDdc2OpNvdHdL4+hXourY2Gj5vF08HmKACH75sENlY+VsvCopMy+XnXeQCe6RCMp1M5/79fToUlhjFyzUgWn1kMwINVHuSvfn/RJqCc/w9+4EnwqQcosPYVeaG+BMhvtEKIzYil5189LR1GnnWD1xHgVLQTxA0bNjBgwADatWvHokWLMBqNzJo1i7i4uEI/x7BhwxgzZgwTJkxg5cqVzJw5k+zsbFavXs3kyZN5/fXX+eqrrxg9ejQNGzakSZPbl0XesWMHiqLg4uJC79692bx5M1qtlk6dOvHxxx9Tt27dIr0/iwtsAff1htNrYct7ULevPGG4A/d9s0EkkanYMj6uP4uSM6niam/psKzW0oNRHLmUDMBbIfXuac4wIQQeDz+MY8uWRL08mexTp4j/5lvSdu0i4KOPsK1WraTClsqTlk/Df3Mg6SJsfBtG/GbpiEqcMSEB1xnfMzRUrXYY5wqf97Mhxusv3oqsT8+gQv6fN5tofPIzAP4x1eP1g76s72xGZyOvud/OZ6FnyMntHv9k+xqWDqfSMZlNzD8xny8PfonBbMBG2DCh6QQeb/A4GlEBjlsbrdoT6Ze+cOk/OPKHWmleKrYKcFRIhfHmm2/i7+/Phg0bGDhwIEOHDmXTpk2kpaUV+jmee+45pkyZQteuXfn888+pUaMGn332GT/99BNPPfUU3bt3Z968eWg0GhYuXHjH54qOVrvKjRkzBn9/f1avXs2cOXM4ceIE7du35/Lly3d8vFV66A1AQPwZ2FfOusmWpavh8I9a9W+ezSAuGD34fGOYhYOyXunZRj5cp84ZFtLYn+bVPUrkefW1ahH05x94PPEEAFmHjxAxYCBJS5cVe3JxqRzT6qHrNHX51Gq4sNuS0ZS41I0bOdc3hNTQjQA49+qJ8vPHxNf0IDUnlcnbJvPajtdIzSnEuNwjf0Ks2no40/Qw565m8OueC6UZfrl2OiaVvw5cAmBi1zo42MqLp2UpMjWSJ9Y/waf7P8VgNhDoHMgvvX7hyYZPVozk8Joa7fNrGYS+BVkplo2nnJN/pYXg6+DLusHrLB1GHl+HopXRTk9PZ+/evbzwwgs3dOd0dXUlJCSE+fPnF+p5+vTpk7cshKBu3brEx8fTrl27vPUeHh74+Phw4cKd/1macydlfvDBB/nxxx/z1jds2JDGjRvz9ddf8+675WxuQd/60OxRODBfHYt4fwi43FoFr9Jb/xqYDeBaDdcWk2B1OH/ui+SxB4Oo5+9i6eiszrdbw4lLzcZOp2FKr5JtWdfY2uL7ymSc2rUlespUjHFxXH7tNdK2bsXv7bfQelrp/FdS6ag/SL14E7UP1r8OYzeBpnyfQJpSUoh9732SV6hd/zXOzvi98Tou/fpRVQiW1mjFm7veZGfUTladW8X+2P180P4Dmvk2K/gJDZn58942GEIDm04c/vciszeGMbBpAG4OBRe3qcw+XHcKswLB3o4Me6CcFUApxxRFYWnYUj7c+yEZRrVy7/D7hvNi8xdx0FXQAkHd34XT6yAtFrZ/BN3fsXRE5ZZMEAtBq9EWuUunNUlMTERRFPz8/G7ZVqWAMt634+FxY8uFra3tLeuurc/KuvN8Wp65J549evS4YX3Dhg2pWrUqB8pr4Yyu0+HUWsiIh3VTYNgvlo7IuoRthDO5F1u6v8OQ++ow998Ywq+kM3XpEZY+2xabe+g+WdFEJmQwJ3dC7qc71CTArXS64Tq2aUONFcuJeettUkNDSd2wgYy9e/F7602ce/aUE1lXFkKoJ1g/94ToA3B8KTQcYumoii1t1y4uv/4GxpgYQD3Oq7z37g3TV3jZe/FNl29YdHoRn+z7hOj0aMasG8Mj9R5hQtMJ2Gtv+pv79ztIuQQ2ttDlTSbpqrDiUDTJmQa+3HyWN/vWK8u3aPX+Cb/KplPqUJZXetRFK7vhlon4zHje3v022y9tB8Db3psZbWfQLqDdXR5ZzrlWhfYvwZZ3Yc+30Owx8Kpt6ajKJfmXWgm4u7sjhCAm95/k9SzVlbNhw4a33aYoCpryetXawQN6vKcun1gOYaGWjceaGHPUpBnUqq/1+mOr1fD+QPVYOHwpmQX/nLdYeNZo5rpT5BjN+LnYMa5jcKm+ltbdnYAvPqfKBx+gcXbGlJhI1KQXiXphIsb4+FJ9bcmKVH9Q7f0AsHE6GO58sc8amTMyiJkxg8gnx2KMiUHY2+P71psEzv2xwLkNhRCMrDuSP/v+yf0e96OgsODEAgavHMzemL35O6ZfhR2fqsstnwb3ILyc9DzXWa3IOf+f80TEp5fBOywfsgwmXl+udsVtVs2NHvWL1vtJKp4N5zcwcMXAvOSwV1AvlvVfVvGTw2vaTAD3ILWn0t+vyoI1xVROz8KlonB0dKRly5b89ddfZGdn561PSUlh1apVFompVatW+Pv7s3bt2hvWHz58mKioKFq1amWRuEpEo+FQo4O6vOZFyMm48/6VxX9z4GoYCA30nKm2VgCtgj0Z2TIQgI/WnyY6KdOSUVqN/yISWHNEvYDzao8xD4YAACAASURBVK/7ymTcjhACt4EDCF69CqeOHQFI3bCBc31DSF69Ro5NrCy6TgeNFpIvqi1m5Uj6v/9xbuDAvOkr7Js2JXj5MjwefviuLeHBbsH81uc3JjSdgFajzRu79d6e90g3pMP6qepUIHauaitFrsfbBhHgZo/BpDDz75Ol+v7Kk682n+XclXS0GsG7AxrKngilLC4jjklbJvHStpdIyk7CxdaFDzt8yIcdP8RV72rp8MqOzg56fKAuh2+C039bNp5ySiaIlcQ777xDVFQU3bt3Z/ny5SxZsoSHHnoIJycni8RjY2PDJ598wsaNGxkzZgzr169n/vz59O/fn4CAAJ599lmLxFUihIA+n6pdkJIuwvYPLR2R5aXF5c+v9sAT4Nfghs1Tet6Pl5Oe9BwTb604VukTEbNZYcZqdY7IJoFu9G9ctl3cdb6+VP3uW/xnzUTj6oopKYnol1/m0oQJGK9cKdNYJAvwrAktxqrLW2eqhaWsnCkpieg33uDi6NEYLlxE6HT4vPwS1X9dgG316oV+Hp1Gx9ONnmZx38U08FS/pxadXsSgJb3YfSZ3CqOu09XeIrnsdDZ544PXH4/ln/CrJffGyqkT0Sl8t009bsZ1rCnHl5cis2Jm8ZnFDFg+gI0X1SJMbQPasqz/MnrV6GXh6Czkvl5Qs4u6vH5quewJYWkyQawkunXrxvLly0lKSmL48OG89NJLDB06lCdyKxhawogRI1i8eDFHjx6lf//+TJw4kdatW7Nr1668MYrllldtaPeiurz7S4g9Ydl4LG3TjNwr727Q+fVbNrs66Hg7RB27s/FkHOuO3dodujJZcuASx6LUCmz3Oq1FcQkhcO3fn+BVK3F66CEA0jZuIrxviKx0Whl0fh1cqoIxE1Y8B7mFxayNoiikrF1LeJ++JC/5CwC7xo0I+msJnmPHImxsivW8tdxrsaD3Al5s/iK2GluicxJ5pooP02rUI7Xh4Fv279uoCs2quQHw2rKjZOQUb77iisBoMvPqX0cwmhWCvR15/qFalg6pwopIjuCJ9U8w458ZpBpScdO78X679/m2y7f4OBRuXs8KSQjoNQs0Okg8r56HSUUiKtM/eSFEkqurq2tSUlKB269V3qxehKuNUtkrN5+TIQu+bQMJ4RDYGh7/u9xXBCyW6IMwpzOgQO+PoeVTBe6mKApPzNvLltNX8HHWs/GljrjY6co2VisQn5ZNj8+2czU9hwFN/Jk9oqmlQ1JPwlevIfbddzElq/Mx2j/QnCpvv42+tiwAUGGd3QS/DlKXe3wAD1pXzw5DVBQxM94hbds2ADQODni/+CLuI0cUOzEsSMTSx3n7yk4O2tkBamGbyQ9MpleNXjd0mzx5OYV+X+3EYFIY0yaIaf3ql1gM5cn328L54G91ap7F4x6kRVDJTM0j5TOYDPx8/Ge+P/w9OeYcAPoG92Vyi8l42Mnfd54Nb8LuL0BrD8/vBbdAS0dkNdzc3EhOTk5WFMWtoO2V8GxVksqIzg76qpMpE7kHDi6wbDyWYDLA6kmAAj71ofnjt91VCME7Axpgr7MhLjU7b+6/ykRRFKb8dYSr6Tk422mZ0ut+S4cE5LYmhvQlePUqXHr3BiBz337ODRxE3McfY86Q42wrpFpdoNlodXnTDKvpaqqYTCT88gvhIf3ykkOnhx4ieM1qPB4ZVaLJIWc3UePIUn6+HMcUnw7Ya+2Jz4zn1R2v8tSGpziXdC5v1/uruDCxax0A5u0+z66zla+40/n4dD4NPQPAYw9Wl8lhKTh85TDD1wzny4NfkmPOwd/Rn++6fscH7T+QyeHNOr4CTr5qT4jQNy0dTbkiE0RJKk3BHdWiNaBO3JpWycZvbf1AbUFEQO8PwebOhVaqujvwUnf1BOvXPRfZdz6hDIK0Hov2RrLxpFoS/t0BDfBztbNwRDfSensT8OknBM79UR3XZTRy9ce5hPftS+qmTZYOTyoN3d8F10D1BGv5s2A2WTScjIMHOT9sOLEfzETJyMDG24uA2bOp+vVXBVYovSfZqbDqBQBsqrVhVI8vWdF/BV2qqWOb/o35l8GrBvP5gc/JMKgXSZ7pEEzT3K6mryw5QkqWoWRjsmKKojBl6RGyjWb8Xe14pWfJztta2cVnxvPGzjd4ZO0jhCWGoREaHq33KMv6L6NtQFtLh2ed9M7QbYa6fHwZnLRMYcbySCaIklTaur+nVr3LSoINb1g6mrJzfmd+Sfi2/4OgwpXYHtMmiAYBakGDqUuPkmO0zrFPJe18fDrvrFbHqoY09qd/E+ude9WpbVtqrFyB14TnEba2GKMvc+m554kc/yw5l6IsHZ5UkuxcoN8X6nLkHotVNTXExhH96qtcGPkwWcfVAk5uw4dTc80aXHr2KJ0KmaFvQXIkaO2g/1eg0VDFqQqzO8/m6y5fU9WpKkazkR+P/siAFQPYfHEzNhrBJ0MbY6fTEJWUyburK8/480V7I9lzTr2o997Ahjjp5VTbJcFgNjD/+HxCloWwIlwtlFTXoy6/9f6NV1q8UnEnvS8pjYZDDbUqNyueh+RLlo2nnJAJoiSVNidvteodwJFFcG6bZeMpC5mJsPRpQIEqTaBz4RNjrY2GmYMaoREQFpfGnO3W0a2tNBlNZib+cYiMHBNVXO14t3+Duz/IwjR6Pd7PPUfwqpU4tlOT/7QtWzjXty9XvvxKdjutSGo+BM3HqMubZkD82TJ7aXNODvFzfiC8Vy+SV6wEwK5ePar//htVpk/DxqWUqmNGbId9P6nLD72pVna9ToeqHVjWfxnjGo9Dp9FxOf0yL2x5gec3P4+N/iqv5rae/bnvEhtPxJZOjFYkJjmL99eoU3wMaOJP57qVuEBKCdpzeQ9DVw7lo30fkWZIw1Xvyput32RRn0U08LL+/xNWQQgY+D04eKoX6pc+bfGeEOWBTBAlqSw0Gw2BuXM7rnoBspItG09pUhRYNRFSokDnAIN/BK1tkZ6iQYArT7arAcAXm89y7kpaaURqNb7eEs6hSLV41sdDG+PqUH6K89hWr07gD3MImP0ZWh8flKws4r/+mvAePUn6aymKSf4jrhC6vZPb1TQLVpR+V1NFUUjdvJlzfUO48umnandSDw/83plB0OI/cWjWrPRePCcdVk5Ql6u2gNbjC9zNTmvHc02eU7v4+atd/LZf2s6A5QOI1f5Bi2A9AFOWHiUxPaf04rUwRVF4c8UxUrONeDja8lZI5SzOU5Ki0qKYtGUST214ivDkcDRCw/D7hrN6wGqG3TcMG00JjrOtDFyqwIBv1eULu2DHJ5aNpxyQCaIklQWNBvrOBhs9JEaoY3kqagXhQ7/DieXqcs8P1Ck/imFStzoEuNmTYzQzecmRCtvV9FBkEl9sDgPgyXY1aFvLy8IRFZ0QApeePQleuxbPcc8g9HqMV65w+fXXiRgylPQ9eywdonSv7FygX26p+Mh/Yc+3pfZS2WfPEvnU01x69jkMFy+CVovH6NHUXPc37kOHlmwRmoJsekctjW9jC/2/hrucjFd3qc63Xb/lk46f4O/oj1Ex8tup34h0fBMnn53Ep6fxxopjpRuzBa09GkNobivp2yH18HAs2gVBKV9qTipfHPiC/sv7581p2MynGX/0/YM3Wr+Bm12BBSelwqjTA1rlXuzZ+gFclP+X7kQmiJJUVnzrQZ+P1eVTq2HXbMvGUxquhsPayepy3b75FRCLwcFWy/uDGgKw/0Ji3vi8iiQjx8ikPw5hMivU8XVico/7LB3SPbFxcsRn4kRq/r0Wl5AQALJPnuTimMeJHP8s2eciLByhdE9qds6vRLz5HYgPK9GnN0RFET31Nc7160/6zp0AOLZtS/CK5fhOnVJ63Umvd/G6cZadpoB34f4mhRB0D+rOyoErebH5izjrnEk3piE8V+MY/Cnrz69j5aGKNz43LiWLt1eqye9DdX3o19jfwhGVT9mmbH45/gu9lvbih6M/kG3Kxsfeh5ntZzKv5zzqesiCPyWi23TwbQiKGf4aqw6HkQok50G8TrmZX6+SK/ef08oJcGA+CA08ugyCO1k6opJhMsDc7hB9AJz9YfwucLj3ktufhZ7h803qieiHgxsxrEXFmcfo9WVH+e3fi+hsBCuea0c9/zI4AS5DmUePEjtzFpn796srtFrchw3F85lx6HzlGKVyKTsVvmkDyRehakt4Yt1dW9juxnj1KvHff0/SwkUoBrXqp2316vi8+gpOnTuXTgGagmQmwQ+dIeGcOnZ67Ka7Vl6+ncSsRL4/8j1/nPoDo2JUV2ZXY3bXN+kS3LoEg7acLIOJ4XP2cDgyCSe9lg2TOuDvZm/psMoVk9nEyvCVfHP4G2LSYwBw0Dowuv5oRtcfjaPO0cIRVkBXzsCcjmDIgHr9Yegv6jjFSuZu8yDKBPE65T7xqCTK/edkyIKfesDlQ+qg6ae3VYzJWze9Azs+BgQ8tkKd4qMEmM0KTy/Yz8aTsdjaaPjjmdY0reZeIs9tSZtPxfLEvH0ATOlVl3Eda97lEeWToiikbggl7uOPMURGAiD0etxHDMdz7Fi03t4WjlAqsnNbYX5/dbntC/ll5IvIlJZGws/zSPj557yiRlofH7yefw63gQMRujIci2sywK+DIWKb2rX0qS3gd+9FQC6kXGDmno/ZeXlr3ro2/m0Y33g8TXya3PPzW4qiKLz452GWHYxCCPjxsQfocr+vpcMqNxRFYUvkFr448AXhyWohNq1Gy7A6w3i60dN42ntaOMIK7sD8/HHGIV9A8+L3diqvZIJ4HZkgVgwV4nNKugjfd1C7N/g3U6/Ca/WWjqr4zu+CeX0A5Z5OGG8nNctA/693ce5KOr4uelZNaIePs3XNEVgUV9Oy6TF7B/Fp2bSs4cHCp1pjo6nYVzDNOTkkLVxI/JwfMF29CoCws8N95Eg8xz6J1lOeEJUr66bCnm/U5b6fwQNPFPqh5sxMEv/4g6vffY8p9/+xxtUVr6efwn3UKDR2Zfy3rShq8bADv6j3B/0AjYaV6Et8sWs93x39HBv7yLx1bf3bMr7JeBp7Ny7R1yoL324NZ9a6UwBM7VWXZyroBa6SpigKu6J38d3h7zh85TAAAkHv4N481+Q5Ap0rwMXi8kBRYMnj6tyIWnt4Zluhu5NXFHdLEOUYxEpu2rRpt3TfEUIwbdq0Un3drVu3IoS47c/MmTNL9fUtzq0aDJ4LCLVL5t+vWjqi4rthSovGRZrSorCc7XTMefQBnPRaYlOyefbXA+W2aE2WwcT43w4Qn5aNs17Lp8MaV/jkEEBja4vH6NHUCt2Az+SXsXF3R8nKIuHnnznbtRtxn3yCMVGOByk3ur0DdXqqy2tehrDQuz7ElJJC/Hffc7ZLV+JmzsKUlISwt8dz3DPUCt2A55NPln1yCPDPV/nJYcdXSzw5BPhf2x708XyfjMjRmDLVOU53Re/ikbWPMG7juLxkoTwIPRHLh+vV5HBQswCe7hBs4Yisn1kxE3ohlOGrhzN+4/i8z7tdQDsWhyxmZvuZMjksS0KohQNdq4ExE5Y8ofbukvLIFsTrVIiWqSKaNm0a06dP5/rjYM+ePVStWpWqVauW2uumpKRw4sStRUdmzpzJihUrOHnyJHXrFjwou0J9Tts/gs3vqsv9v4amj1g2nqLKSVe7ZV38R53S4pntxa5aWhihJ2J5ar7aLXNUq2q8N7Bhqb1WaTCYzIxbsJ9Np+IA+HJkU0IqaVEHc3o6Cb/9TsLcuZiS1WlfNA4OuI0cgcejj6Lz87NwhNJdZafBvN5w+TDYOsHjf0OVRrfsZoyPJ+GX+SQuXIg5TZ2yRuh0uA0ditf4cZbtZnxqDSwaBSjQYLB64a6UxiMZTGae/GUf28/EoXM+Te06u4nMOJO3vW1AW8Y1GmfVXU9PxaQw+JvdpOeYaFbNjd+fao2dTk65cDtGs5G/I/7mx6M/ci75XN76Vn6teKbxM7Twa2HB6CQu/gs/9wLFpBbVC/m80oxHlF1MryMTxFsVlCBaSk5ODgEBAdSpU4ddu3bddr8K9TmZzbDoYTjztzoFxpMbwN96Tw5uYMiChcPV8UgAg36ERkNL/WVnbzzD7I1q0ZqZgxoyomW1Un/NkmA2K7y0WB2zA/BW33o8kTvXY2VmSksjccECrv48D3NKirpSq8WlVy88Hx+DXb16lg1QurPUGPihC6RcAucqamEXV7WFLOdSFAk//UTSX3+hZGcDuRcBRozAY/Royxcqij6knhwaMtSCO6NXga50WzDTs42MmLOHo1HJ2OkErww0sy5qPicTTubt08irEY/Wf5Su1bqi1RSvSE5puJqWTf+vd3EpMRN/VzuWP9+2XHf1L005phyWn13OT8d+Iiotv3ptp6qdGNtobLnsVlxhbfsItuReqH/weej+bqVIEmUXUynPqlWraNy4MXq9nqCgIGbOnFlgYnhzF9Nr3VCPHDlC//79cXZ2xtvbm6lTp2I2m9m/fz8dO3bE0dGR2rVrs2DBgmLFt3LlSuLj43niicKPZSn3NBoY+B241wBTNvz5KGQkWDqquzMZYPGY/OSwz6dlkhwC/O+h2nSrpxZDeGvFcQ5ctP5uiYqiMGP1ibzk8H8P1ZLJYS4bJye8xo+n1sZQvCe+gI2XFxiNpKxaRcSgwVwY8zhp27ahmMtnl+IKz9kPRi0GvQukXkb5bSiZe3cT9corhPfoQeLvv6NkZ2Pj5obX/yZQa/MmfF+ZbPnkMCUaFo5Qk0O3ajDi91JPDgEc9Vp+GtOCQA97sgwKX63R80GruXz50JfU91QnmD8Sf4TJ2ybTe2lvfjn+C6k5qaUe193kGM2M/+0AlxIzsdfZMOexB2RyWIDY9Fi+OvgV3Zd055097xCVFoVA0DOoJ0tClvBlly9lcmht2r8ETUapy/98BZtmVNx5qotAtiBe53YtU4rRiCEmttTjKyydny9CW7Srihs2bKBXr160a9eOiRMnYjQamTVrFnFxcURGRt6QKAohePvtt/OSxGutjPfddx9jxozhgQceYOXKlXz55ZdMmjSJ1atXM3nyZKpXr85XX33F6tWrOXDgAE2aFK0lrHfv3mzfvp2YmBicnJxuu1+FakG8JuYY/NhV7Qvv3xRG/QWOVlq0w2xS5w86vlS93/09aPN8mYaQmmVgwNe7CL+Sjo+znhXPt6WKq/WWV/98YxifbVS7kj32YHWm96tfdqX7yxlzTg4pq1aRMG8e2WFn89bb1qyJx5jRuPbti8beej/rysp8fD0pHz5B4hk7shLzJ0rX+vri+cTjuA0disbBwYIRXic7TW05jDmiJrZPbgCf+8s0hIj4dAZ/u5uE9BwCPexZOr4tXk62HIg7wPzj89kSuQUF9f+yg9aBQbUHMer+UVR1Lr2hH7ejKApTlx5l0V61wM63o5rRq2GVMo/DWimKwoG4A/x+8nc2XdyESTEBoBVa+tbsy5MNniTINciyQUp3ZjbBsnFw9E/1fscp0HmqZWMqZbKL6XWKmyDmXIoivGvXUo+vsGpu3Iht1YAiPaZVq1ZER0dz9uxZ9Hq1WmZycjI1atQgMTGxUAniF198wYQJallgRVGoWbMmERER7Nixg3bt2gGQkJCAj48PL730ErNmzSp0fNHR0VSrVo3HHnuMn3766Y77VsgEEeDYX2ripZjBs7Y6R6K1TX9hNquloQ/9qt7v9Bp0skyBnfAraQz4ahep2UYC3OyZ93gLavs6WySWO/ll93neXnkcgH6N/Zk9vAmaSlCU5l4pikL6zp0k/Pwz6bv/yVuvcXLCJaQvbkOGYF+/vgUjlAByLlwgceEikpYtw5w7lhRA7++Ex7Ov4tqvH8LW9g7PUMbMJvjjETi9FoSN2vpZq4tFQjkUmcTIOXvINJhoEODCoqcfxEmvXvy9mHKR307+xrKzy8g0ZgKgERra+rdlUO1BdKzaEZ1N2UwDMmd7OO+vVYvSTOpahxe6lt448/Ikw5DB2oi1LDy1kDOJ+WNJPew8GFJnCEPrDMXPUY6lLjdMRlg6Vq1sCvDQG9BhsmVjKkWyi6lEeno6e/fuZciQIXnJIYCrqyshISGFfp4+ffrkLQshqFu3Ls7OznnJIYCHhwc+Pj55SVxhzZs3D5PJVLm6l96swWAYtkAdi3g1TJ10Pu6UpaPKpyiw7tX85LDN/6DjKxYLp6a3E9892hxHWxuikjIZ/O1u/j131WLxFGTFoai85LDTfd58MqyxTA4LSQiBU/v2VPvpJ2osX4Zr/34InQ5zWhpJCxdxfvAQIgYNJnHhQkyplu+CV5mYs7NJ2bCBi089TXiPniTMm4c5ORmh0+HSIojqXa9Qo/0Z3HwirCs5NObAqv+pySFA7w8tlhwCNAl04+tRTbHRCI5FpTD+1/0YTGpX6mou1ZjaaiqhQ0KZ1HwSPg4+mBUzO6J2MGnrJLou6cpHez8iPCm81OIzmRWmrzqelxz2aVSF/3WpVWqvVx4oisLhK4d5d8+7dF3Slen/TM9LDht5NeL9du8TOiSUCU0nyOSwvLHRqlPc1O2r3t/8Luz63LIxWZD1jH62Yjo/X2pu3GjpMPLo/Io2Ge21FkK/AqoCVqlS+G4iHh4eN9y3tbW9Zd219VlZRSsXPG/ePOrUqXNDslkp3d8XHvkLFo6E1Gj4uSc8vBgCLVzpTFFg4zT4b456v8VYda5DC3eTbFvLiz+eeZDH5+3lSmo2j879j0+HN6ZvI8tXBt1yKo6X/lRLmT9Q3Z1vRzVHZyOvyRWHXd26+M+ahc+UKSSvWEHSkiXknA0n68QJYqbPIHbWh7j07Inb4EHYN2+O0Mjfc0lTzGYy9u4jZfUqUtatx3xdUq71r4L78BG4DRmM1sMDlj0DR/5QT7CSL0GvDy0/z2tqLPz5GETuUe+3flb9HrOwh+r68t6ABkxZepQdYfFM/OMQHw9pjL2tWhnUVe/KEw2e4NF6j7I1citLw5ayO3o3CVkJzD8xn/kn5tPIuxGDag2iZ42eOOocSySu1CwDExYeZOvpKwB0vd+Hj4c0rrRd4yNTI1l9bjVrzq3hQkr+BXBbjS09a/RkZN2RNPBqYMEIpRJho4MhP6v1IM6sg9C3QKODB5+1dGRlTiaIhSC02iJ36bQm7u7uCCGIiYm5Zdvly5ctENGNtm/fTlhYWMWf+7CwarSHx9eo00ekX4H5/dSWxdoW6uasKLDtQ9g1W73f+GHo9ZHFk8NrGgS4snR8Gx6ft5ezcWk8//tBLidlMbZ9DYuczCiKwuJ9l3hzxTGMZoW6fs7MHdMi74RPKj6tuzueY8bgMXo0mYcOkbR4CSl//42SmUny8uUkL1+O1tsb5+7dcenZA/tmzRA28vd+L7JOnyZl1SqSV6/BeP3/EBsbHNu2wX34cJw6dbrx99zvS3Wc3+k1sH8eXD4Cw+Zbrst81H5Y9Ih60Q2g8+vQ/mXLxFKAES2rEZOSxeyNYaw5cpkzMal8M6rZDV3mdRod3ap3o1v1bsSkx7AyfCXLwpZxKe0SR64c4ciVI8z8bybtAtrRrXo3OlTtgJPt7cfy30lkQgZP/rKXM7HqlCTPdAjmlZ51K8V8rddLzk5mw4UNrA5fzYG4Azdsq+Neh5DgEPrV6oeH3a0XyqVyTGurfl8tehjOboT1U9XEseVTlo6sTMkxiNepsGPbgNatWxMdHU1YWFheN9OUlBSCgoIKPQYxMTERN7f8rsoDBgzg0KFDnD9//obXCgoKokmTJixfvrxQsY0ZM4Zff/2VyMjIQrVoVuTP6QZXw2HBQEi6ABotDPweGg4p2xhSomHl/+Bs7iTY9Qao84TZWN+1paSMHJ6ev5//zqtVYMe0CeLNvvXK9KQmIT2HqUuPsP64WtSquqcDi595EB8XWe2vtJjS0khZs5akJUvIOnr0hm023l64dOuOc88eODRvLpPFQlDMZrKOHSNt61ZSQzeSHRZ2w3a7Ro1wDQnBpXcvtJ53KKRlNsPOT3PneVXA3gOG/AQ1O5fuG7jZod9h1US1SrStMwyaA3V7l20MhaAoCj/uiGDWulMYzQp2Og3v9G/A0Adun1SbFTP7Yvax7OwyQi+Ekm3Kztum0+ho69+WrtW70imwE65610LFse98As8s2M/V9By0GsH7AxsyrIWVjYUvRdFp0WyN3MrWyK3sjd2L0WzM2+Zt702f4D70De7LfR73WTBKqUwYMtVKx9eqtbd9ATpNBV3FKJImi9RcpzIniKGhofTs2ZN27doxadIkjEYjM2fOLFIV09JIENPS0vDz86Nz586sWrWqUO+lIn9Ot0iNUVsSY4+p93vOglbPlH7rnaKoJ1brpkJ2buGJhkOh/zfq1TUrlWUw8dKfh1lzVG0Z71nfj9kjmpTJRM7bzlzh5cWHuZKqnqT1bujH+wMb4uZgvb+viibnwgVS1m8gdd06sk6cuGGbjbcXzp064dimDQ6tW6N1d7dQlNbHlJZO+u5dpG3dRtq2bZiu3jiWV1etGq4hIbiG9MU2KKhoT352k1p8KzMBhEZtvWv3ojrFT2kyGWDDG/Dvd+p9j5owciF4W/eJ/YGLiUz4/SBRSWphmkHNAninfwMc9Xe+KJeSk8LWyK2Eng9ld/Rucsw5edu0QkurKq3oHNiZ1v6tqeZcrcDeFUsPXGLKX0fJMZlxc9Dx3SPNaR1spdW0S4hZMXPi6gm2RG5ha+TWG4rNANhr7elSrQshNUNo5dcKG428yFSp5GTA78Pg/A71vkcwhHyh9vQq52SCeJ3KnCCCOg/iG2+8walTp/Dz8+PZZ58lMzOT6dOnWyxBnDt3LmPHjmXp0qUMHDiwUO+jon9Ot8hMUsckXtyt3q/eFrq8DdValc7rpUSrV9zD1qv3HTyhzydQv3Cfj6WZzQrvrz3JjzsjAGhc1ZU3+9bjgaDS6QaUZTAx8+9TzNt9HgAnvZbp/eozqFlApR2vYw1yLl4kukEibwAAFRpJREFUZf16UtetJ+v48Rs3CoFd/fo4tmmDY9u22DdtgsaaiqmUMsVoJOvkKTL27SN9xw7S9+4Fg+GGfWyDg3Hq2BGXnj2wa9To3o7lpIvq+L/og+r9Or3U+V/tCzwvuXfpV2Hx6PyTutrd1eITpfV6JSw5w8DLSw4TekLtiVDT25GvRzWjrp9LoR6fbkhn+6XthF4IZcelHWSZbqwJUMWxCq2rtKZ1lda0rNISd70nn4ae5ust4XmvN3d0C4K8SmY8ozVRFIVLaZfYH7uffTH72BW9i/jM+Bv2cdO70aFqBzoFdqKtf1scdFYyPYtkGcZs2PEJ7PgUzLnfk81Gq3UYysl3SkEsliAKIeoDzwPNgUaAHqihKMr5IjxHN+AdoDGQCiwDXlUUpeAM7+7PV6kTxIqiUn5OhkxY+jScXJm/rk4v6PIm+JZQqX9FgcMLYd0UyMptNazXH3p/Ak7eJfMaZWjuzgjeXXMib77btrU8+d9DtWlVglfEj0cnM3HRIcLi1LE6D1R357PhTQj0kCcU1iQnMpLUDaGk79pFxr59KDk5N2wX9vY4tHgA+yZNsG/YELsGDSpUC6M5PZ3MI0fI2H+AjP37yDx8BCUj48addDocWzyAU6dOOHXsiG1Jf78astQqyPvnqffda0DI51CjQ8n1iMhJh6NLYPtHkKzO2Uf7l6Hza1DOWn4UReHnXef54O+TGEwKeq2Gaf3qM6JFYJGS9QxDBruid7Hxwkb2XN5DQlbCLfvYGP3JTA7GlFmNZn4NmTOye4Xp+WBWzJxLOsf+2P3qT9x+4jLibtmvhmsNOlXtRKfATjT2bixbCqVbxZ5Qp/mK2qfed/KDPh/D/YWfDcCaWDJBHA28BxwAHIGHKEKCKIToBIQCy4HvAX9gFnAOaK8oirkYMckEsQKo1J9T+BbYND3/SjwCGg1T+8V71CjecyoKxIdB6Jtq1S5Qxwv1+QQaDCqRsC1l19l4Plx3isOX8udnax3swf+61ObBYM9itYooisLRqGTWHL3MTzsjMJgUtBrBxK61GdexJlpZqdSqmbOyyNi/n/Rdu0nfvZvsUwVPJaMLDFSTxYYNsW/UELv777eeid7vwJSURHZYGFlhYert0WNknTwJJtMt++r8/XFo1QqnTp1wbNsGG6fiFTUpkgMLYM1L6phAULtsNX0EmowC52JOCxAfBnvnqt3ir3WJ1znAgG+h/oCSidtCDkcm8fzCA0QmqF1O7/N1pl8Tf/o19i/yhSizYiYsMYyN53ey6sxWojJPgCbnlv1c9a7U96xPfc/61POsR33P+vg5+ll9j4gcUw7nks9xJvEMYYlhnEk8w4mrJ0jKvvWcz9nWmeY+zXnA7wE6BXaiukslPJ+Qis5sUqu5b3oHDOnquvv7Qe+Piv/9ZSGWTBA115I4IcRE4DOKliD+B+iA5tc9TzdgAzBCUZQ/ihGTTBArgEr/OSkKnFwFm9+B+NzxEhodNB+jlm33qHH3kvIZCRCxHcI3q0ln8sX8bfeHQJ9Pwcmn1N5CWVIUhW1nrvD5pjAOXsz/228Z5MELXWvTpubdE8Uco5l/zl0l9EQMG0/EEZOS32Ur2MuRz4Y3oXFg+e1qUpkZ4+NJ/+cfMv77j8yjx9SiLAUkUwBaX19sq1fHNihIva2h3uoCA8u0i6o5PR1DbBzG2BgM0dFkh50lOyyM7DNnMF65UvCDhEBfpw4OzZtj37wZDs2aoSvCNEclKvoQrH0ZLu29Lj4bqNMDmj6qdgm9WyEsk1Gdz3DvjxCxLX+9jV69sNVuktWPNyys5EwDU5ceYe3RGyuRP1Ddnf5N/OnTyB8Px7sff2Gxqfy4I4JlB6PIMZkBI3qnS9QLjkXndI7zqWfINGYW+Fg3vRvVXKoR6BxINWf1NtA5kGou1XDXu5dZ8mgwG4jLiONy2mViMmKITovmbNJZwhLDiEiOwKQU/LfraedJc9/meT+13WujEfJinlRMiRdg9SQI36Te1+ggoJk6BCioLQS2Bn0ZXHC7B1YxBrGoCaIQIgC4BLykKMqnN227BOxWFGVYMeKQCWIFID+nXCYjHFkEWz6AlEvXbRDgXAXcq4NbdfXWPQgcvODSf2pSGH0Qbm6Ed64C3d+FBoOtZgqLkqQoCjvPxvP5xjD2XUjMW++s1+LtosfbSY+383U/TmqSvfXMFbadvkJatvGG5wv2cqRvoyqM61QTB1vrq+oqFY85M5OskyfJPHKErKPHyDx2FMOFi3d+kEaDjYcHWnd3bDw90Xq4Y+PugY2HO1oPD2zc3RG2tmoVVa0WodUhdNq8+yigZGZgzrj2k3ndcgbGq/EYY2LVhDAm9oY5CG9HV7Uq+tq10d9XB4dmzbBv0gQbl8KNYSszsSfg4AI4vEgtYnONk5/avd3WEcxG9bvKbFSv3puNagGa8M3501aA+l3X4klo8gg4VrzCKoqicPhSMssPRrH6yGXi0/Irlmo1gva1vWhX2xujyUx6tpG0bBMZOUbSso2kZxtJyDBwODL/3MfNQcejravz6IPV8XFWqyybzCYikiM4fvU4x68e58TVE5xKOHVDddSCOOoc8XPww1XvirudO256N3VZ746r3hU3vRu2NrZohAYbYaPeatRbrdCioJBuSL/hJ82QRoYhg3RDOlezrnI5/TIxaTFcybyCwp3PW930btRxr5P309SnKdVdqlt9C6hUzigKHPlTHZaTeVPXbWED/k1yE8Z2UK012BWuknBZKa8JYg9gHdBDUZQNN237G6iuKEq9Ah53t7GJrq6ursgEsXyTn9NNDFmw7ye1pHz6bVoPCmKjh+oPQs2H1B+f+qVfWdAKKIrCP+FXmb0pjP8ibh2PcztCQNNAN7rV86NbPV9q+Vj31UGp5JiSksg+e5acCxfIOX+enPMX1OULF1Cy73zyXKo0GrTe3uhrBqvJYO6Pbc1a2DiVowIjxmw4tUZNFsO3wF0SgHxCbW1sMRZqdSl34wyLy2hSezSsOBTNumMxt1y8upMgTweebB/MkGZVCzU3q9FsJDwpnDOJZ7iUeonI1Egupl4kMjWywPGMZcnDzgM/Rz+CXYPzksHa7rXxtveWyaBUdrJS4PxOuLBLvY05cusFeI0OplxQL3pZifKaID4M/Aa0UBRl303bfgO6KoriW8DjZIJYCcjP6TbMJkiJUrs+JF2AxPPXLV+AtBjwqZebEHaGam3A1vrHVJWmk5dTuHA1nSup2epPWnb+cmo2adlGWgR50K2eL13u98Xb+S5dd6VKRTGbMcbGknPhAsYrVzAlJGBMSFRvExMw5S6bkpJQDAYUoxHFaLxtF1ah16NxcEBjb4/G0QHh4IDG3gGthztaXz90fr75t35+aL28ENoK1nqddBEO/gaRewChJn0arXpFXpP7I2zUXhFNHyn+2OsKIstgYvOpOFYciuJsXBoOtlocbG1w0mtx1Gtx1NvgaKvFQa+lUYArnev6lNjcsGk5aUSmRhKZGsmVzCskZyeTmJWo3mbfeGswGzArZsy3KR8hEDjoHHDUOao/WkccbdVbdzt3/Bz9qOJYJe/W19EXvY38PpasUFYyXPwXLuyE87vUHlt+DeGZbXd/bBkqkQQxt2DMlkK+preiKDfUDL6HBPEBRVH237TtN6CLoihFHg16ty6mkZGRGAwGgoODi/rUUhk6d+4cOp2OwMDKM3lviTCbK0ULoSRZO0VRIDdZVIxGEAKNvb3a5VSSKjBFUfISRZNiyksY7bR2ckygVDFlp0FaLHjWtHQkN7hbgljYS4+ngMcLue/dB0fc3bVZegsaSOABlEq/Bjs7O9LS0khISMDDo3TmTJPuTUJCAtnZ2Tg7O1s6lPJHJoeSZBWEEKDTIXQ6S4ciSWVKCIGNsMEGG3TI41+qBPROVl+wpiCFShAVRYkB5pVuKDe4NqtxA9SqpddrCOwujRf18vIiOzub2NhYkpKSsJFXc62KyWTKSw69vLwsHY4kSZIkSZIkVThW2aSgKMolYB8wSoj8PgdCiC5AALC0NF5XCEFAQABeXl7o5JVdq6PT6fDy8iIgIEAOQJckSZIkSZKkUlBqo9uFEA5A79y7jXNvewkhrgBXFEXZdt2+CrBNUZRO1z3Fq6ithwuFEHMAf2AW8C+wuBTjxtvbu7SeXpIkSZIkSZIkyWqVZvkzH25N5L7Jvd0GdLrTgxVF2SyE6AtMB9agjm1cDryiKLeZCVWSJEmSJEmSJEkqtlJLEHOrlRaqH6CiKAXupyjKOtT5ECVJkiRJkiRJkqRSZpVjECVJkiRJkiRJkqSyJxNESZIkSZIkSZIkCZAJoiRJkiRJkiRJkpRLKIpi6RjKjBDCDAhXV1dLhyJJkiRJkiRJklTmkpOTARRFUQpsLKxsCaIRtdU0xdKx3ORaxpps0Sikik4eZ1Jpk8eYVBbkcSaVBXmcSaXNkseYC2BWFKXAgqWVKkG0VkKIJABFUdwsHYtUccnjTCpt8hiTyoI8zqSyII8zqbRZ8zEmxyBKkiRJkiRJkiRJgEwQJUmSJEmSJEmSpFwyQZQkSZIkSZIkSZIAmSBKkiRJkiRJkiRJuWSCKEmSJEmSJEmSJAEyQZQkSZIkSZIkSZJyyQRRkiRJkiRJkiRJAuQ8iJIkSZIkSZIkSVIu2YIoSZIkSZIkSZIkATJBlCRJkiRJkiRJknLJBFGSJEmSJEmSJEkCZIIoSZIkSZIkSZIk5ZIJogUJIZyEEF8IIS4LITKFEPuEEP0sHZdU/gghuggh5gkhTgshMoQQl4QQS4UQDQvYt5sQYk/uMRcnhPheCOFmibil8k8IMU0IoQghDhWwTR5rUrEJIToJITYIIZJyv9dOCCGevmmfh4UQh4UQWbnfezOFEHaWilkqP4QQTYUQy4UQ0UKI9Nzja4oQQn/TfvJ7TCoUIURVIcTnQoidQoi03P+NnW6zb6G+u4QQvkKIX4QQ8bnH6Q4hRJvSfi8yQbSsZcAo4A2gD3ACWCaE6G3RqKTyaBxQDfgM6AW8mHt/rxCi9bWdcr+o1gKRQAjwMtAPWCOEkN8HUpEIIeoDrwKxBWzrhDzWpGISQowGNgLhwAjUY+hrwPa6fR4BfgN2oX7vvQ88B8wr43ClckYIURfYDQQBE1GPr6XAe8AP1+3XCfk9JhVeLWAkkAZsut1Ohf3uyk0YNwEdgQnAQCAV2CSEaFry4V/32nKaC8vITQLXAIMURVmWu04AOwBPRVHut2R8UvkihPBRFCXupnVuQASwWVGUwbnr/gN0QHNFUcy567oBG4ARiqL8UbaRS+VV7snRbmAv0BBwUxSlyXXb5bEmFYsQIhA4DUxTFOXD2+xjA1wC/lMUpf91658C5gCtFUX5tyzilcofIcQ04G2glqIo4detX4B6QcJBURSD/B6TikIIobnuOBmA2hDUWVGUrdftU+jvLiHEs6gXxporinIgd50eOAmcVhSlV2m9F3n1w3IGAsnAimsrFDVb/wWoK4SoZ6nApPLn5uQwd10SEAZUBRBCBAAtgAXXvsBy9wsFooDBZROtVEFMQj22Xr95gzzWpHv0ZO7tl3fYpzXgh/o/83q/AQbkMSbdmSH3Nvmm9cm520zye0wqquuPkzsoynfXQODoteQw9zWygYVANyGE871FfHsyQbScBsCJAg6mI9dtl6RiE0J4ox5Hx3JXXTumjhWw+1HkMScVkhAiGJgBPK8oSkoBu8hjTboXHVCvkA/KHVdtum6MzrUupgUeY4qiZKB2S5XHmHQnC4AE4FshRA0hhIsQoj8wGvgk99xMfo9JpaEo310Nbt4v1xHABii13oYyQbQcT9Qvp5slXLddkoolt7vyHNS/8Y9zV187pm533MljTrqr3GPrB2C9oijLb7ObPNake+EP1EZtQfwC6AL8BLzE/9u7mxCtyiiA4/9TEvYdUhAVFARRkYVRizIorSBqUQtbSB/Soo0bF8WkwaQJfagQIbSRopYuCgMjCQomcBFURsRYi4iSihYNBiHWQk+L8wjXt/d958vXF+X/g5c7c+/DzB04PHPPvc89B95tY4wxLVhmHqKe5NwM/EQ9OfwQ2JmZk22YMaZRmE9cjS1XWDKqH6w5GfYCqC+HajF2AI8Bz2Tm9z3HBsWWMae5eBa4g7qwmo2xpoU4B7gYWJuZu9u+qYg4H3g+IjZ3xhpjmreIuBbYC/xBLeP7iyoEsikijneSRDDGNBpzjaux5AomiOMzQ//Mf1nb9rtjIM0qIl6h7rRvyMz3Oodm2nZQ3BlzGioiLge2A68BRzql3pcA57bv/8FY0+LMUE8QP+nZv4+qInk7J8fYTM+4ZVSBLmmQ16mbECsy82jbN1ULJHgpIt7BeUyjMZ+5a2y5gktMx2cauKlPmeQTfev6rTmWhoqIrcCLwERm7uw5PN22/d6bWI4xp9ldA1xKJYiHO5+VVFwdBrZgrGlxvhuwP9r2OANiLCIuAK7HGNNwK6g6EEd79n9FXRvfiPOYRmM+c9d077hmOXAM+GEUJwgmiOO0B7iM6qvT9TRVuvbg6T8lncnasqtJYDIzd/Qez8xfqX9+T3RvTETE/cDVVA8oaZgfgVV9Pt9SL9evAnYZa1qkE/HR2xP4YWpJ1ZfAF9TywKd6xqyl2hIYYxrmd+CWdlHedVfb/uY8phGZz9y1B1geEd0WUue1sZ8OKBJ3StgHcUxaoYfPgFuBCeqR8joqQXw0M/eO8fR0homI56hiNB9RjX67/s3Mb9q41VT/pg+oIjZXAduAQ8DKzDx22k5aZ42ImOL/fRCNNS1YRHwM3E31qpsGVlP/K3dl5vo2Zh3VWPot4H2qot82qoDS42M4bZ0hOj3q9gNvUkVq7qNi7PPMfLCNcx7TvETEmvblnVQ8baHmsCOZua+NmdPcFRFLgQPAUmATtaR0A1W4657M/Hpkf4cJ4vhExCXAq8Aa6mniQWDrkMqAUl/tAv3eAYd/yczrOmMfAl4GbgP+piq3TWTm4RGfps5S/RLEtt9Y04JExIVU7KwFrqAuyN8GtnfbQ0XEk8ALwA3An1Qvsc19lg5KJ4mIB4CN1HK9i4Cfgd3AG5l5pDPOeUxzFhGDEqvea7E5zV0RcSVVePARKlE8AGzMzP2n/uw7v9cEUZIkSZIEvoMoSZIkSWpMECVJkiRJgAmiJEmSJKkxQZQkSZIkASaIkiRJkqTGBFGSJEmSBJggSpIkSZIaE0RJkiRJEgD/AfAfJQXEpbMzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15, 5))\n",
    "pe = PositionalEncoding(20, 0)\n",
    "y = pe.forward(Variable(torch.zeros(1, 100, 20)))\n",
    "plt.plot(np.arange(100), y[0, :, 4:8].data.numpy())\n",
    "plt.legend([\"dim %d\"%p for p in [4,5,6,7]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 完整的模型 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:29:56.083134Z",
     "start_time": "2020-05-04T03:29:56.079217Z"
    }
   },
   "outputs": [],
   "source": [
    "def make_model(src_vocab,\n",
    "               tgt_vocab,\n",
    "               N=6,\n",
    "               d_model=512,\n",
    "               d_ff=2048,\n",
    "               h=8,\n",
    "               dropout=0.1):\n",
    "    \"Helper: Construct a model from hyperparameters.\"\n",
    "    c = copy.deepcopy\n",
    "    attn = MultiHeadedAttention(h, d_model)\n",
    "    ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
    "    position = PositionalEncoding(d_model, dropout)\n",
    "    model = EncoderDecoder(\n",
    "        Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),\n",
    "        Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),\n",
    "        nn.Sequential(Embeddings(d_model, src_vocab), c(position)),\n",
    "        nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),\n",
    "        Generator(d_model, tgt_vocab),\n",
    "    )\n",
    "\n",
    "    # This was important from their code.\n",
    "    # Initialize parameters with Glorot / fan_avg.\n",
    "    for p in model.parameters():\n",
    "        if p.dim() > 1:\n",
    "            nn.init.xavier_uniform_(p)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:30:55.874290Z",
     "start_time": "2020-05-04T03:30:55.799250Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EncoderDecoder(\n",
       "  (encoder): Encoder(\n",
       "    (layers): ModuleList(\n",
       "      (0): EncoderLayer(\n",
       "        (self_attn): MultiHeadedAttention(\n",
       "          (linears): ModuleList(\n",
       "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (feed_forward): PositionwiseFeedForward(\n",
       "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (sublayer): ModuleList(\n",
       "          (0): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): EncoderLayer(\n",
       "        (self_attn): MultiHeadedAttention(\n",
       "          (linears): ModuleList(\n",
       "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (feed_forward): PositionwiseFeedForward(\n",
       "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (sublayer): ModuleList(\n",
       "          (0): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (norm): LayerNorm()\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (layers): ModuleList(\n",
       "      (0): DecoderLayer(\n",
       "        (self_attn): MultiHeadedAttention(\n",
       "          (linears): ModuleList(\n",
       "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (src_attn): MultiHeadedAttention(\n",
       "          (linears): ModuleList(\n",
       "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (feed_forward): PositionwiseFeedForward(\n",
       "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (sublayer): ModuleList(\n",
       "          (0): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): DecoderLayer(\n",
       "        (self_attn): MultiHeadedAttention(\n",
       "          (linears): ModuleList(\n",
       "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (src_attn): MultiHeadedAttention(\n",
       "          (linears): ModuleList(\n",
       "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
       "          )\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (feed_forward): PositionwiseFeedForward(\n",
       "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (sublayer): ModuleList(\n",
       "          (0): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (1): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (2): SublayerConnection(\n",
       "            (norm): LayerNorm()\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (norm): LayerNorm()\n",
       "  )\n",
       "  (src_embed): Sequential(\n",
       "    (0): Embeddings(\n",
       "      (lut): Embedding(10, 512)\n",
       "    )\n",
       "    (1): PositionalEncoding(\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "  )\n",
       "  (tgt_embed): Sequential(\n",
       "    (0): Embeddings(\n",
       "      (lut): Embedding(10, 512)\n",
       "    )\n",
       "    (1): PositionalEncoding(\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "  )\n",
       "  (generator): Generator(\n",
       "    (proj): Linear(in_features=512, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_model = make_model(10, 10, 2)\n",
    "tmp_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T03:31:29.082159Z",
     "start_time": "2020-05-04T03:31:29.076380Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Embeddings(\n",
       "    (lut): Embedding(10, 512)\n",
       "  )\n",
       "  (1): PositionalEncoding(\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_model.src_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据批次，同时创建 mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T12:31:23.297894Z",
     "start_time": "2020-05-03T12:31:23.291417Z"
    }
   },
   "outputs": [],
   "source": [
    "class Batch:\n",
    "    def __init__(self, src, trg=None, pad=0):\n",
    "        \"\"\"\n",
    "        src: 输入序列\n",
    "        trg: 目标序列\n",
    "        \"\"\"\n",
    "        self.src = src\n",
    "        self.src_mask = (src != pad).unsqueeze(-2)\n",
    "        if trg is not None:\n",
    "            self.trg = trg[:, :-1]\n",
    "            self.trg_y = trg[:, 1:]\n",
    "            self.trg_mask = self.make_std_mask(self.trg, pad)\n",
    "            self.ntokens = (self.trg_y != pad).data.sum()\n",
    "\n",
    "    @staticmethod\n",
    "    def make_std_mask(tgt, pad):\n",
    "        \"\"\"\n",
    "        将 pad 产生的 mask，和序列一次预测下一个单词产生的 mask 结合起来\n",
    "        \"\"\"\n",
    "        tgt_mask = (tgt != pad).unsqueeze(-2)\n",
    "        tgt_mask = tgt_mask & Variable(\n",
    "            subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data))\n",
    "        return tgt_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T12:46:59.335301Z",
     "start_time": "2020-05-03T12:46:59.328074Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ True,  True,  True, False, False]],\n",
       "\n",
       "        [[ True,  True,  True,  True, False]]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "src = torch.tensor([[3, 5, 7, 0, 0], [2, 4, 6, 8, 0]])  # batch=2,seq_len=5\n",
    "trg = torch.tensor([[2, 3, 4, 5, 0, 0], [3, 5, 6, 0, 0,\n",
    "                                         0]])  # batch=2,seq_len=6\n",
    "\n",
    "sample = Batch(src, trg)\n",
    "sample.src_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T12:49:13.676741Z",
     "start_time": "2020-05-03T12:49:13.666383Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[ True, False, False, False, False],\n",
       "          [ True,  True, False, False, False],\n",
       "          [ True,  True,  True, False, False],\n",
       "          [ True,  True,  True,  True, False],\n",
       "          [ True,  True,  True,  True, False]],\n",
       " \n",
       "         [[ True, False, False, False, False],\n",
       "          [ True,  True, False, False, False],\n",
       "          [ True,  True,  True, False, False],\n",
       "          [ True,  True,  True, False, False],\n",
       "          [ True,  True,  True, False, False]]]),\n",
       " tensor(5))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample.trg_mask, sample.ntokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:35:18.905422Z",
     "start_time": "2020-05-03T14:35:18.901680Z"
    }
   },
   "outputs": [],
   "source": [
    "def run_epoch(data_iter, model, loss_compute):\n",
    "    start = time.time()\n",
    "    total_tokens = 0\n",
    "    total_loss = 0\n",
    "    tokens = 0\n",
    "    for i, batch in enumerate(data_iter):\n",
    "        out = model.forward(batch.src, batch.trg, batch.src_mask,\n",
    "                            batch.trg_mask)\n",
    "        loss = loss_compute(out, batch.trg_y, batch.ntokens)\n",
    "        total_loss += loss\n",
    "        total_tokens += batch.ntokens  # 总 tokens 数\n",
    "        tokens += batch.ntokens  # 50 批训练时的总 tokens 数\n",
    "        if i % 50 == 1:\n",
    "            elapsed = time.time() - start\n",
    "            print(\"Epoch Step: %d Loss: %f Tokens per Sec: %f\" %\n",
    "                  (i, loss / batch.ntokens, tokens / elapsed))\n",
    "            start = time.time()\n",
    "            tokens = 0\n",
    "    return total_loss / total_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练数据\n",
    " standard WMT 2014 English-German dataset consisting of about 4.5 million sentence pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:35:21.355422Z",
     "start_time": "2020-05-03T14:35:21.349625Z"
    }
   },
   "outputs": [],
   "source": [
    "global max_src_in_batch, max_tgt_in_batch\n",
    "def batch_size_fn(mew, count, sofar):\n",
    "    global max_src_in_batch, max_tgt_in_batch\n",
    "    if count == 1:\n",
    "        max_src_in_batch = 0\n",
    "        max_tgt_in_batch = 0\n",
    "    max_src_in_batch = max(max_src_in_batch,  len(new.src))\n",
    "    max_tgt_in_batch = max(max_tgt_in_batch,  len(new.trg) + 2)\n",
    "    src_elements = count * max_src_in_batch\n",
    "    tgt_elements = count * max_tgt_in_batch\n",
    "    return max(src_elements, tgt_elements)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优化器\n",
    "\n",
    "Adam 优化器，参数$\\beta_1=0.9$，$\\beta_2=0.98$，$\\epsilon=10^{-9}$，变学习率：\n",
    "\n",
    "$$lrate = d_{\\text{model}}^{-0.5} \\cdot\n",
    "  \\min({step\\_num}^{-0.5},\n",
    "    {step\\_num} \\cdot {warmup\\_steps}^{-1.5})$$\n",
    "    \n",
    "其中：    $warmup_{steps}=4000$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T13:09:22.181078Z",
     "start_time": "2020-05-03T13:09:22.176679Z"
    }
   },
   "outputs": [],
   "source": [
    "class NoamOpt:\n",
    "    def __init__(self, model_size, factor, warmup, optimizer):\n",
    "        self.optimizer = optimizer\n",
    "        self._step = 0\n",
    "        self.warmup = warmup\n",
    "        self.factor = factor\n",
    "        self.model_size = model_size\n",
    "        self._rate = 0\n",
    "\n",
    "    def step(self):\n",
    "        self._step += 1\n",
    "        rate = self.rate()\n",
    "        for p in self.optimizer.param_groups:\n",
    "            p['lr'] = rate\n",
    "        self._rate = rate\n",
    "        self.optimizer.step()\n",
    "\n",
    "    def rate(self, step=None):\n",
    "        if step is None:\n",
    "            step = self._step\n",
    "        return self.factor * (self.model_size**(-0.5) *\n",
    "                              min(step**(-0.5), step * self.warmup**(-1.5)))\n",
    "\n",
    "\n",
    "def get_std_opt(model):\n",
    "    return NoamOpt(\n",
    "        model.src_embed[0].d_model, 2, 4000,\n",
    "        torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),\n",
    "                         eps=1e-9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T13:09:22.861490Z",
     "start_time": "2020-05-03T13:09:22.724533Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7ffb17580990>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAEDCAYAAABtd+CqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOydeVyU1f7H34cdlH0RcQHEXRTF3Bdc08o0zax+WrjULa9atth+S7PrknZbvLbdMhVTKxVT08oFt8ByTRTcRUQUZUfZZ87vj4GRnRlEh+W8X6/nxTzn+Z7v8x0Yns+c7XuElBKFQqFQKGoLZqYOQKFQKBQKY1DCpVAoFIpahRIuhUKhUNQqlHApFAqFolahhEuhUCgUtQoLUwdQ2xBC5KMT/HRTx6JQKBS1CAdAK6W8Y90Rajq8cQghtIBwdHQ0dSgKhUJRa0hLSwOQUso77ulTLS7jSXd0dHRMTU01dRwKhUJRa3ByciItLa1aeqoMUj4hREMhxGdCiKtCiCwhxCEhxEgD6/oJITYKIdKEEBlCiK1CiPbl2L4ghDgjhMgRQpwXQrwmhDArYdNXCLFMCHFMCJEnhCi3ySiEsBRCzBFCXCrweVIIMcWQuBUKhUJRMzG0yRYKjAfeAR4CooBQIcSDFVUSQngA+wAfIBh4EnAB9gghmpawfQf4GFgLDAO+Bf4NzCvhdjAwADgHHKsk7i+AWcAnBT63Ad8IIZ6vpJ5CoVAoaiiVjnEViNMvwBgpZWhBmUAnSK5SynYV1P0QmAH4SSnjC8pcgYvA91LKqUXK4oCvpZQvFqn/b+A1wFdKGVdQZial1Ba8/gR4UUopyrh3B+AE8LKU8uMi5d8DDwBeUsrsCt982e8pVXUVKhQKhXEUdBWmSSmd7tSXIS2u0UAa8HNhgdSp3QqgbXndfkXqbi8UrYK6ScBmYEwRu+GATYHPoixHNw6n75YsFC0DeASQQEgZPp2BQQb6USgUCkUNwhDh8geiyhCM40Wul0IIYQv4oWv1lOQ44FHQlVjoQwInixpJKc8CWeXdw4C4r0kpE42MO7WiA1DTCRUKhcKEGDKr0BU4U0Z5cpHrZeEMiCJ25dW9XvAzU0qZU4ZtSgX3qAhXA+6tUCgMQEpJYmIi2dnZaLWGdnoo6gNmZmbY2Njg5uaGbhTp7mPodPiKBsIqWwhmaN07uYcx9WQF16is/1W1uowjV5PLgasH6NG4B9bm1qYOR1EFpJRcuXKFjIwMrK2tMTc3N3VIihpEXl4eN2/eJCcnhyZNmtwT8TJEuJIou3XiUvCzrFYN6FpK0sC6SUADIYR1Ga0u5wruURFJlN0dWBhPVXwqjGTpsaUsO7GMoKZBLBm05J59I1NUH4mJiWRkZNCoUSNcXFwqr6CodyQnJ5OQkEBiYiLu7u53/X6GjHGdBNqVXE8FdCz4WdYYFlLKLOACZYtHR+CGlPJ6kXsIoENRIyFES8C2vHsYELdnwYxFg+NWVB9SSpadWAbAnrg9bLu4zcQRKapCdnY21tbWSrQU5eLi4oK1tTXZ2UZP1K4ShghXKOAEPFyi/GngtJQyqpK6Q4UQnoUFQgiXAl8bithtA3KAp0rUDwby0c1CNJaN6MRwQhk+U4GwKvhUGMHplNPFzhf8tYCU7BQTRaOoKlqtVnUPKirF3Nz8no1/GtJVuBXdQ/7bImuwgoG+wKhCIyHEbiCoxJqqxejEaKsQYg46EXqn4Kd+YbGUMkkIMR/4lxAireB+vYDXgU+klJeL3McdCCo4bVlQNrbgPEZKeajA5wkhxHJgfsG6s6PACHRCNr2gRai4i4TF6r4b2JjbYGFmQUpOCosOLmJev5JryhUKhcJwKhUuKaUUQjyCTmjmoWt9RaFbkFxhS0hKmSCE6IdOwELQtfD2Af2llLElzN9Ht15sGvAmEA+8BywsYdcB+KlEWeH5CmBikfLn0C1sfhlohK7r8h9Syv9VFLeiegi7rBOucW3G4ePow/sR77P5wmYebPEgfZv0NXF0CoWitqKywxuJypxhGFdvXuX+9fcD8N2w7whsFMjk3yZzOOEwjRs0ZsPIDTS0amjiKBWGcOnSJQC8vb1NHImiJlPZ5+ReZ85QKIymsLXlZO1EZ4/OmAkzZveajbW5NVdvXWXhwZINaYXi3rN7926EEGUep06d0tutXLmScePG4efnhxCCAQMGlOlv/fr1PP7447Ro0QJbW1t8fX0JDg4mJiamSvFNnDgRIQSPPPJImddXr15NQEAANjY2NG3alDfeeKPMCRIJCQkEBwfj5uZGgwYN6NevH+Hh4Xfk05Qo4VLcFQqFq3/T/liY6XqkfRx9eKnrSwBsPLeRnZd2miw+haIoCxcuJCIiotjh4+Ojvx4SEsLp06fp378/jRo1KtfPhx9+SHZ2Nu+++y6//vors2fPJjw8nMDAQC5evGhUTNu3b2fdunU4ODiUeX3VqlWMHz+ePn36sG3bNt566y2WLl3KxIkTi9llZ2czePBg9uzZw5IlSwgNDcXe3p7Bgwdz9OjRKvk0OVJKdRhxAKmOjo5SUT7pOemy84rO0n+5v9wes73YNY1WI5/57Rnpv9xf9lvTT97IvGGiKBWGEhMTI2NiYkwdxl0hLCxMAjI0NLRCO41Go38dEBAgg4KCyrRLSEgoVXbhwgUphJAvv/yywXFlZGRIb29vuXjxYunt7S1HjRpV7Hp+fr709PSUI0eOLFb+9ddfS0AeOHBAX7Z06VIJyMOHD+vLsrOzpa+vrxw+fHiVfJZFZZ8TR0dHCaTKangOqxaXotrZf2U/+TIfKzMrenv1LnbNTJgxt89c7K3sSclJ4b3w9wq/ECgUNRYzM8MelR4eHqXKfH19cXNzIy4uzuD7vfnmm7i4uDBz5swyrx84cIBr164RHBxcrHz8+PFYWlqyfv16fVloaCgdO3YkMDBQX2Ztbc2TTz7J9u3bycjIMNqnqVE7ICuqncJp8D29emJnaVfqumcDT/7V81+8tvc19sbt5YfTP/BE2yfudZiKOyBfo+VqWs0Z92jsaIOFedW/hz/33HOMHTtWP/4zZ84cunbtWi2xnThxghs3buDvXzwXw4ABA9izZ0+pL27h4eF89dVXRERElLt+7sQJXf6Ekj7t7Ozw8/PTXy+0HThwYCkfnTp1QqPREB0dTffu3Y3yaWqUcCmqlTxNHvuu7ANgYLPS/yyFPOD7ALsv72brxa0sOriIAPcA2rmWu7WbooZxNS2bfh/WnDX8+14bSDOX0l+SKsPR0ZGZM2cyYMAAXFxciI6OZsGCBfTp04c9e/bQo0ePO4orJyeHKVOm4OrqyvPPF9+/1tzcvJQwFdpPnz69QuFMSkoCKDObiYuLi/56oW15dkV9GePT1CjhUlQrBxMOcjPvJgLBgGYDKrT9V89/cSLxBLEZsbyy5xV+HPGjmiKvuKd06dKFLl266M/79evHyJEj8ff35+2332bHjh1V9q3RaHj66ac5duwYW7ZsKZXDb+fO0pOT5syZQ1ZWFnPnzjXoHuXl/ixZXlGOUENta1KeUSVcimqlsJuwo3tH3GzdKrRtaNWQxUGLmbB1ApczLjM7YjaL+i+qUf8girJp7GjDvtfKb1Hfaxo72lSbL09PT+6//342bdpUZR9arZZJkyaxYcMGfvjhB4YOHVppnejoaBYtWsSKFSvIy8ujcK2oVqvVn9vZ2WFlZYWrqy4Fa1JSkv51IcnJyfj6+urPXV1dy2wtJSfr8owXtrCM8WlqlHApqg0pJbvjdgMVdxMWpZ1rO17v/jpzD8zlt5jfuK/RfWq8qxZgYW5Wpa652oJWq63yFyitVsvkyZNZvXo1q1atYsyYMZVXAk6fPk1+fj7jx48vde3y5cs4OzvzxRdf8Pzzz9Ohgy4f+YkTJ2jdurXeLjMzk/Pnz/Pww7dTy3bo0KHM8anIyEjMzc1p27at3s5Qn6ZGzSpUVBunkk9x7dY1wHDhAnis9WM84PMAAB8e/JC/b/x9V+JTKAzh2rVrbN++nZ49expdV0rJs88+S0hICMuWLeOJJwz/Eta3b1/CwsJKHY0aNaJPnz6EhYUxcuRIAHr27ImnpychISHFfKxZs4a8vLxiYjl69GgiIyM5duyYviw3N5c1a9YwZMgQ/ToxY3yaGtXiUlQbhYuOm9s3p4VjC4PrCSF4r/d7RCdHE5Mew0thL7F2xFo87EpPLVYoqpPx48fTokULAgMDcXZ25tSpUyxcuJCsrCzmz5+vt4uKiiIqSrcRRlpaGnl5eaxbtw6Abt266dMcvfDCCyxbtoxnn32W1q1bc+DAAb0PBwcH2rdvrz8vOavQzc2tzIwchbsLF71mYWHBggULmDhxItOnT2fs2LFER0fz+uuvM3bs2GKiO2XKFJYuXcqYMWOYP38+Li4ufPrpp8THx/Pjjz9WyafJqY7FYPXpQC1ALpexm8ZK/+X+ctFfi6pU/0LqBdnz+57Sf7m/fHLLkzI7P7uaI1RUhbq8AHn+/PkyICBAOjo6SgsLC9moUSP5+OOPy8jIyGJ27733nkS3MW6p47vvvtPbeXt7l2tXctFyUFCQ1D2CK6asBciFhISESH9/f2llZSW9vLzkrFmzZGZmZim7q1evygkTJkhnZ2dpa2sr+/TpI/ft23dHPktyLxcgqyS7RqKS7JZN/M14hq0fBsDy4cvp2qhqa2D2xu1l+s7pSCQj/UbyQZ8P1GQNE6OS7CoMQSXZVdQ6iiXVde9cZT/9m/bnxcAXAdh0fhMhUSGV1FAoFPUNJVyKamH35d2ATnjMze5st9zJ/pP1kzUWH1rMzliVjFehUNxGCZfijknPTefQtUMADGo26I79CSGY02cOndw6IZG8vvd1NdNQoVDoUcKluGP2x+mS6lqbW9PLq1e1+LS1sGXJ4CU0s29GjiaHGTtnEJtectNshUJRH1HCpbhjCse3ejYuO6luVXGxceGLIV/gZO1ESk4KU3dMJSU7pdr8KxSK2okSLsUdkafJY/+V/YBxi44NxdvBmyWDlmBtbk1sRizTdk7jVt6tar+PQqGoPSjhUtwRB6/dTqob1Czortyjs0dnFvRbgJkwIzIxkhm7ZpCdX3O21FAoFPcWJVyKO6Kwm9CQpLp3whDvIczuNRvQieUre14hT5N31+6nUChqLkq4FFVGViGp7p0wutVoXu/2OqBbqPzm/jfRaDV3/b4KhaJmoYRLUWWik6P1SXWrYxq8IUxoP4FpnacB8FvMb7wX/p4SL4WinqGES1FlCrsJvR288XW8d3v1PNfpOSZ2mAjAz+d/5l9//EuJl6JK7N69GyFEmcepU6f0ditXrmTcuHH4+fkhhCgzGS7A+vXrefzxx2nRogW2trb4+voSHBxMTEyMwTGFhYUxZMgQ3N3dcXBwoEuXLnz55ZdotdpStqtXryYgIAAbGxuaNm3KG2+8QXZ26fHfhIQEgoODcXNzo0GDBvTr14/w8PAy72+oT1OissMrqkzhppEDmw28p/kEhRC83PVlNFJDSFQImy9sJl/mM6/vPCzM1EdaYTwLFy6kf//+xcp8fHz0r0NCQrh+/Tr9+/fn1q3yZ7V++OGHeHp68u677+Lr60tMTAwffPABgYGBHD58uNLNGHft2sXQoUPp378/33zzDXZ2dmzYsIGpU6cSHx/P+++/r7ddtWoVTz31FFOnTuWTTz7RZ3KPiYlh7dq1ervs7GwGDx7MzZs3WbJkCa6urnzyyScMHjyY8PDwYjtAG+rT5FRHpt76dKCyw0sppYzLiJP+y/2l/3J/efjaYZPEoNVq5UeHPtLH8XLYyzJXk2uSWOoydTk7fFhYmARkaGhohXYajUb/OiAgoFSm90ISEhJKlV24cEEKIeTLL79caTzBwcHS2tpa3rx5s1h53759pbe3t/48Pz9fenp6ypEjRxaz+/rrryUgDxw4oC9bunSpBOThw7f/T7Ozs6Wvr68cPnx4lXyWxb3MDq+6ChVVojA3obO1MwHuASaJQQjBS4Ev8WzHZwH4/dLvzNozi1xNrkniUdRdzMwMe1R6eJTeQ87X1xc3Nzfi4uIqrW9paYmVlRW2trbFyh0dHbG2ttafHzhwgGvXrhEcHFzMbvz48VhaWrJ+/Xp9WWhoKB07diQwMFBfZm1tzZNPPsn27dvJyMgw2qepMeivIYRoKIT4TAhxVQiRJYQ4JIQYaWBdPyHERiFEmhAiQwixVQjRvhzbF4QQZ4QQOUKI80KI14QQpWI01KcQwlMIsVQIcaEg7otCiC+FEF6GxK4on8LxrepIqnsnCCGY0WUG/wz4JwA7Y3cydcdUbubeNFlM9QJNPqRcqjmHJv+O3s5zzz2HhYUFjo6OjBgxgsOHD1fTLwpOnDjBjRs38Pf3L1Y+YMCAUl3szz33HHl5ebzwwgvEx8eTmprKt99+y2+//casWbOK+QRK+bSzs8PPz09/vdC2pB1Ap06d0Gg0REdHG+3T1Bg6IBAKBAKvAReBiUCoEOJhKeXW8ioJITyAfcB1IBjIB94B9gghukgp44rYvgPMAf4N7AJ6F7x2Ad4w1qcQwgrYU1D/XSAaaAe8DwwWQvhLKXMMfP+KIqTnpnP4mu4fe2Dzuz8NvjKEEEztPBUbCxv+c/g//HXtLyb/NpnPh3x+V9eW1WvSr8CnnUwdxW1ePA7Oxu8X5ujoyMyZMxkwYAAuLi5ER0ezYMEC+vTpw549e+jRo8cdhZWTk8OUKVNwdXXl+eefL3bN3Nwcc/PiX/ruu+8+du/ezaOPPsrSpUsB3c7En332Gc8884zeLikpCQAXF5dS93RxcdFfL7Qtz66oL2N8mppKhUsI8SAwBBgjpQwtKAsDWgAfAeUKF/Aq4AzcJ6WML6gbgU783gamFpS5Fpz/V0r5bkHd3UKIBsBrQoj/FhE5g3yiE77WwDNSym+L+MwFvgF6Absre/+K0uyL23c7qW7j6kmqWx1M8p+Es40zs8NnE50cTfC2YL4a+hVN7ZuaOjRFDaVLly7FJif069ePkSNH4u/vz9tvv82OHTuq7Fuj0fD0009z7NgxtmzZgru7e7HrO3eW3q7n2LFjjB49ml69ejFlyhSsra3ZsmULM2bMwMLCgmeffbaYfXmTokqWVzR5ylDbmrShqyEtrtFAGvBzYYGUUgohVgBfCyHaSymjKqi7vVBgCuomCSE2A2O4LTLDARtgRYn6y4G3gJHA50b6LEyrkFbCZ+G5am1VkcJuwl6Ne1VrUt3q4JGWj+Bk7cSre14lNiOWp7Y9xdLBS2nvWmbvtKKqODTRtXJqCg5Nqs2Vp6cn999/P5s2baqyD61Wy6RJk9iwYQM//PADQ4cONaje1KlTady4MevWrdMLxeDBg0lNTeWll15iwoQJ2Nra4urqCuhaSYWvC0lOTi42e9HV1bXM1lJycjJwu4VljE9TY8gYlz8QJaUsuYjgeJHrpRBC2AJ+QFkdo8cBj4Juv0IfEjhZ1EhKeRbIKryHkT4PAH8Bs4UQ9xWM090HzAb2An+WE3dqRQfgWFa9+kKuJvd2Ut0a0E1YFgOaDeB/9/8Peyt7ErMSmfjrRHbF7jJ1WHULcwtd11xNOcyrdxmEVqutcgtDq9UyefJkVq9eTUhICGPGjDG47tGjRwkMDCx17/vuu49bt25x6dIlADp06ABQatwpMzOT8+fPFxun6tChQ5njU5GRkZibm9O2bVujfZoaQ4TLFUguozy5yPWycAaEgXVdgcxyxpxSitgZ7FNKqQEGA2eBg0BGwc/LwENlCLHCAA5eO8itvFsIBP2b9q+8gono4tGFlcNX0qRhE7Lys5gZNpPlJ5YXLmlQKMrl2rVrbN++nZ49expdV0rJs88+S0hICMuWLeOJJ54wqr6XlxeHDh0qtdg4IiICMzMzPD09AejZsyeenp6EhIQUs1uzZg15eXnFxHL06NFERkZy7NgxfVlubi5r1qxhyJAhODg4GO3T1Bj6NaWi//bKngSG1jXmHpXaCiEsgdXoWmuTgXPoJme8B/wshBgupSyVpVVK6VSBb+p7q6uwm7CTe6caP/GhpXNLvn/we14Me5G/b/zNR4c/4mL6Rd7p8Q6W5pamDk9RAxg/fjwtWrQgMDAQZ2dnTp06xcKFC8nKymL+/Pl6u6ioKKKidCMiaWlp5OXlsW7dOgC6deuGt7duYsgLL7zAsmXLePbZZ2ndujUHDhzQ+3BwcKB9+9td1gMGDGDPnj3Fvky9+OKLzJw5kzFjxjBlyhSsrKzYtGkT33//Pc888wxOTrrHk4WFBQsWLGDixIlMnz6dsWPH6hcLjx07tpjoTpkyhaVLlzJmzBjmz5+Pi4sLn376KfHx8fz44496O2N8mpzKFnoBEUB4GeU90InEuHLq2QJaYF4Z114vqOtRcD6/wNa6DNtM4PMq+Hyu4DyghF1QQfnTVVn4Rj1egKzVauXgHwdL/+X+8pvj35g6HIPJzs+Ws/bM0i9UnvTrJJmUlWTqsGoNdXkB8vz582VAQIB0dHSUFhYWslGjRvLxxx+XkZGRxezee+89WfDcKHV89913ejtvb+9y7UouWg4KCpK6R3BxfvjhB9mrVy/p4uIi7e3tZefOneWSJUtkbm7pxfUhISHS399fWllZSS8vLzlr1iyZmZlZyu7q1atywoQJ0tnZWdra2so+ffrIffv2lfk7MdRnSe7lAmQhK+k6EUJ8AzwKuMoi3WtCiGeA/wEdZDmTM4QQ59CNj40sUb4KuF9K6VFwPgEIAbpKKY8UsWuJrqtvmpTycyN9fglMllJalbBrANwEFkop38BIhBCpjo6OjqmpqcZWrfWcTDrJE1t0XR8/P/IzLRxbmDgiw5FS8uXfX/L537o5Pp4NPPl4wMf4u9WcfvuaSuG4SmGrQqEoi8o+J05OTqSlpaXJSnq1DMGQMa5QwAl4uET508Dp8kSrSN2hQgjPwgIhhEuBrw1F7Lahm+X3VIn6heu0NlfBZzxgKYToQnEK529fqSBuRRkU5ib0cfCpVaIFt9d6LQ5ajK2FLdduXePpbU+z7sw6U4emUCiMxBDh2gqEAd8KISYLIQYKIZYDfQH9Um4hxG4hRMnm22J008+3CiFGCSEeAn5BJ0bzCo2klEnougtnCCFmCyGChBBvoOv++0RKedlYn+im0qehWyj9TEHc04BVQAK68S+FERSOb92LvbfuFsN8hrH6wdX4OPiQp81jTsQcZofPJkejVkcoFLWFSoVL6voSHwHWohOGbUAndAuSN1dSNwHoh24mXwjwA5AK9JdSxpYwfx/d4uLxwO/oxqjeQydeRvsseN0d+ANdZo2twCvoRK57gVgqDCQuI44zKWcA3XTz2kxL55asfmi1fg+x9WfX8/S2p4lNL/mRVCgUNZFKx7gUxamvY1zfR3/Pgr8W4GLjwq7Hdpk0P2F1oZValp1YxpKjS9BKLXYWdrzT8x0e9ivZK16/UWNcCkOoaWNcCoV+fMvUSXWrEzNhxjMdn+Gb+7/Bw9aDzPxM3tr/Fm/vf5vMvExTh6dQKMpBCZeiUtJy0jiUcAio3eNb5dHNsxvrRq4jqGkQAJvOb2LclnFEJVU070ihUJgKJVyKStl3ZR8aqcHG3IZeXjUnqW514mzjzJJBS3ij+xtYmllyKf0S438Zz1d/f0W+9s62zFAoFNWLEi5FpRR2E/b06omthW0l1rUXIQTj241n9UO6WYf5Mp//HvsvE7ZO4HzqeVOHp1AoClDCpaiQYkl162A3YVm0dWnLTw//xFPtn0IgOJl0knGbx7H8xHI0Wo2pw1Mo6j1KuBQVcvDaQTLzM2t8Ut3qxsbChte6vcayYcto0rAJudpcPjr8EZN+m8SFtAumDk+hqNco4VJUSOGi4wD3gBqfVPducJ/nfWwYuYFxrccBcPT6UcZuGssXx74gV5Nr4ugUivqJEi5FuUgpb2fLqKF7b90L7Czt+Fevf/HV0K9o0rAJedo8Pv/7c8ZuHsvhhMOmDk9xB+zcuZOJEyfSpk0b7OzsaNq0KWPGjCEyMrKY3YABAxBClDrK27Zk9+7d3H///Tg5OWFnZ0f79u35+uuvjY5v4sSJCCF45JFHyry+evVqAgICsLGxoWnTprzxxhtkZ2eXsktISCA4OBg3NzcaNGhAv379CA8PvyOfpkQJl6JcopKiuJ55Hag/41sV0durN6GjQpnkPwlzYc7FtItM/HUis8Nnk5ZTcqNtRW3gyy+/JDY2lpdeeolt27bxn//8h9jYWLp161ZsSxKAVq1aERERUez44IMPSvlcsWIFQ4YMwc/Pj7Vr17J582amTZtGbq5xLfTt27ezbt06/X5ZJVm1ahXjx4+nT58+bNu2jbfeeoulS5cyceLEYnbZ2dkMHjyYPXv2sGTJEkJDQ7G3t2fw4MEcPXq0Sj5NTnWkmK9PB/VoW5PPjnwm/Zf7yxEbRpg6lBpHdFK0fGLzE/qtUvqv7S/XnV4n8zX5pg6t2qnL25okJCSUKktJSZFOTk5yzJgx+rKgoCAZEBBQqb/Y2Fhpa2srFy5ceEdxZWRkSG9vb7l48WLp7e0tR40aVex6fn6+9PT0lCNHjixW/vXXX0tAHjhwQF+2dOlSCcjDhw/ry7Kzs6Wvr68cPnx4lXyWxb3c1qR697tW1CnqQlLdu0Vbl7asenAVa0+v5bMjn5GcnczsiNn8eOZH3uz+Jp09Ops6xLtKvjafhMwEU4ehp5FdIyzMjH+ceXh4lCpzcnKiVatWxMXFGe3v22+/BWDGjBlG1y3Km2++iYuLCzNnzmTJkiWlrh84cIBr164RHBxcrHz8+PFMmzaN9evX06NHDwBCQ0Pp2LEjgYGBejtra2uefPJJFi5cSEZGBvb29kb5NDVKuBRlEpcRx9mUs8CdjW+lZ+ex+e94hnfwxLWhdXWFVyMwNzNnfLvxDPUeyseHP2bLhS1EJUXx1LaneLjFw7zU9SXc7dxNHeZdISEzgeHrh5s6DD2/PvorTRo2qRZfN27c4MSJEzz55JPFyk+fPo2zszMZGRn4+voSHBzM66+/jqXl7d209+7dS7t27diwYQPvv/8+586do3HjxkyYMIH3338fK6vb2wOWtQMyQHh4OF999RURERGYm5edXj+QKKIAACAASURBVO3EiRMA+PsX30/Ozs4OPz8//fVC24EDS/8Pd+rUCY1GQ3R0NN27dzfKp6lRY1yKMtl9eTcALjYudHLrVGU/czZF8XboCR77KoLkW3VzFp6HnQfz+81n5QMraefSDoDNFzYzInQE30R+Q3Z+zRrYVpSPlJJ//OMfaLVaXn31VX15v379+PjjjwkNDWXjxo0EBQXx7rvvMm7cuGL14+PjOXv2LDNmzOCFF15g586dTJ48mY8++ohJkyYVszU3Ny8lTDk5OUyZMoXp06fTtWvXcuNMStJtbuHi4lLqmouLi/56oW15dkV9GePT1KgWl6JMCrsJg5oGVTmp7vX0bNYf0XW3XLhxi0nLD7L6mR40sK6bH7suHl1Y89AaQs+F8tmRz0jJSeHTI5+y9tRapnWexki/kXUmQXEju0b8+uivpg5DTyO7RtXiZ9asWWzcuJHvvvuOdu3a6cvnzp1bzG7EiBE0atSIefPmsX//fvr27QuAVqslIyODNWvW6GccDhgwgKysLBYvXsycOXNo2bIloJvRWJI5c+aQlZVV6n7lIYQwqLw8O2NsK/Jxr6mbTxDFHZGWk6af5n0n41urDlwqdv735VSmfn+Eb56+DyuLutnYNzczZ2zrsQz1HspXx79i7am1JGQm8G74u6yMWslLXV+iX5N+NeohUBUszCyqrWuupvD222/z0Ucf8emnnxo0iy44OJh58+YRERGhFy5XV1fOnj3LsGHDitk+8MADLF68mCNHjuiFqyTR0dEsWrSIFStWkJeXR+HWSVqtVn9uZ2eHlZUVrq6ugK6VVPi6kOTkZHx9ffXnrq6uZbaWkpOTgdstLGN8mpq6+fRQ3BF74/bqk+r29OpZJR/ZeRq+/1O3MeOMQS2ZP6ajzveZG8xa9zdabd3eB87R2pHXur3G5tGbeajFQwCcSz3HtJ3TmPzbZCJvRFbiQXEveffdd5k3bx4ffvghL7zwgkF1tFotAGZmtx+jHTt2LNO2cByrqG1JTp8+TX5+PuPHj8fZ2Vl/XL58ma1bt+Ls7MyyZcsA6NChA0CpcafMzEzOnz9fbJyqQ4cOZY5PRUZGYm5uTtu2bY32aWqUcClKUdhNeCdJdTf9HU/SrVwszAQTenrzZPfmvDK0NQA/H4vnrdDIOi9eAE0aNmFBvwX8OOJHejXWZdY/lHCI/9v6f0zfOZ2TSSdNHKFizpw5zJ07l7lz5zJr1iyD661cuRKAnj1vf7kbM2YMAFu3bi1mu3XrVoQQdOvWrVx/ffv2JSwsrNTRqFEj+vTpQ1hYGCNHjtTf09PTk5CQkGI+1qxZQ15enj4OgNGjRxMZGcmxY8f0Zbm5uaxZs4YhQ4bo14kZ49PkVMec+vp0UMfXceXk58juq7pL/+X+csOZDVXyodVq5fBP9krv17fIF9ccKVY+Z9NJ6f36Fun9+hb5xvrjUqPRVlfotYI/4v6QYzeN1a//8l/uL6fvnC6jEqNMHVq51OV1XIsXL5aAHDFihIyIiCh2HDmi++zu3btXPvjgg/Lbb7+VO3bskD///LOcPHmyFELIxx57rJTPBx54QDo6OspPPvlEbt++Xb755pvS3NxcTp06tZhdUFCQ1D2CK6asdVxSSrl8+XIJyGnTpsmwsDD5+eefS3t7ezl27NhidllZWbJdu3bS19dXrl27Vv7+++/yoYcekjY2NvLQoUNV8lkW93Idl8mFoLYddV249sXtk/7L/WXH5R1lYmZilXyEn0vUi9Ox2JRi17RarXx3Y6T++lsbjkuttn6Jl0arkdtjtsvRP48uJmAv7npRnko6ZerwSlGXhatQPMo6vL29pZRSnj17Vj744IOySZMm0traWtra2srOnTvLjz/+WObnl15wfvPmTfnKK69ILy8vaWlpKf38/OT8+fOlRqMp896VUZ5wSSllSEiI9Pf3l1ZWVtLLy0vOmjVLZmZmlrK7evWqnDBhgnR2dpa2trayT58+ct++fXfksyT3UriElHW/u6Y6EUKkOjo6OhYOnNY15kbM5cczP9LFowsrH1hZJR//WHmI36MSCGzuxIZ/9il1XUrJuz+fJKRg8saEns15f6Q/Zma1e8KCsWillu2XtvPl319yLvWcvnxQs0FM6TiFTu5VX4ZQnVy6pPs7eXt7mzgSRU2mss+Jk5MTaWlpaVJKpzu9lxrjUujRSq1+/VZVZxPGJmWyPVqXUWFy37JnIQkhmDOyA+N7NAdg1YFYXvnpb/I02irds7ZiJswY5jOM9SPXs6j/Ilo4tgBg1+VdjN86nsm/TeaPK3+gvlwqFMVRwqXQE5UUxfWsO0uquyIiBimhsaMNwzp4lmtnZiaYO8qfib19AAg9eoXnQw6TnVf/Nmo0E2YM9x3OhpEbWBS0SL+I+eC1gzy/43nGbRnHrxd/VZtYKhQFKOFS6NkVuwsAHwcffBx9jK5/MyefHw9eBuDpXj5Ymlf88TIzE7z3cHtmDmkFwM5T13l62V+kZ+cZfe+6gLmZOcN9hvPDiB/4auhX9PDU5YU7lXyKWXtn8fDGh1lzag2ZeZkmjlShMC1KuBR67nTvrXWHLpORk4+NpRlPdm9mUB0hBDOHtOa9h9sD8NfFZB7/6gBX07KqFENdQAhBb6/efDPsG1Y/uJohzYcgEFzOuMy8P+cx5KchLDq4iLgM45PAKhR1ASVcCgAuZ1zWTxAY1GyQ0fW1Wsny8BgAxgQ2xcnOquIKJZjUx5f/jAvA3EwQfTWdUf/9gxNX1B5XHd078vHAj9n4yEYebfUo1ubWZORlsDJqJQ+FPsTMsJkcvHZQjYMp6hVKuBRA8aS6Hd3KXv1fEWGnrxOTpOvCmlQwbmUsYwKbsmJSd+xtLLiekcNjX0bw+8lrVfJV12jh2ILZvWezfex2Xgx8EQ87D7RSy87YnUz+bTLjtowj9GzoXelGNDMzQ6NR42uKitFoNBVmBqlOlHApgNvdhAOaDahSIthlf1wEoF8rN1o1sq9yHH1bubFham+aOtuSlafhuVWH+d/eC6pFUYCzjTPPdHyGXx/9lUX9FxHgHgDoxsHeDX+XwT8NZt6f8/Rb0lQHNjY25OTk6HPbKRQlSU5OJicnBxsbm3tyP4PWcQkhGgLzgMcAJ+Ak8L6UcpMBdf2Aj4CB6IRyH/CqlDKqDNsXgOmANxAHfAUsllJqS9gZ47MFMAcYCjgD14BfpJT/rPSNl/1+6tw6rrScNIJ+CEIjNSwZtIQBzQYYVf/0tQyGfbIXgO8mdmNg29Kb8xnLjYwc/hFyiKOxut/zyAAvFjzaETsrlRe6JJE3Ivn+1Pf8HvM7edrbE1u6eHThsdaPMdR7KDYWVX+gSCm5cuUKGRkZWFtbl7tHlKJ+otFoyMnJwd7eniZNmpSbQNoU67hCgfHAO8BDQBQQKoR4sKJKQggPdKLiAwQDTwIuwB4hRNMStu8AHwNrgWHAt8C/0QlmVX12Ag4BjdAJ4v0F70FtkFSEokl1ezQ2fofT7wpaWy3cGhDUuno2TnS3t2bNsz15pLMXoMt9OHppOBcTb1WL/7pER/eOLOi3gJ2P7eTV+17F20G3APTo9aO8tf8tBv80mA8Pfsi5lHOVeCobIQRNmjTBzc2t2KaJCgWApaUlbm5uFYpWdVNpi6tAnH4BxkgpQwvKBDrxcJVStqug7ofADMBPShlfUOYKXAS+l1JOLVIWB3wtpXyxSP1/A68BvlLKOCN9CuBvIBZ4WFZTX1NdbHG9vPtltl/azsBmA/ls0GdG1U2+lUuv+TvJydfy/qgOPN3Lp1pjk1KyIjyGD36JJl8rsbex4ONxnRnSvnr2X6qLSCn569pf/HTmJ3Ze2km+zNdf6+DagVEtR/GAzwM42dzxF1+FwmDudYtrNJAG/FxYUCACK4C2Qoj2ldTdXigwBXWTgM1A0VTDwwGbAp9FWY5uz7CRVfAZBHQEFlWXaNVFcjQ57L+yH6jaouM1f8WSk6/F3saCRwObVl7BSIQQTOzjy5p/9MTd3pqM7HyeWXmID7ZEkZOvJgyUhRCCHo17sDhoMdsf003maNpQ97c5mXSSeX/OY9BPg3h598vsubyHfG1+JR4VipqFIcLlD0SVHGcCjhe5XgohhC3gB5TeCEZX16Og26/Qh0Q3dqZHSnkWyCq8h5E++xf8NBNC7BdC5AohUoQQa4QQXmW/VV2LqqIDcCyvbm3kr6t/kZWfhZkwI6hZkFF18zRaVkbEAPBEt2Z3dWfjbj4u/DKjL918nAH4Zv9FHv0inAs3bt61e9YF3GzdeKbjM2wds5Xlw5czptUY7CzsyNPmsf3Sdqbvms6Qn4aw+OBiTiefVpNgFLUCQ4TLFShrOlFyketl4QwIA+u6AplSypwybFOK2Bnjs1CcNgDh6MbNXgOGoBsPsysn7npF4WzCzu6dcbFxMaru1sirJKTnYCao9i7CsvBwsGHNsz15YVBLzAScuJLOiCX7+enQZfXArQQhBF0bdWVO7zmEjQtjXt959GjcA4EgKTuJFVErGLt5LKN+HsUXf3/BpfRLlTtVKEyEoV+RK3oqVPbEMLSuMfcwxLZQlH+QUr5W8DpMCBEPbAH+D/imVOVK+l/rUqvrTpPqfvdHDAD3t/ekmcu9+R5gYW7Gy/e3oXdLN1764RhX07KZte44u8/cYO4of1waGLfwuT5iZ2nHw34P87Dfw1y9eZXNFzaz6fwmLqVf4mLaRT4/9jmfH/uc9q7tecDnAYb7DsezQfl5JxWKe40hLa4kym5VFX49L29xRwo6ETGkbhLQQAhhXYatcxE7Y30C/FbC7ndAAwSWE3e94WTiSW5k3QAwegr8kdgUjl3WTVApLwv83aRnC1e2vdiPYR10kzR+OX6Vof/Zw7bIq/c8ltpM44aN+Uenf7D5kc2sHbGW4PbBNLLT/U6jkqL46PBHDF03lOBtwaw9tZbErEQTR6xQGCZcJ4F2QoiStoXpFcoab0JKmQVcoOwxsI7ADSnl9SL3EECHokZCiJaAbeE9jPQZWcF7Aqhfe2iUQWE3oa+jr9FJdQtbWx28HPTjTvcaJzsrvpzQlQ8f7YS9tQVJt3KZ+v0Rpq8+QvKtXJPEVFsRQtDBtQOvdnuV38f+zvLhy3m8zeM4W+v+tkeuH+Hff/6bQT8OInhbMCtPriT+ZnwlXhWKu4MhwhWKbtHxwyXKnwZOl7Xot0TdoUIIfT+DEMKlwNeGInbbgBzgqRL1g4F8dDMGq+IzCyi51mw4YA78WUHc9QJ9Ul0juwmvpmWxtaBlM7mP7z1bu1EWQgjGdWvGby/1168h21LQ+tr0d7wa+6oCZsKMro268k7Pd9g1bhdfDvmSUX6jsLe0RyI5cv0Iiw4tYtj6YTy+5XH+d/x/XEi7YOqwFfUIQ9ZxCWAn0And5IaL6ATlaWCUlHJzgd1uIEhKKYrUbYRuLVU8uuwV+egWALcGukgpY4vYvgf8C/gACAN6Ae8Dn0opZ1XR5+voFjB/gk7IWgFz0a0Z6y6lNPpreV1Zx3U54zIPbtBpesgDIXT26Gxw3Q9/PcXnu8/j1tCaP94YiLVFzcikIKXkp0NxzN0SRUaObop335ZuzH3EH1+3BiaOrvaTp8njz2t/suPSDsIuh5GcXXyUoIVjCwY3H8zg5oNp59oOs1KdNIr6THWu4zI05ZMDOgEYi671FYUu5dPGIja7KSFcBeWtgMWUTs90soSdAF4EpgHN0QnT18DCMlI+GeSzwPZ54AV00+hT0a1He0NKWaXEa3VFuFaeXMmiQ4twtXFl17hdBj9ksnI19Fqwk9TMPGYOacXMIa3vcqTGczUti/d+PsnvUbqdmK3MzXh+gB//HOCHjWXNENnajkar4cj1I+yM3cmOSztIyEwodt3N1o3+TfvTv2l/ejXuhZ2lmsRb37nnwqW4TV0Rrkm/TuJQwiEebfUos3vPNrjemr9ieXNDJFbmZvzxxiDc7cuaT1Mz2BGVwHubTnIlVbe3l7erHbMf7sCANu4m7d6sa0gpOZF4gh2xO9gVu4uY9Jhi1y3NLOnu2Z3+TfsT1CyIJg2bmCZQhUlRwmVC6oJwpWanEvRjEFqpNSqprpSSYZ/s5UzCTR4NbMpH4wLubqDVQGZuPkt2neN/ey+Qr9V91vu1cuOdh9rTxrPqWewV5ROTFsPeuL3sjdvL4YTDxVJOAbR0akn/pv3p26Qvnd07Y2mu8h/WB5RwmZC6IFybzm/i7f1vY2thy97H9xqcOXz/2UQmfKub07JlRl/8m9Se5WxnEzKYvfkkf5zTrZIwE/Bk9+a8NLQ1bg1rbquxtpORm0F4fDh74/ayL24fKTkpxa7bWtjS3bM7vbx60cerD94O3qo1XEdRwmVC6oJwFSbVHdRsEJ8O+tTgelOWH2Tnqet093Xhx+d63cUI7w5SSnZGX2fe1mguFGSZt7e24J8DWzKxtw+2Vmr8626i0WqITIzUt8ZOp5wuZePVwIteXr3o7dWbHo174Ghde74cKSpGCZcJqe3ClaPJod/afmTlZzG3z1weafmIQfUuJt5i4OLdAHw5IZDh/o3vYpR3lzyNllUHLvHJjrOkZen2r/Kwt2b6oJY80a05VhZqNty9IDErkYj4CMLjwwmPDy81S9FMmOHv6k9Pr5509+xOgHvAHe0rpjAtSrhMSG0Xrr1xe5m2cxpmwozd43bjbGPY4uHZm06yPDyGps627Jk1EHOz2t+dk5qZy5Jd5wg5cIncfN3E1SZOtrw4pBVjujTBwlwJ2L1CK7WcTTlLeHw4f8T/wZGEI8U2xQSwMrOik3snunt2p5tnNzq5d8LKXKX4qi0o4TIhtV245kTMYd2ZdQR6BLLigZK7yJRNenYevebt5FauhrcfbMez/Vvc5SjvLfGpWSzZdY4fD11GUzCBo4V7A14c3IqHOjZWAmYCsvKzOJxwmD+u/MFf1/7iTMqZUjY25jYEeATQ3bM73T2708GtA5ZmaqJHTUUJlwmpzcKllVoG/zSYxKxEXun6ChP9JxpU75t9F/jgl2jsrMyJeHMwjrZ18+EQk3iLT3ac4ee/4yn8t/B2tWNqkB+jA5vUmIXW9ZGU7BQOJRzir6t/cfDaQc6nnS9lY2thSxePLgR6BBLYKBB/N39sLWxNEK2iLJRwmZDaLFzHbxxn/NbxAGwZvUW/xXtFaLSSoEVhxKVk8XQvb94fVeb2a3WK09cy+HTnGbaduKYXME8HG/7RvwVPdG+GndXd23dMYRiJWYkcSjjEwasH+evaX6XWjgFYCAvau7ani0cXujTqQhePLkZv3aOoPpRwmZDaLFyfHfmM/0X+jxaOLfj5kZ8rrwD8dvIaz4UcBmDXK0G0cG94N0OsUZy7nsEXuy+w8dgVfReiSwMrJvb2YXyP5riqafQ1huuZ1zl47SCHEw5z9PpRzqWeK9POx8GHwEaB+pZZM/tmavr9PUIJlwmpzcI1+ufRnEs9xxT/KczsOtOgOo9/FcGfF5MZ2Mad7yZ1v8sR1kwuJ2fy9d4L/HDosn4Sh7WFGaO7NGFyX19aN1ILmWsaaTlpHLt+jCPXj3D0+lFOJJ4oNdkDwMXGhY5uHenk3omObh3xd/PH3kr9Pe8GSrhMSG0Vrsvpl3kwVJdUd9WDqwhwrzzrxcn4NB76bD8AIVO606+V+12NsaZzPSObZftjWP3nJdKzb2eD6NfKjcl9fQlq5Y5ZHZhtWRfJ0eQQlRTFkQSdkB29fpT03PRSdgJBC8cWdHTvSEe3jgS4B+Dn5IeFmeoevlOUcJmQ2ipcK06uYPGhxUYl1X31p79ZdziOVh4N+f2l/qpLpYBbOfmsPxLHd3/EcLFgITOAn3sDxvfw5tHApjja1c0JLHUFrdRyPvU8x28cJzIxkr9v/M351PPIMjZXt7Wwpb1rezq5daKTeyc6uHbAs4Gn+n8wEiVcJqS2CtfEXydyOOGwwUl1E2/m0Hv+LnI1WuaN7sj/9Wh+94OsZWi1krDT1/l2/0XCzyfpy5tYpDOryUna9BhG2y591QOulnAr7xYnE09yPPG4XtDK2/HZxcaFdi7taO/aXn80btBY/a0rQAmXCamNwpWSncKAHweglVr+O+i/BDULqrTOpzvO8vGOMzjZWRLxxmCVDqkSoq+mE3LgEhePhvGJ+IhGQvf5OGfeguTWj9Nu2BTsnep3V2ttQ0rJ1VtXOZ54nMgbkUQmRhKVFEWOJqdMeydrp2JC1t61PV4NvJSYFaCEy4TURuH6+dzPvPPHOwYn1c3J19B3YRg3MnKYOsCP14e3vUeR1nKOrkJueQmhyUWDGebc3kYuR1pywjEI6+4Tad/rQczM1ReB2kieNo8LqReISorSHclRnE4+Xa6YOVo70t6lPe1c29HGuQ1tXNrg7eBdL8fMqlO46t9vrx6y+/JuAHp79TYo19svx69yIyMHczPBUz0rX+tV79Hkwe/vwJ9fIgBcW2H2xGrOxMaRsv9b/JN30EBk0zV9B+zYwZWdjYhpMormAyfSzK+DqaNXGIGlmSVtXHQCNLrVaADytflcSCsiZkk6McvWZJOWk0bE1QgirkbofVibW+Pn5KcXstbOrWnt3FolFDYC1eIyktrW4iqaVPeDPh8wquWoCu2llDz83/2cuJLOiE6N+e//Bd6jSGspt5Lgp2CI2ac7bzUMHv0f2Nx+CKWlpRC9fSVOp9fQNi+6WPUzFm1IazWa1oOextFdbbBYV8jX5nMx7aJeyE4ln+JMyhlu5t0st07jBo31ItbGpQ1tnNvQ3KG5wbuT13RUV6EJqW3CZWxS3YMxyTz2pe7b4fqpvenqbVgS3nrJtROw9klIjdWd93sFBr4NZuV3A8adOcrV3d/gHb8ND25P6MiXZkTZdiW73VjaDnwCB4c7/t9W1DCklFy5eYXTKac5k3KGM8lnOJ1ymssZl8utY2thi5+jH35OfrR0aqn/WRtnNSrhMiG1Tbhmh89m/dn1BifV/ef3h9kaeY2AZk5s/GfvWvfPcc84GQob/wl5mWBpB6OWgv8Yg6tr8/OJOrCNm4fW0D4lDAeRqb+WKa2JbtgD2X4U7fs/ip29+vJQl7mVd4uzKWc5nXya0ym642zKWbLys8qt08CyAX6OfrR0bqn7WSBqHnYeNfZ/VgmXCalNwlU0qe6r971KcIfgCu3jUjLp/2EYWgmfPtGZUZ1V11UptFoI+zfsW6w7d2wOT3wPjTtV2WVW5i2i9/6EiPyJDjcPYCVuL27OkZacatgdTduRtOr3GPZOrnf6DhS1AK3UEpcRx+mU05xLPce5lHOcTz3PpfRL5Mv8cuvZW9rj51S8hebr6Esju0YmFzQlXCakNgnX3zf+ZsLWCQD8MvoXmjtUvBZr/tZovtp7gUYO1ux7bZDaULEk2emw4R9wZpvu3LsvjFsBDdyq7RY3UxM5s/cHzKM30S7zUDERy5XmRNneR1arh2jZZwzuns2q7b6K2kGeJo9L6Zc4l6YTsvOp5zmXeo7Y9Fg0UlNuPTsLO7wdvPF19MXH0QdfR198HXzxdvC+Z5tzKuEyIbVJuD498infRH6Dn6MfGx/ZWKFtZm4+PeftJD07n1fvb830Qa3uUZS1hMRzsPb/ILFgu/nu/4Bh88D87mXISEtN4tTuH7E4vZkOmX9hI27n2tNKwRnLtqQ1G4RXj9E0a3Mf1NAuIsXdJ1eTS0x6DOdTz3M25axO1NLOcznjMlqpLbeeQODV0AsfB52Y6X86+uBu616trTQlXCakNgnXIxsf4XzaeZ7p+AwvBr5YoW3IgUv8a+MJrC3MCH9jkMp8XpSzO2DdZMhJAzNLGPEfCHz6noZwMyOVM/s3YBa1kdYZB7Cj+Lqhq8KDWLf+2Po/ROvuw7Gxtbun8SlqJrmaXGLTY4lJj+Fi2kUupl3Uv65ohiPoxtF8HHzwcfTB28EbHwcfmjs0x9vem4ZWxu8SoYTLhNQW4YpNj+Wh0IcA+P7B7+nkXv4YjFYrGfLxHi7cuMXj9zVj4diqj9fUKaSEPz6FHbMBCQ084PFV0LyHScPKzc7izJ/buBW5meaJe2lM8bREt6QNp+y6kus7kOb3PUyTFmoBuaI4UkqSspP0YnYx7SIX0y8SkxZD/M34MnM2FmXlAyvp4tHFqHuqBciKSgm7HAaAm60b/m4Vb/649+wNLtzQJYud1NfnbodWO8jNhE0z4MQ63blXoE60HE0/YcXKxhb/oDEQNAap1XIm8k8Sj2zC9couWuWd1i12zvoDov6AqA+IE42Jd+uNTduh+HUbTgMHNUuxviOEwM3WDTdbN7p5dit2LTs/m0vpl/Qts0vpl/TnGbkZADRt2NQUYetRwlVH2RW7C4CgpkGVLmBc9kcMAL39XGnr6XC3Q6v5pF7WjWddO647D3gSRnwClvdmENsYhJkZrQN60TqgFwAp1+OIiQhFnt9Fi/S/cOImTeVVmt5YDzfWk7fXnCir9qR49cXZ/35ade6LpaWVid+FoiZhY2Gjzw5SFCklqTmpXEq/hJtt9U1IqgpKuOogKdkpHLtxDIBBzQdVaHvuegZ7z9wAYHIf37seW43nUjj88BRkJoIwg/v/DT2n1pqJD84eTXEeNQOYgSY/n9PHw0k8thXHq/tokxuNpdDQPi8SLkXCpS/I2GLLSbtO5DTpjXvHIfh06ImZhXosKEojhMDZxrnSJAb3AoM+oUKIhsA84DHACTgJvC+l3GRAXT/gI2AgYAbsA16VUkaVYfsCMB3wBuKAr4DFUhafFmOMzyJ1BgC7AAE4Sylr9iDVHbA3bi9aqcXWwpYejSsej/muoLXl7WrHoLYe9yC6GszBb2Hba6DNBxsneGw5+A00dVRVxtzCgjaB/WkT2B+AtNRkLhzcRt6ZnXglhdNUexV7kUXnrD/h3J9w7mMyQu24YBdAVpPeuPkPxrdDD8yVkClqGIZ+IkOBQOA14CIwEQgVQjwspdxaXiUhhAc6LjAvAgAAIABJREFUUbkOBAP5wDvAHiFEFyllXBHbd4A5wL/RCUzvgtcuwBtV8Vmkji3wDXANaGzge661FI5v9fHqg7V5+bMDUzNz2XDkCgATe/vU391783N1gnX4O925ezt4cjW4tDBtXNWMo5MLXYaOh6HjAbgRd5ZLh39HXtxHk9TDeHEdezIJyIyAsxFw9iPSNjTgvF0nsj274dS2Hy069VUzFhUmp1LhEkI8CAwBxkgpQwvKwoAW6Fo95QoX8CrgDNwnpYwvqBuBTvzeBqYWlLkWnP9XSvluQd3dQogGwGtCiP8WESSDfJZgLpABrC2wqbNk52cTHh8OwMDmFbcW1h68TFaehobWFoztatrBVpNx8zr8+DTEFmTvbjsCRn8J1vamjese4N60Fe5NWwHTkFISe+EUV45tx/zSfpqnH8aTRBzFLQKzIuBiBFz8jNytFpyyak26eyC2LfrQvPMAHN28TP1WFPUMQ1pco4E04OfCAimlFEKsAL4WQrSvoItuNLC9UGAK6iYJITYDY7gtMsMBG6BkMr3lwFvASOBzI30CIIToBswA+gIPGfB+azV/Xv2TrPwszIQZ/Zr0K9cuX6NlZXgMAOPua4a9TT3cav7KEfhhAqTrWp0MeAv6zwKz+pcxRAhBc792NPdrB7wAUnL10mmuHvsd7aUIPFOP0VTGYyXyaZsXBfFREL8K9sNl4cVVh05omvbArV0/fNt0xsKyHn6eFPcMQ4TLH4gqOc4EHC96vWSlgu45P+CnMnweB/5PCOEhpbxe4EOiGzvTI6U8K4TIKrhurE+EEJb/396dx0dZ3Ysf/3yzb5ANsieE1bApIqKCKLjvC9aqVUG0vb12/9322l9v7Y+2v6u9Wm9rW9vbeqvFUnevuKJWKyAiuKHsyJaQkIXsQ0LWmTn3jzMDQzJJJkAyM8n3/XrNi9fzzHnOnDnMzDfP85zzPcBjwH8ZYz4WkT4Dl4j0de8rpBfN8V4mPD3j9F5vor617SAVjjZE7GXCYWfzc3a4u7MNYpLg+j/B5KuC3arQIUJ2YRHZhUXAdwCoP3iAks9X07ZvHSl1nzGhczcx4iTfVJDvqADHm7DNM48sdhLN6acRVziLvGnnMipnfNgMcFGhL5DAlQ7s8rO/3ud5f1KxAyHq/Tzne2y1598WY4y/ZUQbfF6jP3WCPVtLwd4DG/Lcxn1k0cgF+b1fJvzLumIALpqcSUH6MLpn4XbBO0vhg9/Z7dSxcMvTkDE5uO0KA2mZeaRdehtg81+2thxmx5Z1OL5YS1zVp4xp2UIqh0iUNqZ1bIbKzVC5HNZDHSkcSCiidfQM4sfOJm/qXNJHZwX3DamwFejgjN6mUfeVeiPQY/vzGn2WFZGp2MB1gzGm99wmvgf3Mavbc0YWkmddW2q3UNdm13i6IL/nYfCbDzTyyf4GAJbMLRyMpoWG1gZ44S7Y+w+7PW4BfOlxSEgLbrvCVHxCIpPPugTOugQA43ZzsGwX5Vvfp6P0E0bWb2Zsx27ipYN0Gklv2QD7N8D+P8JqOEAWVYmn0DF6GkljZpI35SzSMjVxsOpbIIGrDv9nVd5vu7+zH7BnSibAY+uARBGJ9XPWlepTrj91Pgq8DbwvIt5g5J1Bmiwizv4EtHCwqtReJpyQMoH8kT3/AHiHwBdljeCcccNkmYzqnXbRx/p9dvucb8FFP4NIHep9skhEBJljisgcUwR8FYD2jnZ2bv+Uht0biKjYyKhDWxnj3E+UuMmjirzDVXB4DZQAa6CGVKriJ9E6aioxeTPImDSb7DGnIMPwvqPqWSDf2m3ADSIS0eU+13TPv1v9HWSMaRWRfXjuT3UxHajx3ovyvIYAU4GN3kIiMgGI975GP+ucij0zavBTtgT4EDjbX9vDlff+Vm+XCasPtfHaZjuu5c65Y4O+Rs+g2Pm6XY6koxkiY+Ga38FpNwW7VcNCbEwsRTPmwIw5R/a1tTSxb9sGHHs+RA5uJr3pC/KdpUSJm9E0MLr1Qyj7EMqA9XDIJFAWMx5HymQisk8lpXAGBafMICFx6I/8VP4FErhWAHcBV+MzshBYBHzR26Rfz7HfEpEsY0wVgIikeep62qfcG0A7cDs+gYuj87RePY46r/Lz/u7w1Hk1UMEQsv/QfvY57NnE/Pz5PZb724b9dLoMaYkxXDNjiA9jdrvtgo+r7rPbI3Lsoo+5M4PbrmEuLmEEk868GM68+Mi+ttbD7N3xKfV7PsZUbSbt0E7GdO4jXjoYKS1M7dwCNVug5jnYDC4jlEZkU5cwno70IuLyTiVj/Awyx0zRzB/DQCD/wyuBVcBjnvlWxdgf/3OBa72FRGQ1cL4xxvdP+IewwWiliPyMo5OFndhMHMCR4ey/AH4iIg7P650D/BB42BhTdhx1vt/1jXiyZwC8P9QyZ3gvE46OH91jUt22ThdPflgKwK1nFRAXHTlo7Rt07c3w0t2ww5PcJf8s+PJyGJEZ3HYpv+LiE22GD0+WDwBnZycle7dQt+cTnOWfk9Swney2vaRxiEgxFJgKCg5XwOG1UAp8AG0mmvKoAuqTJtCZPpn43GlkTJhJdt5YIiL1cuNQ0Wfg8szZug4bFO7HjtLbjp2Q/Gofxx4UkXnYYLOco+mZzjPGlHYp/nPsfLFvAj/CnhEtBR44gTqHDe9lwvPze06q+8qmCuoOdxAdKdx29pjBbN7gqi+GZ26Fas/sipmL4YpfQpSuMRZOoqKjKSyaSWHRsWfIDdXlVHzxCYdKNxNRs4OUpt3kOfeTKO3ESSfjXXsZ79gLjrdgH7AWmk08FVF5OJLG4UqbQFzOFEaPnU7WmMlEapLhsKPrcfVTKK7HVd9Wz4LnFuA2bn5/4e85L++8bmWMMVz+m7XsrGriuhk5PHxz/9bSCRv7VsPzd9gRhBFRcPkDMOsunUM0xHU6nZQXf0F98We0lW8ltn4now7vIddVTpT0vAJwh4mkKjKH+vhC2lLGE5lRxMj8qWRPmM5IXf7lpNL1uNQxAkmqu2FfPTur7Fo6d547BLPAGwMb/gv+fi8YFySk20uDhXOD3TI1CKKjoiicOJXCiVOP2e9sb6WseCu1JVvpqNxBZP0eUluKyXEeIF46iBEXBe4yCg6X2UuO5cBn9tgq0qmOyedw0hhM6njisyaSlj+Z7MIiYuLiB/9NqiM0cA0B3vtb5+ae22NS3cc9E47PGJPKqXkn/AdPaOlsg9f+D2x6ym5nTYebn4KUguC2SwVdVGw8+UVnkl907GKJTqeTstI9VO/bRFvlTiLrdzOyuZjszlJSOQRAFnVkddRB/ed2ks1ee6zLCJURo6mLzaN1RCEmbTxxmRNJzS8is+AUYmJDb922oUYDV5hrc7axvtImiO1pNGFpXQvv7DgIDMEJx4cq4dlbofxTuz11IVz7e4gZRtlAVL9FRUWRP66I/HFF3Z5z1FZStW8LzQe24ardQ4yjmJTWMrJdlcRKJ5FiyDbVZLdVQ9tGqAG+sMc6TQTlEaOpjcmzZ2ophcRmjCc5ewJZYyYxIlknu58MGrjC3IbKDbQ6W4mUSM7L7X5vC2DZByUYAznJcVw2dQil2Sn7yCbJbT4ICFy0FOZ+T+9nqROSPCqb5FHZMPuSY/a7XS4qDuyldv8Omit3Yer2Et9UQlpbGTnuKmLESZS4yTUHyW0/CO2f2tQKe4/W0cBIaqKyaI7PxTmygMj0scRnjCM1bxKjc8YRFaMDiAKhgSvM+SbVTYnrfgmwqa2T5z6xswluP6eQqKEyJHjjcnj9X8DVAbHJcMOfYdIlfR+n1HGKiIwkZ8wkcsZM6vacs7OTivJ91JfuoKXqC6jbR1xTCSPbKsh0VREvHQCkcohU5yFo2mUXWir3qcNEUBExivrobA4n5HkCWyFJGWNJyxnP6JwxRGvWfUADV1gLJKnuC58eoLndSVx0BLfMHgJ54Fyd8NaP4aM/2e30iTZJ7qiJwW2XGtaioqPJKTyFnMJTuj1n3G7qasqpKf2Cpqo9dNSWENW4n4SWctI7K8g0tUSKIUrc5JhqcjqqoWMTNGLnp3l0mkgqJY2GmCxa4rNwJeUSmVpAQkYhyVnjSM8dR1xiSKZRPek0cIWxzTWbqW+zqRn9LRrpdhuWedbcWjgzj5SEMJ+vcrgOnl8MJWvt9sRL4Yb/hrjh8WVV4UkiIkjPzCc9Mx+7Ju+x2traKD+wh8aKPbRV78VdX0JMUxkjWssZ5aw6MlgkWlxkU0N2Rw10bLGzXsuPrauRJOoiMzgUm0V7QjbukflEpeWTOLqQ1OyxpGflExsT5r8DaOAKa97LhBNSJpA/ovvZ1Ls7q9lf1wLAknBfc6tqCzzzFWj0/Ak67/uw4McQMYSzf6hhIS4ujoIJ0yiY4D/jTXtrE7UH9tJQWUxLTQmuhjKimg4Q31pJSsdBMkwtMeICIIVmUlzN0LIPWoBa7CRsD6eJ4KCk0Bg1msMxGXQkZsGIbKJS80ganU9yZiGjcgqJjg3twU0auMJYX0l1//KBHQI/b+IoJmaGcULSbSvgpW9AZwtEJ9hRg9MWBrtVSg2K2PgR5E6cQe7EGX6f7+h0Ul5ZSmNVMS01xTjrSpFDB4hrqWREexWjXNUkYxfCiBI3mdST6awH5xc2uNV0r7ORETREptMUk0FbfCbupBwik3OITc8nKSOfnLHTiIsPXnDTwBWmShwlFDtsYPIXuHZWHWLdHrs2V9hOOHa7bYLctQ/Z7eQCmyQ3+9TgtkupEBITHUVuwThyC8b1WKatuZG6yhIcB/fTWldKZ0MF0lRBbMtBkjqqSXXVko7jSPkUmkhxNUFrCbTSbfGqXde8wqSZ5w/I+wmEBq4w5T3bGh0/mqmjpnZ7fplnza1xoxI5f+LowWzaydHmsEuR7HrTbhfOgxuXQeKooDZLqXAUl5TS61kbQGtLC7VVpTRWldBaX4azoRxpqiDmcBWJ7dWkOGtINw1Ei4vUrODmOtXAFaa8gWt+/vxuSXXrD3ew4jN713bJ3EIiIsJsXlPtHrvoY+0uuz3763DpfRCpQ4GVGijxCQk9Tsr2crtc1NaUkz46dxBb1p0GrjBU31bP59WfA/4vEz79USntTjcj4qJYODNvsJt3Yna/DS/cBe0OiIiGq34FMxcFu1VKKexctlFZwU+lpoErDK0pW4PBkBCV0C2pbqfLzV/XlwBwy+wCEmPD5L/YGFj3G3jnp4CBpEy46W+QPzvYLVNKhZgw+VVTvryXCefmziUm8tg5GSu3VHLwUDsRAovOCZM1tzpa4JVvw9YX7HbOTDsIY+QQX6FZKXVcNHCFmVZnK+srbFJdf5cJH/cMyrh0ahZ5qaE9FwOAxjI7P6tqs90+7Stw1a8hWjNsK6X808AVZjZUbKDN1WaT6nZZMHJjaQObyuwCl0vmhsEQ+JJ18NwiaKkFiYBL7oOz79YkuUqpXmngCjPey4QzM2eSHHtsqqPH37fzuqbljuTMwhBfvfXjP8MbPwS3E+JS7FD38f4nUiullC8NXGHE5Xax5sAaoPtlwkpHK29srQJgyZyxSKietTg74I1/hU+X2e2MKfZ+VlrPkyeVUsqXBq4wsqV2y5Gkul0Xjfzr+v243IZRSbFcdVp2EFoXgOZqePZ2KNtgtydfDdf9EWKTgtsupVRY0cAVRt4texfonlS3tcPF0x/Z5LO3nV1AbFQIJp4t32gXfTzkSWe94Mcw7wcQMUTWB1NKDRoNXGFkVan/pLorPiunsaWTmMgIbj0rBIfAb3oWXv0OONsgJgmu/xNMvirYrVJKhSkNXGGi2FFMyaESAC4ouODIfmMMf1lnB2VcfVoOo0eE0NLfbhe8sxQ++J3dTh1rF33MmBzcdimlwpoGrjDhHU2YEZ/BlPQpR/a/v6eW3dV2yYIlcwuD0TT/WhvghTthr728yfgL4IbHICEtuO1SSoU9DVxhYnXZaqB7Ut2/eCYczx6bxrTcEFkJuHoHPH0LNNgzQeZ8Gy78KUTqx00pdeL0lyQM1LXWHU2qW3D0/lZx7WHe3VkNwJ2hMuF45+t2OZKOZoiMhWt+B6fdFOxWKaWGEA1cYeC9A+8dSao7O+to0tllnntbeanxXDwlM1jNs9xueO+XsPp+uz0y1ybJzZ0Z3HYppYacgMYii0iSiPxWRCpFpFVEPhGRawI8dryIvCQiDhFpEpGVIjKlh7LfEZFdItIuIntF5B4R6dbGQOoUkUki8isR+cxTrk5E1gba7lDiHQbvm1TX0drJ858eAOCOOYVEBnPNrfZmeH7R0aCVfzZ8bZUGLaXUgAh0Es0K4FbgXuBKYDuwQkSu6O0gEckA1gKFwGLgFiANWCMieV3K3gv8GngGuBR4DLgPuP8467wEuBx4HvgScDtwAHhZRL4X4PsOulZnKxsq7IRd32Hwz39SRkuHi4SYSG6cld/T4QOvvhgeuxh2vGq3Zy6Gxa/CiCCfASqlhqw+LxV6gtNFwEJjzArPvlXAOOA/gZW9HP4DIBWYZYyp8By7HigGfgzc7dmX7tl+xBjz/zzHrhaRROAeEXnEGHOgP3ViA+DvjTHGpz0rRSQLG4Af7uu9h4L1Feu7JdV1uQ3LPigB4MYz8kiOD9LKwHtXwQtL7AjCiCi4/AGYdZcmyVVKDahAzriuBxzAy94dnmDwBFDU02U/n2Pf9gYYz7F1wKvAQp9ylwFxnjp9LcMGV9/LewHVaYyp7RK0vD4G0kUkvpd2hwzvaMIzMs84klT37e0HOdDQCsDiOYWD3yhjYP0f4G8LbdBKGAWLXoEzv6pBSyk14AIJXNOA7cYYd5f9m32e78YTGMYDW/08vRnI8Fz289ZhgG2+hYwxu4FW72v0s05/bRJgAbDPGNPaQ5nG3h7AoI057ymprnfC8QVFGYwbPch5/jrb4KVvwFs/AuOGrFPhn1ZD4dzBbYdSatgKZFRhOrDLz/56n+f9SQXEp1xPx1Z7/m0xxrT7Kdvg8xr9qdOf7wKzgDt7eD6kbK7d3C2p7rYKBx8W232DPuH4UIXNN1j+qd2edgNc8wjEhMGClUqpISPQ4fD+LrkF8lx/ju3Pa/S7PSJyHfAQsMwY85ceDzYmpZe6GcyzLm9uwompE8kbYcedeCccT8xI4twJowajGVbZRzZoNR8EBC5aCnO/p5cGlVKDLpDAVYf/sypv7h5/Zz9gz5RMgMfWAYkiEuvnrCvVp1x/6jxCRK4EngVeBL7aQ3tDjjfNk/cyYU1TO698bm/tLZk7iGtubVwOr/8LuDogNhlu+DNMumRwXlsppboI5B7XNmCyn/lU0z3/+rvfhOce0j783wObDtQYY7yX9LZhLwFO9S0kIhOAeO9r9LNObx2XYwPWG8CtxhiXv/aGmn2OfUeT6ubbpLpPfVhKh8tNSkI015+eO/CNcHXCyn+FV75lg9aoSfC1dzVoKaWCKpDAtQJIAa7usn8R8IUxZnsfx17sGYIOgIikeep60afcG0A7dq6Vr8WAEztisL91IiKXesq/A3zZGNPZS1tDinc0YUaCTarb7nSxfMN+AG6ZXUB8zACvuXW4DpZfDx89arcnXQZffQdGTRjY11VKqT4EcqlwJbAKeMwz36oYG1DOBa71FhKR1cD5xhjf61cPYYPRShH5GTYI3ev598jEYmNMnYj8AviJiDg8r3cO8EPgYWNMWX/rFJFzsUGrHHgQmNnl0tpnPQwGCQm+a2+JCK9vrqS2uZ3ICGHROQO85lblZnjmVnDYxSmZ9wO78KMu+qiUCgF9Bi5jjPEMbLjf80jBZs5YaIx5tY9jD4rIPGywWY49w1sLnGeMKe1S/OfY+WLfBH4EVABLgQeOs86LsJcZxwGr/TRvLFDSW/uDpba1lk01mwA7mtAYw+OeIfCXT8siO3kAp6BtW2GHu3e2QHQCXPcHmHr9wL2eUkr1k/ifo6t6IiKNycnJyY2NjQP2Gi/ufpGlHywlMTqR9256j01lzdz4x/X2uW/MYWZB6sl/UbcbVv07rP1Pu51cALc8BVnTez9OKaUCkJKSgsPhcPQ1cjsQmh0+BHkvE87NsUl1H3/fnm2dlp8yMEGrzWGXItn1pt0unAc3PgGJPU3RU0qp4NHAFWJaOltYX2nPrhYULOBAQwtvbasC4M6BmHBcu9su+li3227P/jpceh9EBin/oVJK9UEDV4hZX7medlc7kRLJvNx5/P4f+3EbyBwZyxXTs0/ui+1+G164C9odEBkDV/4KZnYd2KmUUqFFA1eI8Q6Dn5U5iygSeeYjO95k0TmFREeepFF9xsC6h+GdnwEGkjLtoo/5s/s8VCmlgk0DVwhxuV28d+A9wI4mfHHjAQ61OYmNiuCW2QUn50U6WuyE4q3/Y7dzz7BBa2TOyalfKaUGmAauELKpZtORpLrn581n8Rt7ALj+9FzSEmNO/AUaS+38rCpPYv/TvgJX/Rqi4068bqWUGiQauEKINzfhpNRJ7KmMYV/NYQDuOBmDMkreh+cWQUsdSCRc8u9w9t2aJFcpFXY0cIUIY8wxSXW9WeDnTkinKGvkiVQMnzwGb/wQ3E6IT4Ubl8G4+SfaZKWUCgoNXCGi2FHM/kM2F+HEpLN4aJfNFbxkztjjr9TZASt/ABs9C0tnTIGbn4K0E6hTKaWCTANXiPCebWUkZLBmSywAY9ITuKCoxwWde9d0EJ67Hco+tNuTr4br/gixg7xislJKnWQauEKEN3DNyT6P598qB+COOYVERBzHPajyjXYQRpNdu4sFP7aJcjVJrlJqCNDAFQJqW2vZXGNH+rmap9LW6WZEbBQ3zsrvf2WbnoVXvg2udohJgoWPQtGVJ7nFSikVPBq4QsCasjUYDInRiaz6bATg5MZZ+STF9uO/x+WEd5bC+kfsdto4ez8rY/KAtFkppYJFA1cI8F4mHJ84i3UOJyL2MmHAWurhhTthn62H8RfAlx63IwiVUmqI0cAVZC2dLWyo3ABA9UG7uvBFkzMpSE8IrILqHTZJboPNIM+cb8OFP4VI/a9VSg1N+usWZL5JdXeX5AJw59wAh6vveA1WfB06miEqDq75HZz65QFsrVJKBZ8GriDzrr01klNodCdQlDWCs8el9X6Q2w3v/RJW32+3R+bCzU9CzukD3FqllAo+DVxB5JtU13uZ8M5zxyK9pWFqb4IV/ww7X7Pb+WfDTcsh6TjneymlVJjRwBVEn9d8TkN7AwDtjsmkJ8ZwzWm9ZGmv3wdPfwVqdtjtM+6Ay38JUSchAa9SSoUJDVxB5L1MSEcOxpnKrecVEBcd6b/w3lXw/B3Q1ggRUXD5g3DmXYPWVqWUChUauILEN6luu2My0ZHCbWeP8VcQNvwB/n4vGDckjIIv/xUK5w5yi5VSKjRo4AqSYkcxpU12dWNn8xSuPTWHjJFd1sXqbIPXvgebnrbbWafaScUpx5FRQymlhggNXEHybtm7ALg7k3G35bCk65pbhypsvsGKjXZ72g1wzSMQE+D8LqWUGqI0cAWJ9zKhs2kys8akcWpeytEnyz6CZ2+D5oOAwEU/hbnf1UUflVIKDVxBUdtay5aaLYC9TLjkQp8Jxxv/Cq9/H1wdEJsMX3oMJl4cpJYqpVTo0cAVBKvLVmMwGFcsGVFTuHRqJrg64a1/g48etYVGTYKbn4ZRE4LbWKWUCjEauILg7ZJ/AOBsPoXFcyYQ1VZvh7qXrLUFJl1mlyOJSw5eI5VSKkQFtLKgiCSJyG9FpFJEWkXkExG5JsBjx4vISyLiEJEmEVkpIlN6KPsdEdklIu0isldE7hGRbm0ciDoHS0tnCx9W2VWJpXUqtxY0wqMLjgateT+wZ1oatJRSyq9Af8BXALcC9wJXAtuBFSJyRW8HiUgGsBYoBBYDtwBpwBoRyetS9l7g18AzwKXAY8B9wP0DXedgWlf+AS7TiTER3JMRyYgnrwRHKUQnwI3L4MKf6ErFSinVCzHG9F7ABqfXgYXGmBWefYINHunGmB5XKhSRB4FvA+ONMRWefelAMfCkMeZun30HgEeNMd/1Of4+4B5grDHmwEDV2R8i0picnJzc2NjY30MBuPO17/Nx3d/JbUnizYPb7c6UAjs/K2v6cdWplFKhLiUlBYfD4TDGpPRduneB/Gl/PeAAXvbuMDbaPQEU9XSJzufYt70BxnNsHfAqsNCn3GVAnKdOX8uw9+F8L0sORJ2Dwul28lntOgBub7WTjymcB19brUFLKaUCFEjgmgZsN8a4u+zf7PN8NyISD4wHtvp5ejOQ4bns563DANt8CxljdgOt3tcYiDr9tLuxtwdw3Def3vjkeZxyGIAFLa0w++tw+wpITD/eKpVSatgJJHClA/V+9tf7PO9PKiABHpsOtBhj2v2UbfApNxB1Dpq1G/8AwCntnWRd8Ru44kGIjB7sZiilVFgLdDh8bzfCer9JFvix/XmNgajT7uzj+uuJnHVdfcGDJP/ju5ichUTMvP14qlBKqWEvkMBVh/+zE+8yvf7OfsCe1ZgAj60DEkUk1s8ZUqpPuYGoc9DMm3QO8yZ9NNgvq5RSQ0oglwq3AZP9zH3yjibwd78JY0wrsA//95KmAzXGmGqf1xBgqm8hEZkAxHtfYyDqVEopFV4CCVwrgBTg6i77FwFfGGO293HsxSKS5d0hImmeul70KfcG0A50vX62GHBiRwwOZJ1KKaXCRCCXClcCq4DHfOZLLQbOBa71FhKR1cD5xhjfFOYPYQPHShH5GTZg3Ov598gkYGNMnYj8AviJiDg8r3cO8EPgYWNM2QDXqZRSKkz0GbiMMUZErsMGhfuxZ1/bsROSez1rMcYcFJF52GCzHHuGtxY4zxhT2qX4z7Hzxb4J/AioAJYCDwx0nUoppcJHn5kz1LFONHOGUkoNR4OdOUMppZQKGXrG1U8i4gYkOVmztyulVKAcDgfYu08nfMKkgaufRMSJPVM9dJxVeCOe4+S0aMjT/uof7a/+0f7qnxPpr5GA2xjxtf+BAAAGCElEQVRzwutAauAaZJ7MG31m6FCW9lf/aH/1j/ZX/4RKf+k9LqWUUmFFA5dSSqmwooFLKaVUWNHApZRSKqxo4FJKKRVWNHAppZQKKxq4lFJKhRWdx6WUUiqs6BmXUkqpsKKBSymlVFjRwKWUUiqsaOAaJCKSJCK/FZFKEWkVkU9E5Jpgt2sgiMh8ETE9PIq6lL1YRDZ4+qRaRP4kIt3yoPWn/wKtMxhEJE9EfiMi74tIs6dP5vdQ9isisklE2kTkgIj8h4jE+SmXKSJPiEitiBwWkbUiMmew6hxogfaZiJT08Jn7Dz9lh2SficiFIrJMRL4QkRZPe18Ukel+ygbtu3fCv4fGGH0MwgN4G6gD7gIuAP4KuIArgt22AXiv8wED3AOc3eUR16VcJ/A8cBGwCKgE1gERx9N//akziH1TDbwJvOzpp/l+yt3mee4PwALgG0AT8EyXcnHAVqAEuAW4BFgJtAKnD3SdIdZnJcAaP5+5/OHSZ57P/bvAPwPnA18GPgHagLND5bsXaJ09vs/B/hAOxwdwhefDf73PPgHeB3YEu30D8H7ne97vdX2U+wj4zPdDDVzsOfam4+m/QOsMYt/4tus6fz/CQKTnC/9yl/1f85Q/y2ffNzz7ZvrsiwX2AW8MZJ2h1Gee50qAlwKob8j2GZDhZ18K0AD8j8++oH33+lNnTw+9VDg4rseuX/Oyd4ex/1tPAEUiMiVYDQsWEckFzgSWG2Pc3v3GmLeBcuAGn+IB9V8/6wwK33b14mwgC/v+fD2J/Yu2a99sMcZs9HmNduBp4GIRGTGAdQ6KAPusP4Zsnxljqv3sawR2A3kQEt+9E/491MA1OKYB2/18ATf7PD8U/UlEnCLiEJHXROQMn+e873mrn+O2cGyfBNp//akzlPl9H8aYFmAv3fvG3/vdjD1jmDyAdYaiCzz3wTpEZIuI3C0i0qXMsOozERnNse0L9nfvhH8PNXANjnSg3s/+ep/nhxIH8DDwT9j7Av8KTAHWichZnjLe99xTv/j2SaD91586Q1mw+yZcP6+vAd/BXoq6CdiFvTf1qy7lhk2feYL2o9jf+oe6tCVsP18nvISyClhvKUqGVPoSY8xn2GvdXmtF5BXsX2P3YW/cHineUzV9bB9P2XDr52D2Tdh9Xo0x3+qya4WIPAl8R0QeNsbs9y3eW1V9bPe3XF/PDaRfYu8LLjHG7OjyXNh+vvSMa3DU4f+viDTPv/7++hhSjDFVwN+x9w3A9gn03C++fRJo//WnzlAW7L4ZSp/XJ7C/c7N99g2LPhOR+4DvA981xizzeSrsP18auAbHNmCyiHTtb+/cCn/XhYeiCI7+NbXN86+/69nTObZPAu2//tQZyvy+DxFJAMbTvW96er8uYOcA1hkOvJ8Z3/spQ77PROTnwL8B9xhjftvl6WB/907893CwhmkO5wdwJfYH+9ou+98Ddga7fYPUB1nYv7Te9tn3MfApxw6fvdDTVzcfT/8FWmcoPOh5OHwUdhj2ii777/KU952P803Pvhk++2KwgwfeHMg6Q6nPein/FDZwFAyXPgOWetpyby9lgvbd60+dPbY/mB/C4fLAzlF4F6gF7sQOWFiG/Svw6mC3bwDe75PA/8cOe52PnQxZDLQAs3zKXQA4gWc9H/DbgQpgAxB5PP0XaJ1B7p8veR4PeL7ASz3bl/uUWex57hFPH94NHAKe71JXHLAdO1/oJuy8mdewE1/P6FL2pNcZKn2GnfT7jOf/ewF2+PUKT9kHh0ufYS8NGuBVuk/EPt2nXNC+e/2ps8f3Gewv8XB5ACM9H/4q7Cz2jfQxQTdcH8D/BT4HGrHzXao8PyrT/JS9DPjQ0yc1wH8DqSfSf4HWGcT+MT08SrqUuw07lLgdOxfmQSDeT31ZwHLsvYEW7ETOc3t47ZNeZyj0meeH+R3sGVIHNrvFB8DiHuobkn0GrO7H5yto373+1OnvoetxKaWUCis6OEMppVRY0cCllFIqrGjgUkopFVY0cCmllAorGriUUkqFFQ1cSimlwooGLqWUUmFFA5dSSqmwooFLKaVUWPlfWQcZ4wVyJSYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "opts = [\n",
    "    NoamOpt(512, 1, 4000, None),\n",
    "    NoamOpt(512, 1, 8000, None),\n",
    "    NoamOpt(256, 1, 4000, None),\n",
    "]\n",
    "plt.plot(np.arange(1, 20000),\n",
    "         [[opt.rate(i) for opt in opts] for i in range(1, 20000)])\n",
    "plt.legend([\"512:4000\", \"512:8000\", \"256:4000\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 正则化\n",
    "标签平滑`Label Smoothing`：$\\epsilon_{ls}=0.1$，会降低 perplexity，因为模型将更不确定，但增加精度和`BLEU`分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T13:18:43.129308Z",
     "start_time": "2020-05-03T13:18:43.121664Z"
    }
   },
   "outputs": [],
   "source": [
    "class LabelSmoothing(nn.Module):\n",
    "    def __init__(self, size, padding_idx, smoothing=0.0):\n",
    "        super(LabelSmoothing, self).__init__()\n",
    "        self.criterion = nn.KLDivLoss(size_average=False)\n",
    "        self.padding_idx = padding_idx\n",
    "        self.confidence = 1.0 - smoothing\n",
    "        self.smoothing = smoothing\n",
    "        self.size = size\n",
    "        self.true_dist = None\n",
    "\n",
    "    def forward(self, x, target):\n",
    "        assert x.size(1) == self.size\n",
    "        true_dist = x.data.clone()\n",
    "\n",
    "        true_dist.fill_(self.smoothing / (self.size - 2))\n",
    "        \n",
    "        true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n",
    "        \n",
    "        true_dist[:, self.padding_idx] = 0\n",
    "        mask = torch.nonzero(target.data == self.padding_idx)\n",
    "        if mask.dim() > 0:\n",
    "            true_dist.index_fill_(0, mask.squeeze(), 0.0)\n",
    "        self.true_dist = true_dist\n",
    "        return self.criterion(x, Variable(true_dist, requires_grad=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`nn.KLDivLoss`:\n",
    "输入为`log`概率分布，目标为概率分布；$l(x,y) = L = \\{ l_1,\\dots,l_N \\}, \\quad l_n = y_n \\cdot \\left( \\log y_n - x_n \\right)$   \n",
    "指定`reduction`参数时：\n",
    "$$    \\ell(x, y) = \\begin{cases}\n",
    "    \\operatorname{mean}(L), & \\text{if reduction} = \\text{'mean';} \\\\\n",
    "    \\operatorname{sum}(L),  & \\text{if reduction} = \\text{'sum'.}\n",
    "\\end{cases}$$\n",
    "\n",
    "`Tensor.scatter_(dim, index, src)`：\n",
    "```\n",
    "x = tensor([[0.1333, 0.1333, 0.1333, 0.1333, 0.1333],\n",
    "            [0.1333, 0.1333, 0.1333, 0.1333, 0.1333],\n",
    "            [0.1333, 0.1333, 0.1333, 0.1333, 0.1333]])       \n",
    "index = tensor([[2],\n",
    "                [1],\n",
    "                [0]])\n",
    "x.scatter(1, index, 0.6) -->\n",
    "tensor([[0.1333, 0.1333, 0.6, 0.1333, 0.1333],\n",
    "        [0.1333, 0.6, 0.1333, 0.1333, 0.1333],\n",
    "        [0.6, 0.1333, 0.1333, 0.1333, 0.1333]]) \n",
    "    \n",
    "```\n",
    "例如上述的五分类中，目标序列 `[2，1，0]` 表示类别 2，1，0。将明确的类别转换成概率分布，使概率分布更均匀些，然后与预测概率分布求损失\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:11:50.319664Z",
     "start_time": "2020-05-03T14:11:50.243231Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.1333, 0.6000, 0.1333, 0.1333],\n",
       "        [0.0000, 0.6000, 0.1333, 0.1333, 0.1333],\n",
       "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000]])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAD4CAYAAAAtrdtxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAS5ElEQVR4nO3df/BldV3H8ecLFydqc5ddXIqY2lHHEWMbTW0EVBZ1K1R+ylRMJWnZqOBYoTj5YyBLsqAxmZwaTQfFxixjNXJNEFnDkhESI1jcUBS1EIXdBRZBd9l3f9zzbb99ud9fn73n3u+yz8fMnfO9n3M+n+977yz74nM+59yTqkKSpBYHTboASdL+yxCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ16zVEkhye5ANJ7k7yQJJrkxy7wL6XJqkhr+v6rFmStHDL+ho4yQ8BVwPLgdcC9wC/A1yd5NiqunEBw+wENsxou3+khUqSmvUWIsArgJ8GnlFVXwRI8lngVuBC4MQFjPFwVTnzkKQlqs/TWacB/zkVIABV9X3gw8CGJD/a4++WJI1BnzORo4FrhrTfBDwGOAr4wjxjLE9yF3AY8C3go8D5VbVz2MFJdswz3gqggPvmOU6StNfjgD1V9YjM6DNEVgPbhrRvm7Z/Lv8BfAm4mUHobGCwtvLcJMdV1a7GurKMg1c09n1UqWWPmXQJS8byH/7BpEtYMnZ+77GTLkFLzMO7H4JZzlwtKESSrGf4rGKYx1fV3d3Pc32745zf/FhV75zR9KkkW4H3AL8MfGhIn5VzjZlkxzIOXrE+p8x12AHjBxueNekSloxr3v/eSZewZJzwildOugQtMf961fk8vPuhoWdwFjoT+TLw8gUeO3X11D0Mn22s6rbDZinz+RDwV8AxDAkRSdJ4LShEqurbwKWLHPsWBusiM60DHmYQTIuVbrunoa8kacT6vDprI7AuydOmGpI8FjgT+HRVtSxu/xqDmr3sV5KWgD4X1t8HnA1cnuT3GZy+eh1wBPBL0w9M8nWAqlrbvf8p4DIGlwN/lcHC+guBc4DPAx/psW5J0gL1FiJV9VCS5wMXAX8J/BDwRWBDVf37PN3vA+4G3ggczuA01u3AO4B3VNXuvuqWJC1cnzORqbWUX1/AcWtnvN8OnN5TWZKkEfFbfCVJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNeg2RJMuTXJLkziQPJrkhyckL7PvEJB9Lcm+S+5NsSvLUPuuVJC1O3zORjcCvAm8BXgxsATYmedFcnZKsAa4F1gJnAWcCq4DPJjmyz4IlSQu3rK+Bu6B4IXB6VW3s2q4BngD8GbBpju6vBw4FnllV/9P1/TzwNeDNwKv7qluStHB9zkROA+4FPj7VUFUFfAB4yjynpk4DrpoKkK7vPcAVwOn9lCtJWqw+Q+RoYEtV7ZnRftO0/Y+Q5BDgicDNQ3bfBKzpTncN67tjrhewou2PIkkaps8QWQ1sG9K+bdr+YQ4F0thXkjRGva2JdKpxX1Pfqlo514DORiRptPqcidzD8BnDqm47bKYBsJ1BSLT0lSSNUZ8hcgtwVJKZv2Ndtx225kFVPQjczvA1k3XAd6vqOyOrUpLUrM8Q2QisBE6a0f4yYGtVbZmn74YkPzbVkGRVN9bloy5UktSmzxDZBFwDvC/JK5KckORS4DnAG6YOSrI5ycw1josZXB68KckpSV4MfALYDVzYY82SpEXoLUS6e0JOBf6WwT/8nwR+hsHNh1fM0/cu4LnAN4HLgI8AO4DnVdU3+qpZkrQ4vV6dVVX3Aed0r9mOWT9L+23AKf1UJkkaBb/FV5LUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSs15DJMnyJJckuTPJg0luSHLyAvpdkKSGvL7dZ72SpMVZ1vP4G4GfBc4Dvgb8BrAxyUlVtWkB/TcAO6e9/8HIK5QkNestRJK8CHghcHpVbezargGeAPwZsJAQuaGqdvRVoyRp3/R5Ous04F7g41MNVVXAB4CnJHlqj79bkjQGfYbI0cCWqtozo/2mafvnc2uSh7s1lfcmWTPXwUl2zPUCVjT8OSRJs+hzTWQ18F9D2rdN2z+brwJvAm5ksA5yHIN1lRckeUZVbR9loQeqa97/3kmXsGSc8IpXTroEab/U98J6teyrqstmNH0myXXAlcDZwB/N0m/lXMU4G5Gk0erzdNY9DJ9trOq224bsm1VVXQXcCRyzj3VJkkakzxC5BTgqyczfsa7b3tww5kHAzDUWSdKE9BkiG4GVwEkz2l8GbK2qLYsZLMnPA4cD142mPEnSvupzTWQTcA3wviSrGdxseBbwHOCUqYOSbAaOr6pMa7sR+CCwFdgFHAu8HvgK8O4ea5YkLUJvIVJVleRU4MLutRLYwuDmwyvm6f5l4DXAEcDBwDeBvwb+0JsPJWnp6PXqrKq6Dzine812zPohbWf2WJYkaUT8Fl9JUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSs15DJMmRSd6V5HNJdiapJOsX0f8ZSa5O8kCS7Un+NslP9FiyJGkR+p6JPAk4E9gJXL2YjkmOAjYDAc4AXgk8HdicZPloy5QktVjW8/j/UlVrAJKcCpy8iL5/ANwPnFRVD3Rj3AzcApwN/MmIa5UkLVKvM5Gq2tPSL8nBwEuAj04FSDfel4HrgJeOpkJJ0r7oeybS6gnAIcDNQ/bdBJw1rFOSHfOMu2If65IkTbNUr85a3W23Ddm3DTgkySFjrEeSNMRSnYlMqcXsq6qVcw3WzVScjUjSiCzVmcg93Xb1kH2rgAer6qEx1iNJGmKphsjtwIPA0UP2rWP4WokkacyWZIhU1S7gE8BLk/zwVHuSJwPHAJdPqjZJ0l69r4kkOaP78Vnd9vgkhwEPVNUnu2O+DlBVa6d1PR/4AvCPSS4GfgR4O/B14N191y1Jmt84Ftb/fsb7C7rtHcDa2TpV1ZYkJzC4qfAfgF3AlcC5VXX/6MuUJC1W7yFSVVnAMWtnab8eeP6oa5IkjcaSXBORJO0fDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ16zVEkhyZ5F1JPpdkZ5JKsn6BfS/tjp/5uq7PmiVJC7es5/GfBJwJfBG4Gjh5kf13AhtmtN0/grokSSPQd4j8S1WtAUhyKosPkYerypmHJC1RvZ7Oqqo9fY4vSZqsvmci+2p5kruAw4BvAR8Fzq+qncMOTrJjnvFWjLg+STqgLeUQ+Q/gS8DNwGMYrI28FnhukuOqatcki3s0+IUjnjbpEpaMx3L9pEuQlqzUw7PuW7IhUlXvnNH0qSRbgfcAvwx8aEiflXON2c1UnI1I0ojsb/eJfAjYAxwz6UIkSftfiKTbumAvSUvA/hYiv8agZi/7laQloPc1kSRndD8+q9sen+Qw4IGq+mR3zNcBqmpt9/6ngMuADwNfZbCw/kLgHODzwEf6rluSNL9xLKz//Yz3F3TbO4C1s/S5D7gbeCNwOIPTWLcD7wDeUVW7R16lJGnReg+RqsoCjlk74/124PS+apIkjcb+tiYiSVpCDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc16C5EkL0hyaZKtSb6X5FtJLk+yboH9n5jkY0nuTXJ/kk1JntpXvZKkxetzJvIq4CeBdwInAr/Xvb8+ybPn6phkDXAtsBY4CzgTWAV8NsmRPdYsSVqEZT2OfXZVfWd6Q5Irga8BbwBeOkff1wOHAs+sqv/p+n6+6/tm4NW9VCxJWpTeZiIzA6Rr2wHcBsw3mzgNuGoqQLq+9wBXAKePsk5JUruxLqwneTxwNHDzHMccAjxxlmNuAtZ0p7uG9d0x1wtYMYI/hiSpM7YQSRLgPd3vvHiOQw8FAmwbsm+qbfVoq5MktehzTWSmi4BTgZdX1a0LOL4Wu6+qVs41oLMRSRqtscxEkrwdOBd4XVVdOs/h2xmExLDZxqpuO2yWIkkas95DJMnbgDcB51XVJfMdX1UPArczWDuZaR3w3WGL9pKk8es1RJKcD7wVeGtVXbSIrhuBDUl+bNpYq4CTgMtHW6UkqVWfd6yfC1wA/BPw6STPnvZ6+rTjNieZucZxMXAvsCnJKUleDHwC2A1c2FfNkqTF6XNh/aRu+5LuNd0dDO5GH6qq7kryXAZhchmDsLsWeF5VfWP0pUqSWvQWIlW1fl+Oq6rbgFNGWJIkacT8Fl9JUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc0MEUlSM0NEktTMEJEkNTNEJEnNDBFJUjNDRJLUzBCRJDUzRCRJzQwRSVIzQ0SS1MwQkSQ1M0QkSc16C5EkL0hyaZKtSb6X5FtJLk+ybgF9L0hSQ17f7qteSdLiLetx7FcBq4F3ArcChwPnAdcnWV9V1y1gjA3AzmnvfzDyKiVJzfoMkbOr6jvTG5JcCXwNeAPw0gWMcUNV7eijOEnSvuvtdNbMAOnadgC3AUf29XslSePT50zkEZI8Hjga+PACu9yaZA3wHeCfgDcPC6dp4883a1mxm11sro8v8NdLknazC+Bxw/alqsZSRJIAlwMnAk+vqlvnOPbXGcxWbmSwDnIcg/WU7wLPqKrts/SbN0SAAu5b9B9gdFZ023snWMNS4Wexl5/FXn4Wey2Vz+JxwJ6qesTEY5whcjFwLvDyqrq0of8G4ErgrVX1RyMub2ymgq6qVk66lknzs9jLz2IvP4u99ofPYiz3iSR5O4MAeV1LgABU1VXAncAxIyxNkrQPeg+RJG8D3gScV1WX7ONwBwF79r0qSdIo9BoiSc4H3srgFNRF+zjWzzO412Qh95dIksagt6uzkpwLXMDgqqpPJ3n2tN3fr6obu+M2A8dXVab1vRH4ILAV2AUcC7we+Arw7r5qliQtTp+X+J7UbV/Svaa7A1g7R98vA68BjgAOBr4J/DXwh958KElLx9iuztLA/nC1xbj4WezlZ7GXn8Ve+8Nn4bf4SpKaORORJDVzJiJJamaISJKaGSKSpGaGiCSpmSEyJkmWJ7kkyZ1JHkxyQ5KTJ13XJCQ5Msm7knwuyc7u0cfrJ13XuO3LI6QfbZIcm+RTSf47yUNJvpvkM0lOnHRtS8G0R4Z/adK1zGSIjM9G4FeBtwAvBrYAG5O8aKJVTcaTgDMZPPr46gnXMkmvAn6SwSOkTwR+r3t//YxveDgQHMrgGyrOBX4R+G3g+8CmJL8yycImLclPA28E7pp0LcN4ie8YdEHxCeD0qtrYtQW4FlhdVUdNsr5xS3JQVe3pfj6VQcCeUFWbJ1rYmCVZM+QR0isZPEL6M1W1kEdIP2olWcbgs7itqp4/6XomIclBwL8B1wPrgJVV9bTJVvX/ORMZj9MYPFTm/x6pWIP0/gDwlCRPnVRhkzAVIAc6HyE9t6razeC/m12TrmWCfpfB34U3T7qQ2Rgi43E0sGXIP543TdsvTX+E9M2TrmUSkhyUZFmSI5L8AfBkBqf7DjhJngC8DTinqib5NNY5jfUZ6wew1cB/DWnfNm2/DnDdKc73MPifu4snXM6k/B0wdRrvPuCXquqfJ1jPRHR/F94LfKqqPjbpeubiTGR85lp8cmFKABcBpwKvqqpbJ13MhJwH/BxwMrAJ+LskZ062pIl4JfBM4LWTLmQ+zkTG4x6GzzZWddttQ/bpADKKR0g/GlTV7cDt3dsrklwBvDvJRw6UtbQkhwF/Cvwx8EB3sQUM/r1+TPf+oap6aFI1TudMZDxuAY7qrrSYbup+gAPy/LcGRvwI6UebLzC4/Pfxky5kjI4EVjAIke3TXscxWC/bzuCBf0uCM5Hx2Aj8JoMHdX18WvvLgK1VtWUiVWniRvkI6Uebbl1gPbCDwWz+QPEV4IQh7X8OLAd+C/jGWCuagyEyHpuAa4D3JVnN4Nr3s4DnAKdMsrBJSXJG9+Ozuu3x3TT+gar65ITKGquFPkL6QJDkbxg88fTfgbuBH2fw38jzgdd2l/seEKpqJ7B5Zvu0B1Q9Yt8kebPhmCR5HHAhcAawksEd629b6lde9CXJbH/x7qiqteOsZVKSbAaOn2X3AfM5ACQ5h8E3OjyZwamce4EbgL+oqismWdtS0f19WXI3GxoikqRmLqxLkpoZIpKkZoaIJKmZISJJamaISJKaGSKSpGaGiCSpmSEiSWr2v7632w881/tXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "crit = LabelSmoothing(size=5, padding_idx=0, smoothing=0.4)\n",
    "predict = torch.FloatTensor([\n",
    "    [0, 0.2, 0.7, 0.1, 0],\n",
    "    [0, 0.2, 0.7, 0.1, 0],\n",
    "    [0, 0.2, 0.7, 0.1, 0],\n",
    "])\n",
    "v = crit(Variable(predict.log()), Variable(torch.LongTensor([2, 1, 0])))\n",
    "plt.imshow(crit.true_dist)\n",
    "crit.true_dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:14:12.351226Z",
     "start_time": "2020-05-03T14:14:12.275640Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7ffb16742c90>]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEDCAYAAADeP8iwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5RcZZ3u8e+vqvqW7qSbbiCRa0AdE0KEwDiigwoYFKMBgrejKHhfzgyzZs7R4xmP4oXlQQ/KrDPg8jo4oMNRx0sTOQY8eiCKF9QISEhLuIMhAXLrTjp9ra7f+WPv3b277n2tdO3ns1at6t61d/X7ViXvU+9+33eXuTsiIiJxqVoXQEREDj8KBxERKaBwEBGRAgoHEREpoHAQEZECmVoXYDrMLEsQbAdqXRYRkQVkCZBz94ptvy3EqaxmlgOsvb291kUREVkw+vr6ANzdK541WpA9B+BAe3t7e29vb63LISKyYHR0dNDX11fVGReNOYiISAGFg4iIFFA4iIhIAYWDiIgUUDiIiEgBhYOIiBRYqFNZp6373h0MjuQ4d8VRPK+9pdbFERE5LCUuHP7bD7Yyks3xb+9+icJBRKSExJ1WamlIAzA8OlbjkoiIHL4SFw7NDUGVBxUOIiIlJS4cop7D0GiuxiURETl8JS4cmsNwGBxRz0FEpJTEhsNQVuEgIlJK4sJh/LSSeg4iIiUlLhyiAemhrMYcRERKSVw4tDRqzEFEpJLEhUNzJgwHTWUVESkpeeHQGE1lVTiIiJSSvHDIKBxERCpJXDi0NIYD0loEJyJSUuLCQWMOIiKVJS4cWjTmICJSUeLCYfzyGQoHEZGSEhsOwxpzEBEpKXHh0KKeg4hIRYkLh/Hvc9AKaRGRkhIXDi2xq7K6e41LIyJyeEpcODSF4eAOw7r4nohIUYkLh6jnABqUFhEpJXHhEI05gAalRURKSVw4RIvgQAvhRERKSVw4RJfPAPUcRERKSVw4qOcgIlJZ4sKhKaMxBxGRSqoKBzNrM7PrzGyXmQ2a2RYzu7DKY99oZr82s/3h7Tdm9paZFXv6zGzie6QVDiIiRVXbc+gGLgU+Drwe6AG6zWxduYPM7HLg+8BO4O3h7Wngu2b2nukWeqbGF8JpKquISFGZSjuEAbAWuMTdu8NtdwInA9cCm8oc/m7gSeAt7p4Lj/0J8BhwGfCNGZV+moKL743qEhoiIiVU03PYAPQBG6MNHlx34iZghZmdUubYUaA/Cobw2BzQDwxPq8SzIH4JDRERKVSx5wCcCvTEG/jQ/fHHSxz7ReCHZvYx4KuAAR8AXgR8uNQfNLPeCmVqr1TocqJLaKjnICJSXDXh0AU8VGT7vtjjRbn7xnDg+t+Bz4SbDwFvdvfbp1LQ2dQSDkjr2koiIsVVEw4A5S5fWvIxMzsf+N/At4EfAGmCge1vm9mb3P3HRZ/QvaNcYcKexbR7D83qOYiIlFVNOOyleO+gM7zfV+QxzMwIxiXucPcPxh663cyOA64HiobDXJuYraRwEBEpppoB6W3ASjPL33d1eP9AieOWAs8DthR5bAtwkpk1V1XKWdbcqG+DExEpp5pw6AY6gPV52y8Dtrt7qcHo/cAQ8FdFHjsL2OvuQ9UWdDZF11dSOIiIFFfNaaVNwJ3ADWbWBTwOXA6cDVwU7WRmm4FXubsBuPuwmX0F+Ecz+1eCxXBpglA5m2BBXU20NIYD0loEJyJSVMVwcHc3s4uBq8NbB8HU1Uvc/dYKh38YeJBg+uqbgBzBzKd3AjfPoNwzop6DiEh5Vc1WcvcDwBXhrdQ+5xTZNkawvuGr0yzfnIiuzKoBaRGR4hJ3VVaITWVVOIiIFJXocNCF90REiktoOOiS3SIi5SQyHLQITkSkvESGg8YcRETKS2Q4qOcgIlJeIsMhPiCdy5W7pqCISDIlNBwmqq3LdouIFEpkOESL4ECnlkREiklkOESXzwANSouIFJPIcFDPQUSkvESGg3oOIiLlJTMcGieqrUtoiIgUSmQ4NKZTmAU/67SSiEihRIaDmWkhnIhIGYkMB9AlNEREyklsOEQ9h8ERhYOISL7EhsP4Zbu1QlpEpECCwyEcc1DPQUSkQGLDQQPSIiKlJTYcNCAtIlJa4sNBi+BERAolOByCqqvnICJSKLHhEI05DCscREQKJDYcNOYgIlJaYsMhumy3wkFEpFBiw6FZU1lFREpKcDhEA9KarSQikq+qcDCzNjO7zsx2mdmgmW0xswurPNbM7ANm9gczGzCzXjO728xePrOiz4wGpEVESstUuV83cAbwEeBx4F1At5mtd/dNFY79V+CNwDXAr4FW4MzwvmY0IC0iUlrFcDCzdcBa4BJ37w633QmcDFwLlAwHM3sjQZCc7e6/iT304xmUeVbo8hkiIqVVc1ppA9AHbIw2uLsDNwErzOyUMsf+PfCLvGA4LIyPOejCeyIiBaoJh1OBHnfPH7m9P/Z4ATNrAM4CtprZ1Wb2rJllzWybmV0+/SLPjvHZSrpkt4hIgWrGHLqAh4ps3xd7vNRxTcDlwA7gCqAXeC9wo5k1uvvXix1oZr0VytReqdCVROEwks0xlnPSKZvpU4qI1I1qB6R9Go9FvZJmYJ27PwlgZj8jGK/4BFA0HOZDNOYAwbhDa1O1L4WISP2r5rTSXor3DjrD+31FHgPYTxAcD0bBAOPjFbcDx5nZ0cUOdPeOcjeCMZAZac4LBxERmVBNOGwDVppZ/r6rw/sHih3k7oPAIyWeMzqHU7MT/vGeg6aziohMVk04dAMdwPq87ZcB2929p8yxPyQIluXRBjMz4HXAY+6+Z0qlnUXNjRNV13c6iIhMVs2J9k3AncANZtZFsAjucuBs4KJoJzPbDLzK3eMju58HLgVuN7NPMzEgfSbwn2ajAtOl00oiIqVVDAd3dzO7GLg6vHUAPQSL4m6tcOxeM3sFQUh8CWgBtgIb3P2WmRZ+JvIHpEVEZEJVU3Tc/QDBVNQryuxzTontTwBvnkbZ5lRDOkU6ZYzlXGMOIiJ5EntVVohfQkNjDiIicYkOB32PtIhIcQkPh7DnoOsriYhMkuhwaAtXRR8YGq1xSUREDi+JDoeutkYA9h4aqXFJREQOL4kOh87WJgD29SscRETiEh0OXa3qOYiIFJPocOgMw2HfoeEal0RE5PCicAD2qecgIjJJosNBp5VERIpLdDhEPYeDQ1lG9HWhIiLjEh0O0VRWgP0D6j2IiEQSHQ7RVFaAvZrOKiIyLtHh0NHSQCr89gn1HEREJiQ6HFIp44hFGpQWEcmX6HAAOCKaztqvtQ4iIpHEh4PWOoiIFEp8OGitg4hIocSHg3oOIiKFEh8O6jmIiBRKfDio5yAiUkjh0BZ+p4PCQURkXOLDITqttH9ghLGc17g0IiKHh8SHQ3RayR16tUpaRARQOIz3HECnlkREIokPhyNi4aAZSyIigcSHQ0M6xZLmDKCeg4hIJPHhANAVzlhSz0FEJKBwILbWQd/pICICVBkOZtZmZteZ2S4zGzSzLWZ24VT+kAXuMDM3s/81veLOjYmFcLoyq4gIVN9z6AYuBT4OvB7oAbrNbN0U/tb7gRVTK9780CU0REQmy1TaIQyAtcAl7t4dbrsTOBm4FthUxXMcC1wDvBf4/kwKPBd0CQ0Rkcmq6TlsAPqAjdEGd3fgJmCFmZ1SxXN8GfiFu/9gWqWcYwoHEZHJKvYcgFOBHnfP5W2/P/54qYPN7G3AuUA1IRId01thl/Zqn6saXW06rSQiEldNz6EL2Fdk+77Y40WZ2ZHAvwAfc/c/T7148yP6Hun9h0YIOkUiIslWTc8BoFyLWe6x64DHgS9WXSLA3TvKPR72LGat99DVGqxzyOacA4NZ2hc1zNZTi4gsSNWEw16K9w46w/tivQrM7HzgrcB5wBIziz/cZGYdQL+7Z6sv7tzobItfQmNY4SAiiVfNaaVtwEozy993dXj/QInjVoXPvxnYH7sBfDD8ee1UCjtXdPE9EZHJqgmHbqADWJ+3/TJgu7uXGoz+PsFAdP4N4Afhz7+baoHnQnNDmkWNaUCD0iIiUN1ppU3AncANZtZFMIZwOXA2cFG0k5ltBl7l7gbg7juAHflPFp5e2uHum2dY9lnV2drIwMgg+xUOIiKVew7hmoaLge8AVwO3AS8mWBR369wWb/5olbSIyISqZiu5+wHgivBWap9zqnwuq7zX/FvW3swfd/SxY/9grYsiIlJzuipraPmRrQA8ufdQjUsiIlJ7CofQ8q4gHJ7Yo3AQEVE4hKJw2Nk3xNDoWI1LIyJSWwqH0PIjF43//NS+gRqWRESk9hQOoaWLm2luCF4OnVoSkaRTOIRSKePEznDcQYPSIpJwCoeY6NTSE3t1WklEkk3hEKMZSyIiAYVDzMRaB/UcRCTZFA4xJ3YFp5V29g1qOquIJJrCIeaksOfgDn/WdFYRSTCFQ8zSxc00ZYKX5HGNO4hIgikcYlIpmxiU1nRWEUkwhUMeTWcVEVE4FNB0VhERhUMBTWcVEVE4FNB0VhERhUMBTWcVEVE4FNB0VhERhUOB+HRWjTuISFIpHIqIxh0e3d1f45KIiNSGwqGIVce0A3Dfn3trXBIRkdpQOBSx5oQOAB569iD9w9kal0ZEZP4pHIo4/YQOzCDncL96DyKSQAqHIpY0N/DCo9sAuOep/TUujYjI/FM4lHDGCUcAcM9T6jmISPIoHEqIwuHep/bj7jUujYjI/FI4lHDGicGg9P6BUS2GE5HEqSoczKzNzK4zs11mNmhmW8zswiqOe5+Z/cjMngyPezh8nqNmXvS5dfKRbSxpzgA6tSQiyVNtz6EbuBT4OPB6oAfoNrN1FY77NHAA+ChwAfDPwFuA35tZx7RKPE9SKeP08XEHDUqLSLJkKu0QBsBa4BJ37w633QmcDFwLbCpz+Bp3fy72+8/NrAfYDLwTuH6a5Z4XZ5zQwS8e2s296jmISMJU03PYAPQBG6MNHozQ3gSsMLNTSh2YFwyR34f3x02hnDURDUpvf+aAFsOJSKJUEw6nAj3unsvbfn/s8ak4L7x/YIrHzTsthhORpKomHLqAfUW274s9XhUz6wSuAx4G/qPMfr3lbkB7tX9zJrQYTkSSqtoB6XIT/ataBGBmi4BbgE7gTe4+XOXfrqk1xwenln77eLF8FBGpT9WEw16K9w46w/uKraaZtQA/AtYA69z9/nL7u3tHuRvBGMi8eOVfBLNu735sL32Do/P1Z0VEaqqacNgGrDSz/H1Xh/dlxw7MrJlgMPtlwBvc/ddTLmUNnfOio2jKpBgdc+548NlaF0dEZF5UEw7dQAewPm/7ZcB2d+8pdaCZNRGcSnoFcJG7/3y6Ba2V1qbMeO/h9geeqXFpRETmR8V1DgTrGO4EbjCzLuBx4HLgbOCiaCcz2wy8yt0tduz3gdcCVwH9ZnZW7LHd7v7ozIo/Py5YtYyf9jzLzx/azcBIlkWN1bxsIiILV8WeQ7im4WLgO8DVwG3AiwkWxd1a4fA3hPefAH6Td7tymmWed2tXLiWTMoZGc2zevrvWxRERmXNVzVZy9wPufoW7L3P3Znc/w91vydvnnLxeA+5uZW7vmsV6zKn2RQ28/AVHAjq1JCLJoKuyVumCVcsAuOPB5xjOjtW4NCIic0vhUKXXrFpKyqB/OMuvHtlT6+KIiMwphUOVjmxr4iXLg6Udt23VqSURqW8KhylYt/p5APx46y4tiBORuqZwmIKL1xxLa2OagZExvv27p2pdHBGROaNwmIL2lgbe+pITALjxV08wks2/UK2ISH1QOEzRu/96OSmDZw4MsWnrrloXR0RkTigcpuj4zkW87tRg7OHrdz1GsEZQRKS+KBym4X2vOAmAbTsPcPdjupS3iNQfhcM0rDnhCP7yxOB7Hr72iwVxeSgRkSlROEzT+15xMgB3bt/NXQ/reksiUl8UDtP02lVL+atwUdwnN27TJTVEpK4oHKbJzLjq4lWkU8Zjew5xwy8fr3WRRERmjcJhBlYsW8K7Xr4cgOv/3yPs7B2sbYFERGaJwmGG/nHtCzlqcRODo2NcdWuPpraKSF1QOMzQ4uYGPv76lQDcvu0Zbv6tLqshIgufwmEWXHjaMaxbHXzfw6dv3cYfntxf4xKJiMyMwmEWmBnXvOk0XnB0G6Njzt/e/Ad2HxyudbFERKZN4TBL2poyfOUdZ9LWlOHZA8P83c33MDSq6a0isjApHGbRC45u4wtvPg2A3z2xj/d/c4sCQkQWJIXDLLvg1GVc+YZTALjr4T2876YtDI4oIERkYVE4zIH3nn0Sn1wfBMQvH9nDe278vb45TkRmxehYbl7OSNhCnJdvZr3t7e3tvb29tS5KWd/8zRN8YuM2AE7sWsRX3nEmK5+3pLaFEpF54+4MjIxxaDjLoeh+OMuhkSyHhvO2j4SPhdsHRsboH84yEO07kmVgeIyRsRyXnHEs//yW06dcno6ODvr6+vrcvaPSvplp1ViqctnLltPamOGj3Vt5cu8AG770Kz57yWo2rDmu1kUTkSLcneFsjv7hLP1DWfrDxjxqqKPfJ+6LN/jjx42OMRefvweG1XMoaqH0HCIPPN3HB//9D+zYH1xe44JVy/jUhatY1t5c45KJLHzuztBojoPDoxwaHhtv1KMG+mDUkOdtn/RzFAQjY4zl5r5NbMqkaGvK0NqUYVFjetLPrU0Z2poytITbFzWmaW0MH28Kfj56cRPLj2yd8t+dSs9B4TBPegdG+C//8UfuePA5IJj6+qHX/AXvOOtEGtIa+pHkGcs5h0YmGuaDQ6McjBrw8W0TDfnBeAMfa+j7h7Nz3qDHG/Og8Q4b7qYMbY0T21rH90mzqDEzfkz8sUUNaTI1+j+vcDhMuTsb79vJZ37cw57+EQBO6FzEFee9gA1rjlVIyIKQixr1sPEObqPjv/cPBQ35waHRSY34wVgI9A8Fn9LnUmv0Kbw5bKQbYz+HjfXipngDHjzeGt/WGOxbq8Z8tikcDnN9A6N87vYH+e7vnyL6wHNC5yLe9fLlvPGM42hf1FDbAkrdGsnmJjXkB8JP60GjPvHJ/UBegx816AeHsvSPZOfkPDpAQ9pY3NxAa1OaxU0N44354lijvTjWiJf6ubUxQzplc1PIBUzhsEA8truf6+94hI33PT0eEk2ZFOtPO4YNa47lpSd11s0nFpm5odGx8YY6/ok9auTjDfmBwSwHhyca9KixH87m5qRsDWkLG/EGFo836A1ho55mcXPDeCMfPB4EwJLY9tamDM0N6TkpnwRmPRzMrA24Gngz0AFsA65y9x9VcezzgWuBcwnWVdwFfNjdeyr+4dLPWRfhEHl0dz/f+OXj3HLv05O62p2tjbx21VLOW7GUs07uZHGzehQLVbGGPfjUPjreePfHHx8OG/jY/iNjc9OwT260g0a9rTnDkuawgQ8/kec38PHfmzIpzPRJ/XA3F+HwU+AM4CPA48C7gEuB9e6+qcxxRwP3Ac8BnwKywMeB5wNr3H1HxT9e/HnrKhwi/cNZNt73NN/bsoP7/jy5bpmUcfrxHbz05E7WHH8Ep5/QwZFtTTUqaXK4O4dGxsYb7mKnWyY+xU9u3CdO18xNw54yJn1CX5LfqE+6z+T9HjTsbU06/ZIksxoOZrYO+DFwibt3h9uMoAfQ5e4ryxx7DfD3wPPdfWe4rYsgYG5297+psk75z1uX4RD3dO8gt23dxU+2PcO9T/WSLTIb49iOFl60bDErli3mhUvbWN7VyvKuVo5obaxBiQ8vI+Fc9fic82JTGsdnxQxPnHePth8M95mLM6/FGvZ4A76kpbCBz2/0WxvT+rQuUzLb4fB1gtNJne6ei21/P/A1YFWpU0Rm9jDwJ3e/MG/7zcBad19asTbFn7fuwyGufzjL3Y/u5VeP7uGep3rp2dnH6Fjp921xc4Zj2ls4pqOZZe0tHNXWyJGLm+hqbaJjUQPtLcFtSTjwV4txjVzOGQkvAzA0Gt5ng58HR8YYGh1jYGSMgZEsQ6NjHBoJfh8cyYbbJ1aRTlpZGv5c7vWZqWjQNP8Uy/in96b8T+mFn9rVsEstzPYK6VOBnngwhO6PP55/kJm1EJw++l6R57wfeLuZHe3uzxU5tlKr316x1HWkrSnD2lOWsvaUIEuHRsfYtrOPnl0H2f7MAR7cdZDH9hxi36FgeuzBoSzbhw6y/dmDVT1/UyZFS2OaloY0zQ1pGtMpGjJGYzpFJpUinTLSKcMs+O6KlIE7OMFpF/dgznrOnbGck8052VyO7JgzOpZjNLwfyQa34Wxuzs6fl9MYzlWPzz0fP70SNeDhY4UDqBONu86vSxJUEw5dwENFtu+LPV7MEYDF9it1bEE4SHnNDWnOPLGTM0/snLS9b3CUJ/YcYsf+QXb1DbKzd4hnDwyxu3+YPf3D7O0f4cDQaMFpkuGwwe7l8Lk4YFMmxaIosMIVoi2NaRaFt2jFaLFVpK1N6Ulz16P7xoxmfolUq9prK5Xro1fqv0/52EpdnrBnkajeQzXaWxo47fgOTju+9MuXyzkHh7L0Do6E5+LH6B8eHT+1Mzg6xkg2N/5JP5tzcmFvIOfgBD0FAzAwgp5EOmWkLLhl0kZD2kinUjRmUjSmjYZ09HNw39yQpikz8XNzQ5rmsAfTnEmT0iCpSE1VEw57Kd47iD62FusZAOwnaPync6zMkVTKaF/UoIV2IlJWNf3sbcBKM8vfd3V4/0Cxg9x9EHiMYEwi32pgd7HxBhERqb1qwqGbYOHb+rztlwHbKyxm6wbON7Nl0QYz6wyf64dTLKuIiMyTasJhE3AncIOZvcfMzjWzG4Gzgf8a7WRmm80sfwzhC0AfsMnMLjKz1xOsmcgSrLgWEZHDUMVw8GAhxMXAdwga9NuAFxMsiru1wrHPAq8A/gx8C/gu0Au80t2fmlnRRURkrujCeyIiCTGVRXCa+C0iIgUWas8hB1h7u5Y6iIhUq6+vD4LRgoodg4UaDlmCXs+BKg+JUqRvbkp02FK9kyWp9Ybk1n2q9V4C5Ny94hq3BRkOUxVdq6ma82z1RPVWvZMiqXWfy3przEFERAooHEREpIDCQURECigcRESkgMJBREQKKBxERKSAwkFERAokYp2DiIhMjXoOIiJSQOEgIiIFFA4iIlKgrsPBzNrM7Doz22Vmg2a2xcwurHW5ZouZvdrMbjSz7WY2YGY7zOyHZra6yL7nm9nd4evwnJl91czq5jo0ZvYpM3Mzu6/IY3VVdzM7x8z+r5n1hu97j5l9IG+ft5vZH81sKPx38Tkza65VmWfKzNaY2S1mttPMDoV1/icza8rbb8G+12Z2nJn9i5n90sz6w3/P55TYt6r318yWmtlNZrYnfN3uMrOXV1Ugd6/bG/BTYC/wXuA84JvAGLCu1mWbpfp9D7gD+CDwKuAtwBZgCDgrtt85wGi4/1qC7//eBfwKSNW6HrPwOqwCBoFngPvyHqurugOXE3zN7peBC4BXA38HXBHb5x2AA18CzgX+FjgIfKfW5Z9mnVeE7+994b/x84DPhP+Xv1kv73VY/ueA24GN4Xt4TpH9qnp/gWbgAeAJ4G3Aawi+9nkQWFOxPLV+QebwhV4XvoAbYtsM+CXwp1qXb5bqeHSRbR3AfuAHsW2/A+6N/wcBzg9fn7fWuh4zfA1SwN3A9cDmIuFQN3UHjgcGgI+U2ScdNogb87a/P6zzS2tdj2nU+1Nh2Z+ft/1bYRg01MN7nVfui4uFw1Te3zA0HDgjtq0JeAy4rVJ56vm00gaCa5xvjDZ48OrcBKwws1NqVbDZ4u7PFdnWCzwMHAdgZscCLwG+5e652H4/BZ4G3jg/pZ0z/5mgrh/Lf6AO6/7e8P76MvucBSwj+HcedzNBQ7rQ6gxBuaHwOwv6wsfG6uG9jpe7jKm8vxuAre5+T+xvDAPfBs43s8Xl/lA9h8OpQE+RF/z+2ON1x8yOIqjbA+GmqJ4PFNl9Kwv4dTCzk4GrCE6pFPvip3qr+yuBPwGXhONMY7HzzY3hPkXr7O4DwKMsvDpD0EPYB3zZzE4ysyVmdhHBKbZrw//j9fZelzKV9/fU/P1C9xP0QFaW+0P1HA5dBP+g8u2LPV5XzMyArxG8r18IN0f1LPVaLMjXIazr14GfuPstJXart7ofA7yQoOdwHcF4wzeADwH/Fu5Tb3XG3Z8i+MR8CsEpkT7gFuA6d78y3K3u6l3CVOo5ozaw4lfFLXDlln/X49LwzxOcq3y3u/8p77FS9V2or8P7gb8kaDAqqZe6p4DFwNvc/Tvhts1m1gJ82Mw+Gdu3XuqMmZ0I3Eow4WAD0EswAeOjZpaLBQTUUb0rqLae024D6zkc9lI8GTvD+2KJumCZ2f8g+AT5D+5+Y+yhveF9qddiwb0OZnYkcA3wWeBQbKpiBkiHvw9Rf3XfS9Bz+Ene9tuADwNnMLnOe/P26wQen8sCzpHPEYTiGncfDLdtDjqPfMLMbqD+3utSpvL+zqgNrOfTStuAlWaWX8doDUCxc3ELkpldBfx3glks1+U9vC28L3bOdTUL83U4juCL1T9LMDMruv01QT33E8xwqbe6by2x3cL7HCXqbGaLgOez8OoMsIZg/HAwb/sWgjZsBfX3Xpcylfd3W/5+odUE04AfLPeH6jkcugmmda7P234ZsN3de+a/SLMvPJVwJXClu38+/3F330Hwn+jSeFCa2auBY4EfzldZZ9EjBPO7829/JBiUOxf4Wh3WPSrvurzt0bTt3xNM630GeGfePm8DGlh4dQbYCZwaNoBxLwvvn67D97qUqby/3cBqMzs92hBOXHgb8LMSkzgm1Hpu7xzOGTaCBWJ7gPcQNBg3Eny6Wl/r8s1SHT9E0CjcSjBgF7+tie13HsHCqe8SDGK+k+A/3N1Autb1mMXXYzOF6xzqqu4Ei5h6gX8gWOh1dVi/L8X2uTz8d/FFgoVVfwMcAL5X6/JPs87RnP+7CKZqriVYBDcC/LSe3mvgTeHtf4Z1/mT4++um+v4SLILrIRjEfyvBmo//Q7AI7syKZan1izHHL/SS8AV8huAc9D3AxbUu1yzWb3P4j6TY7Ym8fS8Afhu+DrsJZvocUes6zMHrcV+R7XVTd6CVYCba02Hj+O4lHHoAAAB8SURBVAjwT+StACZYRbsVGA73vQZoqXX5Z1DvtcDPgGeBQwSnTK4EWuvpvZ7C/+eq3l+CNRHRVOABgkXAZ1dTFn2fg4iIFKjnMQcREZkmhYOIiBRQOIiISAGFg4iIFFA4iIhIAYWDiIgUUDiIiEgBhYOIiBRQOIiISIH/Dz2DYXcxi7CnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "crit = LabelSmoothing(5, 0, 0.1)\n",
    "\n",
    "\n",
    "def loss(x):\n",
    "    d = x + 3 * 1\n",
    "    predict = torch.FloatTensor([\n",
    "        [0, x / d, 1 / d, 1 / d, 1 / d],  # 概率分布，x 的值越大，标签 1 的概率越大\n",
    "    ])\n",
    "    #print(predict)\n",
    "    return crit(\n",
    "        Variable(predict.log()),\n",
    "        Variable(torch.LongTensor([1])),  # 真实标签为 1\n",
    "    ).item()\n",
    "\n",
    "\n",
    "plt.plot(np.arange(1, 100), [loss(x) for x in range(1, 100)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:19:32.586591Z",
     "start_time": "2020-05-03T14:19:32.580827Z"
    }
   },
   "outputs": [],
   "source": [
    "# 生成随机数据\n",
    "def data_gen(V, batch, nbatches):\n",
    "    for i in range(nbatches):\n",
    "        data = torch.from_numpy(np.random.randint(1, V, size=(batch, 10)))\n",
    "        data[:, 0] = 1\n",
    "        src = Variable(data, requires_grad=False)\n",
    "        tgt = Variable(data, requires_grad=False)\n",
    "        yield Batch(src, tgt, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:34:24.835306Z",
     "start_time": "2020-05-03T14:34:24.828742Z"
    }
   },
   "outputs": [],
   "source": [
    "class SimpleLossCompute:\n",
    "    def __init__(self, generator, criterion, opt=None):\n",
    "        self.generator = generator  # 模型最后的输出层\n",
    "        self.criterion = criterion\n",
    "        self.opt = opt\n",
    "\n",
    "    def __call__(self, x, y, norm):\n",
    "        x = self.generator(x)\n",
    "        loss = self.criterion(x.contiguous().view(-1, x.size(-1)),\n",
    "                              y.contiguous().view(-1)) / norm\n",
    "        loss.backward()\n",
    "        if self.opt is not None:\n",
    "            self.opt.step()\n",
    "            self.opt.optimizer.zero_grad()\n",
    "        return loss.data.item() * norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:35:58.256149Z",
     "start_time": "2020-05-03T14:35:28.198504Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Step: 1 Loss: 3.041517 Tokens per Sec: 2320.010498\n",
      "Epoch Step: 1 Loss: 2.116642 Tokens per Sec: 3337.003174\n",
      "tensor(2.0963)\n",
      "Epoch Step: 1 Loss: 2.251899 Tokens per Sec: 2198.680176\n",
      "Epoch Step: 1 Loss: 1.861713 Tokens per Sec: 3322.143066\n",
      "tensor(1.9056)\n",
      "Epoch Step: 1 Loss: 2.097054 Tokens per Sec: 2199.775635\n",
      "Epoch Step: 1 Loss: 1.919798 Tokens per Sec: 3104.268799\n",
      "tensor(1.9386)\n",
      "Epoch Step: 1 Loss: 1.955538 Tokens per Sec: 2096.228027\n",
      "Epoch Step: 1 Loss: 1.854445 Tokens per Sec: 3121.148926\n",
      "tensor(1.8574)\n",
      "Epoch Step: 1 Loss: 2.210659 Tokens per Sec: 2088.976074\n",
      "Epoch Step: 1 Loss: 1.806028 Tokens per Sec: 3116.300537\n",
      "tensor(1.7957)\n",
      "Epoch Step: 1 Loss: 2.442501 Tokens per Sec: 2085.300537\n",
      "Epoch Step: 1 Loss: 1.931611 Tokens per Sec: 3112.506348\n",
      "tensor(1.9629)\n",
      "Epoch Step: 1 Loss: 1.962478 Tokens per Sec: 2092.424561\n",
      "Epoch Step: 1 Loss: 1.450812 Tokens per Sec: 3114.582275\n",
      "tensor(1.5110)\n",
      "Epoch Step: 1 Loss: 1.954074 Tokens per Sec: 2087.149170\n",
      "Epoch Step: 1 Loss: 1.519574 Tokens per Sec: 3094.708252\n",
      "tensor(1.4481)\n",
      "Epoch Step: 1 Loss: 1.859929 Tokens per Sec: 2039.030151\n",
      "Epoch Step: 1 Loss: 1.412592 Tokens per Sec: 3088.327881\n",
      "tensor(1.3660)\n",
      "Epoch Step: 1 Loss: 1.943988 Tokens per Sec: 2065.341797\n",
      "Epoch Step: 1 Loss: 1.302344 Tokens per Sec: 2979.704590\n",
      "tensor(1.3880)\n"
     ]
    }
   ],
   "source": [
    "V = 11\n",
    "criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0)\n",
    "model = make_model(V, V, N=2)\n",
    "model_opt = NoamOpt(\n",
    "    model.src_embed[0].d_model, 1, 400,\n",
    "    torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))\n",
    "\n",
    "for epoch in range(10):\n",
    "    model.train()\n",
    "    run_epoch(data_gen(V, 30, 20), model,\n",
    "              SimpleLossCompute(model.generator, criterion, model_opt))\n",
    "    model.eval()\n",
    "    print(\n",
    "        run_epoch(data_gen(V, 30, 5), model,\n",
    "                  SimpleLossCompute(model.generator, criterion, None)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 解码算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-03T14:48:37.582292Z",
     "start_time": "2020-05-03T14:48:37.536568Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1,  2,  4,  3,  7,  9, 10,  8,  9,  8]])\n"
     ]
    }
   ],
   "source": [
    "def greedy_decode(model, src, src_mask, max_len, start_symbol):\n",
    "    memory = model.encode(src, src_mask)\n",
    "    ys = torch.ones(1, 1).fill_(start_symbol).type_as(src.data)\n",
    "    for i in range(max_len - 1):\n",
    "        out = model.decode(\n",
    "            memory, src_mask, Variable(ys),\n",
    "            Variable(subsequent_mask(ys.size(1)).type_as(src.data)))\n",
    "        prob = model.generator(out[:, -1])\n",
    "        _, next_word = torch.max(prob, dim=1)\n",
    "        next_word = next_word.item()\n",
    "        ys = torch.cat(\n",
    "            [ys, torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)\n",
    "    return ys\n",
    "\n",
    "\n",
    "model.eval()\n",
    "src = Variable(torch.LongTensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]))\n",
    "src_mask = Variable(torch.ones(1, 1, 10))\n",
    "print(greedy_decode(model, src, src_mask, max_len=10, start_symbol=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 实战"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-04T08:37:54.090487Z",
     "start_time": "2020-05-04T08:37:54.077145Z"
    }
   },
   "outputs": [],
   "source": [
    "from torchtext import data, datasets\n",
    "\n",
    "if True:\n",
    "    import spacy\n",
    "    spacy_de = spacy.load(\"de_core_news_sm\")\n",
    "    spacy_en = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "    def tokenize_de(text):\n",
    "        return [tok.text for tok in spacy_de.tokenizer(text)]\n",
    "\n",
    "    def tokenize_en(text):\n",
    "        return [tok.text for tok in spacy_en.tokenizer(text)]\n",
    "\n",
    "    BOS_WORD = '<s>'\n",
    "    EOS_WORD = '</s>'\n",
    "    BLANK_WORD = \"<blank>\"\n",
    "    SRC = data.Field(tokenize=tokenize_de,\n",
    "                     pad_token=BLANK_WORD)  # 定义预处理流程，分词、填充、\n",
    "    TGT = data.Field(tokenize=tokenize_en,\n",
    "                     init_token=BOS_WORD,\n",
    "                     eos_token=EOS_WORD,\n",
    "                     pad_token=BLANK_WORD)\n",
    "\n",
    "    MAX_LEN = 100\n",
    "\n",
    "    # 数据集\n",
    "    train, val, test = datasets.IWSLT.splits(\n",
    "        exts=('.de', '.en'),\n",
    "        fields=(SRC, TGT),\n",
    "        filter_pred=lambda x: len(vars(x)['src']) <= MAX_LEN and len(\n",
    "            vars(x)['trg']) <= MAX_LEN)\n",
    "    MIN_FREQ = 2\n",
    "\n",
    "    # 创建词汇表\n",
    "    SRC.build_vocab(train.src, min_freq=MIN_FREQ)\n",
    "    TGT.build_vocab(train.trg, min_freq=MIN_FREQ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据分批对训练速度很重要：需要拆分成均匀的批次，最小的填充\n",
    "\n",
    "class MyIterator(data.Iterator):\n",
    "    def create_batches(self):\n",
    "        if self.train: # 训练模式，数据分批，然后打乱顺序\n",
    "\n",
    "            def pool(d, random_shuffler):\n",
    "                for p in data.batch(d, self.batch_size * 100):\n",
    "                    p_batch = data.batch(sorted(p, key=self.sort_key),\n",
    "                                         self.batch_size, self.batch_size_fn)\n",
    "                    for b in random_shuffler(list(p_batch)):\n",
    "                        yield b\n",
    "\n",
    "            self.batches = pool(self.data(), self.random_shuffler)\n",
    "\n",
    "        else:\n",
    "            self.batches = []\n",
    "            for b in data.batch(self.data(), self.batch_size,\n",
    "                                self.batch_size_fn):\n",
    "                self.batches.append(sorted(b, key=self.sort_key))\n",
    "\n",
    "\n",
    "def rebatch(pad_idx, batch): # batch first --> True\n",
    "    \"Fix order in torchtext to match ours\"\n",
    "    src, trg = batch.src.transpose(0, 1), batch.trg.transpose(0, 1)\n",
    "    return Batch(src, trg, pad_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 并行计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用 multi-gpu 加速训练速度：将单词生成拆分成块，便于并行处理\n",
    "\n",
    "\n",
    "class MultiGPULossCompute:\n",
    "    def __init__(self, generator, criterion, devices, opt=None, chunk_size=5):\n",
    "        self.generator = generator\n",
    "        self.criterion = nn.parallel.replicate(criterion, devices=devices)\n",
    "        self.opt = opt\n",
    "        self.devices = devices\n",
    "        self.chunk_size = chunk_size\n",
    "\n",
    "    def __call__(self, out, targets, normalize):\n",
    "        \n",
    "        total = 0.0\n",
    "        \n",
    "        # 将最终的线性输出层 并行 到多个 gpu中\n",
    "        generator = nn.parallel.replicate(self.generator, devices=devices)\n",
    "        \n",
    "        # 将 transformer 的输出张量 并行 多个 gpu 中\n",
    "        out_scatter = nn.parallel.scatter(out, target_gpus=self.devices)\n",
    "        out_grad = [[] for _ in out_scatter]\n",
    "        \n",
    "        # 将目标 并行 到多个 gpu 中\n",
    "        targets = nn.parallel.scatter(targets, target_gpus=self.devices)\n",
    "\n",
    "        # 将生成拆分成块？？\n",
    "        chunk_size = self.chunk_size\n",
    "        for i in range(0, out_scatter[0].size(1), chunk_size):\n",
    "\n",
    "            # 预测分布\n",
    "            out_column = [[\n",
    "                Variable(o[:, i:i + chunk_size].data,\n",
    "                         requires_grad=self.opt is not None)\n",
    "            ] for o in out_scatter]\n",
    "            gen = nn.parallel.parallel_apply(generator, out_column)\n",
    "\n",
    "            # 计算损失\n",
    "            y = [(g.contiguous().view(-1, g.size(-1)),\n",
    "                  t[:, i:i + chunk_size].contiguous().view(-1))\n",
    "                 for g, t in zip(gen, targets)]\n",
    "            loss = nn.parallel.parallel_apply(self.criterion, y)\n",
    "\n",
    "            # 损失求和并归一化\n",
    "            l = nn.parallel.gather(loss, target_device=self.devices[0])\n",
    "            l = l.sum()[0] / normalize\n",
    "            total += l.data[0]\n",
    "\n",
    "            # 反向传播\n",
    "            if self.opt is not None:\n",
    "                l.backward()\n",
    "                for j, l in enumerate(loss):\n",
    "                    out_grad[j].append(out_column[j][0].grad.data.clone())\n",
    "\n",
    "        # 反向传播整个模型\n",
    "        if self.opt is not None:\n",
    "            out_grad = [Variable(torch.cat(og, dim=1)) for og in out_grad]\n",
    "            o1 = out\n",
    "            o2 = nn.parallel.gather(out_grad, target_device=self.devices[0])\n",
    "            o1.backward(gradient=o2)\n",
    "            self.opt.step()\n",
    "            self.opt.optimizer.zero_grad()\n",
    "        return total * normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-05-05T02:38:54.986125Z",
     "start_time": "2020-05-05T02:38:54.980483Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "devices = [0, 1, 2, 3]\n",
    "if True:\n",
    "    pad_idx = TGT.vocab.stoi[\"<blank>\"]\n",
    "    model = make_model(len(SRC.vocab), len(TGT.vocab), N=6)\n",
    "    model.cuda()\n",
    "    criterion = LabelSmoothing(size=len(TGT.vocab),\n",
    "                               padding_idx=pad_idx,\n",
    "                               smoothing=0.1)\n",
    "    criterion.cuda()\n",
    "    BATCH_SIZE = 12000\n",
    "    train_iter = MyIterator(train,\n",
    "                            batch_size=BATCH_SIZE,\n",
    "                            device=0,\n",
    "                            repeat=False,\n",
    "                            sort_key=lambda x: (len(x.src), len(x.trg)),\n",
    "                            batch_size_fn=batch_size_fn,\n",
    "                            train=True)\n",
    "    valid_iter = MyIterator(val,\n",
    "                            batch_size=BATCH_SIZE,\n",
    "                            device=0,\n",
    "                            repeat=False,\n",
    "                            sort_key=lambda x: (len(x.src), len(x.trg)),\n",
    "                            batch_size_fn=batch_size_fn,\n",
    "                            train=False)\n",
    "    model_par = nn.DataParallel(model, device_ids=devices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    model_opt = NoamOpt(\n",
    "        model.src_embed[0].d_model, 1, 2000,\n",
    "        torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98),\n",
    "                         eps=1e-9))\n",
    "    for epoch in range(10):\n",
    "        model_par.train()\n",
    "        run_epoch((rebatch(pad_idx, b) for b in train_iter), model_par,\n",
    "                  MultiGPULossCompute(model.generator,\n",
    "                                      criterion,\n",
    "                                      devices=devices,\n",
    "                                      opt=model_opt))\n",
    "        model_par.eval()\n",
    "        loss = run_epoch((rebatch(pad_idx, b) for b in valid_iter), model_par,\n",
    "                         MultiGPULossCompute(model.generator,\n",
    "                                             criterion,\n",
    "                                             devices=devices,\n",
    "                                             opt=None))\n",
    "        print(loss)\n",
    "else:\n",
    "    model = torch.load(\"iwslt.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, batch in enumerate(valid_iter):\n",
    "    src = batch.src.transpose(0, 1)[:1]\n",
    "    src_mask = (src != SRC.vocab.stoi[\"<blank>\"]).unsqueeze(-2)\n",
    "    out = greedy_decode(model,\n",
    "                        src,\n",
    "                        src_mask,\n",
    "                        max_len=60,\n",
    "                        start_symbol=TGT.vocab.stoi[\"<s>\"])\n",
    "    print(\"Translation:\", end=\"\\t\")\n",
    "    for i in range(1, out.size(1)):\n",
    "        sym = TGT.vocab.itos[out[0, i]]\n",
    "        if sym == \"</s>\": break\n",
    "        print(sym, end=\" \")\n",
    "    print()\n",
    "    print(\"Target:\", end=\"\\t\")\n",
    "    for i in range(1, batch.trg.size(0)):\n",
    "        sym = TGT.vocab.itos[batch.trg.data[i, 0]]\n",
    "        if sym == \"</s>\": break\n",
    "        print(sym, end=\" \")\n",
    "    print()\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 额外的组件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `BPE/ Word-piece`：将单词拆分成 子词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 共享权重：输入与目标的嵌入矩阵相同"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    model.src_embed[0].lut.weight = model.tgt_embeddings[0].lut.weight\n",
    "    model.generator.lut.weight = model.tgt_embed[0].lut.weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `Beam Search`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/OpenNMT/OpenNMT-py/blob/master/onmt/translate/beam_search.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `Model Averaging`：将最后 k 个 checkpoint 平均，创造组合模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average(model, models):\n",
    "    \"Average models into model\"\n",
    "    for ps in zip(*[m.params() for m in [model] + models]):\n",
    "        p[0].copy_(torch.sum(*ps[1:]) / len(ps[1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 注意力可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tgt_sent = trans.split()\n",
    "\n",
    "\n",
    "def draw(data, x, y, ax):\n",
    "    seaborn.heatmap(data,\n",
    "                    xticklabels=x,\n",
    "                    square=True,\n",
    "                    yticklabels=y,\n",
    "                    vmin=0.0,\n",
    "                    vmax=1.0,\n",
    "                    cbar=False,\n",
    "                    ax=ax)\n",
    "\n",
    "\n",
    "for layer in range(1, 6, 2):\n",
    "    fig, axs = plt.subplots(1, 4, figsize=(20, 10))\n",
    "    print(\"Encoder Layer\", layer + 1)\n",
    "    for h in range(4):\n",
    "        draw(model.encoder.layers[layer].self_attn.attn[0, h].data,\n",
    "             sent,\n",
    "             sent if h == 0 else [],\n",
    "             ax=axs[h])\n",
    "    plt.show()\n",
    "\n",
    "for layer in range(1, 6, 2):\n",
    "    fig, axs = plt.subplots(1, 4, figsize=(20, 10))\n",
    "    print(\"Decoder Self Layer\", layer + 1)\n",
    "    for h in range(4):\n",
    "        draw(model.decoder.layers[layer].self_attn.attn[0, h].\n",
    "             data[:len(tgt_sent), :len(tgt_sent)],\n",
    "             tgt_sent,\n",
    "             tgt_sent if h == 0 else [],\n",
    "             ax=axs[h])\n",
    "    plt.show()\n",
    "    print(\"Decoder Src Layer\", layer + 1)\n",
    "    fig, axs = plt.subplots(1, 4, figsize=(20, 10))\n",
    "    for h in range(4):\n",
    "        draw(model.decoder.layers[layer].self_attn.attn[0, h].\n",
    "             data[:len(tgt_sent), :len(sent)],\n",
    "             sent,\n",
    "             tgt_sent if h == 0 else [],\n",
    "             ax=axs[h])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}