06-自然语言处理/31-基于Transformer的中译英(TensorFlow).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于`transformer`的中译英模型\n",
    "[原始Google链接](https://tensorflow.google.cn/tutorials/text/transformer#%E4%BC%98%E5%8C%96%E5%99%A8%EF%BC%88optimizer%EF%BC%89)\n",
    "  \n",
    "目录：\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import re\n",
    "import jieba\n",
    "import tensorflow as tf\n",
    "import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import zhconv\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif']=['SimHei']\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一、语料数据处理及向量化\n",
    "原始语料每一行为一条中文短句及对应的英文翻译，两者通过 `\\t` 隔开"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '../H/datasets/cmn-eng/cmn.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hi.\\t嗨。\\tCC-BY 2.0 (France) Attribution: tatoeba.org #538123 (CM) & #891077 (Martha)\\n',\n",
       " 'Hi.\\t你好。\\tCC-BY 2.0 (France) Attribution: tatoeba.org #538123 (CM) & #4857568 (musclegirlxyp)\\n',\n",
       " 'Run.\\t你用跑的。\\tCC-BY 2.0 (France) Attribution: tatoeba.org #4008918 (JSakuragi) & #3748344 (egg0073)\\n',\n",
       " 'Wait!\\t等等！\\tCC-BY 2.0 (France) Attribution: tatoeba.org #1744314 (belgavox) & #4970122 (wzhd)\\n',\n",
       " 'Wait!\\t等一下！\\tCC-BY 2.0 (France) Attribution: tatoeba.org #1744314 (belgavox) & #5092613 (mirrorvan)\\n']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "io.open(data_dir, encoding='utf-8').readlines()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 英文处理\n",
    "def preprocess_eng(w):\n",
    "    w = w.lower().strip()\n",
    "\n",
    "    # 在单词与跟在其后的标点符号之间插入一个空格\n",
    "    # 例如： \"he is a boy.\" => \"he is a boy .\"\n",
    "    w = re.sub(r\"([?.!,¿])\", r\"\\1 \", w)\n",
    "    w = re.sub(r\"[' ']+\", \" \", w)\n",
    "\n",
    "    # 除了 (a-z, A-Z, \".\", \"?\", \"!\", \",\")，将所有字符替换为空格\n",
    "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "\n",
    "    w = w.rstrip().strip()\n",
    "\n",
    "    # 给句子加上开始和结束标记，以便模型知道何时开始和结束预测\n",
    "    w = '<start> ' + w + ' <end>'\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 中文处理\n",
    "def preprocess_zh(w):\n",
    "    w = zhconv.convert(w, \"zh-cn\")\n",
    "    w = ' '.join(jieba.cut(w))\n",
    "    w = w.rstrip().strip()\n",
    "    w = \"<start> \" + w + \" <end>\"\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 原始语料预处理\n",
    "def create_dataset(data_dir, num_examples):\n",
    "    lines = io.open(data_dir, encoding='utf-8').read().strip().split('\\n')\n",
    "    eng, zh = [], []\n",
    "    for line in lines[:num_examples]:\n",
    "        word_pairs = line.split(\"\\t\")\n",
    "        eng.append(preprocess_eng(word_pairs[0]))\n",
    "        zh.append(preprocess_zh(word_pairs[1]))\n",
    "    return [eng, zh]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<start> if a person has not had a chance to acquire his target language by the time he s an adult, he s unlikely to be able to reach native speaker level in that language. <end>\n",
      "<start> 如果 一个 人 在 成人 前 没有 机会 习得 目标语言 ， 他 对 该 语言 的 认识 达到 母语 者 程度 的 机会 是 相当 小 的 。 <end>\n"
     ]
    }
   ],
   "source": [
    "eng, zh = create_dataset(data_dir, None)\n",
    "print(eng[-1])\n",
    "print(zh[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文本向量化\n",
    "def tokenize(lang):\n",
    "    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters=' ')\n",
    "    lang_tokenizer.fit_on_texts(lang)\n",
    "    tensor = lang_tokenizer.texts_to_sequences(lang)\n",
    "    tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n",
    "                                                           padding=\"post\")\n",
    "    return tensor, lang_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(data_dir, num_examples=None):\n",
    "    eng, zh = create_dataset(data_dir, num_examples)\n",
    "    zh_tensor, zh_tokenizer = tokenize(zh)\n",
    "    eng_tensor, eng_tokenizer = tokenize(eng)\n",
    "    return zh_tensor, zh_tokenizer, eng_tensor, eng_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_examples = None\n",
    "zh_tensor, zh_tokenizer, eng_tensor, eng_tokenizer = load_dataset(data_dir, num_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# KEY：将分词器保存，方便后续使用\n",
    "\n",
    "tokenizer_dir = '../H/save/zh2eng_transformer_tokenizer/'\n",
    "zh_tokenizer_dir = tokenizer_dir + 'zh_tokenizer.json'\n",
    "eng_tokenizer_dir = tokenizer_dir + 'eng_tokenizer.json'\n",
    "\n",
    "tokenizer_json = zh_tokenizer.to_json()\n",
    "with io.open(zh_tokenizer_dir, 'w', encoding='utf-8') as f:\n",
    "    f.write(json.dumps(tokenizer_json, ensure_ascii=False))\n",
    "\n",
    "tokenizer_json = eng_tokenizer.to_json()\n",
    "with io.open(eng_tokenizer_dir, 'w', encoding='utf-8') as f:\n",
    "    f.write(json.dumps(tokenizer_json, ensure_ascii=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 载入分词器\n",
    "# # from tensorflow.keras.preprocessing.text import tokenizer_from_json\n",
    "# # 2.0.0 版本没有这个函数\n",
    "\n",
    "# from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "# import json\n",
    "# def tokenizer_from_json(json_string):\n",
    "#     \"\"\"Parses a JSON tokenizer configuration file and returns a\n",
    "#     tokenizer instance.\n",
    "\n",
    "#     # Arguments\n",
    "#         json_string: JSON string encoding a tokenizer configuration.\n",
    "\n",
    "#     # Returns\n",
    "#         A Keras Tokenizer instance\n",
    "#     \"\"\"\n",
    "#     tokenizer_config = json.loads(json_string)\n",
    "#     config = tokenizer_config.get('config')\n",
    "\n",
    "#     word_counts = json.loads(config.pop('word_counts'))\n",
    "#     word_docs = json.loads(config.pop('word_docs'))\n",
    "#     index_docs = json.loads(config.pop('index_docs'))\n",
    "#     # Integer indexing gets converted to strings with json.dumps()\n",
    "#     index_docs = {int(k): v for k, v in index_docs.items()}\n",
    "#     index_word = json.loads(config.pop('index_word'))\n",
    "#     index_word = {int(k): v for k, v in index_word.items()}\n",
    "#     word_index = json.loads(config.pop('word_index'))\n",
    "\n",
    "#     tokenizer = Tokenizer(**config)\n",
    "#     tokenizer.word_counts = word_counts\n",
    "#     tokenizer.word_docs = word_docs\n",
    "#     tokenizer.index_docs = index_docs\n",
    "#     tokenizer.word_index = word_index\n",
    "#     tokenizer.index_word = index_word\n",
    "\n",
    "#     return tokenizer\n",
    "\n",
    "\n",
    "# tokenizer_dir = '../H/save/zh2eng_transformer_tokenizer/'\n",
    "# zh_tokenizer_dir = tokenizer_dir + 'zh_tokenizer.json'\n",
    "# eng_tokenizer_dir = tokenizer_dir + 'eng_tokenizer.json'\n",
    "\n",
    "# with open(zh_tokenizer_dir) as f:\n",
    "#     data = json.load(f)\n",
    "#     zh_tokenizer = tokenizer_from_json(data)\n",
    "\n",
    "# with open(eng_tokenizer_dir) as f:\n",
    "#     data = json.load(f)\n",
    "#     zh_tokenizer = tokenizer_from_json(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21621, 30) (21621, 36)\n"
     ]
    }
   ],
   "source": [
    "print(zh_tensor.shape, eng_tensor.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 拆分训练集与测试集\n",
    "zh_tensor_train, zh_tensor_val, eng_tensor_train, eng_tensor_val = train_test_split(\n",
    "    zh_tensor, eng_tensor, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17296 4325\n"
     ]
    }
   ],
   "source": [
    "print(len(zh_tensor_train), len(zh_tensor_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Language; index to word mapping\n",
      "1 ---> <start>\n",
      "4 ---> 我\n",
      "12 ---> 是\n",
      "7 ---> 你\n",
      "5 ---> 的\n",
      "2381 ---> 伙伴\n",
      "3 ---> 。\n",
      "2 ---> <end>\n",
      "\n",
      "Target Language; index to word mapping\n",
      "1 ---> <start>\n",
      "3 ---> i\n",
      "36 ---> m\n",
      "27 ---> your\n",
      "3917 ---> partner.\n",
      "2 ---> <end>\n"
     ]
    }
   ],
   "source": [
    "def convert(tokenizer, tensor):\n",
    "    for t in tensor:\n",
    "        if t != 0:\n",
    "            print(\"%d ---> %s\" % (t, tokenizer.index_word[t]))\n",
    "\n",
    "\n",
    "print(\"Input Language; index to word mapping\")\n",
    "convert(zh_tokenizer, zh_tensor_train[0])\n",
    "print()\n",
    "print(\"Target Language; index to word mapping\")\n",
    "convert(eng_tokenizer, eng_tensor_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10696, 9756)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BUFFER_SIZE = len(zh_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "steps_per_epoch = len(zh_tensor_train) // BATCH_SIZE\n",
    "\n",
    "embedding_size = 256\n",
    "# units = 1024\n",
    "zh_vocab_size = len(zh_tokenizer.word_index) + 1\n",
    "eng_vocab_size = len(eng_tokenizer.word_index) + 1\n",
    "\n",
    "zh_vocab_size, eng_vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建 tf.dataset 数据集\n",
    "dataset = tf.data.Dataset.from_tensor_slices(\n",
    "    (zh_tensor_train, eng_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True).prefetch(\n",
    "    tf.data.experimental.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 30) (64, 36)\n"
     ]
    }
   ],
   "source": [
    "example_zh_batch, example_eng_batch = next(iter(dataset))\n",
    "print(example_zh_batch.shape, example_eng_batch.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二、位置编码(Positional encoding)\n",
    "对 token 序列中，每个 token 所处的位置信息进行编码，每个位置编码成一个维度为 d_model 的向量\n",
    "      \n",
    "$PE_{(pos, 2i)} = sin(pos/10000^{2i/d\\_model})$\n",
    "     \n",
    "$PE_{(pos, 2i+1)} = sin(pos/10000^{2i/d\\_model})$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_angles(pos, i, d_model):\n",
    "    \"\"\"\n",
    "    pos:     [seq_len, 1]\n",
    "    i:       [1, d_model]\n",
    "    d_model: int \n",
    "    return:  [seq_len, d_model]\n",
    "    \"\"\"\n",
    "    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model))\n",
    "    return pos * angle_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positional_encoding(position, d_model):\n",
    "    # 位置总数 position 个，每个位置编码成 d_model 维向量\n",
    "    angle_rads = get_angles(\n",
    "        np.arange(position)[:, np.newaxis],\n",
    "        np.arange(d_model)[np.newaxis, :], d_model)\n",
    "\n",
    "    # 将 sin 应用于 d_model 维中的偶数索引\n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "    # 将 cos 应用于 d_model 维中的偶数索引\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    # 返回：[1, position, d_model]\n",
    "    pos_encoding = angle_rads[np.newaxis, ...]\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 30, 512)\n"
     ]
    }
   ],
   "source": [
    "pos_encoding = positional_encoding(30, 512)\n",
    "print(pos_encoding.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yangbin7/anaconda3/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py:211: RuntimeWarning: Glyph 8722 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/home/yangbin7/anaconda3/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py:180: RuntimeWarning: Glyph 8722 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZwcdZ3/8denqrtnemaSmZ4zJwkJubgCIcZwOiAgKCqiAt7nBlwWd9fVVYT97aqoLO6PnxewsqKCq3iAIsh9ySUEEkhiIAeQg9wkk0nP1T3dVfX9/VHVMz2TOXomPZ2Zns/z8ahHV1fXmUy+U/n2uz5fMcaglFKq+FiH+wSUUkqNDG3glVKqSGkDr5RSRUobeKWUKlLawCulVJHSBl4ppYrUiDTwIlItIueISO1I7F8ppdTg8t7Ai0gM+DOwBHhCROpE5FYReU5Ersn38ZRSajQRkQYReXqAz8Micq+IPCsin+1vWT6MxB388cCXjDHfBh4CzgJsY8zJwCwRmTMCx1RKqcMuuMG9DSgfYLUrgZXGmFOBD4nIhH6WHbJQPnaSzRjzJICInIF/F18N/C74+GHgNOC17G1EZBmwDKBUrJMqCTF13jQ2toWYW1dOy9r1tKRdGiZNIFU3jU3bm3E6OyivrmZuXTnJLW+wrzlJ7NgFlCWaiG/dR4vjUWYLVZMmYtdPZ0dLkubmdpxEG2KHmGPaae1IkzKGiAgToiGidROxqmppcy32tXXS3p7GSbbjOWlOnDuFpLFpTqRpaU+TSqZwUwmM6/jXYIewwyXYkQhTm7aT9AyOZ3CDa7SAkAgRSwiHhFBJCLs0RPv+BGnP4BiDZ8DL/JkEky1giRASf/vd9UfgeR6ea/A8F+O5GNcF42GMwXgeYMAYTpwzGSwbxMIFPA88/OO4nsE1Bs8zuAZ27GxCRECsXq+CiCBCMA8i4h/LAJlX/+/efx8cPzMfi01AgusIdokgWMG+/J8B/3rf3L4Xej9dHbzv/cz13FmTCbb2X4N99PjZypp59bXt/f7cZh8H4Ni5R2T9fPa/iQBrNrw58H57OX7+ET3PbQCr1+e+7xPmHzH4SllWDWnfMwZfKbigVeu2Duk8TliQw77pe98m0bTPGFM3pAP2Yk2cZnCSOa1rEk2vANkr32KMuSXrvQtcAvxpgN00Al8L5p8CFvez7ImcTmoAeW/gAcT/V3sJ0Iz/73JH8NF+YFHv9YM/oFsA5kbLzQfcBr71s+s59/laHlq2iEfnn8pDO1v58qcb2X7Zf/Hhq+6i6fWXOOmjH+GhZYtY/5mL+cUf1nPRnx5j8ZrbuWfZ//DE3g4WTSjl/V88hwlXfo+rHnyNu+58gbdefZbSyjpudf7KYyt3sTPpML00zOnH1nHC5WdT+t5lPBcv5SfPbubF5dvZu2ElHU07ef7h/2C9U8mda3bx8ModbF+3jfi2dSTjewEomVDNhMmzqZ4+nW/971d5vS3Nnk6HNsdvsqO2UBsJMTUaYnJNlJqjYsTm1LH812vYnXTYl3JpczxSnt/I2AIRS6gIWUwM2VRHLKojNtdf8UM6WjrpaO2ks72NdHucVHscN5XATSVxOhMYz8VzUiy//yq8aCVeyQQ6HENH2qPd8UikPeJJh3inQ1unQ2vK5ap/vw27JIodimCFItiRKFY4QihSgmVbhCI2obCNHRJCYZt0p4vreLiuhxe8uo6H56T846dTXefxwUvPIBKyKAlZRHpPtoUlQtgWbBG++JWbMZ7/azHz6mXeu26Pn5uf3fHvWOJvZ4lgW92/RCy6f3FYwS+V48//lwF/bjPHA/jzYz/s+u9t5peQJZn33dtYwLTGKwfcb2+PP/njHvvLJr1+m9SddkXO+33qmRuHdB7Vp+a+72eevWnQdTKnXnny3w/pPJ7NYd8ZvfedXvXzof026YuTJDTvfTmtml7186QxZnF/nxtjWuDgv8deyunZJjb0s+yQjciXrMZ3BbAGOAWIBh9VjNQxlVJqWEQQy85pypM2Dm4T+1p2yEbiS9avisgng7dVwHX43TIAC4Et+T6mUkoNn2AF/3MdbMqTlRzcJva17JCNRBfNLcDvROTzwFrgbuApEZkCnA8sHYFjKqXU8AR38COzazkLONoY8+OsxbcB94vI6cDRwHL87pneyw7ZSHzJ2gyck71MRBqDZdcbY+L5PqZSSg2XAGLnt4E3xjQGr48Dj/f6bKuInIN/x/5/jDEu0NeyQzYiX7L2FjT6vxt0RaWUKjQRrBG6g++PMWYnvdrEvpYdqoI08EopNZqNVBfN4TbqEi0tSYdvPvwtzn+xnr/efhuPH3Ma9+9o4Z//+TT2fOEGLr76bvZtfJG3XXwJD16+hI2fv4Rf/nEDlWGbxa/+hvsu+ylP7O1gYWUpF/zLmUz84ve45uHX+cMfVvLWq89SMqGaI5e+oysiOaU0xOnH1bFw2VlEL/g8y1ui/OTZzax8cUdXRNIKRdjoVPHHtbt5eOUOdmzcScuOjT0ikhUNM6mePp1JM6sGjUjGZlURm1NH9fyZ7E46NKcHjkhWhv2IZEWs9KCIZDrZ1mdEEsg5ItmSTGOFIzlHJG3byjkiaTw354hk2JKcI5LGc4cUkRxMdkQSyDkiOVy5RCRHkpXnYxXw1POv8CmagtE7eKXUuCYIVih8uE9jRGgDr5Qa30YwRXO4aQOvlBr3tIFXSqliJJL3mORooQ28UmpcE/QOXimlipNY2PkrQzCqjLqY5NR5Uzlv1XSe/vnPOfnjn+SBbS185SvvYM+VP+ADV/2BveufZ+lHP8YjVyzhtc9+kNvuXE9FyOJjnz6Bez57I4++1c6iqlIu/Oo7qfqX/8fXHniN39+5gj1rn6JkQjWzTjmLv//AMX4VyWiYxuPrOeHysyl73zKeby3n5mc28eLy7exZt6IrIlkxaSZ3rd3FAy9uZ8fGnRzYspZE826gOyJZM2Mmk2ZWceaC+kEjkjXz6qmeP5Po7DnsS7nE0wNXkawr8SOS5fXlB1WRdBJtfUYkgZwjkvGONKFINOeIZChi5xyR9Dw354ikFWQHc4lI+n9WuUckrQGK8/aOSGZoRDI3YzoiCX6ZaY1JKqVU8RE0RaOUUkVLG3illCpGmoNXSqlipQ28UkoVJRHBChdnikYbeKXU+KZdNEopVby0gS+QDW1hUrf9gjP/7nPc35hmd9u5rPvotXziy7+mectaTvvUJ7n/08ew9pIL+d8HXicWtvn45UuY8p1b+b//vYC3xUp537+dT2TZd/iX+zZwz13L2bv+eUor65h9aiNXfuAYPjZvIs1lYU4/cRILv3AOkXcv4+kmmxufep3VK3awd/0KEs27sUIRJkyZTf3s+dy/fBs7Nmwnvm1dV5ng0sq6IAN/BFOCDPzJR8S4I8jAV4QsYmG7KwNfc1SM6nkNVC+YSXTWHMIzFwxYJrg60l0muLy+nPKG8h4Z+P7KBGe0O4ZEDhn41qRzUAY+FLaxQlafGXjLloMy8Jn8e+8MfKZccC4Z+OxywYNl4IEhZeD7y2ofagY+H/n1/vbRV17+UGkGvm/WSPxhjwKjroFXSqlCEhFEG3illCpOtp2fh/pF5Fb8QbPvM8Zc28fnXwAuCd5W4Q+ufQWwKZgArjTG/C0f5zPqShUopVRBCYglOU0D7kbkIsA2xpwMzBKROb3XMcbcbIxpDAblfhr4H+B44I7M8nw17qANvFJqnPOrSR56Aw800j1o9sPAaf0eU2Qq0GCMWQEsBS4QkRdE5FYRyVvPijbwSqlxzi9al8sE1IrIiqxpWdaOyoEdwfx+oGGAg14B3BzMvwicbYxZAoSBd+fryrQPXik1vgVdNDnaZ4xZ3M9nbUA0mK+gnxtoEbGAM4Grg0VrjDGdwfwK4KCuneEadQ18R/N+Pnn9P3DL9I3csOT/MP3ZJ/iHL91Kx76dvP+Kz/K/59fx4nsv5FdPv8nMsggf/cqZRL/0/7j0V6v5SF0Z5333IhIfuoov3LWWx//0V/ZvWk1ZzRTmnH4GX/nAsVw43aL9f6/jzFOmcdyy87HPW8bD2zu56cnXWPfSTpo2vkgyvhc7EmXClNk0zJnHCcc38Ph9L9GyYyOdrfsRy6ZkQjUTJs+mdsY0jpgV48wF9SydXsWcav/vtyJkURvxI5KT6sqoPqqa6nmTqF4wg9Ij5xKeMR8nNq1HRDJqW0FE0qIybFFXEqKsOkp5QxkVDeWU1U8ktX0/6UQbTrIdJ5XAS6e6Yom9tac9v1RwyiPemaYt5RJPOrSlHOKJNG1JhwMdado6HexIFLskih0K9YhIhsIWdldc0iIUtrBsCyft4nmmR0Qycx7ZEcmDYpJZEcmwZWELhGz/NRPh6ysi2fv6jOcOGpHsvay3/iKSGRqRHFixRCRhSA38QFbid8s8DywENvSz3unAcmOMCd7/UkS+DawFLgS+k4+TgVHYwCulVCGJgB3KSwN/N/C0iEwBzgcuFZFrjTHX9FrvXcBTWe+/Cfwa/+uAe4wxj+bjZEAbeKWUysv/xowxLSLSCJwDXG+M2Q2s7mO9r/d6vxY/SZN3eW/gRaQS+A1gA+34mc/XGYGMp1JKHSoRyduTrMaYZrqTNIfdSNzBfwy4wRjziIjcDHwNP+P51RE4llJKHTJ9kjVHxpibst7WAdvwM55nAn8DLjPGOPk+rlJKDVexNvAjloMXkZOBGPAII5TxVEqpQxYkuHLMwY8pI/Ilq4hUAz8CPgjsHizjGTwssAygqmEKPzZ/5lvn3MbEkM1l/3wjAP/w9cu47uh2Hmv8EHetb2JRVSkX/+dFxD/4dS766Yu8fO9D3PHTy9lx8me4/Jcv8/L9T9K66w0mTJ7N0WeewjXvO4Z3TozT9D8/5OWfPEPjjV/ANH6SP25o4idPvMEbq7bS9PpLpNvjhEorqJw2lynzj2LJwsm879hJ/OG/f0W6PY5YNtFYAxMmH0X9zEkcNbuaxvn1LJlaxexYhIqWbVSGLWojIY4oC1E3qYLqOTGq504htmAGJUfOx542Fyc2jTa7Ajg4IlkdsaktCVFWG6W8oZzy+jLK6iuJ1sdIbYj7FSQHiUiKZdOR9mjtdIl3OrR2OrR0OrSmHNqSTlcVybZOh9ZkGrskSigS9itGdsUk+45IRiI2ruP4cchBIpLGdYlGbGxLiNhWVuXInhHJcBCfzDUiCT0jkl2VIzUi2cf+8t8wjcG2rl+CYIWK85nPvF+ViESA3wNXGWO24mc8F4qIjZ/x7Otb5VuMMYuNMYvLK6vzfUpKKdU/8csF5zKNNSPxa+tzwCLgahH5C/AK8EtgFfBcPjOeSimVDyKS0zTWjMSXrDfTXWMh4xv5Po5SSuWDX2zscJ/FyNAHnZRS45voiE5KKVWkBCtPA36MNtrAK6XGNdE7+MKpju/i65/8OUuro1zy5E187+qX+N41H+bS+BP8/uTreGJvB++eVMG5P/8ifzv6w1z+/WdY9+j9GM/l5eP/hS/9ZDnrHn+cRPNuao5axOJzTuJb71nA8cmNbP2/P2LV/77Es00JTjrl49y5aje3P/4GW1dvpHnLWtxUgpIJ1VROX8C0BTM488QpvHtBAydNLifdHscKRSirmcLEafNoOKKaY+fWcsacWhZPqeTIqggl+17Dee1lppSGmRoNUTt9IrXzqonNnUZs/kzCMxdgTZmNUzWduBdmb5tDxBKitlBuW1SGg4G2o+GuiGRFfTnR+irKJ1UTrY/hJNtxgwqOfUUkxbK7pgNJp6t6ZFvKpTXlxyPjHWlaOx3akn5UMpFyCUXCBw2sHYrYXbHJzMDboaAqpOekMG7PeGRfEclMNcnM4NrhrGqSlkiPiKQd/BvLJSIJ9IhIZsca+4pISh/bD6Z70O7sZT0bguG2CxqRHF2K9UGnUdfAK6VUIYmArQ28UkoVJ23glVKqCAmiDbxSShUjEYgUaakCbeCVUuOaCIT0Dl4ppYqPoH3wSilVnET74Atm155WLjppDqc8di/v/NkrPHrj55hy5zf54dfvZUtHio8tncqpt13Pr9pm8I3rnmDb8gcoraxjwVln8Xc//Cubn3sEz0kx5aR3ccH5C/jXM2cxacNDvPrjn7H8gTdYHe/ENYYfPLuVPz+9mR2vvELrzjfwnBRlNVOIzTqBGQvqeM+iqZw3t465FQb71ccIlVZQVjuF2BHzaDiiiiXz6zh1VjULJ01getQjvHMNnete5MDfXmV2RYTYrCpq5tcQmzudiXNnEZ6xABqOJF01jaZO2NuRZnNzgoqQRbntlwiujlhUTijxM/D15ZQ3lFNWF6NscjXRuhh2rB6nc22P7Hm27Ay8FY7QnEjTmnJpSaZp6fTLBB/oSNPWKwOf7nT8jHt2WeBMmWDbIhSxsG2LkohNJGRRErIOysC72Xl4t/vcjOd2lQbunYEPW/4/rOz5XDPw0H8Gvkfp4My6InnNwB9KezBWM/DFmH+HzB18fvrgReRW4GjgPmPMtX18HsIfvrTHEKYi8g38sTJeMMZckZeTYQQH/FBKqbHCDm4wBpsGIiIXAbYx5mRglogcNPYF/uDadxhjGoPpbyJyEnAasAR4S0TOztd1aQOvlBrXLBEiwRPag02DaKR7wO2H8Rvt3pbiD2H6gojcGtzRvwO4yxhjgIeA0/NzZdrAK6UUtkhOE1ArIiuypmVZuykHdgTz+4GGPg71IgcPYZrLdsMy6vrglVKqkIZYqmCfMWZxP5+1AdFgvoK+b6DX9DGEaS7bDYvewSulxr189MEDK+nullkIbOljnb6GMM1lu2HRO3il1LiWxwed7gaeFpEpwPnApSJyrTHmmqx1vgn8Gj+8c48x5lERsYDvisgPgPOCKS9GXQM/qb6Cyfc/xHHXPM7mZ+7BWnE91/1uHRUhiy9+fhEz/+unfPGR7dz567vYv2k1VTOP5fT3nsoN7z+GOWd/kZIJ1cw+/V38/UXH8JmFDXj3fJ8Xf3Q/f121hzfaU0Rt4W2xKNfev4E961bQ0bQTKxRh4rS51B11NPOOqeeDi6Zxxowqpjh78Zb/hT3PPk/ltOOomTGTSTOrOHNBPSfPiLGgtowapxnrjVdJrl/JvlUbaVq3g/rj66iZV0/1/JlEZ88hMnM+bmw6neV17O1w2NWW4s14ki37O4iFbSrDFnUlNhWxUsrryymrjVIxuZJoXYyy+hgl9bXYsXrsWD2e8xKekzroz60rHhmKILaNHeoZk4x3+LHI7IhkZ8ol1engpD3CJaGuksA9SgYH0clIyCIasSkJWURCNp7TXa64rxLBPWKStnSVC7as4FX80sG2+F9y2cHyjMEiktBdDrg70th/RHI4RiIi2edxRiB+qBHJ3AmSl1IFxpgWEWkEzgGuN8bsxr9Dz15nLX6SJnuZFyRn3gP8wBiz+ZBPJjDqGnillCqkfJYLNsY0052kGcp2CeDOvJxEFm3glVLjmpYqUEqpYqUDfiilVHHSevBKKVXEtIFXSqkiZOmAH4VzIDaFUz79IzqadnLKJz/Fj7/0KU6tiXLRjz7G9nd+kbNuWsGa++8nnWjjiJMv4O8/spArTqih5af/RuURCzj2zLfzzfcdw8mR3ey5/p9Y9dPnePatdvanXCaVhnh7QznzPnAM2198knR7nHB5JZVT5zJl/ixOPWEK7z12EosnlzPxrVdJvvgIO59+me3Pb2Pqey5i7uxqzppfz5JpVRxZFSHavAVv02pa166i6ZVN7H11Dwc2HeDYjy8mtmAGkZnzsafOxYlNo9UqY29rmu0tnWxp7mBLUwdbm9o5t9QmFglR3lBGWW0Z5fVllE+upqy+imhdjHBdA3asDjtWD+WxfiOSViiCWDZ2OIIVimCFwuwP4pFtSYcDiTRtyTQdKZe2pEMq5ZLudHHSLq7rEQr70UjbDqpIhgTLtogEFSQjIYuyiE0k5L8fLCLZHZP0uqpJhuyeVSW75/1Hxfuq+NhfBUjjuV0RSSsIRPaoLEl3VHA4Eb9cI5KHet832itIjgvaB6+UUsVJ6KozU3S0gVdKjXvF+j8fbeCVUuOaAHZxtu/5b+BFpBL4DWAD7cAlwM0MMMqJUkodNgJWkfbBj8RXxx8DbjDGnAvsBi5l8FFOlFLqsBAgbFk5TWNN3u/gjTE3Zb2tAz4OfD94nxnl5LV8H1cppYZDu2iGQUROBmL4tY2zRytZ1Me6ywB/ZJRIBQ3HVPC973+ZL1RuZcXZR7L4p9/nll2V/OfXH2Dnyocoq5nCCe99DzdccgKL2laz7vIv8ti9r/P5O/7IP542g9jKu1h746947rGtrG1JAnDsxBJOOmkSCy49lQnvuoT0hT+gvG461bOOZ+bR9bzvpKmcM7uWo0qTyCsP0/TXJ9nxzKvsWrmbN5qTnLV4GqfNruG4+nKmRNKEt79E5/qVNK9ZR9MrW2l6rZl921rYnXRoXLKQ8Iz5UO8Psr0v4bKnJcWWAx1sPZBg01vtbG1qp7k5SV1lKeX1ZVQ0lFNWX0FZfYxofRVlDbVYsfquiKQXrcSLVvb8c+s1yLYdxCOtUAQrHGFvS+dBFSQ7kg5O2sVJezip7phkSWk4GGjbwg4dPMh2NBLy45K2H5kcaJBtf/K63oft7gqSttVzPhORzAzEna2viGT2ssEG2c6sA8P7r+pIxiP72udBxx/y/oq0lRppItpFMxQiUg38CPgsOYxWYoy5xRiz2BizWEKlI3FKSinVp8wzE7lMY03eG3gRiQC/B64yxmxlBEcrUUqpfLAlt2msGYkums/hd8NcLSJXAz8HPpE1ysnSETimUkoNiwRPVBejkfiS9Wb8WGQXEbmH7lFO4vk+plJKDVd2WYtiU5AHnYY7yolSShXCWOx+yYU+yaqUGteE/H2BKiK3MsBDnf08COoBm4IJ4EpjzN/ycT7F2fGklFK5CqpJ5jINuBuRixj8oc7eD4Kehz8I9x3GmMZgykvjDqPwDr6sqppVP/s87ve/xA3fe4KLt77EO29fycv3/pZkfC9T3/ZuPvPh4/jX044g+ctv8ch/PsCjb8Zpczx+vChM00++xhM/eYZnd7Swt9OlrsTm7dVlzL9oAdM/9H7Mkgt5cmcHNUctYtLc2Sw9cQrvO3YSS6ZOoKppI53PPsqup1aw4/k32b7pAK+3pdiXcvnUCVOZHYtQ0bINb8NqWtevYd+a19n36h6aNx1g94Eku5MOzWmXyHGn4cam0WZXsLc1zY6WTrYcSLC1qYNNe9vYuT9B24Ek7S2dxGZVUV5fRll9JdH6GOWTagjX1GIHGXgqanCDDLwbLuv6c+qvRHAmAx+KRGlqS9HW6dCaTHeVCO7KwKddnJSL5xqclEv5xFJCYWvAEsER28/ER0LWoCWC/Vd/WdiSHiWCrSD3nikRbFsEn0nXdn3pvTyTgc8uEQz+vnrn32WId2iHs0RwkfYWjFp+H3xedtVId1d0nw919vEg6Fv4wZMLRORM4G/AZcYYJx8npHfwSqlxbYilCmpFZEXWtCxrV+X0fKizod9jBg+CGmOeB14EzjbGLAHCwLvzdW2j7g5eKaUKSmAIKcl9xpjF/Xw26EOd0ONB0A8Gi9YYYzqD+RVA3up16R28Umpcy+OTrIM+1NnHg6AAvxSRhSJiAxcCq/NxXaANvFJq3PO//8llGsTd+A913gBcDLwiIr2TNNkPgv5FRC4Bvgn8ElgFPGeMeTRfV6ZdNEqpcS1fDzoZY1pEpJHuhzp30+tuvK8HQQPHH/IJ9EEbeKXUuOaXKshPjGa0PdQ56hr4eRMdVi06jbs3NTO7PMJp/3Qne9Y+xYTJszntwvP54YcXMnf7X1j7kSt59NEtvNGeoq7E5l1H1bDqc5fz7DPb2djWiS3CoqpSTnj7FBZ89AyiZ3+EzaEp3LtyN396YRvHn3kiF500jbOOrGZGqBVW3cNbzzzNzr9uZPeqPbwe72RnMk087Uf+jpuQwn5zFZ3rVrB/zQaa1m1n/2v7adrZxo6Ew76UQ5vjkXANqclH81a7w1stnWw+kGBrcweb9razfX8Hzc1J2uIJEq0pku0dVB9VQ7Q+RrSuimhDXVd5YKuqrqtEsFcygaQntCfdQUsEhyLRIDIZYW9rko6U22+JYCfl4boenuMRLrEJhfuPR2bKBmc+99KpAUsEZ7+GbeugEsFhy+oRj8zEJQcrEZwtE5EcrETwUCOSGYPFI0eiwqxGJA+PIq1UMPoaeKWUKjSrSH+1agOvlBrXhHF+By8i9cBZQCSzzBhz+0idlFJKFVKRDuiUc0zyQeAogl92aFehUqpYiH8Hn8s01uTaRdPaV2U0pZQa64ScMu5jUq4N/NMicgdwO36JS4wxT43YWSmlVAEVaxdNrg18GlgPvA2/e8YAI9LA79iwnUftqXzqzBks+fE3+I9lf+Lo8z/E1y89gYtqW9n+/Sv5za0v8Pz+BBFLOLOujJMuPpaZf/d5vnLSZSRcw8yyMEtmx5h/8Uk0XHQJB6Yv4b5NzfzmxVfY8Mpb7H39VR648TKOqyslvHk5bc89yo4nV7Nz5W427WxlWyLN/pSLa/yBACrDNt4L9xJ/ZS1Nr2ymaX0TzZsOsK0jzd5OhxbHI+F6uMa/htebO9l6IMm2eILNe9vZ2tTO7qYO2ls6aW/pJNmeItW6n1RHnKrG6UTrY4Riddg1k7FjdXhlVbglE/CilaSsCO1pj460SyJtuuKQVlBNMhORtEui2KEIdiTqxyeDapJ+1cigemQwuY7Bc/yIpOt4eK5HSUmIaMTOikIeXEEye+qrgmTveKQXvJbYVlc8sq8Kktnve+svIgndEcnB4pHD+cebvU1fm+e7QSjS9mXMKNY//1z74L+DX7u4Gr9a2ndG7IyUUqqA8liLZtTJtYH/GX7pyweAqfgDaSulVFEY71+yTjfGfCKYf0hEnhypE1JKqUIr1qqLuTbwO0XkKmA5/ugjOwZZXymlxgQJhuwrRrn+4vo00IJfoP5A8F4ppYrCuO6iMcakgBtH+FyUUqrgBO2iKZiJEZtr77maTcd/mHfe8TLf+u4/cMUJNbT+/Fs89L1HeGJ3GwnXY2FlKSefN4u5f3cpqaUf5lfr9lEZtjl3WjnzPnAM0y7+IO6J7+HRrRE9RU0AACAASURBVC387r4NrFi1iz2vvUbLzjdwkm2c5LxB8p5H2Pz0y2x/fhvbthxgc3uafSmXlGeCeKRFbSTEEWUhtt/zEHtf3cOBTQfY2dLJ7qRLi+PS5nTHI22BqG2xfNsBtjR1sLWpne37OugI4pGJtk46Ww+Q6ojjJNpwU0kq5hzVVUGS8pg/wHbpRNKhKB1pj0SnS3vaoz3lEu90CJVE+xxgOxOVtEIRQpEwtm2RbE9nVY70q0n2jke6joPxXCpKQ/0OsJ092ZYQsS08JwUcPMB2RiYiaVy33wG2s9+L9Cz4NFA8MiPzcMpA1SO7BuQe5t3XSFeQHIM3hUVpuBVHR7sBG3gRucEY8yUReQI/+w5BDt4Yc9aIn51SSo00GacPOhljvhS8nlmY01FKqcIS/P99F6NR10WjlFKFVqxdNMP6bkFEThvk8wYReTqYnyoi24MBZv8iInXDOaZSSo0E/0nW3KZB9yVyq4g8JyLXDGWdXLYbjpwaeBF5pNei7w6wbgy4DSgPFr0d+LYxpjGY9g7rTJVSaoRIjtOA+xC5CLCNMScDs0RkTi7r5LLdcA32JevxwInAVBH5ZLC4HEgOsJkLXAL8KXi/FHiniPwd8KAx5uuHdspKKZVPeasz00j3gNsPA6cBr+Wwzok5bDcsg93BSx+vTcDF/W1gjGkxxsSzFj2Af1FvA04Ofmn0PIjIMhFZISIr4q6T67krpdShG9qAH7WZtiqYlmXtqZzup/z349fv6q2vdXLZblgGS9GsBlaLyLxDGKLvr8aYTgAReRmYA6zpdZxbgFsAjjthkXn/6/NZ+aObaNm+kT9NnMJfPnc/j285QDztcezEEk49cwYLln0Ac9Zn+NPG/fzkpyt47aUtPP65RUz/0Pvhbe/jud2d/ObPG3j+5Z3s3vgGLbveIN0eRyybspopbP7Bf7HzhW1se705yL87JIJAeyb/PjUaYtLkCqrnxHj9gdfYfSDJ7qRDc9ol4RpSnr9+Jv9eEbKYGLJ4cuNedu5P0HYg6ZcH7kjR2dpCuiNOqj2Om0ripBJ46RSRme+Eihq8aCVetBI3XOaXB064JByP9pRHvDNNPOnQlnIJlZZ35d+7ygMHGfhM/j0UtrFCFp3JNE7KxXMNTsrtM//upVMYz6WqLNJv/t22pOuzsCVYluSUf88IW9ag+ffMDVQu+ffMeoc7/34oX8wd7q/0ivQ7xWERY5Acf+6AfcaYxf181gZEg/kK+r6B7mudXLYblpx2dIjdKg+JyGQRKQPOBdYewr6UUirvxHg5TYNYid+9ArAQ2JLjOrlsNyyFiEl+A3gCSAH/bYzZUIBjKqVUjgwM3njn4m780e+mAOcDl4rItcaYawZYZ6l/Agcty4sRe5LVGNMYvD4BzM/DuSql1MgwZvB1Bt2FaRGRRuAc4HpjzG5g9SDrxAH6WpYP+iSrUmp8M3m7g8cY00x3IibndXLZbjj0SVal1LiXQ//6mJRTAy8iFv63ux3A6cAKY0zrSJ6YUkoVhgGvOOPZud7B/x5/HNZ34Q+8fTVw9kic0LrNe9jwP7cycdpcTvnkp/jW5Z8h4XocO7GUU849kvnLLiZ9yqX8fn0Tt968nDde2sz+zatJt8eZ9tfbeHp7G3fc+zor1+xi94bu8sCZeOTEafOom17HX2++a8DywPVTJ1A9p5rquVOomjudBx/8Fc3pvssDZ+KR1RGb2pIQv93U3G954Ew80nP8eKJXPxuvdGJ3PLLD6VEeuLXToaXToTXlEO9IE4pW9FseOBOPDIUtQhGbVMIZNB6ZOY+KktCA5YEz8ciwZWFLbvHIzOd9lQvuHY+0chxQIfuYucYjDyVzlu94JOQekRyJQZ41HtkHQ966aEabXH/2a4wxfwbmGGM+RndmUymlxjgDnpfbNMbkegffKiJ3AytF5N2Ads8opYrGuO6DBz4MHG2MeUlEFuLXmlFKqeIwzht4B1gsIp8AXgkmpZQa+4yB3EsVjCm59sH/HJgMPAhMDd4rpVRRyFOpglEn1zv46caYTwTzD4nIX0bofJRSqsDy96DTaJNrA79TRK4CluPXSdg5Uidk2SE+8I+X843z5zOn6SXu/G4pJ118LDM/91n2zDydG9fu5rf/9QxbV79KfPtG3FSC0so6ahaeyaW/Ws2GV95i7+uv0v7WNtxUAjsSZcLk2UycNo9JM2Mcd1QNjXNqefbbCVzjRx2rIzYNJX48smZGJbXzaqiaO42qeUcSnjkfmTybbYlfdMUjI5YQtYVy249GVkdsYuVhorVlVNSX8db2OKnW/T3jkZ2JrkhidtQvWdHgxyPb0yTSxo9DJh3inQ5tKT8mGe9I05b050sqqv1KkpEodsiPRtq2H4sMha0gJukva92f6BGP9JwUxnV7nIfxXFwnxYTS0MERSRHClhC2LSwRwrYfdQxb4kcss66jr3hkRnY1yex4ZHakMTsy2Vt/FSYzccXsKGPvCpPZ6w3FSMQjcz92fo+j0cgcjMcGXkQmAVcC64B24IP4/e+fHvEzU0qpQshjqYLRZrA7+F8CvwBiwNuzummUUqooCOM3JhkxxvwKQEQ+VIDzUUqpAjPgFmeKZrAGvk5EPor/S64+mAfAGPPrET0zpZQqhCIuVTBYA/9b/CH2es8fevFkpZQaJcZlF40x5huFOhGllDo8xu+XrAV37JG1/HzCU6y6+MvcsHI3/7TpYVYmJvKNpzfxws8eYc+6FXQ07cQKRSivm07dnGOZc0w9HzppGl/8ys0k43sxnkvJhGqqjlhA9fQZTJ4V44x5dZwys5oFtWXUmTgvWkIsbDM1GmJqLEr1nBg18+qpmjOd8qPmEJ65AK96Op0VDezt8EuJRm0JKkfaVIYt6kpsKmKllNWUUd5QRnn9BMom1dC64vUe8chM1cbexLLZ3ebQnnaJJx1aUy4tyXTXa7wjTWvSoa3ToS3pz5dMqMIKYpF+PLJnVNKyxX8fsuhMpLurVvaqHullxSSN61JZFu6qHhm2rK4KkN1VJIOIpO1Xk/SC7bL1jjNm3kdCQZwxKx7ZHWfsWVFyoP31NljlyL6qTA5Vf9HI4e5vIBqPPIy0gVdKqSJUxKUKtIFXSo1zBuOk87InEXmU/tvV7caYj+flQDnSBl4pNb4Z8nkHf50x5tG+PhCRC4PXW4GjgfuMMdf2s24l8BvAxn/I9BLAAzYFE8CVxpi/DXQyhzLYjVJKjXkGg3HdnKZDJSIXAbYx5mRglojM6WfVjwE3GGPOBXYD5wHHA3cYYxqDacDGHfQOXik13hmGMlpTrYisyHp/izHmliEcrRH4XTD/MHAa8NpBp2TMTVlv64C38OuAXSAiZwJ/Ay4zxgw4mKw28EqpcW5IX7LuM8YsPoSDlQM7gvn9wKKBVhaRk4GYMeZ5EXGBs40xu0TkduDdwD0Dba8NvFJqfDP5+5I1B210j2ldwQDd5CJSDfwIv8gjwBpjTGcwv4LuB0/7Neoa+ObVr3L1JTeScA2zyyOcfOMG3lzzCi3bN+I5KUor65h84tkcsWAy5y2aynvm17OgysZ+/Tkub49T0TCTqiPmM2lmjBPn1HLaUTWcOHkCMypsIrvXkVr+EgfWvsKZdeXEZlZSO7+GqrnTqZx7JJGZC2DSLNyqabyVttjb4bBlS5w34wmmlIapDFtdpYHLG8opq4lS3lBO+aQaovVVROtjhGom0fHg2j5LA4Offc9MVjjCpuZEn6WBDyTStCXTdKRc2pIOTtrFSXlEK0qCMsE9SwOHIpb/GrKIRmxKQhYbEm09zsPNLhMc9Cdm3k8sDWMLfZYGtq3e8/TYPltf2XVbpN/SwF3L8XPgg2Xfe/xZ5lAaeLTn3zX7PhqYIf3cHaKV+N0yzwMLgQ19rSQiEeD3wFXGmK3B4l+KyLeBtcCFwHcGO9ioa+CVUqqg8puiGczdwNMiMgU4H1gqIkcDHzXGXJO13ufwu2+uFpGrgZuBbwK/xr8fuqe/tE42beCVUuOcGcqXrIO5TESu6eezVcaYu0WkETgHuN4YEwfiQI9tjDE34zfqvR0/lJPRBl4pNb4Z8hKBBDDGfDiHdZrpTtKMqBHJwYtIg4g8HcyHReReEXlWRD47EsdTSqnhC1I0uUxjTN4beBGJAbfhx4HAH/JvpTHmVOBDIjIh38dUSqlhC1I0uUxjzUjcwbv4j9W2BO8b6f7vyFPAQRlSEVkmIitEZEXrwLl9pZTKs+K9g897H7wxpgV6xMx6B/sb+tjmFuAWgGl2qXnv0XXMv/gkGi66hKs/djullXXUH3Mq0+dP5ewTp3DBggaOqyslvHk5bQ/ezqZn1rDjhV0s/Mh3OX5uLY1zalk0ZSIzJ4YJ79lAetWDtL6ylqZXNtO0vonmTQdYfMXpVM07kvDM+cjk2bhV09jnhtjb4bD1zQ62xRNs3tvO1qZ2djd18LX6MqK1ZVTUl1HeUE60PkZZXRVlk2sIxeqwYvWEaiZhohNxks/3uL7e0UjLsrFCEaxQmPX72rqika1ZZYGzo5FO2u2aohMiA0YjIyGLSMgmErJwkm0945G9opGZaJjxPMrC9qDRSD/uKNhWzzhkfxGzzHLbGjga6f+85PoT1i1zdzJQNDIfqcHRHo3M0IjkMBU2RVNQhfiSNRPsj+MH+9sKcEyllMqJwWDyl6IZVQpRbCwT7Ac/2L+lAMdUSqncZO7gtYtmWG4D7heR0/FLZC4vwDGVUio3xmDSB4+4VgxG7A7eGNMYvG7FD/U/i18oZ+z9GlRKFbHgQadcpjGmIA86GWN2UqBgv1JKDdkY7H7JhT7JqpQa30xBi40V1Khr4BuOmc3Mxx/jD6/t486H3uSMz32WDy6exlmzqjky3AHrn+XA725m3TPr2LlyN6/HO9mZTBNPe/z2C0uZHEpi71pH6rkVNK1ez/7129i3vommnW3sSDg0p13iaZd3ff7LOFVT2dXhsLfdYcvmdrYeSLDpLT8a2dycpL0lSaItRaK1nVnnzqasPkbZpGqiddVdsUirqg4vWolXOoFUaSVJL4j99YpG2kEs0gpF/KhkKEIoEuWVHS1d0ciOTDQy7eGk/Fik63o4KQ/X9fAcj6q6ckJhO4hDWpSELKKRkP/e7l4WCVmkk23+SDRe73ik1/U+81oRsbEtPxbpRyG7o5GZ+GR/McmM/v6RZKpJZlJ8vaOR/cUdB5NZv3eMsfdehhNzHGwbTSQWl2JN0Yy6Bl4ppQrKGIyrDbxSShUdYwxeujifoNcGXik1vhn0Dl4ppYqVNvBKKVWEjDF4eaoHP9poA6+UGvc0RVMgr+7pZMlnbqT9rW24qQSJX32Stud+ys6frOGpF3axdVcrWzrS7E+5uAZsgcqwzYIJJZTd/m9syqoYuTORZnfSocXxSLgervGPEbGEh5or2LZld4+Kke0tnSRaU3S0dZJq3U862Ua6PY6bSjLjX8/BitVjx+qhvAqvtBI3WknCitCe9uhIeyTiLq0ph1BpBXYQhcxEI+2SKHYogh2J+rHJSBQ7ZLFxR/ygipGuY/AcPxrpOh6e6+E6Dp6ToiE2pUfFyIhtZVWR7Dm5nYkeFSOzY5EAXlascUJJ6KCKkb2jkZZI16DZGbnkh8PWwLHI4VZrzK5K2d9n+aKxyCKWxxSNiDxK/+3qdmPMx/NyoByNugZeKaUKKc8pmuv6GwxbRC4MXm/Fr8t1nzHm2n7WDQGbggngSmPM30TkG8C7gReMMVcMdjKFqCaplFKjmud6OU2HSkQuAmxjzMnALBGZ08+qxwN3GGMag+lvInISfmXeJcBbInL2YMfTBl4pNb4FMclcJqA2M/pcMC0b4tEa6a7L9TDdpdR7WwpcICIviMitwR39O4C7jDEGeAg4fbCDaReNUmp8G1of/D5jzEHDjg5B7xHuFvWz3ov41Xd3icjt+N0y5cAbWdseNDpeb9rAK6XGNUNBUzSZEe7AH+Guv16UNcaYzmB+BTBnCNt20S4apdT4ZgxeyslpyoNcR7j7pYgsFBEbuBBYPYRtu+gdvFJqfDPgFe4O/m7gaRGZApwPLBWRo4GPGmOuyVrvm8Cv8RO69xhjHhURC/iuiPwAOC+YBjTqGvhUeysTQhFmLD2XSTOr+PHSZV3lgMHPsFdHbBZWljK1IkL1nBjVR9VQvWAGv/73+7vKAScyoXcgaguVYZuJIYvKsE1dic1197zaoxxwuj1OqiOOk2jDTSVx0yk8J9VVYtda+o94pZV0eEJ72pBwPDpaPOLJDuKdDm0ph7ZOh5ZOh7LaKV3lgDMZeCsUIRS2CUVsbNsiFLGwbYsDe9t7lAPuyr4Hx/bS3efgOSmmxsoOyr3blhAJWYQti7DtZ9fDluCmkkDfuXeT9eReV7ngXpl3oCv3LuJnyzNZ9oHy770/szN59V659+zqwMP5r2R3+eE+Phti6eH+9n04HOKpqyEy5LWa5GUick0/n60yxtwtIo34o9xdb4yJA3GgxzbGmLX4SZrsZV6QnHkP8ANjzObBTmbUNfBKKVVQpucNzyHtypgP57BOM8Mc4c4YkwDuzHV9beCVUuOc0VIFSilVlLRcsFJKFSdjDG5+EjKjjjbwSqlxTrtolFKqOGkXTeHMmjmJx25dxiSrA3vnq3z76y6zyyNMryyhek411UfVEFswg4qjjiI8cwFezQzSEyezt8Nh3Zf+SNQWorZFQ4lFdcSmOmIzobKEivpyyhvKKK+fQLQ+xvpnlvcbicwmlo0VirA+WU78QLIrEhlPOrQk07QlHQ50pGnrdGhLpulIuVROnYNlWwdFIkNhGytkEQpb2CH//RtrdvcbifSCeeO5GNd/nVFbhm3JQZFIywpeRYLPBddJAQdHIrNl3pdHbKDvSGRmGQTL+9h+ILYl/UYis+OMQy3xa/WKXw60Tr5Yec4waiRyFDBgsmLVxWTUNfBKKVVIBpOXSpGjkTbwSqnxzYDx9A5+WPorXD/Sx1VKqVwYA25Kx2Qdrkzh+q8W4FhKKTU0xhRtH3whqkn2VbheKaVGDc81OU1jTSEa+Ezh+iVAGL9wvVJKjQ5DG9FpTCnE3XRfhet7CIa9WgYwKVrKhlPfwZN7O9iddPnqvVcTnrkAp2oaiWiNH4dsTbEtnmDzrg42rTnA1n07aG/p5FtH11FWG6W8oZzy+gmUTaqhrD5GpKYau2YydqwOmViLF62k5fxv9zwHy0YsGzsSRWwbOxTBCoW7qkH+bvVOWpMO8USaRMqhNemQSDo4aRcn7eGkXTzHw0l71E6ZSChiY9lCKGxjhyyiEZtIyAoqQfrz0bDN2idW9hmHzMQP/Xmva37yhFJs8eN6Ydvqmrd7xCT9ZV461XV9fcUZs5dFbOkRkYTsao3ZlSX7319/bBk4DjncpGDv6pT9rjeMfec7DplNo5GjiwG8Iv2StRB38H0Vru/BGHOLMWaxMWZxLBIuwCkppVTAGNyUm9M01hTiDv6gwvUFOKZSSuXE6INOw9dX4XqllBo1tIFXSqlipU+yKqVUcdInWZVSqjgZyFvGXUQepf92dbsx5uN5OVCORl0Dvy/eyf1t+6kIWUwM2fz9vhPYvrGDlgMb6GjppKO1k872Nn+Q7PY4biqBm0ridCZo/PW3u2KQpnQCbulEOtIe+9IeCccjkfaIJx3i+x1KK+t6xCD9WGQEOxLFCkcIRUqyqj/aPPTCtq4YpBsMju2kXH+wgGCQ7Ew1yLPfc6I/ILZtURbEIzMDY3dNtoUlQjK+t0cMMvs1M0h2djXIyRUlXTHI3oNkZwbIBj+K5zkpcmE8l1Lb6hGD9PfR/yDZQxHqlWPM1yDZveOcSg2LMXj5S8hc11+QREQuDF5vBY4G7jPGXNvPul8ALgneVgHLgSsYYtmXQsQklVJq1DKmcE+yishFgG2MORmYJSIHPRfkn5O52RjTaIxpBJ4G/ofusi+NwTRoTS9t4JVS457xvJwmoFZEVmRNy4Z4qEbgd8H8w8BpA60sIlOBBmPMCoZR9mXUddEopVRBmSHdne8zxiw+hKOVAzuC+f3AokHWvwK4OZjPlH3ZJSK345d9uWegjbWBV0qNb4XNwbcB0WC+ggF6UUTEAs4Erg4WDVr2pTftolFKjWuGghYbW0l3t8xCYMsA654OLDfGZH77DFr2pTe9g1dKjW/G4KYK9qDT3cDTIjIFOB9YKiJHAx81xlzTa913AU9lvR9y2Rdt4JVS45ox4Jm8ddFcJiK9G+qMVcaYu0WkETgHuN4YEwfiwEHbGGO+3uv9kMu+jLoGfuqMGr7z4y9jx+qwY/WUfeKmPkvTimVjhSJ+id9whEh5Jb9yFtC62yHekaYtuY8DiV20JdN0pFzakg6plEu608VJu8xYcgahsEUoYmPbVlDWV7Bsi0iQXY+EMjl2mwf++HxXGd/+yvtmzvOMuecQtvxSvqGgpG84yL13z4MtQqq95aDr668Ur/Fc6srDB+Xds/Pg2aV9h1LStyQkfWbg4dDL+9o55OCHo3c543zSkr7ji5unBt4Y8+Ec1mmmO0kzokZdA6+UUoVkgCKtNaYNvFJK5esOfrTRBl4pNa55BlJabEwppYqTdtEopVQRMhjtolFKqWKkX7IW0JtSyUd2n0TbFgcn7XLkaRcEEUaLUNjqKt9r2xahiOW/hiyiEZt/+9FfMK6L56S6Yotu1nym7K7xXL7575/oijJmIoxh2y+/G7YE2+o5f8cP1nWd42CxxrdPreoZXwxijZnoXXas0U0lhhRnrCq1/X1kLetdanc4McRSW/ot2XuosUa71/b5ijVmR0KVOhTawCulVBEyRlM0SilVlAyaolFKqaKkffBKKVXEtItGKaWKkN8Hf7jPYmRoA6+UGvf0Dr5Amvfs5f4bb+l6H3/uppy3rczabjCfO3HykM7LSbblvO6sWCTndYcSkQSoLLGHtH6uSkIjN/ZL72qS+aLxSJUPBihYNfgCG3UNvFJKFZLBaIpGKaWKkZ+i0QZeKaWKj37JeuhE5FbgaOA+Y8y1hTquUkoNJJ938CLyKP23q9uNMR/Py4FyVJAGXkQuAmxjzMki8jMRmWOMea0Qx1ZKqcHk8Q7+uv4GwxaRC4PXBuBOY8zp/e1ERMLAH4Bq4FZjzM/6WjbYyYxcdKKnRrrHIHwYOK1Ax1VKqQF5+KUKcpkOlYjEgNuA8kFWvRJYaYw5FfiQiEzoZ9nAxzMF+HIh6J75oTFmtYicCywyxlyX9fkyYFnw9lhg7YifVGHVAvsO90nkkV7P6Fds19Tf9cwwxtQdyo5F5MFg/7koBZJZ728xxnTls0Xk7EHu4B/HLyT7J2NM4wDndA/wNWPMqyLyNWA58M+9lxljnhjoZAvVB98GRIP5Cnr9zyH4A7oFQERWGGMWF+i8CqLYrkmvZ/Qrtmsayesxxpw3Evvt51gtcHCJ7z6UAzuC+f1AQz/LBlSoLpqVdHfLLAS2FOi4Sik1FvV1UzzgjXJfCtXA3w18QkRuAC4G7ivQcZVSaizq66Z4yDfKBemiMca0iEgjcA5wvTEmPsDqudcbGDuK7Zr0eka/YrumYrueLiJyFnC0MebHWYtvA+4XkdPx4+XL8btnei8beN+F+JJVKaXGAxH5PdDfl76rjDH/NIR9TcG/Y38oc1Pc17IB96ENvFJKFadC9cErddiJSLWInCMiuUbilBrTRlUDLyK3ishzInLN4T6X4RCRBhF5OpgPi8i9IvKsiHy2v2WjlYhUisgDIvKwiPxRRCJ9/f2Mlb+z4AGTPwNLgCdEpG4sX09G8DP3cjA/pq9HREIi8qaI/CWYjhORb4jIiyJyY9Z6By1TfRs1DXx2OQNglojMOdznNBR9PKGWlyfRDqOPATcYY84FdgOX0uvvZ4z9nR0PfMkY823gIeAsxvb1ZPwXEO3r3Mfg9RwP3GGMaQweAorg9zcvAd4SkbNF5KTeyw7b2Y4Bo6aBZ+yXM3CBS4CW4H0j3dfzFLC4n2WjkjHmJmPMI8HbOuDjHPz309jHslHJGPOkMeZ5ETkDv3F4F2P4eqArfdGO/wu4kTF+PcBS4AIReSF4+v2dwF3G/6LwIeB04B19LFP9GE0N/JCf0hpNjDEtvb7VzsuTaIebiJwMxIBtjPHrEf/xwUuAZvwigmP2ekQkAvwb8LVgUTH8vL0InG2MWQKE8R/qGevXdFiNpgZ+yE9pjXJ5eRLtcBKRauBHwGcpgusxviuANcApjO3r+RpwkzHmQPB+zP/9AGuMMbuC+RUUxzUdVqPpD6fYyhnk5Um0wyW4Q/w9cJUxZitj/3q+KiKfDN5WAdcxhq8HOBu4QkT+ApwAvJexfT0AvxSRhSJiAxfi362P9Ws6rEZNDl5EJgJPA48B5wNLcwnyjzYi8hdjTKOIzADuBx7Fv1tcCkzrvcwYM7RRtwtERL4AfAdYHSz6OfAlsv5+8Ls5xsTfWfAl+O+AEvxqpVfhfw8yJq8nW9DIv49e584Yux4RORb4NX61xXvwu6Cexr+bPy+YtvZeZozZfFhOeAwYNQ08dP0jPAd4yhiz+3Cfz6HKx5Noo0lffz9j+e9Mr2f0E5Eo8B7gJWPMpv6Wqb6NqgZeKaVU/oymPnillFJ5pA28UkoVKW3g1WEhIr8QkVUiskJE/m6I254gIidkvf+PoBy1UipLoYbsU6ov/wCsA1aLyHJjzJoct8s07qtG5rSUKg56B68OK2NME/4IXx8Iipv9VUSugq67/D8Exdl+HCz7Lv5DPl8TkceydnWOiDwV/K9gUsEvRKlRSBt4NRo04Weef2uMOQW4UERqgs/uDIqzHSkiJxljrsJ/SOk6Y8w7s/ZxlDHmDOAP+IXElBr3tIFXo0E1YANfCB7aKQemBJ+tDF7XADMH2Mftweub+FUIlRr3tIFXh5WIVOE/ZfkQ8LWgTOx1+IWkwK/8CH6/+xvBfAIoC7aXYFl7Ic5XqbFEG3h1OP0IeBD4KvBpukQCzAAAAGVJREFU4Msi8iz+I+h7gnUuCJatN8ZkvlR9BLgoWK7lYpXqhz7JqkYtEfkF8B/GmC2H+VSUGpO0gVdKqSKlXTRKKVWktIFXSqkipQ28UkoVKW3glVKqSGkDr5RSRUobeKWUKlL/H3yaTwPVeK9mAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n",
    "plt.xlabel('Depth')\n",
    "plt.xlim((0, 512))\n",
    "plt.ylabel('Position')\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 三、遮挡（Masking）：遮挡序列中满足特定条件的标记\n",
    "1. 遮挡值为 0 的标记\n",
    "2. 前瞻遮挡（look-ahead mask）用于遮挡一个序列中的后续标记（future tokens）。\n",
    "   \n",
    ">例如序列生成中：   \n",
    "当要预测第三个词时，将仅使用第一个和第二个词，后续所有词被遮挡；     \n",
    "与此类似，预测第四个词，仅使用第一个，第二个和第三个词，依此类推。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=56, shape=(3, 1, 1, 5), dtype=float32, numpy=\n",
       "array([[[[0., 0., 1., 1., 0.]]],\n",
       "\n",
       "\n",
       "       [[[0., 0., 0., 1., 1.]]],\n",
       "\n",
       "\n",
       "       [[[1., 1., 1., 0., 0.]]]], dtype=float32)>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_padding_mask(seq):\n",
    "    # 将 seq 序列中为 0 的位置标记为 1，其它的位置标记为 0\n",
    "    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
    "    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)\n",
    "\n",
    "# 增加到 4 维，是因为注意力权重形状为 [batch_size, num_heads, seq_len, seq_len]\n",
    "\n",
    "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n",
    "create_padding_mask(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[0. 1. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 0.]], shape=(3, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "def create_look_ahead_mask(size):\n",
    "    # tf.linalg.band_part: 以对角线为中心，取指定带宽的上三角和下三角，其它数据 0 填充\n",
    "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
    "    return mask\n",
    "\n",
    "\n",
    "x = tf.random.uniform((1, 3))\n",
    "print(create_look_ahead_mask(x.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 四、按比例缩放的点积注意力(Scaled dot product attention)\n",
    "- 原理如下图所示：\n",
    "![](images/scaled_attention.png)\n",
    "注意力函数有三个输入：Q（请求（query））、K（主键（key））、V（数值（value））.用于计算注意力权重的等式为：\n",
    "   \n",
    "$$Attention(Q,K,V)=softmax(\\frac{QK^T}{\\sqrt{d_k}})V $$\n",
    "    \n",
    "> 点积注意力被缩小了深度的平方根倍 $\\sqrt{d_k}$。这样做是因为对于较大的深度值，点积的大小会增大，从而推动 softmax 函数往仅有很小的梯度的方向靠拢，导致了一种很硬的（hard）softmax。\n",
    "\n",
    "- 在机器翻译中，点积注意力的用法如下所示\n",
    "\n",
    "\n",
    "![](images/scaled-dot-product-attention.png)\n",
    "\n",
    "> 图中向量 $q_t$ 与 $K=[k_1, k_2, k_3, k_4]$ 每个向量做点积，然后转换为权重，权重值越大可以认为越“关注”该处的信息，然后每个位置的 $v$ 与该处的权重相乘，然后求和得到 $q_t$ 对 $K$ 做注意力的输出向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scaled_dot_product_attention(q, k, v, mask):\n",
    "    \"\"\"\n",
    "    :param q: (batch_size, seq_len_q, depth)\n",
    "    :param k: (batch_size, seq_len_k, depth)\n",
    "    :param v: (batch_size, seq_len_v, depth_v)\n",
    "    :param mask: (batch_size, seq_len_q, deq_len_k)\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
    "    # (..., seq_len_q, seq_len_k)\n",
    "\n",
    "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "    \n",
    "    print(\"shape of mask:\", mask.shape)\n",
    "    print(\"shape of atten:\", scaled_attention_logits.shape)\n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += (mask * -1e9)\n",
    "        # 在待遮挡处减去一个较大值，softmax 即可忽略该位置\n",
    "\n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
    "    # (..., seq_len_q, seq_len_k)\n",
    "\n",
    "    output = tf.matmul(attention_weights, v)\n",
    "    # (..., seq_len_q, depth_v)\n",
    "\n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_out(q, k, v):\n",
    "    temp_out, temp_attn = scaled_dot_product_attention(q, k, v, None)\n",
    "    print('Attention weights are:')\n",
    "    print(temp_attn)\n",
    "    print('Output is:')\n",
    "    print(temp_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention weights are:\n",
      "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
      "Output is:\n",
      "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True)\n",
    "\n",
    "temp_k = tf.constant([[10, 0, 0], [0, 10, 0], [0, 0, 10], [0, 0, 10]],\n",
    "                     dtype=tf.float32)  # (4, 3)\n",
    "\n",
    "temp_v = tf.constant([[1, 0], [10, 0], [100, 5], [1000, 6]],\n",
    "                     dtype=tf.float32)  # (4, 2)\n",
    "\n",
    "# 这条 `请求（query）符合第二个`主键（key）`，\n",
    "# 因此返回了第二个`数值（value）`。\n",
    "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
    "print_out(temp_q, temp_k, temp_v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 五、多头注意力层（Multi-head attention）\n",
    "- 多头注意力如下图所示：\n",
    "![多头注意力](../images/multi_head_attention.png)\n",
    "> Q、K、和 V 被拆分到了多个头，允许模型共同注意来自不同表示空间的不同位置的信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "\n",
    "        assert self.d_model % self.num_heads == 0\n",
    "\n",
    "        self.depth = d_model // self.num_heads\n",
    "\n",
    "        self.wq = tf.keras.layers.Dense(self.d_model)\n",
    "        self.wk = tf.keras.layers.Dense(self.d_model)\n",
    "        self.wv = tf.keras.layers.Dense(self.d_model)\n",
    "\n",
    "        self.dense = tf.keras.layers.Dense(d_model)\n",
    "\n",
    "    def split_heads(self, x, batch_size):\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(x, perm=(0, 2, 1, 3)) \n",
    "        #  (batch_size, num_heads, seq_len, depth)，便于计算\n",
    "\n",
    "    def call(self, v, k, q, mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "\n",
    "        q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
    "        k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
    "        v = self.wv(v)\n",
    "\n",
    "        q = self.split_heads(\n",
    "            q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
    "        k = self.split_heads(k, batch_size)\n",
    "        v = self.split_heads(v, batch_size)\n",
    "\n",
    "        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
    "        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
    "        scaled_attention, attention_weights = scaled_dot_product_attention(\n",
    "            q, k, v, mask)\n",
    "\n",
    "        scaled_attention = tf.transpose(\n",
    "            scaled_attention,\n",
    "            perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
    "        concat_attention = tf.reshape(scaled_attention,\n",
    "                                      (batch_size, -1, self.d_model))\n",
    "\n",
    "        output = self.dense(\n",
    "            concat_attention)  # (batch_size, seq_len_q, d_model)\n",
    "        return output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n",
    "y = tf.random.uniform((1, 60, 512))  # (batch_size, encoder_sequence, d_model)\n",
    "out, attn = temp_mha(y, k=y, q=y, mask=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out.shape, attn.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 六、点式前馈网络（Point wise feed forward network）\n",
    "由两层全联接层组成，两层之间有一个 ReLU 激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 50, 512)\n"
     ]
    }
   ],
   "source": [
    "def point_wise_feed_forward_network(d_model, dff):\n",
    "    return tf.keras.Sequential([\n",
    "        tf.keras.layers.Dense(dff, activation='relu'),\n",
    "        # (batch_size, seq_len, dff)\n",
    "        tf.keras.layers.Dense(d_model)\n",
    "        # (batch_size, seq_len, d_model)\n",
    "    ])\n",
    "\n",
    "\n",
    "sample_ffn = point_wise_feed_forward_network(512, 2048)\n",
    "print(sample_ffn(tf.random.uniform((64, 50, 512))).shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 七、编码与解码（Encoder and decoder）\n",
    "Transformer 模型与标准的具有注意力机制的序列到序列模型（sequence to sequence with attention model），遵循相同的一般模式。其结构如下所示：\n",
    "![transformer](images/transformer.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. 编码器层（Encoder layer）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "\n",
    "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, x, training, mask):\n",
    "        attn_output, _ = self.mha(x, x, x,mask)  # (batch_size, input_seq_len, d_model)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n",
    "\n",
    "        ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
    "\n",
    "        return out2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 43, 512)\n"
     ]
    }
   ],
   "source": [
    "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n",
    "\n",
    "sample_encoder_layer_output = sample_encoder_layer(\n",
    "    tf.random.uniform((64, 43, 512)), False, None)\n",
    "\n",
    "print(sample_encoder_layer_output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. 解码器层（Decoder layer）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "        \n",
    "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
    "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
    "        \n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "        \n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        \n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
    "\n",
    "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
    "        attn1 = self.dropout1(attn1, training=training)\n",
    "        out1 = self.layernorm1(attn1 + x)\n",
    "\n",
    "        attn2, attn_weights_block2 = self.mha2(\n",
    "            enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
    "        attn2 = self.dropout2(attn2, training=training)\n",
    "        out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
    "\n",
    "        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
    "        ffn_output = self.dropout3(ffn_output, training=training)\n",
    "        out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
    "\n",
    "        return out3, attn_weights_block1, attn_weights_block2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 50, 512)\n"
     ]
    }
   ],
   "source": [
    "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n",
    "\n",
    "sample_decoder_layer_output, _, _ = sample_decoder_layer(\n",
    "    tf.random.uniform((64, 50, 512)), sample_encoder_layer_output, False, None,\n",
    "    None)\n",
    "\n",
    "print(sample_decoder_layer_output.shape)  # (batch_size, target_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3. 编码器\n",
    "\n",
    "- 输入嵌入（Input Embedding）\n",
    "- 位置编码（Positional Encoding）\n",
    "- N 个编码器层（encoder layers）\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "    def __init__(self,\n",
    "                 num_layers,\n",
    "                 d_model,\n",
    "                 num_heads,\n",
    "                 dff,\n",
    "                 input_vocab_size,\n",
    "                 maximum_position_encoding,\n",
    "                 rate=0.1):\n",
    "        super(Encoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding,\n",
    "                                                self.d_model)\n",
    "\n",
    "        self.enc_layers = [\n",
    "            EncoderLayer(d_model, num_heads, dff, rate)\n",
    "            for _ in range(num_layers)\n",
    "        ]\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, x, training, mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x = self.enc_layers[i](x, training, mask)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 62, 512)\n"
     ]
    }
   ],
   "source": [
    "sample_encoder = Encoder(num_layers=2,\n",
    "                         d_model=512,\n",
    "                         num_heads=8,\n",
    "                         dff=2048,\n",
    "                         input_vocab_size=8500,\n",
    "                         maximum_position_encoding=10000)\n",
    "\n",
    "sample_encoder_output = sample_encoder(tf.random.uniform((64, 62)),\n",
    "                                       training=False,\n",
    "                                       mask=None)\n",
    "\n",
    "print(sample_encoder_output.shape)  # (batch_size, input_seq_len, d_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4. 解码器（Decoder）\n",
    "- 输出嵌入（Output Embedding）\n",
    "- 位置编码（Positional Encoding）\n",
    "- N 个解码器层（decoder layers）\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "    def __init__(self,\n",
    "                 num_layers,\n",
    "                 d_model,\n",
    "                 num_heads,\n",
    "                 dff,\n",
    "                 target_vocab_size,\n",
    "                 maximum_position_encoding,\n",
    "                 rate=0.1):\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding,\n",
    "                                                d_model)\n",
    "\n",
    "        self.dec_layers = [\n",
    "            DecoderLayer(d_model, num_heads, dff, rate)\n",
    "            for _ in range(num_layers)\n",
    "        ]\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "        attention_weights = {}\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
    "                                                   look_ahead_mask,\n",
    "                                                   padding_mask)\n",
    "            attention_weights['decoder_layer{}_block1'.format(i + 1)] = block1\n",
    "            attention_weights['decoder_layer{}_block2'.format(i + 1)] = block2\n",
    "\n",
    "        return x, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 26, 512) (64, 8, 26, 62)\n"
     ]
    }
   ],
   "source": [
    "sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8, dff=2048,\n",
    "                         target_vocab_size=8000, maximum_position_encoding=5000)\n",
    "\n",
    "output, attn = sample_decoder(tf.random.uniform((64, 26)),\n",
    "                              enc_output=sample_encoder_output, training=False,\n",
    "                              look_ahead_mask=None, padding_mask=None)\n",
    "\n",
    "print(output.shape, attn['decoder_layer2_block2'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 八、Transformer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Transformer(tf.keras.Model):\n",
    "    def __init__(self,\n",
    "                 num_layers,\n",
    "                 d_model,\n",
    "                 num_heads,\n",
    "                 dff,\n",
    "                 input_vocab_size,\n",
    "                 target_vocab_size,\n",
    "                 pe_input,\n",
    "                 pe_target,\n",
    "                 rate=0.1):\n",
    "        super(Transformer, self).__init__()\n",
    "        self.encoder = Encoder(num_layers, d_model, num_heads, dff,\n",
    "                               input_vocab_size, pe_input, rate)\n",
    "        self.decoder = Decoder(num_layers, d_model, num_heads, dff,\n",
    "                               target_vocab_size, pe_target, rate)\n",
    "        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
    "\n",
    "    def call(self, inp, tar, training, enc_padding_mask, look_ahead_mask,\n",
    "             dec_padding_mask):\n",
    "        enc_output = self.encoder(inp, training, enc_padding_mask)\n",
    "        dec_ouput, attention_weights = self.decoder(tar, enc_output, training,\n",
    "                                                    look_ahead_mask,\n",
    "                                                    dec_padding_mask)\n",
    "        final_output = self.final_layer(dec_ouput)\n",
    "        return final_output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 26, 8000)\n"
     ]
    }
   ],
   "source": [
    "sample_transformer = Transformer(num_layers=2,\n",
    "                                 d_model=512,\n",
    "                                 num_heads=8,\n",
    "                                 dff=2048,\n",
    "                                 input_vocab_size=8500,\n",
    "                                 target_vocab_size=8000,\n",
    "                                 pe_input=10000,\n",
    "                                 pe_target=6000)\n",
    "\n",
    "temp_input = tf.random.uniform((64, 62))\n",
    "temp_target = tf.random.uniform((64, 26))\n",
    "\n",
    "fn_out, _ = sample_transformer(temp_input,\n",
    "                               temp_target,\n",
    "                               training=False,\n",
    "                               enc_padding_mask=None,\n",
    "                               look_ahead_mask=None,\n",
    "                               dec_padding_mask=None)\n",
    "\n",
    "print(fn_out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 九、配置超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_layers = 4  \n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "\n",
    "input_vocab_size = len(zh_tokenizer.word_index) + 1\n",
    "target_vocab_size = len(eng_tokenizer.word_index) + 1\n",
    "dropout_rate = 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 十、优化器（Optimizer）\n",
    "将 Adam 优化器与自定义的学习速率调度程序（scheduler）配合使用: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "        self.warmup_steps = warmup_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps**-1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(d_model)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate,\n",
    "                                     beta_1=0.9,\n",
    "                                     beta_2=0.98,\n",
    "                                     epsilon=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAECCAYAAADpdjDfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3zU5Zn//9eVc0hCCORAOBOOosjBiKCoeKz2qNSf2sO2Vl2/23Pr9rG1u7T99bt2v3UPdqtb29q6rtrard1a/arVaqsIVVBBQURBSDgfzEQgISHhlOv7x0xCCJPkM8lMZpK8n49HHnzm5p7PXDOOXLk/9/25bnN3REREgkpLdgAiItK/KHGIiEhMlDhERCQmShwiIhITJQ4REYlJRrIDSLTi4mKfMGFCssMQEelXVq9eXevuJdH+bsAnjgkTJrBq1apkhyEi0q+Y2bbO/k6XqkREJCZKHCIiEhMlDhERiYkSh4iIxESJQ0REYqLEISIiMUlI4jCz+8xshZktiaVPJ21lZrY8yvPPMLPn4h+9iIh0Je6Jw8wWA+nuvgCoMLMpQfp00lYEPADkdXi+AXcCmfGOPxUs3VjD5pqGZIchIhJVIkYci4BHIsfPAgsD9onWdhy4Dqjv8PzPAS90FoCZ3WJmq8xsVSgUivkNJJO7c8P9r3HpnS8mOxQRkagSkTjygF2R431AWcA+p7S5e72717V/opmNAD4N/GtnAbj7ve5e6e6VJSVR75hPWTUHD7cdHzh0JImRiIhEl4jE0QDkRo7zO3mNaH2CPA/gB8C33P1oXKJNMVWhE5eonnv7vSRGIiISXSISx2pOXJ6aBWwN2CfI8wAuBO4ws6XAbDO7vbcBp5KqUCMAaQZPv7U3ydGIiJwqEUUOHwOWm9ko4ErgejO73d2XdNFnPuBR2k7h7lNbj81saYfz9nvVoQZyM9P5xLxx/HLlNuqbjzI0Z0CuARCRfiruIw53ryc80b0SuMjd13b8xz1Kn7pobe36L+rktaK292dVoUYqSvL40JkjOXK8heffqUl2SCIiJ0nIfRzuvt/dH3H3Tq+1ROsT5HkDXXWogYqSfOaMLaJsaDZPv7Un2SGJiJxEd46nkOajx9l1oIlJJXmkpRlXnlHO0o0h6psH5DoAEemnlDhSyJbaRtyhoiQfgI/NHsXhYy08s27QDsBEJAUpcaSQ6siKqkkl4RvlZ48dxsTiPH7/xq6uniYi0qeUOFJI6z0cE4vDicPMuGr2aFZueZ/dB5qSGZqISBsljhRSHWpgVGEOQ7JOrJK+es5o3OGxNRp1iEhqUOJIIdW1jUwqzT+pbdyIIZw1vojfv74Ld09SZCIiJyhxpAh3p6qmgYrivFP+7uo5o9lU08D63R1rPYqI9D0ljhRRc/AwjUeOnzLiAPjIrFFkZ6Tx61e3JyEyEZGTKXGkiKrI/hsVxacmjsLcTD585igeX7ObxsPH+jo0EZGTKHGkiKra8FLcipJTL1UBfPKcsTQcPsYTa3f3ZVgiIqdQ4kgRVTUNDMlKZ+TQnKh/P3dcEdPKCnhYl6tEJMmUOFJEdW0jE4vDpUaiMTM+MW8sb+6s461ddVH7iIj0BSWOFFEdamBSyanzG+1dPXcM2Rlp/OqVbX0UlYjIqZQ4UkBrccPO5jdaFeZmctXs0fz+jV3sb9S2siKSHEocKaC1uGF3Iw6AGxdOpPloi+Y6RCRplDhSQGuNqu5GHADTRhZw/pRiHnh5K0eOtSQ6NBGRUyhxpIDWqrgTo9w1Hs1NCydSc/AwT76ppbki0veUOFJAdaiB0cNyTypu2JULp5YwuTSf+/6yRfWrRKTPKXGkgNZ9xoMyM248byLrd9fzctX7CYxMRORUCUkcZnafma0wsyWx9OmkrczMlrd7PM7MlprZ82Z2r5lFv/Ghn3D3QEtxO1o8dzSlBdnc/fymBEUmIhJd3BOHmS0G0t19AVBhZlOC9OmkrQh4AGj/6/j/Aj7v7hcDY4GZ8X4Pfem9+nBxw1hGHAA5mencckEFK6v38drWfQmKTkTkVIkYcSwCHokcPwssDNgnWttx4DqgrZ64u/+Du78TeTgCqO14cjO7xcxWmdmqUCjUi7eSeNWhzosbdudT54xnRF4Wdz+/Od5hiYh0KhGJIw9o3a5uH1AWsM8pbe5e7+5R62uY2XXAenc/ZWmRu9/r7pXuXllSUtLzd9IHWpfiTiqNbcQBkJuVzs3nV7Ds3RBrdhyId2giIlElInE0ALmR4/xOXiNanyDPA8DMKoBvAF+LQ7xJVRVq7LK4YXf+asF4CnMzufvPmusQkb6RiMSxmhOXp2YBWwP2CfI8IvMevwZu7Gw00p9U14ZXVPV0jj8/O4O/Pn8if95Qw+ptmusQkcRLROJ4DPgrM7sTuBZYb2a3d9PnqU7aorkNGAfcHVlddWEC3kOfCW8XG/v8Rns3LpxISUE2dzy9Ufd1iEjCxT1xuHs94YnulcBF7r7W3Zd006cuWlu7/ovaHX/T3cvdfVHk58V4v4e+0nTkOLvrmmJeitvRkKwMvnLJFF7duo8XNtbEKToRkegSch+Hu+9390fcfW8sfYI8byBpLW4Y61LcaK4/eywTRgzhn5/ZyPEWjTpEJHF053gSVdcGL27Yncz0NP728mls2HuQx97Y1f0TRER6SIkjiVqLG/Z2jqPVh2aWM2tMIXc8s4GGw8fick4RkY6UOJKoKlLcMDcrPS7nS0szvvvR06k5eJj/0E2BIpIgShxJVB1jccMg5o4r4uNzx3DfX6rZUtsY13OLiIASR9L0tLhhEN+8chrZGen845Nvx/3cIiJKHEnS0+KGQZQW5PDVS6bw/IYa/vT2e3E/v4gMbkocSdJWoyoBIw6Az547gall+Xz78bc0US4icaXEkSTVMewz3hNZGWn84ONnsre+mX9+ZkNCXkNEBicljiTpbXHDIOaOK+KGcyfw0MptrNKeHSISJ0ocSVIVauhVccOgvnH5NEYV5nLbo+s4fOx4Ql9LRAYHJY4kqQ41Jmx+o7287Az+afFMNtc08MPnVHpdRHpPiSMJmo4cZ9eBprjdMd6dC6eW8Il5Y/nZsipeqX6/T15TRAYuJY4kaL0xL1ET49Es+dAMxg8fwq2PrKW++Wifva6IDDxKHEnQWtywLy5VtcrLzuCH181mb30z3318fZ+9rogMPEocSVBVEx5xTCzuuxEHwJxxRXz54sn8/o1dPL5GFXRFpGeUOJKguja+xQ1j8aWLJlM5voi/f3Qdm2sa+vz1RaT/U+JIgtaluMmQkZ7Gf3xyLjmZ6Xz+l6s5dER3lYtIbJQ4+li4uGHfLMXtzMjCHH50/Rw2hxr4+0fXaZ9yEYmJEkcf21vfzKEjx5mUpBFHq4VTirn10qk8tmY3v3xle1JjEZH+JSGJw8zuM7MVZrYklj6dtJWZ2fJ2jzPN7Akze8nMbkxE/InUtutfEkccrb540WQumlbC9/7velZU6f4OEQkm7onDzBYD6e6+AKgwsylB+nTSVgQ8ALT/9fzLwGp3Pw+4xswK4v0eEqk6wVVxY5GWZvzoE3OYUJzH53+1mq3a+ElEAkjEiGMR8Ejk+FlgYcA+0dqOA9cB9Z08dxlQ2fHkZnaLma0ys1WhUKhn7yJBqkKN5GWlUzY0O9mhADA0J5P7PluJATc98Bp1Tbo5UES6lojEkQe03iSwDygL2OeUNnevd/e6WM/v7ve6e6W7V5aUlPT4jSRCeEVVfsKLG8Zi/Ig8fvrps9i+7xBfevh1jh5vSXZIIpLCEpE4GoDcyHF+J68RrU+Q5wU9f8pKxD7j8XBOxQi+f9VMlm+q5bbfaaWViHQuEf/orubE5alZwNaAfYI8L+j5U1JfFzeM1bVnj+XWy6byu9d38oOntfmTiESXkYBzPgYsN7NRwJXA9WZ2u7sv6aLPfMCjtEXzAPAHMzsfmAG8koD3kBCtxQ0nlabeiKPVly+eTG3DYX62rJri/Gz++oKKZIckIikm7iMOd68nPIG9ErjI3dd2SBrR+tRFa2vXf1G7423AZcBLwKXu3m92J2rdZzxVRxwAZsZ3P3I6H5pZzvf/8A6/XbUj2SGJSIpJxIgDd9/PiZVPgfsEeV6k3+4g/VJNdagRs74vbhir9DTjzutmUdd0lL/73ZtkpBtXzxmT7LBEJEX0q4nl/q4q1MCowuQUN4xVdkY6P/9MJfMnjuBvH1mraroi0kaJow9V1yavuGFP5Galc98NlcybOJyv/2YNT6zdneyQRCQFKHH0kVQobtgTQ7Iy+M8bzqZywnC+9ps1GnmIiBJHX0mV4oY9MSQrg/tvOJuzJxTxtd+s4cEVW5MdkogkkRJHH2ktbtjfRhyt8rIz+K/PzeOS6WV85/H13PXnTbpJUGSQCpQ4zKzIzE43s3IzU7LpgbaluP00cQDkZKbz00/PZfHc0dz53Lv87yffpqVFyUNksOl2Oa6ZfRO4GhgC3AF8APhMguMacKpTrLhhT2Wkp/Gv18yiMDeT+1/aSk39Yf7t2lnkZKb+SjERiY8go4ePuPt84H13/xWgW4l7IBWLG/ZUWprxnQ/P4FtXTucPb+3h+ntXEjp4ONlhiUgfCZI46s3sM0COmV0IHEhwTANSqhY37Ckz439dOImffGouG/bWc/U9L7HpvYPJDktE+kCQxHEDMAfYD3wMuCmRAQ1ErcUN++vEeFeuOKOc39yygMPHWlh8z8u8sLEm2SGJSIJ1mzjcvcbdv+7uH3T3Wzl5Nz4JoLq2dWJ8YH50s8YO47EvnseY4UO48b9e464/b9KkucgA1m3iMLOHOjT9MkGxDFj9fSluEKOH5fLo58/lqtnhFVd//eAq7SYoMkB1mjjMbFxkTuN0M7sg8nMloH8NYlQVaugXxQ17KzcrnTuvncX3Pno6L74b4qP/8Rfe2VPf/RNFpF/pasQxkXCZ86LInxcBM4EbEx7VAFMdamRUYe6gWLJqZnz23An89y3zaTpynI/9+CUeXLFVNwuKDCCd3sfh7i8CL5rZeHf/330Y04BTFWpgUunAvUwVTeWE4Tz1lfP5xm/X8p3H17Ps3Vr+5ZozKcrLSnZoItJLQSbHTxphmFl54sIZeNydLbWNVAzwy1TRlBRkc/8NZ7PkQ6fx4rs1XPGjZbxcVZvssESkl4JMjv+jma01syozqwL+2AdxDRhtxQ0H2YijVVqacfP5Ffz+C+eRl53Bp37xCt9/6m2aj/abjRtFpIMg93FcAJwLvAqcCYQSGtEAU1UTWVE1CEcc7Z0xupAnv7yQT84bx8+Xb+GDP1rO6m37kh2WiPRAkMSRBswC8gknjpKERjTAtN7DMVhHHO0Nycrg+1fP5Fc3n8PhYy1c89MV/OOTb9N0RKMPkf4kSOK4FjgCfBv4PPCPCY1ogKmqaSAvK53Sgv5d3DCezptczB+/fgGfOmcc9/1lCx+8azkvbdbch0h/0dV9HOlm9gFguruvcvc1hMuPdLuu0szuM7MVZrYklj5B2iIl3v9gZqvM7GfB3mbyVNc2DpjihvGUn53B7VfN5OGbz+F4i/OpX7zCV//7DWoONic7NBHpRlcjjoeB64DPm9ldZvZVYC2wsKsTmtliIN3dFwAVZjYlSJ+gbcBfAb9y90qgwMwqY3/bfSe8Xezgnt/oyrmTi3n26xfwlUum8PS6vVzyby/y4IqtHFfJEpGU1VXiGBtZinsd8FEgGzjf3b/WzTkXAY9Ejp8leqKJ1ido2/vAGWY2DBgL7Oh4cjO7JTIiWRUKJW8u/9CRY+w60NSvN2/qCzmZ6dx62VSe+dr5zBozjO88vp6rfvySJs9FUlRXiSPHzBYAC4B9wF+AGWZ2bjfnzAN2RY73AWUB+wRt+wswHvgK8E6k/STufq+7V7p7ZUlJ8ubyt9QO/BpV8VRRks9DN83jrk/M4b36Zj7+kxV88eHX2bHvULJDE5F2utoBcC1wS7vjv44cO/ByF89rAHIjx/lET07R+gRt+y7wN+5eb2a3Ap8D7u0inqSpihQ3HKhVcRPBzPjorFFcelopP3uxmp8tq+K5t9/jpoUT+cKiSRTkZCY7RJFBr6uSI5/r4TlXE76ktJLwMt6NAfvsDNh2NjDTzFYC5wB/6mGcCVc9SIobJsKQrAy+ftlUrp83ln/540Z+srSKR17bwdcuncJ1Z48jKyPIgkARSYRu9xzvgceA5WY2CrgSuN7Mbnf3JV30mU94JBOkbTNwP+HLVSuAXyfgPcRFVaiR0cMGR3HDRCkvzOXOa2dzw7kTuP2pd/j24+v52bJqvnbpVK6aPYqMdCUQkb5miahaamZFwGXAMnffG7RP0LZYVFZW+qpVq3r2RnrpQ3ctZ0R+Ng/eOC8prz/QuDsvvhvi3559l3W76phUksfXL5vKB88oJy1Ny51F4snMVkdWr54iESMO3H0/J1ZDBe4TtK0/aGlxqkONzJs4PNmhDBhmxqJppVw4tYQ/rn+PO5/byJcefoPTyqv46iWTuXzGSCUQkT6gcX6C7K1vpunocS3FTQAz44ozRvL0Vy/g36+bTdORY/zNL1/n8n9fxqOv7+To8ZZkhygyoAWpjntbh8czzOyCxIU0MJzYLlYT44mSnmZcNWc0f7r1Qu76xBwy0oxbH1nLon9ZykMrtqoCr0iCBBlxzDSzlWZ2feTxt4FvJDCmAaEqFCluqBFHwmWkp/HRWaN4+qvnc99nKykdms23H1/Pwjte4McvbGZ/45FkhygyoASZ46ggvCT2ReC/gVLCRQ+lC9UhFTfsa2bGJaeVcfH0UlZW7+OepZv5lz9u5O7nN7F47hg+d+4EppQVJDtMkX4vSOLYD/yY8J3kHwOmEv3eDGmnKtTIpFIVN0wGM2PBpBEsmDSCDXvruf8vW/mf1Tt5+JXtXDC1hBvPm8AFU0o0kS7SQ0ESx2JgOuHSH1dw4n4K6UJ1qEErqlLA9JFDueOaM/m7K6bx8CvbeXDlNm64/zUmleTx6fnjWTxnDIVDdDe6SCyCzHEMJZw4roz0n+vuv0hoVP3coSPH2F3XrPmNFDIiP5svXzKFl755MT+8bhb52Rl874m3Oef//Im/fWQtq7ftJxH3NIkMREFGHM8Aj3KiCq3G992obqtRpcSRarIy0rh6zhiunjOGt3bV8fCr23n8jV387vWdTB9ZwCfPGcdVc0YzVDWxRDoVJHEcdPfbEx7JAFLdWhW3VEtxU9kZowv5p6tn8vcfPI3/u2Y3D7+6je88vp5/+sM7XHH6SD5+1hjOnVRMuuZCRE4SJHEsN7NfAw8CjQDuviyhUfVzVTXh4oYTRihx9Af52Rl88pxxfPKccby58wC/eW0HT6zdzWNrdjNyaA5XzRnNx+eO1ooskYggieMosAFoLbjkgBJHF6prVdywvzpzzDDOHDOMb394Bs9vqOF3q3fy8+XV/PTFKmaOLuTjc0fzkVmjGJGvZdYyeCWkyGEqSUaRww/dtZzi/GweUHHDAaG24TCPr9nNo6/vZP3uetLTjAUVI/jwmeV84PSRFOVlJTtEkbjr8yKHg5mKGw48xfnZ3LRwIjctnMiGvfU8uXYPT765m9seXceSx97ivMnFfOjMcj4wY6SW9sqg0GniMLM73f1WM3uB8OUpCK+ocne/uE+i64daixtqKe7ANH3kUKaPHMrfXj6V9bvrefLNcBL5u/95k39IX8f5U0r44MxyLpleqpGIDFhd7QB4a+TPi/ounP6vtUaVtosd2MyMM0YXcsboQr55xTTW7qzjqTd389Sbe3h+Qw1pBmdPGM5lM8q4fMZIxo0YkuyQReJGl6ri7ERVXI04BgszY/bYYcweO4xvXXka63bV8dzb7/Hc2+9x+1PvcPtT7zB9ZEFbEjlj9FCVopF+rdvEYWalwMVA27jb3R9MZFD9WVWogfzsDBU3HKTS0oxZY4cxa+wwvvGBaWx7v5Hn3n6PZ99+jx+/sJm7n99MeWEOi6aVcOHUUs6bPIIC3Wwo/UxP7hyXLlSHGqkoydNvlALA+BF53Hx+BTefX8G+xiM8v6GGP739Hk+s3cOvX91BRppx1viitp0NTysv0HdHUp7uHI+z6lAD51SMSHYYkoKG52VxzVljuOasMRw93sLqbft58d0QSzeGuOOZDdzxzAZKC7K5cGoJi6aVcu6kEZpgl5SUkDvHzew+YAbwVGdJJ1qfoG2R9nuAp939iQDvoU+0FjesKNbEuHQtMz2N+RUjmF8xgm9eMZ336pt58d0QL24M8cz6vfx29U7MYEb5UM6dNIJzJxczb8Jw8rI1LSnJF/c7x81sMZDu7gvM7D/NbIq7b+quDzAzSJu7bzKz84GRqZQ0oN3EeKkmxiU2ZUNzuLZyLNdWjuXY8RbW7jzAS5vf5+WqWh54eRs/X76FjLTwJPy5k4s5d9II5owbRnaGqhNI3+s2cbj792I85yLgkcjxs4R3D9wUoM+cIG1mthX4OfAHM/uYuz/eMQAzuwW4BWDcuHExht9zWoor8ZCRnsZZ44dz1vjhfOWSKTQdOc7qbft5qaqWl6ve5z+e38Rdf95ETmYaleOHM2/icConFDFnbBG5WUokknhBVlU97e5XxnDOPMKbPgHsA+YG7BO07TPA28A/A182s3Hufnf7k7v7vcC9EC45EkPsvVIdalRxQ4m73Kx0Fk4pZuGUYgDqmo7y6pZ9vFxVy4qq9/nhn97FHTLTw/eWzJswnLMnhJPJsCGaI5H4C3Kpal1nv9l3ogHIjRznE32zqGh9grbNAe51971m9kvg+8BJiSNZqmsbGVOk4oaSWIW5mVw2o4zLZpQB4UTy+rb9vLp1H69t2cf9L23lZ8uqAZhals/ZE8KjkrnjihhTlKtVW9JrQRLH2YR/s19HeHK8u5IjqwlfZloJzCL6/uTR+uwM2HYIqIicpxLYFuA99ImqmgYqijW/IX2rMDeTi6aXctH0UgCajx5n7Y4DvLZ1H69t3c/ja3bzq1e2A1Ccn8XsscOYM66IOWOHcebYYeRrwl1iFGSOI9aSI48RXok1ivB2s9eb2e3uvqSLPvMJT7oHaWsB/tPMrgcygWtijC8hWlqcLbWNzNdSXEmynMx0zqkY0bYs/HiLs2FvPW9sP8Ab2w+wZsd+/vRODQBmMLW0IJJMhjF73DCmlBZo8yrpUqCy6mZWwolLRqPdfUU3/YuAy4Bl7r43aJ+gbbHoq7Lquw40cd4Pnuf2q87g0/PHJ/z1RHqj7tBR1uw8wJrtB3hjx37W7DjAgUNHAcjLSueM0YXMHF3IzDGFnD6qkIriPNKUTAaVXpVVj9xHMREoInyZyAlfPuqUu+/nxGqowH2CtqWiaq2okn6kcEgmF04t4cKpJQC4O1vfP8Qb2/fzxvYDrNtVx0Mrt3H4WAsQTianjyrk9NFDwwlldCEVJfkamQxSQS5uTib8G/+vgE8Azyc0on6qqiacOCaruKH0Q2bGxOI8JhbnsXjuGACOHm9hc00D63bVsX5XHet21fHrV7dz/9FwMsnNTGfGqHAimVE+lOnlBUwtK9DikEEgSOI4BFwCpAP/H+GRh3RQXdtIfnYGJSpuKANEZnoap5UP5bTyoVA5FoBjx1uoCjXyViSRvLWrjt+8toOmo8cBSDOYWJzH9PKhnDayILx/SXkBo4dpNddAEiRxXAOUA18HbgK+kNCI+qnqUCOTVNxQBriM9DSmjSxg2sgCPn5WeGRyvMXZvu8QG/bU887eg2zYU8+6nXU89eaetucV5GRwWiSJtCaTaWUFKqHSTwVZVdVoZjnAJODXqEpuVFWhBq2okkEpPe3EZa4rZ5a3tTccPsbGvfW8s+cgG/bWs2HPQR59fRcNh0+soB9TlMuU0nymlBUwuTS/7WeoSs2ntCCT43cDowhPkH8buAP4aILj6lcaDx9jT10zkzQxLtImPzujrXRKK3dn5/4mNkRGJptqGthU08BLVe9zJDIRDzByaA5TyvKZVJLPlLJ8ppQWMKU0X9WCU0SQceJMd19kZs+7+1Nm9ncJj6qf2VIbLm5YoYlxkS6ZGWOHD2Hs8CFtd75D+HLXjn2H2FTTwOaaBjbVHGRzTQOPrNrBoSPH2/qNyMtqG5VUlORTERnpjCnKJSM9WpEKSYQgiSNkZt8Biszss0DM91IMdCpuKNI76WnGhOI8JhTnnZRQWlqc3XVNbKppoKqmgU3vhZPKE2t3U998rK1fRpoxbsSQtkQysTificV5VJTkUVqQrbnHOAuSOD5DuNLsCqAQuCGRAfVHKm4okhhpacaYoiGMKRrCRdNK29rdnf2HjrKltoHqUCNbak/8LN9U23b/CcCQrPS2OZiKSHIaPyI86inJV1LpiSCT403Aj1ofm9kXgHsSGVR/UxVqUHFDkT5kZgzPy2J43slzKBAepeypb2ZLqDGcWCIJZd2uOv6wbg8t7Ypl5GamMy5y6Wz8iCGMGz6EcZE/xxTlar+TTvRkLdwNKHGcJLwUV/MbIqkgLc0YPSyX0cNy20rRtzpyrIUd+w+xfd8htr8f/nPb+4fYse8QL22ubbsfBcJ1vMqH5rQlkvEj8hg7/ERSGZGXNWhHK1pE3UstLU51rZbiivQHWRlpTCrJj/qLnrsTajh8SkLZtu8QL2wMETq486T+OZlp4QRVNITRw3IZUxT+CbflUlqQM2BLsnSaOMzsk9GageFR2getPfXNNB9t0cS4SD9nZpQW5FBakEPlhFP/mTt05Bg79jWx7f1Gdh1oYtf+JnYdaGLn/ibe2lXHvsYjJ/XPTDfKC3Pbksrootbj8IhlZGEOmf10JVhXI44pnbQ/lIhA+qvWGlW6VCUysA3Jymi7az6aQ0eOsftAEzv2n5xUdu0/xLJNIWoOHqZ9MfI0g9KCHEYW5jBqWA4jh+ZSXtjucWEupQXZKZlcOk0cPdhrfFBqrYqrm/9EBrchWRlMLi1gcmn0xHL42HH2HGhuG63s3H+IPXXN7KlrZuPegyzdGDrpnhUIz7OU5GdTPiyX8qHhpFJemBN+XJjDyKE5lA3NISujb5OL5jh6qbq2kQIVNxSRbmRnpLfdqxKNu1PffIy9dc3sqWtib10zu+ua2VvXxJ66ZqpCDfxlcy0Nh4+d9DwzKM7PbksiZRpqW+cAAAzESURBVEMjx4U5nDmmkOkjh8b9vShx9FJVqIEKFTcUkV4yMwpzMynMzez0chjAweajkeQSTjB76prZc6CZvfXN7Nx/iNXb9rE/sinXFxZNYvoVShwppzrUyAKtqBKRPlKQk0lBTiZTyjpPLs1Hj1NTf5iczMRcwlLi6IXW4oZaUSUiqSQnM51xI4Yk7PypN13fj6i4oYgMRglJHGZ2n5mtMLMlsfQJ2hZpLzOzNxIRf1BVIS3FFZHBJ+6Jw8wWA+nuvgCoMLNT7geJ1idoW7vT/CuQG+/4Y1EVKW44PoFDQhGRVJOIEcci4JHI8bPAwoB9grZhZhcDjXRS4t3MbjGzVWa2KhQK9fiNdKc61MDYoiEqbigig0oiEkcesCtyvA8oC9gnUJuZZRHeifC2zgJw93vdvdLdK0tKSnrxVrpWFWrUxLiIDDqJSBwNnLiElN/Ja0TrE7TtNuAedz8Q98hj0NLibKltoKJY8xsiMrgkInGs5sTlqVnA1oB9grZdCnzRzJYCs83sF3GMPbDddU00H21hUqlGHCIyuCTiPo7HgOVmNgq4ErjezG539yVd9JkPeJA2d3+49SRmttTdb07Ae+hWdSiyFFcjDhEZZOI+4nD3esKT2iuBi9x9bYekEa1PXdC2DudZFO/4g2orbqgRh4gMMgm5c9zd93NiNVTgPkHbUkFVKFLcMF/FDUVkcNGd4z1UXdtARWm+ihuKyKCjxNFDVTWNTOqkPLKIyECmxNEDjYePsbdexQ1FZHBS4uiB1uKGqlElIoOREkcPtBY3VFVcERmMlDh6oCrUSJqKG4rIIKXE0QNVoQbGqLihiAxSShw9UK3ihiIyiClxxKi1uKEmxkVksFLiiFFrcUONOERksFLiiFFrcUONOERksFLiiNGJpbgacYjI4KTEEaPqUCMFOSpuKCKDlxJHjKpCDVSUqLihiAxeShwxqg6puKGIDG5KHDFoiBQ3nFSqiXERGbyUOGKwpW27WI04RGTwUuKIQXVt63axGnGIyOClxBGDqpoGFTcUkUFPiSMGVbWNjCkaQnaGihuKyOCVkMRhZveZ2QozWxJLnyBtZlZoZk+b2bNm9nszy0rEe4imqqaBSbrxT0QGubgnDjNbDKS7+wKgwsymBOkTtA34FHCnu18O7AWuiPd7iKalxdn6fqM2bxKRQS8jAedcBDwSOX4WWAhsCtBnTpA2d7+n3XlKgJqOAZjZLcAtAOPGjevxG2mvtbihalSJyGCXiEtVecCuyPE+oCxgn6BtAJjZAqDI3Vd2PLm73+vule5eWVJS0rt3E1HVuhRXl6pEZJBLxIijAciNHOcTPTlF6xO0DTMbDtwNfDzOsXeqOlLcUCMOERnsEjHiWE34MhPALGBrwD6B2iKT4b8FvuXu2+IbeueqQg0U5GRQnN9nc/EiIikpESOOx4DlZjYKuBK43sxud/clXfSZD3jAtpuAucA/mNk/AD9x998k4H2cJLxdrIobiojEfcTh7vWEJ79XAhe5+9oOSSNan7oY2n7i7kXuvijyk/CkAZHihprfEBFJyIgDd9/PidVQgfsEbetrbcUNNb8hIqI7x4PY0rZdrEYcIiJKHAGc2C5WIw4RESWOAKpDKm4oItJKiSOAqlAjY4eruKGICChxBFIVatDmTSIiEUoc3WhpcbbUNmpFlYhIhBJHN3YdaOLwsRZNjIuIRChxdKO6VktxRUTaU+LoRlWNluKKiLSnxNGN6loVNxQRaU+JoxtVNeGJcRU3FBEJU+LoRnVtgzZvEhFpR4mjCw2Hj/Fe/WEtxRURaUeJowsndv3TiENEpJUSRxeq26riasQhItJKiaMLVZHihuNU3FBEpI0SRxeqVdxQROQUShxdqAo16DKViEgHShydaC1uqKq4IiInS0jiMLP7zGyFmS2JpU9v2uKttbjhpFKNOERE2ot74jCzxUC6uy8AKsxsSpA+vWmL93uAdtvFasQhInKSRIw4FgGPRI6fBRYG7NObtpOY2S1mtsrMVoVCoR69ifzsDC6bUcZkjThERE6SiMSRB+yKHO8DygL26U3bSdz9XnevdPfKkpKSHr2JygnD+flnKhmRn92j54uIDFSJSBwNQG7kOL+T14jWpzdtIiLSRxLxj+5qTlw+mgVsDdinN20iItJHMhJwzseA5WY2CrgSuN7Mbnf3JV30mQ94L9pERKSPxH3E4e71hCewVwIXufvaDkkjWp+63rTF+z2IiEjnzN2THUNCVVZW+qpVq5IdhohIv2Jmq929MtrfaWJZRERiosQhIiIxUeIQEZGYDPg5DjMLAdt6+PRioDaO4cRLqsYFqRub4oqN4orNQIxrvLtHvYN6wCeO3jCzVZ1NDiVTqsYFqRub4oqN4orNYItLl6pERCQmShwiIhITJY6u3ZvsADqRqnFB6samuGKjuGIzqOLSHIeIiMREIw4REYmJEoeIJJWZDTezy8ysONmxtJeqcaUCJY5O9MW+5p28boaZbTezpZGfmWb2PTN7zcx+3K5foLY4xVRmZssjx5lm9oSZvWRmN/a2LY5xjTazne0+t5JIe5/tY29mhWb2tJk9a2a/N7Os3rx+vL6DncR10ncs0q/Pv2dmVgQ8CcwDXjCzkhT5zKLFlSqfWZmZvRE5TspnpcQRhfXRvuadOBP4tbsvcvdFQBbh/UfmATVmdqmZnRWkLR7BRP4HeoDwzosAXwZWu/t5wDVmVtDLtnjFdQ7w/dbPzd1D0f47Bm3rYVifAu5098uBvcD1PX39OH8HO8Z1G+2+Y+6+Luh3KgHfszOBW939+8AfgYtJjc+sY1w3kjqf2b8Cub35XHr7WSlxRLeI7vdNT5T5wIfN7FUzuw+4BPidh1cx/BE4H7gwYFs8HAeuA+ojjxdx4rNZBlT2si1ecc0Hbjaz183sn6LE2qt97INw93vc/bnIwxLg0714/bjE1Elcx2j3HTOzDIJ/p+L6PXP3F919pZldQPgf1g+QGp9Zx7iaSIHPzMwuBhoJ/wKwiCR9Vkoc0QXZNz1RXgMudfd5QCbhbXLjshd7T7h7fYc9TxK6X3wv4nqa8P8MZwMLzOzMZMQFYGYLgCJgRy9eP+7/PdvF9Rwnf8c+mOS4jPAvAfsJb9SWEp9Zh7jeIMmfmZllAd8mPGKkl6/fq5iUOKJL5r7mb7r7nsjxqk5iSeZe7Km6X/zL7n7Q3Y8T/p98SjLiMrPhwN2EL22kzGfVIa6O37GkfFatPOyLwJvAub2II66xdYhrVAp8ZrcB97j7gcjjpH2/lDiiS+a+5g+Z2SwzSweuIvybQSrtxZ6q+8X/0czKzWwIcDnwVl/HFfmN8LfAt9x9Wy9fP26fVZS4On7H1iYjrkhs3zSzz0QeDgN+0Is44vmZdYzrpynwmV0KfNHMlgKzgY/04vV7F5O766fDDzCU8BfjTuAdoLAPX/sMwr/hrAO+Tzi5vwT8CNgITAzaFue4lkb+HA+sj7zOa0B6b9riGNdFwIbIZ/elzv47Bm3rYSyfJ3xZY2nk57M9ff14fgejxPXd9t+xSJ+kfM84celsGXBP5L2nwmfWMa6ZqfKZtX7ve/O59Pazits/LAPtJ/LFuRYYmQKx5ALXABWxtiUonlGRz6YwHm19/d8xaFuyX7+vv4Op8j3TZ5b6n5VKjoiISEw0xyEiIjFR4hARkZgocYiISEyUOEQ6YWZfitQlaor8eXUPzvHvPXhOnoVrSr1oZg9FbkTDzGab2exYzycSb5ocF+mGmW1298l9+Hp/Awx19382s18AP3P318zsBgB3/6++ikUkmoxkByDS30RuwHoNONPdP2Bm+cD/EL5Zc7O7f659Xw8Xq8TM/n/C5SrOJ7yO/gp33xvlJXYBnzWz37v7zZHn/h/g6sjxX7n7JZGbHR8ESoF17v7FyGucAwwBQsD17n4szh+BDHK6VCUSu/nACnf/QORxOeFyHpcCE8ysq7o/k939AuBRwpVgT+HuTwA/BB41s7vMLN3dv0X4ruofuPslka63AG9Fzlceqc8FsNzdLwTeAz7W87cpEp0Sh0js3nL3R9s9PgrcDPwKGM6JGkDRPBj5czvhkvmniJS4foZwWYnWKrvRTAOujoyAKoDRkfbVkT/fBCZ0EYtIjyhxiMSuocPjmwhfqvoE4ZLXXenu7yGchK72cMHGt4CcSHsT4UtQrZVbNwL/HrkUtoRwMoJwGXCAOcDmAK8nEhMlDpHeew74FvB85PHoLvoG8SPghshIYh7wULvXWWxmLxGeJ/k5cKWZLQP+hnAZd4CzI88dRngXO5G40qoqkQEkMjm+1N2XJjkUGcCUOEREJCa6VCUiIjFR4hARkZgocYiISEyUOEREJCZKHCIiEpP/Bzd1s6kj/Lh6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "temp_learning_rate_schedule = CustomSchedule(d_model)\n",
    "\n",
    "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.xlabel(\"Train Step\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 十一、损失函数与指标（Loss and metrics）\n",
    "由于目标序列是填充（padded）过的，因此在计算损失函数时，应用填充遮挡非常重要。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True,\n",
    "                                                            reduction='none')\n",
    "\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "\n",
    "    return tf.reduce_mean(loss_)\n",
    "\n",
    "\n",
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
    "    name='train_accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 十一、训练与检查点(Training and checkpointing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer = Transformer(num_layers,\n",
    "                          d_model,\n",
    "                          num_heads,\n",
    "                          dff,\n",
    "                          input_vocab_size,\n",
    "                          target_vocab_size,\n",
    "                          pe_input=input_vocab_size,\n",
    "                          pe_target=target_vocab_size,\n",
    "                          rate=dropout_rate)\n",
    "\n",
    "\n",
    "# TODO：各有什么作用？？\n",
    "def create_masks(inp, tar):\n",
    "    # inp:(batch_size, seq_len_inp)\n",
    "    # tar:(batch_size, seq_len_tar - 1)\n",
    "    \n",
    "    # 1,编码器填充遮挡，将输入序列 pad 的部分填充掉:\n",
    "    # (batch_size, 1, 1, seq_len_inp)\n",
    "    enc_padding_mask = create_padding_mask(inp)\n",
    "    \n",
    "    # 2,用于解码器对编码器的输入进行注意力做 mask，因此维度为 seq_len_inp\n",
    "    # (batch_size, 1, 1, seq_len_inp)，    \n",
    "    dec_padding_mask = create_padding_mask(inp)\n",
    "\n",
    "    # 3. 不仅遮挡输入末尾填充（pad）的 0，还要遮挡输入的后续标记（future tokens）：\n",
    "    # (seq_len_tar - 1, seq_len_tar - 1)\n",
    "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
    "    \n",
    "    # (batch_size, 1, 1, seq_len_tar - 1)\n",
    "    dec_target_padding_mask = create_padding_mask(tar)\n",
    "    \n",
    "    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
    "    return enc_padding_mask, combined_mask, dec_padding_mask\n",
    "\n",
    "\n",
    "checkpoint_path = \"../H/save/zh2eng_transformer\"\n",
    "\n",
    "ckpt = tf.train.Checkpoint(transformer=transformer, optimizer=optimizer)\n",
    "\n",
    "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
    "\n",
    "# 如果检查点存在，则恢复最新的检查点。\n",
    "if ckpt_manager.latest_checkpoint:\n",
    "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
    "    print('Latest checkpoint restored!!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "序列长度为 5 的 `look_ahead_mask`:\n",
    "$$\n",
    "[0, 1, 1, 1, 1] \\\\\n",
    "[0, 0, 1, 1, 1] \\\\\n",
    "[0, 0, 0, 1, 1] \\\\ \n",
    "[0, 0, 0, 0, 1] \\\\\n",
    "[0, 0, 0, 0, 0] \\\\\n",
    "$$\n",
    "    \n",
    "对应的代表填充后 3 位得到的序列，`dec_target_padding_mask[x,:,:,:]`:\n",
    "$$[0, 0, 1, 1, 1] $$\n",
    "    \n",
    "最终得到的 `combined_mask` 为：\n",
    "$$\n",
    "[0, 1, 1, 1, 1] \\\\\n",
    "[0, 0, 1, 1, 1] \\\\\n",
    "[0, 0, 1, 1, 1] \\\\ \n",
    "[0, 0, 1, 1, 1] \\\\\n",
    "[0, 0, 1, 1, 1] \\\\\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n",
    "    example_zh_batch, tar_inp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 30, 30)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 30, 30)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 30, 30)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 30, 30)\n",
      "shape of mask: (64, 1, 35, 35)\n",
      "shape of atten: (64, 8, 35, 35)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 35, 30)\n",
      "shape of mask: (64, 1, 35, 35)\n",
      "shape of atten: (64, 8, 35, 35)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 35, 30)\n",
      "shape of mask: (64, 1, 35, 35)\n",
      "shape of atten: (64, 8, 35, 35)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 35, 30)\n",
      "shape of mask: (64, 1, 35, 35)\n",
      "shape of atten: (64, 8, 35, 35)\n",
      "shape of mask: (64, 1, 1, 30)\n",
      "shape of atten: (64, 8, 35, 30)\n"
     ]
    }
   ],
   "source": [
    "predictions, _ = transformer(example_zh_batch, tar_inp, True, enc_padding_mask,\n",
    "                                 combined_mask, dec_padding_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=9139, shape=(64, 35, 9756), dtype=float32, numpy=\n",
       "array([[[-0.1282706 ,  0.10864322, -0.24248649, ...,  0.16296358,\n",
       "          0.1459565 , -0.34566465],\n",
       "        [-0.0717675 ,  0.18643032, -0.08001041, ...,  0.09116092,\n",
       "          0.11926436, -0.34716186],\n",
       "        [-0.02932605,  0.0366094 , -0.21726583, ...,  0.1092973 ,\n",
       "          0.21624133, -0.35436305],\n",
       "        ...,\n",
       "        [-0.1644766 ,  0.13415962, -0.15722702, ...,  0.13606192,\n",
       "          0.1721753 , -0.3274525 ],\n",
       "        [-0.2355859 ,  0.09242162, -0.1662194 , ...,  0.09706165,\n",
       "          0.0351093 , -0.30270767],\n",
       "        [-0.16192722,  0.10823155, -0.15517114, ...,  0.11812331,\n",
       "          0.18167779, -0.40581176]],\n",
       "\n",
       "       [[-0.32617697,  0.22187786, -0.09803833, ...,  0.08348731,\n",
       "          0.04196617, -0.2595157 ],\n",
       "        [-0.12240703,  0.2136595 , -0.12586969, ...,  0.05328099,\n",
       "          0.05000972, -0.2488268 ],\n",
       "        [-0.12685798,  0.09253548, -0.05003348, ...,  0.01079748,\n",
       "          0.13053499, -0.31164265],\n",
       "        ...,\n",
       "        [-0.23358443,  0.23125868, -0.17388429, ...,  0.09450406,\n",
       "          0.02134542, -0.35398316],\n",
       "        [-0.25675273,  0.17063819, -0.20740515, ...,  0.12528966,\n",
       "          0.07416631, -0.25018227],\n",
       "        [-0.28282622,  0.13860644, -0.17433897, ...,  0.11155581,\n",
       "         -0.01655334, -0.3270413 ]],\n",
       "\n",
       "       [[-0.21840023,  0.06015635, -0.04322842, ...,  0.06896798,\n",
       "          0.02794765, -0.22268985],\n",
       "        [-0.17766161, -0.04175738, -0.04316607, ...,  0.22023344,\n",
       "          0.10613605, -0.31021938],\n",
       "        [-0.17850605,  0.06827846, -0.10122703, ...,  0.06556543,\n",
       "          0.06886267, -0.20053634],\n",
       "        ...,\n",
       "        [-0.22859651,  0.08425389, -0.08617701, ...,  0.10340382,\n",
       "          0.01203229, -0.16140257],\n",
       "        [-0.16902383,  0.08090241, -0.04938472, ...,  0.18718097,\n",
       "          0.16317068, -0.2359052 ],\n",
       "        [-0.17250344,  0.01791168, -0.10564379, ...,  0.17990048,\n",
       "          0.13179553, -0.367264  ]],\n",
       "\n",
       "       ...,\n",
       "\n",
       "       [[-0.10422491, -0.02271232, -0.1643712 , ...,  0.09678642,\n",
       "          0.07728803, -0.4102392 ],\n",
       "        [-0.21040101,  0.12451793, -0.15451105, ...,  0.04963348,\n",
       "          0.21309398, -0.3020317 ],\n",
       "        [-0.29859206,  0.07001271, -0.0927428 , ...,  0.15649518,\n",
       "          0.19731031, -0.2828086 ],\n",
       "        ...,\n",
       "        [-0.3092833 ,  0.03744534, -0.07875454, ...,  0.15936223,\n",
       "          0.12852998, -0.21115161],\n",
       "        [-0.19534431,  0.0048671 , -0.06850849, ...,  0.09544557,\n",
       "          0.10667858, -0.25607175],\n",
       "        [-0.09220203,  0.09771182, -0.17135058, ...,  0.08397103,\n",
       "          0.18899764, -0.31485498]],\n",
       "\n",
       "       [[-0.2607268 ,  0.02192224, -0.08926166, ...,  0.05996825,\n",
       "          0.09215534, -0.3057335 ],\n",
       "        [-0.24648269,  0.08431543, -0.24813487, ...,  0.00661636,\n",
       "          0.26635742, -0.40189534],\n",
       "        [-0.2595802 ,  0.02175296, -0.2056845 , ..., -0.02899096,\n",
       "          0.16077825, -0.36374655],\n",
       "        ...,\n",
       "        [-0.26787338,  0.16958177, -0.21922566, ...,  0.09383419,\n",
       "          0.10387039, -0.41843277],\n",
       "        [-0.3175898 ,  0.09149323, -0.2579514 , ...,  0.11508129,\n",
       "          0.20097505, -0.40567046],\n",
       "        [-0.20694615,  0.12027481, -0.16593759, ...,  0.05556691,\n",
       "          0.30633566, -0.41706493]],\n",
       "\n",
       "       [[ 0.01923838,  0.21714151, -0.19829923, ...,  0.14570265,\n",
       "          0.16789883, -0.27872226],\n",
       "        [-0.18276507,  0.19583578, -0.20111662, ...,  0.10464481,\n",
       "          0.050662  , -0.12767582],\n",
       "        [-0.15539445,  0.19552206, -0.13164912, ...,  0.21786754,\n",
       "          0.08099304, -0.24668398],\n",
       "        ...,\n",
       "        [-0.26000267,  0.17794685, -0.22024378, ...,  0.09811572,\n",
       "          0.14509456, -0.07260724],\n",
       "        [-0.1970233 ,  0.24301168, -0.27220955, ...,  0.10705914,\n",
       "          0.15928268, -0.28742662],\n",
       "        [-0.18685852,  0.13866998, -0.15299535, ...,  0.22941297,\n",
       "          0.13347127, -0.31046546]]], dtype=float32)>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 该 @tf.function 将追踪-编译 train_step 到 TF 图中，以便更快地\n",
    "# 执行。该函数专用于参数张量的精确形状。为了避免由于可变序列长度或可变\n",
    "# 批次大小（最后一批次较小）导致的再追踪，使用 input_signature 指定\n",
    "# 更多的通用形状。\n",
    "\n",
    "\n",
    "@tf.function()\n",
    "def train_step(inp, tar):\n",
    "    tar_inp = tar[:, :-1]\n",
    "    tar_real = tar[:, 1:]   # 给定前面的词，预测下一个词\n",
    "\n",
    "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n",
    "        inp, tar_inp)\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions, _ = transformer(inp, tar_inp, True, enc_padding_mask,\n",
    "                                     combined_mask, dec_padding_mask)\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "\n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "\n",
    "    train_loss(loss)\n",
    "    train_accuracy(tar_real, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 2.0904 Accuracy 0.0000\n",
      "Epoch 1 Batch 50 Loss 1.9386 Accuracy 0.0105\n",
      "Epoch 1 Batch 100 Loss 1.9165 Accuracy 0.0194\n",
      "Epoch 1 Batch 150 Loss 1.8870 Accuracy 0.0225\n",
      "Epoch 1 Batch 200 Loss 1.8540 Accuracy 0.0240\n",
      "Epoch 1 Batch 250 Loss 1.8171 Accuracy 0.0249\n",
      "Epoch 1 Loss 1.7999 Accuracy 0.0252\n",
      "Time taken for 1 epoch: 15.697981119155884 secs\n",
      "\n",
      "Epoch 2 Batch 0 Loss 1.5492 Accuracy 0.0286\n",
      "Epoch 2 Batch 50 Loss 1.5150 Accuracy 0.0286\n",
      "Epoch 2 Batch 100 Loss 1.4675 Accuracy 0.0286\n",
      "Epoch 2 Batch 150 Loss 1.4216 Accuracy 0.0286\n",
      "Epoch 2 Batch 200 Loss 1.3878 Accuracy 0.0286\n",
      "Epoch 2 Batch 250 Loss 1.3627 Accuracy 0.0295\n",
      "Epoch 2 Loss 1.3537 Accuracy 0.0300\n",
      "Time taken for 1 epoch: 8.945199728012085 secs\n",
      "\n",
      "Epoch 3 Batch 0 Loss 1.1658 Accuracy 0.0375\n",
      "Epoch 3 Batch 50 Loss 1.1896 Accuracy 0.0433\n",
      "Epoch 3 Batch 100 Loss 1.1671 Accuracy 0.0470\n",
      "Epoch 3 Batch 150 Loss 1.1501 Accuracy 0.0492\n",
      "Epoch 3 Batch 200 Loss 1.1343 Accuracy 0.0512\n",
      "Epoch 3 Batch 250 Loss 1.1194 Accuracy 0.0529\n",
      "Epoch 3 Loss 1.1122 Accuracy 0.0535\n",
      "Time taken for 1 epoch: 8.95915150642395 secs\n",
      "\n",
      "Epoch 4 Batch 0 Loss 0.9786 Accuracy 0.0607\n",
      "Epoch 4 Batch 50 Loss 1.0140 Accuracy 0.0636\n",
      "Epoch 4 Batch 100 Loss 0.9922 Accuracy 0.0643\n",
      "Epoch 4 Batch 150 Loss 0.9829 Accuracy 0.0655\n",
      "Epoch 4 Batch 200 Loss 0.9748 Accuracy 0.0661\n",
      "Epoch 4 Batch 250 Loss 0.9659 Accuracy 0.0669\n",
      "Epoch 4 Loss 0.9622 Accuracy 0.0670\n",
      "Time taken for 1 epoch: 8.963302373886108 secs\n",
      "\n",
      "Epoch 5 Batch 0 Loss 0.8791 Accuracy 0.0683\n",
      "Epoch 5 Batch 50 Loss 0.8899 Accuracy 0.0726\n",
      "Epoch 5 Batch 100 Loss 0.8847 Accuracy 0.0727\n",
      "Epoch 5 Batch 150 Loss 0.8818 Accuracy 0.0731\n",
      "Epoch 5 Batch 200 Loss 0.8770 Accuracy 0.0734\n",
      "Epoch 5 Batch 250 Loss 0.8745 Accuracy 0.0738\n",
      "Saving checkpoint for epoch 5 at ../H/save/zh2eng_transformer/ckpt-1\n",
      "Epoch 5 Loss 0.8740 Accuracy 0.0742\n",
      "Time taken for 1 epoch: 9.066415548324585 secs\n",
      "\n",
      "Epoch 6 Batch 0 Loss 0.7689 Accuracy 0.0799\n",
      "Epoch 6 Batch 50 Loss 0.8208 Accuracy 0.0787\n",
      "Epoch 6 Batch 100 Loss 0.8177 Accuracy 0.0794\n",
      "Epoch 6 Batch 150 Loss 0.8143 Accuracy 0.0796\n",
      "Epoch 6 Batch 200 Loss 0.8126 Accuracy 0.0800\n",
      "Epoch 6 Batch 250 Loss 0.8086 Accuracy 0.0800\n",
      "Epoch 6 Loss 0.8089 Accuracy 0.0803\n",
      "Time taken for 1 epoch: 8.942633152008057 secs\n",
      "\n",
      "Epoch 7 Batch 0 Loss 0.8576 Accuracy 0.0799\n",
      "Epoch 7 Batch 50 Loss 0.7621 Accuracy 0.0851\n",
      "Epoch 7 Batch 100 Loss 0.7603 Accuracy 0.0849\n",
      "Epoch 7 Batch 150 Loss 0.7567 Accuracy 0.0854\n",
      "Epoch 7 Batch 200 Loss 0.7540 Accuracy 0.0856\n",
      "Epoch 7 Batch 250 Loss 0.7512 Accuracy 0.0859\n",
      "Epoch 7 Loss 0.7503 Accuracy 0.0860\n",
      "Time taken for 1 epoch: 8.969208240509033 secs\n",
      "\n",
      "Epoch 8 Batch 0 Loss 0.6805 Accuracy 0.0942\n",
      "Epoch 8 Batch 50 Loss 0.7008 Accuracy 0.0906\n",
      "Epoch 8 Batch 100 Loss 0.6971 Accuracy 0.0904\n",
      "Epoch 8 Batch 150 Loss 0.6978 Accuracy 0.0909\n",
      "Epoch 8 Batch 200 Loss 0.6987 Accuracy 0.0912\n",
      "Epoch 8 Batch 250 Loss 0.6977 Accuracy 0.0912\n",
      "Epoch 8 Loss 0.6968 Accuracy 0.0912\n",
      "Time taken for 1 epoch: 8.950708627700806 secs\n",
      "\n",
      "Epoch 9 Batch 0 Loss 0.6195 Accuracy 0.0942\n",
      "Epoch 9 Batch 50 Loss 0.6450 Accuracy 0.0968\n",
      "Epoch 9 Batch 100 Loss 0.6401 Accuracy 0.0968\n",
      "Epoch 9 Batch 150 Loss 0.6432 Accuracy 0.0967\n",
      "Epoch 9 Batch 200 Loss 0.6415 Accuracy 0.0969\n",
      "Epoch 9 Batch 250 Loss 0.6428 Accuracy 0.0970\n",
      "Epoch 9 Loss 0.6436 Accuracy 0.0971\n",
      "Time taken for 1 epoch: 8.951661825180054 secs\n",
      "\n",
      "Epoch 10 Batch 0 Loss 0.5692 Accuracy 0.1036\n",
      "Epoch 10 Batch 50 Loss 0.5847 Accuracy 0.1031\n",
      "Epoch 10 Batch 100 Loss 0.5865 Accuracy 0.1036\n",
      "Epoch 10 Batch 150 Loss 0.5857 Accuracy 0.1033\n",
      "Epoch 10 Batch 200 Loss 0.5863 Accuracy 0.1035\n",
      "Epoch 10 Batch 250 Loss 0.5886 Accuracy 0.1037\n",
      "Saving checkpoint for epoch 10 at ../H/save/zh2eng_transformer/ckpt-2\n",
      "Epoch 10 Loss 0.5897 Accuracy 0.1036\n",
      "Time taken for 1 epoch: 9.093379497528076 secs\n",
      "\n",
      "Epoch 11 Batch 0 Loss 0.4876 Accuracy 0.1143\n",
      "Epoch 11 Batch 50 Loss 0.5258 Accuracy 0.1113\n",
      "Epoch 11 Batch 100 Loss 0.5216 Accuracy 0.1110\n",
      "Epoch 11 Batch 150 Loss 0.5331 Accuracy 0.1101\n",
      "Epoch 11 Batch 200 Loss 0.5353 Accuracy 0.1102\n",
      "Epoch 11 Batch 250 Loss 0.5368 Accuracy 0.1102\n",
      "Epoch 11 Loss 0.5375 Accuracy 0.1102\n",
      "Time taken for 1 epoch: 8.97883677482605 secs\n",
      "\n",
      "Epoch 12 Batch 0 Loss 0.4288 Accuracy 0.1237\n",
      "Epoch 12 Batch 50 Loss 0.4670 Accuracy 0.1185\n",
      "Epoch 12 Batch 100 Loss 0.4754 Accuracy 0.1179\n",
      "Epoch 12 Batch 150 Loss 0.4809 Accuracy 0.1170\n",
      "Epoch 12 Batch 200 Loss 0.4861 Accuracy 0.1172\n",
      "Epoch 12 Batch 250 Loss 0.4889 Accuracy 0.1167\n",
      "Epoch 12 Loss 0.4894 Accuracy 0.1167\n",
      "Time taken for 1 epoch: 8.94973349571228 secs\n",
      "\n",
      "Epoch 13 Batch 0 Loss 0.3634 Accuracy 0.1268\n",
      "Epoch 13 Batch 50 Loss 0.4260 Accuracy 0.1249\n",
      "Epoch 13 Batch 100 Loss 0.4308 Accuracy 0.1245\n",
      "Epoch 13 Batch 150 Loss 0.4346 Accuracy 0.1235\n",
      "Epoch 13 Batch 200 Loss 0.4388 Accuracy 0.1228\n",
      "Epoch 13 Batch 250 Loss 0.4430 Accuracy 0.1226\n",
      "Epoch 13 Loss 0.4448 Accuracy 0.1225\n",
      "Time taken for 1 epoch: 8.998482942581177 secs\n",
      "\n",
      "Epoch 14 Batch 0 Loss 0.4024 Accuracy 0.1384\n",
      "Epoch 14 Batch 50 Loss 0.3744 Accuracy 0.1324\n",
      "Epoch 14 Batch 100 Loss 0.3830 Accuracy 0.1304\n",
      "Epoch 14 Batch 150 Loss 0.3882 Accuracy 0.1292\n",
      "Epoch 14 Batch 200 Loss 0.3968 Accuracy 0.1287\n",
      "Epoch 14 Batch 250 Loss 0.4027 Accuracy 0.1284\n",
      "Epoch 14 Loss 0.4057 Accuracy 0.1282\n",
      "Time taken for 1 epoch: 8.97952389717102 secs\n",
      "\n",
      "Epoch 15 Batch 0 Loss 0.3361 Accuracy 0.1496\n",
      "Epoch 15 Batch 50 Loss 0.3336 Accuracy 0.1386\n",
      "Epoch 15 Batch 100 Loss 0.3441 Accuracy 0.1368\n",
      "Epoch 15 Batch 150 Loss 0.3556 Accuracy 0.1350\n",
      "Epoch 15 Batch 200 Loss 0.3626 Accuracy 0.1336\n",
      "Epoch 15 Batch 250 Loss 0.3691 Accuracy 0.1331\n",
      "Saving checkpoint for epoch 15 at ../H/save/zh2eng_transformer/ckpt-3\n",
      "Epoch 15 Loss 0.3720 Accuracy 0.1329\n",
      "Time taken for 1 epoch: 9.101390600204468 secs\n",
      "\n",
      "Epoch 16 Batch 0 Loss 0.3165 Accuracy 0.1362\n",
      "Epoch 16 Batch 50 Loss 0.3003 Accuracy 0.1412\n",
      "Epoch 16 Batch 100 Loss 0.3087 Accuracy 0.1409\n",
      "Epoch 16 Batch 150 Loss 0.3182 Accuracy 0.1406\n",
      "Epoch 16 Batch 200 Loss 0.3261 Accuracy 0.1395\n",
      "Epoch 16 Batch 250 Loss 0.3326 Accuracy 0.1388\n",
      "Epoch 16 Loss 0.3353 Accuracy 0.1384\n",
      "Time taken for 1 epoch: 8.975188255310059 secs\n",
      "\n",
      "Epoch 17 Batch 0 Loss 0.3117 Accuracy 0.1567\n",
      "Epoch 17 Batch 50 Loss 0.2679 Accuracy 0.1512\n",
      "Epoch 17 Batch 100 Loss 0.2702 Accuracy 0.1489\n",
      "Epoch 17 Batch 150 Loss 0.2812 Accuracy 0.1473\n",
      "Epoch 17 Batch 200 Loss 0.2883 Accuracy 0.1461\n",
      "Epoch 17 Batch 250 Loss 0.2948 Accuracy 0.1450\n",
      "Epoch 17 Loss 0.2972 Accuracy 0.1448\n",
      "Time taken for 1 epoch: 8.990416765213013 secs\n",
      "\n",
      "Epoch 18 Batch 0 Loss 0.2540 Accuracy 0.1625\n",
      "Epoch 18 Batch 50 Loss 0.2280 Accuracy 0.1585\n",
      "Epoch 18 Batch 100 Loss 0.2371 Accuracy 0.1568\n",
      "Epoch 18 Batch 150 Loss 0.2444 Accuracy 0.1548\n",
      "Epoch 18 Batch 200 Loss 0.2507 Accuracy 0.1530\n",
      "Epoch 18 Batch 250 Loss 0.2584 Accuracy 0.1516\n",
      "Epoch 18 Loss 0.2613 Accuracy 0.1510\n",
      "Time taken for 1 epoch: 8.969774961471558 secs\n",
      "\n",
      "Epoch 19 Batch 0 Loss 0.2235 Accuracy 0.1634\n",
      "Epoch 19 Batch 50 Loss 0.2022 Accuracy 0.1613\n",
      "Epoch 19 Batch 100 Loss 0.2103 Accuracy 0.1608\n",
      "Epoch 19 Batch 150 Loss 0.2188 Accuracy 0.1591\n",
      "Epoch 19 Batch 200 Loss 0.2255 Accuracy 0.1572\n",
      "Epoch 19 Batch 250 Loss 0.2311 Accuracy 0.1562\n",
      "Epoch 19 Loss 0.2334 Accuracy 0.1558\n",
      "Time taken for 1 epoch: 8.98022985458374 secs\n",
      "\n",
      "Epoch 20 Batch 0 Loss 0.1704 Accuracy 0.1710\n",
      "Epoch 20 Batch 50 Loss 0.1777 Accuracy 0.1687\n",
      "Epoch 20 Batch 100 Loss 0.1814 Accuracy 0.1664\n",
      "Epoch 20 Batch 150 Loss 0.1895 Accuracy 0.1647\n",
      "Epoch 20 Batch 200 Loss 0.1978 Accuracy 0.1626\n",
      "Epoch 20 Batch 250 Loss 0.2039 Accuracy 0.1615\n",
      "Saving checkpoint for epoch 20 at ../H/save/zh2eng_transformer/ckpt-4\n",
      "Epoch 20 Loss 0.2067 Accuracy 0.1611\n",
      "Time taken for 1 epoch: 9.083451271057129 secs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "\n",
    "    # inp -> zh, tar -> english\n",
    "    for (batch, (inp, tar)) in enumerate(dataset):\n",
    "        train_step(inp, tar)\n",
    "\n",
    "        if batch % 50 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
    "                epoch + 1, batch, train_loss.result(),\n",
    "                train_accuracy.result()))\n",
    "\n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        ckpt_save_path = ckpt_manager.save()\n",
    "        print('Saving checkpoint for epoch {} at {}'.format(\n",
    "            epoch + 1, ckpt_save_path))\n",
    "\n",
    "    print('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
    "        epoch + 1, train_loss.result(), train_accuracy.result()))\n",
    "\n",
    "    print('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 十二、评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length_ch = zh_tensor.shape[-1]\n",
    "max_length_eng = eng_tensor.shape[-1]\n",
    "\n",
    "\n",
    "def evaluate(inp_sentence):\n",
    "    sentence = preprocess_zh(inp_sentence)\n",
    "    inputs = [zh_tokenizer.word_index[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "        [inputs], maxlen=max_length_ch, padding='post')\n",
    "    encoder_input = tf.convert_to_tensor(inputs)\n",
    "\n",
    "    # encoder_input = tf.expand_dims(inp_sentence, 0)\n",
    "\n",
    "    # 因为目标是英语，输入 transformer 的第一个词应该是\n",
    "    # 英语的开始标记。\n",
    "    decoder_input = [eng_tokenizer.word_index['<start>']]\n",
    "    output = tf.expand_dims(decoder_input, 0)\n",
    "\n",
    "    for i in range(max_length_eng):\n",
    "        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n",
    "            encoder_input, output)\n",
    "\n",
    "        # predictions.shape == (batch_size, seq_len, vocab_size)\n",
    "        predictions, attention_weights = transformer(encoder_input, output,\n",
    "                                                     False, enc_padding_mask,\n",
    "                                                     combined_mask,\n",
    "                                                     dec_padding_mask)\n",
    "\n",
    "        # 从 seq_len 维度选择最后一个词\n",
    "        predictions = predictions[:, -1:, :]  # (batch_size, 1, vocab_size)\n",
    "\n",
    "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "        # 如果 predicted_id 等于结束标记，就返回结果\n",
    "        if predicted_id == eng_tokenizer.word_index['<end>']:\n",
    "            return tf.squeeze(output, axis=0), attention_weights\n",
    "\n",
    "        # 连接 predicted_id 与输出，作为解码器的输入传递到解码器。\n",
    "        output = tf.concat([output, predicted_id], axis=-1)\n",
    "\n",
    "    return tf.squeeze(output, axis=0), attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_attention_weights(attention, sentence, result, layer):\n",
    "    print(result)\n",
    "\n",
    "    fig = plt.figure(figsize=(16, 8))\n",
    "\n",
    "    sentence = preprocess_zh(sentence)\n",
    "    sentence = [zh_tokenizer.word_index[i] for i in sentence.split(' ')]\n",
    "\n",
    "    attention = tf.squeeze(attention[layer], axis=0)\n",
    "\n",
    "    for head in range(attention.shape[0]):\n",
    "        ax = fig.add_subplot(2, 4, head + 1)\n",
    "\n",
    "        # 画出注意力权重\n",
    "        ax.matshow(attention[head][:-1, :], cmap='viridis')\n",
    "\n",
    "        fontdict = {'fontsize': 10}\n",
    "\n",
    "        ax.set_xticks(range(len(sentence) + 2))\n",
    "        ax.set_yticks(range(len(result)))\n",
    "\n",
    "        ax.set_ylim(len(result) - 1.5, -0.5)\n",
    "        ax.set_xlim(-0.5, len(sentence))\n",
    "\n",
    "        ax.set_xticklabels([zh_tokenizer.index_word[i] for i in sentence],\n",
    "                           fontdict=fontdict)\n",
    "\n",
    "        ax.set_yticklabels(\n",
    "            [eng_tokenizer.index_word[i] for i in result.numpy()] + ['<end>'],\n",
    "            fontdict=fontdict)\n",
    "\n",
    "        ax.set_xlabel('Head {}'.format(head + 1))\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(sentence, plot=''):\n",
    "    result, attention_weights = evaluate(sentence)\n",
    "\n",
    "    predicted_sentence = ' '.join(\n",
    "        [eng_tokenizer.index_word[i] for i in result.numpy()[1:]])\n",
    "\n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(predicted_sentence))\n",
    "\n",
    "    if plot:\n",
    "        plot_attention_weights(attention_weights, sentence, result, plot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: 你应该立即开始。\n",
      "Predicted translation: port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port port\n",
      "tf.Tensor(\n",
      "[   1 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771\n",
      " 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771 6771\n",
      " 6771 6771 6771 6771 6771 6771 6771 6771 6771], shape=(37,), dtype=int32)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3oAAAI4CAYAAAA8tJurAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdf5hkZ1nn//dd1T0zyUwyMwkJIUNgUCEaENFFV7OCwy9BXXWVRS9WcWX1C8hlYPECsiCrcZVf8l2DoKisrgQXcJesXxHWrBBlQsDFXSA/lEWQBeOCAiGTzGSSSaa76vn+cc7MmRnqOTPdXc/pOlXv11z3le46daqqu3M+fZ6q6vuOlBKSJEmSpPkx2OwHIEmSJEmaLhd6kiRJkjRnXOhJkiRJ0pxxoSdJkiRJc8aFniRJkiTNGRd6kiRJkjRnFnKhFxFbI+KSCZfHKZ/viYgHTLpeRGw74bZ2nLJtqcTjljRdZoEkWFMWXBgRD4yI4aTrmgdSv81bFszsQq/+5j1xSrf18IjYFhFnR8RFwCOA36u3fXdE7Kyv+tmI2HfCrr8G/O+IWI2IuyLi9ohIwM3AlyLiWuBvgNedsM8zgT/awGM98fFIYnp50JYF9fZjx99vRsT+iNhdbzqWBZ+LiKN1FqwCR4EPRcTHgD8GPhcRW+t9zAJpymYgCy4DPgDc4LmBtHlKrROAb2OOsiBmcWB6HbqvA16YUjowYftjAFJKN5/h7f0J1Tf9+4BdwGeAZeAAcDbwtcA/rrf/U+D9wLcDAQyBxwN/Cfwi8A7gycD3A68CPgz8B+Dt9fU/XG97KHBjSunOM3yMVwH7qf6HeA1wRUrprjPZV5pn08yDU7LgbOBzwM76vwPgPOD/AK8EfgP4OPDgevdl4KuBC4F/Afw74D7gRcAzgE8ALwa+o76+WSBN0Yxlwf3AU4BfSCm9KiLuwXMDqRMF1gl/DdxDdexfBNzBnGTBzL2iFxEXA79M9QV8xQ+v9pi6zuT2vh74AtUP8C5gBXgA1QnaNuDTVD+0DwLfC4yoFnZbgX9E9UtgleqH+TP1PsdWx/uAh1Ct/j8E/C7wppTSLfXtv+nMvupGSunzwEuAN5zwrIG0kKaZBxOy4E6qLFimOq6fR5UDFwLXAUeoju8twOXAP1Ad54epsuBrqJ7RS/X1Xkj1RNIHMQukqZrBLHgZ1QneD0XEfjw3kDpRaJ1whOpJ23+gWiccy4KtwC30OAtm6hW9iNhDtWJ+QUrpUEScBbwTOJdqdf0MqlfVfqDe5fMppSfV7329FtgOfDql9Oz69m6gejvGPcANVN/wMbADuBe4HXh9fXs3AC8Hfotqpfw/gJ+u7/fjwHellP70hJX6PwN+lmrF/23AjVS/OL4K+BLVL41twAdTSk+rV+LLwOPqr+dpVP9jvJPqVcMArkop7a8f+4VUz1a8qOV/ZGluTTMPImK53ucvqZ5Fu5XqSZ5nAEtUof1c4LFUx+W3ALuBp1I9I7cMvJrq5O+OlNJlEXEzVVC/mOpJoodSPcu3F7NAmpoZyoJXUh27/x54K7AHeGZKab/nBlJ5BdcJW4CfB36K6pX9s6iy4cHAu+hxFszaK3r7gI+llA7Vn18GjFNKj6daBe9IKb2MaiH2mpTSk+rrPQh4I9U3dm9EPLC+/Nhq+z9RfZPfSLUa/wzwReDngCcAD6P6gb6P6iTtKcCjgOup3o41BN4dEXdS/fDfBDwL+Of1tqcBtwHfCZwP/CbV3/ScDxyJiEfXj+dr6q/lD4AnAs8B3pNSegLVMwjHpZS+VD/Ob137t1GaC/uYXh48iyrgPwccAt4LPJzqGLuL6pfDE6gC/Zup3j9/B1Wg/1C9/yeoTgQfXmfBo4BvBN5M9fbvHVTBbRZI07WP2ciC76XKg1upsuBC4I88N5A6s48y64RPUv1Jxm1ULwAtAT9C9Xu/11kwUwu9lNLbgIMRcUV90ceAv4qI91J9Y+/N7LoC/CTwNqr30p5VX34L1VusoHr27g6qt2buAD5FFfaXUb3s+ttU78H9BNVq+unAvwVeQLWifmt9H5+gWvG/lWp1voXqh3wTcE19u1dSPct/iGoRuad+DG+t//t39X4Pqx8jwEdO/IIi4hXALSmlP85+w6Q5NuU8eAvVs/afoMmCVaos+CJ1FqSUbgReS/W2iL9PKb2dKpB/lyoLvkT1it5u4P9SPRP4r+qPL6jv3yyQpmiGsuCtVH+n/wKqV/pHVM/Oe24gdaDgOuFcqoXeEPg64MtUx/MFfc+CmVroAaSUrgG+HBEvBb4B+FBK6TupXi59XH21I1R/PH2s3elPUL0k+0yqt2kec5gqwKH6Jv4M8CdUz7pfC1xB9Srek6hePv0M1Uuj30O1ij5M9d79rVQ/lCdQvdf2mMuofjG8ur7NYX3/r61v+2bgFVQ/ME55bNSXP7L++Ph7iSPiVcBNKaU/bP9uSfNtWnmQUhrT/AI4lgW/Vd/OrTRZAFUebAG+WL93/7nAR6my4MHAbfXbRXYAv1Pvc5jqBPLhmAXS1M1YFryf6u1hR+rb9dxA6kihdcIO4CBwCc1bK0fHboMeZ8HMLfQAUkrvoHrG7WuBF0TEn1N1wTm2mn0f8IMR8SGqH+r7qF6V+7N6+56Tb5Flqm/2y6nexhn19R9N9QO5AXgp1UuoL6Fa8P0Y1ft0f4fqjyqP/UBXqF6CPXa7t1C9vLuN6kTwoVSLxNdS/Q/4PKpn+yd5M/D0+o83zwWIiOcB708p/bcz+FZJc2/KeXBiFtxM9Td3j6DqsjuMiOdTHedvAf4l1VszrgHOoXrV/8NUYfsuqmD/W6pnEQG+m+oJIbNAKmBGsuAw1Sv8F1G9KvAuPDeQOjXlLNhG9Yrbf6RqyHIF1fH608DRiPhVepwFM9WMpYSIeDzVy6O3Ua3YH031rPzfUK2uL6Za+H2K6pW7cb3rQ6lW79cA76Zapd9E9ceaz6d6n+8HgKeklF5UP2PwfOD+lNJv1/f9sZTSN3XwZUo6jdNkwRJViF5CPgtuo/rD66dRvX/+56jeG//1VL8QPm8WSLNvg1lwJ1V33bcDDwT+IKX0toj4FJ4bSL0TEa+lGrF2B9WibSvV8X9snbCHHmfB3C/0AOq3WW1bw6yKbVTfmyMnXBbpNN+sqCfdp5RW264naXOYBZKguyyor2ceSDNq3rNgIRZ6kiRJkrRIZvJv9CRJkiRJ6zeXC72IeM5aLi+xrcR9SVqbWTl2p31fktZmVo7dad+XpLWb9eN6mucGc7nQo+pms5bLS2wrcV+S1mZWjt1p35ektZmVY3fa9yVp7Wb9uJ7aucG8LvQkSZIkaWGtqRlLRDwQeGRK6c9Oe+UNiIjvphqAePBMrj/csT0tnb/7+Oejw/cw3LEdgJ3bjzfF4cid93HW7m3HP794+e7jH99xx5jzz6/WvXeMmusA3HNghe3nLQNw/vC+k7aduN/9KY5ffteBEbvOGx7//DOHLpj4+AC23nbv8Y/v5s7DKaVzTvtFS5toVrMAYLh9e1redR4Ao3vuYbi9OdZSc0h+xXEYdQ+sU/fZdu79xz8+etcRtuw66/jnX7W1eVhfvmPEA85v7uDT9+06/vHKwSMs72z2+5ptd03cB+DTt1b3fS+HWE2rgTTDZjoLztmeli6ojsPx3fcwOKc5rs/asnL841OP611LzXnD3XeucM7u6vf/ecOjxy8/9di9/4RzqTsPjNl9XvM8+pdXdxz/+Mid93PW7q3HP3/Q8uHjH594PgHwmVur/cwC9cEsZ8HyzrPStot2Hv985a57Wd5VzULffsJxfe9d93P2rub4PHGFdOKxe/bgKCc6MSfOGdx/0rYDB8acV+fB3eOtE/cBOLDS5NPRg0fYcsI5Q/pUk1drXScsnekVI+Ii4HXACzPbHwOQUrr5TG+zZd9bgF+LiCtSSnedbv+l83dz0c9OfFh872Nvyu73Cw+8YeLl1xy6LLvPj5778ey221aXs9v++Z89P7vtET/xkeMfX5+u/WT2itIMmOUsAFjedR6XPP9FE7etnpN/Ymvr7ZPf4PCI7/w/2X3+89e8J7vt+/76B7Lb/r9L/yC77Qce/C0A/EX60+x1pFkw61mwdMEuHvyqyb97v/7Bn8/u9z0X/OXEy595zt9l9/ns6ii77T/e8U+y21524Y3ZbT9ySbWfWaBZN+tZsO2inXzTm3504rbHnp8/rlfScOLl37T9b7P7PPHs/LY/u3dvdtvv/8O3ZLeNnvD3xz9e6zrhjN66GREXA78MXJFSOpC52mPqWo+T9k0pfR54CfCGiNid3UtSp8wCSWAWSKqYBbPttK/oRcQe4JXAT6eUDtWDBd8JnEs1Rf4ZwC8CP1Bf/1kppSdFxA7gWmA78OmU0rPr7fuB/wU8OqX01Ih49an7AqSUvhARLwZeHxEvavmfR1IHzAJJYBZIqpgFs+9M3rq5D/hYSulQ/fllwDil9PiI+D5gR0rpZRHxSYCU0lvq6z0IeCNwPfDfI+KBKaUvAt8KvCGl9JL6+pP2pf78SxHxmXqfPz5xW91e9DkAw/N2Iam4fcxgFsDJebC00yf4pML20YcseMDOUzdLmq599CALtl64uK0vTvvWzZTS24CDEXFFfdHHgL+KiPcCTwXuzey6Avwk8DbgPODYXxX+VUop/wcqJ4iIVwC3pJS+4geYUnpzSumxKaXHnthQQVIZs5oF9WNr8mC7eSCV1JcsOLH5iqTp60sWHGu8sojOqBlLSumaiHhmRLwUeC9Vt5uXR8TbgccBfwocAc4HiIgAfoLqZdn/ApzY9eQwX+mkfVNKKSJeVd/Pf1vfl1YZREvzhZj85S9H/o+q29w+yj9jEEvjdd2mNEv6kAUpYLwls22Qz4Px1snb7hvlY/K21aPZbW3GmAfqtz5kQZulQf4Y3DWcfG569iATLMDOwaQvoZJr6ADwv1dcjKrfepEFCVKa3Li27Zw/d+yee0oH/jM1bnl97UhLQ8d88pzeGc/RSym9A/g08LXACyLiz4GLgGMtI98H/GBEfIjqB/s+4GXAsTare1pu/qR9I+J5wPunEeaSpssskARmgaSKWTC7zni8AsAJL6f+/oRtB4Ann3LxoyZcb98Z7PuBiNgbEftSSvvX8hgllWcWSAKzQFKlyywAiIi9wF7zoN0Zv6K3CfZS/ZGnpMW2F7NAklkgqbEX8+C0ii70IuKqiLguIm6IiGsjYiki3hgRN0bEu6OefxER+yPiRRFxa/35C4HXAz9eb7tgwm0/JyI+EhEfGR2+p+SXIWmDSmZBfb3jeTC+xzyQZlWnWXC3WSDNsq7WCSsHcz1h5l8Xr+jdmFL6DuCLwI8A21JKjwP+K3BlfZ0HASml9GiqD34V+NfAW1JK+1JKt596o3bdlHqnSBbU12s67dl1U5p13WSBXTelPii+TljeubhdN7tY6H20/u+tVH+Y+Rf15x8Gvq7++CDwhg4ei6TNYxZIArNAUsM8KKiLhd631P/9RuCvqQYbUv/34/XH96aUTu11fAQ4G463Ym2XJteQcbZy2vYZQ7aOpmG2YpCyJS2IbrIgWqpFikylyNZKGmQrImVLWnCdZEEEDIbjyRUpW9sGRydWm5WUr3EaZOve8dZsSQuim3ODjLYsGKeYWAPG2WpbJ2yGLhZ63xwR+4FdwHuAIxHxQeDpwOta9jsAXB4RNwI/XPxRSirNLJAEZoGkhnlQ0JrGK6zTr5zS+vSKU68wqZ0q1UyN61NKV5V5WJI6ZhZIArNAUsM8KKiLV/SuLNVdS1KvmAWSwCyQ1DAPCupt181wvILUN8U67Z3UUt08kGZdJ1kwOmQWSD1QfJ3geIWyinTTcbyC1DvFOmud1FLdPJBmXSdZMDzXLJB6oPg6wfEKZW1q1802K4wm1nLk62hK2RrGOFsxIFvSgtjUzlqnlenUOSayNWqptk5e0oLrqOtmYnl5NLEGpGxti5WJtZJG2bo3DbPVpq1bt7QgiudBgmwHzbZz95z17AMwSoNslTLLXTdvAi61m440N8wCSWAWSGqYBwXNctfNi4GrT9lXUn+ZBZLALJDUMA8KKrrQ22DL073APmD/FB6KpE1kFkgCs0BSwzwor+hbNyPiqoi4zrap0mIzCySBWSCpYR6U53gFSV1xvIIk6CgLVg8tbkt1qUeKrxNWDx7p5AuZRY5XkNQVxytIgo6yYOncxW2pLvVI8XXC0s6zpvJA+2hOxisEMZ5cIwbZWknjidXWNnUM2VpmlK2IlC1pQcz0eIU0mFxtcu2ax6n9YY5I2ZIWQGdZMByMJ1bb2JTceKW28Qr3pWG22owZZEtaEJ3kwXp+V+eyoG1MwiiRrTalRjI5XkFSV8wCSWAWSGqYBwU5XkFSV8wCSWAWSGqYBwU5XkFScWaBJDALJDXMg/IcryCpOLNAEpgFkhrmQXlzMl7hcAdfhqQNcryCJOgoC0aOV5D6oPg6YcXxCkV1MF5hx1QeqKSiHK8gCTrKgqHjFaQ+KL5OWHa8QlHl26amfLW1P8+NSWiz3FJbYpQtSbM9XoGYXOsdoSApq5MsiEj58QrrOK7HLf/aLA1G2ZLUTR7klgptpj3uoG3km+MVJPWdWSAJzAJJDfOgIMcrSOqKWSAJzAJJDfOgIMcrSCrOLJAEZoGkhnlQXm/HK9h1U+qP0i2U7bop9UOXWbB60K6b0izrap1g182yirRNteum1DvFWqrbdVPqlU6yYGmnXTelHii+TrDrZlnF2ihL6hWzQBKYBZIa5kFB8zFeAVrGKwyyNUppYg0YZ2s5IluDGGdLUnfjFVJMrlaZ8QptLY9HRLZaR7uklC1pAWz6qJX243PyOcN6DRlna5QG2ZIWRPE8SC2/k9uOweUYTaz8KmHAMMhWW+4MSNnaiFker3AAuNy2qdLcMAskgVkgqWEeFDTL4xX2ANdvsCOPpNlhFkgCs0BSwzwoqItX9K4s1V1LUq+YBZLALJDUMA8K6m3XzXC8gtQ3xTrtnZQH9zheQZpxnWSB4xWkXii+TljkLOht103HK0i9U6yz1kl5sN3xCtKM6yQLHK8g9ULxdcIiZ8HcdN2McUysNiuZGkbKF5Gttg6fkjrstBcpU2Qr07h33VKKbI1I2ZIWwKZ33WxzNA0n1qDl35CUrfV27pUWxKbmQWsnzNyx29adH7K1kpayVcosd928CbjUbjrS3DALJIFZIKlhHhQ0y103LwauPmVfSf1lFkgCs0BSwzwoqOhCb4MtT/cC+4D9U3gokjaRWSAJzAJJDfOgvKJv3YyIqyLiOtumSovNLJAEZoGkhnlQ3pyMV7CdutQDjleQBI5XkNRwvEJBczJewXbqUg84XkESOF5BUsPxCgXNzXiF9fRGH6XJtRyrLTXIVlub5IiULWlBzHRL9fVoO+bHLSUtuE3PgraRB7nG6eu9PUmtNjUPRgyytR7DiGxtxjgVxytI6opZIAnMAkkN86AgxytI6opZIAnMAkkN86AgxytIKs4skARmgaSGeVCe4xUkFWcWSAKzQFLDPCjP8QqSuuJ4BUngeAVJDccrFOR4BUldcbyCJHC8gqSG4xUKmo/xCgliPLnaWpyvZGpLjLI1aPnXJgYpW9KC2PSW6q0iUy3GaZCtNkMiW9IC6CwLZmGs0ThFtiRt7rnBkHG2lmM0sYYxzlablTTMVqmRTI5XkNQVs0ASmAWSGuZBQY5XkNQVs0ASmAWSGuZBQY5XkFScWSAJzAJJDfOgvN6OV7DLntQfpVsomwdSP3SZBYvcaU/qg67WCYucBb0dr2CXPal3irVUNw+kXukkCxa5057UI8XXCYucBb0dryCpd8wCSWAWSGqYBwXNx3gFgDS5RinWXG2GEdlqMxikbEkLYrbHK2S0tUYfka8BKV8R2ZIWwExnwYDxxFqvtpbqQ1K2pAWxqXkwiJSt3AiFURq0VMrWekcybcQsj1c4AFxu21RpbpgFksAskNQwDwqa5fEKe4DrN9iRR9LsMAskgVkgqWEeFNTFK3pXluquJalXzAJJYBZIapgHBfW266bt1KXeKdZpzzyQeqWTLFjklupSjxRfJyxyFvS266bt1KXeKdZZyzyQeqWTLFjklupSjxRfJyxyFsxP182Mtg43w0gTq62bTts/Sa26y4IUk0vSLOgsC1KKidUm12mvdZ+W7pmb0WlP6pFO8iCXBbnjfRhjlmM0sSb35a1qvdo6fG/ELHfdvAm41G460twwCySBWSCpYR4UNMtdNy8Grj5lX0n9ZRZIArNAUsM8KKjoQm+DLU/3AvuA/VN4KJI2kVkgCcwCSQ3zoLyib92MiKsi4jrbpkqLzSyQBGaBpIZ5UJ7jFSR1xfEKksDxCpIajlcoyPEKkrrieAVJ4HgFSQ3HKxQ0N+MV0mByrceIyNagpdqMx5EtaUFs6qgVAFJLrUNbS/WIfEkLbtOzoK2Vefa4jsjWerWdb0gLonwepPwxPyBlK6dtDNuscbyCpK6YBZLALJDUMA8KcryCpK6YBZLALJDUMA8KcryCpOLMAklgFkhqmAflOV5BUnFmgSQwCyQ1zIPyHK8gqSuOV5AEjleQ1Cg/XuHQ4maB4xUkdcXxCpLA8QqSGuXHK5y7uFkwH+MVguormVCDGGdrmTSxVtJSttq07TceDbIlLYhNb6neaopjF05nnFK2pAWw6VnQNl5hOVYnVpu2Fu1t5yHrPd+Q5sim5sGYyFZuhMIwxtlajshWWxa0ZdJGOF5BUlfMAklgFkhqmAcFOV5BUlfMAklgFkhqmAcFOV5BUnFmgSQwCyQ1zIPyejte4aQue4ftsifNstItlO26KfVDl1lg101ptnW1TrDrZllF2qae1GVvh132pB4o1lLdrptSr3SSBXbdlHqh+DrBrptlFWujLKlXzAJJYBZIapgHBc3HeAUgxeRajnG2hsHEGqXIVpsRka1c53abqWuBbHpLdaKlMgaRstVmvftJC2Dzs2DK2lq0mwVSq03Ng9wIhVEasJKGE6vNcgyyNSRlq5RZHq9wALjctqnS3DALJIFZIKlhHhQ0y+MV9gDXb7Ajj6TZYRZIArNAUsM8KKiLV/SuLNVdS1KvmAWSwCyQ1DAPCupt181wvILUN8U67TleQeqVTrLA8QpSLxRfJzheoawi3XQcryD1TrHOWo5XkHqlkyxwvILUC8XXCY5XKGvuumtJWhezQBKYBZIa5kFBs9x18ybg0o120xnEOF8wsdarrUVrdv7DaUY2SHOksyyINLla98nUeo1TZEtacB1lQZDS5FqPUUr5ahmvtByjbEnqIA8iP/Ko7Xd1bmRKmyGRrc0wy103LwauPmVfSf1lFkgCs0BSwzwoqOhCb4MtT/cC+4D9U3gokjaRWSAJzAJJDfOgvKJv3YyIqyLiOtumSovNLJAEZoGkhnlQnuMVJHWlk5bqY/NAmnUdjVcwC6QeKD9eYYFHrTheQVJXOmmpPjAPpFnX0XgFs0DqgfLjFRZ41MrcjFdYT5e9Ya4iZWtMvtb1AE/3IKX50U0L5dRSudaaHTfDGkRkS1oAm95OPdeBbxCJ+9LyxBq3/FvvfS3HarakBbGpeTBikK3sPi1d9tv+bYa5H68gaWaYBZLALJDUMA8KcryCpK6YBZLALJDUMA8KcryCpOLMAklgFkhqmAflOV5BUnFmgSQwCyQ1zIPyHK8gqSvdjFe4xzyQZpzjFSQd43iFghyvIKkr3YxX2G4eSDPO8QqSjnG8QkFzM15hmkYpstXWXnkY+ZqRru7SZupdFpzOIMbZSimytcwwW9ICmOksWElLE6vNkJQtSa3K50GCcYqJNWScrfVoWye0nTOU4ngFSV0xCySBWSCpYR4U5HgFSV0xCySBWSCpYR4U5HgFScWZBZLALJDUMA/K6+14BbtuSv1RuoWyXTelfugyC+y6Kc22rtYJq4fsullSkbapdt2UeqdYS3W7bkq90kkW2HVT6oXi64Slc+26WVKxNsqSesUskARmgaSGeVDQ3I9XWE3DbE1bW3vlGIyzJS2I7rIgZWodci2Zq7bMtlSX1mGmxyvkDFr+Lcc4W20ZIql8HiRgNI6J1WaUBhOrTVtObMY5wyyPVzgAXG7bVGlumAWSwCyQ1DAPCprl8Qp7gOs32JFH0uwwCySBWSCpYR4U1MUreleW6q4lqVfMAklgFkhqmAcF9bbrZjheQeqbYp32HK8g9UonWeB4BakXiq8TVg86XqGkIt10HK8g9U6xzlqOV5B6pZMscLyC1AvF1wlLOx2vUFIvu2tJmjqzQBKYBZIa5kFBs9x18ybg0pLddFYyNWaQrTb5vcZEkC1pQcxsFrRZ73iFiHxJC66TLEip/fjNVe73+DAiXy1ZsJKG2ZLURR4E4/FgYq3HMMbZWo5htgYxzlYps9x182Lg6lP2ldRfZoEkMAskNcyDgoou9DbY8nQvsA/YP4WHImkTmQWSwCyQ1DAPyiv61s2IuCoirrNtqrTYzAJJYBZIapgH5TleQVJXHK8gCTrKgtGhxW2pLvVIB+MVFve8wPEKkrrieAVJ0FEWDM9d3JbqUo90MF5hcc8LFnq8wihNrhLswCfNQBakdVSL5Rhnq8245Z+0ADY9C1bTMFtbYjSx1qutw6ek8nmQEoxSTKw2a+3MW1Vka8g4WylFtjZioccrSOqUWSAJzAJJDfOgIMcrSOqKWSAJzAJJDfOgIMcrSCrOLJAEZoGkhnlQnuMVJBVnFkgCs0BSwzwoz/EKkrrieAVJ4HgFSY3y64QFzgLHK0jqiuMVJIHjFSQ1yq8TFjgL5me8wjpao48z1dY2tf328ntK2vyW6usdo5AzIGVLUlYnWRABg0gTq23kwXKsTqw2IyJb4zTI1pCULWlBbP65QcaIwcRqM4xBtjaD4xUkdcUskARmgaSGeVCQ4xUkdcUskARmgaSGeVCQ4xUkFWcWSAKzQFLDPCivt+MV7Lop9UfpFsp23ZT6ocssWD1oFkizrLN1gl03iyrSNtWum1LvFGupbtdNqVc6yYKlnWaB1APl1wl23SyqWBtlSb1iFkgCs0BSwzwoaD7GK0y5ZfqWGGVr0PKvrb1yBNmSFsTMtlAGICZXrj37IBJjIlsp5WtEypa0AGY6CyJ6cqoAACAASURBVHLjDkaprSJbgxhna9hS0oLoZtxKptoMGU+sEuPU2ka+bMQsj1c4AFxu21RpbpgFksAskNQwDwqa5fEKe4DrN9iRR9LsMAskgVkgqWEeFNTFK3pXluquJalXzAJJYBZIapgHBfW26+ZJbVNtpy71QbFOe45XkHqlkyxwvILUC+XXCY5XKKpIN52T2qbaTl3qg2KdtRyvIPVKJ1ngeAWpF8qvExyvUNRMd9eS1BmzQBKYBZIa5kFBs9x18ybg0o1202lteQwTq4SU8iUtiE3NAiDfX7nlV0TbeIU2baMXhi0lLYDNz4IOtWXIKA2yJS2I4nkQkRgOxhOrzXqO21EaZ2s993W6c43TmeWumxcDV5+yr6T+MgskgVkgqWEeFFR0obfBlqd7gX3A/ik8FEmbyCyQBGaBpIZ5UF7R9wZExFURcZ1tU6XFZhZIArNAUsM8KM/xCpK64ngFSeB4BUmN4uuE1YOOVyjJ8QqSwPEKkiqOV5B0TPF1wtJOxyuUZNtUSWAWSKqYBZKOMQ8KmvvxCm2GMbnWfXukbKUU2ZIWxMxmAUDKVJuVNMiWpKxNz4JximyNmFxthpGytRyjbK2kYbakBdFJHkSkidVmGOOJlcuIEcGYlK3N4HgFSV0xCySBWSCpYR4U5HgFScWZBZLALJDUMA/Kc7yCpOLMAklgFkhqmAflOV5BUlccryAJHK8gqeF4hYIcryCpK45XkASOV5DUcLxCQfMxXiHyNU6DbA1gYo0YZKvNgHG2UiJb0oLoZQvl9XTnO12HvkHLP2kBdJQFk7vsRSQGLZXtoB2RrfW6Ly1nS1oQvTw3WI/1ri82YqHHK0jqlFkgCcwCSQ3zoCDHK0jqilkgCcwCSQ3zoCDHK0gqziyQBGaBpIZ5UF5vxyuc1HXzsJ21pFlWuoWyXTelfugyCxa5057UB12tExY5C3o7XuGkrps77Kwl9UCxlup23ZR6pZMsWOROe1KPFF8nLHIW9Ha8gqTeMQskgVkgqWEeFDQf4xVaDGKcrTFMrFGKbElat81voZzylZvS0jZeoc0wxi01/TbtUo90lgW58QptlmM0sdY7FqUtQ46mpWxJC6KTPEgpJlabAWlite8T2Wob+VbKLI9XOABcbttUaW6YBZLALJDUMA8KmuXxCnuA6zfYkUfS7DALJIFZIKlhHhTUxSt6V5bqriWpV8wCSWAWSGqYBwX1tutmOF5B6ptinfYcryD1SidZsMgt1aUeKb5OWOQs6G3XTccrSL1TrLOW4xWkXukkCxa5pbrUI8XXCYucBXPfdVPSzDALJIFZIKlhHhQ0y103bwIu3Wg3nWGkbI0SE6vNuOVfm8EgZUtaEJuaBQCkaCkm1zoNIuVrnW3apTnRSRYEMIjJ1WZImlhtv//bxjKtpGG2JHWRB5NHK5xuvEJuRFKfzHLXzYuBq0/ZV1J/mQWSwCyQ1DAPCiq60Ntgy9O9wD5g/xQeiqRNZBZIArNAUsM8KK/o+4Qi4qqIuM62qdJiMwskgVkgqWEelOd4BUldcbyCJHC8gqRGB+MVFve8wPEKkrrieAVJ4HgFSY0Oxiss7nmB4xUkdcUskARmgaSGeVDQfIxXyLVFTzAgZStnzCBbbf/a9osgW9KC6KylerZSvtYzXmE5xtlqH68Q2ZIWwOaPWpmyMZGvNMjW9sH92ZIWRAd5kIiYXG1y64eVtJStMSlbIyJbucd3usd4Oo5XkNQVs0ASmAWSGuZBQY5XkFScWSAJzAJJDfOgPMcrSCrOLJAEZoGkhnlQ3nyMV7CdutQHnbRUNw+kmed4BUnHdDBeYXGzYD7GK9hOXeqDTlqqmwfSzHO8gqRjOhivsLhZMDfjFbKd9NbhNP0zs9XWTUdShy2Ux5lq6a6Zy5C27plD8tVmGINsSQtg09upj1Nk6yiDiTVKKVvrZddNqaN1QqajZdvv+JxRimytpFG22qzncZyJ+RivIKkPzAJJYBZIapgHBTleQVJXzAJJYBZIapgHBTleQVJxZoEkMAskNcyD8no7XsEue1J/lG6hbB5I/dBlFixypz2pD7paJyxyFvR2vIJd9qTeKdZS3TyQeqWTLFjkTntSjxRfJyxyFvR2vIKk3jELJIFZIKlhHhQ0N+MVcsZEtpaDidU2XKGtLfoo5SslsiUtiE1vqd4qN3qhxYCULUlZnWRBAkbjmFht7kvLE6tNWxYMYpyt5VjNlrQgiudBAMNBmlwxztZ6jNv+pUG2Spnl8QoHgMttmyrNDbNAEpgFkhrmQUGzPF5hD3D9BjvySJodZoEkMAskNcyDgrp4Re/KUt21JPWKWSAJzAJJDfOgoN523bSdutQ7xTrtmQdSr3SSBYvcUl3qkeLrhNVDi5sFve26aTt1qXeKddYyD6Re6SQLFrmlutQjxdcJS+cubhbMfddNSTPDLJAEZoGkhnlQ0Cx33bwJuPRMu+mkmFxttkRMrBLGo0G2pAXRSRYARJpc0zaMlK1BS0kLrpvzghSMxoOJNU7RUmtvf96WBcsxylbbWCZpQXR2bjDJtMckjUgtFdkqZZa7bl4MXH3KvpL6yyyQBGaBpIZ5UFDRhd4GW57uBfYB+6fwUCRtIrNAEpgFkhrmQXlF3xsQEVdFxHW2TZUWm1kgCcwCSQ3zoDzHK0jqiuMVJEFXWbDALdWlHnG8QkGOV5DUFccrSIKusmCBW6pLPeJ4hYIcryCpK2aBJDALJDXMg4LmY7xCUH0lk6rFOnZpNWaQrZQiW9KC6KyF8nrGrazHKEW2JGV1lwWZapNrfz6MliJla2kwzlbbeYO0IDZ1vMIwxtkaExOrdZwKw2zlRrdUlR/5shGOV5DUFbNAEpgFkhrmQUGOV5BUnFkgCcwCSQ3zoDzHK0gqziyQBGaBpIZ5UN58jFc4bDt1qQccryAJHK8gqeF4hYLmY7zCDtupSz3geAVJ4HgFSQ3HKxQ0N+MVcl32BqRsrcdKGmVLUqvNb6Eca6+2TlgrDLIlKWvzs2AdBi3/1muUBtmSFkQ364RM5/v1HINDxtlajmG22u6rVHf++RivIKkPzAJJYBZIapgHBTleQVJXzAJJYBZIapgHBTleQVJxZoEkMAskNcyD8no7XsGum1J/lG6hbNdNqR86zYIF7rQn9UFX64TVg4ubBb0dr2DXTal3irVUt+um1CvdZMECd9qTeqT4OmFp5+JmQW/HK0jqHbNAEpgFkhrmQUFzM14h0uQaEy3FxFqvAeNsRaRsSQuiu5bqmVEJuTEsbTWIlK220QttJS24zrIgNzml7bjOGbf8O5oG2WqzHKvZkhZE8TxIwDhlqmWdsJ7xbG3jFdruq5RZHq9wALjctqnS3DALJIFZIKlhHhQ0y+MV9gDXb7Ajj6TZYRZIArNAUsM8KKiLV/SuLNVdS1KvmAWSwCyQ1DAPCupt181wvILUN8U67TleQeqVbrLA8QpSHxRfJzheoawi3XQcryD1TrHOWo5XkHqlmyxwvILUB8XXCY5XKKu7TnuSZplZIAnMAkkN86CgSKlce/+IuAr4JuBc4AtUL8m+nuqHeSfwYymlOyNi/6l/aBkRW4B3AucBv55S+v3c/Tz2sY9NH/nIR4p8DV2LiI+mlB672Y9DmqausgDmJw/MAs0js2DtzALNK9cJa7fWPJjlrpsXA1efsq+k/jILJIFZIKlhHhRUdKG3wZane4F9wP4pPBRJm8gskARmgaSGeVBe0b/Ri4irIuI626ZKi80skARmgaSGeVDeXIxXuP32iV2WJc2WTlqqmwfSzDMLJB3jOqGguRivcMEFLuSlHuikpbp5IM08s0DSMa4TCnK8gqSumAWSwCyQ1DAPCuqi6+Y3R8R+qrap7wGeHBEfpG6b2rLfTcDLI+JG4NeBbNvUj370o1+OiNtOuOgBwJcnXDV3eYlt6729SzOXS31XPAvgK/Kgy2N32vdlFmhe9SUL2rZ1mRNmgeaZ64S1bVtTHnQxR29/161PI+Ijk2ZM5C4vsa3EfUl9NWtZ0LatDzkh9VWfsqBt26zkhNRns5YHs3JcT/PcYJbHK0iaE2aBJDALJDXMg/K6+Bs9SZIkSVKH5nWh9+Y1Xl5iW4n7krQ2s3LsTvu+JK3Nph+7EfEg4AMRcc4U70vS2vX59/+a8qDo3+jNooh4C/DbKaUPRsQrgM+llN6yztu5KqX0t5ntXwe8JqX0/et/tJJK6SILIuIhwFuBMfBp4Llp0UJXmnEdZcEjgN8G/hT4fuBbU0pHN/CwJRXQ1Tqhvs6jgKtTSk9Z36M9vXl9RW9TRcRXA68Ddm72Y5G0qZ4L/FRK6YnAJcDXb/LjkbQ5Hg08O6X0C8BngIdt8uORtInqkRC/AiyXvB8XekBEPDAirouIP4+Il9WXXRwRH4yIGyPilfVlD6uvcz1wWctN3g08vYOHLmmKpp0FKaWfTSl9ov70fPLtkiXNkAJZcC1wW0R8D7Cb6hV+ST1QYJ0A8Gzg/YUf+sIu9N5Yz+z4ifrzlwH/OaV0OfDPIuJ8YA/wb4DvAr63vt5LgV8Gngbk3mNPSulLKaX7Cz12SdNTNAuOiYgfBj6eUvr76T58SVPSRRbsAH4IuA3wLdzS7CqaB/X+Pwr8v0Ue/Qm6GJg+i6444b23UA0f/LaI+HFgO3AxsAr8PHCY5of1MOCWlNJqRNzc8WOWNH3FsyAivgp4MfDkAo9f0nQUz4KU0l3Av4yI3wO+GfiL6X8ZkqagdB68BnhZSmmlegdnOYu60DvVJ4F3pZTeHxE/Chyg+iG8GrgFuLW+3t8Bj4yIv8O/tZHm0VSzICJ2A+8A/lVK6WDRRy5pmqadBb8BvCOl9AFgF3BXyQcvaaqmvU74DuDh9SLvMRHxSymlV7Rcf91c6FVeA/xORPwS8Fng94H3AL8J3A7cGxF7qF6O/U/AzwB2y5Lmz7Sz4N8AD6F6GwjAz6eUbij38CVNybSz4JeB34uIBLw3pfTJkg9e0lRNNQ9SSo849nFE7C+1yIMFHK8gSZIkSfNuUZuxSJIkSdLccqEnSZIkSXPGhZ4kSZIkzRkXepIkSZI0Z1zoSZIkSdKccaEnSZIkSXPGhZ4kSZIkzRkXepIkSZI0Z1zoSZIkSdKccaEnSZIkSXPGhZ4kSZIkzZmFXOhFxNaIuGTC5XHK53si4gGTrhcR2064rR2nbFsq8bglTZdZIAnWlAUXRsQDI2I46brmgdRv85YFM7vQq795T5zSbT08IrZFxNkRcRHwCOD36m3fHRE766t+NiL2nbDrrwH/OyJWI+KuiLg9IhJwM/CliLgW+BvgdSfs80zgjzbwWE98PJKYXh60ZUG9/djx95sRsT8idtebjmXB5yLiaJ0Fq8BR4EMR8THgj4HPRcTWeh+zQJqyGciCy4APADd4biBtnlLrBODbmKMsiJTSeu+rmDp0Xwe8MKV0YML2xwCklG4+w9v7E6pv+vcBu4DPAMvAAeBs4GuBf1xv/6fA+4FvBwIYAo8H/hL4ReAdwJOB7wdeBXwY+A/A2+vrf7je9lDgxpTSnWf4GK8C9lP9D/Ea4IqU0l1nsq80z6aZB6dkwdnA54Cd9X8HwHnA/wFeCfwG8HHgwfXuy8BXAxcC/wL4d8B9wIuAZwCfAF4MfEd9fbNAmqIZy4L7gacAv5BSelVE3IPnBlInCqwT/hq4h+rYvwi4gznJgpl7RS8iLgZ+meoL+IofXu0xdZ3J7X098AWqH+BdwArwAKoTtG3Ap6l+aB8EvhcYUS3stgL/iOqXwCrVD/Nn6n2OrY73AQ+hWv1/CPhd4E0ppVvq23/TmX3VjZTS54GXAG844VkDaSFNMw8mZMGdVFmwTHVcP48qBy4ErgOOUB3fW4DLgX+gOs4PU2XB11A9o5fq672Q6omkD2IWSFM1g1nwMqoTvB+KiP14biB1otA64QjVk7b/QLVOOJYFW4Fb6HEWzNQrehGxh2rF/IKU0qGIOAt4J3Au1er6GVSvqv1AvcvnU0pPqt/7ei2wHfh0SunZ9e3dQPV2jHuAG6i+4WNgB3AvcDvw+vr2bgBeDvwW1Ur5fwA/Xd/vx4HvSin96Qkr9X8G/CzViv/bgBupfnF8FfAlql8a24APppSeVq/El4HH1V/P06j+x3gn1auGAVyVUtpfP/YLqZ6teFHL/8jS3JpmHkTEcr3PX1I9i3Yr1ZM8zwCWqEL7ucBjqY7LbwF2A0+lekZuGXg11cnfHSmlyyLiZqqgfjHVk0QPpXqWby9mgTQ1M5QFr6Q6dv898FZgD/DMlNJ+zw2k8gquE7YAPw/8FNUr+2dRZcODgXfR4yyYtVf09gEfSykdqj+/DBinlB5PtQrekVJ6GdVC7DUppSfV13sQ8Eaqb+zeiHhgffmx1fZ/ovomv5FqNf4Z4IvAzwFPAB5G9QN9H9VJ2lOARwHXU70dawi8OyLupPrhvwl4FvDP621PA24DvhM4H/hNqr/pOR84EhGPrh/P19Rfyx8ATwSeA7wnpfQEqmcQjkspfal+nN+69m+jNBf2Mb08eBZVwH8OOAS8F3g41TF2F9UvhydQBfo3U71//g6qQP+hev9PUJ0IPrzOgkcB3wi8mert3zuogtsskKZrH7ORBd9LlQe3UmXBhcAfeW4gdWYfZdYJn6T6k4zbqF4AWgJ+hOr3fq+zYKYWeimltwEHI+KK+qKPAX8VEe+l+sbem9l1BfhJ4G1U76U9q778Fqq3WEH17N0dVG/N3AF8iirsL6N62fW3qd6D+wmq1fTTgX8LvIBqRf3W+j4+QbXifyvV6nwL1Q/5JuCa+navpHqW/xDVInJP/RjeWv/37+r9HlY/RoCPnPgFRcQrgFtSSn+c/YZJc2zKefAWqmftP0GTBatUWfBF6ixIKd0IvJbqbRF/n1J6O1Ug/y5VFnyJ6hW93cD/pXom8F/VH19Q379ZIE3RDGXBW6n+Tv8FVK/0j6ienffcQOpAwXXCuVQLvSHwdcCXqY7nC/qeBTO10ANIKV0DfDkiXgp8A/ChlNJ3Ur1c+rj6akeo/nj6WLvTn6B6SfaZVG/TPOYwVYBD9U38GeBPqJ51vxa4gupVvCdRvXz6GaqXRr+HahV9mOq9+1upfihPoHqv7TGXUf1ieHV9m8P6/l9b3/bNwCuofmCc8tioL39k/fHx9xJHxKuAm1JKf9j+3ZLm27TyIKU0pvkFcCwLfqu+nVtpsgCqPNgCfLF+7/5zgY9SZcGDgdvqt4vsAH6n3ucw1QnkwzELpKmbsSx4P9Xbw47Ut+u5gdSRQuuEHcBB4BKat1aOjt0GPc6CmVvoAaSU3kH1jNvXAi+IiD+n6oJzbDX7PuAHI+JDVD/U91G9Kvdn9fY9J98iy1Tf7JdTvY0z6us/muoHcgPwUqqXUF9CteD7Mar36f4O1R9VHvuBrlC9BHvsdm+henl3G9WJ4EOpFomvpfof8HlUz/ZP8mbg6fUfb54LEBHPA96fUvpvZ/CtkubelPPgxCy4mepv7h5B1WV3GBHPpzrO3wL8S6q3ZlwDnEP1qv+HqcL2XVTB/rdUzyICfDfVE0JmgVTAjGTBYapX+C+ielXgXXhuIHVqylmwjeoVt/9I1ZDlCqrj9aeBoxHxq/Q4C2aqGUsJEfF4qpdHb6NasT+a6ln5v6FaXV9MtfD7FNUrd+N614dSrd6vAd5NtUq/ieqPNZ9P9T7fDwBPSSm9qH7G4PnA/Sml367v+2MppW/q4MuUdBqnyYIlqhC9hHwW3Eb1h9dPo3r//M9RvTf+66l+IXzeLJBm3waz4E6q7rpvBx4I/EFK6W0R8Sk8N5B6JyJeSzVi7Q6qRdtWquP/2DphDz3Ogrlf6AHUb7PatoZZFduovjdHTrgs0mm+WVFPuk8prbZdT9LmMAskQXdZUF/PPJBm1LxnwUIs9CRJkiRpkczk3+hJkiRJktZvLhd6EfGctVxeYluJ+5K0NrNy7E77viStzawcu9O+L0lrN+vH9TTPDeZyoUfVzWYtl5fYVuK+JK3NrBy7074vSWszK8futO9L0trN+nE9tXODNS30IuKBEfHEteyzHhHx3RGxs/T9SFofs0ASmAWSKmbBbDrjZiwRcRHwOuCFKaUDE7Y/BiCldPOaH8Qp+0bEHuA1wBUppbtOt//wnO1p6fzdxz8f3X0Pw3O2V5+Mo7n88D0Md2w//vmWbSvHP145eC/LO6u5iKPU7AOwevBeluptO5aPnrTtyJ33cdbubdW24X3HL7/7wCrnnLd0/PPzBqPjH99+x4gLzh8e//xTt57d7Medh1NK55zua5Y2yyxnAcADzhumvZcsA195rB0YNx+feozeO95a/ffO+zl799bjlx9e2XL845WDR1jeedbxzxNNVpyYEwAD0gn7NfkCsGVYNdy676772LZr20mPf+Wvq+7N93KI1bR6chhJM2TWs2Dp3LPT8oW7ABgdupfhuc0xuOes5iYOHVjl3BOy4P/edf7xj0f33MNwe33esDxuLj/xPAM4/6xmzvG9dx7l7N1Nbtxx747sfhdsv/v4x/ccOMr285r9Dn28yiuzQLNu1rNg265taceDmuPwxN+994+aY//U39Xnbzl8/OPDd66wY3d1bvHlo81tVfs15wZnLa2ctO2+O+9nW31O8dDlJidOPT/5y4MPOP7xqTmx9bZ7j3+81nXC0umvAhFxMaf/ph6b2L7mH+Kp+6aUPh8RLwHeEBEvPF3L06Xzd3PRv33BxG2Dw8OJlwNc8sgvTLz87vu3TLwc4J886LPZbd9+7qey235ox8HstqdefHzYPdenaz+ZvaK0yWY9CwD2XrLM//yTSyZu+y+H808CfvSeh028/ENf/KrsPqc+KXSircNRdttDzvmK34PHffHbDgHwF+lPs9eRNlsfsmD5wl189a/85MRtv/TId2X3+5k/+rHJGx503+TLgWc98n9mt11z87dmt/3UP7ohu+36R1XncmaBZlkfsmDHg3bwPdd838Rtnz10/sTLAX78IX8+8fL/8Lffnt3nG87/++y2N+35cHbbw979/2S3PeK5/+v4x2tdJ5x2oVevnF8J/HRK6VA9b+KdVANF7wCeAfwi8AP19Z+VUnpSROwArgW2A59OKT273r4f+F/Ao1NKT42IV5+6L0BK6QsR8WLg9RHxoknPEEjqjlkgCcwCSRWzYPadySt6+4CPpZQO1Z9fBoxTSo+PiO8DdqSUXhYRnwRIKb2lvt6DgDcC1wP/PSIemFL6IvCtwBtSSi+prz9pX+rPvxQRn6n3+eMTt9VdZ54DMDxv15q+aEnrso8ZzAI4OQ8esueM3qggaf320YMsWL7AP+ORCttHD7Jg+0XbT928ME7bjCWl9DbgYERcUV/0MeCvIuK9wFOBezO7rgA/CbwNOA849octf5VS+oMzeXAR8QrglpTSV/wAU0pvTik9NqX02BPfxyqpjFnNgvqxHc+DE9/zLmn6+pIFJ/5NnqTp60sWnPq38IvkjLpuppSuAb4cES8FvgH4UErpO4HdwOPqqx0BzgaIiAB+gupl2WcC95xwc4f5SqfuS0S8CrgppfSHa/yaJBViFkgCs0BSxSyYbWf8HqeU0jsi4geBrwWeHRE/C9wHfKS+yvuA/xIRPwK8rP78TcDz6u17gL/N3PxJ+0bEZcD7U0rvO+OvZJDpHtrSVLSt6UrOWcOV7Lbzh5P+/6zcn/L7SX0y61lwfxrx2ZXJx+Jdo4uz+62OJz/vddZy/tgdtzZjWc1uO3fp/uy2L2a3SLNl1rNgMEicvWXy8dv2+zpl3hSQVvLPjd9+NN8EL3LnJ8C9o63ZbVJfzHoWJIKV8eQDezgYT7wc4O7RWRMv39LSbG0p8tsOj/MNndiSfxwbsaY/Zjnh5dTfn7DtAPDkUy5+1ITr7TuDfT8QEXsjYl9Kaf9aHqOk8swCSWAWSKp0mQUAEbEX2GsetFvTwPSO7aX6I09Ji20vZoEks0BSYy/mwWkVXehFxFURcV1E3BAR10bEUkS8MSJujIh3R8Tu+nr7I+JFEXFr/fkLgdcDP15vu6Dk45RUllkgCcwCSQ3zoLwuXtG7MaX0HVR/evIjwLaU0uOA/wpcWV/nQUBKKT2a6oNfBf418JaU0r6U0u2n3mhEPCciPhIRHxkdvufUzZJmT5EsgJPz4MCBMu9zlzQ1nWTB6sFcwz9JM6T4OuG+u1r+Nm7OdbHQ+2j931uBi4C/qD//MPB19ccHgTes5UZPaqG8w/EKUg8UyQI4OQ/OO2+W35EuiY6yYGmn4xWkHii+TnC8QlnfUv/3G4G/phpsSP3fj9cf35tSOvVp+K9opyqp18wCSWAWSGqYBwWtqevmOn1zROwHvgC8B3hyRHwQuBP4sZb9bgJeHhE3Ar/OhC4+xwXE0trfrjXOtFOPyLdCbmunPmyZ5bCS8u1WpQVRPguAMcF9afKx3Xb85gzaZrS03NygLUfadpTmXydZMCCxbWnymJNzBkez+6XMOIQ0yj83fv84fzqVWnLncOt4hZbskeZHJ3mQ03ZeMMr8rl5qGcmwmpvPAqx8xVq10TaGZSO6WOj9yimtT6849QqT2qkCFwNX2zZVmhtmgSQwCyQ1zIOCii70UkpXbWD3vVRtU/dP4aFI2kRmgSQwCyQ1zIPyHK8gqTizQBKYBZIa5kF58zFe4W7HK0g90ElL9TsdryDNuk6yYMXxClIfOF6hoPkYr3CO4xWkHuikpfpuxytIs66TLFh2vILUB45XKGg+xitEIoaTiyBfuZuLlK1ximwdTcNsjUjZkhZEL1sorzcPJGV1kgURieXhaGLtGqxmiwGTaxTZWk2DbDGObB0ZbcmWtCA6yIPEIMaZStnKGcY4W21WSNkiRb42oIuF3rG2qbuo2qYeqdumPh14Xct+NwGX1m1Tf7j4o5RUmlkgCcwCSQ3zoCDHK0jqilkgCcwCSQ3zoCDHK0gqziyQBGaBpIZ5UF5vz0d1qgAAHqZJREFUxyvYdVPqj9ItlO26KfVDl1lw9OCR7r4wSWvW1TrBrptlFWmbatdNqXeKtVS366bUK51kwZadZxX/QiRtWPF1gl03yyrWRllSr5gFksAskNQwDwqaj/EKQAzSxFrPeIVB5GslDfNFvsYpZUtaEN1kATCMNLHajBhMrPVaT8tmaUF0Nmol1wL9nBhkKw3SxGJMtlbGw2ylcWRrJQ2yJS2ITR291Pa7epwGE6ttnwH5atOWExsxy+MVDgCX2zZVmhtmgSQwCyQ1zIOCZnm8wh7g+g125JE0O8wCSWAWSGqYBwV18YrelaW6a0nqFbNAEpgFkhrmQUG97boZjleQ+qZYpz3HK0i90kkWrDheQeqD4usExyuUVaSbjuMVpN4p1lnL8QpSr3SSBcuOV5D6oPg6wfEKZW1qNx1JM8MskARmgaSGeVDQLHfdvAm49Ey76eQmKKTIV85wMM5Wm5W0lC9StqQF0UkWJGCUYmK1WR0PJ9Z6xySMU2RrKUbZkhZAZ+cFuWN3ayxliwGTq+WEou14bxvLkMud1fFwfd9ZqX86yYP1GBETq83SYJStUUrZIpGvDZjlrpsXA1efsq+k/jILJIFZIKlhHhRUdKG3wZane4F9wP4pPBRJm8gskARmgaSGeVBe0bduRsRVEXGdbVOlxWYWSAKzQFLDPChvLsYrjB2vIPWB4xUkQUdZcNTxClIfOF6hoLkYrzBwvILUB45XkAQdZcEWxytIfeB4hYIcryCpK2aBJDALJDXMg4LmZrzCekSkidU2XmGcBtlqczSlbEkLorPxCisMJtaopbq0dbCaLWkBbPp5wTAiW+sxJrJVYj9pjnSQB5E9dx+QspXTNnZpyDhbI8hWdk7cBqPA8QqSumIWSAKzQFLDPCjI8QqSijMLJIFZIKlhHpTneAVJxZkFksAskNQwD8pzvIKkrnTSUv0uxytIs87xCpKOcbxCQY5XkNSVTlqq73K8gjTrHK8g6RjHKxS00OMVcl03Sxi3lLQgOsmCRLCSBpkaZms9ne/aOm+1lV03teC6OS+I/DG6LpGyNU6RrTajFNmSFkTxPEjAahpMrNxaoG090Pb7vc1KylcMUrY2YqHHK0jqlFkgCcwCSQ3zoCDHK0jqilkgCcwCSQ3zoCDHK0gqziyQBGaBpIZ5UF5vxyvYdVPqj9ItlE/uujnq7guTtCZdZsHRu+y6Kc2yrtYJ999p182SirRNteum1DvFWqqf3HVzWPwLkbQhnWTBll123ZR6oPg6Yetuu26WVKyNsqReMQskgVkgqWEeFLTQ4xUkdaqzLBgRE2ucBtnKWW/b9AEpW8sxypa0ADb9vGCUUrayIl+tOdG639ozSZoz5ccrpPwx2iZ3bLb9fm+9vZZqG9+yEbM8XuEAcLltU6W5YRZIArNAUsM8KGiWxyvsAa7fYEceSbPDLJAEZoGkhnlQUBev6F1ZqruWpF4xCySBWSCpYR4U1Nuum+F4BalvinXac7yC1CudZIHjFaReKL5OuP8uxyuUVKSbjuMVpN4p1lnL8QpSr3SSBY5XkHqh+Dph6y7HK5S06d21JM0Es0ASmAWSGuZBQbPcdfMm4NKS3XQGMbna9xlnS1KrTrIgkW+JnBu7MGJ9vyPaWiy3ZcWYyJa0ADb9vGAYkS1JneogD/LjTwaRspWz3rXA0TTIVimz3HXzYuDqU/aV1F9mgSQwCyQ1zIOCii70NtjydC+wD9g/hYciaROZBZLALJDUMA/KK/rWzYi4KiKus22qtNjMAklgFkhqmAflOV5BUle6Ga9wh+MVpBnneAVJx3QwXmFxs8DxCpK60s14hfMdryDNOMcrSDqmg/EKi5sFjleQ1BWzQBKYBZIa5kFBczNeIWUqUr5yoqXW04ZVEjADLdVzYxfG62xtHJGytTQYZ2uUBtmSFsCmZ8H/397dxcp63WcBf9bsc+yoENu4lBYbKqeALBW1EqgR1BJgUPi64ENUUCEgQqJ3FBCowgkqxBUqrVJVligR4UOocBW+bgrCQC1qQVtxkWBA3FRqUcuHFJRyHDuoTXzOzOLi7K1325r1umdm1pp5Z36/6C/Ze+/Zeyb2PPMu7zn/ZzXzv6bWhcYeL/+tle+b6rqVi3HUPJi7rm+9Ts+9vs99v9lqpToze1CvAIwiC4BEFgATedCRegWgO1kAJLIAmMiD/tQrAN3JAiCRBcBEHvR3FvUKa/UKsATqFYBEvQIwUa/Q0VnUK1ypV4AlUK8AJOoVgIl6hY7Opl6hbsrWmduStWnM3Macq2yaA8wakwUpeSdXW2fOupatM5cHc3OnbJpj0x4Xbsw69drearmTma14u27ktskbRuTBbs+z5obMDmptzz7Opl4BOHmyAEhkATCRBx2pVwBGkQVAIguAiTzoSL0C0J0sABJZAEzkQX+LrVewdROWo/cK5dt58Jatm3CyRmbBO29d7qY9WIJR54SvfPHLYx/YCVlsvYKtm7A43Vaq386DJ23dhFM3JAsee/JyN+3BgnQ/Jzz+1AeGPJBTtNh6BWBxZAGQyAJgIg86OpN6he3VCu9Xr9Cy6zr1VTbNWdc0By7EmHqFWvLlzd2tM2dTV1vnzmrTnF2zAi7cmHoFYAm650Ep7Wv7kdYpzSklzdnHKdcr3EvygrWpcDZkAZDIAmAiDzo65XqFZ5O8tudGHuB0yAIgkQXARB50NOI3ei/12q4FLIosABJZAEzkQUeL3bp5e23q+m31CrAA3Tbtvate4d6D7g8E2MuQLFCvAIvQ/ZzwlTfVK/TUZZvOu+oVnlCvAAvQbbPWu+oVnh7xjnRgD0OyQL0CLEL3c8Ljv0K9Qk+2awGJLAAekgXADXnQ0Slv3XwjyfM9t+msyvaZv42V6bCjIVlQk9zP1dZZlU1zWu6UdXPkAexkWBZsatk6s0rdPkAPRz0ntDLifXPi0N+vlTt7Zs8pb918Jskr77ktsFyyAEhkATCRBx11PejtufL0uSQvJnn9AHcFOCJZACSyAJjIg/66vnWzlPJyKeVVa1PhsskCIJEFwEQe9KdeARhlUL3CuvsDAfYyJAvuq1eAJVCv0JF6BWCUQfUKV3vfUaCrIVlwV70CLIF6hY7UKwCjyAIgkQXARB50dCb1CjVltX3mXK02W2duNequa1M3MwMXYtBK9ZJ1XW2dq9TmtNxZbZozZy4r7ter5sAFOHrt0iqlOSMderU7LNDR86CldS2xq7lrkNY55v3OMu9HvQIwiiwAElkATORBR+oVgO5kAZDIAmAiD/pTrwB0JwuARBYAE3nQ33nUK3xJvQIswJCV6m/fe9D9gQB7Ua8A3FCv0NF51Ct8UL0CLMCQlepPPD3ijx4De1CvANxQr9CRegVgFFkAJLIAmMiDjs6kXiEpZfvM36ZunV3Xom+yag4wboVy63l4t6yb03rOr0ptzq7UK3Dhjr5O/aqsmpOSxtT27GiT0hy4EEfNg12u+eess2rOnLJqzz7UKwCjyAIgkQXARB50pF4B6E4WAIksACbyoL/F1ivYugnL0XuF8u08eMvWTThZI7PA1k04baPOCbZu9tVlbaqtm7A43Vaq386DJ23dhFM3JAts3YRF6H5OsHWzr25rlIFFkQVAIguAiTzo6GzqFVobNNvbs5KrUrdOTZrzYHPVnHVdNeedmYELcfQVyquyaU5r890qtTm7mtvyBRdgWBbUWrbOUDMXFQ82q+bAheieB7W2X3dbGVFnXqfntnFfZdOc2euJDlt9k9OuV7iX5IVDrFEGToIsABJZAEzkQUenXK/wbJLX9tzIA5wOWQAksgCYyIOORvxG76Ve27WARZEFQCILgIk86GixWzdvr03dqFeAJei2aU+9AizKkCxQrwCL0P2c8JUvqlfoqcs2ndtrU1fqFWAJum3WUq8AizIkC9QrwCJ0Pyc8/pR6hZ6OvmkPOAmyAEhkATCRBx2d8tbNN5I8/0vZplNKu16hrtKcO6vN1plpZGiuYN+kZD0z97NqDlyI7lnwfubWHt/fXG2duTXKc6uS524HF+7oWbCTmYuDXZ/vcgJONw/WWW2dObs+p+fOHvs45a2bzyR55T23BZZLFgCJLAAm8qCjrge9PVeePpfkxSSvH+CuAEckC4BEFgATedBf1/cNllJeLqW8am0qXDZZACSyAJjIg/7Ool5h/bZ6BVgA9QpAol4BmKhX6Ogs6hWunlCvAAugXgFI1CsAE/UKHalXAEaRBUAiC4CJPOjoLOoVZpXanoYe647v11Vz4EIMyYKaknVdbZ/GquR1VtnUsnXmrMpmp4ELN+y6oFWHdL+um5Oa7TNjrmpl9nbqFWBAHmx/fd/UdmXaZqbYoPm9asndsm7OMahXAEaRBUAiC4CJPOhIvQLQnSwAElkATORBf+oVgO5kAZDIAmAiD/pTrwCMMmSl+tvqFeDUDcmCd9QrwBIMqFe43CxQrwCMMmSl+hPqFeDUDcmCx9QrwBIMqFe43CxQrwCMIguARBYAE3nQ0VnUK9Sa1Fq2TkqaU0rdOnPrjnddoXy/XjUHLsSgeoVknbJ1djG3RnkuK65mZlNXzYELMKxqpfXc/Uq935zUsn1mridmaxJWac5jqwfNgQtx3Bq2GVfZbJ1d7VLlsC/1CsAosgBIZAEwkQcdqVcAupMFQCILgIk86G+x9Qq3t+lsvmTrJpyy3iuUb+fBl2zdhJM1Mgvuf/EXxj0w4JGNOifYutlXl7Wpt7fprD5o6yYsQLeV6rfz4IO2bsKpG5IFd5/6qu4PBNhb93OCrZt9dVujDCyKLAASWQBM5EFHZ1OvUBszkk16MGtIFqxS81hZb52552hrY96DumrOro6xeQtOyNHXqX+5rpuzywXFqmyak1Kbc7dsmgMX4uh50NLcpLvDbValNjeCr1OasbPvWeaU6xXuJXmh59pUYChZACSyAJjIg45OuV7h2SSv7bmRBzgdsgBIZAEwkQcdjfiN3ku9tmsBiyILgEQWABN50NFit27eXpu6Vq8AS9Bt097tPHhLvQKcuiFZoF4BFqH7OUG9Ql9dtuncXpt6pV4BlqDbZq3befCkegU4dUOyQL0CLEL3c4J6hb5OdpsOMJQsABJZAEzkQUenvHXzjSTP771NZ2Zfaa1l62xmZlcfKA+aAxdiWBasstk6c+6u1ltnV+tadhq4AMe9Lkhyv9bm7GLXmoS5VexwIY6aB1dl05yWHs/buinN2ccpb918Jskr77ktsFyyAEhkATCRBx11PejtufL0uSQvJnn9AHcFOCJZACSyAJjIg/66vnWzlPJyKeVVa1PhsskCIJEFwEQe9KdeARhlyEr1t9UrwKlTrwDcUK/QkXoFYJQhK9WfUK8Ap069AnBDvUJH6hWAUWQBkMgCYCIPOrqAeoXSnLkaheakPXMeL+vmwIUYkgUlNY+V9dZZlU1z7jRmzlxWPNhc7TRwAY5er7CemWYt04w7q3VzgFlHz4OWQ1etXaU2Z64Obh/qFYBRZAGQyAJgIg86Uq8AdCcLgEQWABN50J96BaA7WQAksgCYyIP+1CsAowxZqf7WPX8mBk6cegXghnqFjtQrAKMMWan+5NMWmsCJU68A3FCv0JF6BWAUWQAksgCYyIOOzqZeobSmtqelx4bTD5RNc+BCHH2F8lU2zWlXL9TmbOqqOQ9mBi7csCxorUe/SppTatk6c65Kbc5czRNw/GuDlnVWW2eV2pw5c7ertTRnH+oVgFFkAZDIAmAiDzpSrwB0JwuARBYAE3nQ32LrFW5v09nYugknrfcKZVs3YRlGZoGtm3DaRp0TbN3sq8va1NvbdFa2bsISdFupbusmLMqQLLB1Exah+znB1s2+uq1RBhZFFgCJLAAm8qAj9QrAKLIASGQBMJEHHZ1yvcK9JC/svTa1R1dCQ3s9+yZXJc2BCzEsC9YpW2cXd8qmOcBOhmVBqxrlqpTmjNSqf9ioXuBydM+DmvZzbU7z+Zn2zHmsbJrTyynXKzyb5LU9N/IAp0MWAIksACbyoKMRv9F7qdd2LWBRZAGQyAJgIg86WuzWzaJeAZam26Y99QqwKEOyQL0CLEL3c8I76hW66rJNR70CLE63zVrqFWBRhmSBegVYhO7nhMfUK3Rlmw6QyALgIVkA3JAHHZ3y1s03kjy/pK2bc9a1PXAhhmRBTcm6rrbOoe26eau1CXBVBAIX4bjXBbsqtT3Aro6aB7WW5hz6dXo1M6XU5uzjlLduPpPklffcFlguWQAksgCYyIOOuh709lx5+lySF5O8foC7AhyRLAASWQBM5EF/Xd+6WUp5uZTyqrWpcNlkAZDIAmAiD/pTrwCMMmSl+tv3HnR/IMBe1CsAN9QrdKReARhlyEr1J54e8UePgT2oVwBuqFfoSL0CMIosABJZAEzkQUfnX68wY5e1qetamgPMOnoWjKw1UKEATUfPgnWtzdnt+7k2gB0dPQ9arrLZOptadppjUK8AjCILgEQWABN50JF6BaA7WQAksgCYyIP+1CsA3ckCIJEFwEQe9KdeARhFvQKQqFcAJuoVOlKvAIyiXgFI1CsAE/UKHalXAEaRBUAiC4CJPOjo/OsVSnuuVputs6v79U5zgNNdoTxn11XJp7ZiGU7IIrNgZ3VmgJPNg1ZF0tzr+zqr5lyVNKcX9QrAKLIASGQBMJEHHalXALqTBUAiC4CJPOhvsfUKtm7CcvReoWzrJizDyCywdRNO26hzgq2bfXVZm2rrJixOt5Xqtm7CogzJAls3YRG6nxNs3eyr2xplYFFkAZDIAmAiDzpSrwCMIguARBYAE3nQ0Yj3ON2sTf18Hq5N/cj12tQ3k3x05nb3krxwvTb1U0k+s9NPn/lHvyqPvtt4U9tn4/v16pG/H1yQIVlQUnNVttekXOXR61M2cyEyd7uZGgUVC1y4414XvI+6w7XBrna5DoEzc7J50LqW2PV5uz7C0/2U6xWeTfLanht5gNMhC4BEFgATedDRiLduvtRruxawKLIASGQBMJEHHS1262ZRrwBL023T3u08eOveuvsDAfYyJAvUK8AidD8nqFfoq8s2HfUKsDjdNmvdzoMnn/ZnZeHEDckC9QqwCN3PCeoV+rJNB0hkAfCQLABuyIOORhz0brbpPJWH23R+8Xqbzrcl+YGZ272R5PnrbTrf/n4/pDZmzip165SkOauyac66rpqzSZoDF2JIFsy5KpvmtLRy4v3mweaqOZuU5sAFGJYFm1q2zqFdldqc5gWKhZuQnMC1QUvrmn7OVTbNmTsL1Fqas49T3rr5TJJX3nNbYLlkAZDIAmAiDzrqetDbc+Xpc0leTPL6Ae4KcESyAEhkATCRB/11fetmKeXlUsqr1qbCZZMFQCILgIk86O8s6hXW6hVgCdQrAIl6BWCiXqGjs6hXuFKvAEugXgFI1CsAE/UKHalXAEaRBUAiC4CJPOjofOoVNmXr7GJVanNmVygDc46+QvluWTen5c5q3Zy5upUHddWcdS3NgQswJgtqe2X5/aQ5u5irWplzd7VuDlyIo14b7FJ3NHdOmJt1SnNKqc3Zh3oFYBRZACSyAJjIg47UKwDdyQIgkQXARB70p14B6E4WAIksACbyoD/1CsAo6hWAZFS9wlvqFWAB1Ct0pF4BGEW9ApCMqld4Ur0CLIB6hY7UKwCjyAIgkQXARB50dDb1Cqk7zA7ulHVzrsqmOXMrVeFCjKlaScn9emfrzGnVptwtm+bM2dQyM6vmwAU4etXKuran1LJ1dlbas2stA5yRIXmwy2tu6zV87nk7exaYqVYqqzRnH+oVgFFkAZDIAmAiDzpSrwB0JwuARBYAE3nQ32LrFd61dfNtWzfhlPVeofzurZsPxj0w4JGMzAJbN+G0jTon2LrZV5e1qe/auvmErZuwAN1Wqr976+aId6QDexiSBbZuwiJ0PyfYutlXtzXKwKLIAiCRBcBEHnSkXgEYRRYAiSwAJvKgoxHvcbpZm/r5PFyb+pHrtalvJvnozO3uJXnhem3qp5J8pv2lJbWx+niXf/KbmTXKq9JeeXy3rJufW++zmhnOw4AseFivsG6sTL7KfCXCNnPP+R63gwswJAtSktJ4Hs4mQeupO/Myvpn7pJd/mDMkD2afowfUuv543/vQ6ZrhlOsVnk3y2p4beYDTIQuARBYAE3nQ0Yi3br7Ua7sWsCiyAEhkATCRBx0tdutmUa8AS9Nt097tPHhbvQKcuiFZcP+L6hVgAbqfE9Qr9NVlm456BVicbpu1bufBE+oV4NQNyYK7T6lXgAXofk5Qr9CXbTpAIguAh2QBcEMedDTioHezTeepPNym84vX23S+LckPzNzujSTPX2/T+fadf3qdmYGuSm0OXIjjZsH7WNeydTazs2rOqtTmwIUblgU7PQd3uGZo5ce6lofb9FoDnPS1wdKd8tbNZ5K88p7bAsslC4BEFgATedBR14PenitPn0vyYpLXD3BXgCOSBUAiC4CJPOiv61s3Sykvl1JetTYVLpssABJZAEzkQX/qFYBR1CsAiXoFYKJeoSP1CsAo6hWARL0CMFGv0JF6BWAUWQAksgCYyIOOzr9e4cDmVq3PrVe+m9ocuBBDsqCk5qpsts46q+Y0qxJSmjNnldqesmkOXICjXxesZmYXc1Urs7fbMV/gjAzJg9nX5OZr9aPXs8ydE2bV0p49qFcARpEFQCILgIk86Ei9AtCdLAASWQBM5EF/6hWA7mQBkMgCYCIP+lOvAIwyZKX6W+oV4NSpVwBuqFfoSL0CMMqQlepPqleAU6deAbihXqEj9QrAKLIASGQBMJEHHY34T983a1M/n4drUz9yvTb1zSQfnbndG0n+yvXa1E8l+UzrC7/p6a/LZ//kxw93jwf70Vv/6vp3lTPWPQuS5KnHvzF/5Nd99pHv3J965Ft0cp0HpZTPzX8hLNaQLHj+g78mP/a7frDx2dbHk5/9s3PfdQcf3vF2soDL0D0Pfv0v//r8yG/7W4e7xx3896+f+eQfn/7yUc8J6hWAUWQBkMgCYCIPOlKvAHQnC4BEFgATedDfYusVbm/T+cIXti7fAk5E7xXK8gCWQRYAN5wT+ltsvcLtbTpf8zXqM2ABuq1UlwewKLIAuOGc0NFi6xWAxZEFQCILgIk86Ei9AjCKLAASWQBM5EFHZ1Gv8LnPfe7nSyk/d+tDvzLJz2/50tbHe3xu1+/3fOPjsHRDVqq/Jw9GPncP/bNkAedqKVkw97mROSELOGfOCY/2uUfKg1JrfZSvfySllJeTvD569Wkp5bO11m/5pX68x+d6/CxYqlPLgrnPLSEnYKmWlAVznzuVnIAlO7U8OJXn9SGvDU65XgE4E7IASGQBMJEH/Y34M3oAAAAMdK4Hvb/7iB/v8bkePwt4NKfy3D30zwIezak8dw/9s4BHd+rP64NdG3T9M3qnqJTyw0n+fq31x0sp353kf9Vaf3jH7/NyrfVnt3zu2TxcD/vT1x/6o63OH+A4RmTBra/5F0n+aq31P+92b4FeBl0XfE+S33H9t1+X5B/WWr9v1/sM9DEoD74hyd9L8tVJ/nmt9a/vc5/njNi6eYl+S5LvrbX+7WPfEeC4Sil/IsnPOOTB5aq1fuLmr0sp/yzJPzri3QGO6zuT/LVa60+UUn68lPLpXr8QOte3bj6SUsrXllJeLaX8ZCnl49cfe+b6//z/UEr53uuPfej6a15L8o0z3/K3JvmOUsp/KqX8jQEPATiAQ2dBKeXpJD+Y5M1Syu8c8iCAvXW4Lrj5vh/Ow98Q/O++jwA4lA558H+TfHMp5WuTPJ7ki73u+6Ue9H7ourPjz1z//ceT/ONa6wtJ/nAp5auTPJvkY0l+f5I/cP11fznJJ5P8viQfnPn+ryZ5McmHk3xrKeWbD/0AgIPonQV/Mck/TfJ3kny0lPIHD/4IgEPonQU3/kKSHzrg/QYOr3ce/Os8/KXQn0/y75I8OPQDuHGpb938c7fee5s8LB/81lLKn07yy5I8k4f/p38iyf/L9A/rQ0n+S631QSll7m1YP1lr/UqSlFLeSPIbkvzXwz8MYE+9s+A3JfmuWuvnSyn/JMnvTvIjHR4HsJ/eWZBSylNJflWt9Wd6PADgYHrnwceS/LFaay2l/M08vDb4tx0ex8X+Ru+9firJx2qtLyb5/iT3kvylJN+X5DuS3Gys+R9JfmMp5SrJN818v39TSvnVpZSvSvJ7kvy3XnccOKhDZ8FPJ/mG67/+liQ/1+E+A4d36CxIkj+U5F91ubdAT4fOgw8l+bWllA8k+c23bn9wDnoPfX+S7yql/EQe/rr1/yT5l0k+nYf/9f0XrjdpfjLJdyf50STvzHy/70nyY0n+Y5JP11p/quN9Bw7n0FnwySTfef39fnuSf9DxvgOHc+gsSJLfm+Tfd7vHQC+HzoNPJHk9yReS/M88fPtmFxdXrwAAAHDu/EYPAADgzDjoAQAAnBkHPQAAgDPjoAcAAHBmHPQAAADOjIMeAADAmfn/PMX6NM79y5wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x576 with 8 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Real translation: you should begin right away.\n"
     ]
    }
   ],
   "source": [
    "translate(\"你应该立即开始。\", plot='decoder_layer4_block2')\n",
    "print(\"Real translation: you should begin right away.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 十三、重用保存的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 不需要再训练时，不用提供优化器？？？？\n",
    "num_layers = 4  \n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "\n",
    "input_vocab_size = len(zh_tokenizer.word_index) + 1\n",
    "target_vocab_size = len(eng_tokenizer.word_index) + 1\n",
    "dropout_rate = 0.1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "        self.warmup_steps = warmup_steps\n",
    "\n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps**-1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n",
    "\n",
    "\n",
    "learning_rate = CustomSchedule(d_model)\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate,\n",
    "                                     beta_1=0.9,\n",
    "                                     beta_2=0.98,\n",
    "                                     epsilon=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Latest checkpoint restored!!\n"
     ]
    }
   ],
   "source": [
    "# 创建 transformer 并加载模型\n",
    "transformer = Transformer(num_layers,\n",
    "                          d_model,\n",
    "                          num_heads,\n",
    "                          dff,\n",
    "                          input_vocab_size,\n",
    "                          target_vocab_size,\n",
    "                          pe_input=input_vocab_size,\n",
    "                          pe_target=target_vocab_size,\n",
    "                          rate=dropout_rate)\n",
    "\n",
    "checkpoint_path = \"../H/save/zh2eng_transformer\"\n",
    "\n",
    "ckpt = tf.train.Checkpoint(transformer=transformer, optimizer=optimizer)\n",
    "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
    "\n",
    "# 如果检查点存在，则恢复最新的检查点。\n",
    "if ckpt_manager.latest_checkpoint:\n",
    "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
    "    print('Latest checkpoint restored!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载分词器\n",
    "# from tensorflow.keras.preprocessing.text import tokenizer_from_json # tf.2.0没有这个函数\n",
    "\n",
    "tokenizer_dir = '../H/save/zh2eng_transformer_tokenizer/'\n",
    "zh_tokenizer_dir = tokenizer_dir + 'zh_tokenizer.json'\n",
    "eng_tokenizer_dir = tokenizer_dir + 'eng_tokenizer.json'\n",
    "\n",
    "with open(zh_tokenizer_dir) as f:\n",
    "    data = json.load(f)\n",
    "    zh_tokenizer = tokenizer_from_json(data)\n",
    "\n",
    "with open(eng_tokenizer_dir) as f:\n",
    "    data = json.load(f)\n",
    "    eng_tokenizer = tokenizer_from_json(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/yangbin7/anaconda3/lib/python3.7/site-packages/matplotlib/mpl-data/matplotlibrc'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib\n",
    "matplotlib.matplotlib_fname()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Kalimati', 'cmex10', 'DejaVu Serif Display', 'STIXGeneral', 'padmaa', 'Keraleeyam', 'Abyssinica SIL', 'Noto Serif CJK JP', 'DejaVu Serif', 'DejaVu Sans Display', 'LKLUG', 'STIXNonUnicode', 'Loma', 'Nakula', 'Kalapi', 'Sahadeva', 'Garuda', 'FreeSans', 'Tlwg Typewriter', 'KacstFarsi', 'Vemana2000', 'Lohit Malayalam', 'Umpush', 'AR PL UKai CN', 'OpenSymbol', 'AR PL UMing CN', 'Laksaman', 'Rachana', 'mry_KacstQurn', 'KacstOffice', 'Samanata', 'STIXSizeThreeSym', 'Microsoft YaHei', 'cmss10', 'DejaVu Sans Mono', 'Sarai', 'FreeMono', 'Liberation Sans', 'KacstScreen', 'Lohit Tamil', 'Samyak Tamil', 'cmsy10', 'Liberation Serif', 'aakar', 'Sawasdee', 'KacstOne', 'Waree', 'Droid Sans Fallback', 'Ani', 'FreeSerif', 'Tlwg Typist', 'KacstPen', 'Padauk', 'AnjaliOldLipi', 'Meera', 'Phetsarath OT', 'ori1Uni', 'Noto Mono', 'KacstLetter', 'Navilu', 'Likhan', 'Mukti Narrow', 'KacstArt', 'Saab', 'Noto Sans CJK JP', 'Purisa', 'Karumbi', 'Suruma', 'KacstTitleL', 'KacstDecorative', 'Pothana2000', 'Lohit Kannada', 'Chandas', 'Padauk Book', 'Tlwg Mono', 'Uroob', 'cmmi10', 'Pagul', 'Lohit Telugu', 'Gubbi', 'KacstNaskh', 'Manjari', 'Jamrul', 'KacstQurn', 'Dyuthi', 'Lohit Bengali', 'Gargi', 'KacstBook', 'Mitra Mono', 'Khmer OS', 'Norasi', 'Lohit Assamese', 'KacstPoster', 'Ubuntu Mono', 'cmb10', 'Lohit Gurmukhi', 'KacstTitle', 'Samyak Devanagari', 'cmr10', 'Khmer OS System', 'Lohit Devanagari', 'RaghuMalayalam', 'Liberation Mono', 'Lohit Gujarati', 'Samyak Gujarati', 'STIXSizeFiveSym', 'KacstDigital', 'STIXSizeTwoSym', 'Lohit Odia', 'Ubuntu', 'Tibetan Machine Uni', 'padmaa-Bold.1.1', 'DejaVu Sans', 'Liberation Sans Narrow', 'Chilanka', 'SimHei', 'Kinnari', 'Rekha', 'Lohit Tamil Classical', 'Ubuntu Condensed', 'cmtt10', 'STIXSizeFourSym', 'Tlwg Typo', 'Samyak Malayalam', 'STIXSizeOneSym'}\n",
      "********** 系统可用的中文字体 **********\n",
      "Noto Serif CJK TC\n",
      "Noto Serif CJK SC\n",
      "Noto Serif CJK TC\n",
      "Noto Serif CJK KR\n",
      "Noto Serif CJK JP\n",
      "Noto Serif CJK SC\n",
      "AR PL UKai CN\n",
      "微软雅黑,Microsoft YaHei\n",
      "Noto Sans Mono CJK TC\n",
      "Noto Sans CJK JP\n",
      "AR PL UKai TW\n",
      "Noto Sans CJK HK\n",
      "Noto Sans CJK KR\n",
      "Noto Sans CJK SC\n",
      "Noto Sans Mono CJK SC\n",
      "Noto Sans Mono CJK KR\n",
      "Noto Sans Mono CJK JP\n",
      "Noto Sans Mono CJK HK\n",
      "Fixed\n",
      "Noto Sans Mono CJK JP\n",
      "Noto Sans Mono CJK HK\n",
      "Noto Sans Mono CJK KR\n",
      "AR PL UMing CN\n",
      "Noto Sans CJK KR\n",
      "AR PL UMing TW\n",
      "Noto Sans CJK HK\n",
      "Noto Serif CJK KR\n",
      "Noto Sans CJK JP\n",
      "AR PL UMing HK\n",
      "Noto Serif CJK JP\n",
      "Noto Sans CJK TC\n",
      "Droid Sans Fallback\n",
      "Noto Sans Mono CJK TC\n",
      "Noto Sans Mono CJK SC\n",
      "AR PL UMing TW MBE\n",
      "Noto Sans CJK TC\n",
      "AR PL UKai HK\n",
      "AR PL UKai TW MBE\n",
      "Noto Sans CJK SC\n",
      "Fixed\n",
      "\n",
      "********** 可用的字体 **********\n",
      "Droid Sans Fallback\n",
      "Noto Sans CJK JP\n",
      "AR PL UMing CN\n",
      "Noto Serif CJK JP\n",
      "AR PL UKai CN\n"
     ]
    }
   ],
   "source": [
    "from matplotlib.font_manager import FontManager\n",
    "import subprocess\n",
    "fm = FontManager()\n",
    "mat_fonts = set(f.name for f in fm.ttflist)\n",
    "print(mat_fonts)\n",
    "output = subprocess.check_output('fc-list :lang=zh -f \"%{family}\\n\"',\n",
    "                                 shell=True)\n",
    "print('*' * 10, '系统可用的中文字体', '*' * 10)\n",
    "print(output.decode('utf-8'))\n",
    "zh_fonts = set(f.split(',', 1)[0] for f in output.decode('utf-8').split('\\n'))\n",
    "available = mat_fonts & zh_fonts\n",
    "print('*' * 10, '可用的字体', '*' * 10)\n",
    "for f in available:\n",
    "    print(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAETCAYAAADZHBoWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydd3xc1Zn3v4+qVSxZzZItV9lywQ1bNpYbWEDApAIpkGSTJSQhlU2yyb6QhGxCXtLYvOyGkGRTKAkJOIQASYDYYCzjKveOmyzLXZbVJavOzPP+MTNGyKMy0szcOzPn+/nMRzP3nnPu716de597nucUUVUMBoPBYACIsVqAwWAwGOyDMQoGg8FguIwxCgaDwWC4jDEKBoPBYLiMMQoGg8FguIwxCgaDwWC4jDEKEYqIPCMiP+32+3si8j0LJQUVEXlBRBYHoJx1IrI8AJICQqDOy26IyGdF5IyIXBCRpQNI/5SI3NXt93IRWRdMjdFKnNUCDEGjBKgKRsEiMgFYrqpPBaP8waCqt1utIRj4Oq9gX/8Q/X8fBmbjrqNJQTyOwU9MSyECEZEZwClghIhkBeEQE4C7glCuYWBMILjXP9jlA4xQ1dOq2qWqTUE+lsEPjFGITG4A1gEbcLcYvMwSkX0iclJEbvFuFJH/EJFTInLEu72nu0lEKkVkgohsAF4AFotIlYj8ti8hIhIjIr8RkXMiUi4iN/XYft5z7NsGWI7P9D3dPiKiIvIHETkhIj8VkRoRud6T7o8iclpEdorIlD6vpLusb3mOeVJE3heK8+3jvPy6/p48j3iO+4qIvCEiX+jtvHorf7D6fWj5iYhUeb5XicjBbvuuqIcDLPOjIrJJRJI8vz/n+f9Wi8hDg9EZ1aiq+UTYB3gJeDfwaeBXnm3fA04CWUARcAEYBtwIHAAygKtwN+dzPem/163MSmCC5/tyYN0AtSwEnsftqiwGtnm2zwPOAQnAdOCX/ZTTZ3rcRnB5t98KLAV2AA8APwX+05PuedwvRPcCq/op5xbgDSAFmAqcB+KDfb696RnE9Z+Ou9UYBzwLfKa/8/JV/mD196FLe/z2WQ+77X8KuKvnNQCuA/YAWd32NQEzcdfvvwDDrb4nw+ljYgoRhojEAtcCi3A/+Oq77X5BVWuBWhGpBabgfjj8UVXrgXoR2Qos81X0YPSo6lYReQT4PvAuIMez6zjgAv4LWAt8tZ+i/E0PsAVo8fxdxtst4z+oqktEngX+bz9l3Ags8BwfIBkYjdvAXkEAzzdQdOD+38XhfqB7r4Ff50Xw9fdWD5/vI8844Glgt6dee9kI/AD3y9EXVLU5wFojGuM+ijyKgBOqmquqOUCqiIz17Os++6GLt///3bdrj9+ISBzu1oPfiMjHgV8Au4EvXz6IaiPuN8INwMeA1/sqx9/0njxOz1dnj11eAxeD+zr0eQrAD1Q1T1XzcD+IzvaaOEDnG0AacLcCyoF44I9eqfhxXiHS32c99EEW8D4gR0SWdNv+fuBR3C2gAyKS4yuzwTfGKEQe1wNbu/3e6tkG8AERyRCRObgf8keBfwIfF5ERIjINt/tjI+4muNeY3AMkdiuzBhgjIrGe8mL70LMIWA38DfiAd6OI3AA8gftt7pvANSLSa2vE3/T9cJeIxAAfBTb3k3YN8BERSROR0bjfmEf0kT4g59sP/lz/9wJbVXWMqr5fVVsHcF5XlB9g/b7orR72xW5V3Qt8B3cLBhFJxu2G2oXbXdgCTA6gzojHGIXI4wauNAo3eL4f9/x+Gfisqraq6hrcTfB9wN+Bu1X1ArASmC8ir+E2IJfdCqp6APdD5SzuGzChDz1PAR8BTnjS5YhIOvAm0OwpYwPwf9TjEO4Ff9P3RT3uGMkn6McNoqqv4n4QHgA2Afeqak0fWZ4iMOfblyZ/rn8p8Elxjwc4KiK/EpG4vs6rl/J96heRsSKybzDn0eOcequHA8n7BtAuIrd7jN4vgf3AaY/WbUPVF03I4O8rgyH8EPeAp++p6jqLpYQET3xjl6r+0fMWvRH4lOcNO1DH+Imq3heo8gzWYloKBkNk8yrwTRE5BxzCHXQ/2HeWgSMiCfQdDDaEGaalYDAYDIbLmJaCwWAwGC5jjILBYDAYLmOMgsFgMBguE/YjmrOzs3XChAmDynvp0iVSUlICKygAGF3+Y1dtRpd/2FUX2FfbYHXt3LmzxjPA9Z1YPc/GUD9FRUU6WEpLSwedN5gYXf5jV21Gl3/YVZeqfbUNVhewQ308U437yGAwGAyXMUbBYDAYDJcxRsFgMBgMlzFGwWAwGAyXMUbBYDAYDJcJilEQkXQR+aeIvCYiL3rmR/GV7nER2SIiD/S1zWAwGAyhIVgthY8Dj6jqTbiX1VvRM4GI3A7EquoioEBECn1tC5I+W6KqNLR2Utfuwukyc1IZAo+qUnepk/p2Fy5Txww+CMrgNVX9ZbefOUC1j2TLgec831/DvZ7uXB/bjgVDo504UXOJ32+u5B97z1F7qROA+zesYllhNh8vHkfJ1JEEdj0TQ7Rx7EIzT22u5JX952lo7QLgvo2rKJmaw78Uj2dZoVmczOAmqLOkisgi4CFVvcHHvseBR1V1r4jchHth8MKe21T1xz7y3oN7NTByc3OLVq5cOSh9LS0tpKamDipvIHCp8nJFF38v7wKBeSNjKUiPRR0d1Dvi2VblpKFDmZMTy10zEsgYZm0IyOrr1Rd21Wa1LodLeam8i3+e6CJWoCg3lgnpsbi6Oqjpimd7lYOmTpifG8snZySSlmDty4fV16sv7KptsLpKSkp2qur8K3b4GtEWiA+QCewAxvey/2dAsef77cC3fG3r7zjhOqK5rdOhn/39dh1/38v6xT/t1AtNbVfo6nI49bfrj+u0B/6pxT9co4fPN1mk9p267IhdtVmpq7m9S//ld2U6/r6X9evP7dHalo4rdHV0OfWxtce08Nuv6rUPr9UTF1ssUvtOXXbErtrCYkSzJ7D8F+Cbqnqyl2Q7cbuHAObgXh7R17aIo8Ph5O6ntvP6oQt8731X8YuPzWPk8GFXpIuLjeEzywr46xcW41Llw/+7mSNVzRYoNoQbbZ1OPvH4VjYfr+XhD83mpx+eQ2bKlf09EuJi+FLJZP58TzHN7Q4+9L+bqay5ZIFig10Ilj/i07jdQd8WkXUi8l0ReahHmpeAT3iWC/wI8Eov2yIKl0v5P8/vY/PxWn76oTnctWRiv3muGp3G859fzLD4WD715DaqGttDoNQQrjhdyr3P7mLv6QZ+8bG5fGT+2H7zzB2XwXOfW4TTpdz15DZqWzpCoNRgR4JiFFT1V6qaoarLPZ8HVfWBHmmacAeby4ASVW30tS0Y+qzk8Y0n+Nuec/zHzVP5YNGYAecbm5nMk59aQGNbF1/8004cTlcQVRrCmZ+9cYw1h6r53vtnsGLmqAHnmzwyld/96wLON7bzlZV7TO+kKMXSyKWq1qvqc6pa1de2SGHfmQYeXn2Ym2fk8sXlk/zOP2N0Oj/64Gx2nWrgZ29EfKcswyAoq6jlsbXH+OC8MXxy0QS/8xeNz+B775/BxvIafrOhIvACDbbHjGgOER0OJ1/78x6yUxP5yQdnD7qL6fvnjOYj88fwWGk5u0/VB1ilIZxp7XTw9ef2Mj4rhe9/YMagy7lzwVjeM2sUP119xMSwohBjFELEb96s4PjFS/zo9lmMSPY5wHvAfOe9VzFyeCLffvGAcSMZLvOzNcc429DGwx+aTUri4IcgiQj/99aZDB8Wx7df3G/cSFGGMQoh4FRtKz8vLec9s0axfOrIIZc3fFg8333fDN4638RTmyuHLtAQ9hypauZ3G09wx/yxLJiQOeTyMlMS+Oa7p7PjZD1/2Xk6AAoN4YIxCiHgJ6sPEyvCf77vqoCVecvMPK6dksPP15bT6BmhaohefvTPQ6QmxnH/LdMCVuaHi8ZQND6Dn752lNZOR8DKNdgbYxSCzN7TDbyy7zyfXTaR3LQrxyIMFhHhm7dMo6m9i1+uKw9YuYbwY/PxGtYduciXSiaR4WMswmAREb717mlcbO7g8Q0nAlauwd4YoxBkfrLqMFkpCdxznf+9jfpj+qg0bpubz5ObKznf2Bbw8g32R1X5yaoj5I9IGlRvo/4oGp/JTVfl8uv1FdR75uUyRDbGKASRHZV1bD5eyxeWTyJ1CIG/vvjajVNwupTfrDfdB6OR9cdq2Hu6gXuvn8yw+NigHOMbN0+lpcPBk5tMayEaMEYhiDxWWk5mSgIfWzguaMcYm5nMbXPzeXbbKS42m1Go0YSq8vM3jjE6fRi3zxv4QEh/mZI7nBUz8nhycyVN7SZ+FekYoxAkDpxtZN2Ri3x66USSE4LTSvDyxeWT6HC4eHyjeZOLJradqGPHyXo+v3wSCXHBvZW/fP1kmtsdPL2lt6nMDJGCMQpB4ncbKkhNjOMTi8YH/VgFOam8e+Yontl6kksdppdItPDbDSfITEkY0NxGQ2VmfjrXTsnhqc2VdDrM2JhIxhiFIHChqZ2X953nw/PHkDYsPiTHvHvpRJraHbyw60xIjmewlsqaS7xx+AL/snBc0GIJPfn00olcbO7glf3nQnI8gzUYoxAEnt5yEqcqn1rc/wyogWLeuBHMGTuCJzdVmhGoUcBTmyuJixH+pTj4LVEv1xZmM3lkKo9vPOFdE8UQgRijEGA6HS6e3XaKG6fnMi4rOWTHFRHuXjKBippLrD92MWTHNYSelg4Hf9lxmvfOHs3IAI596Q8R4VNLJnDgbBO7zLxbEYsxCgHmjUMXqL3UGdQeR71xy8xRZKUk8MzWUyE/tiF0/H3POS51OkPaSvBy69X5pCbG8SdTxyIWYxQCzJ93nGZU+jCutWAh9IS4GD40fwxvHK7mQpNZiCdSeWbbSablDWfeuBEhP3ZKYhwfuHo0r+w7b6ZXiVCCahREJFdENvSx/0HPymzrROSwiHxTRPJF5Ey37aF/ug6S841trD96kQ8VjSE2xpoF0D+6YBxOl/LcdjOJWSSy/0wjB8428fGF4wY9/fpQ+djCcXQ4XLyw23RqiESCZhREJAP4PZDSWxpV/a53dTbgAPAHYCHwg26rtoWNg/z5HWdwKXy4KPhdBHtjQnYKSyZn8dzO0yYYGIE8t+M0w+Jj+MDcfMs0zBidzpwx6fzZvHhEJMFsKTiBO4Cm/hKKyALgjKqeBYqBz4jILhH5YRD1BRSXS3lu52kWT8oKaYDZF7fPHcPpujYTDIwwOh0uXt53jpuuygtZV+feuH3eGA5XNXO4qt/b2xBmSLDfJkVknacl0FeaPwLfVdXjIlIC7ABagTXAV1R1X4/09wD3AOTm5hatXLlyUNpaWlpITU0dVN6evFXr5OHt7XxudiKLRg9tBPNQdbU5lK+sbWVpfhyfnJE4JC2B1BVM7KotkLp2Vzv42a4OvlaUyJwca+tYU6fy1dJWVkyI5yNTAzczq13/j2BfbYPVVVJSslNV51+xQ1WD+gHW9bN/BPBat9+J3b4/Anywr/xFRUU6WEpLSwedtyf/9uwunfXdVdrW6RhyWYHQde8zu3TOg6u1o8s55LK8BPJ6BRq7agukri/+cafO+/5r2ukY+v80ELo+9eQ2Lf7hGnU6XUMuy4td/4+q9tU2WF3ADvXxTLVD76MPAK92+71aREaJSDJwE+5Yg61p6XCw6kAVH7g6P2SjS/vjtrn5NLR2se5ItdVSDAGgqb2L1w9d4H1zRhMfa4fbFm6dm8/5xnbKTtRaLcUQQEJWu0TkehH5so9dNwPru/1+ECgFyoD/VdUjodA3FN44dIEOh4v3Xz3aaimXWVaYTVZKAi/tOWu1FEMAWLW/ik6Hi1stDDD35F3Tc0lNjOOl3aaORRJBNwrqiSeo6lpVfczH/o+p6q5uv0tVdZqqzvaV3o78Y+958tKGUTQuw2opl4mLjeF9c0az5lA1jW2mP3m488LuM0zMTmHOmHSrpVwmKSGWFTPz+Of+Ktq7nFbLMQQIe7RDw5im9i7WH73Iu2eNIsaisQm9cdvcfDodLv65/7zVUgxD4GxDG2UVddx6db5lYxN647a5+TR3OFhz6ILVUgwBwhiFIfL6wQt0Ol28d84oq6Vcwewx6RRkp/Ciad6HNf/Y656V9Na59nFPeikuyCI3LdG4kCIIYxSGyCv7z5M/Iom5Y0M/5UB/iAjvnT2K7ZV11LSYVdnClX8eqGL2mHTGZ/U6DtQyYmOEd88axfpjNbSYtTwiAmMUhkBjaxcbjl3kPbNH2a5Z7+XmmXm4FNa8ZZr34ci5hjb2nm5gxcw8q6X0yooZeXQ6XKanW4RgjMIQWH2wii6n8t7Z9nMdeblqVBpjM5NYdbDKaimGQfCa5/+2YoZ9jcL8CZlkpSSw6oCpY5GAMQpD4OX95xmXmcysfPv0COmJiLBiRh6bymvMouthyKqDVRSOTKUgx34jab3Exgg3zcil9HC16YUUARijMEgaW7vYVF7Du2fZ13XkZcXMPLqcSulh07wPJ2pbOth2os7WriMvN8/I41Knk03lNVZLMQwRYxQGybqj1Thdyk0zcq2W0i9zx2aQMzyR1caFFFasOXQBl7ofuHZn8aRshifGGRdSBGCMwiB5/a0LZKcmcPUY+/U66klMjHDzjFxKD180zfswYtWBKsZkJDFjdJrVUvolIS6GG6aPZM2hCzicLqvlGIaAMQqDoNPh4s0jF7lhWq7tBqz1xooZo2jrcrL+aNgsTxHVNLd3sam8lhUz8mzvnvSyYmYe9a1dbKuss1qKYQgYozAItp2oo7nDwY1X2d915GVhQSbpSfGsPmi6poYDpUcu0ul0hUU8wcu1U3JIjIthtXEhhTXGKAyCNYcukBgXw9LJ2VZLGTDxsTHcMG0kbxy+gNNlVmSzO68drCI7NZF5NppPqz+SE+K4dkoOr711waz6F8YYo+Anqsrrb11gWWE2SQn2mCZ7oFw/fSQNrV3sNiuy2RqH08X6oxcpmZoTNu5JLzdMG8n5xnYOVzVbLcUwSIxR8JPDVc2cbWjjXWHkOvKyrDCH2Bhhremaamt2nWqgqd3B9dNGWi3Fb0o8mkvN6OawxRgFP1nz1gVE4Ppp4WcU0pPimT8+g9IjJthsZ9YeriYuRlhSGD7uSS+5acOYMTrNjIkJY4xR8JM1hy5w9dgR5AwP3NrHoaRk2kgOnW/ifGOb1VIMvVB6uJoFEzJJGxZvtZRBUTJ1JDtP1tPQ2mm1FMMgMEbBD6qb29l7ppEbp4dfK8GL1yVReti0FuzI2YY2jlxoDkvXkZeSaSNxKaw/ZkY3hyPGKPjBhqPuSr58ao7FSgZP4chU8kckmbiCTfG6XUqmhW8du3rsCDJTEowLKUwJqlEQkVwR2dDH/nwROSMi6zyfHM/2x0Vki4g8EEx9/vLm0YvkDE/kqlH2H2HaGyLC9dNGsqm8hg6HGd1sN0oPVzM2M4lJNp4Arz9iY4TrpuSw7ki16f4chgTNKIhIBvB7oK+VQRYCP1DV5Z7PRRG5HYhV1UVAgYgUBkujPzhdyoZjF7m2MCdsRpj2Rsm0HNq6nGytMCNP7UR7l5NNx2u4furIsK9jy6fmUN/axd4zDVZLMfiJBGuQiYikAQL8TVWX95LmYeAGT7pVqvotEXnU8/1VEbkTSFLVJ3vkuwe4ByA3N7do5cqVg9LY0tJCaurA3sgqGpx8v6ydz89JpHhU3KCOFwxdg6HDqXz5jVaWj43j49MHHjAPtq6hYFdt/ujad9HBIzs7+PeiRGbnhHcda+lU7l3bynsnxfPBwgTb6BoKdtU2WF0lJSU7VXX+FTtUNagfYF0f+0qA4UAsUArMBh4H5nj23wTc31f5RUVFOlhKS0sHnPZ/Xj+qE+5/WetaOgZ9vIHij67BctcTW/Xah9f6lScUugaLXbX5o+u7fzugUx94Vds6HcET5CEU1+tDv9qk7/7Zer/y2PX/qGpfbYPVBexQH89UqwPNm1W1WVWdwG6gEGgBkjz7U7FJMPzNo9XMGTOCjJSBv/XYmeum5HCytpWTtZeslmLwsP7oRYoLshgWH14j5Xvjuik5HDzXZNYHDzOsfuCuFpFRIpKMu1VwANgJLPXsnwNUWqTtMg2tnew53cB1U8K3R0hPlnnOZYPpNmgLztS3UlFziWWFEVTHPOdiFt4JL0JmFETkehH5co/ND+J2G5UB/6uqR4CXgE+IyCPAR4BXQqWxNzaW1+BSuC6Mu6L2pCA7hfwRSWw0RsEWeP8P14bhKObemJmfzojkePPiEWYEN5oFqCfIrKprgbU99pUC03psaxKR5cC7gIdVtTHYGvvjzSMXSU+KZ04YLKgzUESEpZOzefXAeRxOF3GxVjcao5sNx2rISxvG5JH2C2QOltgYYcnkbDYcu4iqhn2PqmjBlk8CVa1X1edU1fKJ2VWVN49eZFlhNrFhNmNlfyybkk1zu4O9Zyy3u1GN06VsLK9hWWF2xD04ry3M5kJTB8eqW6yWYhggtjQKduLIhWaqmzu4NoJ8vV6WTMpGBDYcM1NeWMmBs400tnWxNIJcR16Weu4bs+Jf+GCMQj9sKq8FCMsZK/sjIyWB2fnpJq5gMV6jHE6LNg2U/BFJFOSksNEEm8MGYxT6YXN5DRM9QdlIZGlhNrtPN9DU3mW1lKhl/bEaZuankZUanjPv9se1hTmUVdSaaVXCBGMU+sDhdLH1RB2LJ2VZLSVoLCvMwelSthyvtVpKVNLS4WDXyfqI6orak2WF2bR3udhZaVb8CweMUeiDvWcaaelwsCQCm/Ve5o3LIDkh1sQVLGJrRS0Ol7IsAt2TXooLsoiPFTOVdphgjEIfbC6vQQQWFURuSyEhLobigizTl9wiNhyrISk+lqLxGVZLCRopiXHMHZdhXjzCBGMU+mBjeQ1XjUqLmKktemPp5GxO1rZypr7VailRx+bjNSyYmEliXGRMbdEbyyZn89b5JuovmdXY7I4xCr3Q1ulk96mGiHYdeVk82d0SMnGF0FLT0sHRCy0R3RL1snhyFqqw9YSpY3bHGIVe2F5ZR6fTFdFBZi9TRg4nKyXBGIUQ413Porgg02IlwWf2mBEkJ8Sy2dQx22OMQi9sOl5DfKxwzcTIv2FjYoTigiw2H6/1TmluCAFlFbWkJMQyMz/dailBJz42hgUTMo1RCAOMUeiFzeW1zB2XQXJC0KeHsgWLJmVR1dROZa2JK4SKsopa5k/IJD5K5p1aPCmL8uoWqpvbrZZi6IPoqI1+0tDayYFzjSyZFPnxBC9eN9nm46YXUiioaXHPB7QoCtyTXhZ77ifjprQ3xij4oKyiFlVYMjl6btiJ2SnkpiWa5n2IeDueED117KrRaaQNizNGweYYo+CDjeU1pCTEMmds5EyV3R8iwuJJ2ZSZuEJIuBxPGJ1mtZSQERsjLPTErgz2xRgFH2wur+WaidHj6/WyaFIWtZc6OXrBTHMcbLZU1LJgYmbUrWOxeFIWp+rMmBg7E9QaKSK5IrKhj/3jRGSdiKwVkd+Im3wROePZvk5EQjopzPnGNipqLkXF+ISemLhCaLjY3EF5dUtUuY68mLiC/RmQURCRJSLyZRH5DxG5U0T67acpIhnA74GUPpJ9DviCql4PjAVmAQuBH6jqcs8npGPjvVNlL46iILOXMRnJjM1MMs37IOMdwBWNRmFKbqoZE2Nz+jQKIrJMRP4CzAQ2An8HGoAfisj3RaSvsflO4A6gqbcEqvptVT3k+ZkF1ADFwGdEZJeI/HDgpxIYNpfXkJmSwLS84aE+tC1YXJBNWUUtTpeJKwSLsopaUhPjoiqe4EVEKJ5kxsTYGentHyMiE4EvAPerqsvH/iXAAlX9nz4PILLOu05zH2nuAFao6qdEpATYAbQCa4CvqOq+HunvAe4ByM3NLVq5cmVfxfdKS0sLqalvr4mrqnz9zTYmjYjhS1cPG1SZgaCnrlCy5ZyDX+/r4HuLhjEh/Z0230pd/WFXbb50fWtDK9nJMfx7UXTWsdJTXfz+rU5+vCyJvJR3vpfa9f8I9tU2WF0lJSU7VXX+FTtUtc8P8B3gGjwGxN8PsK6f/QXAdiDd8zux275HgA/2lb+oqEgHS2lp6Tt+n6y5pOPve1n/sPnEoMsMBD11hZILjW06/r6X9X/XlV+xz0pd/WFXbT11XWjq/fqGEiuvV8XFFh1/38v6x7LKK/bZ9f+oal9tg9UF7FAfz9SBxBQagG8Bx0XkWRG5S0RG+W2WfOCJOzwL3K2q3tXjV4vIKBFJBm4CDgTiWAOhrCJ6fb1eRqYNY1JOiokrBIloHJ/QkwlZyeSlDTN1zKb0axRU9eeqeiswGfgfYB5wQkT2+nMgEbleRL7cY/P9wDjg556eRtcBDwKlQBnwv6p6xJ/jDIWyE7VkpSQweaT9moihZPGkbLZX1tHlvMJraBgi3njCjCiMJ3hxj4nJMmNibEq/E/uISBJwHXAjcAPQCTwMrB7IAdQTT1DVtcDaHvvuA+7zkW3aQMoOJKrK1oo6FhZkIiKhPrytWDwpi6fLTrLvTANF4yN/QsBQUlZRy4IJGVE3PqEniyZl8cLusxy90MLUKO3UYVcGUjMP4e5FtAO4UVUXqup/quqm4EoLLWfq2zjb0BbVzXov3mvg7Z5rCAzVze0cv3gpquY76g3vNdhUbsbE2I2BuI8mqOqngHTgYRF5QkSeFJEngi8vdGwx8YTLZKQkMH1U2uUYiyEwmHjC24zJSGZcZrKpYzbEn3mh7wI+BriAiHMEbq2oIzMlgcIojyd4KS7I5Jmtp+hwOCN+qchQsaWiluGJcVw1KnrjCd0pLshk9cELuFxKTEx0u2zthD+OzQu4xw08iXuk8lPBEGQVZRW1LJxo4gleiguy6HC42Hu6sf/EhgFRFqXzHfVGcUEWjW1dHK5qtlqKoRv+1M54YJaqXq+qJeqemiIiOF3XauIJPXAbSEzzPkBUN7VTcfFSVCy9OVAWeu43U8fshT9GIRfY7pm8rlRE1vabI0zYesLt611obtjLjEhOYHqeiSsEijJPHVtUEH1zavVG/ogkxmUmX47nGexBr0ZBRMaLyK3e36o6X1VneFsKwB0icmdIVAaZsopaMpLjmTLSdAfoTDwAACAASURBVI3rTnFBFjtP1tPhcFotJewp88YTonh8gi8WFWSx7UQdLjPXlm3o1Sio6klgiog8KiJTvdtFJFlEPgn8HFgfAo1Bxx1PyDLBrh4UF2TS4XCx51SD1VLCnrLj7jU6Yk0dewfFkzJpbOviUFWv82YaQkyf7iNVfRj4f8CHPd1QnwT+C6hV1TtV9VwoRAaTM/WtnKlvM75eH1xzOa5QZ7WUsOZCUzsVNZdMzMoHCyd64wqmjtmFfrukeloMD4VAiyV4+44vNDfsFXSPK3yFQqvlhC1mTq3eGT0iifFZ7vEKn1460Wo5BsxynJRV1DIiOZ6puSae4ItFk7LYdaqe9i4TVxgsZRV1DB9m4gm9UTzRxBXshDEKJ9zjE0w8wTdvj1cwcYXBstUzBsbEE3yzaJJ7vMJb501cwQ5EtVGobXNxuq7tsl/TcCXXTHDHFUy3wcFR3+4y8YR+8HYFN92f7UFUG4XDdW6XiLlheyc9OZ6rzDxIg+ZwnXv6cVPHemdUehITspJNsNkmRLlRcJGeFB+16zEPlOKCLHadaqDTaXy+/nK4zsnwYXFMN/Md9UlxQRbbTtTiMusrWE6UGwUn15h4Qr8sKsii0+GiotEsuuMvh+ucJp4wAIoLsmhqd3C62dQxq/HLKIjITBG5WUSmi0hYTyd6rqGNi21qmvUDYIFnvILX3WYYGFWN7VxoNXVsIHiv0aFaYxSsZsBGQUR+jnupzB8BBcAzA8iTKyIb+tgfLyL/EJFNInJ3b9uCwdYT3r7jZtBaf6QnxTNjdJoxCn7ydh0zRqE/8tKHMTE7xdQxG+BPS2GWqn4QaFDVV3AvutMrIpKBe4rtlD6S3QvsVNUlwIdEZHgv2wJO2fE6UuJhep7x9Q6E4olZlDe4zHgFP9hyvJbkOEw8YYAUF2RypN6J04xXsBR/Ftm5KCL/CWSIyL8CVf2kd+JexvNvfaRZDtzv+b4emN/LttLumUTkHuAegNzcXNatWzfQc7hM6VutFAxX1q9/0++8waalpWVQ5xRMUlodOFzw5N/XMT3Lfovu2PGalR5sZVKassHUsQGR1u6gzQFP/2MtE9JNHRsogdblj1H4JO4H8RbcrYRP9ZVYVZuA/hatSQHOer7X4Z6e29e2nmX/BvgNwPz583X58uUDPIW3+fvcdtZu2Mxg8gabdevW2U7X3LYuHt31Gu1pY1m+fIrVcq7AbtfsfGMbF1at5fpxibbS5cVu1wtgWmM7v973Bo7MiSxfVmC1nCuw4zWDwOvyx330FVX9map+UVUfBSaIyLVDPH4LkOT5nurR42tbwMlLH8bo1KjufOUX6UnxjE+LMeMVBoh3Tq1pmaaODZS89GHkJgtbjps6ZiV+xRREpKzbGgrfAb4xxOPvBJZ6vs8BKnvZZrAB0zJj2H26wcQVBkBZRS1pw+IYO9wYBX+YnhnLthN1Jq5gIf7U2ALcD+t7Pb9H4l6ic0CIyPUi8uUem38PPCgiPwOuArb2ss1gA6ZlxtLpcLHrVL3VUmxPWUUtCwuyiDFrfvvFtMxYmjscvHXOzINkFf4YhXrgF8AwEfkAMIUBGAVVXe75u1ZVH+ux7yTwLmATcKOqOn1t80OjIYhMyYglxqyv0C/nG9uorG01XVEHwVSPu824KfumobWTmpaOoJTtj1G4HfgVsAJIA24BVg5VgKqeU9XnVLWxr20G60mOF2bmp5sbth/eXj/BjIHxl4xhMRRkp5g61g/P7zzD/IfWUN3cHvCyB2wUVLUdd6+gJOBNYLiq/i7gigy2prggiz2nTFyhL8qO15GeFG/GwAyShZ51mx1OM7q5N8oq6piQlczI4cMCXrY/I5ofB/6Me9zBs7iX5TREGcUFmXQ6TVyhL8pO1Jo5tYZAcUGmO65g1lfwiculbK+sC9qU//64jybjdh2VA9cBxoxHIfMnZJq4Qh+ca2jjpIknDIlFBd51m40LyReHqppobOuieFJw3JP+GIVW4AYgFvgwkBEURQZbkzYs3h1XMH3JfeJ9kC0yRmHQjEwbRkFOinnx6IXL68rboKXwIeAY8DVgOvDFoCgy2J5FBVnsOd1AW6eJK/SkrKLWrNERAIoLsthu4go+KauoZVxmMqNHJPWfeBD4E2i+pKrlqnpSVf8TMKNLopTigiw6nS52m7jCFZRV1Jk1vwNAcUGWiSv4wOVStlXWBbVnmz+B5td7bPpRgLUYwoT5EzI8cQXjQurO2YY2TtWZeEIgKJ7ofuiZKS/eyZELzTS0dgV1Xfl+J8QTkdnAXCBfRD7p2ZwCBL6DrCEsGD4snln56cbn24OtFWb9hEAxMm0Yk3Lc4xU+d90kq+XYBu+L2EKLWwri428t8JGgKDKEBcUmrnAFZRW1jEg28YRAUVyQxfbKehNX6MbWijrGZCQxJiM5aMfo1yio6l5V/T3woqr+QVV/7xltbBzKUUzxpCwzXqEHWypqTTwhgBQXZNHS4eCgmQcJcMcTtp6oDXpL1J9A87eCKcQQXswfn0FsjJi4gocz9a2crmszrqMA4nWRmDrm5lh1C/WtXfYxCgZDd4Z7xyuYGxZ4u++4MQqBY+Rwd1xhi6ljQLd4wsTgzqk1kEBzKVd2PxVAVfX6oKgyhAXFBZk8sfEEbZ1OkhLst3xiKNlSUUtGcjxTc008IZAUF2Tx0u6zOJwu4mKj+x12y/Fa8kckMTYzePEEGFhMoURVr+/xKTEGwVBckEWXU6M+rqCqbDnu9vWaeEJgWTQpi0udTg5EeVzB5VK2VNSyeFLwW6LRbXoNQ2LBhEwTVwBO1bVytqEtJDdstOHtjx/tdeyt8+75jhZPtpFREJE4EblHRP5bRD4jIv26ngyRTWpiHLPy06N+gNFmz/kvmpRtsZLII2d4IpNHpka9UXh7Tq3g1zF/WgpPAqOAVUC+53eviMjjIrJFRB7oZf8XRGSd57NHRH7tMTynum2f5Yc+gwUUF2Sx90wDrZ0Oq6VYxubjtYwcnsiknBSrpUQkxQWZbD9RR1cUj1fYfLyWgpwU8tIDv35CT/wxCmNU9UFVXa2qDwJje0soIrcDsaq6CCgQkcKeaVT1V6q63LNc5wbgt8Bs4FnvdlXd79fZGEJOcUGmO65wssFqKZbgjScsmpSFmPWYg0JxgSeucDY6F2LscrrYWlEbspl3/XEBnReRbwJbgWLgXB9plwPPeb6/BizFPcPqFYhIPpCrqjtE5IvAe0WkBNgPfE5Vr3gFFZF7gHsAcnNzWbdunR+n8TYtLS2DzhtMwklXm0OJEfhz6S4cZxOsEYZ11+xsi4ualg4yHTU+jx9O/0s74EuXq8Pd+fGZNdtpLIi+Olbe4ORSp5P0jgusW3elGy3gulR1QB8gAfgS8Avc02Yn9JH2cWCO5/tNwP19pP0hUOL5vgAY5fn+B+D9/ekqKirSwVJaWjrovMEk3HTd+ouN+sFfbgqtmB5Ydc2e2nRCx9/3sp6qveRzf7j9L62mN103/r91+snHt4ZWTA+sumaPrT2m4+97WWua233uH6wuYIf6eKb64z56H/CEqn5JVX+pqp19pG3BvZYzQCq9uKlEJAYoAdZ5Nu1T1fOe7zuAK9xOBvsRzXGFzcdrGJMR/L7j0U5xQRY7KqMzrrDleC3T8oaTlZoYkuP5YxQKgb+KyJ9E5A4R6SuqthO3ywhgDlDZS7plwFaP1QJ4WkTmiEgscCuw1w99BovwjlfYeTK6xiu4XEpZRZ1ZZS0EeOMK+6MsrtDhcLK9so5FIezu7M/cRz9W1XcDnwemACf7SP4S8AkReQT3bKoHReQhH+luBtZ3+/194GlgD7BFVdcMVJ/BOqJ1HqRQ9h2PdqJ1HqTdpxrocLhYHMLuzgMONIvI+4FbgDHANtxv+T5R1SYRWQ68C3hYVavw8davPSbZU9UDuHsgGcKIlMQ4Zo+JvvUVvOMzQtF3PNrJTk2kcGQqZRV1fHG51WpCx5bjtcQIXBPk+Y6640/vo5nAI6rqsxdRT9Q9tfZz/SY0RASLCrL4zfoKWjsdJCdEx7jGzcdrQtZ33OCe8uL5nWfocrqIj5J5kLYcr2VWfjrpSfEhO6Y/7qMfDtQgGKKP4oIsHK7oiSt0OV1sO2HiCaGkuCCL1iiKK7R2Oth9up7iEE+fEh3m1hB0isZnEBcjUTPlxf6zjVzqdIbU1xvteF0o0RJX2FFZT5dTQ17HjFEwBIS34wrRccN6jV9xENfKNbyT7NREpuSmRs2Lx5aKWuJihAUTMkJ6XGMUDAGjuCCLfWcaudQR+eMVNh+vCWnfcYMb93iF+qgYr7D5eC1zx40IeYzOGAVDwFg0yR1X2BHhcYX2Lic7KutD2nfc4GZRQRZtXU72nYnsubaa2rvYf6bBkpiVMQqGgDF/fCYJsTFsKq+xWkpQ2Xmyng6HiyUmnhByiguyEIGNxyLbhbTleC0uhcWTQ1/HjFEwBIykhFjmT8hg/dGLVksJKuuPXSQ+VkxLwQIyUhKYlZ/OhmORXcc2HLtISkIs88aFNp4AxigYAsyywhwOVzVT3dxutZSgseFoDfPGZZCSGB3jMezGssJsdp9uoKm9y2opQWPDsRoWTcoiIS70j2hjFAwBZVmhu7kbqS6kmpYO3jrfdPk8DaFnWWEOTpdSFqG9kE7VtnKytpWlFriOwBgFQ4C5alQaWSkJbDgamUbBa+yWFeZYrCR6mTcug+SEWDYci8w6tqHc7RpbNsWaOmaMgiGgxMQISyZns6G8hrcnv40c1h+tYURyPDPz062WErUkxMVQXJAVsXGFDUdryB+RREG2Ncu7GqNgCDjLCrO52NzBkQvNVksJKKrKxvKLLJmcTWyMWXrTSpYVZlNZ28rpularpQQUh9PF5uM1LCvMtmx5V2MUDAHH61qJNBfSseoWLjR1sMwiX6/hbbwxnUhzIe0720hTu4OlFsasjFEwBJy89GEUjkxlfYQ1771dba28YQ1uJuWkMip9WMS5kDYcrUEES8fAGKNgCArLCnPYdqKO9i6n1VICxsZy91TZYzLM0ptWIyIsK8xmU3kNTlfkxK42ll9kdn46GSkJlmkImlEQkcdFZIuIPNDL/jgROSUi6zyfWZ7tD4rIdhH5RbC0GYLPsinZdDhcbK+MjIV3OhxOyipqudb0OrINywpzaGp3sDdCprxobu9i16kGy3u2BcUoiMjtQKyqLgIKRKTQR7LZwLOqutzz2S8iRbjXdr4GqBaRG4OhzxB8Fk7MJCEuhjePREbz3t3qcZnxCTZi6eRsRIiYOuZt9Vhdx4LVUljO26uuvYb7Qd+TYuC9IrLN06qIA64D/qruvoyr6WPJT4O9SU6Io7ggi7VHqq2WEhDWHq4mMS7GrJ9gIzJSEpg7dgSlEVTH0obFUTQ+9FNbdEeC0ZdcRB4HHlXVvSJyEzBPVX/cI80C4IyqnheRPwDPA3OAfar6NxGZAvy7qn7eR/n3APcA5ObmFq1cuXJQOltaWkhNTR1U3mASKbpeP9nFnw518vC1SYxMDm74KtjX7L71reQmx/Dv8/1bejNS/pehwl9dfz/eyQvHuvifkiRGJIZvHXOp8rV1bUzNiOGLV4emjpWUlOxU1flX7FDVgH+AnwHFnu+3A9/ykSax2/d/A74OfA2407NtHvCb/o5VVFSkg6W0tHTQeYNJpOiqrGnR8fe9rE9urAiOoG4E85pVXHSfx1ObTvidN1L+l6HCX137zzTo+Pte1ue2nwqOoG4E85p5z+P5Haf9zjtYXcAO9fFMDZZp3cnbLqM5QKWPNE+LyBwRiQVuBfYOMJ8hTBiflUJBTgprw9znu/aw2z1x/bSRFisx9GTG6DRy0xLD3oW09nA1IrB8qvUdGYJlFF4CPiEijwAfAQ6KyEM90nwfeBrYA2xR1TXARmCuiPwMuB94Nkj6DCHi+qkjKauopbUzfFdjKz1czeSRqYzNNF1R7YaIUDJ1JBuO1oT1amxrD1czZ8wIW6zkFxSjoKpNuIPNZUCJqu5V1Qd6pDmgqrNVdZaqftuzzQXcCGwAblHVE8HQZwgdJdNG0ulwsak8PGe0bOlwsPVELSU2eIMz+Gb51JE0dzjCtvtzbUsHe880UDLVHi3RoEVmVLVeVZ9T1So/87Wp6vOqWhEsbYbQsWBCJqmJcWHbvN9UXkOXUykxriPbsrQwm/hYYV2YuinfPHoRVfu4J82IZkNQSYiLYenkbEoPV4flrKmlh6tJTYxjwYRMq6UYeiE1MY6FE7Mux37CjbWHq8kZnsiM0WlWSwGMUTCEgJJpOZxvbOdwVXjNmqqqlB6pZllhNvGx5laxM8un5lBe3RJ2s6Y6nC7WH73I8ik5xNhk5l1T0w1Bx+srfePQBYuV+MfBc01caOowrqMwwOt6WRNmdWzHyXqa2h22cR2BMQqGEDAybRhzx41g9cHwumFXH6wiRuDG6blWSzH0Q0FOKoUjU3ktzOrYqgNVJMbFcK1Fq6z5whgFQ0i4eUYe+882cqY+fJr3qw5Ucc3ETDItnLHSMHBunpHH1hO11F3qtFrKgFBVXjtYxbLCHFIS46yWcxljFAwh4eYZeQBh8yZXXt3CseoWVnh0G+zPipl5uBTWvBUedWz/2UbONbazYqa96pgxCoaQMDE7hWl5w1l10K8eypax2qPzJmMUwoYZo9PIH5EUNnVs1YEqYmOEG6fbJ54AxigYQsjNM/LYXlnHxeYOq6X0y+qDVcwZO4LRI5KslmIYICLCipl5bDxWQ3N7l9Vy+kRVWXWgikUFWYxItpd70hgFQ8hYMTMPVfv3EDnb0Ma+M43GdRSGrJiZR6fTRanNB7KVV7dQUXOJm23mOgJjFAwhZFrecMZnJbPqgL2b96s9+m6eYXodhRvzxmWQnZp4+X9oV1YdqEIEbr7KfnXMGAVDyBARVszIY/PxGhrb7Nu8X3Wwiqm5wynIsd96A4a+iY0RbpqRS+mRaluvD77qYBXzxmUwMs2/tRNCgTEKhpBy88w8upzK2sP2dCHVtHSwo7LOls16w8BYMSOP1k4nG47VWC3FJ6frWjl4rsm27kljFAwh5eoxIxiVPox/7D1vtRSfvLr/PC6FW4xRCFuKC7IYkRzPP/aes1qKT/7u0WW3rqhejFEwhJSYGOH9V4/mzaMXqW2xXy+kF3efZVrecKaPssfkZAb/SYiL4T2zRvHaW1W0dNhrHQ9V5cXdZ5k/PsO263MYo2AIObfNzcfpUl7eZ6/WQmXNJXafauC2uflWSzEMkdvn5dPe5bJdwPnguSbKq1u41cZ1zBgFQ8iZlpfG9FFpvLj7rNVS3sGLu88iAu+/erTVUgxDZN64DMZmJtmujr20+yzxscJ7Z4+yWkqvBM0oiMjjIrJFRB7oZX+6iPxTRF4TkRdFJEFE4kTklIis83xmBUufwVpumzuaPacbOFFzyWopgLtZ/9KesywqyGJUuhmwFu6ICLddnc+m4zVcaGq3Wg4ATpfyt73nKJk60nYD1roTFKMgIrcDsaq6CCgQkUIfyT4OPKKqNwFVwApgNvCsqi73fPYHQ5/Bet4/Jx8R95uTHdh9uoGTta22btYb/OPWufmowt/32CPgvPl4DRebO2zvngxWS2E58Jzn+2vA0p4JVPWXqvq652cOUA0UA+8VkW2eloZ9pg40BJS89GEsnpTFS3vO2mJFtpd2nyUxLsb0OoogCnJSmTN2hG1cSC/uPsvwYXG2X59DgnFDisjjwKOquldEbgLmqeqPe0m7CHhIVW8QkQXAGVU9LyJ/AJ5X1b/7yHMPcA9Abm5u0cqVKwels6WlhdRU+w1QihZdG8508fiBTh4oHsbkEbFDKmso2hwu5aulrVyVFcsXrw7sYKJo+V8GikDrev1kF3861MlDS5IYM3xo78BD0dbhUL5S2so1o+K4e2bikHQESldJSclOVZ1/xQ5VDfgH+BlQ7Pl+O/CtXtJlAjuA8Z7fid32/Rvw9f6OVVRUpIOltLR00HmDSbToamrr1KkPvKrffGHfkMsairbVB87r+Pte1jVvVQ1ZR0+i5X8ZKAKt62JzuxZ88xX9wStvDbmsoWh7YddpHX/fy7rleM2QdfRksLqAHerjmRos99FO3nYZzQEqeyYQkQTgL8A3VfWkZ/PTIjJHRGKBW4G9QdJnsAHDh8Xz7lmj+Puec1yysD/5s9tOkZuWyHU2Wv3KEBiyUxO5cfpInt95hg6HddNePLP1FBOykrlmQqZlGgZKsIzCS8AnROQR4CPAQRF5qEeaTwPzgG97ehrdAXwfeBrYA2xR1TVB0mewCR9fOI6WDodlo0/P1Ley7uhF7pg/lrhY00M7EvnYwvHUXeq0bIGnoxea2V5Zz0evGUdMjFiiwR+CEshV1SYRWQ68C3hYVavo8davqr8CfuUj++xgaDLYk3njMpiaO5xntp3izmvGhfz4f95+GoA7LDi2ITQsm5zNmIwkntl6ivfNCf0YlGe2niI+VvhQ0ZiQH3swBO3VSFXrVfU5j0EwGHwiInxs4Tj2nWlkz+mGkB67w+Fk5fbTLJ+SQ75ZTCdiiYkRPnrNOLZU1HLsQnNIj32pw8Ffd53h5hl5ZKUGNsAcLEx72WA5Hywaw/DEOJ7YeCKkx31573kuNnfwqSUTQ3pcQ+i5c8FYEuNieGJTZUiP+9ddZ2hud4RVHTNGwWA5qYlx3LFgLK/uP8/5xraQHFNVeWLTCQpHprKsMDskxzRYR1ZqIrfNzeeFXWeov9QZkmO6XMqTmyq5euwIisZnhOSYgcAYBYMt+NfFE3Cp8vvNJ/tPHAC2nqjj4Lkm7l46ERH7B/8MQ+fupRPpcLj409bQ1LHSI9WcqLnE3UvDp5UAxigYbMLYzGRumTWKP5adpLE1+Kuy/aK0nOzUBG692t5TDhgCx5Tc4Vw3JYcnNlXS2hncLtCqymOl5eSPSAq7UfLGKBhsw5dLJtPS4eD3WyqDepw9pxvYcKyGzywrIClhaCOpDeHFvddPpu5SJ89uOx3U42w5XsvuUw18Yfkk4sOsq3N4qTVENNNHpXHj9Fye2HQiqIujPLb2GOlJ8fxL8figHcNgT+ZPyKS4IJNfv3k8qGs4P7r2GLlpiWHTDbU7xigYbMW/3TCZhtYufrO+Iijl7zxZx5pD1Xxm6URSE818i9HIv91QSHVzB3/YUhmU8tcfvUhZRR2fv24Sw+LDryVqjILBVsweM4L3zB7F7zZUUN0c2HnwVZUfvXqY7NREPr0svIJ/hsCxeFI2103J4RelxwMev3K5lB//8zBjM5P42MLwHBBpjILBdvzHTVPpdLj479ePBrTc1QcvsONkPV+9sZDkBNNKiGbuWzGNpvYuHis9FtByX9x9lrfON/GNm6aSGBd+rQQwRsFgQyZkp/Cviyewcvtpdp2qD0iZlzocfP8fB5maO5w7FowNSJmG8OWq0Wl8pGgsT26q5HBVU0DKbGjt5IevHuLqsSN43+zwXdLVGAWDLfnau6aQlzaMb72wny6na8jl/c+ao5xrbOeHt88Mu94ghuBw/y3TSEuK51sv7MflGvq6Mj9ZdZiGti5+eNussJj4rjfM3WGwJamJcXzv/TM4XNXMo28MrYm/taKWxzee4KPXjKNovP2nLjaEhoyUBL797unsOtXAbzcMrWND6eFqnt12mruXTOCq0WkBUmgNxigYbMvNM/L4cNEYHistZ/PxmkGV0dDayVf/vIfxWSl8+z3TA6zQEO7cPi+fW2bm8V+rjwx6Qsbqpna+8Ze9TMsbztdvmhpghaHHGAWDrXnwAzOYmJ3Cvc/sprLmkl95OxxOPv/HndS0dPDonXNNF1TDFYgIP759Nrlpw/j80zv9nnurrdPJZ5/eyaVOB499bG5YdkHtiTEKBluTnBDH7z45H5cqdz25jeqmgXVTdThdfOMv+yirqOO/PjSHWWPSg6zUEK6kJ8fzu3+dT0uHg7ue2D7gCfM6HE7ufXYX+8808Oidc5k8cniQlYYGYxQMtqcgJ5Xf/esCqps7uO2XmymvbukzfWung889vZN/7D3H/bdM49a5Zn4jQ99MH5XGrz9RxInaS3zwV5s5VdvaZ/qm9i7uemI7aw5V8+AHZnLTjPCa36gvgmYURORxEdkiIg/4k2Yg+QzRR9H4DP58zyI6HC7e9/ON/G5DxRXTFKgq645Us+J/NlB6pJof3DaTz183ySLFhnBjyeRsnvnMQmovdfKeRzfw9JZKOh3v7PnmcimvHaxixX+vZ8fJOv77jjl8IsKmSwmKk1VEbgdiVXWRiDwhIoWqeqy/NMCs/vIZopdZY9L5x71L+M5LB3jolUP8fG05JVNzyM9I4mB5B9/f8SYVNZcoyE7h2c8Ws7Agy2rJhjBj/oRMXr53Kd96cT/f+dtB/nvNMZZPzWF0ehL7j3XwnW2lnK5rY2rucH7x8XnMHRc+6yQMFFEdev/cKwoVeRRYpaqvisidQJKqPtlfGmBuf/k8ee8B7gHIzc0tWrly5aB0trS0kJqaOqi8wcTo6htV5VCdi/Vnujha76K+XUmOU8alxbJ4dBwLR8WREGuPfuJ2uWY9Mbr6RlXZX+Nk/RkHxxtcNHQoKfHK+LRYluTHc01eLHE2GYsw2GtWUlKyU1XnX7FDVQP+AR4H5ni+3wTcP5A0A8nX81NUVKSDpbS0dNB5g4nR5R9Op8u22owu/7CrrkisY8AO9fFMDVZMoQX3mz9AKr5jF77SDCSfwfAOwnn0qCE8iKY6FqyH7k5gqef7HKBygGkGks9gMBgMQSJYo3leAjaIyGjgFuBOEXlIVR/oI00xoD62GQwGgyFEBKWloKpNwHKgDChR1b09DIKvNI2+tgVDn8FgMBh8E7Rx/6paDzznb5qB5DMYDAZDcDCBXIPBYDBcxhgFg8FgMFzGGAWDwWAwXCYoI5pDiYhcBE4OMns2MLiJ+oOL0eU/dtVmdPmHXXWBfbUNVtd4Vc3puTHsqNDWhwAACZNJREFUjcJQEJEd6muYt8UYXf5jV21Gl3/YVRfYV1ugdRn3kcFgMBguY4yCwWAwGC4T7UbhN1YL6AWjy3/sqs3o8g+76gL7aguorqiOKRgMBoPhnUR7S8FgMBgM3TBGwWAwGAyXiVijYNc1ovsrX0TSReSfIvKaiLwoIgkiEicip0RkneczyyJtPnWIyIMisl1EfmGRri9007RHRH4dwmuWKyIb+tgfLyL/EJFNInJ3b9ss0DXOc13WishvxE2+iJzpds2u6MMeAl0+NYTgvuxP14PdNB0WkW8G+3r5ehb0ki6gz7GINArSbf1noEDc6z/3m2Yg+YKtC/g48Iiq3gRUASuA2cCzqrrc89kfSF1+aLtCh4gU4V4D4xqgWkRuDLUuVf2VVxOwAfitL62B1OXRlgH8HkjpI9m9wE5VXQJ8SESG97It1Lo+B3xBVa8HxuJeH30h8INu1+yiBbqu0BCC+7JfXar63W517ADwB19aA6kL38+CntoD/hyLSKOAe/pt70yrr/H2wj39pRlIvqDqUtVfqurrnp85QDXudSXeKyLbPG8AwZjdtl9tvei4DvirZ3m/1cAyC3QB7rdMIFdVd/SiNdA4gTuApj7SLOdt/euB+b1sC6kuVf22qh7y/MzCPSK2GPiMiOwSkR8GWNOAdPWiYTnBvS8HogsAEVkAnFHVs71oDRi9PAt6spwAP8ci1SikAGc93+uA3AGmGUi+YOsCQEQWARmqWgZsB25U1WuAeODdAdY1UG2+dNjmmgFfAn7Vh9aAoqpNA1jzI+T1bIC6ABCRO4CDqnoO+CfuB8oCYJGIzLZAly8NtrlewFeAn3u+B/V6eenxLOhJwOtXpBoFu64RPaDyRSQTd8Xz+pv3qep5z/cdQECbz35o86XDLtcsBigB1vWh1Qpsuxa5iBQA3wC+6tm0WVWbVdUJ7Maaa+ZLg12u1whgpKoe92wK+vXy8SzoScDrV6QaBbuuEd1v+Z5g0l+Ab6qqd6K/p0VkjojEArcCewOsa0DaetFh+TXzsAzYqm8PvAnFNRsItlyL3ONHfxa4u9tb8moRGSUiycBNuH3nocaXBsuvl4cPAK92+x3U69XLs6Anga9fqhpxHyAN90PgEeCQ58I81E+adF/bLND1BaAe9xvvOty+zpnAPmA/7sCWVdfsCh24Xyw2AT8DjgATQ63Lk+6HwO19aQ1ifVvn+Xs98OUe+8YDBz3XZzsQ62ubBbp+ApzvVs+uw93SOuy5bl8OhqYB6LpCQ7Dvy4Ho8mx/BpjXl9YA6+n5LPiuj3sy4M+xiB3R7HkTehewXlWrBppmIPmCrcsqBqtNRJKA9wC7VLXCLrrsgoiMxv3mtlo9b+W+thl6J9zrQDAJ9HMsYo2CwWAwGPwnUmMKBoPBYBgExigYDAaD4TLGKBgiAs9IzmEikiwieQEoTwKha5DHjh1guoxgazFEH8YoGCKFx4AJwC3Ad/pKKCL/IiLv6WP/VOA1EckSNy+LyAOerofd0/1ARCZ6vieIyF/7Oe5/ishn+kmTAbzuGXfRH8+ISK+joUVkjdfAiMidIvLZAZRpiHKCMfTfYAgp4p7srkpVD4vII8BoEVnXLcknVfVUt9/jAJePcgR3N9HRwJ+ADkCAF3APBnpdRK5V92AlgCLeNkDvAlpFZJrnd4WqdvY4RAfQcxveKThU1aGq9SJSCszDPeiue8tBcU9Hsatb9h93a9TMxT3Nh0NEioHjuF/8nJ6Pw1NejOd4V1wDg8H0PjKENSISD2wGXgHWAJ/GPdlbNu4+3qXAEiBRVVs9eb4D7FHVf3gekMlAu+dhOlZVT4vILcD9uB/C/7+9swvNsgzj+O8v8wuD7OOkDrYiaFjBrIMOxINJwwpKKSLCjJkR2QdRFmwkFEWQSURISkjWMA0kD4piJ+rAA6EPTOdBX6MCKSgqlbHhcrN/B9f9vHv3biKDOti8fkfv+94fz/MMdv+f6+O+7j7b2yTNtT1aFuk/iVzwxcCnxII8SCz6txJ7Jr6T9C3jJQeaS/tvwAJgyPYdkh4BOoHrS98zpX8r8AuxmL9SnvFdopzHiBv+eSVtBTaW59gF9AH3ELnrVU2qw8Ao8Iz/hyKBycwnLYVkpvMQsUADbAC6gKuAV4k6Nd2EW2mPpLHS71pgWFJX+T4feBQ4BpyStIxYRE8CI8BKSX8A/ZJ+tn1W0te2b5PUDjxBbAR7zvagooT432XuUdsdAJKeJyya3ZKuIVxe2N4J7JT0AiFWvaX/cWCZ7UokANZLehu4qc5CWErko68q424kaj312V4t6XZC+MaArbb3Tf/PnFwspCgkM50e4Ceg3fZaSSsJC+EcUXpgv+2/iIqWAEg6CJyw/XD9RJLaCDHpBz6yfbT8fjWxyHYD7xHluduKi2ox8DHwAbAbWEWIzHCZdjoumr3Ai0CvpCWEC+pMYyfbT5X7uq70/7DcV8US4LXSZ0657z3lXp6W1FtZTUnSSIpCMqOx/U9lAUi6lCjf8GBpHiJiBGur/pJuIBbsyyW12v6+bq5+4G5JnwDtperl8dLcBjTbPlW+H7PdUSyFdts/SjpXxixkXBSaJB0on5uBs5LWEe6jCaWayxwq8YCXgJfr20tQ+U3GheZKoqRBC7CmxETesL1P0n2EYO0gyif/ShRHe50QnU6fv55OchGTopDMJt4n3ooHAWx/JukxSR22D0iaD2wjqoKOADsk3dn41mx7NYCkz20vL5+/Ak5f4PpPAr8TolDNud72l2WOevfRAiKG0MhGIpB8yJNLJR8BVlSBbkl3Acttd5fvTUzMKFxBWE0twJrSdpQQlmYgRSGZRIpCMhuYQ2QJddke0MSTptYDp0tW0DtAT7EIkLSdyChaZ3tgqokVpYufJZIy6gO7Nze4j3CcSYCkK6q+lSA03Ce2Rxi3QqprtQGbiFjDLSV2sN32N2WMCbdYbQiRkURpH2Mi+233FDG8lxCrXVP0S5IauU8hmQ00AXOLILQSWUiHARxHJN5PnAq32XZPNcj2XmALcFBSyxTzXmL7JHCISDkFaimiRxxHMz5OsQokdUoaAE5MMReE+2bSObuSlkr6gjimc5PtLbYfIMRms6Qfqv0QdWM2AG8xMT110tTlOavU2nkpCMmFyJTUZNYhqal+8StpqwttT3ncoqRFtoenapvmdS8DmjzNs3pLLGCR7aHztM9r3PNQ3v7H6vZMJMl/QopCkiRJUiPdR0mSJEmNFIUkSZKkRopCkiRJUiNFIUmSJKmRopAkSZLU+BfEoXh5bmwUdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Data for plotting\n",
    "t = np.arange(0.0, 2.0, 0.01)\n",
    "s = 1 + np.sin(2 * np.pi * t)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(t, s)\n",
    "\n",
    "ax.set(xlabel='你大爷的中文字体', ylabel='voltage (mV)',\n",
    "       title='About as simple as it gets, folks')\n",
    "ax.grid()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "453px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}