language_translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from io import open\n",
    "import os\n",
    "import numpy as np\n",
    "import time\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "春 に なれ ば 花 が 咲き 、 夏 に は ほととぎす が 鳴き 、 秋 に は 月 が 美しく 、 冬 に は 雪 が 降っ て 身 が ひきしまる \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# MeCab is an open-source text segmentation library for use with text written in the Japanese.\n",
    "import MeCab # pip install mecab-python3\n",
    "wakati = MeCab.Tagger(\"-Owakati\")\n",
    "\n",
    "example = \"春になれば花が咲き、夏にはほととぎすが鳴き、秋には月が美しく、冬には雪が降って身がひきしまる。\".strip('。')\n",
    "print(wakati.parse(example))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "MAX_LEN_SENTENCE = 10\n",
    "SOS = 0\n",
    "EOS = 1\n",
    "\n",
    "class Language:\n",
    "    def __init__(self, name_lang):\n",
    "        self.name = name_lang\n",
    "        self.word2index = {}\n",
    "        self.index2word = {SOS: \"SOS\", EOS: \"EOS\"}\n",
    "        self.vocab_size = 2\n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.vocab_size\n",
    "        \n",
    "    def addWords(self, sentence):\n",
    "        for word in sentence.split(' '):\n",
    "            if word not in self.word2index:\n",
    "                self.index2word[self.vocab_size] = word\n",
    "                self.word2index[word] = self.vocab_size\n",
    "                self.vocab_size += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class LanguageTranslate():\n",
    "    def __init__(self, input_lang, target_lang, file_path='./dictionary', limit_size=None):\n",
    "        self.input_lang = input_lang\n",
    "        self.target_lang = target_lang\n",
    "        self.pair_sentences = []\n",
    "        \n",
    "        file_name = \"{}-{}.txt\".format(self.input_lang.name, self.target_lang.name)\n",
    "        file_name = os.path.join(file_path, file_name)\n",
    "        lines = open(file_name).read().strip().split('\\n')\n",
    "        \n",
    "        self.pair_sentences = [[s for s in line.split('\\t')[:2]] for line in lines]\n",
    "        \n",
    "        # if the language is japanese do text segmentation\n",
    "        for s in self.pair_sentences:\n",
    "            if self.target_lang.name == 'jpn':\n",
    "                s[1] = wakati.parse(s[1]).strip(' \\n')\n",
    "                s[1] = s[1].strip('。')\n",
    "            else : s[1] = self._normalize(s[1])\n",
    "            s[0] = self._normalize(s[0])\n",
    "        \n",
    "        for pairs in self.pair_sentences: \n",
    "            self.input_lang.addWords(pairs[0])\n",
    "            self.target_lang.addWords(pairs[1])\n",
    "        \n",
    "        print('Pairs size (before filter):', len(self.pair_sentences))\n",
    "        self.pair_sentences = self._filterpairs(self.pair_sentences, lang_1=self.input_lang.name) \n",
    "        print('Pairs size (after filter):', len(self.pair_sentences))\n",
    "       \n",
    "    def _filterpairs(self, pairs, lang_1='eng'):\n",
    "        filter_prefixes = (\"the\", \"this\", \"that\", \"these\", \"those\") # must be lower case\n",
    "        pairs_filtered = []\n",
    "        # the filter is built over the first pair of\n",
    "        for pair in pairs:\n",
    "            if (len(pair[0].split(' ')) < MAX_LEN_SENTENCE) and (len(pair[1].split(' ')) < MAX_LEN_SENTENCE):\n",
    "                if (lang_1=='eng' and pair[0].startswith(filter_prefixes)):\n",
    "                    pairs_filtered.append(pair)\n",
    "        return pairs_filtered\n",
    "                    \n",
    "    \n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pair_sentences)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        pair = self.pair_sentences[idx]\n",
    "        \n",
    "        inp_lang = [self.input_lang.word2index[i] for i in pair[0].split(' ')]\n",
    "        inp_lang.append(EOS)\n",
    "        targ_lang = [self.target_lang.word2index[i] for i in pair[1].split(' ')]\n",
    "        targ_lang.append(EOS)\n",
    "        \n",
    "        inp_lang = torch.tensor(inp_lang, dtype=torch.long).view(-1, 1)\n",
    "        targ_lang = torch.tensor(targ_lang, dtype=torch.long).view(-1, 1)\n",
    "       \n",
    "        return inp_lang, targ_lang\n",
    "        \n",
    " \n",
    "    def _normalize(self, s):\n",
    "        s = s.lower() # everything to lower case\n",
    "        s = s.translate(str.maketrans('', '', string.punctuation)) # remove all punctuations\n",
    "        return s\n",
    "    \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pairs size (before filter): 45815\n",
      "Pairs size (after filter): 2304\n",
      "Input language vocabulary size: 9390\n",
      "Target language vocabulary size: 13935\n"
     ]
    }
   ],
   "source": [
    "data_path = './data'\n",
    "input_language = Language('eng')\n",
    "target_language = Language('jpn')\n",
    "\n",
    "dataset = LanguageTranslate(input_language, target_language, data_path)\n",
    "\n",
    "input_lang_vocab_size = dataset.input_lang.vocab_size\n",
    "target_lang_vocab_size = dataset.target_lang.vocab_size\n",
    "print(\"Input language vocabulary size:\", input_lang_vocab_size)\n",
    "print(\"Target language vocabulary size:\", target_lang_vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of pairs of sentences: 2304\n"
     ]
    }
   ],
   "source": [
    "train_size = len(dataset)\n",
    "batch_size = 1\n",
    "train_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True)\n",
    "print('Number of pairs of sentences:', train_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder model\n",
    "![Encoder model](./images/encoder.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class Encoder(torch.nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, embedding_dim):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        \n",
    "        self.embedding = torch.nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_dim)\n",
    "        self.rnn = torch.nn.LSTM(embedding_dim, hidden_size, num_layers=1)\n",
    "        \n",
    "    def forward(self, inputs, last_hidden):\n",
    "        embedded = self.embedding(inputs).view(1, 1, -1)\n",
    "        output, hidden = self.rnn(embedded, last_hidden)\n",
    "        return output, hidden\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        return (torch.zeros(1, 1, self.hidden_size, device=device),\n",
    "                torch.zeros(1, 1, self.hidden_size, device=device))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder model\n",
    "![Decoder model](./images/decoder.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderAttn(nn.Module):\n",
    "    def __init__(self, hidden_size, output_vocab_size, embedding_dim, dropout_p=0.1, max_length=MAX_LEN_SENTENCE):\n",
    "        super(DecoderAttn, self).__init__()\n",
    "\n",
    "        self.embedding = nn.Embedding(output_vocab_size, embedding_dim)\n",
    "        self.enc_hidden = nn.Linear(2*hidden_size, embedding_dim)\n",
    "        self.attn = nn.Linear(hidden_size + embedding_dim, max_length)\n",
    "        self.attn_combine = nn.Linear(hidden_size + embedding_dim, hidden_size)\n",
    "        self.rnn = nn.LSTM(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, output_vocab_size)\n",
    "\n",
    "    def forward(self, input, hidden, all_encoder_outputs):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "\n",
    "        attn_hidden = torch.cat((hidden[0][0].view(-1), hidden[1][0].view(-1)))\n",
    "        attn_hidden = self.enc_hidden(attn_hidden)\n",
    "        \n",
    "        attn_weights = torch.cat((embedded[0], attn_hidden.view(1, -1)), 1)\n",
    "        attn_weights = self.attn(attn_weights)\n",
    "        attn_weights = F.softmax(attn_weights, dim=1)\n",
    "        \n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0), all_encoder_outputs.unsqueeze(0))\n",
    "\n",
    "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "\n",
    "        output = F.relu(output)\n",
    "        output, hidden = self.rnn(output, hidden)\n",
    "\n",
    "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
    "        return output, hidden, attn_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using gpu: True\n"
     ]
    }
   ],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "print(\"Using gpu:\", use_cuda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# hidden NN size and embedding dimensions definition\n",
    "embedding_dim = 256\n",
    "hidden_size = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "encoder = Encoder(input_lang_vocab_size, hidden_size, embedding_dim).to(device)\n",
    "decoder = DecoderAttn(hidden_size, target_lang_vocab_size, embedding_dim).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "\n",
    "encoder_optim = torch.optim.SGD(encoder.parameters(), lr=learning_rate)\n",
    "decoder_optim = torch.optim.SGD(decoder.parameters(), lr=learning_rate)\n",
    "\n",
    "criterion = torch.nn.NLLLoss()\n",
    "\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "writer = SummaryWriter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def train(encoder_model, decoder_model, encoder_optim, decoder_optim, criterion, epochs=10, num_samples=75000, plots_per_samples=5000):\n",
    "    \n",
    "    cost = []\n",
    "    start_time = time.time()\n",
    "    loss = 0\n",
    "    \n",
    "    sample_acc_avg = 0\n",
    "    sample_loss_avg = 0\n",
    "\n",
    "    for epoch in range(1, epochs+1):\n",
    "        for idx, (inp_sentence, targ_sentence) in enumerate(train_loader):\n",
    "            sample_acc = 0\n",
    "            sample_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            loss = 0\n",
    "            inp_sentence = inp_sentence[0].to(device)\n",
    "            targ_sentence = targ_sentence[0].to(device)\n",
    "            \n",
    "            encoder_model.zero_grad()\n",
    "            decoder_model.zero_grad()\n",
    "\n",
    "            encoder_hidden = encoder_model.init_hidden()\n",
    "\n",
    "            all_encoder_outputs= torch.zeros(MAX_LEN_SENTENCE, encoder.hidden_size, device=device)\n",
    "            # run input sentence though encoder model\n",
    "            for i in range(inp_sentence.size(0)):\n",
    "                encoder_output, encoder_hidden = encoder_model(inp_sentence[i], encoder_hidden)\n",
    "                all_encoder_outputs[i] = encoder_output[0, 0] # get output in one dimension\n",
    "                \n",
    "\n",
    "            decoder_input = torch.tensor([[SOS]], dtype=torch.long).to(device)\n",
    "            decoder_hidden = encoder_hidden # the decoder use the same hidden layer used by the encoder\n",
    "\n",
    "            # run target sentence though decoder model\n",
    "            for i in range(targ_sentence.size(0)):\n",
    "                decoder_output, decoder_hidden, _ = decoder_model(decoder_input, decoder_hidden, all_encoder_outputs)\n",
    "\n",
    "                _, guess = decoder_output.topk(1)\n",
    "                decoder_input = guess.squeeze().detach()  # detach from history as input\n",
    "\n",
    "                loss += criterion(decoder_output, targ_sentence[i])\n",
    "                guess = guess[0].item()\n",
    "                if guess == int(targ_sentence[i]):\n",
    "                    running_corrects += 1\n",
    "\n",
    "                if decoder_input.item() == EOS:\n",
    "                    break\n",
    "\n",
    "            loss.backward() # backpropagation through time\n",
    "\n",
    "            encoder_optim.step() # update gradients of encoder\n",
    "            decoder_optim.step() # update gradients of decoder\n",
    "\n",
    "            sample_acc = running_corrects / targ_sentence.size(0)\n",
    "            sample_loss = loss.item() / targ_sentence.size(0)\n",
    "\n",
    "            writer.add_scalar('Loss/Train', sample_loss, epoch*len(train_loader) + idx) \n",
    "            \n",
    "            cost.append(sample_loss)\n",
    "\n",
    "            sample_acc_avg += sample_acc\n",
    "            sample_loss_avg += sample_loss\n",
    "\n",
    "        sample_acc_avg /= train_size\n",
    "        sample_loss_avg /= train_size\n",
    "        \n",
    "\n",
    "        time_elapsed = time.time() - start_time\n",
    "        print('Epoch [{}] -> Loss: {:.4f}  Acc: {:.4f} ({:.0f}m {:.0f}s)'.format(\n",
    "            epoch, sample_loss_avg, sample_acc_avg, time_elapsed//60, time_elapsed%60))\n",
    "        sample_acc_avg = 0\n",
    "        sample_loss_avg = 0\n",
    "\n",
    "    time_elapsed = time.time() - start_time\n",
    "    print()\n",
    "    print('Training completed in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "    torch.save(encoder_model.state_dict(), './encoder_model.pth')\n",
    "    torch.save(decoder_model.state_dict(), './decoder_model.pth')\n",
    "    writer.close()\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1] -> Loss: 4.8429  Acc: 0.1707 (0m 35s)\n",
      "Epoch [2] -> Loss: 4.4040  Acc: 0.2181 (1m 12s)\n",
      "Epoch [3] -> Loss: 4.0849  Acc: 0.2567 (1m 51s)\n",
      "Epoch [4] -> Loss: 3.8010  Acc: 0.2908 (2m 28s)\n",
      "Epoch [5] -> Loss: 3.5197  Acc: 0.3274 (3m 4s)\n",
      "Epoch [6] -> Loss: 3.2330  Acc: 0.3592 (3m 41s)\n",
      "Epoch [7] -> Loss: 2.9279  Acc: 0.3930 (4m 18s)\n",
      "Epoch [8] -> Loss: 2.6441  Acc: 0.4335 (4m 55s)\n",
      "Epoch [9] -> Loss: 2.3596  Acc: 0.4803 (5m 32s)\n",
      "Epoch [10] -> Loss: 2.1081  Acc: 0.5201 (6m 10s)\n",
      "Epoch [11] -> Loss: 1.8454  Acc: 0.5698 (6m 48s)\n",
      "Epoch [12] -> Loss: 1.6180  Acc: 0.6055 (7m 24s)\n",
      "Epoch [13] -> Loss: 1.3995  Acc: 0.6495 (8m 1s)\n",
      "Epoch [14] -> Loss: 1.2065  Acc: 0.6879 (8m 37s)\n",
      "Epoch [15] -> Loss: 1.0331  Acc: 0.7304 (9m 14s)\n",
      "Epoch [16] -> Loss: 0.8688  Acc: 0.7715 (9m 51s)\n",
      "Epoch [17] -> Loss: 0.7239  Acc: 0.8131 (10m 29s)\n",
      "Epoch [18] -> Loss: 0.6105  Acc: 0.8444 (11m 6s)\n",
      "Epoch [19] -> Loss: 0.5067  Acc: 0.8683 (11m 42s)\n",
      "Epoch [20] -> Loss: 0.4212  Acc: 0.8906 (12m 19s)\n",
      "Epoch [21] -> Loss: 0.3644  Acc: 0.9029 (12m 56s)\n",
      "Epoch [22] -> Loss: 0.3189  Acc: 0.9102 (13m 33s)\n",
      "Epoch [23] -> Loss: 0.2827  Acc: 0.9161 (14m 11s)\n",
      "Epoch [24] -> Loss: 0.2689  Acc: 0.9176 (14m 48s)\n",
      "Epoch [25] -> Loss: 0.2545  Acc: 0.9178 (15m 25s)\n",
      "Epoch [26] -> Loss: 0.2365  Acc: 0.9192 (16m 4s)\n",
      "Epoch [27] -> Loss: 0.2224  Acc: 0.9215 (16m 41s)\n",
      "Epoch [28] -> Loss: 0.2166  Acc: 0.9213 (17m 20s)\n",
      "Epoch [29] -> Loss: 0.2102  Acc: 0.9198 (17m 58s)\n",
      "Epoch [30] -> Loss: 0.2027  Acc: 0.9242 (18m 38s)\n",
      "Epoch [31] -> Loss: 0.1960  Acc: 0.9260 (19m 17s)\n",
      "Epoch [32] -> Loss: 0.1908  Acc: 0.9247 (19m 56s)\n",
      "Epoch [33] -> Loss: 0.1920  Acc: 0.9256 (20m 35s)\n",
      "Epoch [34] -> Loss: 0.1897  Acc: 0.9228 (21m 14s)\n",
      "\n",
      "Training completed in 21m 14s\n"
     ]
    }
   ],
   "source": [
    "#epochsc = int(100000/train_size)\n",
    "#print('Total epochs:', epochsc)\n",
    "epochsc = 34\n",
    "\n",
    "cost = train(encoder, decoder, encoder_optim, decoder_optim, criterion, epochs=epochsc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f0cbe273a10>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD6CAYAAABamQdMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3wUdf4/8NeHEBJ6kWChRU7FUxBLQFDPw96Pn/f1Tr373lm/3NnOs5yH7ezi2fXwTrGLinoWQEE6SC+hhhACgQQSUiGV9PL5/bGzyWZ3dndmZ2ZnZvf1fDx4sJmd8k6yee9n3/MpQkoJIiKKLV3sDoCIiMzH5E5EFIOY3ImIYhCTOxFRDGJyJyKKQUzuREQxKGxyF0J8IIQoFULs8Nk2QAixSAixR/m/v7VhEhGRHiJcP3chxPkAjgD4REo5Stn2IoByKeULQogpAPpLKf8e7mIDBw6UqampxqMmIoojmzZtOiSlTNFzTNdwO0gpVwghUv02TwIwUXn8MYDlAMIm99TUVKSnp+uJj4go7gkh9us9JtKa+9FSyiIAUP4fFCKoyUKIdCFEellZWYSXIyIiPSy/oSqlnC6lTJNSpqWk6PpUQUREEYo0uZcIIY4FAOX/UvNCIiIioyJN7nMA3KQ8vgnAbHPCISIiM2jpCjkTwFoAI4UQBUKI2wC8AOASIcQeAJcoXxMRkUNo6S1zY5CnLjI5FiIiMglHqBIRxaCYTe5Vdc34YXuh3WEQEdkibFnGCWas248jDS24Y+LPNB9z98zNWLnnEMYM6YehA3pYGB0RkfO4ouU+b3sRvkrP13VMYWU9AKCxpdWKkIiIHM0VLfe1+w7bHQIRkau4ouVORET6MLkTEcUgJnciohjE5E5EFINiNrmHXoKEiCi2xWxy7yDsDoCIKOriILmzDU9E8Sdmkzvb60QUz2I2uRMRxTMmdyKiGBSzyd1bab/41RXILKwy5ZxLskpw/1dbTTkXEZGVYja5+/phe5Ep57nt43R8u/mgKeciIrJSXCR3IqJ4w+RORBSDXDHlrx6lNQ3o2oXvWUQU31yX3OduL0J2cTXuv3Sk6vPjnlsCABiR0lPzOfMO1WJI/+7omsA3BSKKDa7LZnd9vhlvLs0Ju5/WQUz55XWY+PJyvLQg21hgREQO4rrkrpXWSQcOHWkEAKzLLbcuGCKiKIvZ5E5EFM9cm9xLaxrsDoGIyLFcm9wve22F6efMKKiClJxFkojcz7XJvaKu2dTzbcuvxDXTVuGD1XmmnpeIyA6uTe56hGqMC9G5X012cbXF0RARWS8uknsoLMMQUSyK++RORBSLYiK51zQ0I3XKXMzfoT7749s/7cWuGCu3SClx52ebsHbvYbtDISIHionk/tqiPQCAacuCj1z9b3pBtMKJirqmVszLKMZtH2+0OxQiciBDyV0IcZ8QIlMIsUMIMVMIkWxWYHp8sDrXjssSETlWxMldCDEYwF8ApEkpRwFIAHCDWYGp+Wl3menn9O8t09DcZvo1iIiizWhZpiuA7kKIrgB6ACg0HlJw93y+WfO++8pqO32ttVPMkcYWPSERETlSxMldSnkQwMsADgAoAlAlpVxoVmBqqhvMT7xu7QrpzqiJKFqMlGX6A5gE4HgAxwHoKYT4X5X9Jgsh0oUQ6WVl5pdVnOL0pxfi4W+3GzrHY7MykDplrq5jtE5tTETxxUhZ5mIAuVLKMillM4BvAZzjv5OUcrqUMk1KmZaSkmLgcuE1tQSvl0uL27qVdc2YuSHf0Dk+XXfApGiIKN4ZSe4HAIwXQvQQnruSFwHIMiesyOwuORL0uQ9X52H6ir0B2/1vqGp1sLIeS7JKwu73VXo+TvnHfLS2sZBCRNFjpOa+HsDXADYDyFDONd2kuCzx/Lxdpp3ryjdW4raP08Pu99ScTNQ1taK+udW0awPuvVdARNFhaA1VKeUTAJ4wKZaINJicNNXa8bWNLeiemNBpW1W9ubNSEhGZyXULZPubs9XS3peoqm/GmKcW4p4LT7D0OpGKtKxERLHN9dMPGLlR2tTShtow/dor65oAALMtfhMhIjKT61vuRvzu3XVI319hdxhERKZzfcvdCDcndt5OJaJQ4jq5q1myqxQ5pYFdKg+U12k+R05pjWpvlls/2oh7Zm4xFJ8/VtyJSI2rk3tDcyteWbjb9PNO+aZjpKneHofr9x3Gxa+uwKfr9gc8t3RXKb7fxto9EVnP1cn9n/N3obSm0e4wOtl/2NPC315QZXMkRBTPXJ3cP1ydZ8l5JYAdB81Nzhx0RETR5OrkbpVN+ytw9b9WYd2+yJew86Zyq/qh872CiEJhcg+hoKJef6+UaN/h5B1VIlIRl/3cF2QWY8TAnnaHQURkmbhM7n+asSlq17K61l5jwQImROR+LMtYxD+nm1579zn/xrxyc89NRK4Xd8m9uKrB0vOvyTlk6fnV7C6pifo1icjZ4i65j5+6RPO+kZRUZnGCMSJygLhL7tFi9bJ+VpuxNg/LskvtDoOIIsTkHiX+nwLqm1pRUm1ticiIx2dn4pYPN9odBhFFiMk9AnrKNbVN6itFnf/SMpz9fOgSUVVdM2oa1Fd8UvtkkFNaY0vNn4ich8k9DLOW8fNvpZcpc+IUVtYHPWbM0wtx5jOLNF/j4ldX4HfvrY8sQCKKKUzuYTw2a0dkB/o1rJta1Fv7bWE+BTS3urt2T0T2YHIPY2t+pd0hEBHpxuQeggTQ2hbYcjZz0Gmkg5t8YxCcYIaI/MTl9APRoDX/CwBbDlTgYIjaOxGRXkzuIUSrPXztv9dE6UpEFC9YlonAfV9tNe1cFk33TkRxjsk9ArM1TDEgpcThI+pLAO4tC1yAm4jITCzLWGTW1sKg88xc9MpP7Y8jvRnKDpJEFApb7iHEWgKVUqKppc3uMIgoCpjcbWZFzb2itkl1+2uL9+Ckx35EbSMX+CCKdUzuIczRUFufsTbP8jjC8X+DOOOZRVi2K3BGx6825gMAqoPMV0NEsYPJPYRVGibhenx2JuqaIm8JR9pwDzd5Wfp+rs5EFM+Y3E1w9+db7A6BiKgTQ8ldCNFPCPG1EGKXECJLCDHBrMDcJF3DGqZXvrlS/Qn2cyciCxjtCvkGgPlSyuuEEN0A9DAhJtepbjD3BmVBRR3O++cyU89JRPEl4pa7EKIPgPMBvA8AUsomKSWnUNRJrZ/7rqLABa8bmluxJKskGiEFKKqqx67ialuuTUSRMVKWGQGgDMCHQogtQoj3hBA9TYqL/Dw7dydu+zgdWw5UAIhuH/wJU5fi8teDlJWIyJGMJPeuAM4E8B8p5RkAagFM8d9JCDFZCJEuhEgvKyszcLnYpLWf+4Fyz6yRVfXaujGaOS1x6OtIvLUsB6U1zl0PligeGUnuBQAKpJTedd2+hifZdyKlnC6lTJNSpqWkpBi4XGyK9H7qun2HDZ/DDBkHq/DSgmzc96V5k6kRkXERJ3cpZTGAfCHESGXTRQB2mhIVheWU7pctymImq3MOI7+8zuZoiMjLaD/3ewB8JoTYDuB0AM8bDym+RLoSkxO9vyrX7hCISGGoK6SUciuANJNiISIik3CEqs30ttu13idduNOebpNE5AxM7i4gpWx/E6hvag07rwwA5JQewYZczi9DFK+Y3G2mpeReUt2xotOdn23WfG6t3SaNiJ07BkSxhcndZr99Z23Atk/W7Q/YFupNYMq3GWaGREQxgMndZrtLAtdTXbG782CvnNLYWXN18ifpmL+jyO4wYkruoVrcM3MLV9miTpjcXSD3kPHknlFQheJqY6NIU6fMxdQfswydY+HOEvz5U+2lJQrv799sx/fbCrFZmZqCCGBydyWtUwv43ni9Ztqq9scTX1qONxbvUT2mrKYRZTWNqs8BwDs/7Qv6XGNLG279aCP2llnzSSPvUC1Sp8zFGg2LqBDFOyZ3N/AruI94ZJ6mw4K9BzS2tOG1xbtVnxv73GKMfW6xjtA6Ylu/7zCW7irFU99bM1B5fa5nyoVZWw9acn6iWGJ0PneKBp+ukHrtLqlBv+6JpoYTTDRnqpRSoqVNIjGB7RMiNfzLcAEtSXNhZrHq9ktfW4Gzpy4Jeez6fYdddzPuy435OPHRH3Gwst7uUIgcick9RkyesSlgm7fkHqpGv7OwGtdPX4fn5xm7UWrUkcYWvLdyH9ratLX/v99eCADILau1Miwi12Jyj3MVdU0APOUbPV5ekI0/vL++0zYtI2eDeW5uFp6dm4XFGlabitZc9aGU1TQ651ODA34e5DxM7i6xLFv/Qid//jSwNW+WactysHLPIdV7AZHcH6hu8IymbQxRHlJbktDf5gMVqFTesKw09rnFOPeFpZZfRw+OFiZfTO4uMGdrod0hBDXprdUB26I56Er6NVt//e81uGH6OtOv88WGA1F50yAyC5O7C6Tvt2ZwipQSX28qAOApM8zdbs7I0WiUK0K14ncV6ysxhbO7pAZTvs3AvV9wtSlyDyb3ODZnWyG+2+LpM76n9Aju+pwjR9U0NntKRYdrgw/uInIaJvc45rsOqxmidV/P9zrBbq42t7axjEJxjck9js3ckG93CPr4VGLCTZV8z+dbcPrTi0y9vBN66ajxv+9ABHCEKplo/+HgC2Rvy6/U3d3SiPlBBnVFwi3L3MbSerxkHJM7RYVarxq9Hvp6uwmREMUHlmXI1ViQIFLH5E4hnf/isoBtqVPmhj3ujcV7MNui2Rutrn23tLZhvcrNZqfW3InUMLlTSAfKg9fRQ3lt8W59/cIjTJxGpjwI5tVFu3H99HVc/IJcjcmdHEXrPUErbx56lz48FGLRkmhqa5N4bdFuHD6iHg8/UZAaJndynW82F2CzMmr35g834r2VwVeHigVr9x3GG0v2hF0I3cj73Yx1+3EgRG8nch8md7LdgsxibD9YGfT5wsr6gEW1jzS2tD+eviI6yd2uBnKLMg1yQ3OrJedvbGnF47N24Lq311hy/mjJKKhqn4COmNzJJruKq9sf/2nGJuSXB5+P5tp/r7Z1UW29LeKqumbkR3ivwg7esk5lvXsTo5QS10xbhZs/2NBp+9ztRXhyTqZNUdmLyZ1scfnrK1W3q9WPS6rNr30/9X0m0vPKTT8vAFz06k/4hUovo1iWWVjliLLO5gOdPwHe9flmfLQmz55gbMbkTpaqrGvCZ+v3W3qNSMolH67Ow3Vvr9V3HY13Lg8FufEZy656cxXOfym+3tC0KK5qsG2OI45QJUs9+N/tmlZXcoLSmgY88u0O1PjVbbUsEmKlcFdnZxnnGj91CZITu2DXM1dE/dpM7mSpchdNk/vW0hzXvBGp4cwyztTQbM/i8yzLkOuxnzdRICZ3cpRN+yuQUVBl+nmPNLagpVVfC4rvGeRmhpO7ECJBCLFFCPGDGQFRfPtoTR6umbbK9POOemJBxLNKRmsm3eMfnovHZgUfqMRPKKSHGS33ewFkmXAeikHRmWNcW9b7dov5E5lNW7rHtAXBpQQ+XXcgYDunaadIGEruQoghAK4C8J454VCs2WTR4t5O8fLC3bj+HX1dKs1m2uRp/GSgW2ubRGl1g91hqDLacn8dwEMAghYzhRCThRDpQoj0srIyg5cjChQqtz3yXYZp/c6DXaepRVstP5K6vx6RtvDVjrMyTj1Sp8zFywuy7Q4jqBd+zMK455cEndTNThEndyHE1QBKpZSbQu0npZwupUyTUqalpKREejmiiHy+/gCe/WGnoXOYVRYZ9cQC/M0Fq0ktzCzGCY/+iKyi6vA7R8G0ZTl2hxDUkqxSAM6cusFIy/1cAL8SQuQB+ALAhUKIT02JikiHcNWEcM8vyCzG5E/SsTU/+ORlRs7v6zsDdX+zFsJ+56e9SJ0yF61t6ufzJqxtEf48yBkiHsQkpXwYwMMAIISYCOBBKeX/mhQXxYFVew7hjk9DfvCLij/N0BaDWclVL7NHyL6yaDcAoLm1DQldEkw9NzkH+7mTbV5ZlI0an6l7fWUX15h2Hd9aeUVtEzbk6pswLFxybWhuxUmP/Yh5GUUBz+UeqnVMeaOd8vN44Ktt+HSdtfP+xAsndlM1JblLKZdLKa8241xEgGdBDq309Ba58d11+K2O3i1FVfX4dkvoWFraJJpa2vCSyo2/C15ejiveUJ8B0yyR5pW5GUV4bNYO088bVxzcTZVzy5BtIq1xG7FL5yeC37+7HvsO1VoUjT7m9XgMfSLvDWQntkatcKSxBV27CCQnxlaJimUZso3Tk4eUQKnPOqp2xavWWyerqLp9QRCzG49WDZpqaW2zbfrbUEY9sQDnx+D8+0zu5EimDcxB9MoLZuXEYL1YfF3xxkr84sVlqG9qDVigAvAsqv3+qlzUN5m7NN/z87Jwy4cbwu+o4vHZmTj96UVobLFmuUAjSiNYDH1JVgn2lXk/1TmvpcLkTo700Zo87DhYpSkRWPlnZUdJdfZW7d0ll2eXqm7/IaMIz/ywE68sDLwPoPV9U618M33FPizLDj0YcV5GEf4xO7CWP0f5vppbnZcII3Hbx+l2hxASa+7kSM2tElf/axVuP+94S68Tdq1Tn+weNiVF+E5wxRsrcdLRvdq/rjewEHZlXRO+316ExC6eYPQsGN2RzD3HRvrh6c7PPOvdPj1pVGQn0KG0pgF9khOR1JXtVH/8iZCjFVYFXzjby0gFJ9zHcT35OtJWflZRNWZvLQy7n5bv84GvtuHxWTuQWRi8+6X/afy7eppZc9+WX4kjSndXK9rr455bgj+8vz7sfk6cHsBqTO7kaPnlWpJ76LRhZv3eDnpy7eFazw3LZmVuGCPfutGfWl1TCya9tTpgoJrZpa6NeRV45LvgXToB4OJXfzL5qp058SXG5E6OlnHQ/IU79PCdsrigInQJJzrTG/tfs9NXQbZ3Fu7NzqzvwltbV+vymlNag+vfWYu6JvVBbHrN3BA4VbKvijrnzf1iNSZ3Io3CrYXphPEspjYgTW6O+p7uublZWJ9bjnX7DgMADlbWo6CiDiXVDViz95Cp141XvKFKrufAT8T2UTKo91OE2s8m3M/L9A8gfhdUO/+5LywFAAzslYRDRxqR98JVJgcRf9hyJ9eraQj90f6n3cG77oVLZFU2TOX66Hc7VHvxeHuzdF4cQv0b0JOfW9o8n0j8u9e3Sc+UyWrdUYuq6sOuQNU+0jXEPv4fDsyaez/anNjAYMudYl6o5L96T6gSgL4/WTNbvOv2HcbQAT2UE3d+bnuIBcS1ROyfUF9d6Jkl0jt4ytt7Zm5GETbkliO/og5/v/zkTsdMmOppaT9y5ck47wT1dRq8YavV+O24PxFvmNwprnmnvzWDgEBpjTOWXPPmTi3D/d9blau63fumWFEb/BzPz9sFYFeQGDoncLVBUU7sZRIrWJahmJBRUKV5uTut/vzpZt3HPBamS55WWnOeb/4UoiNZelvfi7PUR7BqisGkzKvWr94b9u2fpKNIw1gGt2lrk+3dUe3CljvFhGumrcIfxg+3NYbyuiZkl6jPOrmvrKM+XdvYgpzSIxgztJ+u82vJtd7WsZGqh1kVk46yTOj9lhh4A9Ijv7yuo9RlMv/v8f6vtmKWhoFpVmLLnWLG/MxiW69fFmK064WvdAyiOfWJBZj01mocrNTYYlUSR4OOTyZG8rNZ1XD/N4lgSb7BwHQLelz++oqoXAeA7YkdcHlyn/l/4+0OgRwkVHK1Q3FV6Pp7yHi9Cb25Fd9v9yQKtTVN9SRiKSWmzsvScYT3ON2HdD7erzAjROfE/+xc/TFForapFS2tbZizrbC95GR36cRKrk3un91+tt0hEIU0fuqSkM//uCNwWT4vb0I8+fH5mLkhv317el65pq6FCzJLAp6rb27FOyv2hYxJTcbBKqTn6VuaEOio+0f65mBF4n1/VS7+MnNL+0Llz/yw0/RrqLFjmmPXJnch7FuwmOJX6pS5SJ0y15RzNTa3oUbHrI0AcN3boZcI9CbS4urIe+20D4BSzrWzqDrsddXP4/m/USkn+bXfwx7/1y+36r5mOCXVnk9L5UoPoPS8ipD7a32D2VlUFXJMxMdr8rQFaCLXJncit/toTR5GP7lQ9bm5GcHvH/imRd/uhmb3HA92czha5m4P/snGLLUh5rZZs/cQTnz0R2zU8Knlvi+34cbp64I+b3ZPLi1c21ume2JCwIg6olixJif44KqQZZkQz/lP7Rur0vPKkZY6IOx+VXXN2H84+GRwC5XS1vp9hzHW53zXv7MWR/dJDth/Z1HwaZbt4Lrk/tSvTkXPpK44Y1h/SCkxcWQKlodZGYYolny5MfQMiP4KK+tRWdeM4wf2tCgidb43TT9andup9RpJd8sZ6/bjlGN746zhoRP3dW+vxeopF2Jwv+4h9ysPM8DrI5VSSml1A9bn6r//YMdgLdcl9z9OGN7+UVQIgY9uGdf+nFm1UCK7hcoFvgOT/HOk2sCjc5RJuab97gxN1zbSz70qyNS6T37fceMy0vM/PsszQEzLpGJ1jZFNJVzT0IzRTy7Ei/9zWvs23x/puOdD3yR3EtfV3LXOSTF6cF+LIyGyTluETb26EAti3/35lrDHf5WeH7R8My8jfA38uXk+STxEGSjcX/F/0/PD7BGZL3w+9ajFUFjpuRH93ir9vYqcxnXJXaveya77UELUSSRLw0X6puC1YEfwG7netVFDqW82Vnrx8u+iqPdTebBre9/8Dh0JP+eOr9xDta6rDMRscj9zWH+7QyAyRMtC2Z+s29/+WAiBggp752n5flvHyEx7b98K1UFfXm//tBdNKt0c1bpXv7JoN1buMXZfz46+HzGZ3P922Ujcd8lJuGlCx1wjH9481saIiPSREpi2NCfsfitCzFUfqWjMxmv1NVra2jDprdUh9/liQ2Dp590VnhkyI+1ZtONgFb7bUhCw3Y5BTDFZu7jrghMAAKf61N0vOHmQXeEQReSLjdbUnUORsLvFbY4n52SG3efrTYE/3282ByZmPa7+1yrV7W8t24u/XXay6nNWcUXLfcyQyG6OhusKRRRLDpk0t05tiJuyeoTq/GB1n/stB4KXZLyqQyziEgtribii5Z6UmBDRceeeMNDkSIic6/ZP0g2fY8uBClQE6c6ol1X5sbVNIqFL6LObnZy1vFk4jSta7l7duhoLd8MjF5kUCVFsMiuxA0BWsfqIzaLKhrBztoRqVU/XMPmZ2Z8MvBONuYkrkvvYVE/Pl/n3/sLQeQb1ScbPUqI7So8oXl31pnr9ub65FUt2Rb5Ax4bcw2EX0tbS08jflgMdk4jtKrZ3Xh0zRJzchRBDhRDLhBBZQohMIcS9Zgbm6/5LRmLJA7/EiJReYfc9RmXOB18zJ3MOeCI3W5ZdhrRnF7fP7GiWa/+9xtTz2c1Izb0FwANSys1CiN4ANgkhFkkpTZ8gOaGLwM80JPbMpy5TrcX5lnNSeiWZGhsR6WPWPCu/fHGZOSeKUREndyllEYAi5XGNECILwGAA0Zn9XkXPpMBv56XrTsOZwzsGNGmdvoCInK0mwvlj4oUpvWWEEKkAzgCwXuW5yQAmA8CwYcPMuJwuv0kbGvVrElFw98wMP40BGWf4hqoQoheAbwD8VUoZcHtcSjldSpkmpUxLSUkxejkicrm9ZbV2hxAXDCV3IUQiPIn9Mynlt+aEZK+jenazOwQiIsOM9JYRAN4HkCWlfNW8kKx3/knBP0E8fvUpUYyEiMgaRlru5wL4A4ALhRBblX9XmhSXpd7941lBn0tOdEXXfyKikIz0llkFl84xlNQ1AVN/PRoDenbDn2ZssjscIiLTuWJuGSvcOC76PXeIiKKFNYgAHR9GkgzOZUNEZBdmryBOPqY3uiXwx0NE7sTs5aeXMsp1BCcYIyIXi/vkPmZov05fjx7cF5/cOg4vXTdG13lOHBR+7hsiomiJ2xuqXrPvOhcVtU1YlFWCy0cdgz7Jie394Pt0T9Q8f8X4EUdhT+kRK0MlItIs7lvuANC/Zzf8Nm0o+iQndto+8//G44lrtA1qCjUwiogo2pjcQxh2VA/ccu7xmva95JSj8d2d51gcERGRNkzuJjpjWP/wOxERRQGTu04pvZPw/LWj2ZuGiByNyV2ni38+CL87exiWPjDR7lCIiIJicjcgbXhgGebmc1KjHwgRkR8mdx2GDuiO284b0f71Ub0C534fPbhvNEMiIlIV9/3c9Vj50IWdvj5KWWz7urOG2BEOEVFQTO4a3HxOKvokB/6ouihzjI0ZwtY6EYVW09CM3n5jaazEsowGT/7qVNx/6ciA7VIG7iuUhH/tGYM7bb9hLBfqJopnRVUNUb0ek7sZRMc0wVeOPhbXpw3FI1f+vNMu55wwMNpREZGDqDUGrcTkbrLkxAT887rTkNI7CQ9fcXL79lOP62NjVEQUb5jcLXTNmOPaH48Y2BNPTzoVvVVq9/7GpQ6wMiwiskFblJvuTO4W6qnMDT+kf3cIIfDHCanIePIyXDX6WNx/yUnt+00c2XnSsWP6Jkc1TiKyXrSTO3vLGJCgdJcJtkp43+6JWPX3C3B0n87J+q3fnwnAk9T7dk9Et65dMGHqUitDJSKbRbvmzuRuwAOXjERrmwzZz31I/x5BnzttiGehkCMa54wnIvfiDVUX6dsjEc9dOxrJiQmGztMrqSuuT+voKjlsgOcN4ZZzUwO6VBKRO7HmHqf+ed1p7Y/vvfhEfHjzWDxxzano2z16gx6IyDpRbrgzuTtRYkIXXHDyIADADeM6D366/Txti4cQkbOw5R7n/GeaPPmYPsh74ar2r0cN7ovlD04MOO7DW8ZaHRoRGcAbqnFs8+OXoEc39fr99icvxdYDlUHXah15dG/V7TueugyjnlhgWoxEFBnJlnv8GtCzW9Cbs32SEzsl9vdvSsM3d0zAi0qtfkDPbrj7ghMCjuuV1PH+Pah3kskRE5FW0a65s+XuUhf9/GgAwFnDB+C3Sk+bBy8bievHDsXy7FI8Pjuzfd+VD12A5MQEJCV2QV1jK8ZPXWJLzETxrK2NLXcyYOiAHviNkuxvmjC8fVtK7yT0SU7sNPo1bXh/zPvLL2yJkyjesLcMGZacmIC8F67CU5NGqT7/+7OHYdzxA/D1HefglOP64ONbxwU91/ePePIAAAkHSURBVI3jhmLn05dZFSpR3OD0A2S5564d3enrX56UgrwXrkJDcyvGPrsYNcqI2aSuXTD1156a/oK/no/EBIGBvZNw2pML24/t3yMRFXXN7V8vf3AiDtc24n/+szYK3wmRi7hphKoQ4nIhRLYQIkcIMcWsoMgeyYkJePemNIwZ0hfP/L9RmHdvR8lm5DG9MSKlF/okJ7ZPZfzNHROw5R+XdjpH6sCeOGv4ALzymzHt28alDsDi+8/Hxcp9AsCzHi1RPGmNcstdRNo9RwiRAGA3gEsAFADYCOBGKeXOYMekpaXJ9PT0iK5HzrWnpAYb8yowqHcSLj6lI4Evyy5F6lE9cfzAnu3bZm89iKP7JGP8iKNQWt2ApK4J6NsjETmlR5DSKwljnl6odglVowf3RcbBKlO/FyKr/Pv3Z+LK0cdGdKwQYpOUMk3PMUbKMuMA5Egp9ykX/wLAJABBkzvFphOP7o0TVfrZXzByUMC2Sad3zJUzyGe2zBMG9QKA9gFbCzOLsbesFmmp/XHSoN4or2vC8QN7YtP+cuwsqsHvxg1DQheB5tY23PrRRowe3BcPXX4yWtskvtlUgCEDuuO4vt2RlNgFd362GVsOVLZf680bz8DOwmrMWJuH2qZWbH78EnyyNg+vL95j0k+EKFBDc2tUr2ek5X4dgMullLcrX/8BwNlSyrv99psMYDIADBs27Kz9+/cbi5goSqSUKKxqwOB+nhJSbWMLJDxTPCd0EapjEqSUyCk9ghMG9ULe4ToMH9ADBRX1EAIY2CsJyYldcKSxBQsySzC4X3ecclwfJCYIlNU0oqVNYv6OYvwmbQha2yQ27a/AGcP6Iz2vHF2EQGV9M6av2IuxqQPQs1tXLN9diqSuCXjw0pH4cuMBdE3ogj//cgRenJ+N4/p1x3dbDgIIvC/Sv0cihg7ogYbmVhysqEdtU3STTrzyHWmuVyQtdyPJ/TcALvNL7uOklPcEO4ZlGSIi/SJJ7kZuqBYA8J3VagiAQgPnIyIikxhJ7hsBnCiEOF4I0Q3ADQDmmBMWEREZEfENVSllixDibgALACQA+EBKmRnmMCIiigJDg5iklPMAzDMpFiIiMgmnHyAiikFM7kREMYjJnYgoBjG5ExHFoIgHMUV0MSHKAEQ6RHUggEMmhmMmxhYZp8bm1LgAxhYpt8c2XEqpvsZmEFFN7kYIIdL1jtCKFsYWGafG5tS4AMYWqXiMjWUZIqIYxORORBSD3JTcp9sdQAiMLTJOjc2pcQGMLVJxF5trau5ERKSdm1ruRESkEZM7EVEsklI6/h+AywFkA8gBMMWia3wAoBTADp9tAwAsArBH+b+/sl0AeFOJZzuAM32OuUnZfw+Am3y2nwUgQznmTSglMY2xDQWwDEAWgEwA9zolPgDJADYA2KbE9pSy/XgA65XrfAmgm7I9Sfk6R3k+1edcDyvbs+FZCMbw7x+eGUu3APjBYXHlKT/vrQDSnfL7VI7tB+BrALuU19wEJ8QGYKTy8/L+qwbwVyfEphx7Hzx/AzsAzITnb8O215vpSdLsf/D8ce4FMAJAN3iSyCkWXOd8AGeic3J/0ftDBDAFwD+Vx1cC+FF58YwHsN7nj3Of8n9/5bH3hbZB+SMRyrFX6IjtWO8LE0BveBYmP8UJ8Sn791IeJyov1PEAvgJwg7L9bQB3KI/vBPC28vgGAF8qj09RfrdJyh/EXuV3b+j3D+B+AJ+jI7k7Ja48AAP9ttn++1SO/RjA7crjbvAke0fE5pcXigEMd0JsAAYDyAXQ3ed1drOdrzfbk7eGH9oEAAt8vn4YwMMWXSsVnZN7NoBjlcfHAshWHr8D4Eb//QDcCOAdn+3vKNuOBbDLZ3un/SKIczaAS5wWH4AeADYDOBueEXdd/X+H8Mz/P0F53FXZT/j/Xr37Gfn9w7M62BIAFwL4QbmO7XEp++chMLnb/vsE0AeeJCWcFptfPJcCWO2U2OBJ7vnwvGF0VV5vl9n5enNDzd37Q/MqULZFw9FSyiIAUP4fFCamUNsLVLbrJoRIBXAGPC1kR8QnhEgQQmyFp6y1CJ4WRqWUskXlfO0xKM9XATgqgpi1eB3AQwDalK+PckhcACABLBRCbFIWkQec8fscAaAMwIdCiC1CiPeEED0dEpuvG+ApfcAJsUkpDwJ4GcABAEXwvH42wcbXmxuSu1DZJqMeRWfBYtK7Xd9FhegF4BsAf5VSVjslPillq5TydHhayuMA/DzE+aISmxDiagClUspNvpvtjsvHuVLKMwFcAeAuIcT5IfaNZmxd4SlP/kdKeQaAWnhKHU6IzXNBz7KevwLw33C7Ris2IUR/AJPgKaUcB6AnPL/bYOezPDY3JHc7F+IuEUIcCwDK/6VhYgq1fYjKds2EEInwJPbPpJTfOi0+AJBSVgJYDk99s58QwrvSl+/52mNQnu8LoDyCmMM5F8CvhBB5AL6ApzTzugPiAgBIKQuV/0sBfAfPm6ITfp8FAAqklOuVr7+GJ9k7ITavKwBsllKWKF87IbaLAeRKKcuklM0AvgVwDux8vemtdUX7HzwtiX3wvCN6byScatG1UtG55v4SOt+oeVF5fBU636jZoGwfAE+9sr/yLxfAAOW5jcq+3hs1V+qISwD4BMDrftttjw9ACoB+yuPuAFYCuBqeVpXvjaQ7lcd3ofONpK+Ux6ei842kffDcRDL8+wcwER03VG2PC55WXW+fx2vg6Qlh++9TOXYlgJHK4yeVuBwRm3L8FwBucdjfwdnw9JTpoRz7MYB77Hy92Z68Nf7groSnh8heAI9adI2Z8NTKmuF5l7wNnhrYEni6MS3xeQEIAG8p8WQASPM5z63wdFXK8XsBpsHTRWovgGnQ18XqPHg+gm1HRzewK50QH4DT4OlquF05/h/K9hHw9DzIUV7gScr2ZOXrHOX5ET7nelS5fjZ8eikY/f2jc3K3PS4lhm3o6D76qLLd9t+ncuzpANKV3+kseBKgU2LrAeAwgL4+25wS21PwdB/dAWAGPAnattcbpx8gIopBbqi5ExGRTkzuREQxiMmdiCgGMbkTEcUgJnciohjE5E5EFIOY3ImIYtD/B7+6GUzw5bxSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def translate(encoder_model, decoder_model, sentence, remove_spaces= False, toTensor=False):\n",
    "    with torch.no_grad():\n",
    "        sentence_translated = []\n",
    "        encoder_hidden = encoder_model.init_hidden()\n",
    "        \n",
    "        all_encoder_outputs = torch.zeros(MAX_LEN_SENTENCE, encoder.hidden_size, device=device)\n",
    "        for iw in range(sentence.size(0)):\n",
    "            encoder_output, encoder_hidden = encoder_model(sentence[iw], encoder_hidden) \n",
    "            #stores all encoder outputs to be used on the decoder\n",
    "            all_encoder_outputs[iw] = encoder_output[0, 0]\n",
    "        \n",
    "        decoder_hidden = encoder_hidden #the same hidden state is used for decoding\n",
    "        decoder_input = torch.tensor([[SOS]], dtype=torch.long, device=device)\n",
    "        decoder_attn = torch.zeros(MAX_LEN_SENTENCE, MAX_LEN_SENTENCE)\n",
    "        \n",
    "        for iw in range(MAX_LEN_SENTENCE):\n",
    "            decoder_output, decoder_hidden, attn_weights = decoder_model(decoder_input, decoder_hidden, all_encoder_outputs)\n",
    "            decoder_attn[iw] = attn_weights.data\n",
    "            _, guess = decoder_output.topk(1)\n",
    "            decoder_input = guess.squeeze().detach()\n",
    "            \n",
    "            if decoder_input.item() == EOS:\n",
    "                break\n",
    "            else:\n",
    "                sentence_translated.append(target_language.index2word[decoder_input.item()])\n",
    "                \n",
    "        sentence_translated = ' '.join(sentence_translated) \n",
    "        if remove_spaces:\n",
    "            sentence_translated = sentence_translated.replace(\" \", \"\")\n",
    "    return sentence_translated, decoder_attn[:iw + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def tensor2sentence(sentence, dictionary, remove_spaces=False):\n",
    "    s = []\n",
    "    for word in sentence:\n",
    "        s.append(dictionary.index2word[word.item()])\n",
    "        \n",
    "    del s[-1] # remove EOS\n",
    "    s = ' '.join(s)\n",
    "    if remove_spaces:\n",
    "        s = s.replace(\" \", \"\")\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translation evaluation\n",
      "input  : the telephone rang a few minutes later\n",
      "predict: 数分後に電話が鳴った\n",
      "output : 数分後に電話が鳴った\n",
      "-----------------\n",
      "input  : they are gathering nuts\n",
      "predict: 彼らは木の実を拾い集めている\n",
      "output : 彼らは木の実を拾い集めている\n",
      "-----------------\n",
      "input  : they asked for an increase of salary\n",
      "predict: 彼らは賃上げの求めを求め\n",
      "output : 彼らは給料のアップを求めた\n",
      "-----------------\n"
     ]
    }
   ],
   "source": [
    "print('Translation evaluation')\n",
    "for idx, (inp_sentence, targ_sentence) in enumerate(train_loader):\n",
    "    if idx >= 3:\n",
    "        break\n",
    "    \n",
    "    inp_sentence = inp_sentence[0].to(torch.long).to(device)\n",
    "    targ_sentence = targ_sentence[0].to(torch.long).to(device)\n",
    "    \n",
    "    print('input  :', tensor2sentence(inp_sentence, input_language))\n",
    "    sentence_translated, attention = translate(encoder, decoder, inp_sentence, remove_spaces=True)\n",
    "    print('predict:', sentence_translated)\n",
    "    print('output :', tensor2sentence(targ_sentence, target_language, remove_spaces=True))\n",
    "    print('-----------------')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}