ML Language Translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, LSTM, Dense\n",
    "from tensorflow import keras\n",
    "import numpy as np\n",
    "\n",
    "batch_size = 64\n",
    "epochs = 100\n",
    "latent_dim = 256    #Latent dimensionality of encoding space\n",
    "num_samples = 10000\n",
    "\n",
    "data_path = r'fra-eng\\fra.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Vectorize the data\n",
    "input_texts = []\n",
    "target_texts = []\n",
    "input_characters = set()\n",
    "target_characters = set()\n",
    "with open(data_path, 'r', encoding = 'utf-8') as f:\n",
    "    lines = f.read().split('\\n')\n",
    "for line in lines[: min(num_samples,len(lines)-1)]:\n",
    "    input_text, target_text, _ = line.split('\\t')\n",
    "    #we use tab as 'start sequence' character\n",
    "    #for the targets and '\\n' as 'end sequence' character\n",
    "    target_text='\\t'+target_text+'\\n'\n",
    "    input_texts.append(input_text)\n",
    "    target_texts.append(target_text)\n",
    "    for char in input_text:\n",
    "        if char not in input_characters:\n",
    "            input_characters.add(char)\n",
    "    for char in target_text:\n",
    "        if char not in target_characters:\n",
    "            target_characters.add(char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_characters = sorted(list(input_characters))\n",
    "target_characters = sorted(list(target_characters))\n",
    "num_encoder_tokens = len(input_characters)\n",
    "num_decoder_tokens = len(target_characters)\n",
    "max_encoder_seq_length = max([len(txt) for txt in input_texts])\n",
    "max_decoder_seq_length = max([len(txt) for txt in target_texts])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of samples:  10000\n",
      "Number of unique input tokens :  71\n",
      "Number of unique output tokens :  93\n",
      "Max sequence length for inputs :  15\n",
      "Max sequence length for outputs :  59\n"
     ]
    }
   ],
   "source": [
    "print('Number of samples: ',len(input_texts))\n",
    "print('Number of unique input tokens : ', num_encoder_tokens)\n",
    "print('Number of unique output tokens : ', num_decoder_tokens)\n",
    "print('Max sequence length for inputs : ', max_encoder_seq_length)\n",
    "print('Max sequence length for outputs : ', max_decoder_seq_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_token_index = dict([(char,i) for i, char in enumerate(input_characters)])\n",
    "target_token_index = dict([(char,i) for i, char in enumerate(target_characters)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ONE HOT REPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_input_data = np.zeros((len(input_texts),max_encoder_seq_length,num_encoder_tokens),dtype = 'float64')\n",
    "decoder_input_data = np.zeros((len(input_texts),max_decoder_seq_length,num_decoder_tokens), dtype = 'float64')\n",
    "decoder_target_data = np.zeros((len(input_texts),max_decoder_seq_length,num_decoder_tokens), dtype = 'float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (input_text,target_text) in enumerate(zip(input_texts,target_texts)):\n",
    "    for t, char in enumerate(input_text):\n",
    "        encoder_input_data[i, t, input_token_index[char]]=1.\n",
    "    encoder_input_data[i, t+1:, input_token_index[' ']]=1.\n",
    "    for t, char in enumerate(target_text):\n",
    "        #decoder target data is ahead of decoder input data by 1 timestamp\n",
    "        decoder_input_data[i, t, target_token_index[char]]=1.\n",
    "        if t > 0:\n",
    "            #decoder target data will be ahead of 1 timestamp\n",
    "            #and will not include the start character\n",
    "            decoder_target_data[i, t-1, target_token_index[char]]=1.\n",
    "        decoder_input_data[i, t+1:, target_token_index[' ']]=1.\n",
    "        decoder_target_data[i, t:, target_token_index[' ']]=1.\n",
    "         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15, 71)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_input_data[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define an input sequence and process it.\n",
    "encoder_inputs = Input(shape=(None, num_encoder_tokens))\n",
    "encoder = LSTM(latent_dim, return_state=True)\n",
    "encoder_outputs, state_h, state_c = encoder(encoder_inputs)\n",
    "# We discard `encoder_outputs` and only keep the states.\n",
    "encoder_states = [state_h, state_c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up the decoder, using `encoder_states` as initial state.\n",
    "decoder_inputs = Input(shape=(None, num_decoder_tokens))\n",
    "# We set up our decoder to return full output sequences,\n",
    "# and to return internal states as well. We don't use the \n",
    "# return states in the training model, but we will use them in inference.\n",
    "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
    "decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
    "                                     initial_state=encoder_states)\n",
    "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "# Define the model that will turn\n",
    "# encoder_input_data & decoder_input_data into decoder_target_data\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "125/125 [==============================] - 56s 375ms/step - loss: 2.1127 - val_loss: 2.3548\n",
      "Epoch 2/100\n",
      "125/125 [==============================] - 46s 367ms/step - loss: 2.1549 - val_loss: 2.5187\n",
      "Epoch 3/100\n",
      "125/125 [==============================] - 45s 359ms/step - loss: 2.2881 - val_loss: 2.6721\n",
      "Epoch 4/100\n",
      "125/125 [==============================] - 45s 363ms/step - loss: 2.4249 - val_loss: 2.8137\n",
      "Epoch 5/100\n",
      "125/125 [==============================] - 45s 359ms/step - loss: 2.5612 - val_loss: 2.9580\n",
      "Epoch 6/100\n",
      "125/125 [==============================] - 47s 372ms/step - loss: 2.6996 - val_loss: 3.1302\n",
      "Epoch 7/100\n",
      "125/125 [==============================] - 46s 372ms/step - loss: 2.8321 - val_loss: 3.3028\n",
      "Epoch 8/100\n",
      "125/125 [==============================] - 46s 370ms/step - loss: 2.9712 - val_loss: 3.4470\n",
      "Epoch 9/100\n",
      "125/125 [==============================] - 44s 353ms/step - loss: 3.1108 - val_loss: 3.5910\n",
      "Epoch 10/100\n",
      "125/125 [==============================] - 44s 353ms/step - loss: 3.2493 - val_loss: 3.7696\n",
      "Epoch 11/100\n",
      "125/125 [==============================] - 44s 355ms/step - loss: 3.3991 - val_loss: 3.9252\n",
      "Epoch 12/100\n",
      "125/125 [==============================] - 44s 352ms/step - loss: 3.5423 - val_loss: 4.1352\n",
      "Epoch 13/100\n",
      "125/125 [==============================] - 44s 353ms/step - loss: 3.7259 - val_loss: 4.3073\n",
      "Epoch 14/100\n",
      "125/125 [==============================] - 44s 350ms/step - loss: 3.8396 - val_loss: 4.4512\n",
      "Epoch 15/100\n",
      "125/125 [==============================] - 45s 361ms/step - loss: 3.9778 - val_loss: 4.6188\n",
      "Epoch 16/100\n",
      "125/125 [==============================] - 43s 348ms/step - loss: 4.1252 - val_loss: 4.7458\n",
      "Epoch 17/100\n",
      "125/125 [==============================] - 43s 345ms/step - loss: 4.2601 - val_loss: 4.9444\n",
      "Epoch 18/100\n",
      "125/125 [==============================] - 43s 345ms/step - loss: 4.4071 - val_loss: 5.0974\n",
      "Epoch 19/100\n",
      "125/125 [==============================] - 44s 349ms/step - loss: 4.5523 - val_loss: 5.2987\n",
      "Epoch 20/100\n",
      "125/125 [==============================] - 43s 347ms/step - loss: 4.7011 - val_loss: 5.4231\n",
      "Epoch 21/100\n",
      "125/125 [==============================] - 43s 347ms/step - loss: 4.8410 - val_loss: 5.5984\n",
      "Epoch 22/100\n",
      "125/125 [==============================] - 45s 357ms/step - loss: 4.9929 - val_loss: 5.7609\n",
      "Epoch 23/100\n",
      "125/125 [==============================] - 46s 371ms/step - loss: 5.1443 - val_loss: 5.9325\n",
      "Epoch 24/100\n",
      "125/125 [==============================] - 44s 349ms/step - loss: 5.2927 - val_loss: 6.1173\n",
      "Epoch 25/100\n",
      "125/125 [==============================] - 42s 337ms/step - loss: 5.4402 - val_loss: 6.2857\n",
      "Epoch 26/100\n",
      "125/125 [==============================] - 42s 338ms/step - loss: 5.5882 - val_loss: 6.4486\n",
      "Epoch 27/100\n",
      "125/125 [==============================] - 44s 351ms/step - loss: 5.7391 - val_loss: 6.6371\n",
      "Epoch 28/100\n",
      "125/125 [==============================] - 42s 335ms/step - loss: 5.8897 - val_loss: 6.7920\n",
      "Epoch 29/100\n",
      "125/125 [==============================] - 42s 337ms/step - loss: 6.0358 - val_loss: 6.9603\n",
      "Epoch 30/100\n",
      "125/125 [==============================] - 42s 336ms/step - loss: 6.1805 - val_loss: 7.1180\n",
      "Epoch 31/100\n",
      "125/125 [==============================] - 42s 337ms/step - loss: 6.3254 - val_loss: 7.3225\n",
      "Epoch 32/100\n",
      "125/125 [==============================] - 42s 339ms/step - loss: 6.4825 - val_loss: 7.4721\n",
      "Epoch 33/100\n",
      "125/125 [==============================] - 42s 340ms/step - loss: 6.6254 - val_loss: 7.6536\n",
      "Epoch 34/100\n",
      "125/125 [==============================] - 42s 336ms/step - loss: 6.7699 - val_loss: 7.8112\n",
      "Epoch 35/100\n",
      "125/125 [==============================] - 43s 344ms/step - loss: 6.9178 - val_loss: 7.9865\n",
      "Epoch 36/100\n",
      "125/125 [==============================] - 42s 338ms/step - loss: 7.0590 - val_loss: 8.1750\n",
      "Epoch 37/100\n",
      "125/125 [==============================] - 42s 336ms/step - loss: 7.2103 - val_loss: 8.3114\n",
      "Epoch 38/100\n",
      "125/125 [==============================] - 41s 332ms/step - loss: 7.3536 - val_loss: 8.4625\n",
      "Epoch 39/100\n",
      "125/125 [==============================] - 43s 343ms/step - loss: 7.5066 - val_loss: 8.6469\n",
      "Epoch 40/100\n",
      "125/125 [==============================] - 41s 330ms/step - loss: 7.6564 - val_loss: 8.8207\n",
      "Epoch 41/100\n",
      "125/125 [==============================] - 41s 330ms/step - loss: 7.8058 - val_loss: 9.0131\n",
      "Epoch 42/100\n",
      "125/125 [==============================] - 41s 331ms/step - loss: 7.9588 - val_loss: 9.1711\n",
      "Epoch 43/100\n",
      "125/125 [==============================] - 42s 335ms/step - loss: 8.1040 - val_loss: 9.3338\n",
      "Epoch 44/100\n",
      "125/125 [==============================] - 42s 338ms/step - loss: 8.2426 - val_loss: 9.4988\n",
      "Epoch 45/100\n",
      "125/125 [==============================] - 42s 334ms/step - loss: 8.3895 - val_loss: 9.6634\n",
      "Epoch 46/100\n",
      "125/125 [==============================] - 42s 335ms/step - loss: 8.5432 - val_loss: 9.8443\n",
      "Epoch 47/100\n",
      "125/125 [==============================] - 42s 333ms/step - loss: 8.6921 - val_loss: 10.0019\n",
      "Epoch 48/100\n",
      "125/125 [==============================] - 42s 333ms/step - loss: 8.8414 - val_loss: 10.1654\n",
      "Epoch 49/100\n",
      "125/125 [==============================] - 50s 403ms/step - loss: 8.9862 - val_loss: 10.3409\n",
      "Epoch 50/100\n",
      "125/125 [==============================] - 48s 388ms/step - loss: 9.1403 - val_loss: 10.5578\n",
      "Epoch 51/100\n",
      "125/125 [==============================] - 47s 375ms/step - loss: 9.2896 - val_loss: 10.6943\n",
      "Epoch 52/100\n",
      "125/125 [==============================] - 42s 338ms/step - loss: 9.4424 - val_loss: 10.8625\n",
      "Epoch 53/100\n",
      "125/125 [==============================] - 41s 331ms/step - loss: 9.5905 - val_loss: 11.0472\n",
      "Epoch 54/100\n",
      "125/125 [==============================] - 42s 339ms/step - loss: 9.7412 - val_loss: 11.2111\n",
      "Epoch 55/100\n",
      "125/125 [==============================] - 42s 340ms/step - loss: 9.8982 - val_loss: 11.3824\n",
      "Epoch 56/100\n",
      "125/125 [==============================] - 42s 337ms/step - loss: 10.0444 - val_loss: 11.5615\n",
      "Epoch 57/100\n",
      "125/125 [==============================] - 41s 332ms/step - loss: 10.2029 - val_loss: 11.7414\n",
      "Epoch 58/100\n",
      "125/125 [==============================] - 42s 336ms/step - loss: 10.3652 - val_loss: 11.9301\n",
      "Epoch 59/100\n",
      "125/125 [==============================] - 42s 334ms/step - loss: 10.5273 - val_loss: 12.1125\n",
      "Epoch 60/100\n",
      "125/125 [==============================] - 42s 338ms/step - loss: 10.6869 - val_loss: 12.3139\n",
      "Epoch 61/100\n",
      "125/125 [==============================] - 38s 307ms/step - loss: 10.8479 - val_loss: 12.4755\n",
      "Epoch 62/100\n",
      "125/125 [==============================] - 39s 311ms/step - loss: 11.0032 - val_loss: 12.6532\n",
      "Epoch 63/100\n",
      "125/125 [==============================] - 40s 321ms/step - loss: 11.1614 - val_loss: 12.8448\n",
      "Epoch 64/100\n",
      "125/125 [==============================] - 41s 331ms/step - loss: 11.3223 - val_loss: 13.0228\n",
      "Epoch 65/100\n",
      "125/125 [==============================] - 40s 323ms/step - loss: 11.4833 - val_loss: 13.2176\n",
      "Epoch 66/100\n",
      "125/125 [==============================] - 41s 328ms/step - loss: 11.6474 - val_loss: 13.4065\n",
      "Epoch 67/100\n",
      "125/125 [==============================] - 41s 326ms/step - loss: 11.8080 - val_loss: 13.5851\n",
      "Epoch 68/100\n",
      "125/125 [==============================] - 39s 309ms/step - loss: 11.9694 - val_loss: 13.7685\n",
      "Epoch 69/100\n",
      "125/125 [==============================] - 38s 308ms/step - loss: 12.1316 - val_loss: 13.9617\n",
      "Epoch 70/100\n",
      "125/125 [==============================] - 39s 310ms/step - loss: 12.3068 - val_loss: 14.1526\n",
      "Epoch 71/100\n",
      "125/125 [==============================] - 40s 317ms/step - loss: 12.4742 - val_loss: 14.3437\n",
      "Epoch 72/100\n",
      "125/125 [==============================] - 40s 322ms/step - loss: 12.6400 - val_loss: 14.5456\n",
      "Epoch 73/100\n",
      "125/125 [==============================] - 41s 326ms/step - loss: 12.8094 - val_loss: 14.7352\n",
      "Epoch 74/100\n",
      "125/125 [==============================] - 38s 308ms/step - loss: 12.9766 - val_loss: 14.9234\n",
      "Epoch 75/100\n",
      "125/125 [==============================] - 34s 275ms/step - loss: 13.1469 - val_loss: 15.1141\n",
      "Epoch 76/100\n",
      "125/125 [==============================] - 32s 258ms/step - loss: 13.3136 - val_loss: 15.3040\n",
      "Epoch 77/100\n",
      "125/125 [==============================] - 32s 256ms/step - loss: 13.4804 - val_loss: 15.5031\n",
      "Epoch 78/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "125/125 [==============================] - 34s 274ms/step - loss: 13.6451 - val_loss: 15.6747\n",
      "Epoch 79/100\n",
      "125/125 [==============================] - 37s 299ms/step - loss: 13.8106 - val_loss: 15.8775\n",
      "Epoch 80/100\n",
      "125/125 [==============================] - 36s 285ms/step - loss: 13.9771 - val_loss: 16.0487\n",
      "Epoch 81/100\n",
      "125/125 [==============================] - 35s 277ms/step - loss: 14.1390 - val_loss: 16.2516\n",
      "Epoch 82/100\n",
      "125/125 [==============================] - 34s 274ms/step - loss: 14.3079 - val_loss: 16.4492\n",
      "Epoch 83/100\n",
      "125/125 [==============================] - 34s 275ms/step - loss: 14.4737 - val_loss: 16.6229\n",
      "Epoch 84/100\n",
      "125/125 [==============================] - 36s 288ms/step - loss: 14.6369 - val_loss: 16.8316\n",
      "Epoch 85/100\n",
      "125/125 [==============================] - 35s 283ms/step - loss: 14.7991 - val_loss: 17.0134\n",
      "Epoch 86/100\n",
      "125/125 [==============================] - 34s 273ms/step - loss: 14.9677 - val_loss: 17.2024\n",
      "Epoch 87/100\n",
      "125/125 [==============================] - 35s 279ms/step - loss: 15.1347 - val_loss: 17.3918\n",
      "Epoch 88/100\n",
      "125/125 [==============================] - 34s 273ms/step - loss: 15.3048 - val_loss: 17.5883\n",
      "Epoch 89/100\n",
      "125/125 [==============================] - 35s 277ms/step - loss: 15.4722 - val_loss: 17.7791\n",
      "Epoch 90/100\n",
      "125/125 [==============================] - 35s 276ms/step - loss: 15.6418 - val_loss: 17.9548\n",
      "Epoch 91/100\n",
      "125/125 [==============================] - 47s 373ms/step - loss: 15.8025 - val_loss: 18.1523\n",
      "Epoch 92/100\n",
      "125/125 [==============================] - 44s 353ms/step - loss: 15.9729 - val_loss: 18.3509\n",
      "Epoch 93/100\n",
      "125/125 [==============================] - 42s 335ms/step - loss: 16.1363 - val_loss: 18.5379\n",
      "Epoch 94/100\n",
      "125/125 [==============================] - 43s 340ms/step - loss: 16.3029 - val_loss: 18.7237\n",
      "Epoch 95/100\n",
      "125/125 [==============================] - 43s 343ms/step - loss: 16.4715 - val_loss: 18.9267\n",
      "Epoch 96/100\n",
      "125/125 [==============================] - 44s 352ms/step - loss: 16.6377 - val_loss: 19.1118\n",
      "Epoch 97/100\n",
      "125/125 [==============================] - 44s 356ms/step - loss: 16.7993 - val_loss: 19.2882\n",
      "Epoch 98/100\n",
      "125/125 [==============================] - 45s 357ms/step - loss: 16.9618 - val_loss: 19.4852\n",
      "Epoch 99/100\n",
      "125/125 [==============================] - 44s 348ms/step - loss: 17.1335 - val_loss: 19.6748\n",
      "Epoch 100/100\n",
      "125/125 [==============================] - 43s 340ms/step - loss: 17.2967 - val_loss: 19.8703\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x27f459c75e0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Run training\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n",
    "model.fit([encoder_input_data, decoder_input_data], decoder_target_data,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_split=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### INFERENCE SETUP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "encode input and retrieve initial decoder state<br>\n",
    "run one step of decoder with this initial state and a \"start of sequence\" token as target. Output will be the next target token.<br>\n",
    "Repeat with the current target token and current states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define sampling models\n",
    "# Restore the model and construct the encoder and decoder.\n",
    "model = keras.models.load_model(\"s2s\")\n",
    "\n",
    "encoder_inputs = model.input[0]  # input_1\n",
    "encoder_outputs, state_h_enc, state_c_enc = model.layers[2].output  # lstm_1\n",
    "encoder_states = [state_h_enc, state_c_enc]\n",
    "encoder_model = keras.Model(encoder_inputs, encoder_states)\n",
    "\n",
    "decoder_inputs = model.input[1]  # input_2\n",
    "decoder_state_input_h = keras.Input(shape=(latent_dim,))\n",
    "decoder_state_input_c = keras.Input(shape=(latent_dim,))\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "decoder_lstm = model.layers[3]\n",
    "decoder_outputs, state_h_dec, state_c_dec = decoder_lstm(\n",
    "    decoder_inputs, initial_state=decoder_states_inputs\n",
    ")\n",
    "decoder_states = [state_h_dec, state_c_dec]\n",
    "decoder_dense = model.layers[4]\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "decoder_model = keras.Model(\n",
    "    [decoder_inputs] + decoder_states_inputs, [decoder_outputs] + decoder_states\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reverse-lookup token index to decode sequences back to\n",
    "# something readable.\n",
    "reverse_input_char_index = dict((i, char) for char, i in input_token_index.items())\n",
    "reverse_target_char_index = dict((i, char) for char, i in target_token_index.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    target_seq[0, 0, target_token_index[\"\\t\"]] = 1.0\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = \"\"\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
    "        decoded_sentence += sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if sampled_char == \"\\n\" or len(decoded_sentence) > max_decoder_seq_length:\n",
    "            stop_condition = True\n",
    "\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "        target_seq[0, 0, sampled_token_index] = 1.0\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "    return decoded_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for seq_index in range(20):\n",
    "#     # Take one sequence (part of the training set)\n",
    "#     # for trying out decoding.\n",
    "#     input_seq = encoder_input_data[seq_index : seq_index + 1]\n",
    "#     decoded_sentence = decode_sequence(input_seq)\n",
    "#     print(\"-\")\n",
    "#     print(\"Input sentence:\", input_texts[seq_index])\n",
    "#     print(\"Decoded sentence:\", decoded_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}