Language_model_2_words_in,1_word_out.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "Language_model- 2 words in,1 word out.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "i697RFzjO-Jr",
        "colab_type": "code",
        "outputId": "645ace88-718c-4ff3-f87d-116ff2fb33c0",
        "colab": {}
      },
      "source": [
        "from numpy import array\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.utils import to_categorical\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.layers import LSTM\n",
        "from keras.layers import Embedding\n",
        " "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bkH0Mr8O-J8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# source text\n",
        "data = \"\"\" Jack and Jill went up the hill\\n\n",
        "\t\tTo fetch a pail of water\\n\n",
        "\t\tJack fell down and broke his crown\\n\n",
        "\t\tAnd Jill came tumbling after\\n \"\"\"\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hQ7ZYaIxO-KE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# generate a sequence from a language model\n",
        "def generate_seq(model, tokenizer, max_length, seed_text, n_words):\n",
        "\tin_text = seed_text\n",
        "\t# generate a fixed number of words\n",
        "\tfor _ in range(n_words):\n",
        "\t\t# encode the text as integer\n",
        "\t\tencoded = tokenizer.texts_to_sequences([in_text])[0]\n",
        "\t\t# pre-pad sequences to a fixed length\n",
        "\t\tencoded = pad_sequences([encoded], maxlen=max_length, padding='pre')\n",
        "\t\t# predict probabilities for each word\n",
        "\t\tyhat = model.predict_classes(encoded, verbose=0)\n",
        "\t\t# map predicted word index to word\n",
        "\t\tout_word = ''\n",
        "\t\tfor word, index in tokenizer.word_index.items():\n",
        "\t\t\tif index == yhat:\n",
        "\t\t\t\tout_word = word\n",
        "\t\t\t\tbreak\n",
        "\t\t# append to input\n",
        "\t\tin_text += ' ' + out_word\n",
        "\treturn in_text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "umWOc-zGO-KM",
        "colab_type": "code",
        "outputId": "6b77cad6-d1a6-4478-ef47-d55bad9274e2",
        "colab": {}
      },
      "source": [
        "# integer encode sequences of words\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts([data])\n",
        "encoded = tokenizer.texts_to_sequences([data])[0]\n",
        "\n",
        "# retrieve vocabulary size\n",
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "print('Vocabulary Size: %d' % vocab_size)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary Size: 22\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h3nas1YYO-KQ",
        "colab_type": "code",
        "outputId": "05ae6d4a-5959-497d-cdb9-6fceabd94138",
        "colab": {}
      },
      "source": [
        "# encode 2 words -> 1 word\n",
        "sequences = list()\n",
        "for i in range(2, len(encoded)):\n",
        "\tsequence = encoded[i-2:i+1]\n",
        "\tsequences.append(sequence)\n",
        "print('Total Sequences: %d' % len(sequences))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total Sequences: 23\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kyFQ61wgO-KU",
        "colab_type": "code",
        "outputId": "c05be256-1907-4a4f-fe9e-2a3253bf643a",
        "colab": {}
      },
      "source": [
        "# pad sequences\n",
        "max_length = max([len(seq) for seq in sequences])\n",
        "sequences = pad_sequences(sequences, maxlen=max_length, padding='pre')\n",
        "print('Max Sequence Length: %d' % max_length)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max Sequence Length: 3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dwCG3Tx4O-Kl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# split into input and output elements\n",
        "sequences = array(sequences)\n",
        "X, y = sequences[:,:-1],sequences[:,-1]\n",
        "y = to_categorical(y, num_classes=vocab_size)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFUp1CCZO-Kw",
        "colab_type": "code",
        "outputId": "06a06ccc-1bc7-4c23-9f9a-b32c771599f2",
        "colab": {}
      },
      "source": [
        "# define model\n",
        "model = Sequential()\n",
        "model.add(Embedding(vocab_size, 10, input_length=max_length-1))\n",
        "model.add(LSTM(50))\n",
        "model.add(Dense(vocab_size, activation='softmax'))\n",
        "print(model.summary())\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0716 22:01:53.910820  4440 deprecation_wrapper.py:119] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:63: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "W0716 22:01:53.948847  4440 deprecation_wrapper.py:119] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:492: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "W0716 22:01:53.952849  4440 deprecation_wrapper.py:119] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3630: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "W0716 22:01:54.071583  4440 deprecation.py:506] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:1242: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "keep_dims is deprecated, use keepdims instead\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 2, 10)             220       \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 50)                12200     \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 22)                1122      \n",
            "=================================================================\n",
            "Total params: 13,542\n",
            "Trainable params: 13,542\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ps_YkvMlO-K-",
        "colab_type": "code",
        "outputId": "f6315380-0181-447a-a3b7-72ef313ee620",
        "colab": {}
      },
      "source": [
        "# compile network\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "# fit network\n",
        "model.fit(X, y, epochs=500, verbose=2)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0716 22:01:54.469086  4440 deprecation_wrapper.py:119] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\optimizers.py:711: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "W0716 22:01:54.539178  4440 deprecation_wrapper.py:119] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:2884: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n",
            "W0716 22:01:54.761772  4440 deprecation.py:323] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "W0716 22:01:56.124823  4440 deprecation_wrapper.py:119] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:953: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "W0716 22:01:56.166522  4440 deprecation.py:506] From C:\\Users\\RAHUL\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:675: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/500\n",
            " - 1s - loss: 3.0919 - acc: 0.0000e+00\n",
            "Epoch 2/500\n",
            " - 0s - loss: 3.0912 - acc: 0.0000e+00\n",
            "Epoch 3/500\n",
            " - 0s - loss: 3.0904 - acc: 0.0435\n",
            "Epoch 4/500\n",
            " - 0s - loss: 3.0896 - acc: 0.0870\n",
            "Epoch 5/500\n",
            " - 0s - loss: 3.0887 - acc: 0.0870\n",
            "Epoch 6/500\n",
            " - 0s - loss: 3.0878 - acc: 0.0870\n",
            "Epoch 7/500\n",
            " - 0s - loss: 3.0869 - acc: 0.0870\n",
            "Epoch 8/500\n",
            " - 0s - loss: 3.0860 - acc: 0.0870\n",
            "Epoch 9/500\n",
            " - 0s - loss: 3.0851 - acc: 0.0870\n",
            "Epoch 10/500\n",
            " - 0s - loss: 3.0842 - acc: 0.0870\n",
            "Epoch 11/500\n",
            " - 0s - loss: 3.0832 - acc: 0.0870\n",
            "Epoch 12/500\n",
            " - 0s - loss: 3.0823 - acc: 0.0870\n",
            "Epoch 13/500\n",
            " - 0s - loss: 3.0813 - acc: 0.0870\n",
            "Epoch 14/500\n",
            " - 0s - loss: 3.0803 - acc: 0.0870\n",
            "Epoch 15/500\n",
            " - 0s - loss: 3.0792 - acc: 0.0870\n",
            "Epoch 16/500\n",
            " - 0s - loss: 3.0782 - acc: 0.0870\n",
            "Epoch 17/500\n",
            " - 0s - loss: 3.0771 - acc: 0.0870\n",
            "Epoch 18/500\n",
            " - 0s - loss: 3.0760 - acc: 0.0870\n",
            "Epoch 19/500\n",
            " - 0s - loss: 3.0748 - acc: 0.0870\n",
            "Epoch 20/500\n",
            " - 0s - loss: 3.0736 - acc: 0.0870\n",
            "Epoch 21/500\n",
            " - 0s - loss: 3.0724 - acc: 0.0870\n",
            "Epoch 22/500\n",
            " - 0s - loss: 3.0712 - acc: 0.0870\n",
            "Epoch 23/500\n",
            " - 0s - loss: 3.0699 - acc: 0.0870\n",
            "Epoch 24/500\n",
            " - 0s - loss: 3.0685 - acc: 0.0870\n",
            "Epoch 25/500\n",
            " - 0s - loss: 3.0671 - acc: 0.0870\n",
            "Epoch 26/500\n",
            " - 0s - loss: 3.0657 - acc: 0.0870\n",
            "Epoch 27/500\n",
            " - 0s - loss: 3.0642 - acc: 0.0870\n",
            "Epoch 28/500\n",
            " - 0s - loss: 3.0626 - acc: 0.0870\n",
            "Epoch 29/500\n",
            " - 0s - loss: 3.0610 - acc: 0.0870\n",
            "Epoch 30/500\n",
            " - 0s - loss: 3.0594 - acc: 0.0870\n",
            "Epoch 31/500\n",
            " - 0s - loss: 3.0577 - acc: 0.0870\n",
            "Epoch 32/500\n",
            " - 0s - loss: 3.0559 - acc: 0.0870\n",
            "Epoch 33/500\n",
            " - 0s - loss: 3.0540 - acc: 0.0870\n",
            "Epoch 34/500\n",
            " - 0s - loss: 3.0521 - acc: 0.0870\n",
            "Epoch 35/500\n",
            " - 0s - loss: 3.0501 - acc: 0.0870\n",
            "Epoch 36/500\n",
            " - 0s - loss: 3.0480 - acc: 0.0870\n",
            "Epoch 37/500\n",
            " - 0s - loss: 3.0458 - acc: 0.0870\n",
            "Epoch 38/500\n",
            " - 0s - loss: 3.0436 - acc: 0.0870\n",
            "Epoch 39/500\n",
            " - 0s - loss: 3.0413 - acc: 0.0870\n",
            "Epoch 40/500\n",
            " - 0s - loss: 3.0388 - acc: 0.0870\n",
            "Epoch 41/500\n",
            " - 0s - loss: 3.0363 - acc: 0.0870\n",
            "Epoch 42/500\n",
            " - 0s - loss: 3.0337 - acc: 0.0870\n",
            "Epoch 43/500\n",
            " - 0s - loss: 3.0309 - acc: 0.0870\n",
            "Epoch 44/500\n",
            " - 0s - loss: 3.0281 - acc: 0.0870\n",
            "Epoch 45/500\n",
            " - 0s - loss: 3.0251 - acc: 0.0870\n",
            "Epoch 46/500\n",
            " - 0s - loss: 3.0221 - acc: 0.0870\n",
            "Epoch 47/500\n",
            " - 0s - loss: 3.0188 - acc: 0.0870\n",
            "Epoch 48/500\n",
            " - 0s - loss: 3.0155 - acc: 0.0870\n",
            "Epoch 49/500\n",
            " - 0s - loss: 3.0120 - acc: 0.0870\n",
            "Epoch 50/500\n",
            " - 0s - loss: 3.0084 - acc: 0.1304\n",
            "Epoch 51/500\n",
            " - 0s - loss: 3.0047 - acc: 0.1304\n",
            "Epoch 52/500\n",
            " - 0s - loss: 3.0007 - acc: 0.1304\n",
            "Epoch 53/500\n",
            " - 0s - loss: 2.9967 - acc: 0.1304\n",
            "Epoch 54/500\n",
            " - 0s - loss: 2.9924 - acc: 0.1304\n",
            "Epoch 55/500\n",
            " - 0s - loss: 2.9880 - acc: 0.1304\n",
            "Epoch 56/500\n",
            " - 0s - loss: 2.9834 - acc: 0.1304\n",
            "Epoch 57/500\n",
            " - 0s - loss: 2.9786 - acc: 0.1304\n",
            "Epoch 58/500\n",
            " - 0s - loss: 2.9737 - acc: 0.1304\n",
            "Epoch 59/500\n",
            " - 0s - loss: 2.9685 - acc: 0.1304\n",
            "Epoch 60/500\n",
            " - 0s - loss: 2.9631 - acc: 0.1304\n",
            "Epoch 61/500\n",
            " - 0s - loss: 2.9575 - acc: 0.1304\n",
            "Epoch 62/500\n",
            " - 0s - loss: 2.9517 - acc: 0.1304\n",
            "Epoch 63/500\n",
            " - 0s - loss: 2.9456 - acc: 0.1304\n",
            "Epoch 64/500\n",
            " - 0s - loss: 2.9393 - acc: 0.1304\n",
            "Epoch 65/500\n",
            " - 0s - loss: 2.9327 - acc: 0.1304\n",
            "Epoch 66/500\n",
            " - 0s - loss: 2.9259 - acc: 0.1304\n",
            "Epoch 67/500\n",
            " - 0s - loss: 2.9188 - acc: 0.1304\n",
            "Epoch 68/500\n",
            " - 0s - loss: 2.9115 - acc: 0.1304\n",
            "Epoch 69/500\n",
            " - 0s - loss: 2.9038 - acc: 0.1304\n",
            "Epoch 70/500\n",
            " - 0s - loss: 2.8959 - acc: 0.1304\n",
            "Epoch 71/500\n",
            " - 0s - loss: 2.8877 - acc: 0.1304\n",
            "Epoch 72/500\n",
            " - 0s - loss: 2.8792 - acc: 0.1304\n",
            "Epoch 73/500\n",
            " - 0s - loss: 2.8703 - acc: 0.1304\n",
            "Epoch 74/500\n",
            " - 0s - loss: 2.8611 - acc: 0.1304\n",
            "Epoch 75/500\n",
            " - 0s - loss: 2.8515 - acc: 0.1304\n",
            "Epoch 76/500\n",
            " - 0s - loss: 2.8416 - acc: 0.1304\n",
            "Epoch 77/500\n",
            " - 0s - loss: 2.8314 - acc: 0.1304\n",
            "Epoch 78/500\n",
            " - 0s - loss: 2.8207 - acc: 0.1304\n",
            "Epoch 79/500\n",
            " - 0s - loss: 2.8097 - acc: 0.1739\n",
            "Epoch 80/500\n",
            " - 0s - loss: 2.7984 - acc: 0.1739\n",
            "Epoch 81/500\n",
            " - 0s - loss: 2.7866 - acc: 0.1739\n",
            "Epoch 82/500\n",
            " - 0s - loss: 2.7745 - acc: 0.1739\n",
            "Epoch 83/500\n",
            " - 0s - loss: 2.7619 - acc: 0.1739\n",
            "Epoch 84/500\n",
            " - 0s - loss: 2.7490 - acc: 0.1739\n",
            "Epoch 85/500\n",
            " - 0s - loss: 2.7356 - acc: 0.1739\n",
            "Epoch 86/500\n",
            " - 0s - loss: 2.7219 - acc: 0.1739\n",
            "Epoch 87/500\n",
            " - 0s - loss: 2.7077 - acc: 0.1739\n",
            "Epoch 88/500\n",
            " - 0s - loss: 2.6931 - acc: 0.1739\n",
            "Epoch 89/500\n",
            " - 0s - loss: 2.6781 - acc: 0.2174\n",
            "Epoch 90/500\n",
            " - 0s - loss: 2.6627 - acc: 0.2174\n",
            "Epoch 91/500\n",
            " - 0s - loss: 2.6469 - acc: 0.2174\n",
            "Epoch 92/500\n",
            " - 0s - loss: 2.6306 - acc: 0.2609\n",
            "Epoch 93/500\n",
            " - 0s - loss: 2.6140 - acc: 0.2609\n",
            "Epoch 94/500\n",
            " - 0s - loss: 2.5969 - acc: 0.2609\n",
            "Epoch 95/500\n",
            " - 0s - loss: 2.5795 - acc: 0.2609\n",
            "Epoch 96/500\n",
            " - 0s - loss: 2.5617 - acc: 0.2609\n",
            "Epoch 97/500\n",
            " - 0s - loss: 2.5435 - acc: 0.3043\n",
            "Epoch 98/500\n",
            " - 0s - loss: 2.5249 - acc: 0.3043\n",
            "Epoch 99/500\n",
            " - 0s - loss: 2.5059 - acc: 0.3043\n",
            "Epoch 100/500\n",
            " - 0s - loss: 2.4867 - acc: 0.3043\n",
            "Epoch 101/500\n",
            " - 0s - loss: 2.4672 - acc: 0.3043\n",
            "Epoch 102/500\n",
            " - 0s - loss: 2.4474 - acc: 0.3043\n",
            "Epoch 103/500\n",
            " - 0s - loss: 2.4273 - acc: 0.3043\n",
            "Epoch 104/500\n",
            " - 0s - loss: 2.4068 - acc: 0.3043\n",
            "Epoch 105/500\n",
            " - 0s - loss: 2.3862 - acc: 0.3043\n",
            "Epoch 106/500\n",
            " - 0s - loss: 2.3653 - acc: 0.3043\n",
            "Epoch 107/500\n",
            " - 0s - loss: 2.3442 - acc: 0.3478\n",
            "Epoch 108/500\n",
            " - 0s - loss: 2.3229 - acc: 0.3478\n",
            "Epoch 109/500\n",
            " - 0s - loss: 2.3014 - acc: 0.3043\n",
            "Epoch 110/500\n",
            " - 0s - loss: 2.2798 - acc: 0.3478\n",
            "Epoch 111/500\n",
            " - 0s - loss: 2.2580 - acc: 0.3913\n",
            "Epoch 112/500\n",
            " - 0s - loss: 2.2362 - acc: 0.4783\n",
            "Epoch 113/500\n",
            " - 0s - loss: 2.2142 - acc: 0.4783\n",
            "Epoch 114/500\n",
            " - 0s - loss: 2.1923 - acc: 0.4783\n",
            "Epoch 115/500\n",
            " - 0s - loss: 2.1703 - acc: 0.5217\n",
            "Epoch 116/500\n",
            " - 0s - loss: 2.1483 - acc: 0.5217\n",
            "Epoch 117/500\n",
            " - 0s - loss: 2.1262 - acc: 0.5217\n",
            "Epoch 118/500\n",
            " - 0s - loss: 2.1043 - acc: 0.5217\n",
            "Epoch 119/500\n",
            " - 0s - loss: 2.0823 - acc: 0.5217\n",
            "Epoch 120/500\n",
            " - 0s - loss: 2.0605 - acc: 0.5217\n",
            "Epoch 121/500\n",
            " - 0s - loss: 2.0387 - acc: 0.5217\n",
            "Epoch 122/500\n",
            " - 0s - loss: 2.0170 - acc: 0.5217\n",
            "Epoch 123/500\n",
            " - 0s - loss: 1.9954 - acc: 0.5217\n",
            "Epoch 124/500\n",
            " - 0s - loss: 1.9739 - acc: 0.5217\n",
            "Epoch 125/500\n",
            " - 0s - loss: 1.9525 - acc: 0.5217\n",
            "Epoch 126/500\n",
            " - 0s - loss: 1.9313 - acc: 0.5217\n",
            "Epoch 127/500\n",
            " - 0s - loss: 1.9102 - acc: 0.5217\n",
            "Epoch 128/500\n",
            " - 0s - loss: 1.8892 - acc: 0.5217\n",
            "Epoch 129/500\n",
            " - 0s - loss: 1.8684 - acc: 0.5217\n",
            "Epoch 130/500\n",
            " - 0s - loss: 1.8477 - acc: 0.5217\n",
            "Epoch 131/500\n",
            " - 0s - loss: 1.8272 - acc: 0.5652\n",
            "Epoch 132/500\n",
            " - 0s - loss: 1.8068 - acc: 0.5652\n",
            "Epoch 133/500\n",
            " - 0s - loss: 1.7866 - acc: 0.5652\n",
            "Epoch 134/500\n",
            " - 0s - loss: 1.7665 - acc: 0.6087\n",
            "Epoch 135/500\n",
            " - 0s - loss: 1.7466 - acc: 0.6087\n",
            "Epoch 136/500\n",
            " - 0s - loss: 1.7268 - acc: 0.6522\n",
            "Epoch 137/500\n",
            " - 0s - loss: 1.7071 - acc: 0.6522\n",
            "Epoch 138/500\n",
            " - 0s - loss: 1.6876 - acc: 0.6522\n",
            "Epoch 139/500\n",
            " - 0s - loss: 1.6682 - acc: 0.7391\n",
            "Epoch 140/500\n",
            " - 0s - loss: 1.6490 - acc: 0.7826\n",
            "Epoch 141/500\n",
            " - 0s - loss: 1.6298 - acc: 0.7826\n",
            "Epoch 142/500\n",
            " - 0s - loss: 1.6108 - acc: 0.7826\n",
            "Epoch 143/500\n",
            " - 0s - loss: 1.5919 - acc: 0.7826\n",
            "Epoch 144/500\n",
            " - 0s - loss: 1.5730 - acc: 0.7826\n",
            "Epoch 145/500\n",
            " - 0s - loss: 1.5543 - acc: 0.7826\n",
            "Epoch 146/500\n",
            " - 0s - loss: 1.5356 - acc: 0.7826\n",
            "Epoch 147/500\n",
            " - 0s - loss: 1.5170 - acc: 0.7826\n",
            "Epoch 148/500\n",
            " - 0s - loss: 1.4985 - acc: 0.7826\n",
            "Epoch 149/500\n",
            " - 0s - loss: 1.4800 - acc: 0.8261\n",
            "Epoch 150/500\n",
            " - 0s - loss: 1.4616 - acc: 0.8261\n",
            "Epoch 151/500\n",
            " - 0s - loss: 1.4433 - acc: 0.8261\n",
            "Epoch 152/500\n",
            " - 0s - loss: 1.4250 - acc: 0.8261\n",
            "Epoch 153/500\n",
            " - 0s - loss: 1.4068 - acc: 0.8261\n",
            "Epoch 154/500\n",
            " - 0s - loss: 1.3886 - acc: 0.8261\n",
            "Epoch 155/500\n",
            " - 0s - loss: 1.3705 - acc: 0.8261\n",
            "Epoch 156/500\n",
            " - 0s - loss: 1.3525 - acc: 0.8261\n",
            "Epoch 157/500\n",
            " - 0s - loss: 1.3345 - acc: 0.8261\n",
            "Epoch 158/500\n",
            " - 0s - loss: 1.3166 - acc: 0.8261\n",
            "Epoch 159/500\n",
            " - 0s - loss: 1.2987 - acc: 0.8261\n",
            "Epoch 160/500\n",
            " - 0s - loss: 1.2809 - acc: 0.8696\n",
            "Epoch 161/500\n",
            " - 0s - loss: 1.2631 - acc: 0.8696\n",
            "Epoch 162/500\n",
            " - 0s - loss: 1.2454 - acc: 0.8696\n",
            "Epoch 163/500\n",
            " - 0s - loss: 1.2277 - acc: 0.8696\n",
            "Epoch 164/500\n",
            " - 0s - loss: 1.2100 - acc: 0.8696\n",
            "Epoch 165/500\n",
            " - 0s - loss: 1.1924 - acc: 0.8696\n",
            "Epoch 166/500\n",
            " - 0s - loss: 1.1749 - acc: 0.8696\n",
            "Epoch 167/500\n",
            " - 0s - loss: 1.1574 - acc: 0.8696\n",
            "Epoch 168/500\n",
            " - 0s - loss: 1.1399 - acc: 0.8696\n",
            "Epoch 169/500\n",
            " - 0s - loss: 1.1225 - acc: 0.8696\n",
            "Epoch 170/500\n",
            " - 0s - loss: 1.1051 - acc: 0.8696\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 171/500\n",
            " - 0s - loss: 1.0878 - acc: 0.8696\n",
            "Epoch 172/500\n",
            " - 0s - loss: 1.0706 - acc: 0.8696\n",
            "Epoch 173/500\n",
            " - 0s - loss: 1.0533 - acc: 0.8696\n",
            "Epoch 174/500\n",
            " - 0s - loss: 1.0361 - acc: 0.8696\n",
            "Epoch 175/500\n",
            " - 0s - loss: 1.0190 - acc: 0.8696\n",
            "Epoch 176/500\n",
            " - 0s - loss: 1.0019 - acc: 0.8696\n",
            "Epoch 177/500\n",
            " - 0s - loss: 0.9849 - acc: 0.8696\n",
            "Epoch 178/500\n",
            " - 0s - loss: 0.9680 - acc: 0.8696\n",
            "Epoch 179/500\n",
            " - 0s - loss: 0.9511 - acc: 0.8696\n",
            "Epoch 180/500\n",
            " - 0s - loss: 0.9344 - acc: 0.8696\n",
            "Epoch 181/500\n",
            " - 0s - loss: 0.9177 - acc: 0.8696\n",
            "Epoch 182/500\n",
            " - 0s - loss: 0.9012 - acc: 0.8696\n",
            "Epoch 183/500\n",
            " - 0s - loss: 0.8847 - acc: 0.8696\n",
            "Epoch 184/500\n",
            " - 0s - loss: 0.8684 - acc: 0.8696\n",
            "Epoch 185/500\n",
            " - 0s - loss: 0.8522 - acc: 0.8696\n",
            "Epoch 186/500\n",
            " - 0s - loss: 0.8362 - acc: 0.8696\n",
            "Epoch 187/500\n",
            " - 0s - loss: 0.8204 - acc: 0.8696\n",
            "Epoch 188/500\n",
            " - 0s - loss: 0.8047 - acc: 0.8696\n",
            "Epoch 189/500\n",
            " - 0s - loss: 0.7892 - acc: 0.8696\n",
            "Epoch 190/500\n",
            " - 0s - loss: 0.7740 - acc: 0.8696\n",
            "Epoch 191/500\n",
            " - 0s - loss: 0.7589 - acc: 0.8696\n",
            "Epoch 192/500\n",
            " - 0s - loss: 0.7441 - acc: 0.8696\n",
            "Epoch 193/500\n",
            " - 0s - loss: 0.7294 - acc: 0.8696\n",
            "Epoch 194/500\n",
            " - 0s - loss: 0.7149 - acc: 0.8696\n",
            "Epoch 195/500\n",
            " - 0s - loss: 0.7006 - acc: 0.8696\n",
            "Epoch 196/500\n",
            " - 0s - loss: 0.6864 - acc: 0.8696\n",
            "Epoch 197/500\n",
            " - 0s - loss: 0.6724 - acc: 0.8696\n",
            "Epoch 198/500\n",
            " - 0s - loss: 0.6587 - acc: 0.9130\n",
            "Epoch 199/500\n",
            " - 0s - loss: 0.6452 - acc: 0.9130\n",
            "Epoch 200/500\n",
            " - 0s - loss: 0.6319 - acc: 0.9130\n",
            "Epoch 201/500\n",
            " - 0s - loss: 0.6188 - acc: 0.9130\n",
            "Epoch 202/500\n",
            " - 0s - loss: 0.6059 - acc: 0.9130\n",
            "Epoch 203/500\n",
            " - 0s - loss: 0.5933 - acc: 0.9130\n",
            "Epoch 204/500\n",
            " - 0s - loss: 0.5808 - acc: 0.9130\n",
            "Epoch 205/500\n",
            " - 0s - loss: 0.5686 - acc: 0.9130\n",
            "Epoch 206/500\n",
            " - 0s - loss: 0.5566 - acc: 0.9130\n",
            "Epoch 207/500\n",
            " - 0s - loss: 0.5447 - acc: 0.9130\n",
            "Epoch 208/500\n",
            " - 0s - loss: 0.5331 - acc: 0.9130\n",
            "Epoch 209/500\n",
            " - 0s - loss: 0.5217 - acc: 0.9130\n",
            "Epoch 210/500\n",
            " - 0s - loss: 0.5106 - acc: 0.9130\n",
            "Epoch 211/500\n",
            " - 0s - loss: 0.4997 - acc: 0.9130\n",
            "Epoch 212/500\n",
            " - 0s - loss: 0.4890 - acc: 0.9130\n",
            "Epoch 213/500\n",
            " - 0s - loss: 0.4785 - acc: 0.9130\n",
            "Epoch 214/500\n",
            " - 0s - loss: 0.4682 - acc: 0.9130\n",
            "Epoch 215/500\n",
            " - 0s - loss: 0.4582 - acc: 0.9130\n",
            "Epoch 216/500\n",
            " - 0s - loss: 0.4484 - acc: 0.9130\n",
            "Epoch 217/500\n",
            " - 0s - loss: 0.4387 - acc: 0.9130\n",
            "Epoch 218/500\n",
            " - 0s - loss: 0.4293 - acc: 0.9130\n",
            "Epoch 219/500\n",
            " - 0s - loss: 0.4201 - acc: 0.9130\n",
            "Epoch 220/500\n",
            " - 0s - loss: 0.4111 - acc: 0.9130\n",
            "Epoch 221/500\n",
            " - 0s - loss: 0.4023 - acc: 0.9130\n",
            "Epoch 222/500\n",
            " - 0s - loss: 0.3938 - acc: 0.9130\n",
            "Epoch 223/500\n",
            " - 0s - loss: 0.3854 - acc: 0.9130\n",
            "Epoch 224/500\n",
            " - 0s - loss: 0.3772 - acc: 0.9130\n",
            "Epoch 225/500\n",
            " - 0s - loss: 0.3693 - acc: 0.9130\n",
            "Epoch 226/500\n",
            " - 0s - loss: 0.3615 - acc: 0.9130\n",
            "Epoch 227/500\n",
            " - 0s - loss: 0.3540 - acc: 0.9130\n",
            "Epoch 228/500\n",
            " - 0s - loss: 0.3466 - acc: 0.9130\n",
            "Epoch 229/500\n",
            " - 0s - loss: 0.3395 - acc: 0.9565\n",
            "Epoch 230/500\n",
            " - 0s - loss: 0.3325 - acc: 0.9565\n",
            "Epoch 231/500\n",
            " - 0s - loss: 0.3257 - acc: 0.9565\n",
            "Epoch 232/500\n",
            " - 0s - loss: 0.3191 - acc: 0.9565\n",
            "Epoch 233/500\n",
            " - 0s - loss: 0.3126 - acc: 0.9565\n",
            "Epoch 234/500\n",
            " - 0s - loss: 0.3063 - acc: 0.9565\n",
            "Epoch 235/500\n",
            " - 0s - loss: 0.3002 - acc: 0.9565\n",
            "Epoch 236/500\n",
            " - 0s - loss: 0.2943 - acc: 0.9565\n",
            "Epoch 237/500\n",
            " - 0s - loss: 0.2885 - acc: 0.9565\n",
            "Epoch 238/500\n",
            " - 0s - loss: 0.2829 - acc: 0.9565\n",
            "Epoch 239/500\n",
            " - 0s - loss: 0.2774 - acc: 0.9565\n",
            "Epoch 240/500\n",
            " - 0s - loss: 0.2721 - acc: 0.9565\n",
            "Epoch 241/500\n",
            " - 0s - loss: 0.2670 - acc: 0.9565\n",
            "Epoch 242/500\n",
            " - 0s - loss: 0.2620 - acc: 0.9565\n",
            "Epoch 243/500\n",
            " - 0s - loss: 0.2571 - acc: 0.9565\n",
            "Epoch 244/500\n",
            " - 0s - loss: 0.2524 - acc: 0.9565\n",
            "Epoch 245/500\n",
            " - 0s - loss: 0.2478 - acc: 0.9565\n",
            "Epoch 246/500\n",
            " - 0s - loss: 0.2433 - acc: 0.9565\n",
            "Epoch 247/500\n",
            " - 0s - loss: 0.2390 - acc: 0.9565\n",
            "Epoch 248/500\n",
            " - 0s - loss: 0.2348 - acc: 0.9565\n",
            "Epoch 249/500\n",
            " - 0s - loss: 0.2307 - acc: 0.9565\n",
            "Epoch 250/500\n",
            " - 0s - loss: 0.2268 - acc: 0.9565\n",
            "Epoch 251/500\n",
            " - 0s - loss: 0.2230 - acc: 0.9565\n",
            "Epoch 252/500\n",
            " - 0s - loss: 0.2192 - acc: 0.9565\n",
            "Epoch 253/500\n",
            " - 0s - loss: 0.2157 - acc: 0.9565\n",
            "Epoch 254/500\n",
            " - 0s - loss: 0.2122 - acc: 0.9565\n",
            "Epoch 255/500\n",
            " - 0s - loss: 0.2088 - acc: 0.9565\n",
            "Epoch 256/500\n",
            " - 0s - loss: 0.2055 - acc: 0.9565\n",
            "Epoch 257/500\n",
            " - 0s - loss: 0.2023 - acc: 0.9565\n",
            "Epoch 258/500\n",
            " - 0s - loss: 0.1992 - acc: 0.9565\n",
            "Epoch 259/500\n",
            " - 0s - loss: 0.1962 - acc: 0.9565\n",
            "Epoch 260/500\n",
            " - 0s - loss: 0.1934 - acc: 0.9565\n",
            "Epoch 261/500\n",
            " - 0s - loss: 0.1906 - acc: 0.9565\n",
            "Epoch 262/500\n",
            " - 0s - loss: 0.1879 - acc: 0.9565\n",
            "Epoch 263/500\n",
            " - 0s - loss: 0.1852 - acc: 0.9565\n",
            "Epoch 264/500\n",
            " - 0s - loss: 0.1827 - acc: 0.9565\n",
            "Epoch 265/500\n",
            " - 0s - loss: 0.1802 - acc: 0.9565\n",
            "Epoch 266/500\n",
            " - 0s - loss: 0.1778 - acc: 0.9565\n",
            "Epoch 267/500\n",
            " - 0s - loss: 0.1755 - acc: 0.9565\n",
            "Epoch 268/500\n",
            " - 0s - loss: 0.1732 - acc: 0.9565\n",
            "Epoch 269/500\n",
            " - 0s - loss: 0.1711 - acc: 0.9565\n",
            "Epoch 270/500\n",
            " - 0s - loss: 0.1689 - acc: 0.9565\n",
            "Epoch 271/500\n",
            " - 0s - loss: 0.1668 - acc: 0.9565\n",
            "Epoch 272/500\n",
            " - 0s - loss: 0.1648 - acc: 0.9565\n",
            "Epoch 273/500\n",
            " - 0s - loss: 0.1629 - acc: 0.9565\n",
            "Epoch 274/500\n",
            " - 0s - loss: 0.1610 - acc: 0.9565\n",
            "Epoch 275/500\n",
            " - 0s - loss: 0.1591 - acc: 0.9565\n",
            "Epoch 276/500\n",
            " - 0s - loss: 0.1573 - acc: 0.9565\n",
            "Epoch 277/500\n",
            " - 0s - loss: 0.1555 - acc: 0.9565\n",
            "Epoch 278/500\n",
            " - 0s - loss: 0.1538 - acc: 0.9565\n",
            "Epoch 279/500\n",
            " - 0s - loss: 0.1522 - acc: 0.9565\n",
            "Epoch 280/500\n",
            " - 0s - loss: 0.1506 - acc: 0.9565\n",
            "Epoch 281/500\n",
            " - 0s - loss: 0.1490 - acc: 0.9565\n",
            "Epoch 282/500\n",
            " - 0s - loss: 0.1475 - acc: 0.9565\n",
            "Epoch 283/500\n",
            " - 0s - loss: 0.1460 - acc: 0.9565\n",
            "Epoch 284/500\n",
            " - 0s - loss: 0.1445 - acc: 0.9565\n",
            "Epoch 285/500\n",
            " - 0s - loss: 0.1431 - acc: 0.9565\n",
            "Epoch 286/500\n",
            " - 0s - loss: 0.1417 - acc: 0.9565\n",
            "Epoch 287/500\n",
            " - 0s - loss: 0.1403 - acc: 0.9565\n",
            "Epoch 288/500\n",
            " - 0s - loss: 0.1390 - acc: 0.9565\n",
            "Epoch 289/500\n",
            " - 0s - loss: 0.1377 - acc: 0.9565\n",
            "Epoch 290/500\n",
            " - 0s - loss: 0.1365 - acc: 0.9565\n",
            "Epoch 291/500\n",
            " - 0s - loss: 0.1352 - acc: 0.9565\n",
            "Epoch 292/500\n",
            " - 0s - loss: 0.1340 - acc: 0.9565\n",
            "Epoch 293/500\n",
            " - 0s - loss: 0.1329 - acc: 0.9565\n",
            "Epoch 294/500\n",
            " - 0s - loss: 0.1317 - acc: 0.9565\n",
            "Epoch 295/500\n",
            " - 0s - loss: 0.1306 - acc: 0.9565\n",
            "Epoch 296/500\n",
            " - 0s - loss: 0.1295 - acc: 0.9565\n",
            "Epoch 297/500\n",
            " - 0s - loss: 0.1285 - acc: 0.9565\n",
            "Epoch 298/500\n",
            " - 0s - loss: 0.1274 - acc: 0.9565\n",
            "Epoch 299/500\n",
            " - 0s - loss: 0.1264 - acc: 0.9565\n",
            "Epoch 300/500\n",
            " - 0s - loss: 0.1254 - acc: 0.9565\n",
            "Epoch 301/500\n",
            " - 0s - loss: 0.1245 - acc: 0.9565\n",
            "Epoch 302/500\n",
            " - 0s - loss: 0.1235 - acc: 0.9565\n",
            "Epoch 303/500\n",
            " - 0s - loss: 0.1226 - acc: 0.9565\n",
            "Epoch 304/500\n",
            " - 0s - loss: 0.1217 - acc: 0.9565\n",
            "Epoch 305/500\n",
            " - 0s - loss: 0.1208 - acc: 0.9565\n",
            "Epoch 306/500\n",
            " - 0s - loss: 0.1199 - acc: 0.9565\n",
            "Epoch 307/500\n",
            " - 0s - loss: 0.1191 - acc: 0.9565\n",
            "Epoch 308/500\n",
            " - 0s - loss: 0.1183 - acc: 0.9565\n",
            "Epoch 309/500\n",
            " - 0s - loss: 0.1174 - acc: 0.9565\n",
            "Epoch 310/500\n",
            " - 0s - loss: 0.1166 - acc: 0.9565\n",
            "Epoch 311/500\n",
            " - 0s - loss: 0.1159 - acc: 0.9565\n",
            "Epoch 312/500\n",
            " - 0s - loss: 0.1151 - acc: 0.9565\n",
            "Epoch 313/500\n",
            " - 0s - loss: 0.1144 - acc: 0.9565\n",
            "Epoch 314/500\n",
            " - 0s - loss: 0.1136 - acc: 0.9565\n",
            "Epoch 315/500\n",
            " - 0s - loss: 0.1129 - acc: 0.9565\n",
            "Epoch 316/500\n",
            " - 0s - loss: 0.1122 - acc: 0.9565\n",
            "Epoch 317/500\n",
            " - 0s - loss: 0.1115 - acc: 0.9565\n",
            "Epoch 318/500\n",
            " - 0s - loss: 0.1109 - acc: 0.9565\n",
            "Epoch 319/500\n",
            " - 0s - loss: 0.1102 - acc: 0.9565\n",
            "Epoch 320/500\n",
            " - 0s - loss: 0.1096 - acc: 0.9565\n",
            "Epoch 321/500\n",
            " - 0s - loss: 0.1089 - acc: 0.9565\n",
            "Epoch 322/500\n",
            " - 0s - loss: 0.1083 - acc: 0.9565\n",
            "Epoch 323/500\n",
            " - 0s - loss: 0.1077 - acc: 0.9565\n",
            "Epoch 324/500\n",
            " - 0s - loss: 0.1071 - acc: 0.9565\n",
            "Epoch 325/500\n",
            " - 0s - loss: 0.1065 - acc: 0.9565\n",
            "Epoch 326/500\n",
            " - 0s - loss: 0.1059 - acc: 0.9565\n",
            "Epoch 327/500\n",
            " - 0s - loss: 0.1054 - acc: 0.9565\n",
            "Epoch 328/500\n",
            " - 0s - loss: 0.1048 - acc: 0.9565\n",
            "Epoch 329/500\n",
            " - 0s - loss: 0.1043 - acc: 0.9565\n",
            "Epoch 330/500\n",
            " - 0s - loss: 0.1038 - acc: 0.9565\n",
            "Epoch 331/500\n",
            " - 0s - loss: 0.1032 - acc: 0.9565\n",
            "Epoch 332/500\n",
            " - 0s - loss: 0.1027 - acc: 0.9565\n",
            "Epoch 333/500\n",
            " - 0s - loss: 0.1022 - acc: 0.9565\n",
            "Epoch 334/500\n",
            " - 0s - loss: 0.1017 - acc: 0.9565\n",
            "Epoch 335/500\n",
            " - 0s - loss: 0.1012 - acc: 0.9565\n",
            "Epoch 336/500\n",
            " - 0s - loss: 0.1008 - acc: 0.9565\n",
            "Epoch 337/500\n",
            " - 0s - loss: 0.1003 - acc: 0.9565\n",
            "Epoch 338/500\n",
            " - 0s - loss: 0.0998 - acc: 0.9565\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 339/500\n",
            " - 0s - loss: 0.0994 - acc: 0.9565\n",
            "Epoch 340/500\n",
            " - 0s - loss: 0.0990 - acc: 0.9565\n",
            "Epoch 341/500\n",
            " - 0s - loss: 0.0985 - acc: 0.9565\n",
            "Epoch 342/500\n",
            " - 0s - loss: 0.0981 - acc: 0.9565\n",
            "Epoch 343/500\n",
            " - 0s - loss: 0.0977 - acc: 0.9565\n",
            "Epoch 344/500\n",
            " - 0s - loss: 0.0973 - acc: 0.9565\n",
            "Epoch 345/500\n",
            " - 0s - loss: 0.0969 - acc: 0.9565\n",
            "Epoch 346/500\n",
            " - 0s - loss: 0.0965 - acc: 0.9565\n",
            "Epoch 347/500\n",
            " - 0s - loss: 0.0961 - acc: 0.9565\n",
            "Epoch 348/500\n",
            " - 0s - loss: 0.0957 - acc: 0.9565\n",
            "Epoch 349/500\n",
            " - 0s - loss: 0.0953 - acc: 0.9565\n",
            "Epoch 350/500\n",
            " - 0s - loss: 0.0949 - acc: 0.9565\n",
            "Epoch 351/500\n",
            " - 0s - loss: 0.0946 - acc: 0.9565\n",
            "Epoch 352/500\n",
            " - 0s - loss: 0.0942 - acc: 0.9565\n",
            "Epoch 353/500\n",
            " - 0s - loss: 0.0938 - acc: 0.9565\n",
            "Epoch 354/500\n",
            " - 0s - loss: 0.0935 - acc: 0.9565\n",
            "Epoch 355/500\n",
            " - 0s - loss: 0.0932 - acc: 0.9565\n",
            "Epoch 356/500\n",
            " - 0s - loss: 0.0928 - acc: 0.9565\n",
            "Epoch 357/500\n",
            " - 0s - loss: 0.0925 - acc: 0.9565\n",
            "Epoch 358/500\n",
            " - 0s - loss: 0.0922 - acc: 0.9565\n",
            "Epoch 359/500\n",
            " - 0s - loss: 0.0918 - acc: 0.9565\n",
            "Epoch 360/500\n",
            " - 0s - loss: 0.0915 - acc: 0.9565\n",
            "Epoch 361/500\n",
            " - 0s - loss: 0.0912 - acc: 0.9565\n",
            "Epoch 362/500\n",
            " - 0s - loss: 0.0909 - acc: 0.9565\n",
            "Epoch 363/500\n",
            " - 0s - loss: 0.0906 - acc: 0.9565\n",
            "Epoch 364/500\n",
            " - 0s - loss: 0.0903 - acc: 0.9565\n",
            "Epoch 365/500\n",
            " - 0s - loss: 0.0900 - acc: 0.9565\n",
            "Epoch 366/500\n",
            " - 0s - loss: 0.0897 - acc: 0.9565\n",
            "Epoch 367/500\n",
            " - 0s - loss: 0.0895 - acc: 0.9565\n",
            "Epoch 368/500\n",
            " - 0s - loss: 0.0892 - acc: 0.9565\n",
            "Epoch 369/500\n",
            " - 0s - loss: 0.0889 - acc: 0.9565\n",
            "Epoch 370/500\n",
            " - 0s - loss: 0.0887 - acc: 0.9565\n",
            "Epoch 371/500\n",
            " - 0s - loss: 0.0884 - acc: 0.9565\n",
            "Epoch 372/500\n",
            " - 0s - loss: 0.0881 - acc: 0.9565\n",
            "Epoch 373/500\n",
            " - 0s - loss: 0.0879 - acc: 0.9565\n",
            "Epoch 374/500\n",
            " - 0s - loss: 0.0876 - acc: 0.9565\n",
            "Epoch 375/500\n",
            " - 0s - loss: 0.0874 - acc: 0.9565\n",
            "Epoch 376/500\n",
            " - 0s - loss: 0.0871 - acc: 0.9565\n",
            "Epoch 377/500\n",
            " - 0s - loss: 0.0869 - acc: 0.9565\n",
            "Epoch 378/500\n",
            " - 0s - loss: 0.0867 - acc: 0.9565\n",
            "Epoch 379/500\n",
            " - 0s - loss: 0.0864 - acc: 0.9565\n",
            "Epoch 380/500\n",
            " - 0s - loss: 0.0862 - acc: 0.9565\n",
            "Epoch 381/500\n",
            " - 0s - loss: 0.0860 - acc: 0.9565\n",
            "Epoch 382/500\n",
            " - 0s - loss: 0.0858 - acc: 0.9565\n",
            "Epoch 383/500\n",
            " - 0s - loss: 0.0855 - acc: 0.9565\n",
            "Epoch 384/500\n",
            " - 0s - loss: 0.0853 - acc: 0.9565\n",
            "Epoch 385/500\n",
            " - 0s - loss: 0.0851 - acc: 0.9565\n",
            "Epoch 386/500\n",
            " - 0s - loss: 0.0849 - acc: 0.9565\n",
            "Epoch 387/500\n",
            " - 0s - loss: 0.0847 - acc: 0.9565\n",
            "Epoch 388/500\n",
            " - 0s - loss: 0.0845 - acc: 0.9565\n",
            "Epoch 389/500\n",
            " - 0s - loss: 0.0843 - acc: 0.9565\n",
            "Epoch 390/500\n",
            " - 0s - loss: 0.0841 - acc: 0.9565\n",
            "Epoch 391/500\n",
            " - 0s - loss: 0.0839 - acc: 0.9565\n",
            "Epoch 392/500\n",
            " - 0s - loss: 0.0837 - acc: 0.9565\n",
            "Epoch 393/500\n",
            " - 0s - loss: 0.0835 - acc: 0.9565\n",
            "Epoch 394/500\n",
            " - 0s - loss: 0.0833 - acc: 0.9565\n",
            "Epoch 395/500\n",
            " - 0s - loss: 0.0831 - acc: 0.9565\n",
            "Epoch 396/500\n",
            " - 0s - loss: 0.0829 - acc: 0.9565\n",
            "Epoch 397/500\n",
            " - 0s - loss: 0.0828 - acc: 0.9565\n",
            "Epoch 398/500\n",
            " - 0s - loss: 0.0826 - acc: 0.9565\n",
            "Epoch 399/500\n",
            " - 0s - loss: 0.0824 - acc: 0.9565\n",
            "Epoch 400/500\n",
            " - 0s - loss: 0.0822 - acc: 0.9565\n",
            "Epoch 401/500\n",
            " - 0s - loss: 0.0821 - acc: 0.9565\n",
            "Epoch 402/500\n",
            " - 0s - loss: 0.0819 - acc: 0.9565\n",
            "Epoch 403/500\n",
            " - 0s - loss: 0.0817 - acc: 0.9565\n",
            "Epoch 404/500\n",
            " - 0s - loss: 0.0815 - acc: 0.9565\n",
            "Epoch 405/500\n",
            " - 0s - loss: 0.0814 - acc: 0.9565\n",
            "Epoch 406/500\n",
            " - 0s - loss: 0.0812 - acc: 0.9565\n",
            "Epoch 407/500\n",
            " - 0s - loss: 0.0811 - acc: 0.9565\n",
            "Epoch 408/500\n",
            " - 0s - loss: 0.0809 - acc: 0.9565\n",
            "Epoch 409/500\n",
            " - 0s - loss: 0.0807 - acc: 0.9565\n",
            "Epoch 410/500\n",
            " - 0s - loss: 0.0806 - acc: 0.9565\n",
            "Epoch 411/500\n",
            " - 0s - loss: 0.0804 - acc: 0.9565\n",
            "Epoch 412/500\n",
            " - 0s - loss: 0.0803 - acc: 0.9565\n",
            "Epoch 413/500\n",
            " - 0s - loss: 0.0801 - acc: 0.9565\n",
            "Epoch 414/500\n",
            " - 0s - loss: 0.0800 - acc: 0.9565\n",
            "Epoch 415/500\n",
            " - 0s - loss: 0.0798 - acc: 0.9565\n",
            "Epoch 416/500\n",
            " - 0s - loss: 0.0797 - acc: 0.9565\n",
            "Epoch 417/500\n",
            " - 0s - loss: 0.0796 - acc: 0.9565\n",
            "Epoch 418/500\n",
            " - 0s - loss: 0.0794 - acc: 0.9565\n",
            "Epoch 419/500\n",
            " - 0s - loss: 0.0793 - acc: 0.9565\n",
            "Epoch 420/500\n",
            " - 0s - loss: 0.0791 - acc: 0.9565\n",
            "Epoch 421/500\n",
            " - 0s - loss: 0.0790 - acc: 0.9565\n",
            "Epoch 422/500\n",
            " - 0s - loss: 0.0789 - acc: 0.9565\n",
            "Epoch 423/500\n",
            " - 0s - loss: 0.0787 - acc: 0.9565\n",
            "Epoch 424/500\n",
            " - 0s - loss: 0.0786 - acc: 0.9565\n",
            "Epoch 425/500\n",
            " - 0s - loss: 0.0785 - acc: 0.9565\n",
            "Epoch 426/500\n",
            " - 0s - loss: 0.0784 - acc: 0.9565\n",
            "Epoch 427/500\n",
            " - 0s - loss: 0.0782 - acc: 0.9565\n",
            "Epoch 428/500\n",
            " - 0s - loss: 0.0781 - acc: 0.9565\n",
            "Epoch 429/500\n",
            " - 0s - loss: 0.0780 - acc: 0.9565\n",
            "Epoch 430/500\n",
            " - 0s - loss: 0.0779 - acc: 0.9565\n",
            "Epoch 431/500\n",
            " - 0s - loss: 0.0777 - acc: 0.9565\n",
            "Epoch 432/500\n",
            " - 0s - loss: 0.0776 - acc: 0.9565\n",
            "Epoch 433/500\n",
            " - 0s - loss: 0.0775 - acc: 0.9565\n",
            "Epoch 434/500\n",
            " - 0s - loss: 0.0774 - acc: 0.9565\n",
            "Epoch 435/500\n",
            " - 0s - loss: 0.0773 - acc: 0.9565\n",
            "Epoch 436/500\n",
            " - 0s - loss: 0.0772 - acc: 0.9565\n",
            "Epoch 437/500\n",
            " - 0s - loss: 0.0771 - acc: 0.9565\n",
            "Epoch 438/500\n",
            " - 0s - loss: 0.0769 - acc: 0.9565\n",
            "Epoch 439/500\n",
            " - 0s - loss: 0.0768 - acc: 0.9565\n",
            "Epoch 440/500\n",
            " - 0s - loss: 0.0767 - acc: 0.9565\n",
            "Epoch 441/500\n",
            " - 0s - loss: 0.0766 - acc: 0.9565\n",
            "Epoch 442/500\n",
            " - 0s - loss: 0.0765 - acc: 0.9565\n",
            "Epoch 443/500\n",
            " - 0s - loss: 0.0764 - acc: 0.9565\n",
            "Epoch 444/500\n",
            " - 0s - loss: 0.0763 - acc: 0.9565\n",
            "Epoch 445/500\n",
            " - 0s - loss: 0.0762 - acc: 0.9565\n",
            "Epoch 446/500\n",
            " - 0s - loss: 0.0761 - acc: 0.9565\n",
            "Epoch 447/500\n",
            " - 0s - loss: 0.0760 - acc: 0.9565\n",
            "Epoch 448/500\n",
            " - 0s - loss: 0.0759 - acc: 0.9565\n",
            "Epoch 449/500\n",
            " - 0s - loss: 0.0758 - acc: 0.9565\n",
            "Epoch 450/500\n",
            " - 0s - loss: 0.0757 - acc: 0.9565\n",
            "Epoch 451/500\n",
            " - 0s - loss: 0.0756 - acc: 0.9565\n",
            "Epoch 452/500\n",
            " - 0s - loss: 0.0755 - acc: 0.9565\n",
            "Epoch 453/500\n",
            " - 0s - loss: 0.0754 - acc: 0.9565\n",
            "Epoch 454/500\n",
            " - 0s - loss: 0.0753 - acc: 0.9565\n",
            "Epoch 455/500\n",
            " - 0s - loss: 0.0752 - acc: 0.9565\n",
            "Epoch 456/500\n",
            " - 0s - loss: 0.0751 - acc: 0.9565\n",
            "Epoch 457/500\n",
            " - 0s - loss: 0.0750 - acc: 0.9565\n",
            "Epoch 458/500\n",
            " - 0s - loss: 0.0749 - acc: 0.9565\n",
            "Epoch 459/500\n",
            " - 0s - loss: 0.0749 - acc: 0.9565\n",
            "Epoch 460/500\n",
            " - 0s - loss: 0.0748 - acc: 0.9565\n",
            "Epoch 461/500\n",
            " - 0s - loss: 0.0747 - acc: 0.9565\n",
            "Epoch 462/500\n",
            " - 0s - loss: 0.0746 - acc: 0.9565\n",
            "Epoch 463/500\n",
            " - 0s - loss: 0.0745 - acc: 0.9565\n",
            "Epoch 464/500\n",
            " - 0s - loss: 0.0744 - acc: 0.9565\n",
            "Epoch 465/500\n",
            " - 0s - loss: 0.0743 - acc: 0.9565\n",
            "Epoch 466/500\n",
            " - 0s - loss: 0.0742 - acc: 0.9565\n",
            "Epoch 467/500\n",
            " - 0s - loss: 0.0742 - acc: 0.9565\n",
            "Epoch 468/500\n",
            " - 0s - loss: 0.0741 - acc: 0.9565\n",
            "Epoch 469/500\n",
            " - 0s - loss: 0.0740 - acc: 0.9565\n",
            "Epoch 470/500\n",
            " - 0s - loss: 0.0739 - acc: 0.9565\n",
            "Epoch 471/500\n",
            " - 0s - loss: 0.0738 - acc: 0.9565\n",
            "Epoch 472/500\n",
            " - 0s - loss: 0.0738 - acc: 0.9565\n",
            "Epoch 473/500\n",
            " - 0s - loss: 0.0737 - acc: 0.9565\n",
            "Epoch 474/500\n",
            " - 0s - loss: 0.0736 - acc: 0.9565\n",
            "Epoch 475/500\n",
            " - 0s - loss: 0.0735 - acc: 0.9565\n",
            "Epoch 476/500\n",
            " - 0s - loss: 0.0735 - acc: 0.9565\n",
            "Epoch 477/500\n",
            " - 0s - loss: 0.0734 - acc: 0.9565\n",
            "Epoch 478/500\n",
            " - 0s - loss: 0.0733 - acc: 0.9565\n",
            "Epoch 479/500\n",
            " - 0s - loss: 0.0732 - acc: 0.9565\n",
            "Epoch 480/500\n",
            " - 0s - loss: 0.0732 - acc: 0.9565\n",
            "Epoch 481/500\n",
            " - 0s - loss: 0.0731 - acc: 0.9565\n",
            "Epoch 482/500\n",
            " - 0s - loss: 0.0730 - acc: 0.9565\n",
            "Epoch 483/500\n",
            " - 0s - loss: 0.0729 - acc: 0.9565\n",
            "Epoch 484/500\n",
            " - 0s - loss: 0.0729 - acc: 0.9565\n",
            "Epoch 485/500\n",
            " - 0s - loss: 0.0728 - acc: 0.9565\n",
            "Epoch 486/500\n",
            " - 0s - loss: 0.0727 - acc: 0.9565\n",
            "Epoch 487/500\n",
            " - 0s - loss: 0.0727 - acc: 0.9565\n",
            "Epoch 488/500\n",
            " - 0s - loss: 0.0726 - acc: 0.9565\n",
            "Epoch 489/500\n",
            " - 0s - loss: 0.0725 - acc: 0.9565\n",
            "Epoch 490/500\n",
            " - 0s - loss: 0.0725 - acc: 0.9565\n",
            "Epoch 491/500\n",
            " - 0s - loss: 0.0724 - acc: 0.9565\n",
            "Epoch 492/500\n",
            " - 0s - loss: 0.0723 - acc: 0.9565\n",
            "Epoch 493/500\n",
            " - 0s - loss: 0.0723 - acc: 0.9565\n",
            "Epoch 494/500\n",
            " - 0s - loss: 0.0722 - acc: 0.9565\n",
            "Epoch 495/500\n",
            " - 0s - loss: 0.0721 - acc: 0.9565\n",
            "Epoch 496/500\n",
            " - 0s - loss: 0.0721 - acc: 0.9565\n",
            "Epoch 497/500\n",
            " - 0s - loss: 0.0720 - acc: 0.9565\n",
            "Epoch 498/500\n",
            " - 0s - loss: 0.0719 - acc: 0.9565\n",
            "Epoch 499/500\n",
            " - 0s - loss: 0.0719 - acc: 0.9565\n",
            "Epoch 500/500\n",
            " - 0s - loss: 0.0718 - acc: 0.9565\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7460e1cc88>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PokS81THO-LD",
        "colab_type": "code",
        "outputId": "467b5045-b971-4fa1-8603-a0d48fd7fcd2",
        "colab": {}
      },
      "source": [
        "# evaluate model\n",
        "print(generate_seq(model, tokenizer, max_length-1, 'Jack and', 5))\n",
        "print(generate_seq(model, tokenizer, max_length-1, 'And Jill', 3))\n",
        "print(generate_seq(model, tokenizer, max_length-1, 'fell down', 5))\n",
        "print(generate_seq(model, tokenizer, max_length-1, 'pail of', 5))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Jack and jill went up the hill\n",
            "And Jill went up the\n",
            "fell down and broke his crown and\n",
            "pail of water jack fell down and\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q3zCPzisO-LH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}