1_Text_Generation_LSTMs.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text_Generation_LSTMs.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "tEOIqb2-LOe3"
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90VwAU9H5b08"
      },
      "source": [
        "#Step 1 Reading Text Files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "onV3zUGF5bct"
      },
      "source": [
        "def read_txt(file_path):\n",
        "  '''\n",
        "  Input : .txt file\n",
        "  '''\n",
        "  with open(file_path) as f:\n",
        "    str_text = f.read()\n",
        "  return str_text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eYCr870z4mJx"
      },
      "source": [
        "ex_txt_file_path = '/content/drive/My Drive/melville-moby_dick.txt'\n",
        "#these are chapters of a book"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-zTUou57F0s"
      },
      "source": [
        "# read_txt(file_path=ex_txt_file_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jseHQnjy76zz"
      },
      "source": [
        "#Step 2 Tokenizing and Cleaning the data with spacy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vpqm-CyI8Vl0"
      },
      "source": [
        "import spacy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wM6Ju9tQ7Lae"
      },
      "source": [
        "nlp = spacy.load('en',disable=['parser','tagger','ner'])\n",
        "#we are disabling the given features as it would save some time in tokenization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9tMnKYcc8P_M"
      },
      "source": [
        "nlp.max_length = 1198623\n",
        "# upto how many text do we apply tokenization\n",
        "# we can increase the number if we want to cover complete book"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-mekTlyC8-s-"
      },
      "source": [
        "#grabbing the text tokens using list comprehension\n",
        "def separate_punc(doc_text):\n",
        "  '''\n",
        "  text cleaning function\n",
        "  '''\n",
        "  return [token.text.lower() for token in nlp(doc_text) if token.text not in '\\n\\n \\n\\n\\n!\"-#$%&()--.*+,-/:;<=>?@[\\\\]^_`{|}~\\t\\n ']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gNxscgL-96br"
      },
      "source": [
        "d = read_txt('/content/drive/My Drive/moby_dick_four_chapters.txt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_wouI3AYBAuW"
      },
      "source": [
        "# print(d)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80WewqxdBFeD"
      },
      "source": [
        "tokens = separate_punc(d)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4sxu1QbBrmX",
        "outputId": "ad5aa051-af3a-4e10-fe55-0fb61c37670d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(tokens[10:20])\n",
        "print(f'No. of words {len(tokens)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['precisely', 'having', 'little', 'or', 'no', 'money', 'in', 'my', 'purse', 'and']\n",
            "No. of words 11338\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gwUASmLlBxTZ"
      },
      "source": [
        "# so we will have model that takes 25 words as input and outputs the 26th word"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4BYiAxNiChsh"
      },
      "source": [
        "train_len = 25+1\n",
        "\n",
        "text_sequences = [] \n",
        "\n",
        "for i in range(train_len,len(tokens)):\n",
        "  seq = tokens[i-train_len:i]\n",
        "  text_sequences.append(seq)\n",
        "#for this to make sense look at the cell below\n",
        "# A->B->C->D->E\n",
        "# B->C->D->E->F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eeO6hqsHC6nf",
        "outputId": "aad87d29-b7c6-4719-d608-016e4e174f90",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(text_sequences[0])\n",
        "print(text_sequences[1])\n",
        "# so these are sequences of 26 words\n",
        "# same as \n",
        "# A->B->C->D->E\n",
        "# B->C->D->E->F"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['call', 'me', 'ishmael', 'some', 'years', 'ago', 'never', 'mind', 'how', 'long', 'precisely', 'having', 'little', 'or', 'no', 'money', 'in', 'my', 'purse', 'and', 'nothing', 'particular', 'to', 'interest', 'me', 'on']\n",
            "['me', 'ishmael', 'some', 'years', 'ago', 'never', 'mind', 'how', 'long', 'precisely', 'having', 'little', 'or', 'no', 'money', 'in', 'my', 'purse', 'and', 'nothing', 'particular', 'to', 'interest', 'me', 'on', 'shore']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wXt1elrHDRqh"
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qBLf6PzPD8WS"
      },
      "source": [
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(text_sequences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TLR1PYogES2N"
      },
      "source": [
        "sequences = tokenizer.texts_to_sequences(text_sequences)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FCDKy_VCEaM1",
        "outputId": "25c8d7c6-fd5a-4edc-e91c-d4177e6a3bfb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#so what the above code does is it assigns a number to each of the text\n",
        "# like : A->1 B->2 etc.\n",
        "print(sequences[0])\n",
        "print(text_sequences[0])\n",
        "print(sequences[1])\n",
        "print(text_sequences[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[956, 14, 263, 51, 261, 408, 87, 219, 129, 111, 954, 260, 50, 43, 38, 315, 7, 23, 546, 3, 150, 259, 6, 2712, 14, 24]\n",
            "['call', 'me', 'ishmael', 'some', 'years', 'ago', 'never', 'mind', 'how', 'long', 'precisely', 'having', 'little', 'or', 'no', 'money', 'in', 'my', 'purse', 'and', 'nothing', 'particular', 'to', 'interest', 'me', 'on']\n",
            "[14, 263, 51, 261, 408, 87, 219, 129, 111, 954, 260, 50, 43, 38, 315, 7, 23, 546, 3, 150, 259, 6, 2712, 14, 24, 957]\n",
            "['me', 'ishmael', 'some', 'years', 'ago', 'never', 'mind', 'how', 'long', 'precisely', 'having', 'little', 'or', 'no', 'money', 'in', 'my', 'purse', 'and', 'nothing', 'particular', 'to', 'interest', 'me', 'on', 'shore']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1FY5dCUzEsRR",
        "outputId": "cfd9c251-3a29-4388-f93f-2b8b1edb2969",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#these are the tokens assigned to each word\n",
        "for i in sequences[0]:\n",
        "  print(f\"{i} : {tokenizer.index_word[i]}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "956 : call\n",
            "14 : me\n",
            "263 : ishmael\n",
            "51 : some\n",
            "261 : years\n",
            "408 : ago\n",
            "87 : never\n",
            "219 : mind\n",
            "129 : how\n",
            "111 : long\n",
            "954 : precisely\n",
            "260 : having\n",
            "50 : little\n",
            "43 : or\n",
            "38 : no\n",
            "315 : money\n",
            "7 : in\n",
            "23 : my\n",
            "546 : purse\n",
            "3 : and\n",
            "150 : nothing\n",
            "259 : particular\n",
            "6 : to\n",
            "2712 : interest\n",
            "14 : me\n",
            "24 : on\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0gxd7YPuFB3u"
      },
      "source": [
        "sequences = np.array(sequences)\n",
        "# print(sequences)\n",
        "#note the shift"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMZ7CT8GLYye"
      },
      "source": [
        "from keras.utils import to_categorical"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-AOzgehL4QK"
      },
      "source": [
        "X = sequences[:,:-1] #picjking every word except last word"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cyw_bNHIL829"
      },
      "source": [
        "y = sequences[:,-1] # last words"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DPUaIWNSMQhU"
      },
      "source": [
        "#lets check the no. of unique words\n",
        "vocab_size = len(tokenizer.word_counts)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BZqr7K3uMoyh",
        "outputId": "0f8db23e-a20a-4f1f-cf4c-28a5971ff3b2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[  24  957    5 ...    2 2717   26]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6bC9hz3M1MG"
      },
      "source": [
        "#converting to categorical variables\n",
        "y = to_categorical(y,num_classes = vocab_size+1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dd0LMcDoNBds"
      },
      "source": [
        "# print(y)\n",
        "#one hot encoding"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a_5hU3raNEOO",
        "outputId": "7109420f-1556-49b2-ec1c-2e527559329e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "X.shape\n",
        "# 11312 samples with 25 features in each "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(11312, 25)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RF5h23TGNX7V"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense,LSTM,Embedding"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B6r5HfBhN1b6"
      },
      "source": [
        "def create_model(vocab_size,seq_len):\n",
        "  model = Sequential()\n",
        "  model.add(Embedding(vocab_size,seq_len,input_length=seq_len))\n",
        "  model.add(LSTM(75,return_sequences=True))\n",
        "  model.add(LSTM(50))\n",
        "  model.add(Dense(50,activation='relu'))\n",
        "   \n",
        "  model.add(Dense(vocab_size,activation='softmax'))\n",
        "\n",
        "  model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
        "  model.summary()\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TYE8P7lMPgfq",
        "outputId": "9af288db-b4ea-4b06-aa9c-d4208f25a0be",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "model = create_model(vocab_size+1,25)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 25, 25)            67950     \n",
            "_________________________________________________________________\n",
            "lstm (LSTM)                  (None, 25, 75)            30300     \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 50)                25200     \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 50)                2550      \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 2718)              138618    \n",
            "=================================================================\n",
            "Total params: 264,618\n",
            "Trainable params: 264,618\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l07KafkMPuIZ"
      },
      "source": [
        "from pickle import dump,load"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bb9bC1ztQDiS",
        "outputId": "eba3960c-845a-4af4-daac-460c22c044e3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "model.fit(X,y,batch_size=128,epochs=500,verbose=0) #put verbose=1 to view progress"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f89681bf1d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "itjlFmRqQYTk"
      },
      "source": [
        "model.save('book_model_2.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i2zU7GegRZsv"
      },
      "source": [
        "dump(tokenizer,open('my_simpletokenizer','wb'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MbtuR8MiSMb1"
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDPr8LRnRs5K"
      },
      "source": [
        "def generate_text(model,tokenizer,seq_len,seed_text,num_gen_words):\n",
        "\n",
        "  output_text = []\n",
        "\n",
        "  input_text = seed_text\n",
        "  for i in range(num_gen_words):\n",
        "    encoded_text = tokenizer.texts_to_sequences([input_text])[0]\n",
        "    \n",
        "    pad_encoded = pad_sequences([encoded_text],maxlen=seq_len,truncating='pre')\n",
        "    # this will do the padding just in case text sequence is not 25 words long\n",
        "    pred_word_ind = model.predict_classes(pad_encoded,verbose=0)[0]\n",
        "\n",
        "    pred_word = tokenizer.index_word[pred_word_ind]\n",
        "\n",
        "    input_text+=' '+pred_word\n",
        "\n",
        "    output_text.append(pred_word)\n",
        "\n",
        "  return ' '.join(output_text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kdbduEcqZ2OI"
      },
      "source": [
        "import random\n",
        "random.seed(101)\n",
        "random_pick = random.randint(0,len(text_sequences))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aXmoTMOIbOIN"
      },
      "source": [
        "random_seed_text = text_sequences[random_pick]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q7gYa_r8boLp",
        "outputId": "39d29326-c2f5-4393-9ed2-531eb2dfaa40",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "seed_text = ' '.join(random_seed_text)\n",
        "seed_text"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"thought i to myself the man 's a human being just as i am he has just as much reason to fear me as i have\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O9m0auHwP_r_"
      },
      "source": [
        "num_of_words_to_generate = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Q5aAx6SbzgI",
        "outputId": "aced8a90-4ac1-4a5d-8e85-de5383c83502",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "seed_text +' '+ generate_text(model,tokenizer,25,seed_text=seed_text,num_gen_words=num_of_words_to_generate)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"thought i to myself the man 's a human being just as i am he has just as much reason to fear me as i have to be afraid of him that there as if it were complied of mortal once nantucket save the frost whom\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GPG2p0p0cg1N"
      },
      "source": [
        "# THE END"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}