From Glove to Words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys;\n",
    "import re;\n",
    "import operator;\n",
    "import numpy as np;\n",
    "import math;\n",
    "from os import listdir;\n",
    "from keras.utils import np_utils;\n",
    "from keras.regularizers import l2;\n",
    "from keras.preprocessing.text import Tokenizer;\n",
    "from keras.preprocessing.sequence import pad_sequences;\n",
    "from keras.callbacks import ModelCheckpoint;\n",
    "from keras.models import Sequential;\n",
    "from keras.layers import Embedding, LSTM, Dense, Dropout, TimeDistributed, Input;\n",
    "from keras.optimizers import Adam;\n",
    "\n",
    "def cleanText(text):\n",
    "    modifiedString = re.sub(\"\\d\", \"\", text);\n",
    "    modifiedString = modifiedString.lower();\n",
    "    modifiedString = modifiedString.replace(\"\\t\", \" \");\n",
    "    modifiedString = modifiedString.replace(\"\\n\", \" \");\n",
    "    modifiedString = modifiedString.replace('!', \" . \");\n",
    "    modifiedString = modifiedString.replace('\"', \" \");\n",
    "    modifiedString = modifiedString.replace('#', \" \"); \n",
    "    modifiedString = modifiedString.replace(\"'\", \"'\"); \n",
    "    modifiedString = modifiedString.replace('(', \" \"); \n",
    "    modifiedString = modifiedString.replace(')', \" \"); \n",
    "    modifiedString = modifiedString.replace(',', \" , \");\n",
    "    modifiedString = modifiedString.replace('-', \" \"); \n",
    "    modifiedString = modifiedString.replace('.', \" . \"); \n",
    "    modifiedString = modifiedString.replace('/', \" \");\n",
    "    modifiedString = modifiedString.replace(':', \" \");\n",
    "    modifiedString = modifiedString.replace(';', \" ; \"); \n",
    "    modifiedString = modifiedString.replace('?', \" ? \");\n",
    "    modifiedString = modifiedString.replace('–', \" \");\n",
    "    modifiedString = modifiedString.replace('—', \" \");\n",
    "    modifiedString = modifiedString.replace('‘', \"'\"); \n",
    "    modifiedString = modifiedString.replace('…', \" . \");\n",
    "    modifiedString = modifiedString.replace('ç', \"c\");\n",
    "    modifiedString = modifiedString.replace('é', \"e\");\n",
    "    \n",
    "    return modifiedString;\n",
    "\n",
    "def generateData(batchSize, dataX, dataY):\n",
    "    while True:\n",
    "        randomStart = np.random.randint(len(dataX) - batchSize);\n",
    "        X = dataX[randomStart:randomStart+batchSize];\n",
    "        X = np.reshape(X, (batchSize,100));\n",
    "        Y = dataY[randomStart:randomStart+batchSize];\n",
    "        Y = np_utils.to_categorical(Y, nb_classes=len(tokenizer.word_index) + 1);\n",
    "        yield (X, Y);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = \"Stories/\";\n",
    "splitPercentage = 0.2;\n",
    "files = [f for f in listdir(path) if (\"DS_Store\" not in f) and (\"ipynb\" not in f)];\n",
    "rawData = [];\n",
    "for f in files:\n",
    "    text = open(path + f).read();\n",
    "    text = cleanText(text);\n",
    "    rawData.append(text);\n",
    "\n",
    "tokenizer = Tokenizer(filters=\"\");\n",
    "tokenizer.fit_on_texts(rawData);\n",
    "\n",
    "wordDimensions = 100;\n",
    "glovePath = \"GloveData/\";\n",
    "embeddingDic = {};\n",
    "f = open(glovePath + \"tingle-vectors-\" + str(wordDimensions) + \".txt\");\n",
    "for line in f:\n",
    "    values = line.split();\n",
    "    currrentWord = values[0];\n",
    "    currentVector = np.asarray(values[1:], dtype='float32');\n",
    "    embeddingDic[currrentWord] = currentVector;\n",
    "f.close();\n",
    "\n",
    "dataX = np.zeros((len(tokenizer.word_index) + 1, wordDimensions));\n",
    "dataY = np.zeros((len(tokenizer.word_index) + 1, 1));\n",
    "for w, i in tokenizer.word_index.items():\n",
    "    if w in embeddingDic:\n",
    "        dataX[i] = embeddingDic[w];\n",
    "        dataY[i] = [i];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 9.6023 - acc: 0.0265Epoch 00000: loss improved from inf to 9.60074, saving model to GloVeToCategorical/converter-tokenizer-00-9.6007.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 9.6007 - acc: 0.0263     \n",
      "Epoch 2/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 7.8241 - acc: 0.2277Epoch 00001: loss improved from 9.60074 to 7.78426, saving model to GloVeToCategorical/converter-tokenizer-01-7.7843.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 7.7843 - acc: 0.2320     \n",
      "Epoch 3/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 4.9145 - acc: 0.5531Epoch 00002: loss improved from 7.78426 to 4.92723, saving model to GloVeToCategorical/converter-tokenizer-02-4.9272.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 4.9272 - acc: 0.5513     \n",
      "Epoch 4/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 3.4997 - acc: 0.6843Epoch 00003: loss improved from 4.92723 to 3.49278, saving model to GloVeToCategorical/converter-tokenizer-03-3.4928.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 3.4928 - acc: 0.6849     \n",
      "Epoch 5/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 2.5144 - acc: 0.8015Epoch 00004: loss improved from 3.49278 to 2.51345, saving model to GloVeToCategorical/converter-tokenizer-04-2.5134.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 2.5134 - acc: 0.8019     \n",
      "Epoch 6/100\n",
      "19600/20000 [============================>.] - ETA: 0s - loss: 1.9890 - acc: 0.8559Epoch 00005: loss improved from 2.51345 to 1.97522, saving model to GloVeToCategorical/converter-tokenizer-05-1.9752.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 1.9752 - acc: 0.8582     \n",
      "Epoch 7/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 1.6304 - acc: 0.9036Epoch 00006: loss improved from 1.97522 to 1.62673, saving model to GloVeToCategorical/converter-tokenizer-06-1.6267.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 1.6267 - acc: 0.9042     \n",
      "Epoch 8/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 1.5111 - acc: 0.9112Epoch 00007: loss improved from 1.62673 to 1.51917, saving model to GloVeToCategorical/converter-tokenizer-07-1.5192.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 1.5192 - acc: 0.9102     \n",
      "Epoch 9/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 1.4153 - acc: 0.9243Epoch 00008: loss improved from 1.51917 to 1.41885, saving model to GloVeToCategorical/converter-tokenizer-08-1.4188.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 1.4188 - acc: 0.9234     \n",
      "Epoch 10/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 1.1715 - acc: 0.9522Epoch 00009: loss improved from 1.41885 to 1.16822, saving model to GloVeToCategorical/converter-tokenizer-09-1.1682.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 1.1682 - acc: 0.9527     \n",
      "Epoch 11/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 1.0615 - acc: 0.9601Epoch 00010: loss improved from 1.16822 to 1.05932, saving model to GloVeToCategorical/converter-tokenizer-10-1.0593.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 1.0593 - acc: 0.9603     \n",
      "Epoch 12/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 1.0754 - acc: 0.9586Epoch 00011: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 1.0749 - acc: 0.9589     \n",
      "Epoch 13/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 1.0787 - acc: 0.9626Epoch 00012: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 1.0765 - acc: 0.9627     \n",
      "Epoch 14/100\n",
      "19600/20000 [============================>.] - ETA: 0s - loss: 0.9526 - acc: 0.9718Epoch 00013: loss improved from 1.05932 to 0.94976, saving model to GloVeToCategorical/converter-tokenizer-13-0.9498.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.9498 - acc: 0.9721     \n",
      "Epoch 15/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.8342 - acc: 0.9781Epoch 00014: loss improved from 0.94976 to 0.83439, saving model to GloVeToCategorical/converter-tokenizer-14-0.8344.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.8344 - acc: 0.9781     \n",
      "Epoch 16/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.8294 - acc: 0.9801Epoch 00015: loss did not improve\n",
      "20000/20000 [==============================] - 2s - loss: 0.8353 - acc: 0.9790     \n",
      "Epoch 17/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.7968 - acc: 0.9811Epoch 00016: loss improved from 0.83439 to 0.79303, saving model to GloVeToCategorical/converter-tokenizer-16-0.7930.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.7930 - acc: 0.9813     \n",
      "Epoch 18/100\n",
      "19600/20000 [============================>.] - ETA: 0s - loss: 0.8031 - acc: 0.9788Epoch 00017: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.7975 - acc: 0.9792     \n",
      "Epoch 19/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.7810 - acc: 0.9799Epoch 00018: loss improved from 0.79303 to 0.77671, saving model to GloVeToCategorical/converter-tokenizer-18-0.7767.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.7767 - acc: 0.9801     \n",
      "Epoch 20/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.6989 - acc: 0.9856Epoch 00019: loss improved from 0.77671 to 0.69762, saving model to GloVeToCategorical/converter-tokenizer-19-0.6976.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.6976 - acc: 0.9856     \n",
      "Epoch 21/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.6952 - acc: 0.9854Epoch 00020: loss improved from 0.69762 to 0.69592, saving model to GloVeToCategorical/converter-tokenizer-20-0.6959.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.6959 - acc: 0.9853     \n",
      "Epoch 22/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.6879 - acc: 0.9863Epoch 00021: loss improved from 0.69592 to 0.69067, saving model to GloVeToCategorical/converter-tokenizer-21-0.6907.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.6907 - acc: 0.9861     \n",
      "Epoch 23/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.6392 - acc: 0.9874Epoch 00022: loss improved from 0.69067 to 0.64703, saving model to GloVeToCategorical/converter-tokenizer-22-0.6470.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.6470 - acc: 0.9867     \n",
      "Epoch 24/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.6917 - acc: 0.9841Epoch 00023: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.6897 - acc: 0.9840     \n",
      "Epoch 25/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.6355 - acc: 0.9876Epoch 00024: loss improved from 0.64703 to 0.63478, saving model to GloVeToCategorical/converter-tokenizer-24-0.6348.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.6348 - acc: 0.9877     \n",
      "Epoch 26/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.6019 - acc: 0.9899Epoch 00025: loss improved from 0.63478 to 0.60127, saving model to GloVeToCategorical/converter-tokenizer-25-0.6013.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.6013 - acc: 0.9900     \n",
      "Epoch 27/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.6016 - acc: 0.9869Epoch 00026: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.6014 - acc: 0.9869     \n",
      "Epoch 28/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.5670 - acc: 0.9894Epoch 00027: loss improved from 0.60127 to 0.56910, saving model to GloVeToCategorical/converter-tokenizer-27-0.5691.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.5691 - acc: 0.9894     \n",
      "Epoch 29/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.6021 - acc: 0.9852Epoch 00028: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.6002 - acc: 0.9853     \n",
      "Epoch 30/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.5345 - acc: 0.9904Epoch 00029: loss improved from 0.56910 to 0.53320, saving model to GloVeToCategorical/converter-tokenizer-29-0.5332.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.5332 - acc: 0.9904     \n",
      "Epoch 31/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.5246 - acc: 0.9902Epoch 00030: loss improved from 0.53320 to 0.52349, saving model to GloVeToCategorical/converter-tokenizer-30-0.5235.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.5235 - acc: 0.9902     \n",
      "Epoch 32/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.5302 - acc: 0.9906Epoch 00031: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.5295 - acc: 0.9906     \n",
      "Epoch 33/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.5212 - acc: 0.9910Epoch 00032: loss improved from 0.52349 to 0.52042, saving model to GloVeToCategorical/converter-tokenizer-32-0.5204.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.5204 - acc: 0.9910     \n",
      "Epoch 34/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.5108 - acc: 0.9903Epoch 00033: loss improved from 0.52042 to 0.51301, saving model to GloVeToCategorical/converter-tokenizer-33-0.5130.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.5130 - acc: 0.9899     \n",
      "Epoch 35/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.5355 - acc: 0.9875Epoch 00034: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.5374 - acc: 0.9874     \n",
      "Epoch 36/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.4507 - acc: 0.9935Epoch 00035: loss improved from 0.51301 to 0.44980, saving model to GloVeToCategorical/converter-tokenizer-35-0.4498.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.4498 - acc: 0.9935     \n",
      "Epoch 37/100\n",
      "19600/20000 [============================>.] - ETA: 0s - loss: 0.4468 - acc: 0.9930Epoch 00036: loss improved from 0.44980 to 0.44443, saving model to GloVeToCategorical/converter-tokenizer-36-0.4444.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.4444 - acc: 0.9931     \n",
      "Epoch 38/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.4591 - acc: 0.9914Epoch 00037: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4575 - acc: 0.9915     \n",
      "Epoch 39/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.4489 - acc: 0.9928Epoch 00038: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4489 - acc: 0.9929     \n",
      "Epoch 40/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.4585 - acc: 0.9912Epoch 00039: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4564 - acc: 0.9913     \n",
      "Epoch 41/100\n",
      "19600/20000 [============================>.] - ETA: 0s - loss: 0.4364 - acc: 0.9924Epoch 00040: loss improved from 0.44443 to 0.43649, saving model to GloVeToCategorical/converter-tokenizer-40-0.4365.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.4365 - acc: 0.9924     \n",
      "Epoch 42/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.4329 - acc: 0.9925Epoch 00041: loss improved from 0.43649 to 0.43292, saving model to GloVeToCategorical/converter-tokenizer-41-0.4329.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.4329 - acc: 0.9926     \n",
      "Epoch 43/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.3985 - acc: 0.9941Epoch 00042: loss improved from 0.43292 to 0.39822, saving model to GloVeToCategorical/converter-tokenizer-42-0.3982.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3982 - acc: 0.9941     \n",
      "Epoch 44/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.4262 - acc: 0.9921Epoch 00043: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4250 - acc: 0.9920     \n",
      "Epoch 45/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.4178 - acc: 0.9920Epoch 00044: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4171 - acc: 0.9920     \n",
      "Epoch 46/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.3875 - acc: 0.9940Epoch 00045: loss improved from 0.39822 to 0.38543, saving model to GloVeToCategorical/converter-tokenizer-45-0.3854.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3854 - acc: 0.9940     \n",
      "Epoch 47/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.4056 - acc: 0.9910Epoch 00046: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4042 - acc: 0.9911     \n",
      "Epoch 48/100\n",
      "19600/20000 [============================>.] - ETA: 0s - loss: 0.4009 - acc: 0.9916Epoch 00047: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3995 - acc: 0.9916     \n",
      "Epoch 49/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.4130 - acc: 0.9902Epoch 00048: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.4143 - acc: 0.9899     \n",
      "Epoch 50/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.3830 - acc: 0.9915Epoch 00049: loss improved from 0.38543 to 0.38167, saving model to GloVeToCategorical/converter-tokenizer-49-0.3817.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3817 - acc: 0.9916     \n",
      "Epoch 51/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.3656 - acc: 0.9935Epoch 00050: loss improved from 0.38167 to 0.36709, saving model to GloVeToCategorical/converter-tokenizer-50-0.3671.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3671 - acc: 0.9933     \n",
      "Epoch 52/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.3476 - acc: 0.9939Epoch 00051: loss improved from 0.36709 to 0.34779, saving model to GloVeToCategorical/converter-tokenizer-51-0.3478.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3478 - acc: 0.9940     \n",
      "Epoch 53/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.3526 - acc: 0.9936Epoch 00052: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3521 - acc: 0.9936     \n",
      "Epoch 54/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.3663 - acc: 0.9928Epoch 00053: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3659 - acc: 0.9928     \n",
      "Epoch 55/100\n",
      "19700/20000 [============================>.] - ETA: 0s - loss: 0.3426 - acc: 0.9942Epoch 00054: loss improved from 0.34779 to 0.34306, saving model to GloVeToCategorical/converter-tokenizer-54-0.3431.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3431 - acc: 0.9941     \n",
      "Epoch 56/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.3618 - acc: 0.9922Epoch 00055: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3609 - acc: 0.9922     \n",
      "Epoch 57/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.3260 - acc: 0.9942Epoch 00056: loss improved from 0.34306 to 0.32562, saving model to GloVeToCategorical/converter-tokenizer-56-0.3256.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3256 - acc: 0.9942     \n",
      "Epoch 58/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.3277 - acc: 0.9937Epoch 00057: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3272 - acc: 0.9937     \n",
      "Epoch 59/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.3100 - acc: 0.9949Epoch 00058: loss improved from 0.32562 to 0.31079, saving model to GloVeToCategorical/converter-tokenizer-58-0.3108.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3108 - acc: 0.9949     \n",
      "Epoch 60/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.3334 - acc: 0.9930Epoch 00059: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3330 - acc: 0.9930     \n",
      "Epoch 61/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.3355 - acc: 0.9922Epoch 00060: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3349 - acc: 0.9922     \n",
      "Epoch 62/100\n",
      "19800/20000 [============================>.] - ETA: 0s - loss: 0.3224 - acc: 0.9936Epoch 00061: loss did not improve\n",
      "20000/20000 [==============================] - 3s - loss: 0.3214 - acc: 0.9936     \n",
      "Epoch 63/100\n",
      "19900/20000 [============================>.] - ETA: 0s - loss: 0.3003 - acc: 0.9950Epoch 00062: loss improved from 0.31079 to 0.29999, saving model to GloVeToCategorical/converter-tokenizer-62-0.3000.hdf5\n",
      "20000/20000 [==============================] - 3s - loss: 0.3000 - acc: 0.9950     \n",
      "Epoch 64/100\n",
      "  500/20000 [..............................] - ETA: 2s - loss: 0.4484 - acc: 0.9840"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-6337afe668a2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mcallbacks_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_generator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerateData\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataY\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnb_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msamples_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m    933\u001b[0m                                         \u001b[0mnb_worker\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnb_worker\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    934\u001b[0m                                         \u001b[0mpickle_safe\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpickle_safe\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 935\u001b[0;31m                                         initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m    936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    937\u001b[0m     def evaluate_generator(self, generator, val_samples,\n",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch)\u001b[0m\n\u001b[1;32m   1555\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   1556\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1557\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   1558\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1559\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1318\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1320\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1321\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1941\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1942\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 1943\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   1944\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1945\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    935\u001b[0m                 ' to a larger type (e.g. int64).')\n\u001b[1;32m    936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 937\u001b[0;31m           \u001b[0mnp_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubfeed_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubfeed_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    938\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    939\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msubfeed_t\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp_val\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jupyter/anaconda3/envs/py35/lib/python3.5/site-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m    480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m     \"\"\"\n\u001b[0;32m--> 482\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0masanyarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dropOutPercentage = 0.5;\n",
    "model = Sequential();\n",
    "\n",
    "model.add(Dense(10 * wordDimensions, input_dim=wordDimensions, W_regularizer=l2(0.01), activation='relu'));\n",
    "model.add(Dropout(dropOutPercentage));\n",
    "model.add(Dense(len(tokenizer.word_index) + 1, activation='softmax'));\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']);\n",
    "\n",
    "filepath=\"GloVeToCategorical/converter-tokenizer-{epoch:02d}-{loss:.4f}.hdf5\";\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min');\n",
    "callbacks_list = [checkpoint];\n",
    "\n",
    "model.fit_generator(generateData(100, dataX, dataY), nb_epoch=100, samples_per_epoch=20000, callbacks=callbacks_list);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2558587496448308, 0.971199978351593]\n"
     ]
    }
   ],
   "source": [
    "model = Sequential();\n",
    "\n",
    "noisyX = dataX + np.random.normal(0, 0.3, wordDimensions);\n",
    "\n",
    "model.add(Dense(10 * wordDimensions, input_dim=wordDimensions, activation='relu'));\n",
    "model.add(Dense(len(tokenizer.word_index) + 1, activation='softmax'));\n",
    "model.load_weights(\"GloVeToCategorical/converter-tokenizer-0.3000.hdf5\");\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']);\n",
    "f = open(\"GloVeToCategorical/model.json\", \"w\");\n",
    "f.write(model.to_json());\n",
    "f.close();\n",
    "\n",
    "print(model.evaluate_generator(generateData(100, noisyX, dataY), len(dataX)));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'GloVeToCategorical/reverseDic.txt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-748b171236fd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"GloVeToCategorical/reverseDic.txt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mword_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\", \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mw\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'GloVeToCategorical/reverseDic.txt'"
     ]
    }
   ],
   "source": [
    "f = open(\"GloVeToCategorical/reverseDic.txt\", \"w\");\n",
    "for w, i in tokenizer.word_index.items():\n",
    "    f.write(str(i) + \", \" + w + \"\\n\");\n",
    "f.close();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}