Sentiment_Training_with_IMDB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "from sklearn.model_selection import train_test_split\n",
    "import re\n",
    "from time import time\n",
    "\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.models import Sequential, model_from_json\n",
    "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Flatten\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load preprocessed data.\n",
    "wordsList = np.load('wordIndex_all.npy')\n",
    "wordsList = wordsList.tolist() #Originally loaded as numpy array\n",
    "\n",
    "#ids_p = np.load('idsMatrix_pos.npy')\n",
    "#ids_n = np.load('idsMatrix_neg.npy')\n",
    "\n",
    "ids = np.load('idsMatrix_all.npy')\n",
    "\n",
    "# Reverse from integers to words using the DICTIONARY\n",
    "reverse_word_index = dict(\n",
    "[(value, key) for (key, value) in wordsList.items()])\n",
    "\n",
    "\n",
    "X = ids[:,:250]   #word index\n",
    "y = ids[:,250:]   #carwgory label\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this a good episode of the new twilight zone that actually includes interesting ideas and clever stories i note both of them are based on short stories examination day is set in the future year unknown but at a point where they have cake candles that light themselves huge tv looking phones that double as numerous other entertaining machines and distributed only to those of a certain age and the examination day a point where 12 year olds must undergo a government required iq test the kid is this story dickie jordan david UNK is just celebrating his own 12th birthday and is a smart kid so is calm even eager to take the test that he has seen friends pass easily and knows he will excel at based on his school grades his parents christopher UNK and elizabeth UNK on the other hand say he shouldn't have used his birthday wish on getting a good score and while their reason includes that they believe he's capable and he should have no need to worry it's pretty obvious they are worried i won't give anything away in the ending but i will say this there's a point where we get a glimpse of what's to come as far ass why the test is such a heavy subject that evening or another his parents ask dickie whether he'd prefer to watch tv all night by today's standards we'd be pleased he'd say he'd rather read and not just because there's nothing UNK ?\n"
     ]
    }
   ],
   "source": [
    "# decode word ids to text for a given pretrained id sequence\n",
    "decoded_review = ' '.join(\n",
    "[reverse_word_index.get(i, '?') for i in ids[300]])\n",
    "\n",
    "print(decoded_review)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define network parameters\n",
    "\n",
    "MAX_NUM_WORDS = 40000 # only use top n words\n",
    "numDimensions = 50 #Dimensions for each word vector\n",
    "maxSeqLength = X.shape[1] #Maximum number of words for each document\n",
    "\n",
    "batchSize = 24\n",
    "lstmUnits = 64\n",
    "numClasses = 2\n",
    "iterations = 100000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# create the Multi-Layer Perceptron Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 250, 50)           2000000   \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 12500)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                800064    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 2,800,129\n",
      "Trainable params: 2,800,129\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 16750 samples, validate on 8250 samples\n",
      "Epoch 1/2\n",
      " - 4s - loss: 0.5374 - acc: 0.7066 - val_loss: 0.3551 - val_acc: 0.8410\n",
      "Epoch 2/2\n",
      " - 3s - loss: 0.1341 - acc: 0.9533 - val_loss: 0.3457 - val_acc: 0.8553\n",
      "Accuracy: 85.53%\n"
     ]
    }
   ],
   "source": [
    "# create the Multi-Layer Perceptron Model for binary classification\n",
    "model = Sequential()\n",
    "model.add(Embedding(MAX_NUM_WORDS, numDimensions, input_length = maxSeqLength))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(lstmUnits, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "\n",
    "# Fit the Multi-Layer Perceptron Model\n",
    "model.fit(X_train, y_train[:,0], validation_data=(X_test, y_test[:,0]), epochs=2, batch_size=128, verbose=2)\n",
    "# Final evaluation of the model\n",
    "loss, acc = model.evaluate(X_test, y_test[:,0], verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 250, 50)           2000000   \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 12500)             0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 64)                800064    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 2)                 130       \n",
      "=================================================================\n",
      "Total params: 2,800,194\n",
      "Trainable params: 2,800,194\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train on 16750 samples, validate on 8250 samples\n",
      "Epoch 1/10\n",
      " - 3s - loss: 0.5495 - acc: 0.6947 - val_loss: 0.3533 - val_acc: 0.8464\n",
      "Epoch 2/10\n",
      " - 3s - loss: 0.1298 - acc: 0.9543 - val_loss: 0.3650 - val_acc: 0.8543\n",
      "Epoch 3/10\n",
      " - 3s - loss: 0.0134 - acc: 0.9985 - val_loss: 0.3991 - val_acc: 0.8572\n",
      "Epoch 4/10\n",
      " - 3s - loss: 0.0028 - acc: 0.9999 - val_loss: 0.4290 - val_acc: 0.8594\n",
      "Epoch 5/10\n",
      " - 3s - loss: 0.0012 - acc: 1.0000 - val_loss: 0.4550 - val_acc: 0.8585\n",
      "Epoch 6/10\n",
      " - 3s - loss: 7.0491e-04 - acc: 1.0000 - val_loss: 0.4705 - val_acc: 0.8607\n",
      "Epoch 7/10\n",
      " - 3s - loss: 4.5570e-04 - acc: 1.0000 - val_loss: 0.4854 - val_acc: 0.8593\n",
      "Epoch 8/10\n",
      " - 3s - loss: 3.2033e-04 - acc: 1.0000 - val_loss: 0.5006 - val_acc: 0.8601\n",
      "Epoch 9/10\n",
      " - 3s - loss: 2.3209e-04 - acc: 1.0000 - val_loss: 0.5141 - val_acc: 0.8601\n",
      "Epoch 10/10\n",
      " - 3s - loss: 1.7630e-04 - acc: 1.0000 - val_loss: 0.5250 - val_acc: 0.8605\n",
      "Accuracy: 86.05%\n"
     ]
    }
   ],
   "source": [
    "# create the Multi-Layer Perceptron Model for two-classes classification\n",
    "model = Sequential()\n",
    "model.add(Embedding(MAX_NUM_WORDS, numDimensions, input_length = maxSeqLength))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(lstmUnits, activation='relu'))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "print(model.summary())\n",
    "\n",
    "# Fit the Multi-Layer Perceptron Model\n",
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=10, batch_size=128, verbose=2)\n",
    "# Final evaluation of the model\n",
    "loss, acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One-Dimensional Convolutional Neural Network Model for the IMDB Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create the CNN 1D model\n",
    "model = Sequential()\n",
    "model.add(Embedding(MAX_NUM_WORDS, numDimensions, input_length=maxSeqLength))\n",
    "model.add(Conv1D(filters=numDimensions, kernel_size=3, padding='same', activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(250, activation='relu'))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.fit(X_train, y_train[:,0], validation_data=(X_test, y_test[:,0]), epochs=2, batch_size=128, verbose=2)\n",
    "# Final evaluation of the model\n",
    "loss, acc = model.evaluate(X_test, y_test[:,0], verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 250, 50)           2000000   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 64)                29440     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 2)                 130       \n",
      "=================================================================\n",
      "Total params: 2,029,570\n",
      "Trainable params: 2,029,570\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential() \n",
    "model.add(Embedding(MAX_NUM_WORDS, numDimensions, input_length=maxSeqLength))  \n",
    "# number of weights of embeeding is MAX_NUM_WORDS*numDimensions=40000*50=2000000\n",
    "model.add(LSTM(lstmUnits, recurrent_dropout=0.25)) \n",
    "# number of weights of LSTM is (lstmUnits*(numDimensions+lstmUnits) + lstmUnits)*4(number of gates)\n",
    "# = (64*(50+64)+64)*4=29940\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "# number of weights of Classifer is (lstmUnits*2+2(numofClasses)=64*2+2=130\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "print(model.summary()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# define callback list\n",
    "\n",
    "tensorboard = TensorBoard(log_dir=\"logs/{}\".format(time()))\n",
    "\n",
    "# checkpoint\n",
    "filepath=\"weights-improvement-{epoch:02d}-{val_acc:.2f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
    "\n",
    "callbacks_list = [checkpoint, tensorboard]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=100, batch_size=128, callbacks=callbacks_list, verbose=2)\n",
    "# Final evaluation of the model\n",
    "loss, acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"Accuracy: %.2f%%\" % (acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save/Load model and weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "model = load_model('weights-improvement-86-0.83.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 250, 50)           2000000   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 64)                29440     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 130       \n",
      "=================================================================\n",
      "Total params: 2,029,570\n",
      "Trainable params: 2,029,570\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# access the embedding layer through the constructed model \n",
    "# first `0` refers to the position of embedding layer in the `model`\n",
    "embeddings = model.layers[0].get_weights()[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# apply loaded model to test data\n",
    "loss, acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"%s: %.2f%%\" % (model.metrics_names[1], acc*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save model from current model instance \n",
    "# serialize model to JSON\n",
    "model_json = model.to_json()\n",
    "with open(\"IMDB_Sentiment_model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "# serialize weights to HDF5\n",
    "model.save_weights(\"IMDB_Sentiment_weights.h5\")\n",
    "print(\"Saved model to disk\")\n",
    " loss, acc = model.evaluate(X_test, y_test, verbose=0)loss, acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "\n",
    "# load json and create model\n",
    "json_file = open('IMDB_Sentiment_model.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "\n",
    "loaded_model = model_from_json(loaded_model_json)\n",
    "# load weights into new model\n",
    "loaded_model.loadprint(\"%s: %.2f%%\" % (model.metrics_names[1], acc*100))_weights(\"IMDB_Sentiment_weights.h5\")\n",
    "print(\"Loaded model from disk\")\n",
    " \n",
    "# evaluate loaded model on test data\n",
    "loaded_model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
    "loss, acc = loaded_model.evaluate(X_test, y_test, verbose=0)\n",
    "print(\"%s: %.2f%%\" % (loaded_model.metrics_names[1], acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict input doc with pre-trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "maxSeqLength = 250 #Maximum number of words for each document\n",
    "\n",
    "#inputText = \"That movie was terrible.\"\n",
    "inputText = \"That movie was the best one I have ever seen.\"\n",
    "\n",
    "# split to words\n",
    "words = text_to_word_sequence(inputText, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n', lower=True, split=' ')\n",
    "\n",
    "#convert word sequence to ids\n",
    "sentenceMatrix = np.zeros([1,maxSeqLength], dtype='int32')\n",
    "for indexCounter,word in enumerate(words):\n",
    "        try:\n",
    "            sentenceMatrix[0][indexCounter] = wordsList[word]\n",
    "        except KeyError:\n",
    "            sentenceMatrix[0][indexCounter] = 1 #ID for unkown words \"UNK\"\n",
    "\n",
    "# predict sentiment scores for input text\n",
    "predictedSentiment = model.predict(sentenceMatrix)\n",
    "\n",
    "print(predictedSentiment) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}