7. IMDB Movie Reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:04:18.583991Z",
     "start_time": "2020-06-01T08:03:39.259993Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "import keras\n",
    "from keras.datasets import imdb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyper Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:04:18.592995Z",
     "start_time": "2020-06-01T08:04:18.586995Z"
    }
   },
   "outputs": [],
   "source": [
    "n_unique_words = 5000\n",
    "n_words_to_skip = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:06:25.148142Z",
     "start_time": "2020-06-01T08:05:47.331625Z"
    }
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words, skip_top=n_words_to_skip) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:07:43.924695Z",
     "start_time": "2020-06-01T08:07:43.918700Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 2, 2, 2, 2, 530, 973, 1622, 1385, 65, 458, 4468, 66, 3941, 2, 173, 2, 256, 2, 2]\n"
     ]
    }
   ],
   "source": [
    "print(x_train[0][:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0 reserved for padding\n",
    "### 1 would be starting character\n",
    "### 2 is unknown \n",
    "### 3 is most common word\n",
    "### 4 is second most comman word and so on..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:06.941520Z",
     "start_time": "2020-06-01T08:08:06.931515Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([list([2, 2, 2, 2, 2, 530, 973, 1622, 1385, 65, 458, 4468, 66, 3941, 2, 173, 2, 256, 2, 2, 100, 2, 838, 112, 50, 670, 2, 2, 2, 480, 284, 2, 150, 2, 172, 112, 167, 2, 336, 385, 2, 2, 172, 4536, 1111, 2, 546, 2, 2, 447, 2, 192, 50, 2, 2, 147, 2025, 2, 2, 2, 2, 1920, 4613, 469, 2, 2, 71, 87, 2, 2, 2, 530, 2, 76, 2, 2, 1247, 2, 2, 2, 515, 2, 2, 2, 626, 2, 2, 2, 62, 386, 2, 2, 316, 2, 106, 2, 2, 2223, 2, 2, 480, 66, 3785, 2, 2, 130, 2, 2, 2, 619, 2, 2, 124, 51, 2, 135, 2, 2, 1415, 2, 2, 2, 2, 215, 2, 77, 52, 2, 2, 407, 2, 82, 2, 2, 2, 107, 117, 2, 2, 256, 2, 2, 2, 3766, 2, 723, 2, 71, 2, 530, 476, 2, 400, 317, 2, 2, 2, 2, 1029, 2, 104, 88, 2, 381, 2, 297, 98, 2, 2071, 56, 2, 141, 2, 194, 2, 2, 2, 226, 2, 2, 134, 476, 2, 480, 2, 144, 2, 2, 2, 51, 2, 2, 224, 92, 2, 104, 2, 226, 65, 2, 2, 1334, 88, 2, 2, 283, 2, 2, 4472, 113, 103, 2, 2, 2, 2, 2, 178, 2]),\n",
       "       list([2, 194, 1153, 194, 2, 78, 228, 2, 2, 1463, 4369, 2, 134, 2, 2, 715, 2, 118, 1634, 2, 394, 2, 2, 119, 954, 189, 102, 2, 207, 110, 3103, 2, 2, 69, 188, 2, 2, 2, 2, 2, 249, 126, 93, 2, 114, 2, 2300, 1523, 2, 647, 2, 116, 2, 2, 2, 2, 229, 2, 340, 1322, 2, 118, 2, 2, 130, 4901, 2, 2, 1002, 2, 89, 2, 952, 2, 2, 2, 455, 2, 2, 2, 2, 1543, 1905, 398, 2, 1649, 2, 2, 2, 163, 2, 3215, 2, 2, 1153, 2, 194, 775, 2, 2, 2, 349, 2637, 148, 605, 2, 2, 2, 123, 125, 68, 2, 2, 2, 349, 165, 4362, 98, 2, 2, 228, 2, 2, 2, 1157, 2, 299, 120, 2, 120, 174, 2, 220, 175, 136, 50, 2, 4373, 228, 2, 2, 2, 656, 245, 2350, 2, 2, 2, 131, 152, 491, 2, 2, 2, 2, 1212, 2, 2, 2, 371, 78, 2, 625, 64, 1382, 2, 2, 168, 145, 2, 2, 1690, 2, 2, 2, 1355, 2, 2, 2, 52, 154, 462, 2, 89, 78, 285, 2, 145, 95]),\n",
       "       list([2, 2, 2, 2, 2, 2, 2, 2, 249, 108, 2, 2, 2, 54, 61, 369, 2, 71, 149, 2, 2, 112, 2, 2401, 311, 2, 2, 3711, 2, 75, 2, 1829, 296, 2, 86, 320, 2, 534, 2, 263, 4821, 1301, 2, 1873, 2, 89, 78, 2, 66, 2, 2, 360, 2, 2, 58, 316, 334, 2, 2, 1716, 2, 645, 662, 2, 257, 85, 1200, 2, 1228, 2578, 83, 68, 3912, 2, 2, 165, 1539, 278, 2, 69, 2, 780, 2, 106, 2, 2, 1338, 2, 2, 2, 2, 215, 2, 610, 2, 2, 87, 326, 2, 2300, 2, 2, 2, 2, 272, 2, 57, 2, 2, 2, 2, 2, 2, 2307, 51, 2, 170, 2, 595, 116, 595, 1352, 2, 191, 79, 638, 89, 2, 2, 2, 2, 106, 607, 624, 2, 534, 2, 227, 2, 129, 113]),\n",
       "       list([2, 2, 2, 2, 2, 2804, 2, 2040, 432, 111, 153, 103, 2, 1494, 2, 70, 131, 67, 2, 61, 2, 744, 2, 3715, 761, 61, 2, 452, 2, 2, 985, 2, 2, 59, 166, 2, 105, 216, 1239, 2, 1797, 2, 2, 2, 2, 744, 2413, 2, 2, 2, 687, 2, 2, 2, 2, 2, 3693, 2, 2, 2, 121, 59, 456, 2, 2, 2, 265, 2, 575, 111, 153, 159, 59, 2, 1447, 2, 2, 586, 482, 2, 2, 96, 59, 716, 2, 2, 172, 65, 2, 579, 2, 2, 2, 1615, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 464, 2, 314, 2, 2, 2, 719, 605, 2, 2, 202, 2, 310, 2, 3772, 3501, 2, 2722, 58, 2, 2, 537, 2116, 180, 2, 2, 413, 173, 2, 263, 112, 2, 152, 377, 2, 537, 263, 846, 579, 178, 54, 75, 71, 476, 2, 413, 263, 2504, 182, 2, 2, 75, 2306, 922, 2, 279, 131, 2895, 2, 2867, 2, 2, 2, 921, 2, 192, 2, 1219, 3890, 2, 2, 217, 4122, 1710, 537, 2, 1236, 2, 736, 2, 2, 61, 403, 2, 2, 2, 61, 4494, 2, 2, 4494, 159, 90, 263, 2311, 4319, 309, 2, 178, 2, 82, 4319, 2, 65, 2, 2, 145, 143, 2, 2, 2, 537, 746, 537, 537, 2, 2, 2, 2, 594, 2, 2, 94, 2, 3987, 2, 2, 2, 2, 538, 2, 1795, 246, 2, 2, 2, 2, 635, 2, 2, 51, 408, 2, 94, 318, 1382, 2, 2, 2, 2683, 936, 2, 2, 2, 2, 2, 2, 2, 1885, 2, 1118, 2, 80, 126, 842, 2, 2, 2, 2, 4726, 2, 4494, 2, 1550, 3633, 159, 2, 341, 2, 2733, 2, 4185, 173, 2, 90, 2, 2, 2, 2, 2, 1784, 86, 1117, 2, 3261, 2, 2, 2, 2, 2, 2, 2841, 2, 2, 1010, 2, 793, 2, 2, 1386, 1830, 2, 2, 246, 50, 2, 2, 2750, 1944, 746, 90, 2, 2, 2, 124, 2, 882, 2, 882, 496, 2, 2, 2213, 537, 121, 127, 1219, 130, 2, 2, 494, 2, 124, 2, 882, 496, 2, 341, 2, 2, 846, 2, 2, 2, 2, 1906, 2, 97, 2, 236, 2, 1311, 2, 2, 2, 2, 2, 2, 2, 91, 2, 3987, 70, 2, 882, 2, 579, 2, 2, 2, 2, 2, 537, 2, 2, 2, 2, 65, 2, 537, 75, 2, 1775, 3353, 2, 1846, 2, 2, 2, 154, 2, 2, 518, 53, 2, 2, 2, 3211, 882, 2, 399, 2, 75, 257, 3807, 2, 2, 2, 2, 456, 2, 65, 2, 2, 205, 113, 2, 2, 2, 2, 2, 2, 2, 242, 2, 91, 1202, 2, 2, 2070, 307, 2, 2, 2, 126, 93, 2, 2, 2, 188, 1076, 3222, 2, 2, 2, 2, 2348, 537, 2, 53, 537, 2, 82, 2, 2, 2, 2, 2, 280, 2, 219, 2, 2, 431, 758, 859, 2, 953, 1052, 2, 2, 2, 2, 94, 2, 2, 238, 60, 2, 2, 2, 804, 2, 2, 2, 2, 132, 2, 67, 2, 2, 2, 2, 283, 2, 2, 2, 2, 2, 242, 955, 2, 2, 279, 2, 2, 2, 1685, 195, 2, 238, 60, 796, 2, 2, 671, 2, 2804, 2, 2, 559, 154, 888, 2, 726, 50, 2, 2, 2, 2, 566, 2, 579, 2, 64, 2574]),\n",
       "       list([2, 249, 1323, 2, 61, 113, 2, 2, 2, 1637, 2, 2, 56, 2, 2401, 2, 457, 88, 2, 2626, 1400, 2, 3171, 2, 70, 79, 2, 706, 919, 2, 2, 355, 340, 355, 1696, 96, 143, 2, 2, 2, 289, 2, 61, 369, 71, 2359, 2, 2, 2, 131, 2073, 249, 114, 249, 229, 249, 2, 2, 2, 126, 110, 2, 473, 2, 569, 61, 419, 56, 429, 2, 1513, 2, 2, 534, 95, 474, 570, 2, 2, 124, 138, 88, 2, 421, 1543, 52, 725, 2, 61, 419, 2, 2, 1571, 2, 1543, 2, 2, 2, 2, 2, 296, 2, 3524, 2, 2, 421, 128, 74, 233, 334, 207, 126, 224, 2, 562, 298, 2167, 1272, 2, 2601, 2, 516, 988, 2, 2, 79, 120, 2, 595, 2, 784, 2, 3171, 2, 165, 170, 143, 2, 2, 2, 2, 2, 226, 251, 2, 61, 113]),\n",
       "       list([2, 778, 128, 74, 2, 630, 163, 2, 2, 1766, 2, 1051, 2, 2, 85, 156, 2, 2, 148, 139, 121, 664, 665, 2, 2, 1361, 173, 2, 749, 2, 2, 3804, 2, 2, 226, 65, 2, 2, 127, 2, 2, 2, 2])],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:07.490516Z",
     "start_time": "2020-06-01T08:08:07.484523Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "218\n",
      "189\n",
      "141\n",
      "550\n",
      "147\n",
      "43\n"
     ]
    }
   ],
   "source": [
    "for x in x_train[0:6]:\n",
    "    print(len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:08.219512Z",
     "start_time": "2020-06-01T08:08:08.194544Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, 1, 0, 0], dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:08.786512Z",
     "start_time": "2020-06-01T08:08:08.780515Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 25000)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train), len(x_valid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resoring words from the index "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:22.358077Z",
     "start_time": "2020-06-01T08:08:11.788674Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/text-datasets/imdb_word_index.json\n",
      "1646592/1641221 [==============================] - 8s 5us/step\n"
     ]
    }
   ],
   "source": [
    "word_index = keras.datasets.imdb.get_word_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:22.423072Z",
     "start_time": "2020-06-01T08:08:22.362106Z"
    }
   },
   "outputs": [],
   "source": [
    "word_index = {k:(v+3) for k,v in word_index.items()}\n",
    "word_index[\"PAD\"] = 0\n",
    "word_index[\"START\"] = 1\n",
    "word_index[\"UNK\"] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:22.601077Z",
     "start_time": "2020-06-01T08:08:22.428080Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_index[\"this\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:22.720069Z",
     "start_time": "2020-06-01T08:08:22.613078Z"
    }
   },
   "outputs": [],
   "source": [
    "index_word = {v:k for k,v in word_index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:51.795682Z",
     "start_time": "2020-06-01T08:08:51.788702Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 2, 2, 2, 2, 530, 973, 1622, 1385, 65, 458, 4468, 66, 3941, 2, 173, 2, 256, 2, 2]\n"
     ]
    }
   ],
   "source": [
    "print(x_train[0][:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:08:56.486266Z",
     "start_time": "2020-06-01T08:08:56.478270Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"UNK UNK UNK UNK UNK brilliant casting location scenery story direction everyone's really suited UNK part UNK played UNK UNK could UNK imagine being there robert UNK UNK UNK amazing actor UNK now UNK same being director UNK father came UNK UNK same scottish island UNK myself UNK UNK loved UNK fact there UNK UNK real connection UNK UNK UNK UNK witty remarks throughout UNK UNK were great UNK UNK UNK brilliant UNK much UNK UNK bought UNK UNK UNK soon UNK UNK UNK released UNK UNK UNK would recommend UNK UNK everyone UNK watch UNK UNK fly UNK UNK amazing really cried UNK UNK end UNK UNK UNK sad UNK UNK know what UNK say UNK UNK cry UNK UNK UNK UNK must UNK been good UNK UNK definitely UNK also UNK UNK UNK two little UNK UNK played UNK UNK UNK norman UNK paul UNK were UNK brilliant children UNK often left UNK UNK UNK UNK list UNK think because UNK stars UNK play them UNK grown up UNK such UNK big UNK UNK UNK whole UNK UNK these children UNK amazing UNK should UNK UNK UNK what UNK UNK done don't UNK think UNK whole story UNK UNK lovely because UNK UNK true UNK UNK someone's life after UNK UNK UNK UNK UNK us UNK\""
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(index_word[id] for id in x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:07.433821Z",
     "start_time": "2020-06-01T08:08:58.703820Z"
    }
   },
   "outputs": [],
   "source": [
    "(all_x_train,_),(all_x_valid,_) = imdb.load_data() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:07.448826Z",
     "start_time": "2020-06-01T08:09:07.437821Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"START this film was just brilliant casting location scenery story direction everyone's really suited the part they played and you could just imagine being there robert redford's is an amazing actor and now the same being director norman's father came from the same scottish island as myself so i loved the fact there was a real connection with this film the witty remarks throughout the film were great it was just brilliant so much that i bought the film as soon as it was released for retail and would recommend it to everyone to watch and the fly fishing was amazing really cried at the end it was so sad and you know what they say if you cry at a film it must have been good and this definitely was also congratulations to the two little boy's that played the part's of norman and paul they were just brilliant children are often left out of the praising list i think because the stars that play them all grown up are such a big profile for the whole film but these children are amazing and should be praised for what they have done don't you think the whole story was so lovely because it was true and was someone's life after all that was shared with us all\""
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(index_word[id] for id in all_x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:09.436604Z",
     "start_time": "2020-06-01T08:09:09.432634Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:10.068605Z",
     "start_time": "2020-06-01T08:09:10.064604Z"
    }
   },
   "outputs": [],
   "source": [
    "max_review_length = 100\n",
    "pad_type = trunc_type = 'pre'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:11.937169Z",
     "start_time": "2020-06-01T08:09:11.179062Z"
    }
   },
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:12.556338Z",
     "start_time": "2020-06-01T08:09:12.548342Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1415,    2,    2,    2,    2,  215,    2,   77,   52,    2,    2,\n",
       "         407,    2,   82,    2,    2,    2,  107,  117,    2,    2,  256,\n",
       "           2,    2,    2, 3766,    2,  723,    2,   71,    2,  530,  476,\n",
       "           2,  400,  317,    2,    2,    2,    2, 1029,    2,  104,   88,\n",
       "           2,  381,    2,  297,   98,    2, 2071,   56,    2,  141,    2,\n",
       "         194,    2,    2,    2,  226,    2,    2,  134,  476,    2,  480,\n",
       "           2,  144,    2,    2,    2,   51,    2,    2,  224,   92,    2,\n",
       "         104,    2,  226,   65,    2,    2, 1334,   88,    2,    2,  283,\n",
       "           2,    2, 4472,  113,  103,    2,    2,    2,    2,    2,  178,\n",
       "           2],\n",
       "       [ 163,    2, 3215,    2,    2, 1153,    2,  194,  775,    2,    2,\n",
       "           2,  349, 2637,  148,  605,    2,    2,    2,  123,  125,   68,\n",
       "           2,    2,    2,  349,  165, 4362,   98,    2,    2,  228,    2,\n",
       "           2,    2, 1157,    2,  299,  120,    2,  120,  174,    2,  220,\n",
       "         175,  136,   50,    2, 4373,  228,    2,    2,    2,  656,  245,\n",
       "        2350,    2,    2,    2,  131,  152,  491,    2,    2,    2,    2,\n",
       "        1212,    2,    2,    2,  371,   78,    2,  625,   64, 1382,    2,\n",
       "           2,  168,  145,    2,    2, 1690,    2,    2,    2, 1355,    2,\n",
       "           2,    2,   52,  154,  462,    2,   89,   78,  285,    2,  145,\n",
       "          95],\n",
       "       [1301,    2, 1873,    2,   89,   78,    2,   66,    2,    2,  360,\n",
       "           2,    2,   58,  316,  334,    2,    2, 1716,    2,  645,  662,\n",
       "           2,  257,   85, 1200,    2, 1228, 2578,   83,   68, 3912,    2,\n",
       "           2,  165, 1539,  278,    2,   69,    2,  780,    2,  106,    2,\n",
       "           2, 1338,    2,    2,    2,    2,  215,    2,  610,    2,    2,\n",
       "          87,  326,    2, 2300,    2,    2,    2,    2,  272,    2,   57,\n",
       "           2,    2,    2,    2,    2,    2, 2307,   51,    2,  170,    2,\n",
       "         595,  116,  595, 1352,    2,  191,   79,  638,   89,    2,    2,\n",
       "           2,    2,  106,  607,  624,    2,  534,    2,  227,    2,  129,\n",
       "         113],\n",
       "       [   2,    2,    2,  188, 1076, 3222,    2,    2,    2,    2, 2348,\n",
       "         537,    2,   53,  537,    2,   82,    2,    2,    2,    2,    2,\n",
       "         280,    2,  219,    2,    2,  431,  758,  859,    2,  953, 1052,\n",
       "           2,    2,    2,    2,   94,    2,    2,  238,   60,    2,    2,\n",
       "           2,  804,    2,    2,    2,    2,  132,    2,   67,    2,    2,\n",
       "           2,    2,  283,    2,    2,    2,    2,    2,  242,  955,    2,\n",
       "           2,  279,    2,    2,    2, 1685,  195,    2,  238,   60,  796,\n",
       "           2,    2,  671,    2, 2804,    2,    2,  559,  154,  888,    2,\n",
       "         726,   50,    2,    2,    2,    2,  566,    2,  579,    2,   64,\n",
       "        2574],\n",
       "       [   2,    2,  131, 2073,  249,  114,  249,  229,  249,    2,    2,\n",
       "           2,  126,  110,    2,  473,    2,  569,   61,  419,   56,  429,\n",
       "           2, 1513,    2,    2,  534,   95,  474,  570,    2,    2,  124,\n",
       "         138,   88,    2,  421, 1543,   52,  725,    2,   61,  419,    2,\n",
       "           2, 1571,    2, 1543,    2,    2,    2,    2,    2,  296,    2,\n",
       "        3524,    2,    2,  421,  128,   74,  233,  334,  207,  126,  224,\n",
       "           2,  562,  298, 2167, 1272,    2, 2601,    2,  516,  988,    2,\n",
       "           2,   79,  120,    2,  595,    2,  784,    2, 3171,    2,  165,\n",
       "         170,  143,    2,    2,    2,    2,    2,  226,  251,    2,   61,\n",
       "         113],\n",
       "       [   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    2,  778,  128,   74,    2,  630,  163,    2,    2,\n",
       "        1766,    2, 1051,    2,    2,   85,  156,    2,    2,  148,  139,\n",
       "         121,  664,  665,    2,    2, 1361,  173,    2,  749,    2,    2,\n",
       "        3804,    2,    2,  226,   65,    2,    2,  127,    2,    2,    2,\n",
       "           2]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[0:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:15.399102Z",
     "start_time": "2020-06-01T08:09:15.391102Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "for x in x_train[0:6]:\n",
    "    print(len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:16.011472Z",
     "start_time": "2020-06-01T08:09:16.003468Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD PAD UNK begins better than UNK ends funny UNK UNK russian UNK crew UNK UNK other actors UNK UNK those scenes where documentary shots UNK UNK spoiler part UNK message UNK UNK contrary UNK UNK whole story UNK UNK does UNK UNK UNK UNK'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(index_word[id] for id in x_train[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:19.487297Z",
     "start_time": "2020-06-01T08:09:19.482303Z"
    }
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, Dropout\n",
    "from keras.layers import Embedding\n",
    "from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:20.285044Z",
     "start_time": "2020-06-01T08:09:20.281076Z"
    }
   },
   "outputs": [],
   "source": [
    "# vector-space embedding: \n",
    "n_dim = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:21.722652Z",
     "start_time": "2020-06-01T08:09:21.603563Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Admin\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\ops\\resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid')) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:23.054936Z",
     "start_time": "2020-06-01T08:09:23.046972Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 100, 64)           320000    \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 6400)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                409664    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 729,729\n",
      "Trainable params: 729,729\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:23.931964Z",
     "start_time": "2020-06-01T08:09:23.845938Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Admin\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\ops\\nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:25.045327Z",
     "start_time": "2020-06-01T08:09:25.041326Z"
    }
   },
   "outputs": [],
   "source": [
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:56.690684Z",
     "start_time": "2020-06-01T08:09:25.893768Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Admin\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 7s 284us/step - loss: 0.5021 - accuracy: 0.7342 - val_loss: 0.3432 - val_accuracy: 0.8470\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 8s 314us/step - loss: 0.2340 - accuracy: 0.9095 - val_loss: 0.3702 - val_accuracy: 0.8368\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 7s 286us/step - loss: 0.0623 - accuracy: 0.9856 - val_loss: 0.4650 - val_accuracy: 0.8294\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 8s 324us/step - loss: 0.0092 - accuracy: 0.9995 - val_loss: 0.5470 - val_accuracy: 0.8314\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x1db9e819848>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction Phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:57.697706Z",
     "start_time": "2020-06-01T08:09:56.694686Z"
    }
   },
   "outputs": [],
   "source": [
    "y_hat = model.predict_proba(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:57.716686Z",
     "start_time": "2020-06-01T08:09:57.700684Z"
    }
   },
   "outputs": [],
   "source": [
    "float_y_hat = []\n",
    "for y in y_hat:\n",
    "    float_y_hat.append(y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:09:59.338979Z",
     "start_time": "2020-06-01T08:09:57.719686Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:10:00.235987Z",
     "start_time": "2020-06-01T08:09:59.346990Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_hat</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.048598</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.999477</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.945471</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.077232</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.999955</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.926118</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.997006</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.999247</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.886729</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      y_hat  y\n",
       "0  0.048598  0\n",
       "1  0.999477  1\n",
       "2  0.945471  1\n",
       "3  0.077232  0\n",
       "4  0.999955  1\n",
       "5  0.926118  1\n",
       "6  0.997006  1\n",
       "7  0.000131  0\n",
       "8  0.999247  0\n",
       "9  0.886729  1"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ydf = pd.DataFrame(list(zip(float_y_hat, y_valid)), columns=['y_hat', 'y'])\n",
    "ydf.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:10:00.308980Z",
     "start_time": "2020-06-01T08:10:00.238981Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_hat</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.999247</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66</td>\n",
       "      <td>0.973532</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>0.995297</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78</td>\n",
       "      <td>0.944850</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>112</td>\n",
       "      <td>0.975500</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>115</td>\n",
       "      <td>0.935584</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>139</td>\n",
       "      <td>0.993937</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>152</td>\n",
       "      <td>0.999616</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>256</td>\n",
       "      <td>0.993960</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299</td>\n",
       "      <td>0.975120</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        y_hat  y\n",
       "8    0.999247  0\n",
       "66   0.973532  0\n",
       "75   0.995297  0\n",
       "78   0.944850  0\n",
       "112  0.975500  0\n",
       "115  0.935584  0\n",
       "139  0.993937  0\n",
       "152  0.999616  0\n",
       "256  0.993960  0\n",
       "299  0.975120  0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ydf[(ydf.y == 0) & (ydf.y_hat > 0.9)].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-01T08:10:00.392983Z",
     "start_time": "2020-06-01T08:10:00.314017Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"START where do i begin with the killing mind before i mention the good bit this movie is about a young psychological profiling fbi woman or something that for some reason goes to work for the lapd for a wee while there are some recognisable faces like the i love you man bloke from wayne's world playing a cop nice beard age too and two guys who always seem to play cops playing you guessed it cops one of them was one of the cops in gross point blank following john cusack about the place and the other is the fbi guy with specs in final destination who is also in the fugitive as another cop i know the fbi us marshalls cia etc aren't cops but they're all the same they enforce the law to a certain degree that makes them cops in my book feel free to disagree with the definition of a cop all you want they're still seen as the ones who are trying to bring the bad guys down any i digress the woman is working alongside these cops who just seem to sit around in a library in the basement or something and for some unknown reason decides to reopen a case that hasn't been looked at for she saw the dead body as a kid so naturally decided that she can reopen the case cos she was personally attached to it i also like how the person profiling other people's psychological state witnessed a murder scene as a child surely that's the kind of thing that can screw with someone's head she starts asking questions off some journalist who covered the story when it was open even though he appears to be the same age as the woman so he presumably was either writing for a newspaper when he was 10 or he wrote a few school reports on it and she decided he'd be the best person to ask about it br br we then see that as well as reopening homicide cases from 25 years earlier she tries her hand at petty theft as some really inconspicuous bloke through a busy street wearing a balaclava way to blend moron anyway she and the cops she's with eating ice creams in the park give chase after him and whilst crossing a bridge with a woman and child standing on it the thief grabs the baby from the mother's arms and throws the baby in the river this if you haven't already guessed is the best bit of the movie i'm not sadistic and don't have a hatred for babies or anything but the scene just looks so stupid i promise you will wet your pants laughing at it if you should watch this movie it gets better when the cop from final destination the fugitive jumps in to rescue the baby then declares to his friend it's a baby how did he not know that before he jumped in is beyond me br br anyway she chases after the thief but loses him later on she remembers that a random dog didn't bark at the thief so assumed it must be his and decided to track down the thief through the dog the dog had no markings or anything on it so how she knew where to find the dog and thus find the thief is another question i won't go into it was shortly after this that i stopped watching the story was moving really slowly and after i had stopped laughing at the baby throwing bit i had kind of lost interest and missed what was being said br br saying that i'll go ahead and assume that the killer was the journalist bloke because he seemed a bit shifty and she was spending a lot of time with him and anyone who watches columbo knows that the person outside the police force who spends most time with the investigating officer is your bad guy plus other people's reviews say the killer was pretty obvious so i'm sure it was him br br so all in all a pants film but worth watching just for that one glorious scene\""
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(index_word[id] for id in all_x_valid[260]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}