9 - Parallel - LSTM-CONV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parallel LSTM-Convolutional\n",
    "### feed the ratings to one model\n",
    "### feed the word vectors to another network\n",
    "### concatenate the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# data analysis pkg\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Word vectorization and splitting dataset\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# Building sequential model\n",
    "from keras.models import Model\n",
    "from keras.layers import SpatialDropout1D, Conv1D, GlobalMaxPooling1D\n",
    "from keras.layers import Embedding, Dense, Dropout\n",
    "from keras.layers import Input, concatenate\n",
    "\n",
    "# Cofigurations\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "# import metrics\n",
    "from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, recall_score\n",
    "\n",
    "\n",
    "# Avoiding warnings\n",
    "import warnings\n",
    "import os\n",
    "########### Prevent Warnings ###########\n",
    "warnings.filterwarnings(action='ignore')\n",
    "########### Prevent Warnings ###########"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/cleaned/data.csv')\n",
    "df.title = df.title.astype('str')\n",
    "df.text = df.text.astype('str')\n",
    "df.comment = df.comment.astype('str')\n",
    "df.rate = df.rate.astype('category')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting features and Splitting dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# features = df.columns.tolist()[:-1]\n",
    "features = ['text']\n",
    "label = ['verification_status']\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(df[features], df[label], test_size=0.3)\n",
    "\n",
    "X_train = [text for text in X_train.text]\n",
    "Y_train = [label for label in Y_train.verification_status]\n",
    "\n",
    "X_test = [text for text in X_test.text]\n",
    "Y_test = [label for label in Y_test.verification_status]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set hypermeter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir='model_output/parallel_conv/'\n",
    "\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "n_dim = 64\n",
    "n_unique_words = 5000\n",
    "max_text_length = 500\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2\n",
    "\n",
    "n_conv_1 = n_conv_2 = n_conv_3 = 64\n",
    "k_conv_1 = 3\n",
    "k_conv_2 = 2\n",
    "k_conv_3 = 4\n",
    "\n",
    "n_dense = 256\n",
    "dropout = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Option 1 - Tokenizing and convert to sequences using keras tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok = Tokenizer(num_words=n_unique_words, lower=False)\n",
    "tok.fit_on_texts(X_train)\n",
    "\n",
    "train_sequences = tok.texts_to_sequences(X_train)\n",
    "train_sequences_matrix = pad_sequences(train_sequences, maxlen=max_text_length, padding=pad_type)\n",
    "\n",
    "test_sequences = tok.texts_to_sequences(X_test)\n",
    "test_sequences_matrix = pad_sequences(test_sequences, maxlen=max_text_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Option 2 - Tokenizing and convert to sequences using sklearn tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tfidf = TfidfVectorizer(use_idf=True, max_features=n_unique_words, tokenizer=word_tokenize)\n",
    "# train_sequences_matrix = tfidf.fit_transform(X_train)\n",
    "# test_sequences_matrix = tfidf.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125369, 500)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sequences_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Designing Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ParallelConv():\n",
    "    input_layer = Input(shape=(max_text_length,), dtype='int16', name='input')\n",
    "    \n",
    "    embedding_layer = Embedding(n_unique_words, n_dim, input_length=max_text_length, name='embedding')(input_layer)\n",
    "    drop_embedding_layer = SpatialDropout1D(drop_embed, name='drop_embed')(embedding_layer)\n",
    "    \n",
    "    conv_1 = Conv1D(n_conv_1, k_conv_1, activation='relu', name='conv_1')(drop_embedding_layer)\n",
    "    maxp_1 = GlobalMaxPooling1D(name='maxp_1')(conv_1)\n",
    "    \n",
    "    conv_2 = Conv1D(n_conv_2, k_conv_2, activation='relu', name='conv_2')(drop_embedding_layer)\n",
    "    maxp_2 = GlobalMaxPooling1D(name='maxp_2')(conv_2)\n",
    "    \n",
    "    conv_3 = Conv1D(n_conv_3, k_conv_3, activation='relu', name='conv_3')(drop_embedding_layer)\n",
    "    maxp_3 = GlobalMaxPooling1D(name='maxp_3')(conv_3)\n",
    "    \n",
    "    concat = concatenate([maxp_1, maxp_2, maxp_3])\n",
    "    \n",
    "    \n",
    "    dense_layer = Dense(n_dense, activation='relu', name='dense')(concat)\n",
    "    drop_dense_layer = Dropout(dropout, name='drop_dense')(dense_layer)\n",
    "    \n",
    "    dense_layer_2 = Dense(n_dense, activation='relu', name='dense_2')(drop_dense_layer)\n",
    "    drop_dense_layer_2 = Dropout(dropout, name='drop_dense_2')(dense_layer_2)\n",
    "    \n",
    "    predictions = Dense(1, activation='sigmoid', name='output')(drop_dense_layer_2)\n",
    "    \n",
    "    model = Model(input_layer, predictions)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input (InputLayer)              (None, 500)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 500, 64)      320000      input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "drop_embed (SpatialDropout1D)   (None, 500, 64)      0           embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv_1 (Conv1D)                 (None, 498, 64)      12352       drop_embed[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_2 (Conv1D)                 (None, 499, 64)      8256        drop_embed[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv_3 (Conv1D)                 (None, 497, 64)      16448       drop_embed[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "maxp_1 (GlobalMaxPooling1D)     (None, 64)           0           conv_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "maxp_2 (GlobalMaxPooling1D)     (None, 64)           0           conv_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "maxp_3 (GlobalMaxPooling1D)     (None, 64)           0           conv_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 192)          0           maxp_1[0][0]                     \n",
      "                                                                 maxp_2[0][0]                     \n",
      "                                                                 maxp_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256)          49408       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "drop_dense (Dropout)            (None, 256)          0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          65792       drop_dense[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "drop_dense_2 (Dropout)          (None, 256)          0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 1)            257         drop_dense_2[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 472,513\n",
      "Trainable params: 472,513\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = ParallelConv()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configure Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
    "model_ckpt = ModelCheckpoint(filepath=output_dir+'weights.{epoch:02d}.hdf5')\n",
    "\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 125369 samples, validate on 53730 samples\n",
      "Epoch 1/4\n",
      " 61056/125369 [=============>................] - ETA: 1:42 - loss: 12.7955 - accuracy: 0.1655"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-d77c03d096f4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m           \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m           validation_data=(test_sequences_matrix, Y_test), callbacks=[model_ckpt])\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1237\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[1;32m   1240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     def evaluate(self,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[1;32m    194\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3738\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3739\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3740\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3741\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3742\u001b[0m     \u001b[0;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1079\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1080\u001b[0m     \"\"\"\n\u001b[0;32m-> 1081\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1082\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1083\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1119\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[1;32m   1120\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[0;32m-> 1121\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1222\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mexecuting_eagerly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1223\u001b[0m       flat_outputs = forward_function.call(\n\u001b[0;32m-> 1224\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1225\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1226\u001b[0m       \u001b[0mgradient_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delayed_rewrite_functions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    509\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 511\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    512\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(train_sequences_matrix, Y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(test_sequences_matrix, Y_test), callbacks=[model_ckpt])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+'weights.02.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat = model.predict(test_sequences_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQKklEQVR4nO3cf6zddX3H8efLVpRsw1YphLR1ZfG6WM1U7KCZyaJgSluXlSWwlDjpSLMmDI1bzCbuj7GhJPrPMCSo6aShNdNK8AeNK+s6fsRsUuQyFCiM9VoZ3JTYq62IIerK3vvjfLqclXN7z23vPae3fT6Sk/P9vj+f7/d8Pu1tX/f743xTVUiSzmyvGvYAJEnDZxhIkgwDSZJhIEnCMJAkAfOHPYATde6559ayZcumv+FPn+68n/ObMzoeSTrVPfLIIz+qqkW92uZsGCxbtozR0dHpb/gv7+m8v++BmRyOJJ3ykvzXZG2eJpIkGQaSJMNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEnP4G8jSqWrZDf84lM995lPvH8rn6vTgkYEkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEn0GQZJnknyeJLvJhlttdcn2Z1kX3tf2OpJcmuSsSSPJbmoaz8bWv99STZ01d/V9j/Wts1MT1SSNLnpHBm8t6reUVUr2voNwL1VNQLc29YB1gAj7bUJ+Bx0wgO4EbgEuBi48WiAtD6burZbfcIzkiRN28mcJloHbG3LW4EruurbqmMPsCDJBcDlwO6qOlRVh4HdwOrWdk5VPVhVBWzr2pckaQD6DYMC/jnJI0k2tdr5VfU8QHs/r9UXA891bTveaserj/eov0KSTUlGk4xOTEz0OXRJ0lT6fYT1u6vqQJLzgN1J/uM4fXud768TqL+yWLUZ2AywYsWKnn0kSdPX15FBVR1o7weBr9M55//DdoqH9n6wdR8HlnZtvgQ4MEV9SY+6JGlApgyDJL+S5NeOLgOrgCeAHcDRO4I2AHe35R3ANe2uopXAC+000i5gVZKF7cLxKmBXa3sxycp2F9E1XfuSJA1AP6eJzge+3u72nA98qar+KcnDwJ1JNgLPAle1/juBtcAY8BJwLUBVHUryCeDh1u+mqjrUlq8D7gDOBu5pL0nSgEwZBlW1H3h7j/qPgct61Au4fpJ9bQG29KiPAm/rY7ySpFngN5AlSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSUwjDJLMS/Jokm+29QuTPJRkX5KvJDmr1V/T1sda+7KufXy81Z9OcnlXfXWrjSW5YeamJ0nqx3SODD4CPNW1/mnglqoaAQ4DG1t9I3C4qt4E3NL6kWQ5sB54K7Aa+GwLmHnAbcAaYDlwdesrSRqQvsIgyRLg/cAX2nqAS4G7WpetwBVteV1bp7Vf1vqvA7ZX1S+q6gfAGHBxe41V1f6q+iWwvfWVJA1Iv0cGnwH+Eviftv4G4CdVdaStjwOL2/Ji4DmA1v5C6/9/9WO2mawuSRqQKcMgye8BB6vqke5yj641Rdt0673GsinJaJLRiYmJ44xakjQd/RwZvBv4/STP0DmFcymdI4UFSea3PkuAA215HFgK0NpfBxzqrh+zzWT1V6iqzVW1oqpWLFq0qI+hS5L6MWUYVNXHq2pJVS2jcwH4vqr6AHA/cGXrtgG4uy3vaOu09vuqqlp9fbvb6EJgBPgO8DAw0u5OOqt9xo4ZmZ0kqS/zp+4yqY8B25N8EngUuL3Vbwe+mGSMzhHBeoCq2pvkTuBJ4AhwfVW9DJDkQ8AuYB6wpar2nsS4JEnTNK0wqKoHgAfa8n46dwId2+fnwFWTbH8zcHOP+k5g53TGIkmaOX4DWZJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJoo8wSPLaJN9J8r0ke5P8batfmOShJPuSfCXJWa3+mrY+1tqXde3r463+dJLLu+qrW20syQ0zP01J0vH0c2TwC+DSqno78A5gdZKVwKeBW6pqBDgMbGz9NwKHq+pNwC2tH0mWA+uBtwKrgc8mmZdkHnAbsAZYDlzd+kqSBmTKMKiOn7XVV7dXAZcCd7X6VuCKtryurdPaL0uSVt9eVb+oqh8AY8DF7TVWVfur6pfA9tZXkjQgfV0zaL/Bfxc4COwGvg/8pKqOtC7jwOK2vBh4DqC1vwC8obt+zDaT1XuNY1OS0SSjExMT/QxdktSHvsKgql6uqncAS+j8Jv+WXt3aeyZpm2691zg2V9WKqlqxaNGiqQcuSerLtO4mqqqfAA8AK4EFSea3piXAgbY8DiwFaO2vAw5114/ZZrK6JGlA+rmbaFGSBW35bOB9wFPA/cCVrdsG4O62vKOt09rvq6pq9fXtbqMLgRHgO8DDwEi7O+ksOheZd8zE5CRJ/Zk/dRcuALa2u35eBdxZVd9M8iSwPckngUeB21v/24EvJhmjc0SwHqCq9ia5E3gSOAJcX1UvAyT5ELALmAdsqaq9MzZDSdKUpgyDqnoMeGeP+n461w+Orf8cuGqSfd0M3NyjvhPY2cd4JUmzwG8gS5IMA0mSYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ9BEGSZYmuT/JU0n2JvlIq78+ye4k+9r7wlZPkluTjCV5LMlFXfva0PrvS7Khq/6uJI+3bW5NktmYrCSpt36ODI4AH62qtwArgeuTLAduAO6tqhHg3rYOsAYYaa9NwOegEx7AjcAlwMXAjUcDpPXZ1LXd6pOfmiSpX1OGQVU9X1X/3pZfBJ4CFgPrgK2t21bgira8DthWHXuABUkuAC4HdlfVoao6DOwGVre2c6rqwaoqYFvXviRJAzCtawZJlgHvBB4Czq+q56ETGMB5rdti4LmuzcZb7Xj18R71Xp+/KcloktGJiYnpDF2SdBx9h0GSXwW+CvxZVf30eF171OoE6q8sVm2uqhVVtWLRokVTDVmS1Ke+wiDJq+kEwT9U1dda+YftFA/t/WCrjwNLuzZfAhyYor6kR12SNCD93E0U4Hbgqar6u66mHcDRO4I2AHd31a9pdxWtBF5op5F2AauSLGwXjlcBu1rbi0lWts+6pmtfkqQBmN9Hn3cDHwQeT/LdVvsr4FPAnUk2As8CV7W2ncBaYAx4CbgWoKoOJfkE8HDrd1NVHWrL1wF3AGcD97SXJGlApgyDqvpXep/XB7isR/8Crp9kX1uALT3qo8DbphqLJGl2+A1kSZJhIEkyDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRJ9hEGSLUkOJnmiq/b6JLuT7GvvC1s9SW5NMpbksSQXdW2zofXfl2RDV/1dSR5v29yaJDM9SUnS8fVzZHAHsPqY2g3AvVU1Atzb1gHWACPttQn4HHTCA7gRuAS4GLjxaIC0Ppu6tjv2syRJs2zKMKiqbwGHjimvA7a25a3AFV31bdWxB1iQ5ALgcmB3VR2qqsPAbmB1azunqh6sqgK2de1LkjQgJ3rN4Pyqeh6gvZ/X6ouB57r6jbfa8erjPeo9JdmUZDTJ6MTExAkOXZJ0rJm+gNzrfH+dQL2nqtpcVSuqasWiRYtOcIiSpGOdaBj8sJ3iob0fbPVxYGlXvyXAgSnqS3rUJUkDdKJhsAM4ekfQBuDurvo17a6ilcAL7TTSLmBVkoXtwvEqYFdrezHJynYX0TVd+5IkDcj8qTok+TLwHuDcJON07gr6FHBnko3As8BVrftOYC0wBrwEXAtQVYeSfAJ4uPW7qaqOXpS+js4dS2cD97SXJGmApgyDqrp6kqbLevQt4PpJ9rMF2NKjPgq8bapxSJJmj99AliQZBpIkw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSOIXCIMnqJE8nGUtyw7DHI0lnklMiDJLMA24D1gDLgauTLB/uqCTpzHFKhAFwMTBWVfur6pfAdmDdkMckSWeM+cMeQLMYeK5rfRy45NhOSTYBm9rqz5I8feIfmRPf9MSdC/xoGB88RM55QPLpQX/i/+Pf89zw65M1nCph0Ot/5npFoWozsHn2hzM7koxW1Yphj2OQnPOZwTnPfafKaaJxYGnX+hLgwJDGIklnnFMlDB4GRpJcmOQsYD2wY8hjkqQzxilxmqiqjiT5ELALmAdsqaq9Qx7WbJizp7hOgnM+MzjnOS5Vrzg1L0k6w5wqp4kkSUNkGEiSDIPZ0M+jNZL8YZInk+xN8qVBj3GmTTXnJG9Mcn+SR5M8lmTtMMY5U5JsSXIwyROTtCfJre3P47EkFw16jDOtjzl/oM31sSTfTvL2QY9xpk01565+v53k5SRXDmpsM66qfM3gi84F8O8DvwGcBXwPWH5MnxHgUWBhWz9v2OMewJw3A9e15eXAM8Me90nO+XeBi4AnJmlfC9xD5zs0K4GHhj3mAcz5d7p+ptecCXNufeYB9wE7gSuHPeYTfXlkMPP6ebTGnwC3VdVhgKo6OOAxzrR+5lzAOW35dczx75FU1beAQ8fpsg7YVh17gAVJLhjM6GbHVHOuqm8f/ZkG9tD5vtCc1sffM8CHga8Cc/rfsWEw83o9WmPxMX3eDLw5yb8l2ZNk9cBGNzv6mfPfAH+UZJzOb1AfHszQhqafP5PT2UY6R0antSSLgT8APj/ssZwsw2Dm9fNojfl0ThW9B7ga+EKSBbM8rtnUz5yvBu6oqiV0TqF8Mcnp/PPX1yNWTkdJ3ksnDD427LEMwGeAj1XVy8MeyMk6Jb50dprp59Ea48Ceqvpv4AftgXsjdL6JPRf1M+eNwGqAqnowyWvpPOhrTh9aH8cZ+YiVJL8FfAFYU1U/HvZ4BmAFsD0JdH6e1yY5UlXfGO6wpu90/s1sWPp5tMY3gPcCJDmXzmmj/QMd5czqZ87PApcBJHkL8FpgYqCjHKwdwDXtrqKVwAtV9fywBzWbkrwR+Brwwar6z2GPZxCq6sKqWlZVy4C7gD+di0EAHhnMuJrk0RpJbgJGq2pHa1uV5EngZeAv5vJvUX3O+aPA3yf5czqnS/642q0Yc1GSL9M5zXduuw5yI/BqgKr6PJ3rImuBMeAl4NrhjHTm9DHnvwbeAHy2/aZ8pOb4Uz37mPNpw8dRSJI8TSRJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEkC/hdhyNZ0A4BSqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87.74399024307658\n"
     ]
    }
   ],
   "source": [
    "auc = roc_auc_score(Y_test, y_hat)*100.0\n",
    "print(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}