RNN Self-Trained WordVec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras.preprocessing import text, sequence\n",
    "import numpy as np\n",
    "from keras.layers import Input, SpatialDropout1D,Dropout, GlobalAveragePooling1D, GlobalMaxPooling1D, \\\n",
    "                            CuDNNGRU, GRU, Bidirectional, LSTM, Dense, Embedding, concatenate, Embedding, \\\n",
    "                            Flatten, Activation, BatchNormalization, regularizers, Conv1D\n",
    "\n",
    "from keras.initializers import Orthogonal\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LambdaCallback, Callback, LearningRateScheduler\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import pickle\n",
    "import gc; gc.enable()\n",
    "import matplotlib \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords                \n",
    "from nltk.stem.snowball import RussianStemmer\n",
    "from scipy.stats import boxcox\n",
    "import re\n",
    "#from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check GPU Availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 17981027881184451297\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 3174131302\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 8268266985934388495\n",
      "physical_device_desc: \"device: 0, name: GeForce GTX 1050 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1\"\n",
      "]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/job:localhost/replica:0/task:0/device:GPU:0']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "K.tensorflow_backend._get_available_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess Training and Testing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you could download it on the google drive we shared. More columns than in public kernel\n",
    "agg_features_path = 'aggregated_features.csv'\n",
    "train_data_path = 'train.csv'\n",
    "test_data_path = 'test.csv'\n",
    "embedding_file = 'avito_description_300.w2v' #'cc.ru.300.vec'\n",
    "title_embedding_file = 'avito_title_100_ver2.w2v'\n",
    "seed = 411\n",
    "rnn_train_epochs = 10\n",
    "batch_size=64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_features = pd.read_csv(agg_features_path)\n",
    "agg_cols = list(agg_features.columns)[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv', parse_dates=['activation_date']) \\\n",
    "          .sort_values('activation_date') \\\n",
    "          .reset_index(drop=True)\n",
    "    \n",
    "test = pd.read_csv('test.csv', parse_dates=['activation_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.merge(agg_features, on='user_id', how='left')\n",
    "test = test.merge(agg_features, on='user_id', how='left')\n",
    "del agg_features; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge to faciliate data cleaning and transformation\n",
    "train_len = train.shape[0]\n",
    "train_y = train.deal_probability.values\n",
    "\n",
    "train.drop('deal_probability', axis=1, inplace=True)\n",
    "all_features = pd.concat([train, test]).reset_index(drop=True)\n",
    "\n",
    "del train, test; gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = lambda l1,l2: sum([1 for x in l1 if x in l2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = RussianStemmer(ignore_stopwords=False)\n",
    "'''\n",
    "def clean_text(txt):\n",
    "    words = str(txt).lower().strip().split(\" \\t\\r.,!?^+-*/@~:;/\\\\\\\"\\'&{}[]()#$%\") #str(txt).split(\" \") #\n",
    "    words = [stemmer.stem(wrd) for wrd in words \\\n",
    "                if wrd not in stopwords.words('russian') and len(wrd) > 1]\n",
    "    txt = u\" \".join(words)\n",
    "    return txt\n",
    "'''\n",
    "def clean_text(txt):\n",
    "    return u\" \".join([stemmer.stem(re.sub(r'\\b\\d+\\b', '', wrd)) for wrd in str(txt).lower().strip().split(string.punctuation)\n",
    "                         if wrd not in stopwords.words('russian')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features['description'].fillna('unknowndescription', inplace=True)\n",
    "all_features['description'] = [clean_text(txt) for txt in all_features['description'].values] # clean text\n",
    "\n",
    "all_features['title'].fillna('unknowntitle', inplace=True)\n",
    "all_features['title'] = [clean_text(txt) for txt in all_features['title'].values]\n",
    "\n",
    "all_features['weekday'] = pd.to_datetime(all_features['activation_date']).dt.day\n",
    "\n",
    "for col in ['description', 'title']:\n",
    "    all_features['num_words_' + col] = all_features[col].apply(lambda comment: len(comment.split()))\n",
    "    all_features['num_unique_words_' + col] = all_features[col].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "\n",
    "all_features['words_vs_unique_title'] = all_features['num_unique_words_title'] / all_features['num_words_title'] * 100\n",
    "all_features['words_vs_unique_description'] = all_features['num_unique_words_description'] / all_features['num_words_description'] * 100\n",
    "all_features['words_vs_unique_description'] = all_features['words_vs_unique_description'].fillna(0.)\n",
    "\n",
    "all_features['city'] = all_features['region'] + '_' + all_features['city'] # city is repeated in different region\n",
    "all_features['num_desc_punct'] = all_features['description'].apply(lambda x: count(x, set(string.punctuation)))\n",
    "\n",
    "for col in agg_cols:\n",
    "    all_features[col].fillna(-1, inplace=True)\n",
    "\n",
    "for col in ['price', 'image_top_1']:\n",
    "    all_features[col].fillna(-1, inplace=True)\n",
    "\n",
    "for col in ['param_1', 'param_2', 'param_3']:\n",
    "    all_features[col].fillna('unknwonparam', inplace=True)\n",
    "    \n",
    "for col in ['image']:\n",
    "    all_features[col].fillna('no-image', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features.price.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features.item_seq_number.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Since price and item seq number are highly skewed, we'll transform it into more normal like by using boxcox (more robust to outliers compared to np.log1p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in ['price', 'item_seq_number']:\n",
    "    select_filter = all_features[col] > 0\n",
    "    all_features.loc[select_filter, col], _ = boxcox(all_features.loc[select_filter, col])\n",
    "    all_features[col].hist()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "all_features.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "numerical = [\n",
    "    'weekday', 'num_words_title', 'num_words_description',\n",
    "    'num_unique_words_title', 'num_unique_words_description',\n",
    "    'words_vs_unique_title', 'words_vs_unique_description',\n",
    "    'num_desc_punct', 'avg_times_up_user', 'avg_days_up_user', \n",
    "    'med_times_up_user', 'med_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "]\n",
    "\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n",
    "]\n",
    "\n",
    "features = numerical+categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label encode categorical data\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "for feature in categorical:\n",
    "    print('Transforming {}...'.format(feature))\n",
    "    encoder = LabelEncoder()\n",
    "    all_features.loc[:, feature] = encoder.fit_transform(all_features[feature].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features.loc[:, numerical] = all_features[numerical].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features[numerical].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardize numerical data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "all_features.loc[:, numerical] = scaler.fit_transform(all_features[numerical].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare  pre-trained embeddings and text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 500000\n",
    "maxlen = 150\n",
    "embed_size = 300\n",
    "\n",
    "title_max_features = 200000\n",
    "title_maxlen = 80\n",
    "title_embed_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import word2vec\n",
    "\n",
    "def get_embed_matrix(emb_file, texts, max_feat_num, max_len, emb_size):\n",
    "\n",
    "    print('getting embeddings')\n",
    "    embeddings_index = word2vec.Word2Vec.load(emb_file)\n",
    "    \n",
    "    print('fitting tokenizer')\n",
    "    tokenizer = text.Tokenizer(num_words=max_feat_num)\n",
    "    tokenizer.fit_on_texts(texts)\n",
    "    \n",
    "    print('convert to sequences')\n",
    "    texts = tokenizer.texts_to_sequences(texts)\n",
    "\n",
    "    print('padding')\n",
    "    texts = sequence.pad_sequences(texts, maxlen=max_len)\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "    nb_words = min(max_feat_num, len(word_index))\n",
    "    embedding_matrix = np.zeros((nb_words, emb_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_feat_num: continue\n",
    "        try:\n",
    "            embedding_vector = embeddings_index[word]\n",
    "        except KeyError:\n",
    "            embedding_vector = None\n",
    "        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n",
    "      \n",
    "    \n",
    "    return {\n",
    "        'text': texts,\n",
    "        'emb_matrix': embedding_matrix,\n",
    "        'nb_words': nb_words\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "desc_embed_info = get_embed_matrix(embedding_file, \n",
    "                                   all_features.description.values, \n",
    "                                   max_features, \n",
    "                                   maxlen, \n",
    "                                   embed_size)\n",
    "\n",
    "title_embed_info = get_embed_matrix(title_embedding_file, \n",
    "                                    all_features.title.values, \n",
    "                                    title_max_features, \n",
    "                                    title_maxlen, \n",
    "                                    title_embed_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('setup max info for embedding in categorical variables')\n",
    "max_info = dict((col, all_features[col].max()+1) for col in categorical)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square(y_true - y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        print(input_shape)\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_rmse(true, prediction):\n",
    "    return np.sqrt(metrics.mean_squared_error(true, np.clip(prediction, 0., 1.)))\n",
    "    \n",
    "class NBatchEvalLogger(Callback):\n",
    "    def __init__(self, display, val_X, val_y, save_path=None, save_start=1000):\n",
    "        self.step = 0\n",
    "        self.display = display\n",
    "        self.val_X = val_X\n",
    "        self.val_y = val_y\n",
    "        self.best_loss = None\n",
    "        self.save_path = save_path\n",
    "        self.save_start = save_start\n",
    "        self.record_count = 0\n",
    "        \n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.step += 1\n",
    "        if self.step % self.display == 0 and self.step >= self.save_start:\n",
    "            #loss, metric = self.model.evaluate(self.val_X, self.val_y, batch_size=128, verbose=1)\n",
    "            prediction = self.model.predict(self.val_X, batch_size=128, verbose=0)\n",
    "            loss = clip_rmse(self.val_y, prediction)\n",
    "            \n",
    "            if self.best_loss is None:\n",
    "                self.best_loss = loss\n",
    "            else:\n",
    "                if loss < self.best_loss:\n",
    "                    self.best_loss = loss\n",
    "                    if self.save_path is not None:\n",
    "                        self.model.save(self.save_path, overwrite=True)\n",
    "                        self.record_count += 1\n",
    "                    \n",
    "            print('\\rstep: {} val loss={:.5f}, best loss={:.5f}'.format(self.step, loss, self.best_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from copy import deepcopy as cp\n",
    "\n",
    "class DataGenerator(keras.utils.Sequence):\n",
    "    #'Generates data for Keras'\n",
    "    def __init__(self, list_IDs, X, y, batch_size=32, shuffle=True, is_train=True, img_path=None):\n",
    "        #'Initialization'\n",
    "        self.batch_size = batch_size\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.list_IDs = list_IDs\n",
    "        self.shuffle = shuffle\n",
    "        self.img_path = img_path\n",
    "        self.is_train = is_train\n",
    "        self.on_epoch_end()\n",
    "\n",
    "    def __len__(self):\n",
    "        #'Denotes the number of batches per epoch'\n",
    "        return int(np.ceil(len(self.list_IDs) / self.batch_size))\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        #'Generate one batch of data'\n",
    "        # Generate indexes of the batch\n",
    "        start = index*self.batch_size\n",
    "        end = min((index+1)*self.batch_size, len(self.indexes))\n",
    "        indexes = self.indexes[start: end]\n",
    "\n",
    "        # Generate data\n",
    "        return self.__data_generation(indexes)\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        #'Updates indexes after each epoch'\n",
    "        self.indexes = cp(list(self.list_IDs))\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.indexes)\n",
    "\n",
    "    def __data_generation(self, list_IDs_temp):\n",
    "        #'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n",
    "        \n",
    "        # Generate data\n",
    "        X = dict((col, self.X.loc[list_IDs_temp, col].values) for col in features)\n",
    "        X['desc'] = desc_embed_info['text'][list_IDs_temp,:]\n",
    "        X['title'] = title_embed_info['text'][list_IDs_temp,:]\n",
    "        \n",
    "        # add img data\n",
    "        for image_name in self.X.loc[list_IDs_temp, 'image'].values:\n",
    "            if image_name == 'no-image':\n",
    "                pass #TODO: all-zero image\n",
    "            else:\n",
    "                pass #TODO: load real image\n",
    "        \n",
    "        if self.is_train:\n",
    "            y = cp(self.y[list_IDs_temp])\n",
    "            return X, y\n",
    "        else:\n",
    "            return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(X, categorical_features, numerical_features):\n",
    "    \n",
    "    # non-cat features\n",
    "    non_cat_inputs = []\n",
    "    for col in numerical_features:\n",
    "        f = Input(shape=[1], name=col)\n",
    "        non_cat_inputs.append(f)\n",
    "        \n",
    "    # cat features\n",
    "    cat_inputs = []\n",
    "    cat_embeds = []\n",
    "    for col in categorical_features:\n",
    "        f = Input(shape=[1], name=col)\n",
    "        embed_dim = max_info[col].max()\n",
    "        if max_info[col] > 10:\n",
    "            reduced_dim = 10\n",
    "        else:\n",
    "            reduced_dim = 1\n",
    "        embed_f = Embedding(embed_dim, reduced_dim)(f)\n",
    "        flatten_f = Flatten()(embed_f)\n",
    "        cat_inputs.append(f)\n",
    "        cat_embeds.append(flatten_f)\n",
    "      \n",
    "    # text features: architecture of text to try here!!!\n",
    "    \n",
    "    # description\n",
    "    text_inp = Input(shape = (maxlen, ), name='desc')\n",
    "    text_emb = Embedding(desc_embed_info['nb_words'], embed_size, weights = [desc_embed_info['emb_matrix']],\n",
    "                    input_length = maxlen, trainable = False)(text_inp)\n",
    "    text_emb = SpatialDropout1D(0.3)(text_emb)\n",
    "    text_gru = Bidirectional(CuDNNGRU(128, return_sequences = True))(text_emb)\n",
    "    text_gru = Conv1D(64, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(text_gru)\n",
    "    text_gru_avg = GlobalAveragePooling1D()(text_gru)\n",
    "    text_gru_max = GlobalMaxPooling1D()(text_gru)\n",
    "    text_gru = concatenate([text_gru_avg, text_gru_max]) \n",
    "    text_gru = Dropout(0.1)(text_gru)\n",
    "    \n",
    "    # title\n",
    "    title_inp = Input(shape = (title_maxlen, ), name='title')\n",
    "    title_emb = Embedding(title_embed_info['nb_words'], title_embed_size, weights = [title_embed_info['emb_matrix']],\n",
    "                    input_length = title_maxlen, trainable = False)(title_inp)\n",
    "    title_emb = SpatialDropout1D(0.1)(title_emb)\n",
    "    title_gru = Bidirectional(CuDNNGRU(32, return_sequences = True))(title_emb)\n",
    "    title_gru = Conv1D(16, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(title_gru)\n",
    "    title_gru_avg = GlobalAveragePooling1D()(title_gru)\n",
    "    title_gru_max = GlobalMaxPooling1D()(title_gru)\n",
    "    title_gru = concatenate([title_gru_avg, title_gru_max]) \n",
    "    title_gru = Dropout(0.1)(title_gru)\n",
    "    \n",
    "    # add image architecture ???\n",
    "    \n",
    "    # merge each branch: non-cat, cat, text\n",
    "    concat_main = non_cat_inputs+cat_embeds+[text_gru, title_gru]\n",
    "    main = concatenate(concat_main)\n",
    "    main = BatchNormalization()(main)\n",
    "    main = Dropout(0.1)(main)\n",
    "    main = BatchNormalization()(Dense(512, activation='relu')(main))\n",
    "    main = Dropout(0.1)(main)\n",
    "    main = BatchNormalization()(Dense(64, activation='relu')(main))\n",
    "    out = Dense(1, activation = \"sigmoid\")(main)\n",
    "\n",
    "    concat_input = non_cat_inputs+cat_inputs+[text_inp, title_inp]\n",
    "    model = Model(concat_input, out)\n",
    "    model.regularizers = [regularizers.l2(0.0001)]\n",
    "    model.compile(optimizer = Adam(lr=0.001), loss = root_mean_squared_error,\n",
    "                  metrics =[root_mean_squared_error])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import warnings; warnings.filterwarnings('ignore') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indices = np.arange(0, train_len)\n",
    "test_indices = np.arange(train_len, all_features.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from ipywidgets import IntProgress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_fold = 4 # <= 0 for invalid, train from fold 1, > 0: train from fold=start_fold\n",
    "resume_file_prefix = '0617_rnn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if start_fold > 0:\n",
    "    import pickle\n",
    "    ret = pickle.load(open(resume_file_prefix+'_oof_val_pred', 'rb'))\n",
    "    ret_test = pickle.load(open(resume_file_prefix+'_oof_test_pred', 'rb'))\n",
    "    print(ret)\n",
    "    print(ret_test)\n",
    "else:\n",
    "    ret = np.zeros((train_len,))\n",
    "    ret_test = np.zeros((all_features.shape[0]-train_len,))\n",
    "\n",
    "fold = 0    \n",
    "for tr_ix, val_ix in KFold(5, shuffle=True, random_state=seed).split(train_indices):\n",
    "    fold += 1\n",
    "    \n",
    "    if start_fold > 0 and fold < start_fold:\n",
    "        continue\n",
    "    else:\n",
    "        pass\n",
    "    \n",
    "    model = build_model(all_features, categorical, numerical)\n",
    "    file_path = \"rnn_weights/model_self_train_fold_{}.hdf5\".format(fold)\n",
    "    \n",
    "    # customized batch loader\n",
    "    training_generator = DataGenerator(tr_ix, all_features, train_y, batch_size=batch_size, shuffle=True)\n",
    "    validation_generator = DataGenerator(val_ix, all_features, train_y, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    lr_schd = LearningRateScheduler(lambda epoch: 0.001*(0.2**(epoch//6)), verbose=1)\n",
    "    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", mode = \"min\", save_best_only = True, verbose = 1)\n",
    "    history = model.fit_generator(generator=training_generator,\n",
    "                                  validation_data=validation_generator,\n",
    "                                  use_multiprocessing=False,\n",
    "                                  workers=1, \n",
    "                                  epochs=rnn_train_epochs,\n",
    "                                  verbose = 0, \n",
    "                                  callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n",
    "    \n",
    "    \n",
    "    # Predict val + test oofs\n",
    "    model.load_weights(file_path) # load weight with best validation score\n",
    "    \n",
    "    del validation_generator\n",
    "    validation_generator = DataGenerator(val_ix, all_features, None, batch_size=batch_size, shuffle=False, is_train=False)\n",
    "    test_generator = DataGenerator(test_indices, all_features, None, batch_size=batch_size, shuffle=False, is_train=False)\n",
    "    \n",
    "    ret[val_ix] = model.predict_generator(validation_generator, use_multiprocessing=False, workers=1).reshape((len(val_ix),))\n",
    "    ret_test += model.predict_generator(test_generator, use_multiprocessing=False, workers=1).reshape((ret_test.shape[0],))\n",
    "    \n",
    "    del model, history, training_generator, validation_generator, test_generator; gc.collect()\n",
    "    \n",
    "ret_test /= 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment these to dump files if OOM happens\n",
    "import pickle\n",
    "pickle.dump(ret, open(resume_file_prefix+'_oof_val_pred', 'wb'))\n",
    "pickle.dump(ret_test, open(resume_file_prefix+'_oof_test_pred', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# public:  cv = .2220, lb = .2247 \n",
    "# bigru-conv1d: cv =.2185 , lb = .2235\n",
    "# bigru-attention: cv =.2186 , lb = .2235\n",
    "# 2gru: lb: .2239\n",
    "# self-trained wordvec: cv .217232, lb: .2229"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate OOFs and Submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(data=ret, columns=['selftrained_bigru_conv1d_ver2_rnn_pred']).to_csv('selftrained_bigru_conv1d_ver2_rnn_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=ret_test, columns=['selftrained_bigru_conv1d_ver2_rnn_pred']).to_csv('selftrained_bigru_conv1d_ver2_rnn_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subm = pd.read_csv('sample_submission.csv')\n",
    "subm['deal_probability'] = np.clip(ret_test, 0, 1)\n",
    "subm.to_csv('selftrained_bigru_conv1d_ver2_rnn_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.5 tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}