RNN Baseline + Pretrained WordVec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras.preprocessing import text, sequence\n",
    "import numpy as np\n",
    "from keras.layers import Input, SpatialDropout1D,Dropout, GlobalAveragePooling1D, GlobalMaxPooling1D, \\\n",
    "                            CuDNNGRU, GRU, Bidirectional, LSTM, Dense, Embedding, concatenate, Embedding, \\\n",
    "                            Flatten, Activation, BatchNormalization, regularizers, Conv1D\n",
    "\n",
    "from keras.initializers import Orthogonal\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LambdaCallback, Callback, LearningRateScheduler\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import pickle\n",
    "import gc; gc.enable()\n",
    "import matplotlib \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords                \n",
    "from nltk.stem.snowball import RussianStemmer\n",
    "from scipy.stats import boxcox\n",
    "#from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check GPU Availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 9313005261063746220\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 3174131302\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 17551354236519716130\n",
      "physical_device_desc: \"device: 0, name: GeForce GTX 1050 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1\"\n",
      "]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/job:localhost/replica:0/task:0/device:GPU:0']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())\n",
    "K.tensorflow_backend._get_available_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess Training and Testing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you could download it on the google drive we shared. More columns than in public kernel\n",
    "agg_features_path = 'aggregated_features.csv'\n",
    "train_data_path = 'train.csv'\n",
    "test_data_path = 'test.csv'\n",
    "embedding_file = 'cc.ru.300.vec'\n",
    "seed = 411\n",
    "rnn_train_epochs = 10\n",
    "batch_size=128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_features = pd.read_csv(agg_features_path)\n",
    "agg_cols = list(agg_features.columns)[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv', parse_dates=['activation_date']) \\\n",
    "          .sort_values('activation_date') \\\n",
    "          .reset_index(drop=True)\n",
    "    \n",
    "test = pd.read_csv('test.csv', parse_dates=['activation_date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.merge(agg_features, on='user_id', how='left')\n",
    "test = test.merge(agg_features, on='user_id', how='left')\n",
    "del agg_features; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge to faciliate data cleaning and transformation\n",
    "train_len = train.shape[0]\n",
    "train_y = train.deal_probability.values\n",
    "\n",
    "train.drop('deal_probability', axis=1, inplace=True)\n",
    "all_features = pd.concat([train, test]).reset_index(drop=True)\n",
    "\n",
    "del train, test; gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = lambda l1,l2: sum([1 for x in l1 if x in l2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = RussianStemmer(ignore_stopwords=False)\n",
    "def clean_text(txt):\n",
    "    words = str(txt).lower().strip().split(\" \\t\\r.,!?^+-*/@~:;/\\\\\\\"\\'&{}[]()#$%\") #str(txt).split(\" \") #\n",
    "    words = [stemmer.stem(wrd) for wrd in words \\\n",
    "                if wrd not in stopwords.words('russian') and len(wrd) > 1]\n",
    "    txt = u\" \".join(words)\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features['description'].fillna('unknowndescription', inplace=True)\n",
    "all_features['description'] = [clean_text(txt) for txt in all_features['description'].values] # clean text\n",
    "\n",
    "all_features['title'].fillna('unknowntitle', inplace=True)\n",
    "all_features['title'] = all_features['title'].apply(lambda x: x.lower())\n",
    "\n",
    "all_features['weekday'] = pd.to_datetime(all_features['activation_date']).dt.day\n",
    "\n",
    "for col in ['description', 'title']:\n",
    "    all_features['num_words_' + col] = all_features[col].apply(lambda comment: len(comment.split()))\n",
    "    all_features['num_unique_words_' + col] = all_features[col].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "\n",
    "all_features['words_vs_unique_title'] = all_features['num_unique_words_title'] / all_features['num_words_title'] * 100\n",
    "all_features['words_vs_unique_description'] = all_features['num_unique_words_description'] / all_features['num_words_description'] * 100\n",
    "all_features['words_vs_unique_description'] = all_features['words_vs_unique_description'].fillna(0.)\n",
    "\n",
    "all_features['city'] = all_features['region'] + '_' + all_features['city'] # city is repeated in different region\n",
    "all_features['num_desc_punct'] = all_features['description'].apply(lambda x: count(x, set(string.punctuation)))\n",
    "\n",
    "for col in agg_cols:\n",
    "    all_features[col].fillna(-1, inplace=True)\n",
    "\n",
    "for col in ['price', 'image_top_1']:\n",
    "    all_features[col].fillna(-1, inplace=True)\n",
    "\n",
    "for col in ['param_1', 'param_2', 'param_3']:\n",
    "    all_features[col].fillna('unknwonparam', inplace=True)\n",
    "    \n",
    "for col in ['image']:\n",
    "    all_features[col].fillna('no-image', inplace=True)\n",
    "    \n",
    "# TODO: use self train embedding for title and description separately instead of concatenating them together\n",
    "all_features['description'] = all_features['title'] + ' ' + all_features['description']\n",
    "all_features.drop('title', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "item_id                         0\n",
       "user_id                         0\n",
       "region                          0\n",
       "city                            0\n",
       "parent_category_name            0\n",
       "category_name                   0\n",
       "param_1                         0\n",
       "param_2                         0\n",
       "param_3                         0\n",
       "description                     0\n",
       "price                           0\n",
       "item_seq_number                 0\n",
       "activation_date                 0\n",
       "user_type                       0\n",
       "image                           0\n",
       "image_top_1                     0\n",
       "avg_days_up_user                0\n",
       "avg_times_up_user               0\n",
       "med_days_up_user                0\n",
       "med_times_up_user               0\n",
       "n_user_items                    0\n",
       "weekday                         0\n",
       "num_words_description           0\n",
       "num_unique_words_description    0\n",
       "num_words_title                 0\n",
       "num_unique_words_title          0\n",
       "words_vs_unique_title           0\n",
       "words_vs_unique_description     0\n",
       "num_desc_punct                  0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_features.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x14389203d30>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEJCAYAAABR4cpEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHS1JREFUeJzt3X+Q1PWd5/HnKxAN6ioY4xQH7GEubHZRb4nMKbdWUqNkEdxUMFd6B+XJxHA10cKreGfVibm6Iqexytyd8UrLuEvCnJAlEtcfC5XFEIrY66ZKDaKuSNRlRFZHOFgdRCcavXHf98f3M5uvk57p6f7MTDfx9ajq6u739/P5ft9tqa/5/uhvKyIwMzPL8ZFmN2BmZsc+h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWbbJzW5gopx22mkxe/bshuf/8pe/5MQTTxy7hsaI+6pPq/YFrdub+6pfq/bWSF+7du16LSI+UXNgRHwoHvPnz48cDz/8cNb88eK+6tOqfUW0bm/uq36t2lsjfQFPxCj+H+vDXGZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtlqhomkWZIelvScpD2Svpbqp0raLmlvep6W6pJ0u6QeSc9IOqe0rs40fq+kzlJ9vqTdac7tktToNszMbOKNZs9kALguIv4AWACskjQXWA3siIg5wI70HmAJMCc9uoC7oAgGYA1wHnAusGYwHNKYrtK8xale1zbMzKw5aoZJRByMiCfT67eA54AZwFJgfRq2HrgkvV4KbEjfd3kMmCppOnARsD0i+iLiCLAdWJyWnRwRj6YvyGwYsq56tmFmZk1Q1+1UJM0GPgM8DrRFxEEoAkfS6WnYDOCV0rTeVBup3lulTgPbOFjP56nH7leP8uXVfzVeqx/W/lv+ZMK3aWZWr1GHiaSTgPuBayPizXRao+rQKrVooD5iO6OZI6mL4jAYbW1tVCqVGqsdXtsUuO7sgYbnN6pWz/39/Vmfa7y4r/q1am/uq36t2tt49jWqMJH0UYog2RgRD6TyIUnT0x7DdOBwqvcCs0rTZwIHUr1jSL2S6jOrjG9kGx8QEWuBtQDt7e3R0dExdMio3bFxM7funvj7Yu6/vGPE5ZVKhZzPNV7cV/1atTf3Vb9W7W08+xrN1VwC1gHPRcS3S4u2AINXZHUCm0v1FemKqwXA0XSoahuwSNK0dOJ9EbAtLXtL0oK0rRVD1lXPNszMrAlG86f2+cAVwG5JT6fa14FbgHslrQReBi5Ly7YCFwM9wNvAlQAR0SfpJmBnGndjRPSl11cDdwNTgIfSg3q3YWZmzVEzTCLiZ1Q/RwGwsMr4AFYNs65uoLtK/QngrCr11+vdhpmZTTx/A97MzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyjeY34LslHZb0bKn2Q0lPp8f+wZ/zlTRb0julZX9amjNf0m5JPZJuT7/3jqRTJW2XtDc9T0t1pXE9kp6RdE5pXZ1p/F5JnZiZWVONZs/kbmBxuRAR/y4i5kXEPOB+4IHS4hcHl0XEVaX6XUAXMCc9Bte5GtgREXOAHek9wJLS2K40H0mnAmuA84BzgTWDAWRmZs1RM0wi4hGgr9qytHfxb4F7RlqHpOnAyRHxaPr99g3AJWnxUmB9er1+SH1DFB4Dpqb1XARsj4i+iDgCbGdI2JmZ2cTKPWfyWeBQROwt1c6Q9JSkv5b02VSbAfSWxvSmGkBbRBwESM+nl+a8UmXOcHUzM2uSyZnzl/PBvZKDwO9GxOuS5gN/KelMQFXmRo11Dzdn1OuS1EVxiIy2tjYqlUqNTQ6vbQpcd/ZAw/MbVavn/v7+rM81XtxX/Vq1N/dVv1btbTz7ajhMJE0G/g0wf7AWEe8C76bXuyS9CPwexd7DzNL0mcCB9PqQpOkRcTAdxjqc6r3ArCpzeoGOIfVKtR4jYi2wFqC9vT06OjqqDRuVOzZu5tbdudlbv/2Xd4y4vFKpkPO5xov7ql+r9ua+6teqvY1nXzmHuT4PPB8R/3T4StInJE1Krz9JcfJ8Xzp89ZakBek8ywpgc5q2BRi8IqtzSH1FuqprAXA0rWcbsEjStHTifVGqmZlZk9T8U1vSPRR7AqdJ6gXWRMQ6YBm/eeL9c8CNkgaA94GrImLw5P3VFFeGTQEeSg+AW4B7Ja0EXgYuS/WtwMVAD/A2cCVARPRJugnYmcbdWNqGmZk1Qc0wiYjlw9S/XKV2P8WlwtXGPwGcVaX+OrCwSj2AVcOsqxvoHqlvMzObOP4GvJmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWWrGSaSuiUdlvRsqfYNSa9Kejo9Li4tu0FSj6QXJF1Uqi9OtR5Jq0v1MyQ9LmmvpB9KOi7Vj0/ve9Ly2bW2YWZmzTGaPZO7gcVV6rdFxLz02AogaS7Fb8OfmeZ8R9IkSZOAO4ElwFxgeRoL8K20rjnAEWBlqq8EjkTEp4Db0rhht1HfxzYzs7FUM0wi4hGgb5TrWwpsioh3I+IloAc4Nz16ImJfRLwHbAKWShJwIXBfmr8euKS0rvXp9X3AwjR+uG2YmVmTTM6Ye42kFcATwHURcQSYATxWGtObagCvDKmfB3wceCMiBqqMnzE4JyIGJB1N40faxgdI6gK6ANra2qhUKvV/yqRtClx39kDtgWOsVs/9/f1Zn2u8uK/6tWpv7qt+rdrbePbVaJjcBdwERHq+FfgKoCpjg+p7QDHCeEZYNtKcDxYj1gJrAdrb26Ojo6PasFG5Y+Nmbt2dk72N2X95x4jLK5UKOZ9rvLiv+rVqb+6rfq3a23j21dDVXBFxKCLej4h/BL7Lrw8z9QKzSkNnAgdGqL8GTJU0eUj9A+tKy0+hONw23LrMzKxJGgoTSdNLb78EDF7ptQVYlq7EOgOYA/wc2AnMSVduHUdxAn1LRATwMHBpmt8JbC6tqzO9vhT4aRo/3DbMzKxJah63kXQP0AGcJqkXWAN0SJpHcXhpP/BVgIjYI+le4BfAALAqIt5P67kG2AZMArojYk/axPXAJknfBJ4C1qX6OuD7knoo9kiW1dqGmZk1R80wiYjlVcrrqtQGx98M3FylvhXYWqW+jypXY0XEr4DL6tmGmZk1h78Bb2Zm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpatZphI6pZ0WNKzpdr/lPS8pGckPShpaqrPlvSOpKfT409Lc+ZL2i2pR9LtkpTqp0raLmlvep6W6krjetJ2zimtqzON3yupEzMza6rR7JncDSweUtsOnBUR/xL4O+CG0rIXI2JeelxVqt8FdFH8Zvuc0jpXAzsiYg6wI70HWFIa25XmI+lUip8OPo/iFxrXDAaQmZk1R80wiYhHKH6DvVz7SUQMpLePATNHWoek6cDJEfFoRASwAbgkLV4KrE+v1w+pb4jCY8DUtJ6LgO0R0RcRRyiCbWjYmZnZBBqLcyZfAR4qvT9D0lOS/lrSZ1NtBtBbGtObagBtEXEQID2fXprzSpU5w9XNzKxJJudMlvRfgQFgYyodBH43Il6XNB/4S0lnAqoyPWqtfpg5o16XpC6KQ2S0tbVRqVRqbHJ4bVPgurMHag8cY7V67u/vz/pc48V91a9Ve3Nf9WvV3sazr4bDJJ34/gKwMB26IiLeBd5Nr3dJehH4PYq9h/KhsJnAgfT6kKTpEXEwHcY6nOq9wKwqc3qBjiH1SrUeI2ItsBagvb09Ojo6qg0blTs2bubW3VnZ25D9l3eMuLxSqZDzucaL+6pfq/bmvurXqr2NZ18NHeaStBi4HvhiRLxdqn9C0qT0+pMUJ8/3pcNXb0lakK7iWgFsTtO2AINXZHUOqa9IV3UtAI6m9WwDFkmalk68L0o1MzNrkpp/aku6h2JP4DRJvRRXUt0AHA9sT1f4Ppau3PoccKOkAeB94KqIGDx5fzXFlWFTKM6xDJ5nuQW4V9JK4GXgslTfClwM9ABvA1cCRESfpJuAnWncjaVtmJlZE9QMk4hYXqW8bpix9wP3D7PsCeCsKvXXgYVV6gGsGmZd3UD38F2bmdlE8jfgzcwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLNuowkRSt6TDkp4t1U6VtF3S3vQ8LdUl6XZJPZKekXROaU5nGr9XUmepPl/S7jTn9vQ78Q1tw8zMJt5o90zuBhYPqa0GdkTEHGBHeg+wBJiTHl3AXVAEA8Xvx58HnAusGQyHNKarNG9xI9swM7PmGFWYRMQjQN+Q8lJgfXq9HrikVN8QhceAqZKmAxcB2yOiLyKOANuBxWnZyRHxaPrd9w1D1lXPNszMrAlyzpm0RcRBgPR8eqrPAF4pjetNtZHqvVXqjWzDzMyaYPI4rFNVatFAvZFtfHCQ1EVxGIy2tjYqlUqN1Q6vbQpcd/ZAw/MbVavn/v7+rM81XtxX/Vq1N/dVv1btbTz7ygmTQ5KmR8TBdIjpcKr3ArNK42YCB1K9Y0i9kuozq4xvZBsfEBFrgbUA7e3t0dHRMXTIqN2xcTO37h6P7B3Z/ss7RlxeqVTI+VzjxX3Vr1V7c1/1a9XexrOvnMNcW4DBK7I6gc2l+op0xdUC4Gg6RLUNWCRpWjrxvgjYlpa9JWlBuoprxZB11bMNMzNrglH9qS3pHoq9itMk9VJclXULcK+klcDLwGVp+FbgYqAHeBu4EiAi+iTdBOxM426MiMGT+ldTXDE2BXgoPah3G2Zm1hyjCpOIWD7MooVVxgawapj1dAPdVepPAGdVqb9e7zbMzGzi+RvwZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmls1hYmZm2RwmZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmlq3hMJH0aUlPlx5vSrpW0jckvVqqX1yac4OkHkkvSLqoVF+caj2SVpfqZ0h6XNJeST+UdFyqH5/e96Tlsxv9HGZmlq/hMImIFyJiXkTMA+ZT/Bb7g2nxbYPLImIrgKS5wDLgTGAx8B1JkyRNAu4ElgBzgeVpLMC30rrmAEeAlam+EjgSEZ8CbkvjzMysScbqMNdC4MWI+PsRxiwFNkXEuxHxEtADnJsePRGxLyLeAzYBSyUJuBC4L81fD1xSWtf69Po+YGEab2ZmTTBWYbIMuKf0/hpJz0jqljQt1WYAr5TG9KbacPWPA29ExMCQ+gfWlZYfTePNzKwJFBF5KyjOYxwAzoyIQ5LagNeAAG4CpkfEVyTdCTwaEX+e5q0DtlIE2kUR8R9S/QqKvZUb0/hPpfosYGtEnC1pT5rTm5a9CJwbEa8P6a0L6AJoa2ubv2nTpoY/5+G+oxx6p+HpDTt7xikjLu/v7+ekk06aoG5Gz33Vr1V7c1/1a9XeGunrggsu2BUR7bXGTW64q19bAjwZEYcABp8BJH0X+FF62wvMKs2bSRFCDFN/DZgqaXLa+yiPH1xXr6TJwClA39DGImItsBagvb09Ojo6Gv6Qd2zczK27x+IfV332X94x4vJKpULO5xov7qt+rdqb+6pfq/Y2nn2NxWGu5ZQOcUmaXlr2JeDZ9HoLsCxdiXUGMAf4ObATmJOu3DqO4pDZlih2mR4GLk3zO4HNpXV1pteXAj+N3F0sMzNrWNaf2pJOAP4Y+Gqp/D8kzaM4zLV/cFlE7JF0L/ALYABYFRHvp/VcA2wDJgHdEbEnret6YJOkbwJPAetSfR3wfUk9FHsky3I+h5mZ5ckKk4h4myEnviPiihHG3wzcXKW+leL8ydD6PorzJ0PrvwIua6BlMzMbB/4GvJmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWXLDhNJ+yXtlvS0pCdS7VRJ2yXtTc/TUl2SbpfUI+kZSeeU1tOZxu+V1Fmqz0/r70lzNdI2zMxs4o3VnskFETEvItrT+9XAjoiYA+xI7wGWAHPSowu4C4pgANYA51H8TO+aUjjclcYOzltcYxtmZjbBxusw11JgfXq9HrikVN8QhceAqZKmAxcB2yOiLyKOANuBxWnZyRHxaEQEsGHIuqptw8zMJpiK/0dnrEB6CTgCBPBnEbFW0hsRMbU05khETJP0I+CWiPhZqu8Argc6gI9FxDdT/b8B7wCVNP7zqf5Z4PqI+MJw2xjSWxfFXg1tbW3zN23a1PDnPNx3lEPvNDy9YWfPOGXE5f39/Zx00kkT1M3oua/6tWpv7qt+rdpbI31dcMEFu0pHnYY1ueGufu38iDgg6XRgu6TnRxirKrVooD4qEbEWWAvQ3t4eHR0do536G+7YuJlbd4/FP6767L+8Y8TllUqFnM81XtxX/Vq1N/dVv1btbTz7yj7MFREH0vNh4EGKcx6H0iEq0vPhNLwXmFWaPhM4UKM+s0qdEbZhZmYTLCtMJJ0o6XcGXwOLgGeBLcDgFVmdwOb0eguwIl3VtQA4GhEHgW3AIknT0on3RcC2tOwtSQvSVVwrhqyr2jbMzGyC5R63aQMeTFfrTgZ+EBE/lrQTuFfSSuBl4LI0fitwMdADvA1cCRARfZJuAnamcTdGRF96fTVwNzAFeCg9AG4ZZhtmZjbBssIkIvYBf1il/jqwsEo9gFXDrKsb6K5SfwI4a7TbMDOziedvwJuZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllc5iYmVk2h4mZmWVzmJiZWTaHiZmZZXOYmJlZNoeJmZllazhMJM2S9LCk5yTtkfS1VP+GpFclPZ0eF5fm3CCpR9ILki4q1RenWo+k1aX6GZIel7RX0g8lHZfqx6f3PWn57EY/h5mZ5cvZMxkArouIPwAWAKskzU3LbouIeemxFSAtWwacCSwGviNpkqRJwJ3AEmAusLy0nm+ldc0BjgArU30lcCQiPgXclsaZmVmTNBwmEXEwIp5Mr98CngNmjDBlKbApIt6NiJcofgf+3PToiYh9EfEesAlYquKH5S8E7kvz1wOXlNa1Pr2+D1iYxpuZWROMyTmTdJjpM8DjqXSNpGckdUualmozgFdK03pTbbj6x4E3ImJgSP0D60rLj6bxZmbWBJNzVyDpJOB+4NqIeFPSXcBNQKTnW4GvANX2HILqgRYjjKfGsnJvXUAXQFtbG5VKZcTPMpK2KXDd2QO1B46xWj339/dnfa7x4r7q16q9ua/6tWpv49lXVphI+ihFkGyMiAcAIuJQafl3gR+lt73ArNL0mcCB9Lpa/TVgqqTJae+jPH5wXb2SJgOnAH1D+4uItcBagPb29ujo6Gj4s96xcTO37s7O3rrtv7xjxOWVSoWczzVe3Ff9WrU391W/Vu1tPPvKuZpLwDrguYj4dqk+vTTsS8Cz6fUWYFm6EusMYA7wc2AnMCdduXUcxUn6LRERwMPApWl+J7C5tK7O9PpS4KdpvJmZNUHOn9rnA1cAuyU9nWpfp7gaax7FYaf9wFcBImKPpHuBX1BcCbYqIt4HkHQNsA2YBHRHxJ60vuuBTZK+CTxFEV6k5+9L6qHYI1mW8TnMzCxTw2ESET+j+rmLrSPMuRm4uUp9a7V5EbGP4mqvofVfAZfV06+ZmY0ffwPezMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLI5TMzMLJvDxMzMsh3TYSJpsaQXJPVIWt3sfszMPqyO2TCRNAm4E1gCzKX47fm5ze3KzOzD6ZgNE4rfhu+JiH0R8R6wCVja5J7MzD6UjuUwmQG8Unrfm2pmZjbBJje7gQyqUosPDJC6gK70tl/SCxnbOw14LWN+Q/StmkOa0tcouK/6tWpv7qt+rdpbI33989EMOpbDpBeYVXo/EzhQHhARa4G1Y7ExSU9ERPtYrGssua/6tGpf0Lq9ua/6tWpv49nXsXyYaycwR9IZko4DlgFbmtyTmdmH0jG7ZxIRA5KuAbYBk4DuiNjT5LbMzD6UjtkwAYiIrcDWCdrcmBwuGwfuqz6t2he0bm/uq36t2tu49aWIqD3KzMxsBMfyORMzM2sRDpMaWvWWLZK6JR2W9GyzeymTNEvSw5Kek7RH0tea3ROApI9J+rmkv019/fdm91QmaZKkpyT9qNm9lEnaL2m3pKclPdHsfgZJmirpPknPp3/X/nUL9PTp9M9p8PGmpGub3ReApP+U/r1/VtI9kj425tvwYa7hpVu2/B3wxxSXIu8ElkfEL5raGCDpc0A/sCEizmp2P4MkTQemR8STkn4H2AVc0ux/ZpIEnBgR/ZI+CvwM+FpEPNbMvgZJ+s9AO3ByRHyh2f0MkrQfaI+IlvrOhKT1wN9ExPfS1ZwnRMQbze5rUPp/x6vAeRHx903uZQbFv+9zI+IdSfcCWyPi7rHcjvdMRtayt2yJiEeAvmb3MVREHIyIJ9Prt4DnaIE7E0ShP739aHq0xF9SkmYCfwJ8r9m9HAsknQx8DlgHEBHvtVKQJAuBF5sdJCWTgSmSJgMnMOQ7eWPBYTIy37Ilg6TZwGeAx5vbSSEdSnoaOAxsj4iW6Av438B/Af6x2Y1UEcBPJO1Kd5RoBZ8E/gH4P+nQ4PckndjspoZYBtzT7CYAIuJV4H8BLwMHgaMR8ZOx3o7DZGQ1b9li1Uk6CbgfuDYi3mx2PwAR8X5EzKO4W8K5kpp+eFDSF4DDEbGr2b0M4/yIOIfi7tyr0uHVZpsMnAPcFRGfAX4JtNL5zOOALwJ/0exeACRNoziicgbwz4ATJf37sd6Ow2RkNW/ZYr8pnZO4H9gYEQ80u5+h0iGRCrC4ya0AnA98MZ2b2ARcKOnPm9vSr0XEgfR8GHiQ4tBvs/UCvaU9y/sowqVVLAGejIhDzW4k+TzwUkT8Q0T8P+AB4I/GeiMOk5H5li11Sie61wHPRcS3m93PIEmfkDQ1vZ5C8R/Y883tCiLihoiYGRGzKf79+mlEjPlfjY2QdGK6iIJ0GGkR0PSrByPi/wKvSPp0Ki0Emn5RTMlyWuQQV/IysEDSCem/z4UU5zLH1DH9Dfjx1sq3bJF0D9ABnCapF1gTEeua2xVQ/KV9BbA7nZ8A+Hq6W0EzTQfWp6tsPgLcGxEtdRluC2oDHiz+/8Nk4AcR8ePmtvRP/iOwMf2Rtw+4ssn9ACDpBIqrP7/a7F4GRcTjku4DngQGgKcYh2/C+9JgMzPL5sNcZmaWzWFiZmbZHCZmZpbNYWJmZtkcJmZmx7B6bvoq6XOSnpQ0IOnSIcs6Je1Nj856+3CYmJkd2+5m9F/AfRn4MvCDclHSqcAa4DyKL6auSd+cHzWHiZnZMazaTV8l/QtJP073VPsbSb+fxu6PiGf4zfvAXURxv7q+iDgCbKfOO0T4S4tmZr991gJXRcReSecB3wEuHGF89k1tHSZmZr9F0k1W/wj4i3T3AoDja02rUqvrG+0OEzOz3y4fAd5Id8gerV6K2zMNmklxM9S6NmpmZr8l0k8+vCTpMihuvirpD2tM2wYskjQtnXhflGqj5jAxMzuGpZu+Pgp8WlKvpJXA5cBKSX8L7CH9Qqykf5VuDHsZ8GeS9gBERB9wE8Wd0ncCN6ba6PvwjR7NzCyX90zMzCybw8TMzLI5TMzMLJvDxMzMsjlMzMwsm8PEzMyyOUzMzCybw8TMzLL9f4nbkCuZOeo3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_features.price.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x143364325c0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAD8CAYAAACyyUlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG5hJREFUeJzt3X+MXfWZ3/H3p3agbn5hQhhZtrc2zWy1DrQOjMCrNNE0dI3xVmtSQWsLxbPE0iTUqIlEpZjNHyAIEtnKQYImJI4YYSIvPwphbaWmjuVwN4oUfpjgtXEI8eB4w8SWLbBDmLAhHfr0j/vM6nhy78yd+x3PncGfl3R0z33O98e5j+/4mfPjzlVEYGZmVuKfdXoHzMxs9nMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFZvb6R2YLhdccEEsWbKk7f6//e1vee973zt1O/Qu5Ty1xnlqjfPUmjOZp+eff/61iPjwRO3OmmKyZMkS9u7d23b/Wq1Gb2/v1O3Qu5Tz1BrnqTXOU2vOZJ4k/UMr7Xyay8zMirmYmJlZMRcTMzMr5mJiZmbFJiwmkhZLekrSS5IOSvpCxs+XtFvSoXycn3FJukfSoKT9ki6tjNWX7Q9J6qvEL5N0IPvcI0ntzmFmZtOvlSOTEeDmiPgTYAWwUdIyYBOwJyK6gT35HOBqoDuXfuA+qBcG4FbgCuBy4NbR4pBt+iv9VmV8UnOYmVlnTFhMIuJYRPwk198EXgIWAmuArdlsK3BNrq8BHoy6p4HzJC0ArgJ2R8TJiDgF7AZW5bYPRMSPo/61jw+OGWsyc5iZWQdM6pqJpCXAx4BngK6IOAb1ggNcmM0WAq9Wug1lbLz4UIM4bcxhZmYd0PKHFiW9D3gc+GJE/CYvazRs2iAWbcTH3Z1W+kjqp34ajK6uLmq12gTDNjc8PFzU/2zhPLXGeWqN89SamZCnloqJpPdQLyTbIuK7GT4uaUFEHMtTTCcyPgQsrnRfBBzNeO+YeC3jixq0b2eO00TEFmALQE9PT5R8QvTebdvZ/KPftt2/XUfu+vNpn7OEP7HcGuepNc5Ta2ZCnlq5m0vA/cBLEfG1yqYdwOgdWX3A9kp8fd5xtQJ4I09R7QJWSpqfF95XArty25uSVuRc68eMNZk5zMysA1o5Mvk48BnggKR9Gfsr4C7gUUkbgF8C1+W2ncBqYBB4C7gBICJOSroDeC7b3R4RJ3P9RuABYB7wZC5Mdg4zM+uMCYtJRPyIxtcoAK5s0D6AjU3GGgAGGsT3Ahc3iL8+2TnMzGz6+RPwZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFXMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFXMxMTOzYi4mZmZWrJXvgB+QdELSi5XYI5L25XJk9Ot8JS2R9I+Vbd+s9LlM0gFJg5Luye97R9L5knZLOpSP8zOubDcoab+kSytj9WX7Q5L6MDOzjmrlyOQBYFU1EBH/JSKWR8Ry4HHgu5XNr4xui4jPV+L3Af1Ady6jY24C9kREN7AnnwNcXWnbn/2RdD5wK3AFcDlw62gBMjOzzpiwmETED4GTjbbl0cV/Bh4abwxJC4APRMSP8/vbHwSuyc1rgK25vnVM/MGoexo4L8e5CtgdEScj4hSwmzHFzszMplfpNZNPAMcj4lAltlTSC5L+TtInMrYQGKq0GcoYQFdEHAPIxwsrfV5t0KdZ3MzMOmRuYf91nH5Ucgz4o4h4XdJlwN9K+iigBn1jgrGb9Wl5LEn91E+R0dXVRa1Wm2DK5rrmwc2XjLTdv10l+9wJw8PDs26fO8F5ao3z1JqZkKe2i4mkucB/Ai4bjUXE28Dbuf68pFeAP6Z+9LCo0n0RcDTXj0taEBHH8jTWiYwPAYsb9BkCesfEa432MSK2AFsAenp6ore3t1Gzlty7bTubD5TW3sk7cn3vtM9ZolarUZLns4Xz1BrnqTUzIU8lp7n+A/CziPin01eSPixpTq5fRP3i+eE8ffWmpBV5nWU9sD277QBG78jqGxNfn3d1rQDeyHF2ASslzc8L7yszZmZmHTLhr9qSHqJ+JHCBpCHg1oi4H1jLH154/yRwu6QR4B3g8xExevH+Rup3hs0DnswF4C7gUUkbgF8C12V8J7AaGATeAm4AiIiTku4Anst2t1fmMDOzDpiwmETEuibxv2wQe5z6rcKN2u8FLm4Qfx24skE8gI1NxhoABsbbbzMzmz7+BLyZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRWbsJhIGpB0QtKLldhtkn4laV8uqyvbbpE0KOllSVdV4qsyNihpUyW+VNIzkg5JekTSORk/N58P5vYlE81hZmad0cqRyQPAqgbxuyNieS47ASQto/7d8B/NPt+QNEfSHODrwNXAMmBdtgX4ao7VDZwCNmR8A3AqIj4C3J3tms4xuZdtZmZTacJiEhE/BE62ON4a4OGIeDsifgEMApfnMhgRhyPi98DDwBpJAj4FPJb9twLXVMbamuuPAVdm+2ZzmJlZh5RcM7lJ0v48DTY/YwuBVytthjLWLP4h4NcRMTImftpYuf2NbN9sLDMz65C5bfa7D7gDiHzcDHwWUIO2QeOiFeO0Z5xt4/U5jaR+oB+gq6uLWq3WqFlLuubBzZeMTNxwipXscycMDw/Pun3uBOepNc5Ta2ZCntoqJhFxfHRd0reB7+XTIWBxpeki4GiuN4q/BpwnaW4efVTbj441JGku8EHqp9vGm2Psfm4BtgD09PREb2/vpF5n1b3btrP5QLu1t31Hru+d9jlL1Go1SvJ8tnCeWuM8tWYm5Kmt01ySFlSefhoYvdNrB7A278RaCnQDzwLPAd1559Y51C+g74iIAJ4Crs3+fcD2ylh9uX4t8INs32wOMzPrkAl/1Zb0ENALXCBpCLgV6JW0nPrppSPA5wAi4qCkR4GfAiPAxoh4J8e5CdgFzAEGIuJgTvEl4GFJXwFeAO7P+P3AdyQNUj8iWTvRHGZm1hkTFpOIWNcgfH+D2Gj7O4E7G8R3AjsbxA/T4G6siPgdcN1k5jAzs87wJ+DNzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKzYhMVE0oCkE5JerMT+h6SfSdov6QlJ52V8iaR/lLQvl29W+lwm6YCkQUn3SFLGz5e0W9KhfJyfcWW7wZzn0spYfdn+kKQ+zMyso1o5MnkAWDUmthu4OCL+DfBz4JbKtlciYnkun6/E7wP6ge5cRsfcBOyJiG5gTz4HuLrStj/7I+l86t9DfwX1r/u9dbQAmZlZZ0xYTCLih8DJMbHvR8RIPn0aWDTeGJIWAB+IiB9HRAAPAtfk5jXA1lzfOib+YNQ9DZyX41wF7I6IkxFxinphG1vszMxsGk3FNZPPAk9Wni+V9IKkv5P0iYwtBIYqbYYyBtAVEccA8vHCSp9XG/RpFjczsw6ZW9JZ0peBEWBbho4BfxQRr0u6DPhbSR8F1KB7TDR8kz4tjyWpn/opMrq6uqjVahNM2VzXPLj5kpGJG06xkn3uhOHh4Vm3z53gPLXGeWrNTMhT28UkL3z/R+DKPHVFRLwNvJ3rz0t6Bfhj6kcP1VNhi4CjuX5c0oKIOJansU5kfAhY3KDPENA7Jl5rtI8RsQXYAtDT0xO9vb2NmrXk3m3b2XygqPa25cj1vdM+Z4larUZJns8WzlNrnKfWzIQ8tXWaS9Iq4EvAX0TEW5X4hyXNyfWLqF88P5ynr96UtCLv4loPbM9uO4DRO7L6xsTX511dK4A3cpxdwEpJ8/PC+8qMmZlZh0z4q7akh6gfCVwgaYj6nVS3AOcCu/MO36fzzq1PArdLGgHeAT4fEaMX72+kfmfYPOrXWEavs9wFPCppA/BL4LqM7wRWA4PAW8ANABFxUtIdwHPZ7vbKHGZm1gETFpOIWNcgfH+Tto8DjzfZthe4uEH8deDKBvEANjYZawAYaL7XZmY2nfwJeDMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK9ZSMZE0IOmEpBcrsfMl7ZZ0KB/nZ1yS7pE0KGm/pEsrffqy/SFJfZX4ZZIOZJ978nvi25rDzMymX6tHJg8Aq8bENgF7IqIb2JPPAa4GunPpB+6DemGg/v3xVwCXA7eOFods01/pt6qdOczMrDNaKiYR8UPg5JjwGmBrrm8FrqnEH4y6p4HzJC0ArgJ2R8TJiDgF7AZW5bYPRMSP83vfHxwz1mTmMDOzDii5ZtIVEccA8vHCjC8EXq20G8rYePGhBvF25jAzsw6YewbGVINYtBFvZ47TG0n91E+D0dXVRa1Wm2DY5rrmwc2XjLTdv10l+9wJw8PDs26fO8F5ao3z1JqZkKeSYnJc0oKIOJanmE5kfAhYXGm3CDia8d4x8VrGFzVo384cp4mILcAWgJ6enujt7R3bpGX3btvO5gNnovaO78j1vdM+Z4larUZJns8WzlNrnKfWzIQ8lZzm2gGM3pHVB2yvxNfnHVcrgDfyFNUuYKWk+XnhfSWwK7e9KWlF3sW1fsxYk5nDzMw6oKVftSU9RP2o4gJJQ9TvyroLeFTSBuCXwHXZfCewGhgE3gJuAIiIk5LuAJ7LdrdHxOhF/Rup3zE2D3gyFyY7h5mZdUZLxSQi1jXZdGWDtgFsbDLOADDQIL4XuLhB/PXJzmFmZtPPn4A3M7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NibRcTSf9a0r7K8htJX5R0m6RfVeKrK31ukTQo6WVJV1XiqzI2KGlTJb5U0jOSDkl6RNI5GT83nw/m9iXtvg4zMyvXdjGJiJcjYnlELAcuo/5d7E/k5rtHt0XETgBJy4C1wEeBVcA3JM2RNAf4OnA1sAxYl20BvppjdQOngA0Z3wCcioiPAHdnOzMz65CpOs11JfBKRPzDOG3WAA9HxNsR8QtgELg8l8GIOBwRvwceBtZIEvAp4LHsvxW4pjLW1lx/DLgy25uZWQdMVTFZCzxUeX6TpP2SBiTNz9hC4NVKm6GMNYt/CPh1RIyMiZ82Vm5/I9ubmVkHzC0dIK9j/AVwS4buA+4AIh83A58FGh05BI0LWozTngm2VfetH+gH6OrqolarNXsZE+qaBzdfMjJxwylWss+dMDw8POv2uROcp9Y4T62ZCXkqLibUr3X8JCKOA4w+Akj6NvC9fDoELK70WwQczfVG8deA8yTNzaOPavvRsYYkzQU+CJwcu2MRsQXYAtDT0xO9vb1tv8h7t21n84GpSNfkHLm+d9rnLFGr1SjJ89nCeWqN89SamZCnqTjNtY7KKS5JCyrbPg28mOs7gLV5J9ZSoBt4FngO6M47t86hfspsR0QE8BRwbfbvA7ZXxurL9WuBH2R7MzPrgKJftSX9C+DPgM9Vwn8taTn1005HRrdFxEFJjwI/BUaAjRHxTo5zE7ALmAMMRMTBHOtLwMOSvgK8ANyf8fuB70gapH5EsrbkdZiZWZmiYhIRbzHmwndEfGac9ncCdzaI7wR2Nogfpn6319j474Dr2thlMzM7A/wJeDMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIq5mJiZWTEXEzMzK1ZcTCQdkXRA0j5JezN2vqTdkg7l4/yMS9I9kgYl7Zd0aWWcvmx/SFJfJX5Zjj+YfTXeHGZmNv2m6sjk30fE8ojoyeebgD0R0Q3syecAVwPdufQD90G9MAC3AldQ/5reWyvF4b5sO9pv1QRzmJnZNDtTp7nWAFtzfStwTSX+YNQ9DZwnaQFwFbA7Ik5GxClgN7Aqt30gIn4cEQE8OGasRnOYmdk0m4piEsD3JT0vqT9jXRFxDCAfL8z4QuDVSt+hjI0XH2oQH28OMzObZnOnYIyPR8RRSRcCuyX9bJy2ahCLNuItyeLWD9DV1UWtVmu16x/omgc3XzLSdv92lexzJwwPD8+6fe4E56k1zlNrZkKeiotJRBzNxxOSnqB+zeO4pAURcSxPVZ3I5kPA4kr3RcDRjPeOidcyvqhBe8aZo7pvW4AtAD09PdHb2zu2Scvu3badzQemovZOzpHre6d9zhK1Wo2SPJ8tnKfWOE+tmQl5KjrNJem9kt4/ug6sBF4EdgCjd2T1AdtzfQewPu/qWgG8kaeodgErJc3PC+8rgV257U1JK/IurvVjxmo0h5mZTbPSX7W7gCfybt25wN9ExP+R9BzwqKQNwC+B67L9TmA1MAi8BdwAEBEnJd0BPJftbo+Ik7l+I/AAMA94MheAu5rMYWZm06yomETEYeDfNoi/DlzZIB7AxiZjDQADDeJ7gYtbncPMzKafPwFvZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbFXEzMzKyYi4mZmRVzMTEzs2IuJmZmVszFxMzMirmYmJlZMRcTMzMr5mJiZmbF2i4mkhZLekrSS5IOSvpCxm+T9CtJ+3JZXelzi6RBSS9LuqoSX5WxQUmbKvGlkp6RdEjSI5LOyfi5+Xwwty9p93WYmVm5kiOTEeDmiPgTYAWwUdKy3HZ3RCzPZSdAblsLfBRYBXxD0hxJc4CvA1cDy4B1lXG+mmN1A6eADRnfAJyKiI8Ad2c7MzPrkLaLSUQci4if5PqbwEvAwnG6rAEejoi3I+IXwCBweS6DEXE4In4PPAyskSTgU8Bj2X8rcE1lrK25/hhwZbY3M7MOmJJrJnma6WPAMxm6SdJ+SQOS5mdsIfBqpdtQxprFPwT8OiJGxsRPGyu3v5HtzcysA+aWDiDpfcDjwBcj4jeS7gPuACIfNwOfBRodOQSNC1qM054JtlX3rR/oB+jq6qJWq437WsbTNQ9uvmRk4oZTrGSfO2F4eHjW7XMnOE+tcZ5aMxPyVFRMJL2HeiHZFhHfBYiI45Xt3wa+l0+HgMWV7ouAo7neKP4acJ6kuXn0UW0/OtaQpLnAB4GTY/cvIrYAWwB6enqit7e37dd677btbD5QXHsn7cj1vdM+Z4larUZJns8WzlNrnKfWzIQ8ldzNJeB+4KWI+FolvqDS7NPAi7m+A1ibd2ItBbqBZ4HngO68c+sc6hfpd0REAE8B12b/PmB7Zay+XL8W+EG2NzOzDij5VfvjwGeAA5L2ZeyvqN+NtZz6aacjwOcAIuKgpEeBn1K/E2xjRLwDIOkmYBcwBxiIiIM53peAhyV9BXiBevEiH78jaZD6EcnagtdhZmaF2i4mEfEjGl+72DlOnzuBOxvEdzbqFxGHqd/tNTb+O+C6yeyvmZmdOf4EvJmZFXMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFXMxMTOzYi4mZmZWzMXEzMyKuZiYmVkxFxMzMyvmYmJmZsVcTMzMrJiLiZmZFZvVxUTSKkkvSxqUtKnT+2NmdraatcVE0hzg68DVwDLq3z2/rLN7ZWZ2dpq1xYT6d8MPRsThiPg98DCwpsP7ZGZ2Vprb6R0osBB4tfJ8CLiiQ/tyxizZ9L87vQuTcvMlI/zlLNvnUUfu+vNO74LZrDWbi4kaxOK0BlI/0J9PhyW9XDDfBcBrBf3PCv9tFudJX53W6WZtnqaZ89SaM5mnf9lKo9lcTIaAxZXni4Cj1QYRsQXYMhWTSdobET1TMda7mfPUGuepNc5Ta2ZCnmbzNZPngG5JSyWdA6wFdnR4n8zMzkqz9sgkIkYk3QTsAuYAAxFxsMO7ZWZ2Vpq1xQQgInYCO6dpuik5XXYWcJ5a4zy1xnlqTcfzpIiYuJWZmdk4ZvM1EzMzmyFcTCZwtv7JFklHJB2QtE/S3oydL2m3pEP5OD/jknRP5mi/pEsr4/Rl+0OS+irxy3L8wezb6FbvGUfSgKQTkl6sxM54XprNMVM1ydNtkn6V76l9klZXtt2Sr/llSVdV4g1//vLGm2cyH4/kTThIOjefD+b2JdPzitsjabGkpyS9JOmgpC9kfPa9pyLCS5OF+oX9V4CLgHOAvweWdXq/pum1HwEuGBP7a2BTrm8Cvprrq4EnqX/2ZwXwTMbPBw7n4/xcn5/bngX+NPs8CVzd6dfcYl4+CVwKvDideWk2x0xdmuTpNuC/N2i7LH+2zgWW5s/cnPF+/oBHgbW5/k3gxlz/r8A3c30t8EinczFBnhYAl+b6+4GfZz5m3Xuq48mcyUv+A+yqPL8FuKXT+zVNr/0If1hMXgYW5PoC4OVc/xawbmw7YB3wrUr8WxlbAPysEj+t3UxfgCVj/pM843lpNsdMXhrk6TYaF5PTfq6o36H5p81+/vI/xdeAuRn/p3ajfXN9brZTp3MxiZxtB/5sNr6nfJprfI3+ZMvCDu3LdAvg+5KeV/0vCQB0RcQxgHy8MOPN8jRefKhBfLaajrw0m2O2uSlPzwxUTqtMNk8fAn4dESNj4qeNldvfyPYzXp6S+xjwDLPwPeViMr4J/2TLu9jHI+JS6n+VeaOkT47TtlmeJht/t3FeTncf8K+A5cAxYHPGpzJPszKHkt4HPA58MSJ+M17TBrEZ8Z5yMRnfhH+y5d0qIo7m4wngCep/pfm4pAUA+XgimzfL03jxRQ3is9V05KXZHLNGRByPiHci4v8B36b+noLJ5+k14DxJc8fETxsrt38QODn1r2bqSHoP9UKyLSK+m+FZ955yMRnfWfknWyS9V9L7R9eBlcCL1F/76F0ifdTP75Lx9XmnyQrgjTxs3gWslDQ/T2mspH5u+xjwpqQVeWfJ+spYs9F05KXZHLPG6H9c6dPU31NQf21r806spUA39YvGDX/+on6S/yng2uw/NuejeboW+EG2n5Hy3/l+4KWI+Fpl0+x7T3X6gtNMX6jfPfFz6neVfLnT+zNNr/ki6nfO/D1wcPR1Uz/3vAc4lI/nZ1zUv6jsFeAA0FMZ67PAYC43VOI91P8zeQX4n8ySi6TAQ9RP0fxf6r/1bZiOvDSbY6YuTfL0nczDfur/kS2otP9yvuaXqdzZ1+znL9+jz2b+/hdwbsb/eT4fzO0XdToXE+Tp31E/7bQf2JfL6tn4nvIn4M3MrJhPc5mZWTEXEzMzK+ZiYmZmxVxMzMysmIuJmZkVczExM7NiLiZmZlbMxcTMzIr9f6myGMVAdhI1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_features.item_seq_number.hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Since price and item seq number are highly skewed, we'll transform it into more normal like by using boxcox (more robust to outliers compared to np.log1p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGS5JREFUeJzt3X+s3XWd5/Hne1oZkVlsxeGGbZstExtHpFHhBjpjMrlrnVLEWP6QDYQZCmnSjUHFsclYJps0q0OCyTiMJErSSMey64pdRkOj1U6DnmwmUSwgK0IlvYMdeqcdUFuQK1H2Ou/943yqx8v3nPu5F+79nt4+H8nJ+Z739/P9fr7nw7m8+v1xzjcyE0mSavxO2xsgSTp9GBqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqotbXsDXm1vfOMbc/Xq1W1vxiv285//nHPOOaftzRhKjk0zx6U/x6ZZ77g8/PDDP8nM359pmUUXGqtXr+ahhx5qezNesU6nw9jYWNubMZQcm2aOS3+OTbPecYmIf6lZxsNTkqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqL7hvhkl5u9favtdLvkduvaqVfzZ+qPY2I+IuIeDwifhARX4yI10bEhRHxYEQcjogvRcRZpe3vltfjZf7qnvXcWupPRsQVPfWNpTYeEdt76o19SJLaMWNoRMQK4MPAaGZeDCwBrgU+CdyRmWuAk8CWssgW4GRmvgm4o7QjIi4qy70V2Ah8NiKWRMQS4DPAlcBFwHWlLQP6kCS1oPacxlLg7IhYCrwOOA68C7ivzN8NXF2mN5XXlPnrIyJK/d7M/GVm/ggYBy4rj/HMfCozXwLuBTaVZfr1IUlqwYyhkZn/CvwN8DTdsHgeeBh4LjOnSrMJYEWZXgEcLctOlfbn9danLdOvft6APiRJLZjxRHhELKe7l3Ah8Bzwv+keSpouTy3SZ16/elNwDWrftI1bga0AIyMjdDqdpmanlcnJyUXxPuaDY9Ns0LhsWzvVWJ9vw/Lfyc9Ms7mMS83VU+8GfpSZPwaIiC8Dfwwsi4ilZU9gJXCstJ8AVgET5XDW64ETPfVTepdpqv9kQB+/JTN3AjsBRkdHczH8br6//9+fY9Ns0Ljc2NbVU9ePtdLvdH5mms1lXGrOaTwNrIuI15XzDOuBJ4BvAe8vbTYD95fpveU1Zf43MzNL/dpyddWFwBrgu8BBYE25UuosuifL95Zl+vUhSWpBzTmNB+mejH4EeKwssxP4GPDRiBine/7h7rLI3cB5pf5RYHtZz+PAHrqB8w3g5sz8VdmL+CCwHzgE7CltGdCHJKkFVV/uy8wdwI5p5afoXvk0ve0vgGv6rOc24LaG+j5gX0O9sQ9JUjv8GRFJUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1WYMjYh4c0Q82vP4WUR8JCLeEBEHIuJweV5e2kdE3BkR4xHx/Yi4pGddm0v7wxGxuad+aUQ8Vpa5s9xWln59SJLaUXO71ycz8+2Z+XbgUuBF4Ct0b+P6QGauAR4orwGupHv/7zXAVuAu6AYA3bv/XU73bnw7ekLgrtL21HIbS71fH5KkFsz28NR64J8z81+ATcDuUt8NXF2mNwH3ZNd3gGURcQFwBXAgM09k5kngALCxzDs3M7+dmQncM21dTX1Iklow29C4FvhimR7JzOMA5fn8Ul8BHO1ZZqLUBtUnGuqD+pAktWBpbcOIOAt4H3DrTE0bajmHerWI2Er38BYjIyN0Op3ZLD6UJicnF8X7mA+OTbNB47Jt7dTCbkwxLP+d/Mw0m8u4VIcG3XMVj2TmM+X1MxFxQWYeL4eYni31CWBVz3IrgWOlPjat3in1lQ3tB/XxWzJzJ7ATYHR0NMfGxpqanVY6nQ6L4X3MB8em2aBxuXH71xZ2Y4oj14+10u90fmaazWVcZnN46jp+c2gKYC9w6gqozcD9PfUbylVU64Dny6Gl/cCGiFheToBvAPaXeS9ExLpy1dQN09bV1IckqQVVexoR8TrgT4H/2lO+HdgTEVuAp4FrSn0f8B5gnO6VVjcBZOaJiPgEcLC0+3hmnijTHwA+D5wNfL08BvUhSWpBVWhk5ovAedNqP6V7NdX0tgnc3Gc9u4BdDfWHgIsb6o19SJLa4TfCJUnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1apCIyKWRcR9EfHDiDgUEX8UEW+IiAMRcbg8Ly9tIyLujIjxiPh+RFzSs57Npf3hiNjcU780Ih4ry9xZ7hVOvz4kSe2o3dP4NPCNzPxD4G3AIWA78EBmrgEeKK8BrgTWlMdW4C7oBgCwA7gcuAzY0RMCd5W2p5bbWOr9+pAktWDG0IiIc4E/Ae4GyMyXMvM5YBOwuzTbDVxdpjcB92TXd4BlEXEBcAVwIDNPZOZJ4ACwscw7NzO/Xe4vfs+0dTX1IUlqwdKKNn8A/Bj4+4h4G/AwcAswkpnHATLzeEScX9qvAI72LD9RaoPqEw11BvTxWyJiK909FUZGRuh0OhVva7hNTk4uivcxHxybZoPGZdvaqYXdmGJY/jv5mWk2l3GpCY2lwCXAhzLzwYj4NIMPE0VDLedQr5aZO4GdAKOjozk2NjabxYdSp9NhMbyP+eDYNBs0Ljdu/9rCbkxx5PqxVvqdzs9Ms7mMS805jQlgIjMfLK/voxsiz5RDS5TnZ3var+pZfiVwbIb6yoY6A/qQJLVgxtDIzH8DjkbEm0tpPfAEsBc4dQXUZuD+Mr0XuKFcRbUOeL4cYtoPbIiI5eUE+AZgf5n3QkSsK1dN3TBtXU19SJJaUHN4CuBDwBci4izgKeAmuoGzJyK2AE8D15S2+4D3AOPAi6UtmXkiIj4BHCztPp6ZJ8r0B4DPA2cDXy8PgNv79CFJakFVaGTmo8Bow6z1DW0TuLnPenYBuxrqDwEXN9R/2tSHJKkdfiNcklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrWq0IiIIxHxWEQ8GhEPldobIuJARBwuz8tLPSLizogYj4jvR8QlPevZXNofjojNPfVLy/rHy7IxqA9JUjtms6fxnzPz7Zl56g5+24EHMnMN8EB5DXAlsKY8tgJ3QTcAgB3A5cBlwI6eELirtD213MYZ+pAkteCVHJ7aBOwu07uBq3vq92TXd4BlEXEBcAVwIDNPZOZJ4ACwscw7NzO/XW4Ve8+0dTX1IUlqQW1oJPCPEfFwRGwttZHMPA5Qns8v9RXA0Z5lJ0ptUH2ioT6oD0lSC5ZWtntnZh6LiPOBAxHxwwFto6GWc6hXK0G2FWBkZIROpzObxYfS5OTkongf88GxaTZoXLatnVrYjSmG5b+Tn5lmcxmXqtDIzGPl+dmI+ArdcxLPRMQFmXm8HGJ6tjSfAFb1LL4SOFbqY9PqnVJf2dCeAX1M376dwE6A0dHRHBsba2p2Wul0OiyG9zEfHJtmg8blxu1fW9iNKY5cP9ZKv9P5mWk2l3GZ8fBURJwTEf/h1DSwAfgBsBc4dQXUZuD+Mr0XuKFcRbUOeL4cWtoPbIiI5eUE+AZgf5n3QkSsK1dN3TBtXU19SJJaULOnMQJ8pVwFuxT4X5n5jYg4COyJiC3A08A1pf0+4D3AOPAicBNAZp6IiE8AB0u7j2fmiTL9AeDzwNnA18sD4PY+fUiSWjBjaGTmU8DbGuo/BdY31BO4uc+6dgG7GuoPARfX9iFJaoffCJckVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdVqfxpdkmZtdUu/rgtw5ParWut7MXNPQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVK06NCJiSUR8LyK+Wl5fGBEPRsThiPhSRJxV6r9bXo+X+at71nFrqT8ZEVf01DeW2nhEbO+pN/YhSWrHbPY0bgEO9bz+JHBHZq4BTgJbSn0LcDIz3wTcUdoRERcB1wJvBTYCny1BtAT4DHAlcBFwXWk7qA9JUguqQiMiVgJXAZ8rrwN4F3BfabIbuLpMbyqvKfPXl/abgHsz85eZ+SO69xC/rDzGM/OpzHwJuBfYNEMfkqQW1O5p/B3wl8C/l9fnAc9l5lR5PQGsKNMrgKMAZf7zpf2v69OW6Vcf1IckqQUzfiM8It4LPJuZD0fE2KlyQ9OcYV6/elNwDWrftI1bga0AIyMjdDqdpmanlcnJyUXxPuaDY9Ns0LhsWzvVWF/MesfCz0yzuYxLzc+IvBN4X0S8B3gtcC7dPY9lEbG07AmsBI6V9hPAKmAiIpYCrwdO9NRP6V2mqf6TAX38lszcCewEGB0dzbGxsYq3Ndw6nQ6L4X3MB8em2aBxubHFn/Noy5Hrx3497Wem2VzGZcbDU5l5a2auzMzVdE9kfzMzrwe+Bby/NNsM3F+m95bXlPnfzMws9WvL1VUXAmuA7wIHgTXlSqmzSh97yzL9+pAkteCVfE/jY8BHI2Kc7vmHu0v9buC8Uv8osB0gMx8H9gBPAN8Abs7MX5W9iA8C++lenbWntB3UhySpBbP6ldvM7ACdMv0U3Sufprf5BXBNn+VvA25rqO8D9jXUG/uQJLXDb4RLkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKnarO6nIWnuVs/zLVe3rZ06I2/rqoU1455GRLw2Ir4bEf83Ih6PiP9e6hdGxIMRcTgivlRu1Uq5neuXImK8zF/ds65bS/3JiLiip76x1MYjYntPvbEPSVI7ag5P/RJ4V2a+DXg7sDEi1gGfBO7IzDXASWBLab8FOJmZbwLuKO2IiIvo3v/7rcBG4LMRsSQilgCfAa4ELgKuK20Z0IckqQUzhkZ2TZaXrymPBN4F3Ffqu4Gry/Sm8poyf31ERKnfm5m/zMwfAeN0b+V6GTCemU9l5kvAvcCmsky/PiRJLag6EV72CB4FngUOAP8MPJeZU6XJBLCiTK8AjgKU+c8D5/XWpy3Tr37egD4kSS2oOhGemb8C3h4Ry4CvAG9palaeo8+8fvWm4BrU/mUiYiuwFWBkZIROp9PU7LQyOTm5KN7HfDhdx2bb2qmZG70CI2fPfx+nk97PyOn6mZlvcxmXWV09lZnPRUQHWAcsi4ilZU9gJXCsNJsAVgETEbEUeD1woqd+Su8yTfWfDOhj+nbtBHYCjI6O5tjY2Gze1lDqdDoshvcxH07XsZnvK5u2rZ3iU495QeQpR64f+/X06fqZmW9zGZeaq6d+v+xhEBFnA+8GDgHfAt5fmm0G7i/Te8tryvxvZmaW+rXl6qoLgTXAd4GDwJpypdRZdE+W7y3L9OtDktSCmn+WXADsLlc5/Q6wJzO/GhFPAPdGxF8D3wPuLu3vBv5HRIzT3cO4FiAzH4+IPcATwBRwcznsRUR8ENgPLAF2ZebjZV0f69OHJKkFM4ZGZn4feEdD/Sm6Vz5Nr/8CuKbPum4Dbmuo7wP21fYhSWqHPyMiSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKq+ZOYkhal3nuyL+T904/cftWC9NMW9zQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrWae4SviohvRcShiHg8Im4p9TdExIGIOFyel5d6RMSdETEeEd+PiEt61rW5tD8cEZt76pdGxGNlmTsjIgb1IUlqR82exhSwLTPfAqwDbo6Ii4DtwAOZuQZ4oLwGuBJYUx5bgbugGwDADuByurdw3dETAneVtqeW21jq/fqQJLVgxtDIzOOZ+UiZfgE4BKwANgG7S7PdwNVlehNwT3Z9B1gWERcAVwAHMvNEZp4EDgAby7xzM/PbmZnAPdPW1dSHJKkFs/pGeESsBt4BPAiMZOZx6AZLRJxfmq0AjvYsNlFqg+oTDXUG9DF9u7bS3VNhZGSETqczm7c1lCYnJxfF+5gPp+vYbFs7Na/rHzl7/vs4XS3k2JxOn825/C1Vh0ZE/B7wD8BHMvNn5bRDY9OGWs6hXi0zdwI7AUZHR3NsbGw2iw+lTqfDYngf8+F0HZv5/hmLbWun+NRj/jJQk4UcmyPXjy1IP6+GufwtVV09FRGvoRsYX8jML5fyM+XQEuX52VKfAFb1LL4SODZDfWVDfVAfkqQW1Fw9FcDdwKHM/NueWXuBU1dAbQbu76nfUK6iWgc8Xw4x7Qc2RMTycgJ8A7C/zHshItaVvm6Ytq6mPiRJLajZX3sn8OfAYxHxaKn9FXA7sCcitgBPA9eUefuA9wDjwIvATQCZeSIiPgEcLO0+npknyvQHgM8DZwNfLw8G9CFJasGMoZGZ/0TzeQeA9Q3tE7i5z7p2Absa6g8BFzfUf9rUhySpHX4jXJJUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVK1GW/CFBG7gPcCz2bmxaX2BuBLwGrgCPBfMvNkuV3rp+neue9F4MbMfKQssxn4b2W1f52Zu0v9Un5z1759wC2Zmf36eMXveIDV2782n6vv68jtV7XSryTNVs2exueBjdNq24EHMnMN8EB5DXAlsKY8tgJ3wa9DZgdwOXAZsKPcJ5zSZmvPchtn6EOS1JIZQyMz/w9wYlp5E7C7TO8Gru6p35Nd3wGWRcQFwBXAgcw8UfYWDgAby7xzM/Pb5Tax90xbV1MfkqSWzPWcxkhmHgcoz+eX+grgaE+7iVIbVJ9oqA/qQ5LUkhnPacxSNNRyDvXZdRqxle4hLkZGRuh0OrNdBQDb1k7NablXqml7Jycn5/w+FrvTdWzm+/M1cnZ7n+Fht5Bjczp9NufytzTX0HgmIi7IzOPlENOzpT4BrOpptxI4Vupj0+qdUl/Z0H5QHy+TmTuBnQCjo6M5NjbWr+lAN7Z1Ivz6sZfVOp0Oc30fi93pOjbz/fnatnaKTz32av87cHFYyLFp+nseVnP5W5rr4am9wOYyvRm4v6d+Q3StA54vh5b2AxsiYnk5Ab4B2F/mvRAR68qVVzdMW1dTH5KkltRccvtFunsJb4yICbpXQd0O7ImILcDTwDWl+T66l9uO073k9iaAzDwREZ8ADpZ2H8/MUyfXP8BvLrn9enkwoA9JUktmDI3MvK7PrPUNbRO4uc96dgG7GuoPARc31H/a1IckqT1+I1ySVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1f3xfZ5zVLd03RVoMDA1JehW19Y+SI7dftSD9eHhKklTN0JAkVRv60IiIjRHxZESMR8T2trdHks5kQ31OIyKWAJ8B/hSYAA5GxN7MfKLdLXt1NR0D3bZ2ihsX4NjoQh0HlbQ4DHVoAJcB45n5FEBE3AtsAhZVaJyJXsnJwoUKVEkvN+yhsQI42vN6Ari8pW1ZlLz8VNJsDHtoREMtX9YoYiuwtbycjIgn53WrFsCH4Y3AT9rejmHk2DRzXPo7E8YmPjmnxXrH5T/VLDDsoTEBrOp5vRI4Nr1RZu4Edi7URi2EiHgoM0fb3o5h5Ng0c1z6c2yazWVchv3qqYPAmoi4MCLOAq4F9ra8TZJ0xhrqPY3MnIqIDwL7gSXArsx8vOXNkqQz1lCHBkBm7gP2tb0dLVhUh9teZY5NM8elP8em2azHJTJfdl5ZkqRGw35OQ5I0RAyNIeRPp7xcRKyKiG9FxKGIeDwibml7m4ZJRCyJiO9FxFfb3pZhEhHLIuK+iPhh+ez8UdvbNAwi4i/K39EPIuKLEfHa2mUNjSHT89MpVwIXAddFxEXtbtVQmAK2ZeZbgHXAzY7Lb7kFONT2RgyhTwPfyMw/BN6GY0RErAA+DIxm5sV0LzK6tnZ5Q2P4/PqnUzLzJeDUT6ec0TLzeGY+UqZfoPvHv6LdrRoOEbESuAr4XNvbMkwi4lzgT4C7ATLzpcx8rt2tGhpLgbMjYinwOhq+/9aPoTF8mn46xf859oiI1cA7gAfb3ZKh8XfAXwL/3vaGDJk/AH4M/H05dPe5iDin7Y1qW2b+K/A3wNPAceD5zPzH2uUNjeFT9dMpZ6qI+D3gH4CPZObP2t6etkXEe4FnM/PhtrdlCC0FLgHuysx3AD8HzvhzhBGxnO7RiwuB/wicExF/Vru8oTF8qn465UwUEa+hGxhfyMwvt709Q+KdwPsi4gjdQ5nvioj/2e4mDY0JYCIzT+2R3kc3RM507wZ+lJk/zsz/B3wZ+OPahQ2N4eNPpzSIiKB7bPpQZv5t29szLDLz1sxcmZmr6X5WvpmZ1f9qXMwy89+AoxHx5lJaj7dVgO5hqXUR8bryd7WeWVwgMPTfCD/T+NMpfb0T+HPgsYh4tNT+qvxigNTPh4AvlH+APQXc1PL2tC4zH4yI+4BH6F6V+D1m8c1wvxEuSarm4SlJUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdX+P8snVDPgznqBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD8CAYAAACLrvgBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEmxJREFUeJzt3W+MnWWZx/HvZQtSYbEI7IS0zRZjY6w2qzDBbsiaEVwoQiwvJIGwUgybbgwajE2kmGyIf0jqC8RglKShXcsua+2ihgaqbAOduCYqUERrqSwjdmWEpYstyOAfMnrti3MXzw5n5twznc5zTvl+kpM5z3Xu57mvmZz2N8+f80xkJpIk1Xhd0w1IkvqHoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqdr8phuYbaeddlouXbp0Ruu+9NJLnHjiibPb0FHUb/1C//Xcb/1C//Vsv0dfTc+7d+9+LjNP77qxzDymHmeffXbO1K5du2a8bhP6rd/M/uu53/rN7L+e7ffoq+kZeDgr/o/18JQkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSp2jF3GxGpm6Xr760eu27FOFdPY/xU9m+4eFa2IzXJPQ1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUrXq0IiIeRHxo4i4pyyfGRE/jIgnIuLrEXF8qb++LI+U15e2beOGUn88Ii5sq68qtZGIWN9W7ziHJKkZ09nTuA7Y17b8eeCWzFwGHAKuKfVrgEOZ+RbgljKOiFgOXA68HVgFfKUE0Tzgy8BFwHLgijJ2qjkkSQ2oCo2IWAxcDNxelgM4D7irDNkCXFqery7LlNfPL+NXA1sz8w+Z+QtgBDinPEYy88nMfBnYCqzuMockqQG1expfBD4J/Kksnwo8n5njZXkUWFSeLwKeAiivv1DGv1KfsM5k9anmkCQ1YH63ARFxCXAgM3dHxNDhcoeh2eW1yeqdgmuq8Z16XAusBRgYGGB4eLjTsK7GxsZmvG4T+q1f6I2e160Y7z6oGFgwvfFTmavvuxd+xtNhv0ffbPbcNTSAc4EPRMT7gROAk2nteSyMiPllT2Ax8HQZPwosAUYjYj7wRuBgW/2w9nU61Z+bYo7/JzM3AhsBBgcHc2hoqOLberXh4WFmum4T+q1f6I2er15/b/XYdSvGuXlPzT+T7vZfOTQr2+mmF37G02G/R99s9tz18FRm3pCZizNzKa0T2Q9k5pXALuCDZdga4O7yfHtZprz+QGZmqV9erq46E1gGPAg8BCwrV0odX+bYXtaZbA5JUgOO5HMa1wOfiIgRWucfNpX6JuDUUv8EsB4gM/cC24DHgO8A12bmH8texEeB+2hdnbWtjJ1qDklSA6a1352Zw8Bwef4krSufJo75PXDZJOvfBNzUob4D2NGh3nEOSVIz/ES4JKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqs3O/RGkaVo6jVt5SOod7mlIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKq+Zf7pDkyV3+tcN2Kca6eMNf+DRfPydw69rmnIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSarWNTQi4oSIeDAifhwReyPi06V+ZkT8MCKeiIivR8Txpf76sjxSXl/atq0bSv3xiLiwrb6q1EYiYn1bveMckqRm1Oxp/AE4LzP/GngnsCoiVgKfB27JzGXAIeCaMv4a4FBmvgW4pYwjIpYDlwNvB1YBX4mIeRExD/gycBGwHLiijGWKOSRJDegaGtkyVhaPK48EzgPuKvUtwKXl+eqyTHn9/IiIUt+amX/IzF8AI8A55TGSmU9m5svAVmB1WWeyOSRJDag6p1H2CB4FDgA7gZ8Dz2fmeBkyCiwqzxcBTwGU118ATm2vT1hnsvqpU8whSWpA1V1uM/OPwDsjYiHwLeBtnYaVrzHJa5PVOwXXVONfJSLWAmsBBgYGGB4e7jSsq7GxsRmv24R+6xf+3PO6FePdB/eAgQX0Ta+Hdeq5l98n/fY+7rd+YXZ7ntat0TPz+YgYBlYCCyNiftkTWAw8XYaNAkuA0YiYD7wRONhWP6x9nU7156aYY2JfG4GNAIODgzk0NDSdb+sVw8PDzHTdJvRbv/DnnifeurtXrVsxzs17+usvCHTqef+VQ800U6Hf3sf91i/Mbs81V0+dXvYwiIgFwPuAfcAu4INl2Brg7vJ8e1mmvP5AZmapX16urjoTWAY8CDwELCtXSh1P62T59rLOZHNIkhpQ8yvUGcCWcpXT64BtmXlPRDwGbI2IzwE/AjaV8ZuAf4mIEVp7GJcDZObeiNgGPAaMA9eWw15ExEeB+4B5wObM3Fu2df0kc0iSGtA1NDLzJ8C7OtSfpHXl08T674HLJtnWTcBNHeo7gB21c0iSmuEnwiVJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRV668/fqxZt3SO/1b3uhXjffP3wSW9mnsakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqtY1NCJiSUTsioh9EbE3Iq4r9TdFxM6IeKJ8PaXUIyJujYiRiPhJRJzVtq01ZfwTEbGmrX52ROwp69waETHVHJKkZtTsaYwD6zLzbcBK4NqIWA6sB+7PzGXA/WUZ4CJgWXmsBW6DVgAANwLvBs4BbmwLgdvK2MPrrSr1yeaQJDWga2hk5jOZ+Uh5/iKwD1gErAa2lGFbgEvL89XAHdnyA2BhRJwBXAjszMyDmXkI2AmsKq+dnJnfz8wE7piwrU5zSJIaEK3/pysHRywFvgu8A/hlZi5se+1QZp4SEfcAGzLze6V+P3A9MASckJmfK/V/An4HDJfx7yv1vwWuz8xLIuL5TnN06GstrT0VBgYGzt66dWv199TuwMEXePZ3M1r1iKxY9MYZrTc2NsZJJ510RHPv+dULR7T+dA0soJGf8Uz1W7/QueeZvsfmwmy8j+dSv/ULdT2/973v3Z2Zg922Nb920og4CfgG8PHM/E057dBxaIdazqBeLTM3AhsBBgcHc2hoaDqrv+JLd97NzXuqfySzZv+VQzNab3h4mJl+r4ddvf7eI1p/utatGG/kZzxT/dYvdO55pu+xuTAb7+O51G/9wuz2XHX1VEQcRysw7szMb5bys+XQEuXrgVIfBZa0rb4YeLpLfXGH+lRzSJIaUHP1VACbgH2Z+YW2l7YDh6+AWgPc3Va/qlxFtRJ4ITOfAe4DLoiIU8oJ8AuA+8prL0bEyjLXVRO21WkOSVIDava7zwU+BOyJiEdL7VPABmBbRFwD/BK4rLy2A3g/MAL8FvgwQGYejIjPAg+VcZ/JzIPl+UeArwILgG+XB1PMIUlqQNfQKCe0JzuBcX6H8QlcO8m2NgObO9QfpnVyfWL9153mkCQ1w0+ES5KqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkavObbkDS0bd0/b2NzLt/w8WNzKujxz0NSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTNS257wEwvh1y3YpyrG7qUUtJrk3sakqRqhoYkqZqhIUmqZmhIkqoZGpKkal1DIyI2R8SBiPhpW+1NEbEzIp4oX08p9YiIWyNiJCJ+EhFnta2zpox/IiLWtNXPjog9ZZ1bIyKmmkOS1JyaPY2vAqsm1NYD92fmMuD+sgxwEbCsPNYCt0ErAIAbgXcD5wA3toXAbWXs4fVWdZlDktSQrqGRmd8FDk4orwa2lOdbgEvb6ndkyw+AhRFxBnAhsDMzD2bmIWAnsKq8dnJmfj8zE7hjwrY6zSFJashMz2kMZOYzAOXrX5b6IuCptnGjpTZVfbRDfao5JEkNme1PhEeHWs6gPr1JI9bSOsTFwMAAw8PD090EAAMLWp+y7hf91i/0X8/91i/0Vs81/xbHxsZm/G+2Cf3WL8xuzzMNjWcj4ozMfKYcYjpQ6qPAkrZxi4GnS31oQn241Bd3GD/VHK+SmRuBjQCDg4M5NDQ02dApfenOu7l5T//cWWXdivG+6hf6r+d+6xd6q+f9Vw51HTM8PMxM/802od/6hdnteaaHp7YDh6+AWgPc3Va/qlxFtRJ4oRxaug+4ICJOKSfALwDuK6+9GBEry1VTV03YVqc5JEkN6frrSER8jdZewmkRMUrrKqgNwLaIuAb4JXBZGb4DeD8wAvwW+DBAZh6MiM8CD5Vxn8nMwyfXP0LrCq0FwLfLgynmkCQ1pGtoZOYVk7x0foexCVw7yXY2A5s71B8G3tGh/utOc0iSmuMnwiVJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVeuOuZpKOSUvX39t1zLoV41xdMW669m+4eNa3Kfc0JEnTYGhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqnkbEUnHpJpbmMxEt9ueHOu3L3FPQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVfM2IpI0i47W7Uu6mavbl7inIUmqZmhIkqoZGpKkaj0fGhGxKiIej4iRiFjfdD+S9FrW06EREfOALwMXAcuBKyJiebNdSdJrV0+HBnAOMJKZT2bmy8BWYHXDPUnSa1avh8Yi4Km25dFSkyQ1IDKz6R4mFRGXARdm5j+U5Q8B52TmxyaMWwusLYtvBR6f4ZSnAc/NcN0m9Fu/0H8991u/0H892+/RV9PzX2Xm6d021Osf7hsFlrQtLwaenjgoMzcCG490soh4ODMHj3Q7c6Xf+oX+67nf+oX+69l+j77Z7LnXD089BCyLiDMj4njgcmB7wz1J0mtWT+9pZOZ4RHwUuA+YB2zOzL0NtyVJr1k9HRoAmbkD2DFH0x3xIa451m/9Qv/13G/9Qv/1bL9H36z13NMnwiVJvaXXz2lIknqIoVH00+1KImJzRByIiJ823UuNiFgSEbsiYl9E7I2I65ruqZuIOCEiHoyIH5eeP910TzUiYl5E/Cgi7mm6lxoRsT8i9kTEoxHxcNP9dBMRCyPiroj4WXk//03TPU0mIt5afq6HH7+JiI8f8XY9PPXK7Ur+C/g7Wpf5PgRckZmPNdrYJCLiPcAYcEdmvqPpfrqJiDOAMzLzkYj4C2A3cGmv/nwBIiKAEzNzLCKOA74HXJeZP2i4tSlFxCeAQeDkzLyk6X66iYj9wGBm9sXnHiJiC/CfmXl7uaLzDZn5fNN9dVP+j/sV8O7M/O8j2ZZ7Gi19dbuSzPwucLDpPmpl5jOZ+Uh5/iKwjx7/ZH+2jJXF48qjp3/DiojFwMXA7U33ciyKiJOB9wCbADLz5X4IjOJ84OdHGhhgaBzm7UrmSEQsBd4F/LDZTrorh3oeBQ4AOzOz13v+IvBJ4E9NNzINCfxHROwud3boZW8G/hf453II8PaIOLHppipdDnxtNjZkaLREh1pP/1bZjyLiJOAbwMcz8zdN99NNZv4xM99J604E50REzx4KjIhLgAOZubvpXqbp3Mw8i9adrK8th1571XzgLOC2zHwX8BLQ0+c/AcphtA8A/z4b2zM0WqpuV6KZK+cFvgHcmZnfbLqf6SiHIIaBVQ23MpVzgQ+UcwRbgfMi4l+bbam7zHy6fD0AfIvWoeJeNQqMtu1x3kUrRHrdRcAjmfnsbGzM0GjxdiVHUTmpvAnYl5lfaLqfGhFxekQsLM8XAO8DftZsV5PLzBsyc3FmLqX1/n0gM/++4bamFBEnlgsjKId5LgB69orAzPwf4KmIeGspnQ/07MUcba5glg5NQR98Inwu9NvtSiLia8AQcFpEjAI3ZuamZrua0rnAh4A95RwBwKfKp/171RnAlnLVyeuAbZnZF5ex9pEB4Fut3ymYD/xbZn6n2Za6+hhwZ/nl8kngww33M6WIeAOtq0L/cda26SW3kqRaHp6SJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTt/wDOR+pwWRt7JQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for col in ['price', 'item_seq_number']:\n",
    "    select_filter = all_features[col] > 0\n",
    "    all_features.loc[select_filter, col], _ = boxcox(all_features.loc[select_filter, col])\n",
    "    all_features[col].hist()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['item_id',\n",
       " 'user_id',\n",
       " 'region',\n",
       " 'city',\n",
       " 'parent_category_name',\n",
       " 'category_name',\n",
       " 'param_1',\n",
       " 'param_2',\n",
       " 'param_3',\n",
       " 'description',\n",
       " 'price',\n",
       " 'item_seq_number',\n",
       " 'activation_date',\n",
       " 'user_type',\n",
       " 'image',\n",
       " 'image_top_1',\n",
       " 'avg_days_up_user',\n",
       " 'avg_times_up_user',\n",
       " 'med_days_up_user',\n",
       " 'med_times_up_user',\n",
       " 'n_user_items',\n",
       " 'weekday',\n",
       " 'num_words_description',\n",
       " 'num_unique_words_description',\n",
       " 'num_words_title',\n",
       " 'num_unique_words_title',\n",
       " 'words_vs_unique_title',\n",
       " 'words_vs_unique_description',\n",
       " 'num_desc_punct']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_features.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "numerical = [\n",
    "    'weekday', 'num_words_title', 'num_words_description',\n",
    "    'num_unique_words_title', 'num_unique_words_description',\n",
    "    'words_vs_unique_title', 'words_vs_unique_description',\n",
    "    'num_desc_punct', 'avg_times_up_user', 'avg_days_up_user', \n",
    "    'med_times_up_user', 'med_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "]\n",
    "\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n",
    "]\n",
    "\n",
    "features = numerical+categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transforming image_top_1...\n",
      "Transforming param_1...\n",
      "Transforming param_2...\n",
      "Transforming param_3...\n",
      "Transforming city...\n",
      "Transforming region...\n",
      "Transforming category_name...\n",
      "Transforming parent_category_name...\n",
      "Transforming user_type...\n"
     ]
    }
   ],
   "source": [
    "# label encode categorical data\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "for feature in categorical:\n",
    "    print('Transforming {}...'.format(feature))\n",
    "    encoder = LabelEncoder()\n",
    "    all_features.loc[:, feature] = encoder.fit_transform(all_features[feature].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardize numerical data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "all_features.loc[:, numerical] = scaler.fit_transform(all_features[numerical].values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Prepare  pre-trained embeddings and text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 200000\n",
    "maxlen = 150\n",
    "embed_size = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = all_features.description.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fitting tokenizer\n"
     ]
    }
   ],
   "source": [
    "print('fitting tokenizer')\n",
    "tokenizer = text.Tokenizer(num_words=max_features)\n",
    "tokenizer.fit_on_texts(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting embeddings\n"
     ]
    }
   ],
   "source": [
    "print('getting embeddings')\n",
    "def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n",
    "embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(embedding_file, encoding = 'utf8'))\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "nb_words = min(max_features, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words, embed_size))\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setup max info for embedding in categorical variables\n"
     ]
    }
   ],
   "source": [
    "print('setup max info for embedding in categorical variables')\n",
    "max_info = dict((col, all_features[col].max()+1) for col in categorical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convert to sequences\n"
     ]
    }
   ],
   "source": [
    "print('convert to sequences')\n",
    "texts = tokenizer.texts_to_sequences(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "padding\n"
     ]
    }
   ],
   "source": [
    "print('padding')\n",
    "texts = sequence.pad_sequences(texts, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def root_mean_squared_error(y_true, y_pred):\n",
    "    return K.sqrt(K.mean(K.square(y_true - y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        print(input_shape)\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n",
    "                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        if mask is not None:\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clip_rmse(true, prediction):\n",
    "    return np.sqrt(metrics.mean_squared_error(true, np.clip(prediction, 0., 1.)))\n",
    "    \n",
    "class NBatchEvalLogger(Callback):\n",
    "    def __init__(self, display, val_X, val_y, save_path=None, save_start=1000):\n",
    "        self.step = 0\n",
    "        self.display = display\n",
    "        self.val_X = val_X\n",
    "        self.val_y = val_y\n",
    "        self.best_loss = None\n",
    "        self.save_path = save_path\n",
    "        self.save_start = save_start\n",
    "        self.record_count = 0\n",
    "        \n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        self.step += 1\n",
    "        if self.step % self.display == 0 and self.step >= self.save_start:\n",
    "            #loss, metric = self.model.evaluate(self.val_X, self.val_y, batch_size=128, verbose=1)\n",
    "            prediction = self.model.predict(self.val_X, batch_size=128, verbose=0)\n",
    "            loss = clip_rmse(self.val_y, prediction)\n",
    "            \n",
    "            if self.best_loss is None:\n",
    "                self.best_loss = loss\n",
    "            else:\n",
    "                if loss < self.best_loss:\n",
    "                    self.best_loss = loss\n",
    "                    if self.save_path is not None:\n",
    "                        self.model.save(self.save_path, overwrite=True)\n",
    "                        self.record_count += 1\n",
    "                    \n",
    "            print('\\rstep: {} val loss={:.5f}, best loss={:.5f}'.format(self.step, loss, self.best_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from copy import deepcopy as cp\n",
    "\n",
    "class DataGenerator(keras.utils.Sequence):\n",
    "    #'Generates data for Keras'\n",
    "    def __init__(self, list_IDs, X, y, batch_size=32, shuffle=True, is_train=True, img_path=None):\n",
    "        #'Initialization'\n",
    "        self.batch_size = batch_size\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.list_IDs = list_IDs\n",
    "        self.shuffle = shuffle\n",
    "        self.img_path = img_path\n",
    "        self.is_train = is_train\n",
    "        self.on_epoch_end()\n",
    "\n",
    "    def __len__(self):\n",
    "        #'Denotes the number of batches per epoch'\n",
    "        return int(np.ceil(len(self.list_IDs) / self.batch_size))\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        #'Generate one batch of data'\n",
    "        # Generate indexes of the batch\n",
    "        start = index*self.batch_size\n",
    "        end = min((index+1)*self.batch_size, len(self.indexes))\n",
    "        indexes = self.indexes[start: end]\n",
    "\n",
    "        # Generate data\n",
    "        return self.__data_generation(indexes)\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        #'Updates indexes after each epoch'\n",
    "        self.indexes = cp(list(self.list_IDs))\n",
    "        if self.shuffle == True:\n",
    "            np.random.shuffle(self.indexes)\n",
    "\n",
    "    def __data_generation(self, list_IDs_temp):\n",
    "        #'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)\n",
    "        \n",
    "        # Generate data\n",
    "        X = dict((col, self.X.loc[list_IDs_temp, col].values) for col in features)\n",
    "        X['text'] = texts[list_IDs_temp,:]\n",
    "        \n",
    "        # add img data\n",
    "        for image_name in self.X.loc[list_IDs_temp, 'image'].values:\n",
    "            if image_name == 'no-image':\n",
    "                pass #TODO: all-zero image\n",
    "            else:\n",
    "                pass #TODO: load real image\n",
    "        \n",
    "        if self.is_train:\n",
    "            y = cp(self.y[list_IDs_temp])\n",
    "            return X, y\n",
    "        else:\n",
    "            return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(X, categorical_features, numerical_features):\n",
    "    \n",
    "    # non-cat features\n",
    "    non_cat_inputs = []\n",
    "    for col in numerical_features:\n",
    "        f = Input(shape=[1], name=col)\n",
    "        non_cat_inputs.append(f)\n",
    "        \n",
    "    # cat features\n",
    "    cat_inputs = []\n",
    "    cat_embeds = []\n",
    "    for col in categorical_features:\n",
    "        f = Input(shape=[1], name=col)\n",
    "        embed_dim = max_info[col].max()+1\n",
    "        if max_info[col].max() > 10:\n",
    "            reduced_dim = 10\n",
    "        else:\n",
    "            reduced_dim = 1\n",
    "        embed_f = Embedding(embed_dim, reduced_dim)(f)\n",
    "        flatten_f = Flatten()(embed_f)\n",
    "        cat_inputs.append(f)\n",
    "        cat_embeds.append(flatten_f)\n",
    "      \n",
    "    # text features: architecture of text to try here!!!\n",
    "    text_inp = Input(shape = (maxlen, ), name='text')\n",
    "    \n",
    "    text_emb = Embedding(nb_words, embed_size, weights = [embedding_matrix],\n",
    "                    input_length = maxlen, trainable = False)(text_inp)\n",
    "    text_emb = SpatialDropout1D(0.3)(text_emb)\n",
    "    \n",
    "    text_gru = CuDNNGRU(128, return_sequences=True)(text_emb)\n",
    "    text_gru = CuDNNGRU(64)(text_gru)\n",
    "    text_gru = Dropout(0.1)(text_gru)\n",
    "    \n",
    "    #text_gru = Bidirectional(CuDNNGRU(64, return_sequences = True))(text_emb)\n",
    "    #text_gru = Attention(maxlen)(text_gru)\n",
    "    #text_gru = Conv1D(32, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(text_gru)\n",
    "    #text_gru_avg = GlobalAveragePooling1D()(text_gru)\n",
    "    #text_gru_max = GlobalMaxPooling1D()(text_gru)\n",
    "    #text_gru = concatenate([text_gru_avg, text_gru_max]) \n",
    "    #text_gru = Dropout(0.1)(text_gru)\n",
    "    \n",
    "    # add image architecture ???\n",
    "    \n",
    "    # merge each branch: non-cat, cat, text\n",
    "    concat_main = non_cat_inputs+cat_embeds+[text_gru]\n",
    "    main = concatenate(concat_main)\n",
    "    main = BatchNormalization()(main)\n",
    "    main = Dropout(0.1)(main)\n",
    "    main = BatchNormalization()(Dense(512, activation='relu')(main))\n",
    "    main = Dropout(0.1)(main)\n",
    "    main = BatchNormalization()(Dense(64, activation='relu')(main))\n",
    "    out = Dense(1, activation = \"sigmoid\")(main)\n",
    "\n",
    "    concat_input = non_cat_inputs+cat_inputs+[text_inp]\n",
    "    model = Model(concat_input, out)\n",
    "    model.regularizers = [regularizers.l2(0.0001)]\n",
    "    model.compile(optimizer = Adam(lr=0.001), loss = root_mean_squared_error,\n",
    "                  metrics =[root_mean_squared_error])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import warnings; warnings.filterwarnings('ignore') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_indices = np.arange(0, train_len)\n",
    "test_indices = np.arange(train_len, all_features.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras_tqdm import TQDMNotebookCallback\n",
    "from ipywidgets import IntProgress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From E:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text (InputLayer)               (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)        (None, 150, 300)     60000000    text[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_1 (SpatialDro (None, 150, 300)     0           embedding_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "image_top_1 (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_1 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "city (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "region (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_name (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "parent_category_name (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_type (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_1 (CuDNNGRU)          (None, 150, 128)     165120      spatial_dropout1d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 1, 10)        30650       image_top_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 1, 10)        3730        param_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 1, 10)        2790        param_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 1, 10)        12780       param_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 1, 10)        18250       city[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 1, 10)        290         region[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 1, 10)        480         category_name[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 1, 1)         10          parent_category_name[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_9 (Embedding)         (None, 1, 1)         4           user_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_2 (CuDNNGRU)          (None, 64)           37248       cu_dnngru_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "weekday (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_title (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_description (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_title (InputLa (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_description (I (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_title (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_description (In (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_desc_punct (InputLayer)     (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "price (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "item_seq_number (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 10)           0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 10)           0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 10)           0           embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 10)           0           embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 10)           0           embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 10)           0           embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 10)           0           embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_8 (Flatten)             (None, 1)            0           embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 1)            0           embedding_9[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 64)           0           cu_dnngru_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 151)          0           weekday[0][0]                    \n",
      "                                                                 num_words_title[0][0]            \n",
      "                                                                 num_words_description[0][0]      \n",
      "                                                                 num_unique_words_title[0][0]     \n",
      "                                                                 num_unique_words_description[0][0\n",
      "                                                                 words_vs_unique_title[0][0]      \n",
      "                                                                 words_vs_unique_description[0][0]\n",
      "                                                                 num_desc_punct[0][0]             \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 avg_days_up_user[0][0]           \n",
      "                                                                 med_times_up_user[0][0]          \n",
      "                                                                 med_days_up_user[0][0]           \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 price[0][0]                      \n",
      "                                                                 item_seq_number[0][0]            \n",
      "                                                                 flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "                                                                 flatten_3[0][0]                  \n",
      "                                                                 flatten_4[0][0]                  \n",
      "                                                                 flatten_5[0][0]                  \n",
      "                                                                 flatten_6[0][0]                  \n",
      "                                                                 flatten_7[0][0]                  \n",
      "                                                                 flatten_8[0][0]                  \n",
      "                                                                 flatten_9[0][0]                  \n",
      "                                                                 dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 151)          604         concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 151)          0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 512)          77824       dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 512)          2048        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 512)          0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64)           32832       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 64)           256         dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1)            65          batch_normalization_3[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 60,384,981\n",
      "Trainable params: 383,527\n",
      "Non-trainable params: 60,001,454\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3593261f6494368aaa78c6ec05ae1b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Training', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cca56eba4b94b21847a30a5a6f5cd21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 0', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.22470, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a848ed40b6f4144b5f81ab05f8807e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 1', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss improved from 0.22470 to 0.22243, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70a761af0d254b54b521caa1a98185ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 2', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00003: val_loss improved from 0.22243 to 0.22124, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed9cbf6c9543400d908c2ef24a46d003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 3', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00004: val_loss improved from 0.22124 to 0.22081, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c50e3a6bbb554d28baad7a6daad560ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 4', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00005: val_loss improved from 0.22081 to 0.22003, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e6266f000f44279851f7519f9ad3260",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 5', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00006: val_loss improved from 0.22003 to 0.21968, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7230ea3272b440c4b214610b64174883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 6', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00007: val_loss improved from 0.21968 to 0.21942, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31d37503795c441bba23d93ec4b0bcc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 7', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00008: val_loss improved from 0.21942 to 0.21900, saving model to rnn_weights/model_fold_1.hdf5\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e999cca720ce40bf8b7659cf9ee3dade",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 8', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00009: val_loss did not improve from 0.21900\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ae453d97c6d45f7a15a3d9addaf09f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 9', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00010: val_loss did not improve from 0.21900\n",
      "\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text (InputLayer)               (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_20 (Embedding)        (None, 150, 300)     60000000    text[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_2 (SpatialDro (None, 150, 300)     0           embedding_20[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "image_top_1 (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_1 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "city (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "region (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_name (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "parent_category_name (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_type (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_3 (CuDNNGRU)          (None, 150, 128)     165120      spatial_dropout1d_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_11 (Embedding)        (None, 1, 10)        30650       image_top_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_12 (Embedding)        (None, 1, 10)        3730        param_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_13 (Embedding)        (None, 1, 10)        2790        param_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_14 (Embedding)        (None, 1, 10)        12780       param_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_15 (Embedding)        (None, 1, 10)        18250       city[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_16 (Embedding)        (None, 1, 10)        290         region[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_17 (Embedding)        (None, 1, 10)        480         category_name[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_18 (Embedding)        (None, 1, 1)         10          parent_category_name[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_19 (Embedding)        (None, 1, 1)         4           user_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_4 (CuDNNGRU)          (None, 64)           37248       cu_dnngru_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "weekday (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_title (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_description (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_title (InputLa (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_description (I (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_title (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_description (In (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_desc_punct (InputLayer)     (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "price (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "item_seq_number (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_10 (Flatten)            (None, 10)           0           embedding_11[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_11 (Flatten)            (None, 10)           0           embedding_12[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_12 (Flatten)            (None, 10)           0           embedding_13[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_13 (Flatten)            (None, 10)           0           embedding_14[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_14 (Flatten)            (None, 10)           0           embedding_15[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_15 (Flatten)            (None, 10)           0           embedding_16[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_16 (Flatten)            (None, 10)           0           embedding_17[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_17 (Flatten)            (None, 1)            0           embedding_18[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_18 (Flatten)            (None, 1)            0           embedding_19[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 64)           0           cu_dnngru_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 151)          0           weekday[0][0]                    \n",
      "                                                                 num_words_title[0][0]            \n",
      "                                                                 num_words_description[0][0]      \n",
      "                                                                 num_unique_words_title[0][0]     \n",
      "                                                                 num_unique_words_description[0][0\n",
      "                                                                 words_vs_unique_title[0][0]      \n",
      "                                                                 words_vs_unique_description[0][0]\n",
      "                                                                 num_desc_punct[0][0]             \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 avg_days_up_user[0][0]           \n",
      "                                                                 med_times_up_user[0][0]          \n",
      "                                                                 med_days_up_user[0][0]           \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 price[0][0]                      \n",
      "                                                                 item_seq_number[0][0]            \n",
      "                                                                 flatten_10[0][0]                 \n",
      "                                                                 flatten_11[0][0]                 \n",
      "                                                                 flatten_12[0][0]                 \n",
      "                                                                 flatten_13[0][0]                 \n",
      "                                                                 flatten_14[0][0]                 \n",
      "                                                                 flatten_15[0][0]                 \n",
      "                                                                 flatten_16[0][0]                 \n",
      "                                                                 flatten_17[0][0]                 \n",
      "                                                                 flatten_18[0][0]                 \n",
      "                                                                 dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 151)          604         concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 151)          0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 512)          77824       dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 512)          2048        dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 512)          0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 64)           32832       dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 64)           256         dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 1)            65          batch_normalization_6[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 60,384,981\n",
      "Trainable params: 383,527\n",
      "Non-trainable params: 60,001,454\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8641e500b9f141cb9d5441b077f8e360",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Training', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14fdb6bb13e94cf0a7f2694326462609",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 0', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.22700, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b80c013b679445818fd0c8619448fcaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 1', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss improved from 0.22700 to 0.22478, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "954c5d7e09224e81bdf92d183a63a4db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 2', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00003: val_loss improved from 0.22478 to 0.22154, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22539e9f825a4fde9f404faa31225259",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 3', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00004: val_loss did not improve from 0.22154\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "382ffc805fe94fd480e2b340b5b7777b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 4', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00005: val_loss improved from 0.22154 to 0.22048, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "755e4338479d4f31a4d40c0931a252d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 5', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00006: val_loss did not improve from 0.22048\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38421b3009b147ca89c770a63ee1f096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 6', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00007: val_loss improved from 0.22048 to 0.22015, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dc312ed039e4d4aaa909479a3ac8e0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 7', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00008: val_loss improved from 0.22015 to 0.21939, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "559817918d0042b683a473944a1401dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 8', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00009: val_loss did not improve from 0.21939\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0dbc610f50cb4537a84641d97f3ceb97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 9', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00010: val_loss improved from 0.21939 to 0.21926, saving model to rnn_weights/model_fold_2.hdf5\n",
      "\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text (InputLayer)               (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_30 (Embedding)        (None, 150, 300)     60000000    text[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_3 (SpatialDro (None, 150, 300)     0           embedding_30[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "image_top_1 (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_1 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "city (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "region (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_name (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "parent_category_name (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_type (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_5 (CuDNNGRU)          (None, 150, 128)     165120      spatial_dropout1d_3[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_21 (Embedding)        (None, 1, 10)        30650       image_top_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_22 (Embedding)        (None, 1, 10)        3730        param_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_23 (Embedding)        (None, 1, 10)        2790        param_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_24 (Embedding)        (None, 1, 10)        12780       param_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_25 (Embedding)        (None, 1, 10)        18250       city[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_26 (Embedding)        (None, 1, 10)        290         region[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_27 (Embedding)        (None, 1, 10)        480         category_name[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_28 (Embedding)        (None, 1, 1)         10          parent_category_name[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_29 (Embedding)        (None, 1, 1)         4           user_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_6 (CuDNNGRU)          (None, 64)           37248       cu_dnngru_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "weekday (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_title (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_description (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_title (InputLa (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_description (I (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_title (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_description (In (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_desc_punct (InputLayer)     (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "price (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "item_seq_number (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_19 (Flatten)            (None, 10)           0           embedding_21[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_20 (Flatten)            (None, 10)           0           embedding_22[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_21 (Flatten)            (None, 10)           0           embedding_23[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_22 (Flatten)            (None, 10)           0           embedding_24[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_23 (Flatten)            (None, 10)           0           embedding_25[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_24 (Flatten)            (None, 10)           0           embedding_26[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_25 (Flatten)            (None, 10)           0           embedding_27[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_26 (Flatten)            (None, 1)            0           embedding_28[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_27 (Flatten)            (None, 1)            0           embedding_29[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 64)           0           cu_dnngru_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 151)          0           weekday[0][0]                    \n",
      "                                                                 num_words_title[0][0]            \n",
      "                                                                 num_words_description[0][0]      \n",
      "                                                                 num_unique_words_title[0][0]     \n",
      "                                                                 num_unique_words_description[0][0\n",
      "                                                                 words_vs_unique_title[0][0]      \n",
      "                                                                 words_vs_unique_description[0][0]\n",
      "                                                                 num_desc_punct[0][0]             \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 avg_days_up_user[0][0]           \n",
      "                                                                 med_times_up_user[0][0]          \n",
      "                                                                 med_days_up_user[0][0]           \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 price[0][0]                      \n",
      "                                                                 item_seq_number[0][0]            \n",
      "                                                                 flatten_19[0][0]                 \n",
      "                                                                 flatten_20[0][0]                 \n",
      "                                                                 flatten_21[0][0]                 \n",
      "                                                                 flatten_22[0][0]                 \n",
      "                                                                 flatten_23[0][0]                 \n",
      "                                                                 flatten_24[0][0]                 \n",
      "                                                                 flatten_25[0][0]                 \n",
      "                                                                 flatten_26[0][0]                 \n",
      "                                                                 flatten_27[0][0]                 \n",
      "                                                                 dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 151)          604         concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 151)          0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 512)          77824       dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 512)          2048        dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 512)          0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 64)           32832       dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 64)           256         dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 1)            65          batch_normalization_9[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 60,384,981\n",
      "Trainable params: 383,527\n",
      "Non-trainable params: 60,001,454\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bee443986eb443a9b2a837dad6683317",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Training', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e58b46341aff41289f0809aab73a91dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 0', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.22461, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01a68accb9c340f5a1f2f9cc204abe33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 1', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss improved from 0.22461 to 0.22379, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1c8f8129f5b4bdab29df1401c137df7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 2', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00003: val_loss improved from 0.22379 to 0.22294, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12078be2a85d47c299519d6065a8ca2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 3', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00004: val_loss improved from 0.22294 to 0.22134, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "567ec3a31f014898ae3d81f4b233fabb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 4', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00005: val_loss did not improve from 0.22134\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9e0ba3467564ab790124b96b24d71e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 5', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00006: val_loss improved from 0.22134 to 0.22015, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "019a23aa801546feb40c43eb295cffe7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 6', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00007: val_loss improved from 0.22015 to 0.22001, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3029d9350dc7433aa015b7f7706e6c38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 7', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00008: val_loss improved from 0.22001 to 0.21889, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06031a5115d641faa429dfddbae1be18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 8', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00009: val_loss improved from 0.21889 to 0.21848, saving model to rnn_weights/model_fold_3.hdf5\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00ab2901b51e494d83120e119ea81a52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 9', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00010: val_loss did not improve from 0.21848\n",
      "\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text (InputLayer)               (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_40 (Embedding)        (None, 150, 300)     60000000    text[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_4 (SpatialDro (None, 150, 300)     0           embedding_40[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "image_top_1 (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_1 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "city (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "region (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_name (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "parent_category_name (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_type (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_7 (CuDNNGRU)          (None, 150, 128)     165120      spatial_dropout1d_4[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_31 (Embedding)        (None, 1, 10)        30650       image_top_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_32 (Embedding)        (None, 1, 10)        3730        param_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_33 (Embedding)        (None, 1, 10)        2790        param_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_34 (Embedding)        (None, 1, 10)        12780       param_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_35 (Embedding)        (None, 1, 10)        18250       city[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_36 (Embedding)        (None, 1, 10)        290         region[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_37 (Embedding)        (None, 1, 10)        480         category_name[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_38 (Embedding)        (None, 1, 1)         10          parent_category_name[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_39 (Embedding)        (None, 1, 1)         4           user_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_8 (CuDNNGRU)          (None, 64)           37248       cu_dnngru_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "weekday (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_title (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_description (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_title (InputLa (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_description (I (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_title (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_description (In (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_desc_punct (InputLayer)     (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "price (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "item_seq_number (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_28 (Flatten)            (None, 10)           0           embedding_31[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_29 (Flatten)            (None, 10)           0           embedding_32[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_30 (Flatten)            (None, 10)           0           embedding_33[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_31 (Flatten)            (None, 10)           0           embedding_34[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_32 (Flatten)            (None, 10)           0           embedding_35[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_33 (Flatten)            (None, 10)           0           embedding_36[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_34 (Flatten)            (None, 10)           0           embedding_37[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_35 (Flatten)            (None, 1)            0           embedding_38[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_36 (Flatten)            (None, 1)            0           embedding_39[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 64)           0           cu_dnngru_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 151)          0           weekday[0][0]                    \n",
      "                                                                 num_words_title[0][0]            \n",
      "                                                                 num_words_description[0][0]      \n",
      "                                                                 num_unique_words_title[0][0]     \n",
      "                                                                 num_unique_words_description[0][0\n",
      "                                                                 words_vs_unique_title[0][0]      \n",
      "                                                                 words_vs_unique_description[0][0]\n",
      "                                                                 num_desc_punct[0][0]             \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 avg_days_up_user[0][0]           \n",
      "                                                                 med_times_up_user[0][0]          \n",
      "                                                                 med_days_up_user[0][0]           \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 price[0][0]                      \n",
      "                                                                 item_seq_number[0][0]            \n",
      "                                                                 flatten_28[0][0]                 \n",
      "                                                                 flatten_29[0][0]                 \n",
      "                                                                 flatten_30[0][0]                 \n",
      "                                                                 flatten_31[0][0]                 \n",
      "                                                                 flatten_32[0][0]                 \n",
      "                                                                 flatten_33[0][0]                 \n",
      "                                                                 flatten_34[0][0]                 \n",
      "                                                                 flatten_35[0][0]                 \n",
      "                                                                 flatten_36[0][0]                 \n",
      "                                                                 dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 151)          604         concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 151)          0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 512)          77824       dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 512)          2048        dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, 512)          0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 64)           32832       dropout_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 64)           256         dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 1)            65          batch_normalization_12[0][0]     \n",
      "==================================================================================================\n",
      "Total params: 60,384,981\n",
      "Trainable params: 383,527\n",
      "Non-trainable params: 60,001,454\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d11a0401cad848ac8a2b9eaab20f4845",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Training', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05803e19d7c444d984234308ef859ff7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 0', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.22388, saving model to rnn_weights/model_fold_4.hdf5\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "018bc123019b4a0c9f25c47b1ed58dbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 1', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss improved from 0.22388 to 0.22140, saving model to rnn_weights/model_fold_4.hdf5\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f13e69934b1447feb61848cf00ee9ecd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 2', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00003: val_loss improved from 0.22140 to 0.22031, saving model to rnn_weights/model_fold_4.hdf5\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06fc24dcc41b4cb8bd23bcb1bef890bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 3', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00004: val_loss improved from 0.22031 to 0.21914, saving model to rnn_weights/model_fold_4.hdf5\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f2cb0bf6c25477e95cc795ebbcffee7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 4', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00005: val_loss did not improve from 0.21914\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18eac23f84f540399136e7d41440a608",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 5', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00006: val_loss improved from 0.21914 to 0.21861, saving model to rnn_weights/model_fold_4.hdf5\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c70d5e9c58048afa4ee35c81a7e8756",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 6', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00007: val_loss did not improve from 0.21861\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d3ea4266045462aa8828aa5c27a18ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 7', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00008: val_loss improved from 0.21861 to 0.21832, saving model to rnn_weights/model_fold_4.hdf5\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "926fbb7d67d345c1a5caf1d2e1c856c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 8', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00009: val_loss did not improve from 0.21832\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "543408de1f644cc284238ae7a0a9b6d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 9', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00010: val_loss did not improve from 0.21832\n",
      "\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "text (InputLayer)               (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_50 (Embedding)        (None, 150, 300)     60000000    text[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "spatial_dropout1d_5 (SpatialDro (None, 150, 300)     0           embedding_50[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "image_top_1 (InputLayer)        (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_1 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_2 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "param_3 (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "city (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "region (InputLayer)             (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_name (InputLayer)      (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "parent_category_name (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_type (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_9 (CuDNNGRU)          (None, 150, 128)     165120      spatial_dropout1d_5[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_41 (Embedding)        (None, 1, 10)        30650       image_top_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_42 (Embedding)        (None, 1, 10)        3730        param_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_43 (Embedding)        (None, 1, 10)        2790        param_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_44 (Embedding)        (None, 1, 10)        12780       param_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_45 (Embedding)        (None, 1, 10)        18250       city[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_46 (Embedding)        (None, 1, 10)        290         region[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_47 (Embedding)        (None, 1, 10)        480         category_name[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "embedding_48 (Embedding)        (None, 1, 1)         10          parent_category_name[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "embedding_49 (Embedding)        (None, 1, 1)         4           user_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "cu_dnngru_10 (CuDNNGRU)         (None, 64)           37248       cu_dnngru_9[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "weekday (InputLayer)            (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_title (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_words_description (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_title (InputLa (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_unique_words_description (I (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_title (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_vs_unique_description (In (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "num_desc_punct (InputLayer)     (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "avg_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_times_up_user (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "med_days_up_user (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "n_user_items (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "price (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "item_seq_number (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_37 (Flatten)            (None, 10)           0           embedding_41[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_38 (Flatten)            (None, 10)           0           embedding_42[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_39 (Flatten)            (None, 10)           0           embedding_43[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_40 (Flatten)            (None, 10)           0           embedding_44[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_41 (Flatten)            (None, 10)           0           embedding_45[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_42 (Flatten)            (None, 10)           0           embedding_46[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_43 (Flatten)            (None, 10)           0           embedding_47[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_44 (Flatten)            (None, 1)            0           embedding_48[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_45 (Flatten)            (None, 1)            0           embedding_49[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 64)           0           cu_dnngru_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 151)          0           weekday[0][0]                    \n",
      "                                                                 num_words_title[0][0]            \n",
      "                                                                 num_words_description[0][0]      \n",
      "                                                                 num_unique_words_title[0][0]     \n",
      "                                                                 num_unique_words_description[0][0\n",
      "                                                                 words_vs_unique_title[0][0]      \n",
      "                                                                 words_vs_unique_description[0][0]\n",
      "                                                                 num_desc_punct[0][0]             \n",
      "                                                                 avg_times_up_user[0][0]          \n",
      "                                                                 avg_days_up_user[0][0]           \n",
      "                                                                 med_times_up_user[0][0]          \n",
      "                                                                 med_days_up_user[0][0]           \n",
      "                                                                 n_user_items[0][0]               \n",
      "                                                                 price[0][0]                      \n",
      "                                                                 item_seq_number[0][0]            \n",
      "                                                                 flatten_37[0][0]                 \n",
      "                                                                 flatten_38[0][0]                 \n",
      "                                                                 flatten_39[0][0]                 \n",
      "                                                                 flatten_40[0][0]                 \n",
      "                                                                 flatten_41[0][0]                 \n",
      "                                                                 flatten_42[0][0]                 \n",
      "                                                                 flatten_43[0][0]                 \n",
      "                                                                 flatten_44[0][0]                 \n",
      "                                                                 flatten_45[0][0]                 \n",
      "                                                                 dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 151)          604         concatenate_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, 151)          0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 512)          77824       dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 512)          2048        dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, 512)          0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 64)           32832       dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 64)           256         dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 1)            65          batch_normalization_15[0][0]     \n",
      "==================================================================================================\n",
      "Total params: 60,384,981\n",
      "Trainable params: 383,527\n",
      "Non-trainable params: 60,001,454\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "698643fa7b184e59a503e15638e117f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Training', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ba3a37dbb394c79a2dafea8c6a89dd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 0', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.22470, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ac0ebafe0b444c2b7a75b95ffd788a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 1', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00002: val_loss improved from 0.22470 to 0.22249, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb215cf577b64923ad10d2a8b945c449",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 2', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00003: val_loss improved from 0.22249 to 0.22119, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "744057f6d63042f7888fb84ed55c988a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 3', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00004: val_loss improved from 0.22119 to 0.22038, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc58473437e244e2a6b80530c0f117a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 4', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00005: val_loss did not improve from 0.22038\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e8aa1742b314c67b3964e3c594cb519",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 5', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00006: val_loss improved from 0.22038 to 0.22011, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.001.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9ee56d378944487bb74ca8298202b8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 6', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00007: val_loss improved from 0.22011 to 0.21986, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0be2027ae414943b6e7605802761558",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 7', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00008: val_loss improved from 0.21986 to 0.21908, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90bfa0facd354db2b8509e1ba29dd9c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 8', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00009: val_loss did not improve from 0.21908\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0002.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93052bd706cb4da790960a59bfbad947",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch 9', max=9397), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00010: val_loss improved from 0.21908 to 0.21908, saving model to rnn_weights/model_fold_5.hdf5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "fold = 0\n",
    "ret = np.zeros((train_len,))\n",
    "ret_test = np.zeros((all_features.shape[0]-train_len,))\n",
    "\n",
    "for tr_ix, val_ix in KFold(5, shuffle=True, random_state=seed).split(train_indices):\n",
    "    fold += 1\n",
    "    \n",
    "    model = build_model(all_features, categorical, numerical)\n",
    "    file_path = \"rnn_weights/model_fold_{}.hdf5\".format(fold)\n",
    "    \n",
    "    # old style\n",
    "    '''\n",
    "    X_train = dict((col, all_features.loc[tr_ix, col].values) for col in features)\n",
    "    X_train['text'] = texts[tr_ix,:]\n",
    "    y_train = train_y[tr_ix]\n",
    "    X_val = dict((col, all_features.loc[val_ix, col].values) for col in features)\n",
    "    X_val['text'] = texts[val_ix,:]\n",
    "    y_val = train_y[val_ix]\n",
    "    \n",
    "    lr_schd = LearningRateScheduler(lambda epoch: 0.001*(0.2**(epoch//8)), verbose=1)\n",
    "    check_point = NBatchEvalLogger(2000, X_val, y_val, save_path=file_path, save_start=2000)\n",
    "    #check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", mode = \"min\", save_best_only = True, verbose = 1)\n",
    "    \n",
    "    history = model.fit(X_train, y_train, batch_size = 128, epochs = rnn_train_epochs, validation_data = (X_val, y_val),\n",
    "                    verbose = 0, callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n",
    "    \n",
    "    del model, X_train, y_train, X_val, y_val, history; gc.collect()\n",
    "    '''\n",
    "\n",
    "    # customized batch loader\n",
    "    training_generator = DataGenerator(tr_ix, all_features, train_y, batch_size=batch_size, shuffle=True)\n",
    "    validation_generator = DataGenerator(val_ix, all_features, train_y, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    lr_schd = LearningRateScheduler(lambda epoch: 0.001*(0.2**(epoch//7)), verbose=1)\n",
    "    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", mode = \"min\", save_best_only = True, verbose = 1)\n",
    "    history = model.fit_generator(generator=training_generator,\n",
    "                                  validation_data=validation_generator,\n",
    "                                  use_multiprocessing=False,\n",
    "                                  workers=1, \n",
    "                                  epochs=rnn_train_epochs,\n",
    "                                  verbose = 0, \n",
    "                                  callbacks = [lr_schd, check_point, TQDMNotebookCallback(leave_inner=True, leave_outer=True)])\n",
    "    \n",
    "    \n",
    "    # Predict val + test oofs\n",
    "    model.load_weights(file_path) # load weight with best validation score\n",
    "    \n",
    "    del validation_generator\n",
    "    validation_generator = DataGenerator(val_ix, all_features, None, batch_size=batch_size, shuffle=False, is_train=False)\n",
    "    test_generator = DataGenerator(test_indices, all_features, None, batch_size=batch_size, shuffle=False, is_train=False)\n",
    "    \n",
    "    ret[val_ix] = model.predict_generator(validation_generator, use_multiprocessing=False, workers=1).reshape((len(val_ix),))\n",
    "    ret_test += model.predict_generator(test_generator, use_multiprocessing=False, workers=1).reshape((ret_test.shape[0],))\n",
    "    \n",
    "    del model, history, training_generator, validation_generator, test_generator; gc.collect()\n",
    "    \n",
    "ret_test /= 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# public:  cv = .2220, lb = .2247 \n",
    "# bigru-conv1d: cv =.2185 , lb = .2235\n",
    "# bigru-attention: cv =.2186 , lb = .2235"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate OOFs and Submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(data=ret, columns=['pretrained_2gru_rnn_pred']).to_csv('pretrained_2gru_rnn_oof_val_pred.csv', index=False)\n",
    "pd.DataFrame(data=ret_test, columns=['pretrained_2gru_rnn_pred']).to_csv('pretrained_2gru_rnn_oof_test_pred.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "subm = pd.read_csv('sample_submission.csv')\n",
    "subm['deal_probability'] = np.clip(ret_test, 0, 1)\n",
    "subm.to_csv('pretrained_2gru_rnn_submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.5 tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}