Exploratory-c4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import pandas as pd\n",
    "import workalendar.europe.belgium as belgium\n",
    "import sklearn\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data = pd.read_csv(\"./data/dataraw.csv\", parse_dates=[\"time\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nikita/Code/openings/ve/lib/python3.6/site-packages/pandas/core/indexing.py:517: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>is_open</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15152</th>\n",
       "      <td>2014-08-30 22:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-08-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17235</th>\n",
       "      <td>2014-11-25 17:00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>2014-11-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27708</th>\n",
       "      <td>2016-02-05 02:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2016-02-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22388</th>\n",
       "      <td>2015-06-28 10:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2015-06-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12422</th>\n",
       "      <td>2014-05-09 04:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-05-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36954</th>\n",
       "      <td>2017-02-24 08:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2017-02-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>922</th>\n",
       "      <td>2013-01-15 00:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2013-01-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18738</th>\n",
       "      <td>2015-01-27 08:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2015-01-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26015</th>\n",
       "      <td>2015-11-26 13:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2015-11-26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9234</th>\n",
       "      <td>2013-12-27 08:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2013-12-27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     time  is_open        date\n",
       "15152 2014-08-30 22:00:00    False  2014-08-30\n",
       "17235 2014-11-25 17:00:00     True  2014-11-25\n",
       "27708 2016-02-05 02:00:00    False  2016-02-05\n",
       "22388 2015-06-28 10:00:00    False  2015-06-28\n",
       "12422 2014-05-09 04:00:00    False  2014-05-09\n",
       "36954 2017-02-24 08:00:00    False  2017-02-24\n",
       "922   2013-01-15 00:00:00    False  2013-01-15\n",
       "18738 2015-01-27 08:00:00    False  2015-01-27\n",
       "26015 2015-11-26 13:00:00    False  2015-11-26\n",
       "9234  2013-12-27 08:00:00    False  2013-12-27"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_data = raw_data[[\"time\", \"is_open\"]]\n",
    "cleaned_data.loc[cleaned_data.time.dt.hour >= 23, \"is_open\"] = False\n",
    "cleaned_data.loc[cleaned_data.time.dt.hour < 7, \"is_open\"] = False\n",
    "cleaned_data[\"date\"] = cleaned_data.time.dt.date\n",
    "\n",
    "cleaned_data.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "205     2013-06-30\n",
       "1422    2016-10-29\n",
       "1611    2017-05-06\n",
       "220     2013-07-15\n",
       "572     2014-07-02\n",
       "928     2015-06-23\n",
       "1639    2017-06-03\n",
       "21      2012-12-28\n",
       "1072    2015-11-14\n",
       "146     2013-05-02\n",
       "Name: day, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dates = pd.Series(cleaned_data[\"date\"].unique(), name=\"day\")\n",
    "data_dates.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "starting_days = [\n",
    "    datetime.date(2009, 9, 14),\n",
    "    datetime.date(2010, 9, 20),\n",
    "    datetime.date(2011, 9, 19),\n",
    "    datetime.date(2012, 9, 17),\n",
    "    datetime.date(2013, 9, 16),\n",
    "    datetime.date(2014, 9, 15),\n",
    "    datetime.date(2015, 9, 14),\n",
    "    datetime.date(2016, 9, 19),\n",
    "    datetime.date(2017, 9, 18),\n",
    "]\n",
    "\n",
    "assert all([d.weekday() == 0 for d in starting_days])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>day</th>\n",
       "      <th>week_no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>2013-09-11</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>2013-09-12</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>2013-09-13</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>2013-09-14</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>2013-09-15</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>2013-09-16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>284</th>\n",
       "      <td>2013-09-17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>2013-09-18</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>2013-09-19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>2013-09-20</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>2013-09-21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>2013-09-22</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>2013-09-23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>2013-09-24</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>2013-09-25</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           day  week_no\n",
       "278 2013-09-11       51\n",
       "279 2013-09-12       51\n",
       "280 2013-09-13       51\n",
       "281 2013-09-14       51\n",
       "282 2013-09-15       51\n",
       "283 2013-09-16        0\n",
       "284 2013-09-17        0\n",
       "285 2013-09-18        0\n",
       "286 2013-09-19        0\n",
       "287 2013-09-20        0\n",
       "288 2013-09-21        0\n",
       "289 2013-09-22        0\n",
       "290 2013-09-23        1\n",
       "291 2013-09-24        1\n",
       "292 2013-09-25        1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_week_no(date):\n",
    "    possible = [d for d in starting_days if d <= date]\n",
    "    aca_start = possible[-1]\n",
    "    dt = date - aca_start\n",
    "    return math.floor(timedelta64(dt) / timedelta64(1,'W'))\n",
    "\n",
    "week_df = data_dates.to_frame()\n",
    "week_df[\"week_no\"] = week_df[\"day\"].apply(get_week_no)\n",
    "week_df[\"day\"] = pd.to_datetime(week_df[\"day\"])\n",
    "\n",
    "week_df[week_df.day.dt.date > datetime64(\"2013-09-10\")].head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>time</th>\n",
       "      <th>is_open</th>\n",
       "      <th>date</th>\n",
       "      <th>weekday</th>\n",
       "      <th>aca_week_no</th>\n",
       "      <th>week_no</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>working</th>\n",
       "      <th>tampon</th>\n",
       "      <th>exams</th>\n",
       "      <th>vacances</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17576</th>\n",
       "      <td>2014-12-09 22:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2014-12-09</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>50</td>\n",
       "      <td>2014</td>\n",
       "      <td>12</td>\n",
       "      <td>9</td>\n",
       "      <td>22</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30279</th>\n",
       "      <td>2016-05-22 05:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2016-05-22</td>\n",
       "      <td>6</td>\n",
       "      <td>35</td>\n",
       "      <td>20</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32054</th>\n",
       "      <td>2016-08-04 04:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2016-08-04</td>\n",
       "      <td>3</td>\n",
       "      <td>46</td>\n",
       "      <td>31</td>\n",
       "      <td>2016</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33318</th>\n",
       "      <td>2016-09-25 20:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2016-09-25</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>2016</td>\n",
       "      <td>9</td>\n",
       "      <td>25</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18201</th>\n",
       "      <td>2015-01-04 23:00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     time  is_open        date  weekday  aca_week_no  week_no  \\\n",
       "17576 2014-12-09 22:00:00    False  2014-12-09        1           12       50   \n",
       "30279 2016-05-22 05:00:00    False  2016-05-22        6           35       20   \n",
       "32054 2016-08-04 04:00:00    False  2016-08-04        3           46       31   \n",
       "33318 2016-09-25 20:00:00    False  2016-09-25        6            0       38   \n",
       "18201 2015-01-04 23:00:00    False  2015-01-04        6           15        1   \n",
       "\n",
       "       year  month  day  hour  working  tampon  exams  vacances  \n",
       "17576  2014     12    9    22     True   False  False     False  \n",
       "30279  2016      5   22     5    False   False  False     False  \n",
       "32054  2016      8    4     4     True   False  False      True  \n",
       "33318  2016      9   25    20    False   False  False     False  \n",
       "18201  2015      1    4    23    False   False   True     False  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featurized = cleaned_data.copy()\n",
    "\n",
    "# day of the week\n",
    "featurized[\"weekday\"] = featurized[\"time\"].dt.weekday\n",
    "\n",
    "# academic week number\n",
    "featurized[\"aca_week_no\"] = featurized[\"date\"].apply(get_week_no)\n",
    "\n",
    "# civil week number\n",
    "featurized[\"week_no\"] = featurized[\"time\"].dt.week\n",
    "\n",
    "# date\n",
    "featurized[\"year\"] = featurized[\"time\"].dt.year\n",
    "featurized[\"month\"] = featurized[\"time\"].dt.month\n",
    "featurized[\"day\"] = featurized[\"time\"].dt.day\n",
    "featurized[\"hour\"] = featurized[\"time\"].dt.hour\n",
    "\n",
    "# holiday\n",
    "calendar = belgium.Belgium()\n",
    "featurized[\"working\"] = featurized[\"date\"].apply(calendar.is_working_day)\n",
    "### St V\n",
    "featurized.loc[(featurized[\"time\"].dt.month == 11) & (featurized[\"time\"].dt.day == 20), \"working\"] = False\n",
    "\n",
    "# tampon\n",
    "featurized[\"tampon\"] = False\n",
    "featurized.loc[featurized.aca_week_no.isin([6, 13, 26, 34]), \"tampon\"] = True\n",
    "\n",
    "# exams\n",
    "featurized[\"exams\"] = False\n",
    "featurized.loc[featurized.aca_week_no.isin([14, 15, 16, 17, 18]), \"exams\"] = True\n",
    "featurized.loc[featurized.aca_week_no.isin([36, 37, 38, 39, 40]), \"exams\"] = True\n",
    "\n",
    "# vacances\n",
    "featurized[\"vacances\"] = False\n",
    "featurized.loc[featurized.aca_week_no.isin([19] + list(range(41, 52))), \"vacances\"] = True\n",
    "\n",
    "# weather\n",
    "\n",
    "featurized.sort_values(\"time\")\n",
    "featurized.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xcols = list(set(featurized.columns) - set([\"is_open\", \"date\", \"time\"]))\n",
    "one_hot = [\n",
    "    \"weekday\",\n",
    "    \"aca_week_no\", \n",
    "    \"week_no\",\n",
    "    \n",
    "    \"year\",\n",
    "    \"month\",\n",
    "    \"day\",\n",
    "    \"hour\",\n",
    "]\n",
    "\n",
    "I = featurized[\"time\"].values\n",
    "X = pd.get_dummies(featurized[xcols], columns=one_hot).values.astype(bool)\n",
    "Y = featurized[\"is_open\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39965, 190)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39965,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['exams',\n",
       " 'working',\n",
       " 'hour',\n",
       " 'month',\n",
       " 'week_no',\n",
       " 'vacances',\n",
       " 'weekday',\n",
       " 'tampon',\n",
       " 'aca_week_no',\n",
       " 'year',\n",
       " 'day']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List of features\n",
    "xcols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['exams', 'working', 'vacances', 'tampon', 'weekday_0', 'weekday_1',\n",
       "       'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6',\n",
       "       'aca_week_no_0', 'aca_week_no_1', 'aca_week_no_2', 'aca_week_no_3',\n",
       "       'aca_week_no_4', 'aca_week_no_5', 'aca_week_no_6', 'aca_week_no_7',\n",
       "       'aca_week_no_8', 'aca_week_no_9', 'aca_week_no_10',\n",
       "       'aca_week_no_11', 'aca_week_no_12', 'aca_week_no_13',\n",
       "       'aca_week_no_14', 'aca_week_no_15', 'aca_week_no_16',\n",
       "       'aca_week_no_17', 'aca_week_no_18', 'aca_week_no_19',\n",
       "       'aca_week_no_20', 'aca_week_no_21', 'aca_week_no_22',\n",
       "       'aca_week_no_23', 'aca_week_no_24', 'aca_week_no_25',\n",
       "       'aca_week_no_26', 'aca_week_no_27', 'aca_week_no_28',\n",
       "       'aca_week_no_29', 'aca_week_no_30', 'aca_week_no_31',\n",
       "       'aca_week_no_32', 'aca_week_no_33', 'aca_week_no_34',\n",
       "       'aca_week_no_35', 'aca_week_no_36', 'aca_week_no_37',\n",
       "       'aca_week_no_38', 'aca_week_no_39', 'aca_week_no_40',\n",
       "       'aca_week_no_41', 'aca_week_no_42', 'aca_week_no_43',\n",
       "       'aca_week_no_44', 'aca_week_no_45', 'aca_week_no_46',\n",
       "       'aca_week_no_47', 'aca_week_no_48', 'aca_week_no_49',\n",
       "       'aca_week_no_50', 'aca_week_no_51', 'aca_week_no_52', 'week_no_1',\n",
       "       'week_no_2', 'week_no_3', 'week_no_4', 'week_no_5', 'week_no_6',\n",
       "       'week_no_7', 'week_no_8', 'week_no_9', 'week_no_10', 'week_no_11',\n",
       "       'week_no_12', 'week_no_13', 'week_no_14', 'week_no_15',\n",
       "       'week_no_16', 'week_no_17', 'week_no_18', 'week_no_19',\n",
       "       'week_no_20', 'week_no_21', 'week_no_22', 'week_no_23',\n",
       "       'week_no_24', 'week_no_25', 'week_no_26', 'week_no_27',\n",
       "       'week_no_28', 'week_no_29', 'week_no_30', 'week_no_31',\n",
       "       'week_no_32', 'week_no_33', 'week_no_34', 'week_no_35',\n",
       "       'week_no_36', 'week_no_37', 'week_no_38', 'week_no_39',\n",
       "       'week_no_40', 'week_no_41', 'week_no_42', 'week_no_43',\n",
       "       'week_no_44', 'week_no_45', 'week_no_46', 'week_no_47',\n",
       "       'week_no_48', 'week_no_49', 'week_no_50', 'week_no_51',\n",
       "       'week_no_52', 'week_no_53', 'year_2012', 'year_2013', 'year_2014',\n",
       "       'year_2015', 'year_2016', 'year_2017', 'month_1', 'month_2',\n",
       "       'month_3', 'month_4', 'month_5', 'month_6', 'month_7', 'month_8',\n",
       "       'month_9', 'month_10', 'month_11', 'month_12', 'day_1', 'day_2',\n",
       "       'day_3', 'day_4', 'day_5', 'day_6', 'day_7', 'day_8', 'day_9',\n",
       "       'day_10', 'day_11', 'day_12', 'day_13', 'day_14', 'day_15',\n",
       "       'day_16', 'day_17', 'day_18', 'day_19', 'day_20', 'day_21',\n",
       "       'day_22', 'day_23', 'day_24', 'day_25', 'day_26', 'day_27',\n",
       "       'day_28', 'day_29', 'day_30', 'day_31', 'hour_0', 'hour_1',\n",
       "       'hour_2', 'hour_3', 'hour_4', 'hour_5', 'hour_6', 'hour_7',\n",
       "       'hour_8', 'hour_9', 'hour_10', 'hour_11', 'hour_12', 'hour_13',\n",
       "       'hour_14', 'hour_15', 'hour_16', 'hour_17', 'hour_18', 'hour_19',\n",
       "       'hour_20', 'hour_21', 'hour_22', 'hour_23'], dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# List of one-hotted features\n",
    "features = pd.get_dummies(featurized[xcols], columns=one_hot).columns.values\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "122"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(features).index(\"year_2017\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(35650, 190) (35650,)\n",
      "(4315, 190) (4315,)\n"
     ]
    }
   ],
   "source": [
    "test_mask = X[:,122]\n",
    "train_mask = np.invert(test_mask)\n",
    "\n",
    "Xtrain = X[train_mask]\n",
    "Ytrain = Y[train_mask]\n",
    "\n",
    "Xtest = X[test_mask]\n",
    "Ytest = Y[test_mask]\n",
    "\n",
    "print(Xtrain.shape, Ytrain.shape)\n",
    "print(Xtest.shape, Ytest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn.ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None,\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada = sklearn.ensemble.AdaBoostClassifier()\n",
    "ada.fit(Xtrain, Ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Yada = ada.predict(Xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.57227465142780976"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.matthews_corrcoef(Ytest, Yada)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn.neighbors\n",
    "knn = sklearn.neighbors.KNeighborsClassifier(20)\n",
    "knn.fit(Xtrain, Ytrain)\n",
    "Yknn = knn.predict(Xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19378445666356187"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.matthews_corrcoef(Ytest, Yknn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "forest = sklearn.ensemble.RandomForestClassifier()\n",
    "forest.fit(Xtrain, Ytrain)\n",
    "Yforest = forest.predict(Xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53704793106065185"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.matthews_corrcoef(Ytest, Yforest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(39965,) (39965, 190) (39965,)\n"
     ]
    }
   ],
   "source": [
    "print(I.shape, X.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def balance(X, Y):\n",
    "    t_count = Y.sum()\n",
    "    f_count = Y.shape[0] - t_count\n",
    "    \n",
    "    c_max = max(t_count, f_count)\n",
    "    \n",
    "    t = pd.DataFrame(X[Y]).sample(c_max, replace=True).values\n",
    "    f = pd.DataFrame(X[np.logical_not(Y)]).sample(c_max, replace=True).values\n",
    "    \n",
    "    X = np.vstack([t, f])\n",
    "    Y = np.array([True] * c_max + [False] * c_max)\n",
    "    \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "  0%|          | 0/52 [00:00<?, ?week/s]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "  2%|▏         | 1/52 [00:07<06:43,  7.91s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "  4%|▍         | 2/52 [00:15<06:33,  7.86s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "  6%|▌         | 3/52 [00:23<06:25,  7.87s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "  8%|▊         | 4/52 [00:31<06:19,  7.91s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 10%|▉         | 5/52 [00:39<06:13,  7.95s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 12%|█▏        | 6/52 [00:47<06:07,  7.98s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 13%|█▎        | 7/52 [00:56<06:10,  8.24s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 15%|█▌        | 8/52 [01:05<06:07,  8.35s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 17%|█▋        | 9/52 [01:13<06:02,  8.43s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 19%|█▉        | 10/52 [01:22<06:02,  8.64s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 21%|██        | 11/52 [01:32<06:02,  8.84s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 23%|██▎       | 12/52 [01:40<05:51,  8.78s/week, mcc=0.597]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 25%|██▌       | 13/52 [01:49<05:38,  8.68s/week, mcc=0.565]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 27%|██▋       | 14/52 [01:57<05:28,  8.63s/week, mcc=0.739]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 29%|██▉       | 15/52 [02:06<05:18,  8.61s/week, mcc=0.647]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 31%|███       | 16/52 [02:15<05:12,  8.69s/week, mcc=0.607]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 33%|███▎      | 17/52 [02:23<05:04,  8.71s/week, mcc=0.559]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 35%|███▍      | 18/52 [02:32<04:56,  8.73s/week, mcc=0.649]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 37%|███▋      | 19/52 [02:41<04:48,  8.73s/week, mcc=0.568]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 38%|███▊      | 20/52 [02:50<04:41,  8.80s/week, mcc=0.661]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 40%|████      | 21/52 [02:59<04:33,  8.81s/week, mcc=0.615]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 42%|████▏     | 22/52 [03:08<04:27,  8.90s/week, mcc=0.725]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 44%|████▍     | 23/52 [03:17<04:17,  8.89s/week, mcc=0.782]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 46%|████▌     | 24/52 [03:26<04:08,  8.89s/week, mcc=0.774]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 48%|████▊     | 25/52 [03:35<04:00,  8.91s/week, mcc=0.492]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 50%|█████     | 26/52 [03:44<03:53,  8.96s/week, mcc=0]    \u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 52%|█████▏    | 27/52 [03:53<03:44,  8.96s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 54%|█████▍    | 28/52 [04:02<03:35,  8.98s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 56%|█████▌    | 29/52 [04:12<03:33,  9.29s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 58%|█████▊    | 30/52 [04:21<03:25,  9.36s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 60%|█████▉    | 31/52 [04:31<03:17,  9.43s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 62%|██████▏   | 32/52 [04:40<03:09,  9.45s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 63%|██████▎   | 33/52 [04:50<03:00,  9.52s/week, mcc=0.755]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 65%|██████▌   | 34/52 [04:59<02:50,  9.49s/week, mcc=0.725]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 67%|██████▋   | 35/52 [05:09<02:41,  9.52s/week, mcc=0.623]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 69%|██████▉   | 36/52 [05:19<02:33,  9.57s/week, mcc=0.696]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 71%|███████   | 37/52 [05:28<02:24,  9.62s/week, mcc=0.648]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 73%|███████▎  | 38/52 [05:38<02:15,  9.68s/week, mcc=0.712]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 75%|███████▌  | 39/52 [05:50<02:15, 10.42s/week, mcc=0.785]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 77%|███████▋  | 40/52 [06:01<02:07, 10.60s/week, mcc=0.452]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 79%|███████▉  | 41/52 [06:12<01:57, 10.71s/week, mcc=0]    \u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 81%|████████  | 42/52 [06:23<01:47, 10.80s/week, mcc=0.533]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 83%|████████▎ | 43/52 [06:34<01:38, 10.90s/week, mcc=0.584]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 85%|████████▍ | 44/52 [06:46<01:28, 11.01s/week, mcc=0.732]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 87%|████████▋ | 45/52 [06:57<01:17, 11.07s/week, mcc=0.835]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 88%|████████▊ | 46/52 [07:08<01:06, 11.12s/week, mcc=0.594]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 90%|█████████ | 47/52 [07:19<00:55, 11.16s/week, mcc=0]    \u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 92%|█████████▏| 48/52 [07:31<00:44, 11.20s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 94%|█████████▍| 49/52 [07:42<00:33, 11.22s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 96%|█████████▌| 50/52 [07:53<00:22, 11.27s/week, mcc=0]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      " 98%|█████████▊| 51/52 [08:05<00:11, 11.34s/week, mcc=-0.138]\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "100%|██████████| 52/52 [08:17<00:00, 11.47s/week, mcc=0.412] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.411555208351\n",
      "CPU times: user 8min 12s, sys: 832 ms, total: 8min 13s\n",
      "Wall time: 8min 17s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[A\u001b[A\u001b[A\n",
      "\n",
      "\n",
      "\u001b[A\u001b[A\u001b[A"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tqdm.tqdm.monitor_interval = 0\n",
    "\n",
    "validation_start = np.datetime64(cleaned_data[\"time\"].max() - timedelta64(1, \"Y\"))\n",
    "mccs = []\n",
    "\n",
    "t = tqdm.tqdm(range(52), unit=\"week\")\n",
    "for i in t:\n",
    "    cut = validation_start + i * timedelta64(7, \"D\")\n",
    "    end = validation_start + (i + 1) * timedelta64(7, \"D\")\n",
    "    \n",
    "    train_mask = I < cut\n",
    "    test_mask = np.logical_and(I >= cut, I < end)\n",
    "\n",
    "    Xtrain = X[train_mask]\n",
    "    Ytrain = Y[train_mask]\n",
    "    \n",
    "    Xtrain, Ytrain = balance(Xtrain, Ytrain)\n",
    "\n",
    "    Xtest = X[test_mask]\n",
    "    Ytest = Y[test_mask]\n",
    "    \n",
    "    ada = sklearn.ensemble.AdaBoostClassifier()\n",
    "    ada.fit(Xtrain, Ytrain)\n",
    "    Y_pred = ada.predict(Xtest)\n",
    "    \n",
    "    with warnings.catch_warnings():\n",
    "        warnings.simplefilter(\"ignore\")\n",
    "        mcc = sklearn.metrics.matthews_corrcoef(Ytest, Y_pred)\n",
    "    mccs.append(mcc)\n",
    "    t.set_postfix(mcc=mcc)\n",
    "\n",
    "print(mean(mccs))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.41155521])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mcc[mcc > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.64522743728099041"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean([mcc for mcc in mccs if mcc > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83484710993672195"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(mccs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}