vodafone_challenge_v0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM, Dense, Dropout, Conv1D, TimeDistributed, Flatten, GRU\n",
    "from tensorflow.keras.optimizers import Adam, SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_parquet('full_dataset.parquet', engine='pyarrow')\n",
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_identifier</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>event_category_idx_lv1</th>\n",
       "      <th>event_category_idx_lv2</th>\n",
       "      <th>event_category_idx_lv3</th>\n",
       "      <th>event_category_idx_lv4</th>\n",
       "      <th>tobi_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 14:38:14</td>\n",
       "      <td>1_5</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 14:38:14</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:46:56</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:23:50</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_12</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:21:29</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   session_id  user_identifier     event_timestamp event_category_idx_lv1  \\\n",
       "0      182576                0 2020-09-22 14:38:14                    1_5   \n",
       "1      182576                0 2020-09-22 14:38:14                    1_4   \n",
       "2      182576                0 2020-09-22 11:46:56                    1_4   \n",
       "3      182576                0 2020-09-22 11:23:50                    1_4   \n",
       "4      182576                0 2020-09-22 11:21:29                    1_4   \n",
       "\n",
       "  event_category_idx_lv2 event_category_idx_lv3 event_category_idx_lv4  \\\n",
       "0                      2                   None                   None   \n",
       "1                    2_9                    3_5                   None   \n",
       "2                    2_9                    3_6                   None   \n",
       "3                    2_9                   3_12                   None   \n",
       "4                    2_9                    3_6                   None   \n",
       "\n",
       "       tobi_timestamp  \n",
       "0 2020-09-24 02:03:53  \n",
       "1 2020-09-24 02:03:53  \n",
       "2 2020-09-24 02:03:53  \n",
       "3 2020-09-24 02:03:53  \n",
       "4 2020-09-24 02:03:53  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['training_sample'] = data['session_id'].isin(train['session_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "208486"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['training_sample'] == True]['session_id'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "user0 = data[data['user_identifier'] == 0].sort_values(by=['tobi_timestamp', 'session_id','event_timestamp','training_sample', 'event_category_idx_lv2'],\n",
    "                                                       ascending=[True, True, True, False, False])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1941    True\n",
       "1940    True\n",
       "1939    True\n",
       "1938    True\n",
       "1937    True\n",
       "        ... \n",
       "5       True\n",
       "4       True\n",
       "3       True\n",
       "2       True\n",
       "1       True\n",
       "Name: session_id, Length: 1874, dtype: bool"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user0['session_id'].isin(train['session_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_identifier</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>event_category_idx_lv1</th>\n",
       "      <th>event_category_idx_lv2</th>\n",
       "      <th>event_category_idx_lv3</th>\n",
       "      <th>event_category_idx_lv4</th>\n",
       "      <th>tobi_timestamp</th>\n",
       "      <th>training_sample</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1941</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 13:54:51</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1940</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:34:00</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1939</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:54:31</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1938</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 20:40:38</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 20:44:58</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 10:58:23</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_12</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:21:29</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:23:50</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_12</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:46:56</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>182576</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 14:38:14</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 02:03:53</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1874 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      session_id  user_identifier     event_timestamp event_category_idx_lv1  \\\n",
       "1941      940424                0 2020-06-01 13:54:51                    1_4   \n",
       "1940      940424                0 2020-06-01 14:34:00                    1_4   \n",
       "1939      940424                0 2020-06-01 14:54:31                    1_4   \n",
       "1938      940424                0 2020-06-01 20:40:38                    1_4   \n",
       "1937      940424                0 2020-06-01 20:44:58                    1_4   \n",
       "...          ...              ...                 ...                    ...   \n",
       "5         182576                0 2020-09-22 10:58:23                    1_4   \n",
       "4         182576                0 2020-09-22 11:21:29                    1_4   \n",
       "3         182576                0 2020-09-22 11:23:50                    1_4   \n",
       "2         182576                0 2020-09-22 11:46:56                    1_4   \n",
       "1         182576                0 2020-09-22 14:38:14                    1_4   \n",
       "\n",
       "     event_category_idx_lv2 event_category_idx_lv3 event_category_idx_lv4  \\\n",
       "1941                    2_9                    3_5                   None   \n",
       "1940                    2_9                    3_5                   None   \n",
       "1939                    2_9                    3_6                   None   \n",
       "1938                    2_9                    3_5                   None   \n",
       "1937                    2_9                    3_6                   None   \n",
       "...                     ...                    ...                    ...   \n",
       "5                       2_9                   3_12                   None   \n",
       "4                       2_9                    3_6                   None   \n",
       "3                       2_9                   3_12                   None   \n",
       "2                       2_9                    3_6                   None   \n",
       "1                       2_9                    3_5                   None   \n",
       "\n",
       "          tobi_timestamp  training_sample  \n",
       "1941 2020-06-02 12:30:50             True  \n",
       "1940 2020-06-02 12:30:50             True  \n",
       "1939 2020-06-02 12:30:50             True  \n",
       "1938 2020-06-02 12:30:50             True  \n",
       "1937 2020-06-02 12:30:50             True  \n",
       "...                  ...              ...  \n",
       "5    2020-09-24 02:03:53             True  \n",
       "4    2020-09-24 02:03:53             True  \n",
       "3    2020-09-24 02:03:53             True  \n",
       "2    2020-09-24 02:03:53             True  \n",
       "1    2020-09-24 02:03:53             True  \n",
       "\n",
       "[1874 rows x 9 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id\n",
       "1788      [[1788, 0, 2020-06-14 01:07:33, 1_4, 2_9, 3_12...\n",
       "2075      [[2075, 0, 2020-06-03 10:22:32, 1_4, 2_9, 3_6,...\n",
       "6358      [[6358, 0, 2020-07-23 20:40:08, 1_4, 2_9, 3_12...\n",
       "6398      [[6398, 0, 2020-06-03 23:37:08, 1_4, 2_9, 3_13...\n",
       "11453     [[11453, 0, 2020-07-23 20:40:08, 1_4, 2_9, 3_1...\n",
       "                                ...                        \n",
       "739452    [[739452, 0, 2020-06-01 13:54:51, 1_4, 2_9, 3_...\n",
       "741630    [[741630, 0, 2020-06-01 13:54:51, 1_4, 2_9, 3_...\n",
       "756004    [[756004, 0, 2020-06-01 13:54:51, 1_4, 2_9, 3_...\n",
       "756812    [[756812, 0, 2020-06-01 13:54:51, 1_4, 2_9, 3_...\n",
       "940424    [[940424, 0, 2020-06-01 13:54:51, 1_4, 2_9, 3_...\n",
       "Length: 68, dtype: object"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user0.groupby('session_id').apply(np.array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reshape_by_user(dataset):\n",
    "    n_sequences = dataset['session_id'].nunique()\n",
    "    max_length = max(dataset['session_id'].value_counts())\n",
    "    sequences = -np.ones((n_sequences, max_length, 8), dtype=object)\n",
    "    \n",
    "    i = 0\n",
    "    grouped = dataset.groupby('session_id').apply(np.array)\n",
    "    for seq in grouped:\n",
    "        l = len(seq)\n",
    "        sequences[i,:l] = seq[:,:-1]\n",
    "        i += 1\n",
    "    return sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_0 = reshape_by_user(user0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[940424, 0, Timestamp('2020-06-01 13:54:51'), '1_4', '2_9', '3_5',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 14:34:00'), '1_4', '2_9', '3_5',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 14:54:31'), '1_4', '2_9', '3_6',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:40:38'), '1_4', '2_9', '3_5',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:44:58'), '1_4', '2_9', '3_6',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:47:11'), '1_4', '2_9',\n",
       "        '3_12', None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:49:25'), '1_4', '2_9',\n",
       "        '3_13', None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:51:40'), '1_4', '2_9', '3_5',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:53:53'), '1_4', '2_9', '3_5',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 20:58:07'), '1_4', '2_9', '3_6',\n",
       "        None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [940424, 0, Timestamp('2020-06-01 21:00:20'), '1_4', '2_9',\n",
       "        '3_12', None, Timestamp('2020-06-02 12:30:50')],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1],\n",
       "       [-1, -1, -1, -1, -1, -1, -1, -1]], dtype=object)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "int(data.loc[data['session_id'] == 13, ['tobi_timestamp']].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_identifier</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>event_category_idx_lv1</th>\n",
       "      <th>event_category_idx_lv2</th>\n",
       "      <th>event_category_idx_lv3</th>\n",
       "      <th>event_category_idx_lv4</th>\n",
       "      <th>tobi_timestamp</th>\n",
       "      <th>training_sample</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1929</th>\n",
       "      <td>756004</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 13:54:51</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-03 10:11:34</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1899</th>\n",
       "      <td>756812</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 13:54:51</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-03 10:27:00</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1928</th>\n",
       "      <td>756004</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:34:00</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-03 10:11:34</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1898</th>\n",
       "      <td>756812</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:34:00</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-03 10:27:00</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1927</th>\n",
       "      <td>756004</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:54:31</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-03 10:11:34</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>167384</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:23:50</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_12</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 01:53:58</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>107474</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:46:56</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 01:39:54</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>167384</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 11:46:56</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 01:53:58</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>107474</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 14:38:14</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 01:39:54</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>167384</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-09-22 14:38:14</td>\n",
       "      <td>1_4</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-09-24 01:53:58</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>568 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      session_id  user_identifier     event_timestamp event_category_idx_lv1  \\\n",
       "1929      756004                0 2020-06-01 13:54:51                    1_4   \n",
       "1899      756812                0 2020-06-01 13:54:51                    1_4   \n",
       "1928      756004                0 2020-06-01 14:34:00                    1_4   \n",
       "1898      756812                0 2020-06-01 14:34:00                    1_4   \n",
       "1927      756004                0 2020-06-01 14:54:31                    1_4   \n",
       "...          ...              ...                 ...                    ...   \n",
       "34        167384                0 2020-09-22 11:23:50                    1_4   \n",
       "157       107474                0 2020-09-22 11:46:56                    1_4   \n",
       "33        167384                0 2020-09-22 11:46:56                    1_4   \n",
       "156       107474                0 2020-09-22 14:38:14                    1_4   \n",
       "32        167384                0 2020-09-22 14:38:14                    1_4   \n",
       "\n",
       "     event_category_idx_lv2 event_category_idx_lv3 event_category_idx_lv4  \\\n",
       "1929                    2_9                    3_5                   None   \n",
       "1899                    2_9                    3_5                   None   \n",
       "1928                    2_9                    3_5                   None   \n",
       "1898                    2_9                    3_5                   None   \n",
       "1927                    2_9                    3_6                   None   \n",
       "...                     ...                    ...                    ...   \n",
       "34                      2_9                   3_12                   None   \n",
       "157                     2_9                    3_6                   None   \n",
       "33                      2_9                    3_6                   None   \n",
       "156                     2_9                    3_5                   None   \n",
       "32                      2_9                    3_5                   None   \n",
       "\n",
       "          tobi_timestamp  training_sample  \n",
       "1929 2020-06-03 10:11:34            False  \n",
       "1899 2020-06-03 10:27:00            False  \n",
       "1928 2020-06-03 10:11:34            False  \n",
       "1898 2020-06-03 10:27:00            False  \n",
       "1927 2020-06-03 10:11:34            False  \n",
       "...                  ...              ...  \n",
       "34   2020-09-24 01:53:58            False  \n",
       "157  2020-09-24 01:39:54            False  \n",
       "33   2020-09-24 01:53:58            False  \n",
       "156  2020-09-24 01:39:54            False  \n",
       "32   2020-09-24 01:53:58            False  \n",
       "\n",
       "[568 rows x 9 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user0[~user0['training_sample']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True    5676992\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data['tobi_timestamp'] >= data['event_timestamp']).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True    347767\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv1'] == '1_5']['event_category_idx_lv2'].apply(lambda s: s.isnumeric()).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    5329225\n",
       "True      347767\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['event_category_idx_lv2'].apply(lambda s: s.isnumeric() if isinstance(s, str) else False).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1_0     1821342\n",
       "1_1      794503\n",
       "1_2      746991\n",
       "1_4      577190\n",
       "1_3      574892\n",
       "1_5      347767\n",
       "1_6      277829\n",
       "1_7      248115\n",
       "1_8      185776\n",
       "1_9       60585\n",
       "1_10      23189\n",
       "1_11      18813\n",
       "Name: event_category_idx_lv1, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['event_category_idx_lv1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_0      761330\n",
       "2_2      714830\n",
       "2_3      551242\n",
       "2_1      549528\n",
       "2_4      339168\n",
       "          ...  \n",
       "2_366         1\n",
       "2_355         1\n",
       "2_451         1\n",
       "2_316         1\n",
       "2_311         1\n",
       "Name: event_category_idx_lv2, Length: 370, dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3_0     486973\n",
       "3_1     308585\n",
       "3_2     249488\n",
       "3_3     246024\n",
       "3_4     179971\n",
       "         ...  \n",
       "3_71         2\n",
       "3_81         2\n",
       "3_76         2\n",
       "3_72         1\n",
       "3_77         1\n",
       "Name: event_category_idx_lv3, Length: 78, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['event_category_idx_lv3'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4_0    251586\n",
       "4_1    224356\n",
       "4_2    218525\n",
       "4_3    181721\n",
       "4_4    128892\n",
       "4_5     37389\n",
       "4_6       149\n",
       "Name: event_category_idx_lv4, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['event_category_idx_lv4'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1_0     1821342\n",
       "1_1      794503\n",
       "1_2      746991\n",
       "1_4      577190\n",
       "1_3      574892\n",
       "1_5      347767\n",
       "1_6      277829\n",
       "1_7      248115\n",
       "1_10      23189\n",
       "1_11      18813\n",
       "Name: event_category_idx_lv1, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv2'].apply(lambda s: s != None)]['event_category_idx_lv1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3_2     249488\n",
       "3_4     179971\n",
       "3_9     146006\n",
       "3_5     139900\n",
       "3_6     132281\n",
       "         ...  \n",
       "3_71         2\n",
       "3_81         2\n",
       "3_75         2\n",
       "3_72         1\n",
       "3_77         1\n",
       "Name: event_category_idx_lv3, Length: 71, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv4'].apply(lambda s: s == None)]['event_category_idx_lv3'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3_2     249488\n",
       "3_4     179971\n",
       "3_9     146006\n",
       "3_5     139900\n",
       "3_6     132281\n",
       "         ...  \n",
       "3_75         2\n",
       "3_81         2\n",
       "3_71         2\n",
       "3_77         1\n",
       "3_72         1\n",
       "Name: event_category_idx_lv3, Length: 71, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv4'].apply(lambda s: s == None)]['event_category_idx_lv3'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: event_category_idx_lv4, dtype: int64)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv3'] == '3_2']['event_category_idx_lv4'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_2      714830\n",
       "2_3      551242\n",
       "2_4      339168\n",
       "2_7      184244\n",
       "2_6      182341\n",
       "          ...  \n",
       "2_375         1\n",
       "2_313         1\n",
       "2_354         1\n",
       "2_335         1\n",
       "2_346         1\n",
       "Name: event_category_idx_lv2, Length: 334, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv3'].apply(lambda s: s == None)]['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_30    16628\n",
       "2_46     6561\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv1'] == '1_10']['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1_0     1821342\n",
       "1_2      746991\n",
       "1_4      577190\n",
       "1_3      574892\n",
       "1_5      347767\n",
       "1_6      277829\n",
       "1_8      185776\n",
       "1_9       60585\n",
       "1_10      23189\n",
       "1_11      18813\n",
       "Name: event_category_idx_lv1, dtype: int64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv4'].apply(lambda s: s == None)]['event_category_idx_lv1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3_16    16590\n",
       "3_52       38\n",
       "Name: event_category_idx_lv3, dtype: int64"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv2'] == '2_30']['event_category_idx_lv3'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_30    16590\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv3'] == '3_16']['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for +: 'int' and 'str'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-124-1a8976aac79a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'ciao'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'ciaoc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'int' and 'str'"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1_7    38386\n",
       "1_1      227\n",
       "Name: event_category_idx_lv1, dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv2'] == '2_22']['event_category_idx_lv1'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_0      761330\n",
       "2_10      31499\n",
       "2_94        588\n",
       "2_143       242\n",
       "2_22        227\n",
       "2_145       149\n",
       "2_38        124\n",
       "2_98        113\n",
       "2_65         72\n",
       "2_28         45\n",
       "2_53         39\n",
       "2_78         24\n",
       "2_41         24\n",
       "2_59         15\n",
       "2_101         7\n",
       "2_55          2\n",
       "2_105         2\n",
       "2_210         1\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv1'] == '1_1']['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = []\n",
    "for i in data['event_category_idx_lv2'].unique():\n",
    "    if len(data[data['event_category_idx_lv2'] == i]['event_category_idx_lv1'].unique()) != 1:\n",
    "        a.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['2_10', '2_22', '2_28', '2_38', '2_41', '2_55', '2_53', '2_59', '2_65',\n",
       "       '2_78', '2_94', '2_105', '2_98', '2_101', '2_210'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['event_category_idx_lv2'].value_counts().index[~a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_10     80371\n",
       "2_22     38386\n",
       "2_28     24511\n",
       "2_26     20179\n",
       "2_38     11908\n",
       "2_37     10518\n",
       "2_41      9083\n",
       "2_45      8686\n",
       "2_48      8020\n",
       "2_55      6319\n",
       "2_50      5562\n",
       "2_53      5123\n",
       "2_59      4691\n",
       "2_63      3372\n",
       "2_65      3110\n",
       "2_78      3036\n",
       "2_94      1523\n",
       "2_105     1288\n",
       "2_98      1076\n",
       "2_101     1068\n",
       "2_133      204\n",
       "2_197       47\n",
       "2_210       26\n",
       "2_299        6\n",
       "2_294        2\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv1'] == '1_7']['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3_1    111870\n",
       "Name: event_category_idx_lv3, dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv2'] == '2_10']['event_category_idx_lv3'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_0      153474\n",
       "2_10     111870\n",
       "2_45       8686\n",
       "2_38       8113\n",
       "2_26       6328\n",
       "2_37       5596\n",
       "2_50       5538\n",
       "2_63       3372\n",
       "2_22       2739\n",
       "2_28       2103\n",
       "2_78        484\n",
       "2_133       204\n",
       "2_94         68\n",
       "2_48          8\n",
       "2_294         2\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv3'] == '3_1']['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threes = ['1_2', '1_4', '1_10']\n",
    "data[data['event_category_idx_lv1'].isin(threes)]['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "l = []\n",
    "for i in data['event_category_idx_lv3'].value_counts().index:\n",
    "    if len(data[data['event_category_idx_lv3'] == i]['event_category_idx_lv2'].unique()) != 1:\n",
    "        l.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['3_0',\n",
       " '3_1',\n",
       " '3_2',\n",
       " '3_3',\n",
       " '3_4',\n",
       " '3_9',\n",
       " '3_5',\n",
       " '3_6',\n",
       " '3_12',\n",
       " '3_8',\n",
       " '3_7',\n",
       " '3_13',\n",
       " '3_10',\n",
       " '3_11',\n",
       " '3_14',\n",
       " '3_15',\n",
       " '3_17',\n",
       " '3_18',\n",
       " '3_22',\n",
       " '3_31',\n",
       " '3_27',\n",
       " '3_25',\n",
       " '3_35',\n",
       " '3_34',\n",
       " '3_36',\n",
       " '3_38',\n",
       " '3_42',\n",
       " '3_40',\n",
       " '3_44',\n",
       " '3_49',\n",
       " '3_45',\n",
       " '3_47',\n",
       " '3_43',\n",
       " '3_48',\n",
       " '3_57',\n",
       " '3_58',\n",
       " '3_56',\n",
       " '3_60',\n",
       " '3_62',\n",
       " '3_70',\n",
       " '3_65',\n",
       " '3_63',\n",
       " '3_71']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = []\n",
    "for j in l:\n",
    "    n = list(data[data['event_category_idx_lv3'] == j]['event_category_idx_lv2'].value_counts().index)\n",
    "    m.extend(n)\n",
    "m = pd.Series(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = []\n",
    "for j in l:\n",
    "    n = list(data[data['event_category_idx_lv3'] == j]['event_category_idx_lv1'].unique())\n",
    "    h.extend(n)\n",
    "h = pd.Series(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2_0', '2_22', '2_28', '2_41', '2_26', '2_48', '2_53', '2_59',\n",
       "       '2_65', '2_37', '2_38', '2_98', '2_105', '2_101', '2_143', '2_78',\n",
       "       '2_197', '2_210', '2_10', '2_45', '2_50', '2_63', '2_133', '2_94',\n",
       "       '2_294', '2_1', '2_14', '2_55', '2_299', '2_13', '2_9', '2_5',\n",
       "       '2_21'], dtype=object)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_1      31\n",
       "2_13     24\n",
       "2_14     14\n",
       "2_9       5\n",
       "2_5       5\n",
       "2_0       5\n",
       "2_78      4\n",
       "2_22      4\n",
       "2_38      4\n",
       "2_48      3\n",
       "2_37      3\n",
       "2_28      3\n",
       "2_94      3\n",
       "2_26      3\n",
       "2_41      2\n",
       "2_65      2\n",
       "2_21      2\n",
       "2_105     2\n",
       "2_50      2\n",
       "2_59      2\n",
       "2_210     1\n",
       "2_53      1\n",
       "2_55      1\n",
       "2_197     1\n",
       "2_143     1\n",
       "2_133     1\n",
       "2_299     1\n",
       "2_98      1\n",
       "2_10      1\n",
       "2_45      1\n",
       "2_101     1\n",
       "2_63      1\n",
       "2_294     1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2_0      761330\n",
       "2_10      31499\n",
       "2_94        588\n",
       "2_143       242\n",
       "2_22        227\n",
       "2_145       149\n",
       "2_38        124\n",
       "2_98        113\n",
       "2_65         72\n",
       "2_28         45\n",
       "2_53         39\n",
       "2_78         24\n",
       "2_41         24\n",
       "2_59         15\n",
       "2_101         7\n",
       "2_55          2\n",
       "2_105         2\n",
       "2_210         1\n",
       "Name: event_category_idx_lv2, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['event_category_idx_lv1'] == '1_1']['event_category_idx_lv2'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_events(dataset):\n",
    "    d = {1: ['1_8', '1_9'],\n",
    "         2: ['1_0', '1_3', '1_5', '1_6', '1_11'],\n",
    "         3: ['1_10'],\n",
    "         4: ['1_2', '1_4'],\n",
    "         5: ['1_1', '1_7']}\n",
    "    \n",
    "    level1 = dataset['event_category_idx_lv1']\n",
    "    dataset.loc[level1.isin(d[2]), 'event_category_idx_lv1'] = dataset.loc[level1.isin(d[2]), 'event_category_idx_lv2']\n",
    "    dataset.loc[level1.isin(d[3]), 'event_category_idx_lv1'] = dataset.loc[level1.isin(d[3]), 'event_category_idx_lv3']\n",
    "    dataset.loc[level1.isin(d[4]), 'event_category_idx_lv1'] = dataset.loc[level1.isin(d[4]), 'event_category_idx_lv1'] + dataset.loc[level1.isin(d[4]), 'event_category_idx_lv2'] + dataset.loc[level1.isin(d[4]), 'event_category_idx_lv3']\n",
    "    dataset.loc[level1.isin(d[5]), 'event_category_idx_lv1'] = dataset.loc[level1.isin(d[5]), 'event_category_idx_lv1'] + dataset.loc[level1.isin(d[5]), 'event_category_idx_lv2'] + dataset.loc[level1.isin(d[5]), 'event_category_idx_lv3'] + dataset.loc[level1.isin(d[5]), 'event_category_idx_lv4']                                                         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_events(dataset):\n",
    "    d = {1: ['1_8', '1_9'],\n",
    "         2: ['1_0', '1_3', '1_5', '1_6', '1_11'],\n",
    "         3: ['1_10']}\n",
    "    \n",
    "    level1 = dataset['event_category_idx_lv1'].copy()\n",
    "    dataset.loc[level1.isin(d[2]), 'event_category_idx_lv1'] = dataset.loc[level1.isin(d[2]), 'event_category_idx_lv2']\n",
    "    dataset.loc[level1.isin(d[2]), 'event_category_idx_lv2'] = None\n",
    "    dataset.loc[level1.isin(d[3]), 'event_category_idx_lv1'] = dataset.loc[level1.isin(d[3]), 'event_category_idx_lv3']\n",
    "    dataset.loc[level1.isin(d[3]), 'event_category_idx_lv2'] = None\n",
    "    dataset.loc[level1.isin(d[3]), 'event_category_idx_lv3'] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['session_id'].is_monotonic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['session_id'].is_monotonic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.sort_values(by=['user_identifier','tobi_timestamp', 'event_timestamp'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "for user in data['user_identifier'].unique():\n",
    "    baseline_date = min(data.loc[data['user_identifier'] == user, 'tobi_timestamp'])\n",
    "    try:\n",
    "        data.loc[data['user_identifier'] == user, 'tobi_timestamp'] = data.loc[data['user_identifier'] == user, 'tobi_timestamp'] - baseline_date\n",
    "    except:\n",
    "        print(user)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['tobi_timestamp'] = (data['tobi_timestamp'] - min(data['tobi_timestamp'])).apply(lambda td: td.delta) * 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_date = data.groupby('user_identifier', sort=False).min(numeric_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for -: 'numpy.ndarray' and 'Timestamp'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-87-575216e7627e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'user_identifier'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tobi_timestamp'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTimestamp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'2020-06-03 23:36:51'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf\\lib\\site-packages\\pandas\\core\\ops\\common.py\u001b[0m in \u001b[0;36mnew_method\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m     62\u001b[0m         \u001b[0mother\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mitem_from_zerodim\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnew_method\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf\\lib\\site-packages\\pandas\\core\\ops\\__init__.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(left, right)\u001b[0m\n\u001b[0;32m    501\u001b[0m         \u001b[0mlvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mextract_numpy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    502\u001b[0m         \u001b[0mrvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextract_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mright\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mextract_numpy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 503\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marithmetic_op\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr_rep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    504\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    505\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0m_construct_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mres_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf\\lib\\site-packages\\pandas\\core\\ops\\array_ops.py\u001b[0m in \u001b[0;36marithmetic_op\u001b[1;34m(left, right, op, str_rep)\u001b[0m\n\u001b[0;32m    191\u001b[0m         \u001b[1;31m#  by dispatch_to_extension_op.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    192\u001b[0m         \u001b[1;31m# Timedelta is included because numexpr will fail on it, see GH#31457\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 193\u001b[1;33m         \u001b[0mres_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdispatch_to_extension_op\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    194\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    195\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf\\lib\\site-packages\\pandas\\core\\ops\\dispatch.py\u001b[0m in \u001b[0;36mdispatch_to_extension_op\u001b[1;34m(op, left, right)\u001b[0m\n\u001b[0;32m    123\u001b[0m     \u001b[1;31m# The op calls will raise TypeError if the op is not defined\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    124\u001b[0m     \u001b[1;31m# on the ExtensionArray\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 125\u001b[1;33m     \u001b[0mres_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mright\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    126\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mres_values\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: unsupported operand type(s) for -: 'numpy.ndarray' and 'Timestamp'"
     ]
    }
   ],
   "source": [
    "data.loc[data['user_identifier'] == 1, 'tobi_timestamp'] - pd.Timestamp('2020-06-03 23:36:51')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_identifier</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>event_category_idx_lv1</th>\n",
       "      <th>event_category_idx_lv2</th>\n",
       "      <th>event_category_idx_lv3</th>\n",
       "      <th>event_category_idx_lv4</th>\n",
       "      <th>tobi_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1941</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 13:54:51</td>\n",
       "      <td>1_42_93_5</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1940</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:34:00</td>\n",
       "      <td>1_42_93_5</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1939</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 14:54:31</td>\n",
       "      <td>1_42_93_6</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1938</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 20:40:38</td>\n",
       "      <td>1_42_93_5</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_5</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937</th>\n",
       "      <td>940424</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-06-01 20:44:58</td>\n",
       "      <td>1_42_93_6</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_6</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 12:30:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5676962</th>\n",
       "      <td>1556478</td>\n",
       "      <td>871002</td>\n",
       "      <td>2020-06-01 00:00:00</td>\n",
       "      <td>1_22_13_2</td>\n",
       "      <td>2_1</td>\n",
       "      <td>3_2</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-08 18:53:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5676980</th>\n",
       "      <td>1554939</td>\n",
       "      <td>871090</td>\n",
       "      <td>2020-06-28 10:00:00</td>\n",
       "      <td>1_72_373_34_4</td>\n",
       "      <td>2_37</td>\n",
       "      <td>3_3</td>\n",
       "      <td>4_4</td>\n",
       "      <td>2020-07-04 14:56:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5676982</th>\n",
       "      <td>1576694</td>\n",
       "      <td>871093</td>\n",
       "      <td>2020-06-02 12:36:27</td>\n",
       "      <td>2_2</td>\n",
       "      <td>2_2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-05 11:43:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5676984</th>\n",
       "      <td>1549128</td>\n",
       "      <td>871095</td>\n",
       "      <td>2020-06-01 17:22:28</td>\n",
       "      <td>1_42_93_13</td>\n",
       "      <td>2_9</td>\n",
       "      <td>3_13</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-02 16:26:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5676988</th>\n",
       "      <td>1559831</td>\n",
       "      <td>871110</td>\n",
       "      <td>2020-06-01 00:00:00</td>\n",
       "      <td>1_22_143_4</td>\n",
       "      <td>2_14</td>\n",
       "      <td>3_4</td>\n",
       "      <td>None</td>\n",
       "      <td>2020-06-17 10:39:45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3197684 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         session_id  user_identifier     event_timestamp  \\\n",
       "1941         940424                0 2020-06-01 13:54:51   \n",
       "1940         940424                0 2020-06-01 14:34:00   \n",
       "1939         940424                0 2020-06-01 14:54:31   \n",
       "1938         940424                0 2020-06-01 20:40:38   \n",
       "1937         940424                0 2020-06-01 20:44:58   \n",
       "...             ...              ...                 ...   \n",
       "5676962     1556478           871002 2020-06-01 00:00:00   \n",
       "5676980     1554939           871090 2020-06-28 10:00:00   \n",
       "5676982     1576694           871093 2020-06-02 12:36:27   \n",
       "5676984     1549128           871095 2020-06-01 17:22:28   \n",
       "5676988     1559831           871110 2020-06-01 00:00:00   \n",
       "\n",
       "        event_category_idx_lv1 event_category_idx_lv2 event_category_idx_lv3  \\\n",
       "1941                 1_42_93_5                    2_9                    3_5   \n",
       "1940                 1_42_93_5                    2_9                    3_5   \n",
       "1939                 1_42_93_6                    2_9                    3_6   \n",
       "1938                 1_42_93_5                    2_9                    3_5   \n",
       "1937                 1_42_93_6                    2_9                    3_6   \n",
       "...                        ...                    ...                    ...   \n",
       "5676962              1_22_13_2                    2_1                    3_2   \n",
       "5676980          1_72_373_34_4                   2_37                    3_3   \n",
       "5676982                    2_2                    2_2                   None   \n",
       "5676984             1_42_93_13                    2_9                   3_13   \n",
       "5676988             1_22_143_4                   2_14                    3_4   \n",
       "\n",
       "        event_category_idx_lv4      tobi_timestamp  \n",
       "1941                      None 2020-06-02 12:30:50  \n",
       "1940                      None 2020-06-02 12:30:50  \n",
       "1939                      None 2020-06-02 12:30:50  \n",
       "1938                      None 2020-06-02 12:30:50  \n",
       "1937                      None 2020-06-02 12:30:50  \n",
       "...                        ...                 ...  \n",
       "5676962                   None 2020-06-08 18:53:55  \n",
       "5676980                    4_4 2020-07-04 14:56:13  \n",
       "5676982                   None 2020-06-05 11:43:37  \n",
       "5676984                   None 2020-06-02 16:26:54  \n",
       "5676988                   None 2020-06-17 10:39:45  \n",
       "\n",
       "[3197684 rows x 8 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.set_index(train['session_id'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>session_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>session_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606177</th>\n",
       "      <td>3</td>\n",
       "      <td>1606177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606199</th>\n",
       "      <td>2</td>\n",
       "      <td>1606199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606217</th>\n",
       "      <td>2</td>\n",
       "      <td>1606217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606226</th>\n",
       "      <td>1</td>\n",
       "      <td>1606226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606227</th>\n",
       "      <td>2</td>\n",
       "      <td>1606227</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208486 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            label  session_id\n",
       "session_id                   \n",
       "6               2           6\n",
       "13              1          13\n",
       "21              2          21\n",
       "22              2          22\n",
       "32              4          32\n",
       "...           ...         ...\n",
       "1606177         3     1606177\n",
       "1606199         2     1606199\n",
       "1606217         2     1606217\n",
       "1606226         1     1606226\n",
       "1606227         2     1606227\n",
       "\n",
       "[208486 rows x 2 columns]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.iloc[data[filt]['session_id'].to_numpy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['label'] = -1\n",
    "def assign(row):\n",
    "    row['label'] = train[train['session_id'] == row['session_id']]\n",
    "# data[data['session_id'].isin(train['session_id'])]\n",
    "filt = data['session_id'].isin(train['session_id'])\n",
    "data.loc[filt, 'label'] = train.iloc[data.loc[filt, 'session_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1    5329225\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1           True\n",
       "2           True\n",
       "3           True\n",
       "4           True\n",
       "5           True\n",
       "           ...  \n",
       "5676984     True\n",
       "5676986    False\n",
       "5676988     True\n",
       "5676990    False\n",
       "5676991    False\n",
       "Name: session_id, Length: 5329225, dtype: bool"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['session_id'].isin(train['session_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reshape_by_user(dataset): #funziona\n",
    "    n_users = dataset['user_identifier'].nunique()\n",
    "    max_length = max(dataset['session_id'].value_counts())\n",
    "    sequences = -np.ones((n_users, 69, 32), dtype=object)\n",
    "    labels = np.zeros(n_users)\n",
    "    \n",
    "    i = 0\n",
    "    unique_users = dataset['user_identifier'].unique()\n",
    "    grouped = dataset.groupby(['user_identifier', 'session_id'], sort=False).apply(np.array)\n",
    "    for user in unique_users:\n",
    "        sequences[i,0,:] = user\n",
    "        j = 1\n",
    "        length = len(grouped[user])\n",
    "        for session in grouped[user]:\n",
    "            l = session.shape[0]\n",
    "            session = session.reshape((-1, session.shape[-1]))\n",
    "            sequences[i,j,0] = session[0,-1] - minimum_date.loc[user, 'tobi_timestamp']\n",
    "            sequences[i,j,1:l+1] = session[:,3]\n",
    "            \n",
    "            if j != length:\n",
    "                sequences[i,j,-1] = train.loc[session[0,0], 'label'] \n",
    "            else:\n",
    "                labels[i] = train.loc[session[0,0], 'label']\n",
    "            j += 1\n",
    "        i += 1\n",
    "    return sequences, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reshape_by_user_test(dataset):\n",
    "    n_sessions = dataset['session_id'].nunique()\n",
    "    max_length = max(dataset['session_id'].value_counts())\n",
    "    sequences = -np.ones((n_sessions, 69, 32), dtype=object)\n",
    "    labels = np.zeros(n_sessions)\n",
    "    \n",
    "    i = 0\n",
    "    unique_users = dataset['user_identifier'].unique()\n",
    "    grouped = dataset.groupby(['user_identifier', 'session_id'], sort=False).apply(np.array)\n",
    "    for user in unique_users:\n",
    "        for session in grouped[user]:\n",
    "            sequences[i,0,:] = user\n",
    "            l = session.shape[0]\n",
    "            session = session.reshape((-1, session.shape[-1]))\n",
    "            sequences[i,1,0] = session[0,-1] - minimum_date.loc[user, 'tobi_timestamp']\n",
    "            sequences[i,1,1:l+1] = session[:,3]\n",
    "            labels[i] = train.loc[session[0,0], 'label']\n",
    "            i += 1\n",
    "    return sequences, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences, labels = reshape_by_user(training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train, y_test_and_val = train_test_split(train, train_size=0.7, random_state=42)\n",
    "y_test, y_val = train_test_split(y_test_and_val, train_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = training_data[training_data['session_id'].isin(y_train['session_id'])]\n",
    "test_df = training_data[training_data['session_id'].isin(y_test['session_id'])]\n",
    "val_df = training_data[training_data['session_id'].isin(y_val['session_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sequences, train_labels = reshape_by_user(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sequences, test_labels = reshape_by_user_test(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_sequences, val_labels = reshape_by_user_test(val_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, Timestamp('2020-06-09 12:41:36'), -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1], dtype=object)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_sequences[1,:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10, Timestamp('2020-06-03 21:51:59'),\n",
       "       Timestamp('2020-06-03 23:53:05'), Timestamp('2020-06-05 12:09:04'),\n",
       "       Timestamp('2020-06-06 18:06:58'), Timestamp('2020-06-09 17:47:21'),\n",
       "       Timestamp('2020-07-03 10:53:44'), Timestamp('2020-07-04 20:09:59'),\n",
       "       Timestamp('2020-07-24 12:50:23'), -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1], dtype=object)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sequences[10,:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, Timestamp('2020-06-02 12:30:50'),\n",
       "       Timestamp('2020-06-03 10:24:45'), Timestamp('2020-06-03 23:31:44'),\n",
       "       Timestamp('2020-06-03 23:37:08'), Timestamp('2020-06-05 08:49:17'),\n",
       "       Timestamp('2020-06-05 09:02:23'), Timestamp('2020-06-05 12:22:20'),\n",
       "       Timestamp('2020-06-09 12:41:36'), Timestamp('2020-06-11 17:38:21'),\n",
       "       Timestamp('2020-06-13 23:37:53'), Timestamp('2020-06-14 00:43:31'),\n",
       "       Timestamp('2020-06-14 00:56:36'), Timestamp('2020-06-14 01:07:33'),\n",
       "       Timestamp('2020-06-14 01:18:28'), Timestamp('2020-06-14 01:31:35'),\n",
       "       Timestamp('2020-06-14 01:44:34'), Timestamp('2020-06-14 01:57:40'),\n",
       "       Timestamp('2020-06-14 02:24:14'), Timestamp('2020-06-14 02:37:44'),\n",
       "       Timestamp('2020-06-18 19:13:05'), Timestamp('2020-06-25 12:29:55'),\n",
       "       Timestamp('2020-06-25 12:34:19'), Timestamp('2020-06-25 12:42:54'),\n",
       "       Timestamp('2020-06-25 12:45:06'), Timestamp('2020-06-25 12:47:22'),\n",
       "       Timestamp('2020-06-25 12:55:58'), Timestamp('2020-06-25 12:58:10'),\n",
       "       Timestamp('2020-06-25 13:24:30'), Timestamp('2020-06-25 13:26:44'),\n",
       "       Timestamp('2020-06-25 13:59:25'), Timestamp('2020-06-25 14:01:35'),\n",
       "       Timestamp('2020-06-25 14:16:56'), Timestamp('2020-06-25 14:25:29'),\n",
       "       Timestamp('2020-06-26 16:03:44'), Timestamp('2020-07-23 20:40:08'),\n",
       "       Timestamp('2020-07-29 17:10:48'), Timestamp('2020-09-20 11:50:05'),\n",
       "       Timestamp('2020-09-20 11:52:29'), Timestamp('2020-09-21 09:43:06'),\n",
       "       Timestamp('2020-09-21 10:50:50'), Timestamp('2020-09-21 10:52:54'),\n",
       "       Timestamp('2020-09-22 11:21:29'), Timestamp('2020-09-22 11:23:50'),\n",
       "       Timestamp('2020-09-24 01:29:59'), Timestamp('2020-09-24 01:41:58'),\n",
       "       Timestamp('2020-09-24 01:49:53'), Timestamp('2020-09-24 01:51:53'),\n",
       "       Timestamp('2020-09-24 02:03:53'), -1, -1, -1, -1, -1, -1, -1, -1,\n",
       "       -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], dtype=object)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences[0,:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>session_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>session_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606177</th>\n",
       "      <td>3</td>\n",
       "      <td>1606177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606199</th>\n",
       "      <td>2</td>\n",
       "      <td>1606199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606217</th>\n",
       "      <td>2</td>\n",
       "      <td>1606217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606226</th>\n",
       "      <td>1</td>\n",
       "      <td>1606226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606227</th>\n",
       "      <td>2</td>\n",
       "      <td>1606227</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208486 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            label  session_id\n",
       "session_id                   \n",
       "6               2           6\n",
       "13              1          13\n",
       "21              2          21\n",
       "22              2          22\n",
       "32              4          32\n",
       "...           ...         ...\n",
       "1606177         3     1606177\n",
       "1606199         2     1606199\n",
       "1606217         2     1606217\n",
       "1606226         1     1606226\n",
       "1606227         2     1606227\n",
       "\n",
       "[208486 rows x 2 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['tobi_timestamp'] = (data['tobi_timestamp'] - data['event_timestamp']).apply(lambda td: td.days)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "filt = data['event_category_idx_lv2'].apply(lambda s: s.isnumeric() if isinstance(s, str) else False)\n",
    "to_drop = data[filt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(index=to_drop.index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "encode_events(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = data.sort_values(by=['event_category_idx_lv1'])['event_category_idx_lv1'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = LabelEncoder()\n",
    "encoder.fit(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['event_category_idx_lv1'] = encoder.transform(data['event_category_idx_lv1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.drop(columns=['event_category_idx_lv2', 'event_category_idx_lv3', 'event_category_idx_lv4'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_by_user = data.groupby(['user_identifier', 'session_id'], sort=False).apply(np.array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "session_id\n",
       "940424    [[940424, 0, 2020-06-01 13:54:51, 1_42_93_5, 2...\n",
       "756004    [[756004, 0, 2020-06-01 13:54:51, 1_42_93_5, 2...\n",
       "724114    [[724114, 0, 2020-06-01 13:54:51, 1_42_93_5, 2...\n",
       "756812    [[756812, 0, 2020-06-01 13:54:51, 1_42_93_5, 2...\n",
       "739452    [[739452, 0, 2020-06-01 13:54:51, 1_42_93_5, 2...\n",
       "                                ...                        \n",
       "90303     [[90303, 0, 2020-07-23 20:40:08, 1_42_93_12, 2...\n",
       "11453     [[11453, 0, 2020-07-23 20:40:08, 1_42_93_12, 2...\n",
       "102033    [[102033, 0, 2020-07-23 20:40:08, 1_42_93_12, ...\n",
       "167384    [[167384, 0, 2020-07-23 20:40:08, 1_42_93_12, ...\n",
       "182576    [[182576, 0, 2020-07-23 20:40:08, 1_42_93_12, ...\n",
       "Length: 68, dtype: object"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_by_user[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_users = data['user_identifier'].unique()\n",
    "opt = -np.inf\n",
    "best_user = None\n",
    "for user in unique_users:\n",
    "    l = len(grouped_by_user[user])\n",
    "    if l > opt:\n",
    "        opt = l\n",
    "        best_user = user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_identifier  session_id\n",
       "0                182576        [[182576, 0, 2020-09-22 14:38:14, 1_42_93_5, 2...\n",
       "                 167384        [[167384, 0, 2020-09-22 14:38:14, 1_42_93_5, 2...\n",
       "                 102033        [[102033, 0, 2020-09-22 14:38:14, 1_42_93_5, 2...\n",
       "                 11453         [[11453, 0, 2020-09-22 14:38:14, 1_42_93_5, 2_...\n",
       "                 90303         [[90303, 0, 2020-09-22 14:38:14, 1_42_93_5, 2_...\n",
       "                                                     ...                        \n",
       "871095           1549128       [[1549128, 871095, 2020-06-01 17:22:28, 1_42_9...\n",
       "871099           1590753       [[1590753, 871099, 2020-06-01 00:00:00, 1_22_1...\n",
       "871110           1559831       [[1559831, 871110, 2020-06-01 00:00:00, 1_22_1...\n",
       "871111           1586623       [[1586623, 871111, 2020-06-05 17:51:07, 1_8, N...\n",
       "871126           1606224       [[1606224, 871126, 2020-06-01 00:00:00, 1_22_1...\n",
       "Length: 347837, dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped_by_user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Timestamp('2020-04-14 19:16:49')"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(data['event_timestamp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline_date = pd.Timestamp(year=2020, month=4, day=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['event_timestamp'] = (data['event_timestamp'] - baseline_date).apply(lambda ts: ts.delta) * 1e-9 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "event_categories = ['event_category_idx_lv2', 'event_category_idx_lv3', 'event_category_idx_lv4']\n",
    "for col in event_categories:\n",
    "    data[col] = data[col].apply(lambda s: int(s[2:]) if isinstance(s, str) else -1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = LabelEncoder()\n",
    "data['event_category_idx_lv1'] = encoder.fit_transform(data['event_category_idx_lv1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = data[data['session_id'].isin(train['session_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_data = data[data['session_id'].isin(test['session_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_train, seq_test_and_val = train_test_split(train, train_size=0.7, random_state=42)\n",
    "seq_test, seq_val = train_test_split(seq_test_and_val, train_size=0.5, random_state=42)\n",
    "\n",
    "train_df = training_data[training_data['session_id'].isin(seq_train['session_id'])]\n",
    "test_df = training_data[training_data['session_id'].isin(seq_test['session_id'])]\n",
    "val_df = training_data[training_data['session_id'].isin(seq_val['session_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_identifier</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>event_category_idx_lv1</th>\n",
       "      <th>event_category_idx_lv2</th>\n",
       "      <th>event_category_idx_lv3</th>\n",
       "      <th>event_category_idx_lv4</th>\n",
       "      <th>tobi_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1041402</th>\n",
       "      <td>13</td>\n",
       "      <td>0.017192</td>\n",
       "      <td>0.709845</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.219780</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.164835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041401</th>\n",
       "      <td>13</td>\n",
       "      <td>0.017192</td>\n",
       "      <td>0.711627</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.019780</td>\n",
       "      <td>0.168831</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.164835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041400</th>\n",
       "      <td>13</td>\n",
       "      <td>0.017192</td>\n",
       "      <td>0.711782</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.054945</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.164835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041399</th>\n",
       "      <td>13</td>\n",
       "      <td>0.017192</td>\n",
       "      <td>0.712119</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.210989</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.164835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1041398</th>\n",
       "      <td>13</td>\n",
       "      <td>0.017192</td>\n",
       "      <td>0.712531</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008791</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.153846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4644870</th>\n",
       "      <td>1606146</td>\n",
       "      <td>0.385380</td>\n",
       "      <td>0.171370</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.175824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5674132</th>\n",
       "      <td>1606153</td>\n",
       "      <td>0.990610</td>\n",
       "      <td>0.199982</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.030769</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.032967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5648728</th>\n",
       "      <td>1606158</td>\n",
       "      <td>0.933034</td>\n",
       "      <td>0.026270</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.013187</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.164835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154161</th>\n",
       "      <td>1606177</td>\n",
       "      <td>0.020426</td>\n",
       "      <td>0.083601</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.019780</td>\n",
       "      <td>0.077922</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1431141</th>\n",
       "      <td>1606226</td>\n",
       "      <td>0.029303</td>\n",
       "      <td>0.041270</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.010989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2241651 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         session_id  user_identifier  event_timestamp  event_category_idx_lv1  \\\n",
       "1041402          13         0.017192         0.709845                0.545455   \n",
       "1041401          13         0.017192         0.711627                0.363636   \n",
       "1041400          13         0.017192         0.711782                0.545455   \n",
       "1041399          13         0.017192         0.712119                0.545455   \n",
       "1041398          13         0.017192         0.712531                0.000000   \n",
       "...             ...              ...              ...                     ...   \n",
       "4644870     1606146         0.385380         0.171370                0.090909   \n",
       "5674132     1606153         0.990610         0.199982                0.181818   \n",
       "5648728     1606158         0.933034         0.026270                0.272727   \n",
       "1154161     1606177         0.020426         0.083601                0.363636   \n",
       "1431141     1606226         0.029303         0.041270                0.090909   \n",
       "\n",
       "         event_category_idx_lv2  event_category_idx_lv3  \\\n",
       "1041402                0.219780               -1.000000   \n",
       "1041401                0.019780                0.168831   \n",
       "1041400                0.054945               -1.000000   \n",
       "1041399                0.210989               -1.000000   \n",
       "1041398                0.008791               -1.000000   \n",
       "...                         ...                     ...   \n",
       "4644870                0.000000                0.000000   \n",
       "5674132                0.030769                0.025974   \n",
       "5648728                0.013187               -1.000000   \n",
       "1154161                0.019780                0.077922   \n",
       "1431141                0.000000                0.000000   \n",
       "\n",
       "         event_category_idx_lv4  tobi_timestamp  \n",
       "1041402               -1.000000        0.164835  \n",
       "1041401               -1.000000        0.164835  \n",
       "1041400               -1.000000        0.164835  \n",
       "1041399               -1.000000        0.164835  \n",
       "1041398               -1.000000        0.153846  \n",
       "...                         ...             ...  \n",
       "4644870                0.000000        0.175824  \n",
       "5674132               -1.000000        0.032967  \n",
       "5648728               -1.000000        0.164835  \n",
       "1154161               -1.000000        0.142857  \n",
       "1431141                0.166667        0.010989  \n",
       "\n",
       "[2241651 rows x 8 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_identifier</th>\n",
       "      <th>event_timestamp</th>\n",
       "      <th>event_category_idx_lv1</th>\n",
       "      <th>event_category_idx_lv2</th>\n",
       "      <th>event_category_idx_lv3</th>\n",
       "      <th>event_category_idx_lv4</th>\n",
       "      <th>tobi_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4134595</th>\n",
       "      <td>6</td>\n",
       "      <td>237376</td>\n",
       "      <td>7980451.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4134594</th>\n",
       "      <td>6</td>\n",
       "      <td>237376</td>\n",
       "      <td>8021332.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4134593</th>\n",
       "      <td>6</td>\n",
       "      <td>237376</td>\n",
       "      <td>8021360.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4134592</th>\n",
       "      <td>6</td>\n",
       "      <td>237376</td>\n",
       "      <td>8021371.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4134591</th>\n",
       "      <td>6</td>\n",
       "      <td>237376</td>\n",
       "      <td>8021415.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154161</th>\n",
       "      <td>1606177</td>\n",
       "      <td>17793</td>\n",
       "      <td>4902655.0</td>\n",
       "      <td>4</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5670922</th>\n",
       "      <td>1606199</td>\n",
       "      <td>853582</td>\n",
       "      <td>4147200.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5532706</th>\n",
       "      <td>1606217</td>\n",
       "      <td>701497</td>\n",
       "      <td>5657100.0</td>\n",
       "      <td>3</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1431141</th>\n",
       "      <td>1606226</td>\n",
       "      <td>25526</td>\n",
       "      <td>4454640.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5668409</th>\n",
       "      <td>1606227</td>\n",
       "      <td>846547</td>\n",
       "      <td>5529600.0</td>\n",
       "      <td>2</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3197684 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         session_id  user_identifier  event_timestamp  event_category_idx_lv1  \\\n",
       "4134595           6           237376        7980451.0                       0   \n",
       "4134594           6           237376        8021332.0                       0   \n",
       "4134593           6           237376        8021360.0                       0   \n",
       "4134592           6           237376        8021371.0                       0   \n",
       "4134591           6           237376        8021415.0                       0   \n",
       "...             ...              ...              ...                     ...   \n",
       "1154161     1606177            17793        4902655.0                       4   \n",
       "5670922     1606199           853582        4147200.0                       2   \n",
       "5532706     1606217           701497        5657100.0                       3   \n",
       "1431141     1606226            25526        4454640.0                       1   \n",
       "5668409     1606227           846547        5529600.0                       2   \n",
       "\n",
       "         event_category_idx_lv2  event_category_idx_lv3  \\\n",
       "4134595                     3.0                     NaN   \n",
       "4134594                     2.0                     NaN   \n",
       "4134593                     4.0                     NaN   \n",
       "4134592                     4.0                     NaN   \n",
       "4134591                     3.0                     NaN   \n",
       "...                         ...                     ...   \n",
       "1154161                     9.0                     6.0   \n",
       "5670922                     1.0                     2.0   \n",
       "5532706                     6.0                     NaN   \n",
       "1431141                     0.0                     0.0   \n",
       "5668409                    14.0                     4.0   \n",
       "\n",
       "         event_category_idx_lv4  tobi_timestamp  \n",
       "4134595                     NaN               8  \n",
       "4134594                     NaN               7  \n",
       "4134593                     NaN               7  \n",
       "4134592                     NaN               7  \n",
       "4134591                     NaN               7  \n",
       "...                         ...             ...  \n",
       "1154161                     NaN              13  \n",
       "5670922                     NaN               5  \n",
       "5532706                     NaN               1  \n",
       "1431141                     1.0               1  \n",
       "5668409                     NaN              13  \n",
       "\n",
       "[3197684 rows x 8 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "347837"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_sequences = data['session_id'].nunique()\n",
    "n_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reshape_sequences(dataset):\n",
    "    n_sequences = dataset['session_id'].nunique()\n",
    "    max_length = max(dataset['session_id'].value_counts())\n",
    "    sequences = -np.ones((n_sequences, max_length, 7))\n",
    "    \n",
    "    i = 0\n",
    "    grouped = dataset.groupby('session_id').apply(np.array)\n",
    "    for seq in grouped:\n",
    "        l = len(seq)\n",
    "        sequences[i,:l] = seq[:,1:]\n",
    "        i += 1\n",
    "    return sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = reshape_sequences(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = reshape_sequences(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val = reshape_sequences(val_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training = reshape_sequences(training_data, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing = reshape_sequences(testing_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = (seq_train['label'] - 1).to_numpy()\n",
    "y_test = (seq_test['label'] - 1).to_numpy()\n",
    "y_val = (seq_val['label'] - 1).to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels = (train['label'].to_numpy() - 1)  # REMEMBER TO DO +1 ON PREDICTION WHEN SUBMITTING!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, X_test_and_val = train_test_split(training, train_size=0.7, random_state=42)\n",
    "# y_train, y_test_and_val = train_test_split(labels, train_size=0.7, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, X_test_and_val, y_train, y_test_and_val = train_test_split(training, labels, train_size=0.7, random_state=42)\n",
    "# X_test, X_val, y_test, y_val = train_test_split(X_test_and_val, y_test_and_val, train_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaler = Standa()\n",
    "# X_train = scaler.fit_transform(X_train.reshape(-1, X_train.shape[-1])).reshape(X_train.shape)\n",
    "# X_test = scaler.transform(X_test.reshape(-1, X_test.shape[-1])).reshape(X_test.shape)\n",
    "# X_val = scaler.transform(X_val.reshape(-1, X_val.shape[-1])).reshape(X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "#model.add(Embedding(input_dim=uniques,\n",
    "#              output_dim=64,\n",
    "#              trainable=False,\n",
    "#              mask_zero=True))\n",
    "\n",
    "#model.add(Conv1D(10, 2, padding='same', activation='relu', input_shape=(30, 7)))\n",
    "#model.add(TimeDistributed(Flatten()))\n",
    "model.add(GRU(120, activation='softsign', return_sequences=True, dropout=0.1))\n",
    "model.add(GRU(120, activation='softsign', return_sequences=False, dropout=0.1, kernel_initializer='zeros'))\n",
    "model.add(Dense(72, activation='relu', kernel_initializer='random_normal'))\n",
    "model.add(Dropout(0.1))\n",
    "model.add(Dense(4, activation='softmax'))\n",
    "\n",
    "opt = Adam(learning_rate=1e-1)\n",
    "model.compile(\n",
    "    optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31273/31273 [==============================] - 10s 320us/sample - loss: 1.1965 - accuracy: 0.4674\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.19654573027053, 0.467368]"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}