00_customclientdata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp customclientdata\n",
    "%load_ext lab_black\n",
    "# nb_black if running in jupyter\n",
    "%load_ext autoreload\n",
    "# automatically reload python modules if there are changes in the\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CustomClientData\n",
    "\n",
    "> Create custom TFF ClientData from Pandas Dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***input***: none / raw data\n",
    "\n",
    "***output***: function for creating custom tff ClientData\n",
    "\n",
    "***description:***\n",
    "\n",
    "In this notebook we generate customer path data, and transform it into tensorflow federated [ClientData](https://www.tensorflow.org/federated/tutorials/working_with_client_data) format.\n",
    "You can edit the code to load and clean and transform your own client path data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import relevant modules"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uncomment the following cell to run with the newest version of tff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tensorflow_federated_nightly also bring in tf_nightly, which\n",
    "# can causes a duplicate tensorboard install, leading to errors.\n",
    "#!pip uninstall --yes tensorboard tb-nightly\n",
    "\n",
    "#!pip install --quiet --upgrade tensorflow-federated-nightly\n",
    "#!pip install --quiet --upgrade nest-asyncio\n",
    "#!pip install --quiet --upgrade tb-nightly  # or tensorboard, but not both"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": "2021-12-23 11:40:05.852008: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
    }
   ],
   "source": [
    "# export\n",
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import tensorflow_federated as tff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyarrow import feather\n",
    "from scipy.special import softmax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define notebook parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell is tagged with 'parameters' for papermill use\n",
    "seed = 0\n",
    "\n",
    "SHUFFLE_BUFFER = 100\n",
    "NUM_EPOCHS = 1\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "n_customers = 10000  # number of customers (paths, assuming only one path per customer)\n",
    "max_path_length = 100  # limit simulation length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define any immediate derivative operations from the parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set seed\n",
    "np.random.seed(seed)\n",
    "tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (or alternatively load your own data and turn it into an applicable format)\r\n",
    "\r\n",
    "Try for example: https://cseweb.ucsd.edu/~jmcauley/datasets.html\r\n",
    "\r\n",
    "or https://archive.ics.uci.edu/ml/datasets/Entree+Chicago+Recommendation+Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import requests, zipfile, io\n",
    "\n",
    "p = Path().cwd() / \"data\" / \"raw_data\"\n",
    "if not (p / \"customer-journey-unil-ch-datasets\").exists():  # check if already loaded\n",
    "    r = requests.get(\n",
    "        \"http://customer-journey.me/wp-content/uploads/2018/02/customer-journey-unil-ch-datasets.zip\"\n",
    "    )\n",
    "    z = zipfile.ZipFile(io.BytesIO(r.content))\n",
    "    z.extractall(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>trace_id</th>\n      <th>activities</th>\n      <th>owner</th>\n      <th>employed</th>\n      <th>age</th>\n      <th>income</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>activity_09</td>\n      <td>no</td>\n      <td>yes</td>\n      <td>20-39yo</td>\n      <td>high</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>activity_04</td>\n      <td>no</td>\n      <td>yes</td>\n      <td>20-39yo</td>\n      <td>high</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>activity_02</td>\n      <td>no</td>\n      <td>yes</td>\n      <td>20-39yo</td>\n      <td>high</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>activity_09</td>\n      <td>no</td>\n      <td>yes</td>\n      <td>60-79yo</td>\n      <td>high</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>activity_04</td>\n      <td>no</td>\n      <td>yes</td>\n      <td>60-79yo</td>\n      <td>high</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4298</th>\n      <td>4994</td>\n      <td>activity_07</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>40-59yo</td>\n      <td>middle</td>\n    </tr>\n    <tr>\n      <th>4299</th>\n      <td>4994</td>\n      <td>activity_08</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>40-59yo</td>\n      <td>middle</td>\n    </tr>\n    <tr>\n      <th>4300</th>\n      <td>4994</td>\n      <td>activity_09</td>\n      <td>yes</td>\n      <td>no</td>\n      <td>40-59yo</td>\n      <td>middle</td>\n    </tr>\n    <tr>\n      <th>4301</th>\n      <td>4995</td>\n      <td>activity_10</td>\n      <td>yes</td>\n      <td>yes</td>\n      <td>60-79yo</td>\n      <td>middle</td>\n    </tr>\n    <tr>\n      <th>4302</th>\n      <td>4995</td>\n      <td>activity_06</td>\n      <td>yes</td>\n      <td>yes</td>\n      <td>60-79yo</td>\n      <td>middle</td>\n    </tr>\n  </tbody>\n</table>\n<p>21521 rows × 6 columns</p>\n</div>",
      "text/plain": "      trace_id   activities owner employed      age  income\n0            0  activity_09    no      yes  20-39yo    high\n1            0  activity_04    no      yes  20-39yo    high\n2            0  activity_02    no      yes  20-39yo    high\n3            1  activity_09    no      yes  60-79yo    high\n4            1  activity_04    no      yes  60-79yo    high\n...        ...          ...   ...      ...      ...     ...\n4298      4994  activity_07   yes       no  40-59yo  middle\n4299      4994  activity_08   yes       no  40-59yo  middle\n4300      4994  activity_09   yes       no  40-59yo  middle\n4301      4995  activity_10   yes      yes  60-79yo  middle\n4302      4995  activity_06   yes      yes  60-79yo  middle\n\n[21521 rows x 6 columns]"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filepaths = (\n",
    "    p\n",
    "    / \"customer-journey-unil-ch-datasets\"\n",
    "    / \"csv\"\n",
    "    / \"configuration6\"\n",
    "    / \"excluding-solution\"\n",
    ").glob(\"*.csv\")\n",
    "\n",
    "\n",
    "def read_csv_list_to_df(filepaths):\n",
    "    max_trace_id = 0\n",
    "    df_list = []\n",
    "    for f in filepaths:\n",
    "        df = pd.read_csv(f)\n",
    "        df.trace_id += max_trace_id  # user id count begins from 0 at each file\n",
    "        max_trace_id = df.trace_id.max()\n",
    "        df_list.append(df)\n",
    "\n",
    "    return pd.concat(df_list)\n",
    "\n",
    "\n",
    "df = read_csv_list_to_df(filepaths)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "trace_id      4996\nactivities      15\nowner            2\nemployed         2\nage              5\nincome           3\ndtype: int64"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array(['activity_09', 'activity_04', 'activity_02', 'activity_10',\n       'activity_06', 'activity_03', 'activity_07', 'activity_08',\n       'activity_01', 'activity_05', 'noise_2', 'noise_0', 'noise_4',\n       'noise_3', 'noise_1'], dtype=object)"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.activities.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array(['20-39yo', '60-79yo', '0-19yo', '40-59yo', '80yo+'], dtype=object)"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.age.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "array(['high', 'middle', 'low'], dtype=object)"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.income.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.trace_id = df.trace_id.astype(\"int\")\n",
    "df.activities = df.activities.astype(\"category\")\n",
    "df.owner = df.owner.map(lambda x: 1 if \"yes\" else 0).astype(\"uint8\")\n",
    "df.employed = df.employed.map(lambda x: 1 if \"yes\" else 0).astype(\"uint8\")\n",
    "df.age = df.age.map(\n",
    "    {\"0-19yo\": 0, \"20-39yo\": 1, \"40-59yo\": 2, \"60-79yo\": 3, \"80yo+\": 4}\n",
    ").astype(\"uint8\")\n",
    "df.income = df.income.map({\"low\": 0, \"middle\": 1, \"high\": 2}).astype(\"uint8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>action_index</th>\n      <th>trace_id</th>\n      <th>activities</th>\n      <th>owner</th>\n      <th>employed</th>\n      <th>age</th>\n      <th>income</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>activity_09</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0</td>\n      <td>activity_04</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>0</td>\n      <td>activity_02</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>1</td>\n      <td>activity_09</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>1</td>\n      <td>activity_04</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>21516</th>\n      <td>4298</td>\n      <td>4994</td>\n      <td>activity_07</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>21517</th>\n      <td>4299</td>\n      <td>4994</td>\n      <td>activity_08</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>21518</th>\n      <td>4300</td>\n      <td>4994</td>\n      <td>activity_09</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>21519</th>\n      <td>4301</td>\n      <td>4995</td>\n      <td>activity_10</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>21520</th>\n      <td>4302</td>\n      <td>4995</td>\n      <td>activity_06</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>21521 rows × 7 columns</p>\n</div>",
      "text/plain": "       action_index  trace_id   activities  owner  employed  age  income\n0                 0         0  activity_09      1         1    1       2\n1                 1         0  activity_04      1         1    1       2\n2                 2         0  activity_02      1         1    1       2\n3                 3         1  activity_09      1         1    3       2\n4                 4         1  activity_04      1         1    3       2\n...             ...       ...          ...    ...       ...  ...     ...\n21516          4298      4994  activity_07      1         1    2       1\n21517          4299      4994  activity_08      1         1    2       1\n21518          4300      4994  activity_09      1         1    2       1\n21519          4301      4995  activity_10      1         1    3       1\n21520          4302      4995  activity_06      1         1    3       1\n\n[21521 rows x 7 columns]"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.reset_index(inplace=True)\n",
    "df.rename({\"index\": \"action_index\"}, axis=1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# we need extra category for denoting that client activity has stopped\n",
    "activity_ended = f\"activity_{df.activities.nunique()+1:d}\"\n",
    "df.activities = df.activities.cat.add_categories([activity_ended])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- order of events for customer\r\n",
    "\r\n",
    "- next and previous event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "Index(['action_index', 'trace_id', 'activities', 'owner', 'employed', 'age',\n       'income'],\n      dtype='object')"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_next_df = pd.DataFrame(\n",
    "    columns={\n",
    "        \"client_id\": int,\n",
    "        \"action_index\": \"uint8\",\n",
    "        \"prev_activity\": \"category\",\n",
    "        \"owner\": \"uint8\",\n",
    "        \"employed\": \"uint8\",\n",
    "        \"age\": \"uint8\",\n",
    "        \"income\": \"uint8\",\n",
    "        \"next_activity\": \"category\",\n",
    "    }\n",
    ")\n",
    "\n",
    "for client_id, client_data in df.groupby(\"trace_id\"):\n",
    "    client_data.action_index -= client_data.action_index.min()\n",
    "    client_data.action_index = client_data.action_index.astype(int)\n",
    "    # rename columns\n",
    "    buf_df = client_data.rename(\n",
    "        {\"trace_id\": \"client_id\", \"activities\": \"prev_activity\"}, axis=1\n",
    "    )\n",
    "\n",
    "    # add new column for next activity\n",
    "    buf_df[\"next_activity\"] = client_data.activities.shift(\n",
    "        periods=-1, fill_value=activity_ended\n",
    "    ).astype(\"category\")\n",
    "    # add buffer to prev_next_df\n",
    "    prev_next_df = pd.concat((prev_next_df, buf_df), axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>client_id</th>\n      <th>action_index</th>\n      <th>prev_activity</th>\n      <th>owner</th>\n      <th>employed</th>\n      <th>age</th>\n      <th>income</th>\n      <th>next_activity</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>activity_09</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>activity_04</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>1</td>\n      <td>activity_04</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>activity_02</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>2</td>\n      <td>activity_02</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>activity_16</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>0</td>\n      <td>activity_09</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>2</td>\n      <td>activity_04</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>1</td>\n      <td>activity_04</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>2</td>\n      <td>activity_02</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>21516</th>\n      <td>4994</td>\n      <td>2</td>\n      <td>activity_07</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n      <td>activity_08</td>\n    </tr>\n    <tr>\n      <th>21517</th>\n      <td>4994</td>\n      <td>3</td>\n      <td>activity_08</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n      <td>activity_09</td>\n    </tr>\n    <tr>\n      <th>21518</th>\n      <td>4994</td>\n      <td>4</td>\n      <td>activity_09</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n      <td>activity_16</td>\n    </tr>\n    <tr>\n      <th>21519</th>\n      <td>4995</td>\n      <td>0</td>\n      <td>activity_10</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>1</td>\n      <td>activity_06</td>\n    </tr>\n    <tr>\n      <th>21520</th>\n      <td>4995</td>\n      <td>1</td>\n      <td>activity_06</td>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>1</td>\n      <td>activity_16</td>\n    </tr>\n  </tbody>\n</table>\n<p>21521 rows × 8 columns</p>\n</div>",
      "text/plain": "      client_id action_index prev_activity owner employed age income  \\\n0             0            0   activity_09     1        1   1      2   \n1             0            1   activity_04     1        1   1      2   \n2             0            2   activity_02     1        1   1      2   \n3             1            0   activity_09     1        1   3      2   \n4             1            1   activity_04     1        1   3      2   \n...         ...          ...           ...   ...      ...  ..    ...   \n21516      4994            2   activity_07     1        1   2      1   \n21517      4994            3   activity_08     1        1   2      1   \n21518      4994            4   activity_09     1        1   2      1   \n21519      4995            0   activity_10     1        1   3      1   \n21520      4995            1   activity_06     1        1   3      1   \n\n      next_activity  \n0       activity_04  \n1       activity_02  \n2       activity_16  \n3       activity_04  \n4       activity_02  \n...             ...  \n21516   activity_08  \n21517   activity_09  \n21518   activity_16  \n21519   activity_06  \n21520   activity_16  \n\n[21521 rows x 8 columns]"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev_next_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Int64Index([999, 1998, 2997, 3996], dtype='int64', name='client_id')\nInt64Index([ 4295,  4296,  4297,  4298,  4299,  4300,  4301,  4302,  8592,\n             8593,  8594,  8595,  8596,  8597,  8598,  8599,  8600,  8601,\n            12881, 12882, 12883, 12884, 12885, 12886, 12887, 12888, 17213,\n            17214, 17215, 17216, 17217, 17218, 17219],\n           dtype='int64')\n"
    }
   ],
   "source": [
    "# drop outliers\n",
    "grouped_by_action_count = prev_next_df.groupby(\"client_id\").max()[[\"action_index\"]]\n",
    "clients_to_be_dropped = (\n",
    "    grouped_by_action_count[grouped_by_action_count.action_index > 10].dropna().index\n",
    ")\n",
    "# clients_to_be_dropped.to_numpy()\n",
    "print(clients_to_be_dropped)\n",
    "outliers = (\n",
    "    prev_next_df.apply(\n",
    "        lambda row: 1 if row.client_id in clients_to_be_dropped else np.nan, axis=1\n",
    "    )\n",
    "    .dropna()\n",
    "    .index\n",
    ")\n",
    "print(outliers)\n",
    "prev_next_df.drop(outliers, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_next_df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "client_id        4992\naction_index       11\nprev_activity      15\nowner               1\nemployed            1\nage                 5\nincome              3\nnext_activity      16\ndtype: int64"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev_next_df.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop uninformative columns that only contain one value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_next_df.drop([\"owner\", \"employed\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_next_df[\"action_count\"] = prev_next_df.action_index + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA34AAAEHCAYAAADxm5Q0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoDUlEQVR4nO3df7xddX3n+9fbBMSqIyhOawENapw2WoV6jFhulSpqkE5iRzqGjgotnVQvCI61c0PrRUund9D6sL9kxFQzYkdFxR9zLFGGUbA6LZgjIBgwGpBKcumgQEGuFo187h97BTbHc3J2cs7a++x1Xs/HYz+yfnzX2p+998ln7c9e3/VdqSokSZIkSd31sFEHIEmSJElql4WfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHXc8lEHsFDWrFlTn/3sZ0cdhqSFl1EHMF/mJ6mTzE2SFqNZc1Nnzvh997vfHXUIkjQj85OkxcjcJC0tnSn8JEmSJEkzs/CTJEmSpI6z8JMkSZKkjrPwkyRJkqSOs/CTJEmSpI6z8JMkSZKkjmu18EuyJsn2JDuSbJxh/WuTXJ/k2iRfSrKqWb4iyQ+a5dcmuaDNOCVpj7nyVtPm3ya5Icm2JB8adoySlpYkByX5cpKvNnnnD2do8/AkH2ly11VJVowgVEmLWGs3cE+yDDgfeDGwE9iaZLKqbuhr9qGquqBpvxZ4J7CmWXdTVR3VVnySNN0geSvJSuBs4NiquivJvxxNtJKWkPuAF1bVvUkOAL6U5DNVdWVfm9OAu6rqqUnWA28DXjmKYCUtTm2e8VsN7Kiqm6vqh8BFwLr+BlV1T9/sI4FqMR5JmsuceQv498D5VXUXQFXdPuQYJS0x1XNvM3tA85j+nWkdcGEzfTHwoiQZUoiSxkCbhd9hwK198zubZQ+R5PQkNwFvB87sW3VkkmuSfCHJL8/0BEk2JJlKMrWQgUtasgbJW08DnpbkfyW5MskaZmB+krSQkixLci1wO3BZVV01rckD+auqdgN3A4+bYT/mJmmJGvngLlV1flU9Bfi/gDc3i28DnlhVRwNvBD6U5F/MsO2mqpqoqonhRSxpiVsOrASOA04G/irJwdMbmZ8kLaSq+nFzCczhwOokz9jP/ZibpCWqzcJvF3BE3/zhzbLZXAS8HKCq7quqO5rprwA30fuVXZLaNEje2glMVtWPqupbwDfoFYKS1Lqq+ifgch4cE2GPB/JXkuXAY4A7hhqcpEWtzcJvK7AyyZFJDgTWA5P9DZpBEvY4Efhms/zxzSALJHkyvS9VN7cYqyTBAHkL+BS9s30kOZTej1LmJ0mtab4XHdxMP4LeAFRfn9ZsEjilmT4J+HxVOXaCpAe0NqpnVe1OcgZwKbAM2FxV25KcC0xV1SRwRpLjgR8Bd/Fgwno+cG6SHwH3A6+tqjvbilWSYOC8dSnwkiQ3AD8Gfm9PDwVJaskTgAubH8UfBny0qv5mWm56H/DXSXYAd9L74UqSHpCu/Bg0MTFRU1Nepyx10NiPSmd+kjrJ3CRpMZo1N418cBdJkiRJUrss/CRJkiSp4yz8JEmSJKnjLPwkSZIkqeMs/CRJkiSp4yz8JEmSJKnjLPwkSZIkqeMs/CRJkiSp4yz8JEmSJKnjLPwkSZIkqeMs/CRJkiSp4yz8JEmSJKnjLPwkSZIkqeMs/CRJkiSp45aPOgANZsXGS1rb9y3nndjaviVJkiSNnmf8JEmSJKnjLPwkSZIkqeMs/CRJkiSp4yz8JEmSJKnjLPwkSZIkqeNaLfySrEmyPcmOJBtnWP/aJNcnuTbJl5Ks6lt3drPd9iQvbTNOSZIkSeqy1m7nkGQZcD7wYmAnsDXJZFXd0NfsQ1V1QdN+LfBOYE1TAK4Hng78LPA/kzytqn7cVrySpP3T1u1mvNWMJEkLp837+K0GdlTVzQBJLgLWAQ8UflV1T1/7RwLVTK8DLqqq+4BvJdnR7O/vW4x3UfMLkCRJkqT91WZXz8OAW/vmdzbLHiLJ6UluAt4OnLmP225IMpVkasGiliRJkqSOafOM30Cq6nzg/CS/AbwZOGUftt0EbAKYmJioOZqPtba6UoFnEyVJkqSua/OM3y7giL75w5tls7kIePl+bitJC2KAQalOTfKdZlCqa5P89ijilLR0JDkiyeVJbkiyLclZM7Q5LsndfbnpnFHEKmnxavOM31ZgZZIj6RVt64Hf6G+QZGVVfbOZPRHYMz0JfCjJO+kN7rIS+HKLsUrSoINSAXykqs5oIwYHShlvfn5qyW7gd6vq6iSPBr6S5LIZctMXq+pXRxCfpDHQWuFXVbuTnAFcCiwDNlfVtiTnAlNVNQmckeR44EfAXTTdPJt2H6U3EMxu4HRH9JQ0BHMOStW2cfyCP44xt8X3Qm2oqtuA25rp7yW5kd7YB0PLTZLGX6v38auqLVX1tKp6SlX9cbPsnKboo6rOqqqnV9VRVfUrVbWtb9s/brb7V1X1mTbjlKTGQANLAa9Icl2Si5McMcN6B5+S1IokK4CjgatmWP28JF9N8pkkT59le3OTtESNfHAXSRoznwY+XFX3Jfkd4ELghdMb7e/gU+PYVXAcY26L74XalORRwMeBN0y7JRbA1cCTqureJC8DPkXvUpmHWEoD40l6qFbP+EnSmJlzYKmquqO5xyjAe4FnDyk2SUtYkgPoFX0frKpPTF9fVfdU1b3N9BbggCSHDjlMSYuYhZ8kPeiBQamSHEhvUKrJ/gZJntA3uxa4cYjxSVqCkgR4H3BjVb1zljY/07QjyWp63/HuGF6UkhY7u3pKUmPAQanOTLKW3sBTdwKnjixgSUvFscCrgeuTXNss+33giQBVdQFwEvC6JLuBHwDrq8qunJIeYOEnSX2aLlJbpi07p2/6bODsYcclqaet6yhh8V5LWVVfAjJHm3cB7xpORJJmstiv87arpyRJkiR1nIWfJEmSJHWcXT0lSVrCFnvXJEnSwvCMnyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1XKuFX5I1SbYn2ZFk4wzr35jkhiTXJflckif1rftxkmubx2SbcUqSJElSly1va8dJlgHnAy8GdgJbk0xW1Q19za4BJqrq+0leB7wdeGWz7gdVdVRb8al9KzZe0tq+bznvxNb2LUmSJHVNa4UfsBrYUVU3AyS5CFgHPFD4VdXlfe2vBF7VYjxjbRwLnXGMWZIkSeqiNrt6Hgbc2je/s1k2m9OAz/TNH5RkKsmVSV4+0wZJNjRtpuYdrSRJkiR1VJtn/AaW5FXABPCCvsVPqqpdSZ4MfD7J9VV1U/92VbUJ2AQwMTFRQwt4BMax2+Q4xixJkiR1UZtn/HYBR/TNH94se4gkxwN/AKytqvv2LK+qXc2/NwNXAEe3GKskSZIkdVabhd9WYGWSI5McCKwHHjI6Z5KjgffQK/pu71t+SJKHN9OHAsfSd22gJEmSJGlwrXX1rKrdSc4ALgWWAZuraluSc4GpqpoE/gR4FPCxJADfrqq1wM8D70lyP73i9Lxpo4FKkiRJkgbU6jV+VbUF2DJt2Tl908fPst3fAb/QZmySJEmStFS0egN3SZIkSdLoWfhJUp8ka5JsT7Ijyca9tHtFkkoyMcz4JC09SY5IcnmSG5JsS3LWDG2S5C+a3HVdkl8cRaySFi8LP0lqJFkGnA+cAKwCTk6yaoZ2jwbOAq4aboSSlqjdwO9W1SrgGOD0GXLTCcDK5rEBePdwQ5S02C2K+/hJ0iKxGtjR3EaGJBcB6/jJUYX/CHgb8HsLHcA43qNyHGNuyzi+F+MW87jFuxCq6jbgtmb6e0luBA7joblpHfCBqirgyiQHJ3lCs62kIVjs+ckzfpL0oMOAW/vmdzbLHtB0nzqiqi7Z246SbEgylWRq4cOUtFQlWUHv3sbTexzMmb+a7c1N0hLlGT9JGlCShwHvBE6dq21VbQI2AUxMTNSgz7Fi417ryf3W5q+Q4xhzW8bxvRi3mNuKFxb/31ySRwEfB95QVffszz72NzdJmttiz6ee8ZOkB+0CjuibP7xZtsejgWcAVyS5hd61NpMO8CKpbUkOoFf0fbCqPjFDk7nyl6QlzsJPkh60FViZ5MgkBwLrgck9K6vq7qo6tKpWVNUK4EpgbVXZZUpSa5IEeB9wY1W9c5Zmk8BrmtE9jwHu9vo+Sf3s6ilJjaraneQM4FJgGbC5qrYlOReYqqrJve9BklpxLPBq4Pok1zbLfh94IkBVXQBsAV4G7AC+D/zm8MOUtJhZ+ElSn6raQu8LVP+yc2Zpe9wwYpK0tFXVl4DM0aaA04cTkaRxZFdPSZIkSeo4Cz9JkiRJ6jgLP0mSJEnqOAs/SZIkSeo4Cz9JkiRJ6jgLP0mSJEnqOAs/SZIkSeo4Cz9JkiRJ6jgLP0mSJEnqOAs/SZIkSeq45W3uPMka4M+BZcB7q+q8aevfCPw2sBv4DvBbVfUPzbpTgDc3Tf9TVV3YZqxL2YqNl4w6BEmSJEktau2MX5JlwPnACcAq4OQkq6Y1uwaYqKpnAhcDb2+2fSzwFuC5wGrgLUkOaStWSZIkSeqyNrt6rgZ2VNXNVfVD4CJgXX+Dqrq8qr7fzF4JHN5MvxS4rKrurKq7gMuANS3GKkmSJEmd1WZXz8OAW/vmd9I7gzeb04DP7GXbwxY0ujFzy3knjjoESZIkSWNqUQzukuRVwATwJ/u43YYkU0mm2olMkiRJksZfm2f8dgFH9M0f3ix7iCTHA38AvKCq7uvb9rhp214xfduq2gRsApiYmKiFCHqxcgCWh/IMqCRJkjS4Ns/4bQVWJjkyyYHAemCyv0GSo4H3AGur6va+VZcCL0lySDOoy0uaZZIkSZKkfdTaGb+q2p3kDHoF2zJgc1VtS3IuMFVVk/S6dj4K+FgSgG9X1dqqujPJH9ErHgHOrao724pVkiRJkrqs1fv4VdUWYMu0Zef0TR+/l203A5vbi06SJEmSloZFMbiLJEmSJKk9Fn6SJEmS1HH7VPglOSbJZ5NckeTlLcUkSQvCnCVpsTI/SRq2vV7jl+Rnquof+xa9Efg1IMBVwKfaC02S9o05S9JiZX6SNGpzDe5yQZKrgbdX1T8D/wScBNwP3NNybJK0r8xZkhYr85OkkdprV8+qejlwDfA3SV4DvAF4OPA44OUtxyZJ+8ScJWmxMj9JGrU5r/Grqk8DLwUeA3wS+EZV/UVVfaft4CRpX803ZyVZk2R7kh1JNs6w/rVJrk9ybZIvJVm1sK9AUlftb35KsjnJ7Um+Nsv645Lc3eSla5OcM1M7SUvbXgu/JGuTXA58Fvga8EpgXZKLkjxlGAFK0qDmm7OSLAPOB04AVgEnz1DYfaiqfqGqjgLeDrxzIV+DpG6aZ356P7BmjjZfrKqjmse5849YUtfMdY3ffwJWA48ALq2q1cDvJlkJ/DGwvuX4JGlfzDdnrQZ2VNXNAEkuAtYBN+xpUFX91+I8EqiFCx9uOe/EhdzdUIxjzG0Zx/di3GIet3j77Hd+qqq/TbJiKFFK2m+LPT/N1dXzbuDfAK8Abt+zsKq+WVUWfZIWm/nmrMOAW/vmdzbLHiLJ6UluonfG78yZdpRkQ5KpJFP7EL+k7mr7O9Xzknw1yWeSPH22RuYmaema64zfrwEnAz8CfqP9cCRpXoaSs6rqfOD8JL8BvBk4ZYY2m4BNABMTEwOfFVyx8ZKFCvMh2vwVchxjbss4vhfjFnNb8ULrf3Nt5qergSdV1b1JXkbv1hArZ2q4v7lJ0twWez7da+FXVd8F/nJBnkmSWrYAOWsXcETf/OHNstlcBLx7Hs8naYlo8ztVfxf0qtqS5L8kObR5TkkCBhjVU5KWkK3AyiRHJjmQ3jU3k/0Nmutx9jgR+OYQ45Okn5DkZ5KkmV5N7/vdHaONStJiM1dXT0laMqpqd5IzgEuBZcDmqtqW5FxgqqomgTOSHE+vu9ZdzNDNU5IWUpIPA8cBhybZCbwFOACgqi6gdyP41yXZDfwAWF9VduOU9BAWfpLUp6q2AFumLTunb/qsoQclaUmrqpPnWP8u4F1DCkfSmLKrpyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHWchZ8kSZIkdVyrhV+SNUm2J9mRZOMM65+f5Ooku5OcNG3dj5Nc2zwmp28rSZIkSRpMa/fxS7IMOB94MbAT2Jpksqpu6Gv2beBU4E0z7OIHVXVUW/FJkiRJ0lLR5g3cVwM7qupmgCQXAeuABwq/qrqlWXd/i3F0wi3nnTjqECRJkjSGVmy8pLV9+x11fLTZ1fMw4Na++Z3NskEdlGQqyZVJXj5TgyQbmjZT84hTkiRJkjqtzTN+8/WkqtqV5MnA55NcX1U39Teoqk3AJoCJiYkaRZDD0uYvNePIX5ckSZKkwbVZ+O0CjuibP7xZNpCq2tX8e3OSK4CjgZv2upEkSZIWnF0FpfHXZlfPrcDKJEcmORBYDww0OmeSQ5I8vJk+FDiWvmsDJUmSJEmDa63wq6rdwBnApcCNwEeraluSc5OsBUjynCQ7gV8H3pNkW7P5zwNTSb4KXA6cN200UEmSJEnSgFq9xq+qtgBbpi07p296K70uoNO3+zvgF9qMTZIkSZKWilZv4C5JkiRJGj0LP0mSJEnqOAs/SZIkSeo4Cz9JkiRJ6jgLP0mSJEnqOAs/SZIkSeo4Cz9JkiRJ6jgLP0mSJEnqOAs/SZIkSeo4Cz9JkiRJ6jgLP0mSJEnqOAs/SeqTZE2S7Ul2JNk4w/o3JrkhyXVJPpfkSaOIU9LSkmRzktuTfG2W9UnyF03uui7JLw47RkmLm4WfJDWSLAPOB04AVgEnJ1k1rdk1wERVPRO4GHj7cKOUtES9H1izl/UnACubxwbg3UOISdIYSVWNOoYFMTExUVNTU6MOozUrNl4y6hAWlVvOO3HUIWh4MrQnSp4HvLWqXtrMnw1QVf95lvZHA++qqmP3tt+u5ydpiRpabnrgCZMVwN9U1TNmWPce4Iqq+nAzvx04rqpum21/5iapk2bNTZ7xk6QHHQbc2je/s1k2m9OAz8y0IsmGJFNJ/FYlaRgGyl/mJmnpWj7qACRpHCV5FTABvGCm9VW1CdgEvV/VB91vW2f32zxLPo4xt2Uc34txi7nNHjDj+De3rxZbboKl8b6Pmp/fcCz2fGrhJ0kP2gUc0Td/eLPsIZIcD/wB8IKqum9IsUnS3gyUvyQtXXb1lKQHbQVWJjkyyYHAemCyv0FzXd97gLVVdfsIYpSkmUwCr2lG9zwGuHtv1/dJWno84ydJjaraneQM4FJgGbC5qrYlOReYqqpJ4E+ARwEfSwLw7apaO7KgJS0JST4MHAccmmQn8BbgAICqugDYArwM2AF8H/jN0UQqabGy8JOkPlW1hd4XqP5l5/RNHz/0oCQteVV18hzrCzh9SOFIGkOtdvUc4EbIz09ydZLdSU6atu6UJN9sHqe0GackSZIkdVlrhd+AN0L+NnAq8KFp2z6WXheG5wKrgbckOaStWCVJkiSpy9o847ca2FFVN1fVD4GLgHX9Darqlqq6Drh/2rYvBS6rqjur6i7gMmBNi7FKkiRJUme1eY3fTDcSfe48tt3bTZQ7z3ukSJIkSdpfY307hyQbkkwlmRp1LJIkSZK0WLV5xm8+NxLdRW/I4v5tr5jeqKo2AZsAJiYman+CHBcrNl4y6hAWFc+ASpIkSYNr84zfnDdC3otLgZckOaQZ1OUlzTJJkiRJ0j5qrfCrqt3Anhsh3wh8dM+NkJOsBUjynOYmpL8OvCfJtmbbO4E/olc8bgXObZZJkiRJkvZRqzdwH+BGyFvpdeOcadvNwOY245MkSZKkpWCsB3eRJEmSJM3Nwk+SJEmSOs7CT5IkSZI6zsJPkiRJkjrOwk+SJEmSOs7CT5IkSZI6zsJPkiRJkjrOwk+SJEmSOs7CT5IkSZI6zsJPkiRJkjpu+agDkJaCFRsvaW3ft5x3Ymv7liRJUjd4xk+SJEmSOs7CT5IkSZI6zsJPkiRJkjrOwk+SJEmSOs7CT5IkSZI6zsJPkvokWZNke5IdSTbOsP75Sa5OsjvJSaOIUdLSM0BuOjXJd5Jc2zx+exRxSlq8vJ2DJDWSLAPOB14M7AS2Jpmsqhv6mn0bOBV40/AjlLQUDZibAD5SVWcMPUBJY8HCT5IetBrYUVU3AyS5CFgHPPDlqqpuadbd30YA43hfxnGMuS3j+F6MW8zjFu8CmTM3tW2Jvu+d4ec3HIv9fbarpyQ96DDg1r75nc2yfZZkQ5KpJFMLEpmkpWzQ3PSKJNcluTjJETPtyNwkLV2e8ZOkFlTVJmATwMTERA263YqNl7QST5u/Qo5jzG0Zx/di3GJuK14Yz7+5Pp8GPlxV9yX5HeBC4IXTGy223ARj/76PBT+/4Vjs+bTVM34DXIj88CQfadZflWRFs3xFkh/0XaB8QZtxSlJjF9D/K/nhzTJJGqU5c1NV3VFV9zWz7wWePaTYJI2J1gq/vguRTwBWAScnWTWt2WnAXVX1VOBPgbf1rbupqo5qHq9tK05J6rMVWJnkyCQHAuuByRHHJElz5qYkT+ibXQvcOMT4JI2BNrt6DnIh8jrgrc30xcC7kqTFmMaWp9HHm5/feKiq3UnOAC4FlgGbq2pbknOBqaqaTPIc4JPAIcC/TvKHVfX0EYYtqeMGyU3AmUnWAruBO+mNPixJD2iz8JvpQuTnztamSWp3A49r1h2Z5BrgHuDNVfXF6U+QZAOwAeDZz7ZHg6T5q6otwJZpy87pm95Kr5uVJA3NALnpbODsYcclaXws1sFdbgOeWFV3JHk28KkkT6+qe/ob7e8Fym1q8+JZPWjczqB5UbUkSZJGqc3Cb5BBEva02ZlkOfAY4I6qKuA+gKr6SpKbgKcBDj0sYPGPmiRJkiQtJm2O6jnIIAmTwCnN9EnA56uqkjy+GRyGJE8GVgI3txirJEmSJHVWa2f8BrwQ+X3AXyfZQe9C5PXN5s8Hzk3yI+B+4LVVdWdbsUqSJElSl7V6jd8AFyL/M/DrM2z3ceDjbcYmSZIkSUtFqzdwlyRJkiSNnoWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR1nIWfJEmSJHXc8lEHIC0mKzZeMuoQJEmSpAW3ZAs/v+BLkiRJWirs6ilJkiRJHWfhJ0mSJEkdZ+EnSZIkSR1n4SdJkiRJHWfhJ0mSJEkdt2RH9ZS6oq0Ram8578RW9itJkqTh84yfJEmSJHWchZ8kSZIkdVyrhV+SNUm2J9mRZOMM6x+e5CPN+quSrOhbd3azfHuSl7YZpyTtMZ+8JUltMTdJmq/WrvFLsgw4H3gxsBPYmmSyqm7oa3YacFdVPTXJeuBtwCuTrALWA08Hfhb4n0meVlU/biteSQ+1FK8dnE/eGn60kpYKc5OkhZCqamfHyfOAt1bVS5v5swGq6j/3tbm0afP3SZYD/wg8HtjY37a/3WzPNzExUVNTU628FkkjlaE90TzyVu0lmZqfpE4yN0lajGbNTW2O6nkYcGvf/E7gubO1qardSe4GHtcsv3LatodNf4IkG4ANzey9SbYvTOjzcijw3VEHsY+MuX3jFi8snpg/W1VrhvRc88lbD3mv5pGfFsv73hZf33jz9T3I3NQtvr7x5ut70Ky5aaxv51BVm4BNo46jX5KpqpoYdRz7wpjbN27xwnjGvJjsb37q+vvu6xtvvr7xZ26ama9vvPn6BtPm4C67gCP65g9vls3YpumW8BjgjgG3laSFNp+8JUltMTdJmrc2C7+twMokRyY5kN5gLZPT2kwCpzTTJwGfb/qiTwLrmxGqjgRWAl9uMVZJgvnlLUlqi7lJ0ry11tWz6V9+BnApsAzYXFXbkpwLTFXVJPA+4K+T7ADupJfIaNp9FLgB2A2cPkYjei6qrqcDMub2jVu8MJ4xz8t88tYC6vr77usbb76+ETA3DYWvb7z5+gbQ2qiekiRJkqTFodUbuEuSJEmSRs/CT5IkSZI6zsJvgSQ5IsnlSW5Isi3JWaOOaRBJliW5JsnfjDqWQSQ5OMnFSb6e5MbmpraLWpL/0PxNfC3Jh5McNOqYpkuyOcntSb7Wt+yxSS5L8s3m30NGGeNSkGRNku1JdiTZOOp4FtJMf2NdMq7HgEElOSjJl5N8tXl9fzjqmBbauB0P2zJXHmoG3vtIs/6qJCtGEOZ+G+D1nZrkO0mubR6/PYo499dcuTY9f9G8/uuS/OKwY9xfA7y245Lc3ffZnTPsGOdjkOPIfD8/C7+Fsxv43apaBRwDnJ5k1YhjGsRZwI2jDmIf/Dm9G1P+HPAsFnnsSQ4DzgQmquoZ9C7KX+gL7hfC+4HpN/vcCHyuqlYCn2vm1ZIky4DzgROAVcDJY5JDBvV+fvJvrEvG9RgwqPuAF1bVs4CjgDVJjhltSAtu3I6HC27APHQacFdVPRX4U+Btw41y/+1Dnv1IVR3VPN471CDn7/3sPdeeQG+0/JXABuDdQ4hpobyfuY8jX+z77M4dQkwLaZDjyLw+Pwu/BVJVt1XV1c309+gdPA4bbVR7l+Rw4ERgLJJakscAz6c3chlV9cOq+qeRBjWY5cAj0ruv0k8B/++I4/kJVfW39EaB67cOuLCZvhB4+TBjWoJWAzuq6uaq+iFwEb3PoBNm+RvrjHE8BuyL6rm3mT2geXRmdLhxOx62aJA81H9suBh4UZIMMcb56HSehYFy7TrgA83/6SuBg5M8YTjRzY/HEWCen5+FXwuabg9HA1eNOJS5/BnwH4H7RxzHoI4EvgP816Y7znuTPHLUQe1NVe0C3gF8G7gNuLuq/sdooxrYT1fVbc30PwI/PcpgloDDgFv75nfSocJhKRmjY8A+abpCXgvcDlxWVV16fX/GeB0P2zJIHnqgTVXtBu4GHjeU6OZv0Dz7iqYb3cVJjhhOaEPT9WPN85ou6Z9J8vRRB7O/9nIcmdfnZ+G3wJI8Cvg48IaqumfU8cwmya8Ct1fVV0Ydyz5YDvwi8O6qOhr4/1jk3Q+b6+LW0StafxZ4ZJJXjTaqfdfcBLgzv+5LbRmXY8D+qKofV9VRwOHA6iTPGHFIC2JMj4dqz6eBFVX1TOAyHjy7qcXvauBJTZf0vwQ+Ndpw9k+bxxELvwWU5AB6H9QHq+oTo45nDscCa5PcQq+rwwuT/LfRhjSnncDOvl+ZL6ZXCC5mxwPfqqrvVNWPgE8AvzTimAb1v/d0H2j+vX3E8XTdLqD/l+XDm2UaE2N2DNhvTRf7y+nONZvjeDxsyyB56IE2zSUMjwHuGEp08zfn66uqO6rqvmb2vcCzhxTbsHT2WFNV9+zpkl5VW4ADkhw64rD2yQDHkXl9fhZ+C6Tp3/4+4Maqeueo45lLVZ1dVYdX1Qp6g418vqoW9ZmoqvpH4NYk/6pZ9CLghhGGNIhvA8ck+anmb+RFjM/gAZPAKc30KcB/H2EsS8FWYGWSI5McSO//5eSIY9KAxu0YsK+SPD7Jwc30I4AXA18faVALZByPhy0aJA/1HxtOovd+jUuPkDlf37TrpdYyPsfsQU0Cr2lGhzyG3iUot8210ThI8jN7rjdNsppenTMuP0oMehyZ1+e3fAHiVM+xwKuB65trIAB+v/nFQQvn9cAHm4R9M/CbI45nr6rqqiQX0+t+sBu4Btg02qh+UpIPA8cBhybZCbwFOA/4aJLTgH8A/u3oIuy+qtqd5AzgUnqjv26uqm0jDmvBzPQ3VlXvG21UC6rrx4AnABc2oyI+DPhoVS3p2x500Wx5KMm5wFRVTdL7YvrXSXbQG2hjMY5UPaMBX9+ZSdbSO2bfCZw6soD3wyzH8wMAquoCYAvwMmAH8H0W+feofgO8tpOA1yXZDfwAWD9GP0rALMcR4ImwMJ9fxuv9kCRJkiTtK7t6SpIkSVLHWfhJkiRJUsdZ+EmSJElSx1n4SZIkSVLHWfhJkiRJUsdZ+EmSJEmLVJK/G3UM6gYLP7UuyXFJfqlv/rVJXjPKmPZFkoOT/J+jjkOSJC09VfVLc7eS5mbhp2E4DnggaVXVBVX1gdGFs88OBiz8JJHkU0m+kmRbkg3NstOSfCPJl5P8VZJ3Ncsfn+TjSbY2j2NHG72kcZTk3ubf45JckeTiJF9P8sEkadY9J8nfJflqk4seneSgJP81yfVJrknyK03bU5tcdlmSW5KckeSNTZsrkzy2afeUJJ9tct4Xk/zc6N4FLQQLP+23Wb4ArUlydZN4PpdkBfBa4D8kuTbJLyd5a5I3Ne2PapLMdUk+meSQZvkVSd7WJK9vJPnlvcSxLMk7knyt2c/rm+UvapLY9Uk2J3l4s/yWJIc20xNJrmim39q0uyLJzUnObJ7iPOApTfx/0sZ7KWls/FZVPRuYAM5MchjwfwPHAMcC/V+M/hz406p6DvAK4L3DDlZS5xwNvAFYBTwZODbJgcBHgLOq6lnA8cAPgNOBqqpfAE4GLkxyULOfZwD/BngO8MfA96vqaODvgT29sjYBr29y3puA/9L+y1Oblo86AI2136qqO5M8Atia5L8DfwU8v6q+leSxzfoLgHur6h3QK8j69vEBeknlC0nOBd5CL6EBLK+q1Ule1iw/fpY4NgArgKOqaneSxzaJ7f3Ai6rqG0k+ALwO+LM5XtPPAb8CPBrYnuTdwEbgGVV11IDvi6TuOjPJrzXTRwCvBr5QVXcCJPkY8LRm/fHAquYHeYB/keRRVXXvMAOW1ClfrqqdAEmupff9527gtqraClBV9zTr/w/gL5tlX0/yDzyYny6vqu8B30tyN/DpZvn1wDOTPIpeb62P9eWwh7f70tQ2Cz/Nx/QvQBuAv62qbwHs+SI0mySPAQ6uqi80iy4EPtbX5BPNv1+hl9hmczxwQVXt3vO8SZ4FfKuqvtG379OZu/C7pKruA+5Lcjvw03O0l7REJDmOXr55XlV9v+kt8HXg52fZ5GHAMVX1z0MJUNJScF/f9I/Z/+/y/fu5v2/+/mafDwP+yR+9u8Wuntov074APQu4Brh2gZ9mTxKaT2KbyW4e/Ns/aNq6hUqokrrnMcBdTdH3c/S6dz4SeEGSQ5Isp9elc4//Abx+z0ySo4YZrKQlYzvwhCTPAWiu71sOfBH4d82ypwFPbNrOqTlr+K0kv95sn+ZHdY0xCz/tr5m+AB0EPD/JkQB7Lg4Gvkev6+RDVNXdwF191++9GvjC9HYDuAz4nSbJ7Xne7cCKJE+dYd+3AM9upvu/pM1mxvglLTmfBZYnuZHetb9XAruA/wf4MvC/6OWXu5v2ZwITzbXHN9C73lmSFlRV/RB4JfCXSb5K73vRQfSuyXtYkuvpXQN4atOraVD/Djit2ec2YN3CRq5hS1WNOgaNoWaglE/R64K5nd7Il28FHkHvS9DDgNur6sXNr0wX0+s+8HrgRTTX/DW/gF8A/BRwM/CbVXVX04XqTVU11QzEMlVVK2aJZTnwdmAN8CPgr6rqXc21hO+gd9ZuK/C6qrqvKTTfB9wDXAFMVNVxSd7KQ69F/Brwq1V1S5IPAc8EPlNVvzfvN1BSZ+y5bq/JRZ8ENlfVJ0cdlyRJ/Sz8JEmahyTvoNf1/SB63TvPKg+ukqRFxsJPkiRJkjrOgSs0NpK8FHjbtMXfqqpfm6m9JEmSpB7P+EmSJElSxzmqpyRJkiR1nIWfJEmSJHWchZ8kSZIkdZyFnyRJkiR13P8P0zm3MVq8t6MAAAAASUVORK5CYII=\n",
      "text/plain": "<Figure size 1080x288 with 3 Axes>"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axs = plt.subplots(1, 3, figsize=(15, 4))\n",
    "\n",
    "for ax, column in zip(axs, [\"action_count\", \"age\", \"income\"]):\n",
    "    prev_next_df.groupby(\"client_id\").max()[column].hist(ax=ax, density=True)\n",
    "    ax.set_ylabel(\"%\")\n",
    "    ax.set_xlabel(column)\n",
    "    ax.spines[\"top\"].set_visible(False)\n",
    "    ax.spines[\"right\"].set_visible(False)\n",
    "    ax.grid(False)\n",
    "    [ax.axhline(tick, color=\"white\", linewidth=2) for tick in ax.get_yticks()]\n",
    "\n",
    "plt.savefig(\"results/client_histograms.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok, so most clients have from 3 to 5 actions (including end of journey). considering background information, all groups are well represented."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# convert state feature into one hot\n",
    "onehot = pd.get_dummies(prev_next_df.prev_activity, prefix=\"prev\")\n",
    "prev_next_df[onehot.columns] = onehot\n",
    "prev_next_df.drop(\"prev_activity\", axis=1, inplace=True)\n",
    "\n",
    "# convert label categories into numerical format\n",
    "# (this is because for the moment TFF does not support multi-output models)\n",
    "prev_next_df[[\"next_activity\"]] = prev_next_df[[\"next_activity\"]].apply(\n",
    "    lambda x: x.cat.codes\n",
    ")\n",
    "prev_next_df = prev_next_df.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "client_id           int64\naction_index        int64\nage                 int64\nincome              int64\nnext_activity       int64\naction_count        int64\nprev_activity_01    int64\nprev_activity_02    int64\nprev_activity_03    int64\nprev_activity_04    int64\nprev_activity_05    int64\nprev_activity_06    int64\nprev_activity_07    int64\nprev_activity_08    int64\nprev_activity_09    int64\nprev_activity_10    int64\nprev_noise_0        int64\nprev_noise_1        int64\nprev_noise_2        int64\nprev_noise_3        int64\nprev_noise_4        int64\nprev_activity_16    int64\ndtype: object"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev_next_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>client_id</th>\n      <th>action_index</th>\n      <th>age</th>\n      <th>income</th>\n      <th>next_activity</th>\n      <th>action_count</th>\n      <th>prev_activity_01</th>\n      <th>prev_activity_02</th>\n      <th>prev_activity_03</th>\n      <th>prev_activity_04</th>\n      <th>...</th>\n      <th>prev_activity_07</th>\n      <th>prev_activity_08</th>\n      <th>prev_activity_09</th>\n      <th>prev_activity_10</th>\n      <th>prev_noise_0</th>\n      <th>prev_noise_1</th>\n      <th>prev_noise_2</th>\n      <th>prev_noise_3</th>\n      <th>prev_noise_4</th>\n      <th>prev_activity_16</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>2</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n      <td>2</td>\n      <td>15</td>\n      <td>3</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>0</td>\n      <td>3</td>\n      <td>2</td>\n      <td>3</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>1</td>\n      <td>3</td>\n      <td>2</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>21516</th>\n      <td>4994</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>7</td>\n      <td>3</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>21517</th>\n      <td>4994</td>\n      <td>3</td>\n      <td>2</td>\n      <td>1</td>\n      <td>8</td>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>21518</th>\n      <td>4994</td>\n      <td>4</td>\n      <td>2</td>\n      <td>1</td>\n      <td>15</td>\n      <td>5</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>21519</th>\n      <td>4995</td>\n      <td>0</td>\n      <td>3</td>\n      <td>1</td>\n      <td>5</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>21520</th>\n      <td>4995</td>\n      <td>1</td>\n      <td>3</td>\n      <td>1</td>\n      <td>15</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>21488 rows × 22 columns</p>\n</div>",
      "text/plain": "       client_id  action_index  age  income  next_activity  action_count  \\\n0              0             0    1       2              3             1   \n1              0             1    1       2              1             2   \n2              0             2    1       2             15             3   \n3              1             0    3       2              3             1   \n4              1             1    3       2              1             2   \n...          ...           ...  ...     ...            ...           ...   \n21516       4994             2    2       1              7             3   \n21517       4994             3    2       1              8             4   \n21518       4994             4    2       1             15             5   \n21519       4995             0    3       1              5             1   \n21520       4995             1    3       1             15             2   \n\n       prev_activity_01  prev_activity_02  prev_activity_03  prev_activity_04  \\\n0                     0                 0                 0                 0   \n1                     0                 0                 0                 1   \n2                     0                 1                 0                 0   \n3                     0                 0                 0                 0   \n4                     0                 0                 0                 1   \n...                 ...               ...               ...               ...   \n21516                 0                 0                 0                 0   \n21517                 0                 0                 0                 0   \n21518                 0                 0                 0                 0   \n21519                 0                 0                 0                 0   \n21520                 0                 0                 0                 0   \n\n       ...  prev_activity_07  prev_activity_08  prev_activity_09  \\\n0      ...                 0                 0                 1   \n1      ...                 0                 0                 0   \n2      ...                 0                 0                 0   \n3      ...                 0                 0                 1   \n4      ...                 0                 0                 0   \n...    ...               ...               ...               ...   \n21516  ...                 1                 0                 0   \n21517  ...                 0                 1                 0   \n21518  ...                 0                 0                 1   \n21519  ...                 0                 0                 0   \n21520  ...                 0                 0                 0   \n\n       prev_activity_10  prev_noise_0  prev_noise_1  prev_noise_2  \\\n0                     0             0             0             0   \n1                     0             0             0             0   \n2                     0             0             0             0   \n3                     0             0             0             0   \n4                     0             0             0             0   \n...                 ...           ...           ...           ...   \n21516                 0             0             0             0   \n21517                 0             0             0             0   \n21518                 0             0             0             0   \n21519                 1             0             0             0   \n21520                 0             0             0             0   \n\n       prev_noise_3  prev_noise_4  prev_activity_16  \n0                 0             0                 0  \n1                 0             0                 0  \n2                 0             0                 0  \n3                 0             0                 0  \n4                 0             0                 0  \n...             ...           ...               ...  \n21516             0             0                 0  \n21517             0             0                 0  \n21518             0             0                 0  \n21519             0             0                 0  \n21520             0             0                 0  \n\n[21488 rows x 22 columns]"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev_next_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>client_id</th>\n      <th>x</th>\n      <th>y</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>[0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>[1, 1, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>[2, 1, 2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>[0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>[1, 3, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>21516</th>\n      <td>4994</td>\n      <td>[2, 2, 1, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, ...</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>21517</th>\n      <td>4994</td>\n      <td>[3, 2, 1, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>21518</th>\n      <td>4994</td>\n      <td>[4, 2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n      <td>15</td>\n    </tr>\n    <tr>\n      <th>21519</th>\n      <td>4995</td>\n      <td>[0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>21520</th>\n      <td>4995</td>\n      <td>[1, 3, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, ...</td>\n      <td>15</td>\n    </tr>\n  </tbody>\n</table>\n<p>21488 rows × 3 columns</p>\n</div>",
      "text/plain": "       client_id                                                  x   y\n0              0  [0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...   3\n1              0  [1, 1, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...   1\n2              0  [2, 1, 2, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  15\n3              1  [0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...   3\n4              1  [1, 3, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, ...   1\n...          ...                                                ...  ..\n21516       4994  [2, 2, 1, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, ...   7\n21517       4994  [3, 2, 1, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...   8\n21518       4994  [4, 2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...  15\n21519       4995  [0, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...   5\n21520       4995  [1, 3, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, ...  15\n\n[21488 rows x 3 columns]"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "# We need to convert the data into untidy nested format for TFF\n",
    "# so that x is a vector and y is a scalar\n",
    "cxy_df = pd.DataFrame(columns=[\"client_id\", \"x\", \"y\"])\n",
    "cxy_df.client_id = prev_next_df.client_id\n",
    "cxy_df.x = prev_next_df[\n",
    "    prev_next_df.drop([\"client_id\", \"next_activity\"], axis=1).columns\n",
    "].apply(lambda row: row.to_numpy(), axis=1)\n",
    "cxy_df.y = prev_next_df.next_activity\n",
    "\n",
    "cxy_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save data for further use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feather.write_feather(cxy_df, \"data/preprocessed_data/data.f\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create function to convert df into tff ClientData\r\n",
    "\r\n",
    "Following this discussion: https://stackoverflow.com/questions/58965488/how-to-create-federated-dataset-from-a-csv-file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export\n",
    "\n",
    "import collections\n",
    "\n",
    "\n",
    "def create_tff_client_data_from_df(\n",
    "    df,\n",
    "    client_id_col=\"client_id\",\n",
    "    sample_size=1.0,\n",
    "    shuffle_buffer=100,\n",
    "    batch_size=32,\n",
    "    num_epochs=20,\n",
    "    prefetch_buffer=100,\n",
    "    shuffle_seed=42,\n",
    "):\n",
    "    \"\"\"\n",
    "    turn pd dataframe into tff client dataset\n",
    "    \"\"\"\n",
    "\n",
    "    def batch_format_fn(element):\n",
    "        \"\"\"format data into OrderedDict where x denotes features and y labels for a client\"\"\"\n",
    "        return collections.OrderedDict(\n",
    "            x=element[\"x\"],  # tf.reshape(element[xcol], [-1, xshape]),\n",
    "            y=element[\"y\"],  # tf.reshape(element[ycol], [-1, yshape]),\n",
    "        )\n",
    "\n",
    "    def create_tf_dataset_for_client_fn(client_id):\n",
    "        \"\"\"a function which takes a client_id and returns a tf.data.Dataset for that client\"\"\"\n",
    "        client_data = df[df[client_id_col] == int(client_id)]\n",
    "        # create tf dataset\n",
    "        dataset = tf.data.Dataset.from_tensor_slices(client_data.to_dict(\"list\"))\n",
    "        # dataset = dataset.shuffle(shuffle_buffer).batch(num_batch).repeat(num_epochs)\n",
    "        dataset = (\n",
    "            dataset.repeat(num_epochs)\n",
    "            .shuffle(shuffle_buffer, seed=shuffle_seed)\n",
    "            .batch(batch_size)\n",
    "            .map(batch_format_fn)\n",
    "            .prefetch(prefetch_buffer)\n",
    "        )\n",
    "        return dataset\n",
    "\n",
    "    # split client id into train and test clients\n",
    "    client_ids = np.random.choice(\n",
    "        df[client_id_col].unique(),\n",
    "        size=int(sample_size * df[client_id_col].nunique()),\n",
    "    ).tolist()  # proportion of clients to use\n",
    "\n",
    "    # train data\n",
    "    def create_mapping(client_ids):\n",
    "        \"\"\"\n",
    "        create mapping of client ids\n",
    "        \"\"\"\n",
    "        mapping = {}\n",
    "        for client_id in client_ids:\n",
    "            mapping[client_id] = str(client_id)\n",
    "        return mapping\n",
    "\n",
    "    # create CustomClientData\n",
    "    # this is slight misuse of FilePerUserClientData but it works for now,\n",
    "    # and there seems not to be another solution for the moment :D\n",
    "    # (please suggest correction if there is a better solution)\n",
    "    tff_data = tff.simulation.datasets.FilePerUserClientData(\n",
    "        client_ids_to_files=create_mapping(client_ids),\n",
    "        dataset_fn=create_tf_dataset_for_client_fn,\n",
    "    )\n",
    "\n",
    "    return tff_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test (well, at least it should not crash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": "2021-12-23 11:41:39.523696: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcuda.so.1\n2021-12-23 11:41:39.783200: E tensorflow/stream_executor/cuda/cuda_driver.cc:328] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n2021-12-23 11:41:39.783253: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (ml-nuutti-1a-m-mem): /proc/driver/nvidia/version does not exist\n2021-12-23 11:41:39.785298: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
    }
   ],
   "source": [
    "tff_data = create_tff_client_data_from_df(cxy_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "<PrefetchDataset shapes: OrderedDict([(x, (None, 20)), (y, (None,))]), types: OrderedDict([(x, tf.int64), (y, tf.int32)])>"
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tff_data.create_tf_dataset_for_client(tff_data.client_ids[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": "2021-12-23 11:41:39.966178: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:176] None of the MLIR Optimization Passes are enabled (registered 2)\n2021-12-23 11:41:39.966611: I tensorflow/core/platform/profile_utils/cpu_utils.cc:114] CPU Frequency: 2095190000 Hz\n"
    }
   ],
   "source": [
    "train_data, test_data = tff.simulation.datasets.ClientData.train_test_client_split(\n",
    "    tff_data, 500\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fcj38",
   "language": "python",
   "name": "fcj38"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}