03_feature-calc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp data.feature_calc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature calculation\n",
    "\n",
    "> This module calculates features/aggregations that could be useful to detect fraudulent transactions. Results are saved into 2 parquet datasets for features email and customer_id features. These features are calculated:\n",
    "\n",
    "Email aggregations:\n",
    " * \"num_customer_id_by_email\": number of previously known different customer_ids for a given email, up to 10\n",
    " * \"same_customer_id_by_email\": 1 if order contains the same customer_id as the last order from this email\n",
    " * \"known_customer_id_by_email\": 1 if the customer_id has been seen before with the email\n",
    " * \"order_amount_mean_by_email\": mean of the last 10 order amounts on this email\n",
    " * \"order_amount_std_by_email\": std\n",
    " * \"order_amount_min_by_email\": min\n",
    " * \"order_amount_max_by_email\": max\n",
    " * \"order_amount_sum_by_email\": sum\n",
    "  \n",
    "Customer_id aggregations:\n",
    " * \"num_email_by_customer_id\": number of previosuly seen different emails on the order customer_id, up to 10\n",
    " * \"num_ip_addr_by_customer_id\": number of previously seen different IP addresses for the order customer_id, up to 10\n",
    " * \"same_email_by_customer_id\": 1 is the email is the same as latest order for this customer_id\n",
    " * \"same_ip_addr_by_customer_id\": 1 is the ip address is the same\n",
    " * \"known_email_by_customer_id\": 1 if the email was seen before with this customer_id\n",
    " * \"known_ip_addr_by_customer_id\": 1 if the IP address was seen befor with this customer_id\n",
    " * \"order_amount_mean_by_customer_id\": mean of the last 10 order amount on this customer_id\n",
    " * \"order_amount_std_by_customer_id\": std\n",
    " * \"order_amount_min_by_customer_id\": min\n",
    " * \"order_amount_max_by_customer_id\": max\n",
    " * \"order_amount_sum_by_customer_id\": sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import dask.dataframe as dd\n",
    "import numpy as np\n",
    "import json\n",
    "\n",
    "from hopeit.app.context import EventContext\n",
    "from hopeit.app.events import Spawn, SHUFFLE\n",
    "from hopeit.app.api import event_api\n",
    "from hopeit.app.logger import app_logger\n",
    "\n",
    "from fraud_poc.jobs import get_client, FeatureCalcJob, PreprocessingJob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "__steps__ = ['run']\n",
    "\n",
    "logger = app_logger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def calculate(df, count_cols, stat_cols, by):\n",
    "    counts = count_distinct_values(df, count_cols, by)\n",
    "    stats = num_stats(df, stat_cols, by)\n",
    "    right = counts.merge(stats)\n",
    "    df = df.merge(right,\n",
    "                  left_on=[df.index, 'order_id'], \n",
    "                  right_on=[by, 'order_id'],\n",
    "                  suffixes=('', f'_DROP'))\n",
    "    return df[[col for col in df.columns if col[-5:] != '_DROP']]\n",
    "        \n",
    "\n",
    "def count_distinct_values(df, cols, by):\n",
    "    sets = []\n",
    "    for col in cols:\n",
    "        sets.append( \n",
    "            df.groupby([df.index, df.order_date, df.order_id])[col] \\\n",
    "                .apply(list) \\\n",
    "                .sort_index() \\\n",
    "                .groupby(level=0) \\\n",
    "                .apply(np.cumsum) \\\n",
    "                .apply(lambda x: list(x)[-10:]))\n",
    "        \n",
    "    counts = sets[0].to_frame()\n",
    "    for col, result in zip(cols, sets):\n",
    "        counts[col] = result\n",
    "        counts[f'num_{col}_by_{by}'] = counts[col].apply(lambda x: len(set(x)))\n",
    "        counts[f'last_{col}_by_{by}'] = counts[col].apply(lambda x: x[-2] if len(x)>1 else \"\")\n",
    "        counts[f'same_{col}_by_{by}'] = counts[col].apply(lambda x: int(x[-2] == x[-1]) if len(x) > 1 else 0)\n",
    "        counts[f'known_{col}_by_{by}'] = counts[col].apply(lambda x: int(x[-1] in x[:-1]) if len(x) > 1 else 0)\n",
    "        counts[f'{col}_by_{by}'] = counts[col].apply(lambda x: json.dumps(x))\n",
    " \n",
    "    count_cols = [f'{p}{col}_by_{by}' for p in ('', 'num_', 'last_', 'same_', 'known_') for col in cols]\n",
    "    counts = counts.reset_index()[[by, 'order_id', *count_cols]]\n",
    "    return counts\n",
    "\n",
    "def num_stats(df, cols, by):\n",
    "    results = []\n",
    "    for col in cols:\n",
    "        results.append(df.groupby([df.index, df.order_date, df.order_id])[col] \\\n",
    "                .apply(list) \\\n",
    "                .sort_index() \\\n",
    "                .groupby(level=0) \\\n",
    "                .apply(np.cumsum) \\\n",
    "                .apply(lambda x: list(x)[-10:]))\n",
    "        \n",
    "    stats = results[0].to_frame()\n",
    "    for col, result in zip(cols[1:], results[1:]):\n",
    "        stats[col] = result\n",
    "    \n",
    "    stats = stats.reset_index()[[by, 'order_id', *cols]]\n",
    "    for col in cols:\n",
    "        stats[f'{col}_mean_by_{by}'] = stats[col].apply(lambda x: np.mean(x))\n",
    "        stats[f'{col}_std_by_{by}'] = stats[col].apply(lambda x: np.std(x))\n",
    "        stats[f'{col}_min_by_{by}'] = stats[col].apply(lambda x: np.min(x))\n",
    "        stats[f'{col}_max_by_{by}'] = stats[col].apply(lambda x: np.max(x))\n",
    "        stats[f'{col}_sum_by_{by}'] = stats[col].apply(lambda x: np.sum(x))\n",
    "        stats[f'{col}_by_{by}'] = stats[col].apply(lambda x: json.dumps(x))\n",
    "    \n",
    "    return stats\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "def run(job: PreprocessingJob, context: EventContext) -> FeatureCalcJob:\n",
    "    base_path = context.env['data']['features']\n",
    "    client = get_client(context)\n",
    "    features = {}\n",
    "    try:\n",
    "        path = job.partitioned.get('customer_id')\n",
    "        if path:\n",
    "            logger.info(context, \"Calculating features on customer_id...\")\n",
    "            df = dd.read_parquet(path, \n",
    "                         engine='fastparquet', \n",
    "                         columns=['order_id', 'order_date', 'email', 'ip_addr', 'order_amount'])\n",
    "            df = df.map_partitions(calculate, count_cols=['email', 'ip_addr'], stat_cols=['order_amount'], by='customer_id')\n",
    "            save_path = f'{base_path}/customer_id/'\n",
    "            df.to_parquet(save_path)\n",
    "            features['customer_id'] = save_path \n",
    "            logger.info(context, f\"Saved {save_path}.\")\n",
    "        \n",
    "        path = job.partitioned.get('email')\n",
    "        if path:\n",
    "            logger.info(context, \"Calculating features on email...\")\n",
    "            df = dd.read_parquet(path, \n",
    "                         engine='fastparquet', \n",
    "                         columns=['order_id', 'order_date', 'customer_id', 'ip_addr', 'order_amount'])\n",
    "            df = df.map_partitions(calculate, count_cols=['customer_id'], stat_cols=['order_amount'], by='email')\n",
    "            save_path = f'{base_path}/email/'\n",
    "            df.to_parquet(save_path)\n",
    "            features['email'] = save_path \n",
    "            logger.info(context, f\"Saved {save_path}.\")\n",
    "            \n",
    "        return FeatureCalcJob(\n",
    "            sources=job.partitioned,\n",
    "            features=features\n",
    "        )\n",
    "    except Exception as e:\n",
    "        logger.error(context, e)\n",
    "        return None\n",
    "    finally:\n",
    "        client.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test from notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-05-18 20:27:03,391 | INFO | fraud-poc training data.feature-calc ALT00617 75592 | Calculating features on customer_id... | track.operation_id=test_operation_id | track.request_id=test_request_id | track.request_ts=2021-05-18T20:27:02.159375+00:00 | stream.name= | stream.msg_id= | stream.consumer_group=\n",
      "2021-05-18 20:27:04,473 | INFO | fraud-poc training data.feature-calc ALT00617 75592 | Saved ./data/features/customer_id/. | track.operation_id=test_operation_id | track.request_id=test_request_id | track.request_ts=2021-05-18T20:27:02.159375+00:00 | stream.name= | stream.msg_id= | stream.consumer_group=\n",
      "2021-05-18 20:27:04,473 | INFO | fraud-poc training data.feature-calc ALT00617 75592 | Calculating features on email... | track.operation_id=test_operation_id | track.request_id=test_request_id | track.request_ts=2021-05-18T20:27:02.159375+00:00 | stream.name= | stream.msg_id= | stream.consumer_group=\n",
      "2021-05-18 20:27:05,344 | INFO | fraud-poc training data.feature-calc ALT00617 75592 | Saved ./data/features/email/. | track.operation_id=test_operation_id | track.request_id=test_request_id | track.request_ts=2021-05-18T20:27:02.159375+00:00 | stream.name= | stream.msg_id= | stream.consumer_group=\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "FeatureCalcJob(sources={'customer_id': './data/partitioned/customer_id/', 'email': './data/partitioned/email'}, features={'customer_id': './data/features/customer_id/', 'email': './data/features/email/'})"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from hopeit.testing.apps import config, execute_event\n",
    "\n",
    "app_config = config('config/training-pipeline.json')\n",
    "job = PreprocessingJob(source='./data/raw', partitioned={\n",
    "    'customer_id': './data/partitioned/customer_id/', \n",
    "    'email': './data/partitioned/email'\n",
    "})\n",
    "result = await execute_event(app_config, 'data.feature-calc', job)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>order_date</th>\n",
       "      <th>email</th>\n",
       "      <th>ip_addr</th>\n",
       "      <th>order_amount</th>\n",
       "      <th>customer_id</th>\n",
       "      <th>email_by_customer_id</th>\n",
       "      <th>ip_addr_by_customer_id</th>\n",
       "      <th>num_email_by_customer_id</th>\n",
       "      <th>num_ip_addr_by_customer_id</th>\n",
       "      <th>...</th>\n",
       "      <th>same_email_by_customer_id</th>\n",
       "      <th>same_ip_addr_by_customer_id</th>\n",
       "      <th>known_email_by_customer_id</th>\n",
       "      <th>known_ip_addr_by_customer_id</th>\n",
       "      <th>order_amount_mean_by_customer_id</th>\n",
       "      <th>order_amount_std_by_customer_id</th>\n",
       "      <th>order_amount_min_by_customer_id</th>\n",
       "      <th>order_amount_max_by_customer_id</th>\n",
       "      <th>order_amount_sum_by_customer_id</th>\n",
       "      <th>order_amount_by_customer_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7beaa6ea-27ea-4082-a0ca-e216b93521a1</td>\n",
       "      <td>2021-05-07 22:49:55+00:00</td>\n",
       "      <td>af9ec8893d8c212276b9916b2599efe88510207d</td>\n",
       "      <td>e18142d2fcd6626e86d40573fd268c2f5f449a60</td>\n",
       "      <td>301.888156</td>\n",
       "      <td>00c00d42-6923-4297-87c4-107bc70035be</td>\n",
       "      <td>[\"af9ec8893d8c212276b9916b2599efe88510207d\"]</td>\n",
       "      <td>[\"e18142d2fcd6626e86d40573fd268c2f5f449a60\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>301.888156</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>301.888156</td>\n",
       "      <td>301.888156</td>\n",
       "      <td>301.888156</td>\n",
       "      <td>[301.8881562196214]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4069005c-02d0-4f41-b065-ad4b82412f20</td>\n",
       "      <td>2021-05-07 05:29:05+00:00</td>\n",
       "      <td>30650ad8775163a27cc73b642d3de1f46a337aaa</td>\n",
       "      <td>110fef07ad03b67a2f30fcebd6ab190bc2d11252</td>\n",
       "      <td>86.057737</td>\n",
       "      <td>048e9104-8369-407b-a0a1-fb06c06760c1</td>\n",
       "      <td>[\"30650ad8775163a27cc73b642d3de1f46a337aaa\"]</td>\n",
       "      <td>[\"110fef07ad03b67a2f30fcebd6ab190bc2d11252\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>86.057737</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>86.057737</td>\n",
       "      <td>86.057737</td>\n",
       "      <td>86.057737</td>\n",
       "      <td>[86.05773660199678]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3e5bb9cc-bdef-4693-8870-49967d510ae3</td>\n",
       "      <td>2021-05-03 10:08:39+00:00</td>\n",
       "      <td>79cdeec4056ed0714127c4c255d9dafa334ee2a7</td>\n",
       "      <td>ee714f6b18d2b01e2d632f1299d8cb7618099070</td>\n",
       "      <td>945.678360</td>\n",
       "      <td>09da86be-63c3-4265-a799-8fe5efaf270c</td>\n",
       "      <td>[\"79cdeec4056ed0714127c4c255d9dafa334ee2a7\"]</td>\n",
       "      <td>[\"ee714f6b18d2b01e2d632f1299d8cb7618099070\"]</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>945.678360</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>945.678360</td>\n",
       "      <td>945.678360</td>\n",
       "      <td>945.678360</td>\n",
       "      <td>[945.678359760513]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9cc8c9aa-e342-4e7a-8269-4ebb7bc0d208</td>\n",
       "      <td>2021-05-15 04:11:54+00:00</td>\n",
       "      <td>79cdeec4056ed0714127c4c255d9dafa334ee2a7</td>\n",
       "      <td>ee714f6b18d2b01e2d632f1299d8cb7618099070</td>\n",
       "      <td>955.929156</td>\n",
       "      <td>09da86be-63c3-4265-a799-8fe5efaf270c</td>\n",
       "      <td>[\"79cdeec4056ed0714127c4c255d9dafa334ee2a7\", \"...</td>\n",
       "      <td>[\"ee714f6b18d2b01e2d632f1299d8cb7618099070\", \"...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>950.803758</td>\n",
       "      <td>5.125398</td>\n",
       "      <td>945.678360</td>\n",
       "      <td>955.929156</td>\n",
       "      <td>1901.607516</td>\n",
       "      <td>[945.678359760513, 955.9291560497809]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5b3900c1-e2ca-4226-81ee-0aa47d825005</td>\n",
       "      <td>2021-05-07 09:52:48+00:00</td>\n",
       "      <td>86665ef2c644c5f6a50e7702a7d5b0b88ee3c68b</td>\n",
       "      <td>8a799f9e07fa26a47e84fd8f64ea20ae5601a489</td>\n",
       "      <td>525.625661</td>\n",
       "      <td>0c5e0c2c-fb79-4a60-94c4-501bf2d07278</td>\n",
       "      <td>[\"86665ef2c644c5f6a50e7702a7d5b0b88ee3c68b\", \"...</td>\n",
       "      <td>[\"8a799f9e07fa26a47e84fd8f64ea20ae5601a489\", \"...</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>680.036988</td>\n",
       "      <td>236.531382</td>\n",
       "      <td>336.752082</td>\n",
       "      <td>997.119448</td>\n",
       "      <td>4760.258919</td>\n",
       "      <td>[336.7520815122097, 821.1451274507584, 922.876...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               order_id                order_date  \\\n",
       "0  7beaa6ea-27ea-4082-a0ca-e216b93521a1 2021-05-07 22:49:55+00:00   \n",
       "1  4069005c-02d0-4f41-b065-ad4b82412f20 2021-05-07 05:29:05+00:00   \n",
       "2  3e5bb9cc-bdef-4693-8870-49967d510ae3 2021-05-03 10:08:39+00:00   \n",
       "3  9cc8c9aa-e342-4e7a-8269-4ebb7bc0d208 2021-05-15 04:11:54+00:00   \n",
       "4  5b3900c1-e2ca-4226-81ee-0aa47d825005 2021-05-07 09:52:48+00:00   \n",
       "\n",
       "                                      email  \\\n",
       "0  af9ec8893d8c212276b9916b2599efe88510207d   \n",
       "1  30650ad8775163a27cc73b642d3de1f46a337aaa   \n",
       "2  79cdeec4056ed0714127c4c255d9dafa334ee2a7   \n",
       "3  79cdeec4056ed0714127c4c255d9dafa334ee2a7   \n",
       "4  86665ef2c644c5f6a50e7702a7d5b0b88ee3c68b   \n",
       "\n",
       "                                    ip_addr  order_amount  \\\n",
       "0  e18142d2fcd6626e86d40573fd268c2f5f449a60    301.888156   \n",
       "1  110fef07ad03b67a2f30fcebd6ab190bc2d11252     86.057737   \n",
       "2  ee714f6b18d2b01e2d632f1299d8cb7618099070    945.678360   \n",
       "3  ee714f6b18d2b01e2d632f1299d8cb7618099070    955.929156   \n",
       "4  8a799f9e07fa26a47e84fd8f64ea20ae5601a489    525.625661   \n",
       "\n",
       "                            customer_id  \\\n",
       "0  00c00d42-6923-4297-87c4-107bc70035be   \n",
       "1  048e9104-8369-407b-a0a1-fb06c06760c1   \n",
       "2  09da86be-63c3-4265-a799-8fe5efaf270c   \n",
       "3  09da86be-63c3-4265-a799-8fe5efaf270c   \n",
       "4  0c5e0c2c-fb79-4a60-94c4-501bf2d07278   \n",
       "\n",
       "                                email_by_customer_id  \\\n",
       "0       [\"af9ec8893d8c212276b9916b2599efe88510207d\"]   \n",
       "1       [\"30650ad8775163a27cc73b642d3de1f46a337aaa\"]   \n",
       "2       [\"79cdeec4056ed0714127c4c255d9dafa334ee2a7\"]   \n",
       "3  [\"79cdeec4056ed0714127c4c255d9dafa334ee2a7\", \"...   \n",
       "4  [\"86665ef2c644c5f6a50e7702a7d5b0b88ee3c68b\", \"...   \n",
       "\n",
       "                              ip_addr_by_customer_id  \\\n",
       "0       [\"e18142d2fcd6626e86d40573fd268c2f5f449a60\"]   \n",
       "1       [\"110fef07ad03b67a2f30fcebd6ab190bc2d11252\"]   \n",
       "2       [\"ee714f6b18d2b01e2d632f1299d8cb7618099070\"]   \n",
       "3  [\"ee714f6b18d2b01e2d632f1299d8cb7618099070\", \"...   \n",
       "4  [\"8a799f9e07fa26a47e84fd8f64ea20ae5601a489\", \"...   \n",
       "\n",
       "   num_email_by_customer_id  num_ip_addr_by_customer_id  ...  \\\n",
       "0                         1                           1  ...   \n",
       "1                         1                           1  ...   \n",
       "2                         1                           1  ...   \n",
       "3                         1                           1  ...   \n",
       "4                         5                           5  ...   \n",
       "\n",
       "  same_email_by_customer_id same_ip_addr_by_customer_id  \\\n",
       "0                         0                           0   \n",
       "1                         0                           0   \n",
       "2                         0                           0   \n",
       "3                         1                           1   \n",
       "4                         0                           0   \n",
       "\n",
       "   known_email_by_customer_id  known_ip_addr_by_customer_id  \\\n",
       "0                           0                             0   \n",
       "1                           0                             0   \n",
       "2                           0                             0   \n",
       "3                           1                             1   \n",
       "4                           1                             1   \n",
       "\n",
       "   order_amount_mean_by_customer_id  order_amount_std_by_customer_id  \\\n",
       "0                        301.888156                         0.000000   \n",
       "1                         86.057737                         0.000000   \n",
       "2                        945.678360                         0.000000   \n",
       "3                        950.803758                         5.125398   \n",
       "4                        680.036988                       236.531382   \n",
       "\n",
       "   order_amount_min_by_customer_id  order_amount_max_by_customer_id  \\\n",
       "0                       301.888156                       301.888156   \n",
       "1                        86.057737                        86.057737   \n",
       "2                       945.678360                       945.678360   \n",
       "3                       945.678360                       955.929156   \n",
       "4                       336.752082                       997.119448   \n",
       "\n",
       "   order_amount_sum_by_customer_id  \\\n",
       "0                       301.888156   \n",
       "1                        86.057737   \n",
       "2                       945.678360   \n",
       "3                      1901.607516   \n",
       "4                      4760.258919   \n",
       "\n",
       "                         order_amount_by_customer_id  \n",
       "0                                [301.8881562196214]  \n",
       "1                                [86.05773660199678]  \n",
       "2                                 [945.678359760513]  \n",
       "3              [945.678359760513, 955.9291560497809]  \n",
       "4  [336.7520815122097, 821.1451274507584, 922.876...  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dd.read_parquet(result.features['customer_id'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>order_date</th>\n",
       "      <th>customer_id</th>\n",
       "      <th>ip_addr</th>\n",
       "      <th>order_amount</th>\n",
       "      <th>email</th>\n",
       "      <th>customer_id_by_email</th>\n",
       "      <th>num_customer_id_by_email</th>\n",
       "      <th>last_customer_id_by_email</th>\n",
       "      <th>same_customer_id_by_email</th>\n",
       "      <th>known_customer_id_by_email</th>\n",
       "      <th>order_amount_mean_by_email</th>\n",
       "      <th>order_amount_std_by_email</th>\n",
       "      <th>order_amount_min_by_email</th>\n",
       "      <th>order_amount_max_by_email</th>\n",
       "      <th>order_amount_sum_by_email</th>\n",
       "      <th>order_amount_by_email</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>564ca53c-92ae-44e9-b550-857bc3f32299</td>\n",
       "      <td>2021-05-07 16:47:55+00:00</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>2061d686466fd7bdd0c4ff12c0c0431adfb4b584</td>\n",
       "      <td>296.665980</td>\n",
       "      <td>06b27c6a29714eead08e96789b9ed37c6b33c67b</td>\n",
       "      <td>[\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...</td>\n",
       "      <td>1</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>620.601978</td>\n",
       "      <td>323.935998</td>\n",
       "      <td>296.665980</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>1241.203956</td>\n",
       "      <td>[944.5379757024328, 296.6659799710525]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>78210c4e-1997-477f-92f7-c78da8f6e2b0</td>\n",
       "      <td>2021-05-12 00:37:48+00:00</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>e1bf51a3b72ef02c554f338b28ea67a61f2effa9</td>\n",
       "      <td>675.094589</td>\n",
       "      <td>06b27c6a29714eead08e96789b9ed37c6b33c67b</td>\n",
       "      <td>[\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...</td>\n",
       "      <td>1</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>638.766181</td>\n",
       "      <td>265.737145</td>\n",
       "      <td>296.665980</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>1916.298544</td>\n",
       "      <td>[944.5379757024328, 296.6659799710525, 675.094...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bab6d5be-14fe-4705-9765-ad5fc5a14b82</td>\n",
       "      <td>2021-05-15 01:52:11+00:00</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>e1bf51a3b72ef02c554f338b28ea67a61f2effa9</td>\n",
       "      <td>860.661434</td>\n",
       "      <td>06b27c6a29714eead08e96789b9ed37c6b33c67b</td>\n",
       "      <td>[\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...</td>\n",
       "      <td>1</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>582.538511</td>\n",
       "      <td>315.696452</td>\n",
       "      <td>135.732576</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>2912.692555</td>\n",
       "      <td>[944.5379757024328, 296.6659799710525, 675.094...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d12ec21f-c115-45c4-b5c9-904578faa733</td>\n",
       "      <td>2021-05-01 23:42:34+00:00</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>e1bf51a3b72ef02c554f338b28ea67a61f2effa9</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>06b27c6a29714eead08e96789b9ed37c6b33c67b</td>\n",
       "      <td>[\"1161afda-4b7f-498b-b07c-a19c8ff25383\"]</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>[944.5379757024328]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c1adb330-64c2-40c1-931a-86ba8dde545d</td>\n",
       "      <td>2021-05-14 06:56:31+00:00</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>e1bf51a3b72ef02c554f338b28ea67a61f2effa9</td>\n",
       "      <td>135.732576</td>\n",
       "      <td>06b27c6a29714eead08e96789b9ed37c6b33c67b</td>\n",
       "      <td>[\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...</td>\n",
       "      <td>1</td>\n",
       "      <td>1161afda-4b7f-498b-b07c-a19c8ff25383</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>513.007780</td>\n",
       "      <td>316.871739</td>\n",
       "      <td>135.732576</td>\n",
       "      <td>944.537976</td>\n",
       "      <td>2052.031121</td>\n",
       "      <td>[944.5379757024328, 296.6659799710525, 675.094...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               order_id                order_date  \\\n",
       "0  564ca53c-92ae-44e9-b550-857bc3f32299 2021-05-07 16:47:55+00:00   \n",
       "1  78210c4e-1997-477f-92f7-c78da8f6e2b0 2021-05-12 00:37:48+00:00   \n",
       "2  bab6d5be-14fe-4705-9765-ad5fc5a14b82 2021-05-15 01:52:11+00:00   \n",
       "3  d12ec21f-c115-45c4-b5c9-904578faa733 2021-05-01 23:42:34+00:00   \n",
       "4  c1adb330-64c2-40c1-931a-86ba8dde545d 2021-05-14 06:56:31+00:00   \n",
       "\n",
       "                            customer_id  \\\n",
       "0  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "1  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "2  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "3  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "4  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "\n",
       "                                    ip_addr  order_amount  \\\n",
       "0  2061d686466fd7bdd0c4ff12c0c0431adfb4b584    296.665980   \n",
       "1  e1bf51a3b72ef02c554f338b28ea67a61f2effa9    675.094589   \n",
       "2  e1bf51a3b72ef02c554f338b28ea67a61f2effa9    860.661434   \n",
       "3  e1bf51a3b72ef02c554f338b28ea67a61f2effa9    944.537976   \n",
       "4  e1bf51a3b72ef02c554f338b28ea67a61f2effa9    135.732576   \n",
       "\n",
       "                                      email  \\\n",
       "0  06b27c6a29714eead08e96789b9ed37c6b33c67b   \n",
       "1  06b27c6a29714eead08e96789b9ed37c6b33c67b   \n",
       "2  06b27c6a29714eead08e96789b9ed37c6b33c67b   \n",
       "3  06b27c6a29714eead08e96789b9ed37c6b33c67b   \n",
       "4  06b27c6a29714eead08e96789b9ed37c6b33c67b   \n",
       "\n",
       "                                customer_id_by_email  \\\n",
       "0  [\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...   \n",
       "1  [\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...   \n",
       "2  [\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...   \n",
       "3           [\"1161afda-4b7f-498b-b07c-a19c8ff25383\"]   \n",
       "4  [\"1161afda-4b7f-498b-b07c-a19c8ff25383\", \"1161...   \n",
       "\n",
       "   num_customer_id_by_email             last_customer_id_by_email  \\\n",
       "0                         1  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "1                         1  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "2                         1  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "3                         1                                         \n",
       "4                         1  1161afda-4b7f-498b-b07c-a19c8ff25383   \n",
       "\n",
       "   same_customer_id_by_email  known_customer_id_by_email  \\\n",
       "0                          1                           1   \n",
       "1                          1                           1   \n",
       "2                          1                           1   \n",
       "3                          0                           0   \n",
       "4                          1                           1   \n",
       "\n",
       "   order_amount_mean_by_email  order_amount_std_by_email  \\\n",
       "0                  620.601978                 323.935998   \n",
       "1                  638.766181                 265.737145   \n",
       "2                  582.538511                 315.696452   \n",
       "3                  944.537976                   0.000000   \n",
       "4                  513.007780                 316.871739   \n",
       "\n",
       "   order_amount_min_by_email  order_amount_max_by_email  \\\n",
       "0                 296.665980                 944.537976   \n",
       "1                 296.665980                 944.537976   \n",
       "2                 135.732576                 944.537976   \n",
       "3                 944.537976                 944.537976   \n",
       "4                 135.732576                 944.537976   \n",
       "\n",
       "   order_amount_sum_by_email  \\\n",
       "0                1241.203956   \n",
       "1                1916.298544   \n",
       "2                2912.692555   \n",
       "3                 944.537976   \n",
       "4                2052.031121   \n",
       "\n",
       "                               order_amount_by_email  \n",
       "0             [944.5379757024328, 296.6659799710525]  \n",
       "1  [944.5379757024328, 296.6659799710525, 675.094...  \n",
       "2  [944.5379757024328, 296.6659799710525, 675.094...  \n",
       "3                                [944.5379757024328]  \n",
       "4  [944.5379757024328, 296.6659799710525, 675.094...  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd.read_parquet(result.features['email']).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}