tabular-anonymizer-demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Anonymize data with tabular anonymizer\r\n",
        "\r\n",
        "Import libraries."
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tabular_anonymizer import DataFrameAnonymizer\r\n",
        "import pandas as pd\r\n",
        "!pip install matplotlib\r\n",
        "import matplotlib.pyplot as plt"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428137647
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Read data. The dataset is Adult Census data. Label is income information, where 0 corresponds to < 50K $ \r\n",
        "\r\n",
        "per year, and 1 corresponds to >= 50K $."
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file1 = \"./data/adult.csv\"\r\n",
        "df = pd.read_csv(file1, sep=\",\", index_col=0)\r\n",
        "df.reset_index()\r\n",
        "df.index = range(len(df))\r\n",
        "df.head()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428137868
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\r\n",
        "For the purposes of this demonstration, we drop some columns and rows."
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Keep only 1000 rows\r\n",
        "df = df.loc[0:999,]\r\n",
        "df.shape "
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428138008
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Drop some columns\r\n",
        "df = df.drop(columns = [\"fnlwgt\", \"education-num\", \"occupation\", \"race\", \"native-country\"])\r\n",
        "df.head(10)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428138176
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup Sensitive attributes you dont want to alter\r\n",
        "sensitive_columns = ['label']\r\n",
        "# Setup columns that are generalized with average, not interval\r\n",
        "avg_columns = ['age']\r\n",
        "\r\n",
        "# Set k\r\n",
        "k = 10"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428138876
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run the anonymizer."
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Run tabular_anonymizer. Sensitive columns: \", sensitive_columns, \", k=\", k)\r\n",
        "p = DataFrameAnonymizer(df, sensitive_columns, avg_columns=avg_columns)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428138984
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# New anonymized dataframe is formed\r\n",
        "df_anonymized = p.anonymize_k_anonymity(k=k)\r\n",
        "df_anonymized.head()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428139756
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Write anonymized dataframe to file.\r\n",
        "\r\n",
        "If you want to open the files in Excel, use as separator something other than \",\", for example \";\"."
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Original small data to csv\r\n",
        "df.to_csv(\"./data/adult-small.csv\", index=False)\r\n",
        "# Anonymized dataframe to csv\r\n",
        "df_anonymized.to_csv(\"./data/adult-anonymized.csv\", index=False)#, sep=\";\")"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428140599
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Example: Generating semi-synthetic tidy data from the Mondrian-k -anonymized data\r\n",
        "\r\n",
        "You may have noticed, that the Mondrian-k -anonymized data is no longer tidy (each row contains an observation, each cell contains a single value). \r\n",
        "Untidy data is difficult to utilize for analysis and ML. \r\n",
        "Let's generate tidy samples from the anonymized dataset by bootstrapping.\r\n",
        "Note that this data is now synthetic: the data points do not anymore link to individuals. However, the dataset should have similar statistical properties as the original one."
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\r\n",
        "# new dataframe for storing generated data\r\n",
        "bootstrap_c = 1 # constant for tuning bootstrapping\r\n",
        "replace_nan_with = 0\r\n",
        "# amount of new data to be generated = bootstrap_c * N\r\n",
        "float_range = []\r\n",
        "int_range = ['capital-gain', 'capital-loss', 'hours-per-week']\r\n",
        "numeric = ['label', 'age']\r\n",
        "category_list = ['workclass', 'education', 'marital-status', 'relationship', 'sex']\r\n",
        "colnames = df.columns\r\n",
        "df_gen = []#pd.DataFrame(columns = colnames)\r\n",
        "df_gen\r\n",
        "# iterate through anonymized data\r\n",
        "for _, row in df_anonymized.iterrows():\r\n",
        "    # generate new samples from each row\r\n",
        "    # tune bootstrap_c for bootstrapping\r\n",
        "    for _ in range(int(row['label_count']*bootstrap_c)):\r\n",
        "        new_row = {} # new sample\r\n",
        "        for colname in colnames: # go through columns\r\n",
        "            value = replace_nan_with\r\n",
        "            if colname in numeric: # just take the value as-is\r\n",
        "                value = row[colname]\r\n",
        "            elif colname in float_range: # sample from value range\r\n",
        "                value_range = np.array(row[colname][0].split('-')).astype(int)\r\n",
        "                if value_range.shape[0] == 2:\r\n",
        "                    value = np.random.rand()*(value_range[1]-value_range[0]) + value_range[0]\r\n",
        "                else: pass\r\n",
        "            elif colname in int_range: # sample from value range\r\n",
        "                value_range = np.array(row[colname][0].split('-')).astype(int)\r\n",
        "                if value_range.shape[0] == 2:\r\n",
        "                    value = np.random.choice(np.arange(value_range[0],value_range[1]))\r\n",
        "                else: pass\r\n",
        "            elif colname in category_list: # sample from list of alternatives\r\n",
        "                if len(row[colname][0].split(',')) > 0:\r\n",
        "                    value = np.random.choice(row[colname][0].split(','))\r\n",
        "                else: pass\r\n",
        "            else:\r\n",
        "                pass\r\n",
        "            new_row[colname] = value\r\n",
        "        #print(new_row)\r\n",
        "        df_gen.append(new_row)\r\n",
        "df_gen = pd.DataFrame(df_gen, columns=df_anonymized.columns).drop('label_count', axis = 1)\r\n",
        "df_gen = df_gen.sample(frac = 1)\r\n",
        "\r\n",
        "# Exercise:\r\n",
        "# You can try plotting the old and new data\r\n",
        "# what happens to the data quality?\r\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428140844
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_gen"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428140997
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Exercises: Anonymization and data quality\r\n",
        "\r\n",
        "1. Try plotting the data in a representative way before (df) and after anonymization & bootstrapping (df_gen). Try changing the setup: N, k, bootstrap_c, numerical feature treatment (avg / range). What happens?\r\n",
        "\r\n",
        "2. Try fitting some ML model to the data or part of it before and after anonymization & bootstrapping. What happens?\r\n",
        "\r\n",
        "3. Can you explain how anonymization can affect data & model quality?\r\n"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1. Plotting the data (some examples)"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_sorted_line(ori, gen, column_name, ax):\r\n",
        "    \"\"\"\r\n",
        "    line plot of sorted data, original and generated from anonymized\r\n",
        "    \"\"\"\r\n",
        "    # original\r\n",
        "    df[column_name].sort_values().reset_index(drop = True).plot(ax = ax, label = 'original')\r\n",
        "    # anonymized\r\n",
        "    df_gen[column_name].sort_values().reset_index(drop = True).plot(ax = ax, label = 'anonymized')\r\n",
        "    ax.legend(title = column_name)\r\n",
        "    return ax\r\n",
        "\r\n",
        "def plot_scatter(ori, gen, xcol, ycol, ax):\r\n",
        "    \"\"\"\r\n",
        "    scatter plot of two columns of the data, original and generated\r\n",
        "    \"\"\"\r\n",
        "    # original\r\n",
        "    df[[xcol, ycol]].plot(ax = ax, x = xcol, y = ycol, kind = 'scatter', label = 'original', color = 'b', marker = 'o', alpha = 0.5)\r\n",
        "    # anonymized\r\n",
        "    df_gen[[xcol, ycol]].plot(ax = ax, x = xcol, y = ycol, kind = 'scatter', label = 'anonymized', color = 'r', marker = 'x', alpha = 0.5)\r\n",
        "    ax.legend(title = xcol + '-' + ycol)\r\n",
        "    return ax"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428141120
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nrow = 2\r\n",
        "ncol = 2\r\n",
        "fig, axs = plt.subplots(nrow, ncol, figsize = (ncol*8, nrow*5),constrained_layout = True)\r\n",
        "# cap-loss\r\n",
        "plot_sorted_line(df, df_gen, 'capital-loss', ax = axs[0,0])\r\n",
        "# cap-gain\r\n",
        "plot_sorted_line(df, df_gen, 'capital-gain', ax = axs[0,1])\r\n",
        "\r\n",
        "# hours per week\r\n",
        "plot_sorted_line(df, df_gen, 'hours-per-week', ax = axs[1,0]) # capital gain!\r\n",
        "\r\n",
        "# age\r\n",
        "plot_sorted_line(df, df_gen, 'age', ax = axs[1,1])"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428141733
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nrow = 4\r\n",
        "ncol = 4\r\n",
        "fig, axs = plt.subplots(nrow, ncol, figsize = (ncol*4, nrow*4),constrained_layout = True)\r\n",
        "plot_by = ['age', 'hours-per-week', 'capital-loss', 'capital-gain']\r\n",
        "\r\n",
        "for i in range(4):\r\n",
        "    for j in range(4):\r\n",
        "        if i == j:\r\n",
        "            pass\r\n",
        "        else:\r\n",
        "            xcol = plot_by[i]\r\n",
        "            ycol = plot_by[j]\r\n",
        "            plot_scatter(df, df_gen, xcol, ycol, ax = axs[i,j])\r\n",
        "\r\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "source_hidden": false,
          "outputs_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1642428144854
        }
      }
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "py38anontest",
      "language": "python",
      "display_name": "py38anontest"
    },
    "language_info": {
      "name": "python",
      "version": "3.8.12",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kernel_info": {
      "name": "py38anontest"
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}