finite_pool_qbc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Active Learning with SUSY-AI Pool\n",
    "### Include packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Active Learning configuration parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Size of initial dataset (will be chosen at random from remaining pool)\n",
    "size_init = 10000\n",
    "# Step size\n",
    "size_iter = 2500\n",
    "# Number of active learning iterations\n",
    "n_iterations = 1\n",
    "# Test size\n",
    "size_test = 100000\n",
    "# Stop at\n",
    "size_max = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set other configuration variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Location of the data pool\n",
    "data_location = \"alldata_full.npy\"\n",
    "# Output log files\n",
    "output_location_active = \"active.csv\"\n",
    "output_location_random = \"random.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Load data and logbook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = np.load(data_location).astype(np.float)\n",
    "X = source[:,1:21]\n",
    "y = source[:,-6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X -= np.mean(X, axis=0)\n",
    "X /= np.std(X, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_active = open(output_location_active, 'w')\n",
    "log_active.write(\"iteration,trainsize,score,accuracy,auc,brier,f1,precision,recall\\n\")\n",
    "log_active.flush()\n",
    "\n",
    "log_random = open(output_location_random, 'w')\n",
    "log_random.write(\"iteration,trainsize,score,accuracy,auc,brier,f1,precision,recall\\n\")\n",
    "log_random.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Functions\n",
    "### Train classifier and create metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_test_results(log, prediction):\n",
    "    log.write(','.join(map(str, prediction)))\n",
    "    log.write(\"\\n\")\n",
    "    log.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(X_train, y_train, X_test, y_test, do_log=False):\n",
    "    # Create and train algorithm\n",
    "    est = RandomForestClassifier()\n",
    "    est.fit(X_train, y_train)\n",
    "    # Make prediction on test set\n",
    "    y_pred = est.predict(X_test)\n",
    "    # Calculate performance\n",
    "    score = est.score(X_test, y_test)\n",
    "    accuracy = metrics.accuracy_score(y_test, y_pred)\n",
    "    auc = metrics.roc_auc_score(y_test, y_pred)\n",
    "    brier = metrics.brier_score_loss(y_test, y_pred)\n",
    "    fone = metrics.f1_score(y_test, y_pred)\n",
    "    precision = metrics.precision_score(y_test, y_pred)\n",
    "    recall = metrics.recall_score(y_test, y_pred)\n",
    "    # Return performance metrics\n",
    "    return (est, {\n",
    "        \"score\": score,\n",
    "        \"acc\": accuracy,\n",
    "        \"auc\": auc,\n",
    "        \"brier\": brier,\n",
    "        \"f1\": fone,\n",
    "        \"prec\": precision,\n",
    "        \"recall\": recall\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create initial data sets for each iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_sets(X, y, size_init, size_test):\n",
    "    # Split data in test and pool\n",
    "    X_pool, X_test, y_pool, y_test = train_test_split(X, y, test_size=size_test)\n",
    "    # X_pool is now randomly ordered w.r.t. the original X, so we can just use\n",
    "    # slicing to create the training and the pool set\n",
    "    X_train = X_pool[:size_init, :]\n",
    "    y_train = y_pool[:size_init]\n",
    "    X_pool = X_pool[size_init:, :]\n",
    "    y_pool = y_pool[size_init:]\n",
    "    # Return arrays\n",
    "    return (X_train, y_train, X_pool, y_pool, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_results(log, iteration, size, performance):\n",
    "    log.write(\"{},{},{},{},{},{},{},{},{}\\n\".format(\n",
    "        iteration,\n",
    "        size,\n",
    "        performance[\"score\"],\n",
    "        performance[\"acc\"],\n",
    "        performance[\"auc\"],\n",
    "        performance[\"brier\"],\n",
    "        performance[\"f1\"],\n",
    "        performance[\"prec\"],\n",
    "        performance[\"recall\"]\n",
    "    ))\n",
    "    log.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_uncertainties(log, uncertainties):\n",
    "    log.write(','.join(map(str, uncertainties))+\"\\n\")\n",
    "    log.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_label_predtruth(log, prediction, truth):\n",
    "    log.write(','.join(map(str, prediction))+',')\n",
    "    log.write(','.join(map(str, truth)))\n",
    "    log.write(\"\\n\")\n",
    "    log.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Increase data size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def increment_dataset_random(X_train, y_train, X_pool, y_pool, size_iter):\n",
    "    # Get random indices for selection from pool\n",
    "    seed = np.random.rand(X_pool.shape[0])\n",
    "    indices = np.argsort(seed)[::-1]\n",
    "    # Increment data set\n",
    "    X_train, y_train, X_pool, y_pool, y_selected = increment_dataset(indices, X_train, y_train, X_pool, y_pool, size_iter)\n",
    "    # Result new training set and pool\n",
    "    return (X_train, y_train, X_pool, y_pool)\n",
    "\n",
    "def increment_dataset_active(est, X_train, y_train, X_pool, y_pool, size_iter, log_uncertainty):\n",
    "    # Create prediction on pool to get uncertainty of estimator on pool data points\n",
    "    y_pred = est.predict_proba(X_pool)[:,1]\n",
    "    uncertainty = -1*np.abs(y_pred-0.5)+0.5\n",
    "    # Sort pool based on predictions\n",
    "    indices = uncertainty.argsort()[::-1]\n",
    "    # Increment data set\n",
    "    X_train, y_train, X_pool, y_pool, y_selected = increment_dataset(indices, X_train, y_train, X_pool, y_pool, size_iter)\n",
    "    # Return new training set and pool\n",
    "    return (X_train, y_train, X_pool, y_pool)\n",
    "\n",
    "def increment_dataset(indices, X_train, y_train, X_pool, y_pool, size_iter):\n",
    "    # Sort pool based on random indices\n",
    "    X_pool = X_pool[indices,:]\n",
    "    y_pool = y_pool[indices]\n",
    "    # Selected\n",
    "    X_selected = X_pool[:size_iter, :]\n",
    "    y_selected = y_pool[:size_iter]\n",
    "    # Add top [size_iter] points to training data\n",
    "    X_train = np.vstack((X_train, X_selected))\n",
    "    y_train = np.hstack((y_train, y_selected))\n",
    "    # Remove selected points from the pool\n",
    "    X_pool = X_pool[size_iter:, :]\n",
    "    y_pool = y_pool[size_iter:]\n",
    "    # Return training set and pool\n",
    "    return (X_train, y_train, X_pool, y_pool, y_selected)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Main program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for iteration in range(n_iterations):\n",
    "    print(\"ITERATION {}\".format(iteration))\n",
    "    # Create initial data sets\n",
    "    # Create data sets for active learning\n",
    "    X_active, y_active, X_active_pool, y_active_pool, X_test, y_test = create_sets(X, y, size_init, size_test)\n",
    "    # Copy initial states of active learning to create initial states for random sampling\n",
    "    X_random, y_random = copy.deepcopy(X_active), copy.deepcopy(y_active)\n",
    "    X_random_pool, y_random_pool = copy.deepcopy(X_active_pool), copy.deepcopy(y_active_pool)\n",
    "\n",
    "    # Determine stopping criterion\n",
    "    if size_max == -1:\n",
    "        continue_run = X_active_pool.shape[0] >= size_iter\n",
    "    else:\n",
    "        continue_run = X_active_pool.shape[0] >= size_max\n",
    "        \n",
    "    # Run for as long\n",
    "    while continue_run:\n",
    "        \"\"\" Random Sampling \"\"\"\n",
    "        # Get performance of trained estimator\n",
    "        _, performance = train_and_test(X_random, y_random, X_test, y_test, False)\n",
    "        # Log results\n",
    "        log_results(log_random, iteration, X_random.shape[0], performance)\n",
    "        # Increment training datßa by adding data from the pool\n",
    "        X_random, y_random, X_random_pool, y_random_pool = increment_dataset_random(X_random, y_random, X_random_pool, y_random_pool, size_iter)\n",
    "        # Store result for printing\n",
    "        result_random = performance[\"acc\"]\n",
    "\n",
    "        \"\"\" Active Learning \"\"\"\n",
    "        # Get performance of trained estimator\n",
    "        estimator, performance = train_and_test(X_active, y_active, X_test, y_test, True)\n",
    "        # Log results\n",
    "        log_results(log_active, iteration, X_active.shape[0], performance)\n",
    "        # Print results\n",
    "        print(\"  {:<7}{:<10}{:<10}\".format(X_active.shape[0], round(result_random, 5), round(performance[\"acc\"],5)))\n",
    "        # Increment training data by adding data from the pool\n",
    "        X_active, y_active, X_active_pool, y_active_pool = increment_dataset_active(estimator, X_active, y_active, X_active_pool, y_active_pool, size_iter, None)\n",
    "\n",
    "        # Determine stopping criterion\n",
    "        if size_max == -1:\n",
    "            continue_run = X_active_pool.shape[0] >= size_iter\n",
    "        else:\n",
    "            continue_run = X_active_pool.shape[0] >= size_max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Load results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "active = pd.read_csv(output_location_active)\n",
    "random = pd.read_csv(output_location_random)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialise arrays\n",
    "n_iterations = active['iteration'].value_counts().keys().max() + 1\n",
    "n_per_iteration = int(active.shape[0] / n_iterations)\n",
    "al = np.zeros((n_per_iteration, n_iterations))\n",
    "rs = np.zeros((n_per_iteration, n_iterations))\n",
    "\n",
    "# Fill arrays\n",
    "for i in range(n_iterations):\n",
    "    print(active[active['iteration']==i]['accuracy'].shape)\n",
    "    al[:,i] = active[active['iteration']==i]['accuracy']\n",
    "    rs[:,i] = random[random['iteration']==i]['accuracy']\n",
    "    if i == 0:\n",
    "        # Get x axis\n",
    "        x = active[active['iteration'] == i]['trainsize']\n",
    "        \n",
    "# Plot lines and bands\n",
    "plt.clf()\n",
    "plt.figure(figsize=(16,10))\n",
    "for label,arr in zip(('Active learning', 'Random sampling'),(al, rs)):\n",
    "    plt.plot(x,np.mean(arr, axis=1), label=label)\n",
    "    band_min = np.amin(arr, axis=1)\n",
    "    band_max = np.amax(arr, axis=1)\n",
    "    plt.fill_between(x,band_min, band_max, alpha=0.3)\n",
    "plt.legend()\n",
    "plt.xlim([0,309000])\n",
    "plt.xlabel(\"Train size\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}