modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "96e3739d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import preprocessing\n",
    "from sklearn import model_selection\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "10d55d82",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.float_format', lambda x: '%.6f' % x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b9c45243",
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv('../processed_logs.csv',index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "id": "b74bb8fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(575061, 157)"
      ]
     },
     "execution_count": 407,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ecb49542",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>147</th>\n",
       "      <th>148</th>\n",
       "      <th>149</th>\n",
       "      <th>150</th>\n",
       "      <th>151</th>\n",
       "      <th>152</th>\n",
       "      <th>153</th>\n",
       "      <th>154</th>\n",
       "      <th>155</th>\n",
       "      <th>anomaly_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 157 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0        1        2        3        4        5        6        7  \\\n",
       "0 3.000000 1.000000 3.000000 3.000000 3.000000 2.000000 1.000000 3.000000   \n",
       "1 3.000000 1.000000 3.000000 3.000000 3.000000 0.000000 0.000000 3.000000   \n",
       "2 3.000000 1.000000 3.000000 3.000000 3.000000 0.000000 0.000000 3.000000   \n",
       "3 3.000000 1.000000 3.000000 3.000000 3.000000 2.000000 1.000000 3.000000   \n",
       "4 3.000000 1.000000 3.000000 3.000000 3.000000 0.000000 0.000000 0.000000   \n",
       "\n",
       "         8        9  ...      147      148      149      150      151  \\\n",
       "0 3.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000 0.000000   \n",
       "1 3.000000 2.000000  ... 0.000000 0.000000 0.000000 0.000000 0.000000   \n",
       "2 3.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000 0.000000   \n",
       "3 3.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000 0.000000   \n",
       "4 0.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000 0.000000   \n",
       "\n",
       "       152      153      154      155  anomaly_label  \n",
       "0 0.000000 0.000000 0.000000 0.000000              0  \n",
       "1 0.000000 0.000000 0.000000 0.000000              0  \n",
       "2 0.000000 0.000000 0.000000 0.000000              0  \n",
       "3 0.000000 0.000000 0.000000 0.000000              0  \n",
       "4 0.000000 0.000000 0.000000 0.000000              0  \n",
       "\n",
       "[5 rows x 157 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "18cccda8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test=model_selection.train_test_split(data.iloc[:,:-1],data.loc[:,'anomaly_label'],test_size=0.15,random_state=586)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eabe5e61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(488801, 156)\n",
      "(488801,)\n",
      "(86260, 156)\n",
      "(86260,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a141044",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>146</th>\n",
       "      <th>147</th>\n",
       "      <th>148</th>\n",
       "      <th>149</th>\n",
       "      <th>150</th>\n",
       "      <th>151</th>\n",
       "      <th>152</th>\n",
       "      <th>153</th>\n",
       "      <th>154</th>\n",
       "      <th>155</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>480256</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108706</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379236</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573215</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305262</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 156 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              0        1        2        3        4        5        6  \\\n",
       "480256 3.000000 1.000000 3.000000 3.000000 3.000000 0.000000 0.000000   \n",
       "108706 3.000000 1.000000 3.000000 3.000000 3.000000 2.000000 1.000000   \n",
       "379236 3.000000 1.000000 3.000000 3.000000 3.000000 0.000000 0.000000   \n",
       "573215 3.000000 1.000000 3.000000 3.000000 3.000000 2.000000 1.000000   \n",
       "305262 3.000000 1.000000 3.000000 3.000000 3.000000 0.000000 0.000000   \n",
       "\n",
       "              7        8        9  ...      146      147      148      149  \\\n",
       "480256 0.000000 0.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000   \n",
       "108706 3.000000 3.000000 2.000000  ... 0.000000 0.000000 0.000000 0.000000   \n",
       "379236 0.000000 0.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000   \n",
       "573215 3.000000 3.000000 3.000000  ... 0.000000 0.000000 0.000000 0.000000   \n",
       "305262 3.000000 3.000000 0.000000  ... 0.000000 0.000000 0.000000 0.000000   \n",
       "\n",
       "            150      151      152      153      154      155  \n",
       "480256 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000  \n",
       "108706 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000  \n",
       "379236 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000  \n",
       "573215 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000  \n",
       "305262 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000  \n",
       "\n",
       "[5 rows x 156 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "15f5ab5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train percent anamolies: 0.0293\n",
      "Test percent anamolies: 0.029\n"
     ]
    }
   ],
   "source": [
    "print('Train percent anamolies:',str(np.round(y_train.mean(),4)))\n",
    "print('Test percent anamolies:',str(np.round(y_test.mean(),4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdb15328",
   "metadata": {},
   "source": [
    "## Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2c63341e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train mean: [-7.26182494e-16  0.00000000e+00 -4.85647989e-16 -1.95442462e-16\n",
      " -2.94915334e-16]\n",
      "Test mean: [0.00275988 0.         0.00505306 0.00404968 0.00250741]\n",
      "Train std: [1. 0. 1. 1. 1.]\n",
      "Test std: [1.00747758 0.         0.98275836 0.97977846 0.98379632]\n"
     ]
    }
   ],
   "source": [
    "## Perform standardization\n",
    "\n",
    "# Fit StandardScaler on training set\n",
    "scaler=preprocessing.StandardScaler().fit(X_train)\n",
    "\n",
    "# Apply standardization to train/test sets\n",
    "X_train_scaled=scaler.transform(X_train)\n",
    "X_test_scaled=scaler.transform(X_test)\n",
    "\n",
    "# Examine mean/std of first 5\n",
    "print('Train mean:',X_train_scaled.mean(axis=0)[0:5])\n",
    "print('Test mean:',X_test_scaled.mean(axis=0)[0:5])\n",
    "\n",
    "print('Train std:',X_train_scaled.std(axis=0)[0:5])\n",
    "print('Test std:',X_test_scaled.std(axis=0)[0:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b17897ad",
   "metadata": {},
   "source": [
    "## PCA (Dimensionality Reduction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1c5c9d53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform PCA on scaled data\n",
    "pca=PCA()\n",
    "pca.fit(X_train_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c08287ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.04750859 0.08817697 0.11806787 0.14545955 0.17002872 0.19313692\n",
      " 0.21487609 0.23661527 0.25835444 0.28009362 0.30183216 0.32262586\n",
      " 0.34306334 0.36241976 0.38144819 0.40038449 0.41854496 0.43623694\n",
      " 0.45290777 0.46849176 0.48363444 0.49826264 0.51281997 0.52717507\n",
      " 0.54131139 0.55495502 0.56765768 0.57944152 0.59064633 0.60175483\n",
      " 0.61155078 0.62073992 0.6298328  0.63868655 0.64684982 0.65481632\n",
      " 0.66265446 0.67027904 0.67782159 0.68523934 0.69262409 0.69997499\n",
      " 0.70725571 0.71451226 0.7217616  0.72900897 0.7362554  0.7435018\n",
      " 0.7507482  0.7579946  0.76524099 0.77248739 0.77973378 0.78698017\n",
      " 0.79422656 0.80147295 0.80871934 0.81596574 0.82321213 0.83045852\n",
      " 0.83770491 0.8449513  0.85219769 0.85944409 0.86669048 0.87393687\n",
      " 0.88118326 0.88842918 0.89567265 0.90289983 0.9099805  0.91703115\n",
      " 0.92384759 0.93039872 0.93664212 0.94271838 0.94787254 0.95220404\n",
      " 0.9561483  0.959922   0.96354521 0.9670143  0.97039779 0.97350761\n",
      " 0.97656712 0.97951951 0.98236829 0.98513706 0.98777851 0.98998146\n",
      " 0.99201583 0.99365282 0.99524199 0.9964137  0.99730007 0.99795569\n",
      " 0.9985782  0.9989846  0.99927075 0.99951469 0.99967742 0.99978443\n",
      " 0.99988385 0.99995482 0.99998328 0.99999039 0.99999584 0.99999994\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.\n",
      " 1.         1.         1.         1.         1.         1.        ]\n"
     ]
    }
   ],
   "source": [
    "# Examine cumulative proportion of variance explained\n",
    "print(pca.explained_variance_ratio_.cumsum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "571e5f49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Components to explain 99% of variance: 90\n"
     ]
    }
   ],
   "source": [
    "num_components=np.argmax(pca.explained_variance_ratio_.cumsum()>=0.99)\n",
    "print('# Components to explain 99% of variance:',num_components)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "30f6f57b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(488801, 90)\n",
      "(86260, 90)\n"
     ]
    }
   ],
   "source": [
    "# Get component scores for train/test sets\n",
    "X_train_scores=pca.transform(X_train_scaled)[:,:num_components]\n",
    "X_test_scores=pca.transform(X_test_scaled)[:,:num_components]\n",
    "print(X_train_scores.shape)\n",
    "print(X_test_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be62206a",
   "metadata": {},
   "source": [
    "## Define weighted binary cross entropy loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "17a99d87",
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_bce(y_true,y_pred):\n",
    "    sample_weights=(y_true.values==1).astype(float)\n",
    "    sample_weights=sample_weights*(1/np.mean(y_true))\n",
    "    sample_weights=np.where(sample_weights==0,1,sample_weights)\n",
    "    return metrics.log_loss(y_true=y_true,y_pred=y_pred,sample_weight=sample_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ff75e1d",
   "metadata": {},
   "source": [
    "## Logistic regression on first 90 PCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "fd3e5b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform logistic regression on component scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "cfd8fb86",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define dictionary of metrics for CV\n",
    "eval_metrics={'weighted_bce':metrics.make_scorer(weighted_bce),'accuracy':metrics.make_scorer(metrics.accuracy_score),'recall':metrics.make_scorer(metrics.recall_score),'precision':metrics.make_scorer(metrics.precision_score)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "9503f3a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   35.7s remaining:   53.6s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   46.1s finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for logistic regression model\n",
    "logistic_regressor_cv=model_selection.cross_validate(LogisticRegression(max_iter=1000,n_jobs=-1),X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "id": "4859f21e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': array([41.62140703, 44.70747209, 35.00737   , 34.34562516, 33.01395297]),\n",
       " 'score_time': array([0.06046104, 0.04790878, 0.07120609, 0.09732294, 0.09866214]),\n",
       " 'test_weighted_bce': array([0.05894362, 0.0914829 , 0.1376988 , 0.14309481, 0.12045256]),\n",
       " 'test_accuracy': array([0.9996829 , 0.99918167, 0.99925327, 0.99928396, 0.99956015]),\n",
       " 'test_recall': array([0.99686192, 0.99546564, 0.99267527, 0.99232647, 0.99337286]),\n",
       " 'test_precision': array([0.99236376, 0.97706265, 0.98205659, 0.98340823, 0.99164345])}"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_regressor_cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "30380ff7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 37.73917,\n",
       " 'score_time': 0.07511,\n",
       " 'test_weighted_bce': 0.11033,\n",
       " 'test_accuracy': 0.99939,\n",
       " 'test_recall': 0.99414,\n",
       " 'test_precision': 0.98531}"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in logistic_regressor_cv.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af7a4575",
   "metadata": {},
   "source": [
    "## QDA on first 90 PCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "b59045b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   20.1s remaining:   30.2s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   20.3s finished\n"
     ]
    }
   ],
   "source": [
    "qda_model=QuadraticDiscriminantAnalysis()\n",
    "# Get 5-fold CV estimates for QDA\n",
    "qda_model_cv=model_selection.cross_validate(qda_model,X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "id": "cdfb6506",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 18.95564,\n",
       " 'score_time': 0.354,\n",
       " 'test_weighted_bce': 0.90334,\n",
       " 'test_accuracy': 0.99653,\n",
       " 'test_recall': 0.95048,\n",
       " 'test_precision': 0.93374}"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in qda_model_cv.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "037f78d3",
   "metadata": {},
   "source": [
    "## KNN on first 90 PCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "id": "1e4f9f89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create KNN model with default params\n",
    "knn_model=KNeighborsClassifier(n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "id": "726b87d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   25.7s remaining:   38.6s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   25.8s finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for default KNN model\n",
    "# Use 10% of training set (computationally expensive)\n",
    "knn_model_cv=model_selection.cross_validate(knn_model,X=X_train_scores[:int(len(X_train_scores)*0.1),:],y=y_train[:int(len(X_train_scores)*0.1)],scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "id": "f1b2f157",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 0.01714,\n",
       " 'score_time': 24.27379,\n",
       " 'test_weighted_bce': 0.12314,\n",
       " 'test_accuracy': 0.99959,\n",
       " 'test_recall': 0.99318,\n",
       " 'test_precision': 0.99319}"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in knn_model_cv.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "id": "a7530997",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 1 of 50\n",
      "Performed iteration 2 of 50\n",
      "Performed iteration 3 of 50\n",
      "Performed iteration 4 of 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ericphillips/miniforge3/lib/python3.9/site-packages/joblib/externals/loky/process_executor.py:688: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 5 of 50\n",
      "Performed iteration 6 of 50\n",
      "Performed iteration 7 of 50\n",
      "Performed iteration 8 of 50\n",
      "Performed iteration 9 of 50\n",
      "Performed iteration 10 of 50\n",
      "Performed iteration 11 of 50\n",
      "Performed iteration 12 of 50\n",
      "Performed iteration 13 of 50\n",
      "Performed iteration 14 of 50\n",
      "Performed iteration 15 of 50\n",
      "Performed iteration 16 of 50\n",
      "Performed iteration 17 of 50\n",
      "Performed iteration 18 of 50\n",
      "Performed iteration 19 of 50\n",
      "Performed iteration 20 of 50\n",
      "Performed iteration 21 of 50\n",
      "Performed iteration 22 of 50\n",
      "Performed iteration 23 of 50\n",
      "Performed iteration 24 of 50\n",
      "Performed iteration 25 of 50\n",
      "Performed iteration 26 of 50\n",
      "Performed iteration 27 of 50\n",
      "Performed iteration 28 of 50\n",
      "Performed iteration 29 of 50\n",
      "Performed iteration 30 of 50\n",
      "Performed iteration 31 of 50\n",
      "Performed iteration 32 of 50\n",
      "Performed iteration 33 of 50\n",
      "Performed iteration 34 of 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ericphillips/miniforge3/lib/python3.9/site-packages/joblib/externals/loky/process_executor.py:688: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 35 of 50\n",
      "Performed iteration 36 of 50\n",
      "Performed iteration 37 of 50\n",
      "Performed iteration 38 of 50\n",
      "Performed iteration 39 of 50\n",
      "Performed iteration 40 of 50\n",
      "Performed iteration 41 of 50\n",
      "Performed iteration 42 of 50\n",
      "Performed iteration 43 of 50\n",
      "Performed iteration 44 of 50\n",
      "Performed iteration 45 of 50\n",
      "Performed iteration 46 of 50\n",
      "Performed iteration 47 of 50\n",
      "Performed iteration 48 of 50\n",
      "Performed iteration 49 of 50\n",
      "Performed iteration 50 of 50\n"
     ]
    }
   ],
   "source": [
    "# Tune k hyperparameter\n",
    "k_vals=[]\n",
    "knn_cv_weighted_bce=[]\n",
    "counter=0\n",
    "max_k=50\n",
    "for i in np.arange(1,max_k+1):\n",
    "    counter+=1\n",
    "    k_vals.append(i)\n",
    "    # Define KNN model\n",
    "    knn_model_tuner=KNeighborsClassifier(n_neighbors=i,n_jobs=-1)\n",
    "    # Get CV estimate of weighted binary cross entropy loss\n",
    "    knn_model_tuner_cv=model_selection.cross_validate(knn_model_tuner,X=X_train_scores[:int(len(X_train_scores)*0.1),:],y=y_train[:int(len(X_train_scores)*0.1)],scoring=eval_metrics,n_jobs=-1)\n",
    "    knn_cv_weighted_bce.append(np.mean(knn_model_tuner_cv['test_weighted_bce']))\n",
    "    print('Performed iteration '+str(counter)+' of '+str(max_k))\n",
    "knn_grid_results=pd.DataFrame({'k_val':k_vals,'cv_weighted_bce':knn_cv_weighted_bce})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "8f96f701",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k_val</th>\n",
       "      <th>cv_weighted_bce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.051057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.075311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>48</td>\n",
       "      <td>0.094361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>47</td>\n",
       "      <td>0.094361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>49</td>\n",
       "      <td>0.094720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>50</td>\n",
       "      <td>0.094720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.121745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.123140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.135067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.135067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    k_val  cv_weighted_bce\n",
       "0       1         0.051057\n",
       "2       3         0.075311\n",
       "47     48         0.094361\n",
       "46     47         0.094361\n",
       "48     49         0.094720\n",
       "49     50         0.094720\n",
       "1       2         0.121745\n",
       "4       5         0.123140\n",
       "6       7         0.135067\n",
       "5       6         0.135067"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_grid_results.sort_values('cv_weighted_bce').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "32d244c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create KNN model with tuned k param\n",
    "tuned_knn_model=KNeighborsClassifier(n_neighbors=3,n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "d0932f5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed: 34.6min remaining: 51.9min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 34.7min finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for default KNN model on whole dataset\n",
    "tuned_knn_model_cv=model_selection.cross_validate(tuned_knn_model,X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "ee345195",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 0.18673,\n",
       " 'score_time': 2072.15979,\n",
       " 'test_weighted_bce': 0.02739,\n",
       " 'test_accuracy': 0.99986,\n",
       " 'test_recall': 0.99854,\n",
       " 'test_precision': 0.99666}"
      ]
     },
     "execution_count": 334,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in tuned_knn_model_cv.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "046d3008",
   "metadata": {},
   "source": [
    "## Random forest on first 90 PCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "4f36e7d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create random forest with default params\n",
    "random_forest=RandomForestClassifier(n_jobs=-1,random_state=586)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "1150903a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   59.2s remaining:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   59.6s finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for default random forest\n",
    "random_forest_cv=model_selection.cross_validate(random_forest,X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "788fe342",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 58.08247,\n",
       " 'score_time': 0.3241,\n",
       " 'test_weighted_bce': 0.01887,\n",
       " 'test_accuracy': 0.99987,\n",
       " 'test_recall': 0.99902,\n",
       " 'test_precision': 0.99659}"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in random_forest_cv.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "f06ce531",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Tune random forest hyperparameters using OOB weighted binary cross entropy loss\n",
    "rf_params={'criterion':['gini','entropy'],\n",
    "           'max_features':['auto','sqrt','log2'],\n",
    "           'max_depth':[30,40,50,60,None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "543073a0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.4s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 1 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    3.4s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    9.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 2 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    4.8s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   13.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 3 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    4.7s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   13.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 4 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 5 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    3.5s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:    9.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 6 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    4.7s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   13.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 7 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.0s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   13.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 8 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.6s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 9 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.3s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 10 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    3.9s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   10.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 11 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.5s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 12 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    4.7s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   12.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 13 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.2s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 14 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.4s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   14.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 15 of 15\n"
     ]
    }
   ],
   "source": [
    "# Implement random grid search\n",
    "counter=0\n",
    "num_iter=15\n",
    "rf_grid_results=[]\n",
    "for i in range(num_iter):\n",
    "    counter+=1\n",
    "    # Select random parameter set\n",
    "    param_choice={key:np.random.choice(val) for key,val in rf_params.items()}\n",
    "    # Define model with param choices\n",
    "    random_forest_tuner=RandomForestClassifier(\n",
    "                           criterion=param_choice['criterion'],\n",
    "                           max_features=param_choice['max_features'],\n",
    "                           max_depth=param_choice['max_depth'],\n",
    "                           oob_score=True,\n",
    "                           n_jobs=-1,\n",
    "                           verbose=1,\n",
    "                           random_state=586)\n",
    "    # Fit model\n",
    "    random_forest_tuner.fit(X_train_scores,y_train)\n",
    "    # Compute OOB binary weighted cross entropy loss\n",
    "    param_choice['oob_weighted_bce']=weighted_bce(y_train,random_forest_tuner.oob_decision_function_[:, 1])\n",
    "    rf_grid_results.append(param_choice)\n",
    "    print('Performed iteration '+str(counter)+' of '+str(num_iter))\n",
    "rf_grid_results=pd.DataFrame(rf_grid_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "ccb5dc4a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>criterion</th>\n",
       "      <th>max_features</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>oob_weighted_bce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>entropy</td>\n",
       "      <td>auto</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>0.003522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>entropy</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.003522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>entropy</td>\n",
       "      <td>auto</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.003522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>entropy</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.003522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>entropy</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.003522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>gini</td>\n",
       "      <td>log2</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.003659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>entropy</td>\n",
       "      <td>log2</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.004700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>entropy</td>\n",
       "      <td>log2</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.004700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gini</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gini</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>gini</td>\n",
       "      <td>auto</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>gini</td>\n",
       "      <td>auto</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>gini</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>gini</td>\n",
       "      <td>auto</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>gini</td>\n",
       "      <td>auto</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>0.004714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   criterion max_features  max_depth  oob_weighted_bce\n",
       "2    entropy         auto  40.000000          0.003522\n",
       "3    entropy         sqrt  30.000000          0.003522\n",
       "6    entropy         auto        nan          0.003522\n",
       "7    entropy         sqrt  30.000000          0.003522\n",
       "12   entropy         sqrt        nan          0.003522\n",
       "10      gini         log2        nan          0.003659\n",
       "1    entropy         log2  30.000000          0.004700\n",
       "5    entropy         log2        nan          0.004700\n",
       "0       gini         sqrt  30.000000          0.004714\n",
       "4       gini         sqrt        nan          0.004714\n",
       "8       gini         auto  30.000000          0.004714\n",
       "9       gini         auto  40.000000          0.004714\n",
       "11      gini         sqrt  40.000000          0.004714\n",
       "13      gini         auto  30.000000          0.004714\n",
       "14      gini         auto  40.000000          0.004714"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid_results.sort_values('oob_weighted_bce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "1a025285",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit random forest using tuned hyperparameters\n",
    "best_rf_params=rf_grid_results.sort_values('oob_weighted_bce').iloc[0,:]\n",
    "tuned_random_forest=RandomForestClassifier(criterion=best_rf_params['criterion'],\n",
    "                                           max_features=best_rf_params['max_features'],\n",
    "                                           max_depth=best_rf_params['max_depth'],\n",
    "                                           n_jobs=-1,\n",
    "                                           random_state=586)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "id": "a2bf08c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   50.0s remaining:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   50.6s finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for tuned random forest\n",
    "tuned_random_forest_cv=model_selection.cross_validate(tuned_random_forest,X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "fbd9adc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 48.84725,\n",
       " 'score_time': 0.49096,\n",
       " 'test_weighted_bce': 0.01769,\n",
       " 'test_accuracy': 0.99987,\n",
       " 'test_recall': 0.99909,\n",
       " 'test_precision': 0.99652}"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in tuned_random_forest_cv.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7e3aed0",
   "metadata": {},
   "source": [
    "## Gradient boosting classifier on first 90 PCs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "31474d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create gradient boosting classifier with default params\n",
    "gradient_boosting=GradientBoostingClassifier(random_state=586)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "9b8e901d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  5.3min remaining:  7.9min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  5.3min finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for gradient boosting classifier\n",
    "gradient_boosting_cv=model_selection.cross_validate(gradient_boosting,X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "2444eb92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 313.38383,\n",
       " 'score_time': 0.28748,\n",
       " 'test_weighted_bce': 0.01884,\n",
       " 'test_accuracy': 0.99987,\n",
       " 'test_recall': 0.99902,\n",
       " 'test_precision': 0.99666}"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in gradient_boosting_cv.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "id": "c548a78d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Tune gradient boosting classifier hyperparameters\n",
    "gb_params={'n_estimators':[100,200,300,400,500],\n",
    "           'subsample':[0.4,0.6,0.8,1],\n",
    "           'max_depth':[4,6,8,10]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "id": "4019b5a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.6min remaining:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.7min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 1 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.3min remaining:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.3min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 2 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  2.2min remaining:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  2.2min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 3 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.1min remaining:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.1min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 4 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.6min remaining:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.6min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 5 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.6min remaining:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.6min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 6 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   27.6s remaining:   41.3s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   28.1s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 7 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.9min remaining:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  2.1min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 8 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   31.6s remaining:   47.4s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   32.9s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 9 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   17.9s remaining:   26.8s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   18.5s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 10 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  2.1min remaining:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  2.1min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 11 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.1min remaining:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.1min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 12 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   23.7s remaining:   35.6s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   23.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 13 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   24.8s remaining:   37.3s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   25.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 14 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  2.1min remaining:  3.1min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 15 of 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  2.1min finished\n"
     ]
    }
   ],
   "source": [
    "# Implement random grid search\n",
    "# Using 10% sample of dataset -- too computationally expensive otherwise\n",
    "counter=0\n",
    "num_iter=15\n",
    "gb_grid_results=[]\n",
    "for i in range(num_iter):\n",
    "    counter+=1\n",
    "    # Select random parameter set\n",
    "    param_choice={key:np.random.choice(val) for key,val in gb_params.items()}\n",
    "    # Define model with param choices\n",
    "    gradient_boosting_tuner=GradientBoostingClassifier(\n",
    "                           n_estimators=param_choice['n_estimators'],\n",
    "                           subsample=param_choice['subsample'],\n",
    "                           max_depth=param_choice['max_depth'],\n",
    "                           learning_rate=10/param_choice['n_estimators'],\n",
    "                           random_state=586)\n",
    "    # Get CV estimate of weighted binary cross entropy loss\n",
    "    gradient_boosting_tuner_cv=model_selection.cross_validate(gradient_boosting_tuner,X=X_train_scores[:int(len(X_train_scores)*0.1),:],y=y_train[:int(len(X_train_scores)*0.1)],scoring=eval_metrics,n_jobs=-1,verbose=1)\n",
    "    param_choice['learning_rate']=10/param_choice['n_estimators']\n",
    "    param_choice['cv_weighted_bce']=np.mean(gradient_boosting_tuner_cv['test_weighted_bce'])\n",
    "    gb_grid_results.append(param_choice)\n",
    "    print('Performed iteration '+str(counter)+' of '+str(num_iter))\n",
    "gb_grid_results=pd.DataFrame(gb_grid_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "id": "b42fb063",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_estimators</th>\n",
       "      <th>subsample</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>cv_weighted_bce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>100</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>8</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.074275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>500</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.086561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>400</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.025000</td>\n",
       "      <td>0.086601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>400</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.025000</td>\n",
       "      <td>0.086601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>200</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.086641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>400</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.025000</td>\n",
       "      <td>0.098169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>100</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.098169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>300</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>8</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.098209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>500</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>8</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.098608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>300</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.098887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.110136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>10</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.110136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>100</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.110136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>100</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>6</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.122104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.122104</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    n_estimators  subsample  max_depth  learning_rate  cv_weighted_bce\n",
       "6            100   0.800000          8       0.100000         0.074275\n",
       "4            500   0.800000          4       0.020000         0.086561\n",
       "0            400   1.000000          4       0.025000         0.086601\n",
       "5            400   1.000000          4       0.025000         0.086601\n",
       "8            200   0.400000         10       0.050000         0.086641\n",
       "7            400   0.800000         10       0.025000         0.098169\n",
       "13           100   1.000000         10       0.100000         0.098169\n",
       "3            300   0.600000          8       0.033333         0.098209\n",
       "11           500   0.400000          8       0.020000         0.098608\n",
       "1            300   1.000000          4       0.033333         0.098887\n",
       "2            500   1.000000         10       0.020000         0.110136\n",
       "10           500   1.000000         10       0.020000         0.110136\n",
       "12           100   1.000000          6       0.100000         0.110136\n",
       "9            100   0.600000          6       0.100000         0.122104\n",
       "14           500   1.000000          8       0.020000         0.122104"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_grid_results.sort_values('cv_weighted_bce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "id": "aaeaec42",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_gb_params=gb_grid_results.sort_values('cv_weighted_bce').iloc[0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "id": "68ffd23b",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_lr_params={'learning_rate':[0.05,0.06,0.07,0.08,0.09,0.1,0.11,0.12,0.13,0.14,0.15]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "2a03306d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   23.1s remaining:   34.7s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   24.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 1 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   24.2s remaining:   36.4s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   26.2s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 2 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   25.3s remaining:   37.9s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   26.1s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 3 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   25.4s remaining:   38.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   26.5s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 4 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   26.4s remaining:   39.6s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   27.7s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 5 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   56.8s remaining:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   57.3s finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 6 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  4.5min remaining:  6.7min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  4.5min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 7 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.5min remaining:  2.2min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 8 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.5min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  4.6min remaining:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  4.6min finished\n",
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 9 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   30.5s remaining:   45.7s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performed iteration 10 of 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   32.0s finished\n"
     ]
    }
   ],
   "source": [
    "# Implement exhaustive grid search to further tune learning rate\n",
    "counter=0\n",
    "gb_lr_grid_results=[]\n",
    "for i in range(len(gb_lr_params['learning_rate'])):\n",
    "    counter+=1\n",
    "    # Select parameter\n",
    "    param_choice={key:val[i] for key,val in gb_lr_params.items()}\n",
    "    # Define model with param choices\n",
    "    gradient_boosting_lr_tuner=GradientBoostingClassifier(\n",
    "                           n_estimators=int(best_gb_params['n_estimators']),\n",
    "                           subsample=best_gb_params['subsample'],\n",
    "                           max_depth=int(best_gb_params['max_depth']),\n",
    "                           learning_rate=param_choice['learning_rate'],\n",
    "                           random_state=586)\n",
    "    # Get CV estimate of weighted binary cross entropy loss\n",
    "    gradient_boosting_lr_tuner_cv=model_selection.cross_validate(gradient_boosting_lr_tuner,X=X_train_scores[:int(len(X_train_scores)*0.1),:],y=y_train[:int(len(X_train_scores)*0.1)],scoring=eval_metrics,n_jobs=-1,verbose=1)\n",
    "    param_choice['cv_weighted_bce']=np.mean(gradient_boosting_lr_tuner_cv['test_weighted_bce'])\n",
    "    gb_lr_grid_results.append(param_choice)\n",
    "    print('Performed iteration '+str(counter)+' of '+str(len(gb_lr_params['learning_rate'])))\n",
    "gb_lr_grid_results=pd.DataFrame(gb_lr_grid_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "id": "fc0adaa3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>cv_weighted_bce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.074275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.074275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.130000</td>\n",
       "      <td>0.074633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.150000</td>\n",
       "      <td>0.074992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.086242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.086242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.090000</td>\n",
       "      <td>0.086601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.098528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.070000</td>\n",
       "      <td>0.098528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.120000</td>\n",
       "      <td>0.098568</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   learning_rate  cv_weighted_bce\n",
       "5       0.100000         0.074275\n",
       "8       0.140000         0.074275\n",
       "7       0.130000         0.074633\n",
       "9       0.150000         0.074992\n",
       "1       0.060000         0.086242\n",
       "3       0.080000         0.086242\n",
       "4       0.090000         0.086601\n",
       "0       0.050000         0.098528\n",
       "2       0.070000         0.098528\n",
       "6       0.120000         0.098568"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_lr_grid_results.sort_values('cv_weighted_bce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "e485fd9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create gradient boosting classifier with tuned params\n",
    "tuned_gradient_boosting=GradientBoostingClassifier(\n",
    "                                                  n_estimators=int(best_gb_params['n_estimators']),\n",
    "                                                  subsample=best_gb_params['subsample'],\n",
    "                                                  max_depth=int(best_gb_params['max_depth']),\n",
    "                                                  learning_rate=gb_lr_grid_results.sort_values('cv_weighted_bce').iloc[0]['learning_rate'],\n",
    "                                                  random_state=586)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "48defe4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  9.9min remaining: 14.9min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 10.1min finished\n"
     ]
    }
   ],
   "source": [
    "# Get 5-fold CV estimates for tuned gradient boosting classifier\n",
    "tuned_gradient_boosting_cv=model_selection.cross_validate(tuned_gradient_boosting,X=X_train_scores,y=y_train,scoring=eval_metrics,n_jobs=-1,verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "4af06889",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'fit_time': 598.12489,\n",
       " 'score_time': 0.29889,\n",
       " 'test_weighted_bce': 0.01628,\n",
       " 'test_accuracy': 0.99988,\n",
       " 'test_recall': 0.99916,\n",
       " 'test_precision': 0.99687}"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get avg error metrics from folds\n",
    "{key:np.round(np.mean(val),5) for key,val in tuned_gradient_boosting_cv.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdc9c224",
   "metadata": {},
   "source": [
    "## Comparison of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "57ffd919",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>CV F1 Score</th>\n",
       "      <th>CV Binary Cross Entropy Loss</th>\n",
       "      <th>CV Accuracy</th>\n",
       "      <th>CV Recall</th>\n",
       "      <th>CV Precision</th>\n",
       "      <th>Avg Fit Time</th>\n",
       "      <th>Avg Score Time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Tuned Gradient Boosting</td>\n",
       "      <td>0.998014</td>\n",
       "      <td>0.016280</td>\n",
       "      <td>0.999880</td>\n",
       "      <td>0.999160</td>\n",
       "      <td>0.996870</td>\n",
       "      <td>598.124890</td>\n",
       "      <td>0.298890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gradient Boosting</td>\n",
       "      <td>0.997839</td>\n",
       "      <td>0.018840</td>\n",
       "      <td>0.999870</td>\n",
       "      <td>0.999020</td>\n",
       "      <td>0.996660</td>\n",
       "      <td>313.383830</td>\n",
       "      <td>0.287480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.997804</td>\n",
       "      <td>0.018870</td>\n",
       "      <td>0.999870</td>\n",
       "      <td>0.999020</td>\n",
       "      <td>0.996590</td>\n",
       "      <td>58.082470</td>\n",
       "      <td>0.324100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Tuned Random Forest</td>\n",
       "      <td>0.997803</td>\n",
       "      <td>0.017690</td>\n",
       "      <td>0.999870</td>\n",
       "      <td>0.999090</td>\n",
       "      <td>0.996520</td>\n",
       "      <td>48.847250</td>\n",
       "      <td>0.490960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tuned KNN</td>\n",
       "      <td>0.997599</td>\n",
       "      <td>0.027390</td>\n",
       "      <td>0.999860</td>\n",
       "      <td>0.998540</td>\n",
       "      <td>0.996660</td>\n",
       "      <td>0.186730</td>\n",
       "      <td>2072.159790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>KNN</td>\n",
       "      <td>0.993185</td>\n",
       "      <td>0.123140</td>\n",
       "      <td>0.999590</td>\n",
       "      <td>0.993180</td>\n",
       "      <td>0.993190</td>\n",
       "      <td>0.017140</td>\n",
       "      <td>24.273790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>0.989705</td>\n",
       "      <td>0.110330</td>\n",
       "      <td>0.999390</td>\n",
       "      <td>0.994140</td>\n",
       "      <td>0.985310</td>\n",
       "      <td>37.739170</td>\n",
       "      <td>0.075110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>QDA</td>\n",
       "      <td>0.942036</td>\n",
       "      <td>0.903340</td>\n",
       "      <td>0.996530</td>\n",
       "      <td>0.950480</td>\n",
       "      <td>0.933740</td>\n",
       "      <td>18.955640</td>\n",
       "      <td>0.354000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Model  CV F1 Score  CV Binary Cross Entropy Loss  \\\n",
       "7  Tuned Gradient Boosting     0.998014                      0.016280   \n",
       "6        Gradient Boosting     0.997839                      0.018840   \n",
       "4            Random Forest     0.997804                      0.018870   \n",
       "5      Tuned Random Forest     0.997803                      0.017690   \n",
       "3                Tuned KNN     0.997599                      0.027390   \n",
       "2                      KNN     0.993185                      0.123140   \n",
       "0      Logistic Regression     0.989705                      0.110330   \n",
       "1                      QDA     0.942036                      0.903340   \n",
       "\n",
       "   CV Accuracy  CV Recall  CV Precision  Avg Fit Time  Avg Score Time  \n",
       "7     0.999880   0.999160      0.996870    598.124890        0.298890  \n",
       "6     0.999870   0.999020      0.996660    313.383830        0.287480  \n",
       "4     0.999870   0.999020      0.996590     58.082470        0.324100  \n",
       "5     0.999870   0.999090      0.996520     48.847250        0.490960  \n",
       "3     0.999860   0.998540      0.996660      0.186730     2072.159790  \n",
       "2     0.999590   0.993180      0.993190      0.017140       24.273790  \n",
       "0     0.999390   0.994140      0.985310     37.739170        0.075110  \n",
       "1     0.996530   0.950480      0.933740     18.955640        0.354000  "
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_results=[]\n",
    "for model in [logistic_regressor_cv,qda_model_cv,knn_model_cv,tuned_knn_model_cv,random_forest_cv,tuned_random_forest_cv,gradient_boosting_cv,tuned_gradient_boosting_cv]:\n",
    "    all_results.append({key:np.round(np.mean(val),5) for key,val in model.items()})\n",
    "all_results_df=pd.DataFrame(all_results)\n",
    "all_results_df['test_f1_score']=(2*all_results_df['test_precision']*all_results_df['test_recall'])/(all_results_df['test_precision']+all_results_df['test_recall'])\n",
    "all_results_df['model']=['Logistic Regression','QDA','KNN','Tuned KNN','Random Forest','Tuned Random Forest','Gradient Boosting','Tuned Gradient Boosting']\n",
    "all_results_df=all_results_df.rename(columns={'fit_time':'Avg Fit Time','score_time':'Avg Score Time','test_weighted_bce':'CV Binary Cross Entropy Loss','test_accuracy':'CV Accuracy','test_recall':'CV Recall','test_precision':'CV Precision','test_f1_score':'CV F1 Score','model':'Model'})\n",
    "all_results_df=all_results_df.sort_values('CV F1 Score',ascending=False)[['Model','CV F1 Score','CV Binary Cross Entropy Loss','CV Accuracy','CV Recall','CV Precision','Avg Fit Time','Avg Score Time']]\n",
    "all_results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3af2a49d",
   "metadata": {},
   "source": [
    "## Evaluate best model (tuned gradient boosting classifier) on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "aedc8208",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define gradient boosting model with tuned hyperparameters\n",
    "tuned_gb=GradientBoostingClassifier(\n",
    "                                   n_estimators=int(best_gb_params['n_estimators']),\n",
    "                                   subsample=best_gb_params['subsample'],\n",
    "                                   max_depth=int(best_gb_params['max_depth']),\n",
    "                                   learning_rate=gb_lr_grid_results.sort_values('cv_weighted_bce').iloc[0]['learning_rate'],\n",
    "                                   random_state=586)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "83b89bc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(max_depth=8, random_state=586, subsample=0.8)"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit model\n",
    "tuned_gb.fit(X_train_scores,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "id": "87c2f38e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Weighted Binary Cross Entropy</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Precision</th>\n",
       "      <th>F1 Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004872</td>\n",
       "      <td>0.999872</td>\n",
       "      <td>0.998401</td>\n",
       "      <td>0.997206</td>\n",
       "      <td>0.997803</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Weighted Binary Cross Entropy  Accuracy   Recall  Precision  F1 Score\n",
       "0                       0.004872  0.999872 0.998401   0.997206  0.997803"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate predictions+probabilities on test set\n",
    "test_preds=tuned_gb.predict(X_test_scores)\n",
    "test_probs=tuned_gb.predict_proba(X_test_scores)\n",
    "\n",
    "# Compute error metrics\n",
    "test_bce=weighted_bce(y_test,test_probs)\n",
    "test_acc=metrics.accuracy_score(y_test,test_preds)\n",
    "test_recall=metrics.recall_score(y_test,test_preds)\n",
    "test_precision=metrics.precision_score(y_test,test_preds)\n",
    "\n",
    "# Compile results into DataFrame\n",
    "test_results=pd.DataFrame({'Weighted Binary Cross Entropy':test_bce,'Accuracy':test_acc,'Recall':test_recall,'Precision':test_precision},index=[0])\n",
    "test_results['F1 Score']=(2*test_results['Precision']*test_results['Recall'])/(test_results['Precision']+test_results['Recall'])\n",
    "test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5d42f22",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}