Importance of feature selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importance of Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook I aim to study the importance of feature selection while building a Deep Learning model. I hypothesize that a model would be able to find out the most importance features. So, we can throw all variables in the deep learning models and its performance will not be deteriorated. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I will check and compare the performance of models built using all features."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Required Libraries and Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#to plot the data\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams.update(mpl.rcParamsDefault)\n",
    "\n",
    "import time\n",
    "import os\n",
    "os.chdir(\"C:/Data/aircraft/\")\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler #to normalize data\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_curve\n",
    "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score, precision_score, recall_score\n",
    "\n",
    "#for deep learning\n",
    "import keras\n",
    "import keras.backend as k\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, LSTM, Activation, Masking, Dropout\n",
    "from keras.optimizers import RMSprop,adam\n",
    "from keras.callbacks import History\n",
    "from keras import callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data(drop_cols = True):\n",
    "    dependent_var = ['RUL']\n",
    "    index_columns_names =  [\"UnitNumber\",\"Cycle\"]\n",
    "    operational_settings_columns_names = [\"OpSet\"+str(i) for i in range(1,4)]\n",
    "    sensor_measure_columns_names =[\"SensorMeasure\"+str(i) for i in range(1,22)]\n",
    "    input_file_column_names = index_columns_names + operational_settings_columns_names + sensor_measure_columns_names\n",
    "\n",
    "    cols_to_drop = ['OpSet3', 'SensorMeasure1', 'SensorMeasure5', 'SensorMeasure6', 'SensorMeasure10', 'SensorMeasure14',\n",
    "     'SensorMeasure16', 'SensorMeasure18', 'SensorMeasure19']\n",
    "\n",
    "    df_train = pd.read_csv('train_FD001.txt',delim_whitespace=True,names=input_file_column_names)\n",
    "\n",
    "    rul = pd.DataFrame(df_train.groupby('UnitNumber')['Cycle'].max()).reset_index()\n",
    "    rul.columns = ['UnitNumber', 'max']\n",
    "    df_train = df_train.merge(rul, on=['UnitNumber'], how='left')\n",
    "    df_train['RUL'] = df_train['max'] - df_train['Cycle']\n",
    "    df_train.drop('max', axis=1, inplace=True)\n",
    "\n",
    "    df_test = pd.read_csv('test_FD001.txt', delim_whitespace=True, names=input_file_column_names)\n",
    "    \n",
    "    if(drop_cols == True):\n",
    "        df_train = df_train.drop(cols_to_drop, axis = 1)\n",
    "        df_test = df_test.drop(cols_to_drop, axis = 1)\n",
    "\n",
    "    y_true = pd.read_csv('RUL_FD001.txt', delim_whitespace=True,names=[\"RUL\"])\n",
    "    y_true[\"UnitNumber\"] = y_true.index\n",
    "    \n",
    "    return df_train, df_test, y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((20631, 27), (13096, 26), (100, 2))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train, df_test, y_true = prepare_data(drop_cols=False)\n",
    "df_train.shape, df_test.shape, y_true.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = df_train.columns.drop(['UnitNumber', 'Cycle', 'RUL'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaler = MinMaxScaler(feature_range=(-1,1))\n",
    "\n",
    "df_train[feats] = min_max_scaler.fit_transform(df_train[feats])\n",
    "df_test[feats] = min_max_scaler.transform(df_test[feats])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['failure'] = [1 if i < 50 else 0 for i in df_train.RUL]\n",
    "y_true['failure'] = [1 if i < 50 else 0 for i in y_true.RUL]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UnitNumber</th>\n",
       "      <th>Cycle</th>\n",
       "      <th>OpSet1</th>\n",
       "      <th>OpSet2</th>\n",
       "      <th>OpSet3</th>\n",
       "      <th>SensorMeasure1</th>\n",
       "      <th>SensorMeasure2</th>\n",
       "      <th>SensorMeasure3</th>\n",
       "      <th>SensorMeasure4</th>\n",
       "      <th>SensorMeasure5</th>\n",
       "      <th>...</th>\n",
       "      <th>SensorMeasure14</th>\n",
       "      <th>SensorMeasure15</th>\n",
       "      <th>SensorMeasure16</th>\n",
       "      <th>SensorMeasure17</th>\n",
       "      <th>SensorMeasure18</th>\n",
       "      <th>SensorMeasure19</th>\n",
       "      <th>SensorMeasure20</th>\n",
       "      <th>SensorMeasure21</th>\n",
       "      <th>RUL</th>\n",
       "      <th>failure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.080460</td>\n",
       "      <td>-0.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.632530</td>\n",
       "      <td>-0.186396</td>\n",
       "      <td>-0.380486</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.600784</td>\n",
       "      <td>-0.272028</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.426357</td>\n",
       "      <td>0.449323</td>\n",
       "      <td>191</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.218391</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.433735</td>\n",
       "      <td>-0.093961</td>\n",
       "      <td>-0.294733</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.674373</td>\n",
       "      <td>-0.177376</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.462027</td>\n",
       "      <td>190</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.494253</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.313253</td>\n",
       "      <td>-0.260955</td>\n",
       "      <td>-0.258947</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.656414</td>\n",
       "      <td>-0.285110</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.255814</td>\n",
       "      <td>0.242751</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.080460</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.313253</td>\n",
       "      <td>-0.487683</td>\n",
       "      <td>-0.337610</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.650222</td>\n",
       "      <td>-0.666795</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.147287</td>\n",
       "      <td>0.324772</td>\n",
       "      <td>188</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.218391</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.301205</td>\n",
       "      <td>-0.485066</td>\n",
       "      <td>-0.190749</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.650532</td>\n",
       "      <td>-0.195845</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.178295</td>\n",
       "      <td>0.409003</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   UnitNumber  Cycle    OpSet1    OpSet2  OpSet3  SensorMeasure1  \\\n",
       "0           1      1 -0.080460 -0.666667    -1.0            -1.0   \n",
       "1           1      2  0.218391 -0.500000    -1.0            -1.0   \n",
       "2           1      3 -0.494253  0.500000    -1.0            -1.0   \n",
       "3           1      4  0.080460  0.000000    -1.0            -1.0   \n",
       "4           1      5 -0.218391 -0.333333    -1.0            -1.0   \n",
       "\n",
       "   SensorMeasure2  SensorMeasure3  SensorMeasure4  SensorMeasure5  ...  \\\n",
       "0       -0.632530       -0.186396       -0.380486            -1.0  ...   \n",
       "1       -0.433735       -0.093961       -0.294733            -1.0  ...   \n",
       "2       -0.313253       -0.260955       -0.258947            -1.0  ...   \n",
       "3       -0.313253       -0.487683       -0.337610            -1.0  ...   \n",
       "4       -0.301205       -0.485066       -0.190749            -1.0  ...   \n",
       "\n",
       "   SensorMeasure14  SensorMeasure15  SensorMeasure16  SensorMeasure17  \\\n",
       "0        -0.600784        -0.272028             -1.0        -0.333333   \n",
       "1        -0.674373        -0.177376             -1.0        -0.333333   \n",
       "2        -0.656414        -0.285110             -1.0        -0.666667   \n",
       "3        -0.650222        -0.666795             -1.0        -0.333333   \n",
       "4        -0.650532        -0.195845             -1.0        -0.166667   \n",
       "\n",
       "   SensorMeasure18  SensorMeasure19  SensorMeasure20  SensorMeasure21  RUL  \\\n",
       "0             -1.0             -1.0         0.426357         0.449323  191   \n",
       "1             -1.0             -1.0         0.333333         0.462027  190   \n",
       "2             -1.0             -1.0         0.255814         0.242751  189   \n",
       "3             -1.0             -1.0         0.147287         0.324772  188   \n",
       "4             -1.0             -1.0         0.178295         0.409003  187   \n",
       "\n",
       "   failure  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UnitNumber</th>\n",
       "      <th>Cycle</th>\n",
       "      <th>OpSet1</th>\n",
       "      <th>OpSet2</th>\n",
       "      <th>OpSet3</th>\n",
       "      <th>SensorMeasure1</th>\n",
       "      <th>SensorMeasure2</th>\n",
       "      <th>SensorMeasure3</th>\n",
       "      <th>SensorMeasure4</th>\n",
       "      <th>SensorMeasure5</th>\n",
       "      <th>...</th>\n",
       "      <th>SensorMeasure12</th>\n",
       "      <th>SensorMeasure13</th>\n",
       "      <th>SensorMeasure14</th>\n",
       "      <th>SensorMeasure15</th>\n",
       "      <th>SensorMeasure16</th>\n",
       "      <th>SensorMeasure17</th>\n",
       "      <th>SensorMeasure18</th>\n",
       "      <th>SensorMeasure19</th>\n",
       "      <th>SensorMeasure20</th>\n",
       "      <th>SensorMeasure21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.264368</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.090361</td>\n",
       "      <td>-0.378679</td>\n",
       "      <td>-0.461175</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.292111</td>\n",
       "      <td>-0.558824</td>\n",
       "      <td>-0.735680</td>\n",
       "      <td>-0.382070</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.116279</td>\n",
       "      <td>0.323667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.310345</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.698795</td>\n",
       "      <td>-0.240898</td>\n",
       "      <td>-0.555368</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.479744</td>\n",
       "      <td>-0.470588</td>\n",
       "      <td>-0.590463</td>\n",
       "      <td>-0.573682</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.364341</td>\n",
       "      <td>0.373654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.246988</td>\n",
       "      <td>-0.306736</td>\n",
       "      <td>-0.355503</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.398721</td>\n",
       "      <td>-0.558824</td>\n",
       "      <td>-0.688719</td>\n",
       "      <td>-0.082724</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.166667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.457364</td>\n",
       "      <td>0.442695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.482759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.259036</td>\n",
       "      <td>-0.429693</td>\n",
       "      <td>-0.183997</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.147122</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.659820</td>\n",
       "      <td>-0.485956</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.324220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.160920</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.216867</td>\n",
       "      <td>-0.295836</td>\n",
       "      <td>-0.335922</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.475480</td>\n",
       "      <td>-0.558824</td>\n",
       "      <td>-0.694499</td>\n",
       "      <td>-0.398230</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.317829</td>\n",
       "      <td>0.432753</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   UnitNumber  Cycle    OpSet1    OpSet2  OpSet3  SensorMeasure1  \\\n",
       "0           1      1  0.264368  0.500000    -1.0            -1.0   \n",
       "1           1      2 -0.310345 -0.500000    -1.0            -1.0   \n",
       "2           1      3  0.034483  0.166667    -1.0            -1.0   \n",
       "3           1      4  0.482759  0.000000    -1.0            -1.0   \n",
       "4           1      5  0.160920  0.000000    -1.0            -1.0   \n",
       "\n",
       "   SensorMeasure2  SensorMeasure3  SensorMeasure4  SensorMeasure5  ...  \\\n",
       "0        0.090361       -0.378679       -0.461175            -1.0  ...   \n",
       "1       -0.698795       -0.240898       -0.555368            -1.0  ...   \n",
       "2       -0.246988       -0.306736       -0.355503            -1.0  ...   \n",
       "3       -0.259036       -0.429693       -0.183997            -1.0  ...   \n",
       "4       -0.216867       -0.295836       -0.335922            -1.0  ...   \n",
       "\n",
       "   SensorMeasure12  SensorMeasure13  SensorMeasure14  SensorMeasure15  \\\n",
       "0         0.292111        -0.558824        -0.735680        -0.382070   \n",
       "1         0.479744        -0.470588        -0.590463        -0.573682   \n",
       "2         0.398721        -0.558824        -0.688719        -0.082724   \n",
       "3         0.147122        -0.500000        -0.659820        -0.485956   \n",
       "4         0.475480        -0.558824        -0.694499        -0.398230   \n",
       "\n",
       "   SensorMeasure16  SensorMeasure17  SensorMeasure18  SensorMeasure19  \\\n",
       "0             -1.0        -0.333333             -1.0             -1.0   \n",
       "1             -1.0        -0.166667             -1.0             -1.0   \n",
       "2             -1.0        -0.166667             -1.0             -1.0   \n",
       "3             -1.0        -0.500000             -1.0             -1.0   \n",
       "4             -1.0        -0.666667             -1.0             -1.0   \n",
       "\n",
       "   SensorMeasure20  SensorMeasure21  \n",
       "0         0.116279         0.323667  \n",
       "1         0.364341         0.373654  \n",
       "2         0.457364         0.442695  \n",
       "3         0.333333         0.324220  \n",
       "4         0.317829         0.432753  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RUL</th>\n",
       "      <th>UnitNumber</th>\n",
       "      <th>failure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>98</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>69</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>82</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>91</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RUL  UnitNumber  failure\n",
       "0  112           0        0\n",
       "1   98           1        0\n",
       "2   69           2        0\n",
       "3   82           3        0\n",
       "4   91           4        0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "LSTM expects an input in the shape of a numpy array of 3 dimensions and I will need to convert train and test data accordingly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_train(id_df, seq_length, seq_cols):\n",
    "    \"\"\"\n",
    "        function to prepare train data into (samples, time steps, features)\n",
    "        id_df = train dataframe\n",
    "        seq_length = look back period\n",
    "        seq_cols = feature columns\n",
    "    \"\"\"\n",
    "        \n",
    "    data_array = id_df[seq_cols].values\n",
    "    num_elements = data_array.shape[0]\n",
    "    lstm_array=[]\n",
    "    \n",
    "    for start, stop in zip(range(0, num_elements-seq_length+1), range(seq_length, num_elements+1)):\n",
    "        lstm_array.append(data_array[start:stop, :])\n",
    "    \n",
    "    return np.array(lstm_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_target(id_df, seq_length, label):\n",
    "    data_array = id_df[label].values\n",
    "    num_elements = data_array.shape[0]\n",
    "    return data_array[seq_length-1:num_elements+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_test(id_df, seq_length, seq_cols, mask_value):\n",
    "    \"\"\"\n",
    "        function to prepare test data into (samples, time steps, features)\n",
    "        function only returns last sequence of data for every unit\n",
    "        id_df = test dataframe\n",
    "        seq_length = look back period\n",
    "        seq_cols = feature columns\n",
    "    \"\"\"\n",
    "    df_mask = pd.DataFrame(np.zeros((seq_length-1,id_df.shape[1])),columns=id_df.columns)\n",
    "    df_mask[:] = mask_value\n",
    "    \n",
    "    id_df = df_mask.append(id_df,ignore_index=True)\n",
    "    \n",
    "    data_array = id_df[seq_cols].values\n",
    "    num_elements = data_array.shape[0]\n",
    "    lstm_array=[]\n",
    "    \n",
    "    start = num_elements-seq_length\n",
    "    stop = num_elements\n",
    "    \n",
    "    lstm_array.append(data_array[start:stop, :])\n",
    "    \n",
    "    return np.array(lstm_array)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Let's define look back period and mask_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_length = 50 #predicting using last 30 cycle values\n",
    "mask_value = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Let's prepare data using above functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15731, 50, 24)\n"
     ]
    }
   ],
   "source": [
    "#generate train\n",
    "x_train=np.concatenate(list(list(gen_train(df_train[df_train['UnitNumber']==unit], sequence_length, feats)) for unit in df_train['UnitNumber'].unique()))\n",
    "print(x_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15731,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#generate target of train\n",
    "y_train = np.concatenate(list(list(gen_target(df_train[df_train['UnitNumber']==unit], sequence_length, \"failure\")) for unit in df_train['UnitNumber'].unique()))\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 50, 24)\n"
     ]
    }
   ],
   "source": [
    "#generate test\n",
    "x_test=np.concatenate(list(list(gen_test(df_test[df_test['UnitNumber']==unit], sequence_length, feats, mask_value)) for unit in df_test['UnitNumber'].unique()))\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#true target of test \n",
    "y_test = y_true.RUL.values\n",
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb_features = x_train.shape[2]\n",
    "nb_out = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_0 = pd.Series(y_train).value_counts()[0]\n",
    "class_1 = pd.Series(y_train).value_counts()[1]\n",
    "total = class_0 + class_1\n",
    "\n",
    "cls_wt = {0: class_1/total, 1: class_0/total}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.3178437480134766, 1: 0.6821562519865234}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_wt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_exp_model(X, y, X_test, y_test, nruns = 10):\n",
    "    \n",
    "    \"\"\"\n",
    "    Parameters:\n",
    "    X, y = training pandas dataframe\n",
    "    X_test, y_test = testing pandas dataframe\n",
    "    nruns = number of time the models should rn\n",
    "    \n",
    "    return:\n",
    "    Dataframe with Model Run, Accuracy, Precision, Recall, F1 Score, AUC Score\n",
    "    \"\"\"\n",
    "    \n",
    "    score_df = pd.DataFrame(columns=[\"Model\", \"Accuracy\", \"Precision\", \"Recall\", \"F1 Score\", \"AUC Score\"])\n",
    "\n",
    "    for i in range(0, nruns):\n",
    "        start = time.time()\n",
    "\n",
    "        model = Sequential()\n",
    "        model.add(LSTM(\n",
    "                 units=8,\n",
    "                 return_sequences=True,\n",
    "                 input_shape=(sequence_length, nb_features)))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(LSTM(\n",
    "                  units=4,\n",
    "                  return_sequences=False))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(Dense(units=1, activation='sigmoid'))\n",
    "\n",
    "        model.compile(loss='binary_crossentropy', optimizer = \"adam\", metrics=[\"accuracy\"])\n",
    "\n",
    "        model.fit(X, y, epochs=100, batch_size=64, validation_split = 0.1, verbose=0, class_weight=cls_wt,\n",
    "              shuffle= True ,callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=10, \n",
    "                                                                  verbose=0, mode='auto')])\n",
    "\n",
    "        predictions = model.predict_classes(X_test)\n",
    "\n",
    "        acc = accuracy_score(y_test,predictions)\n",
    "        pr = precision_score(y_test,predictions)\n",
    "        rc = recall_score(y_test,predictions)\n",
    "        f1 = f1_score(y_test,predictions)\n",
    "        auc = roc_auc_score(y_test,predictions)\n",
    "\n",
    "        score_df = score_df.append({'Model':i + 1, 'Accuracy': acc, 'Precision': pr, 'Recall': rc,\n",
    "                                    'F1 Score': f1, 'AUC Score': auc}, ignore_index=True)\n",
    "        end = time.time()\n",
    "        print(\"Run:\", i+1, \" and Runtime:\", np.round(end - start, 3), \" Seconds\", sep = \"\")        \n",
    "    \n",
    "    return score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run:1 and Runtime:292.915 Seconds\n",
      "Run:2 and Runtime:249.298 Seconds\n",
      "Run:3 and Runtime:252.413 Seconds\n",
      "Run:4 and Runtime:286.292 Seconds\n",
      "Run:5 and Runtime:1440.235 Seconds\n",
      "Run:6 and Runtime:224.546 Seconds\n",
      "Run:7 and Runtime:226.172 Seconds\n",
      "Run:8 and Runtime:282.199 Seconds\n",
      "Run:9 and Runtime:5506.732 Seconds\n",
      "Run:10 and Runtime:574.399 Seconds\n"
     ]
    }
   ],
   "source": [
    "result_df = run_exp_model(x_train, y_train, x_test, y_true.failure, nruns = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>AUC Score</th>\n",
       "      <th>F1 Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.985714</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.976190</td>\n",
       "      <td>0.966667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978571</td>\n",
       "      <td>0.952381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.937500</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.985714</td>\n",
       "      <td>0.967742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.923077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978571</td>\n",
       "      <td>0.952381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.923077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.0</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.923077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0.97</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978571</td>\n",
       "      <td>0.952381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.0</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.878788</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.954762</td>\n",
       "      <td>0.920635</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Model  Accuracy  Precision    Recall  AUC Score  F1 Score\n",
       "0    1.0      0.98   0.937500  1.000000   0.985714  0.967742\n",
       "1    2.0      0.98   0.966667  0.966667   0.976190  0.966667\n",
       "2    3.0      0.97   0.909091  1.000000   0.978571  0.952381\n",
       "3    4.0      0.98   0.937500  1.000000   0.985714  0.967742\n",
       "4    5.0      0.95   0.857143  1.000000   0.964286  0.923077\n",
       "5    6.0      0.97   0.909091  1.000000   0.978571  0.952381\n",
       "6    7.0      0.95   0.857143  1.000000   0.964286  0.923077\n",
       "7    8.0      0.95   0.857143  1.000000   0.964286  0.923077\n",
       "8    9.0      0.97   0.909091  1.000000   0.978571  0.952381\n",
       "9   10.0      0.95   0.878788  0.966667   0.954762  0.920635"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Value')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAF1CAYAAADvMUN6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYJXV97/H3h30bAWFEYXAGFZWJ4JIRuVEDkUhQI4ioATcwBsJNQL1qZFyiCCKQmM2VC4qAekUkmotCROQCLgFlBAEBRyYEnBHRUQQBCev3/lHVcjjTPdPd00VP17xfz3OerqpfLb/zO1X1qa3PSVUhSZL6a53proAkSeqWYS9JUs8Z9pIk9ZxhL0lSzxn2kiT1nGEvSVLPGfarIcmJSf52uusxHkkuSvIXHS/jziRPaLs3TvKVJLcn+WKS1yT5egfLfH6SxVM9364kOTjJt8c57qlJPtB1ndplPSXJFUnuSPKmR2KZUpK9kywZ6L8lyfPGGHezJP+e5DdJPvPI1bIfDPuVSHJjkrvbEPt1knOSbD9SXlWHVdUx01nHEUk2SHJUkuuT3NXW/ZQk8x6pOlTVZlV1Q9v7CmAbYKuqemVVfa6q9lrdZSSpJE8aWOa3quopqzvfUZYzr13W5UPDt05yb5Ibp3qZE9EeNDzQrpu/SfKDJH+6GrN8B3BRVc2qqg9PVT37Lsnj289g5FXt9jfS//zVmPeYwdeW753kwYFlLU3y+STPnMAyjk/yycnWcQLLOaxtm31WYzYHApsBW1bV61azPo/I+16TGPar9tKq2gx4HPBz4CNdLzDJepOY7CxgH+DVwObA04HvA3tOYdUmYi7w46q6f5qWP1U2TfK0gf5XA/81XZUZckm7bm4BfAo4M8mjJzKDgXVtLnDNZCoxyfW1F6rqJ+1B7mbtZwHw9IFh3+q4Cje0y30U8Ac06+Z/rM5BRkcOAm5t/07WXGBxVT0wNVWavBm5zleVrzFewI3AHw/0v5gmwEb6TwU+0HbvASwD3gb8AvgZ8IaBcV8CXAH8BlgKHDVQNg8o4I3AT4BvAucARwzV5yrgZaPU84+Bu4HtV/JeLgL+ou1+IvD/gF8BvwQ+B2wxMO5C4D+BO4Brgf0Gyp4EXAzc3k77hYGyasvfD9wL3Afc2b6vg4FvD4z7e8D5NDuAnwPvaofvClwC3Na24UeBDdqyb7bLuKud75+NtPvAfHdq3+ttNOG1z9Dn9bG2be8Avgs8cYz2GvlM3gP8/cDwRcC7gRvHucytgLPbz/17wDFD7fDUgXZYDLxqtPVrlPoNt+embX0XtP1/CvygrdN/ALsMrddH0qxP97TrwgPAf7ft+mSaA8bTgeXATW07rDOw7O8A/9TW+wNDw24DbqAJn4Np1vdfAAdNcHs4iGZ7+CXw7oHydYF38dA6+n3adX9l7TlKG27bfja3AkuAQwbKjgLObNvgjvZzXTCOfUYBTxoatjHwz+37vIXmhGHDtuyxwNfaNvsV8P/a4V8EHgR+234mbxplWXsDS0YZ/smhdeMTNPumkXVwt3b4y3j4dvq9dvhfAj9q3/cS4M9Xcz/65Pa9vKJd3x491nto2+d5o8zjhKG6vmagrovbz/AcYLvVeN8PWzZwPPDJgfXqfuCQ9nP8ejv8+TT7kduAy4HnDkx/CM22dgfN9vDK1WnH1X1N24JnwouBsAc2AU4DTh8oP5WHh/39wNHA+jQHBr+lueQ0Ur4zzdWUXWgC7mVt2TyancTpNDvtjYFXAd8dWNbTaXYGG4xSz+OBi1fxXi7iobB/EvBCYENgNk2I/vPAuK+k2RGuQxOodwGPa8s+TxN26wAbDW0cv9vR0ewsPztQdjDtDgiYRRPkb2vnMQt4Tlv2+8BuwHptu1wHvGW0ZQy067K2e32andO7gA2AF7Qb2lMGPq9baQ4o1qM5yDljjPYa+Uzm0Wzc69KE+mKag6sbx7nMM2hCY1PgacBPB9ph03beb2jr8yyaYPu94fVrlPoNtud6wJvb5W7ezucXwHPaeh9Esy6PBMyNNAcC2wMbD68fbf/pwP9tP5t5wI+BNw4s+37giHbZGw8Me0O7zA/QBPXHaNazvdr6bTaB7eHkdt5PpwmJndryvwGuBp4CpC3falXtOUobXgx8nGYdfAbNgc2eA+vvf9Nsx+sCxwGXjmOfMVrYn0hz5W2L9vM5D3hfW/ZPwL+09d0A+MOB6UYNvoHyscL+xe1nsX7b/3pgS5p19d1tG42U/S7QBqbfB9ihbduRE4mx2nBHmqB7zErqeSzwzbb7x8BfjfUeVvaeh+sKHECzf3hy+94+AFw4UD7R972qsC+aA6lN2vVyHs0++Y9p1uMXt+vQlu3rNtqTCWA72vV3ul7TtuCZ8KLZKd7Zfmj3AzcDOw+Un8rDw/5uYL2B8l/QHk2OMu9/Bv6p7Z7XrkhPGCjfkCaYdmz7PwR8fIx5ncwYoTUwzkUM7MyHyl4GXLGSaX8A7Nt2nw6cBMwZZbzxhv2BK1ve0DzfAnx5tGUMtPtI2D+/3WDXGSj/PO1ZY/t5De4sXgz8aIzljnwm6wHfAP6k3fjfzcPDfsxl0oTEfcBTB8o+ONAOfwZ8a2i5/5uHguB369co9Tu4XSdvowm0S3nowPQTwDFD4y8Gdh9Yr/98qPx360db73uA+QPlf0lzT39k2T8ZpT7XD/Tv3LbfNgPDfgU8YwLbw5yB8u8BBwy8l31HmcdK23No+PY0VzNmDQw7Djh1YP39xkDZfODucayvw+vnejRnkYNnnH8EXNd2/x3NWfwTRpnXZMP+GW09thqlLDQnISMHoyuE3ijTfA34y/Fsr6NMuw7NQd9hbf/7efhJzOqE/YW0Z/ht//o029s2o0y7yvc9vGxGD/ttB8rfB5w8NI+L2/VwJOz3BTaaTNtN9ct79qv2sqragiZ8DwcuTvLYMcb9VT38HvVvaR4oIclzklyYZHmS24HDgK2Hpl860lFV99CcEb42yTo0ATnWE6i/onmmYFySPCbJGUl+muQ3wGcH65Lk9e0DX7cluY3mjHSk/B00G873klyT5M/Hu9wB29Ncgh2tbk9O8tX24aTf0ITjcDuNZVtgaVU9ODDsJpqj6hG3DHT/7vNZhdNpwuxAmrYa7zJn0+zslw6VjZgLPGekndu2fg3Npd3xuLSqtqiqratqt6r6xsB83zY03+3buo5YusLcHrI1zVnmYF2H23G06X8+0H03QFUND5vI9jDWZzXW+jOR9twWuLWq7ljJexxe/kaTuFe7LU0IXTNQp38DHtOWH0tzEnFhkiVJ3jrB+Y9mO5oDmd8AJHlnksVtO/+a5krGmNtUkn2SfC/JrW19X7Cy8Vfhj2ja/4tt/+eAXZM8dZLzGzQXOHGgXZfTHADPgYm/73F4sKpuHlr+a4fWtwU0BwS/pln33gTckuTswQeLp4NhP05V9UBVfYlmIxrzCdmV+D809we3r6rNaS7tZXgxQ/2n0awwewK/rapLxpj3N2g2oDnjrMtx7bJ2qapHAa8dqUuSuTRXCg6nOTPYAvjhSHlV3VJVh1TVtjRnex+fxEq8lOa5gdF8guZ+4Y5t3d7Fiu00lpuB7duDoxGPp7l0vjr+leYe8w1VddNQ2cqWObLz2X6obMRSmtsvWwy8Nquq/7ma9V0KHDs0302q6vMD4wyva4N+SXOGNHeo3oPtuLLpx2M828NYxlp/JtKeNwOPTjJrYNhUrCvDfkazDjxxoE6bV9VWAFV1e1W9uarmAvsD70ny3HbaybbxfjQHgvcleSHN7Zb9aG4jPJrmoGukrR+2jCSb0gTzMTSX5regeaZjvJ/NsINocuaaJLfw0HM3r5/k/AYtBQ4e+rw3rqrvT/R9t+6iuUQ/YvggcXiapTRn/oPL37Sq/gmgqs6pqj1pDvh+QrNvmzaG/TilsS/N5ZnrJjGLWTRnEv+dZFeap7pXqg33B4F/YOyzetozuvOBLyf5/STrJZnV/rvLaGfes2hvTyTZjuYe6IiRB72WAyR5A82ZPW3/KwcOKn7djjvRp2O/Cjw2yVuSbNjW9TkDdfsNcGd79D+8o/458IQx5vtdmg32HUnWT7IH8FKa++aTVlV30ZzdjPY9BWMus5qnhr8EHJVkkyTzefjTyF8Fnpzkde206yd5dpKdVqe+NAdrh7Vnz0myaZKXDAXbmNp6nwkc2342c4G3suJVjdUx4e1hwCeBY5Ls2L6/XZJsxQTas6qW0jy4eFySjZLsQvMg6edW/609bDn3AacA/5Lm3zaTZPs2jEbOondIEpqHXh/goe1pZev6w7TznZPkGJqD93e3RbNoDtyW01ytOZrmDHfEz4GR5UNzL3p9mluQD6b5V7k9JvHWSbIZ8HKaq2LPGHi9HXjd0AHyZJxIc3D0lHZ5WybZvy2b6PuG5nblge3+czeaS/ArcxrwyiR7Jlk3zXeL7JnksUm2a7e5TWhuid3JxPeTU8qwX7WvJLmTJoCOpXmieDL/ovRXwNFJ7gDeS7MzHY/Tae5/rmpH+wrgXOALNDuNH9JcUvrGKOO+n+bhpdtpnmD90khBVV1Lc3BxCc0GsTPNU9Yjng18t22Ts4E3V9V/jfO9jCzjDpoHBF9Kc6n0eprLfdDsCF5N8zDXye37GXQUcFp72exVQ/O9l+bhohfRnJ1+HHh9Vf1oIvUbo86LqmqFS8fjWObhNJefb6G5B//pgWnvoHlw7QCaM81baJ463nB160rzJPBHaQ7IltDscCfiCJqDmBuAb9OciZ+yOvUaMtntAeAf2/G/TrNdformQcOJtueBNM8H3Ax8mebe/vkTfier9pZ2GYtotrmv0TwkCw/9J8cdNGe9H6qqS9uyY2kOuG5LcvgY835Cuy3eSXPg+RSa+84Xt+Vfaef7nzSf5S9pD+RbZ9Cczd6a5D+q6pc02+BXaG4PvoxmvzKqNLfd7kzymFGKX0Hz3NHn2yuCt1TVLTTP/DyK5gB60torVR8FvpTmlt8PaPYrMMH33Q57F83+7jbgnaziJKGa7xTZn2Z/+kua20BvpsnVddt53ELTjs+m2aamTdqHCrSGSvJ64NCqmsytA0mSPLNfk7WXgP6K5khYkqRJMezXUEn+hOay089pLqFKkjQpXsaXJKnnPLOXJKnnDHtJknpu5v1yzxi23nrrmjdv3nRXQ5KkR8z3v//9X1bV7FWN15uwnzdvHosWLZruakiS9IhJMvytnqPyMr4kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPddZ2Cc5JckvkvxwjPIk+XCSJUmuSvKsgbKDklzfvg7qqo6SJK0NujyzPxXYeyXlLwJ2bF+HAp8ASPJo4H3Ac4Bdgfcl2bLDekqS1GudhX1VfRO4dSWj7AucXo1LgS2SPA74E+D8qrq1qn4NnM/KDxokSdJKTOcP4WwHLB3oX9YOG2v4CpIcSnNVgMc//vHd1FLqiZ1P23m6qzApVx909XRXQZrxpjPsM8qwWsnwFQdWnQScBLBgwYJRx5HUMDSltdd0Po2/DNh+oH8OcPNKhkuSpEmYzrA/G3h9+1T+bsDtVfUz4DxgryRbtg/m7dUOkyRJk9DZZfwknwf2ALZOsozmCfv1AarqROBc4MXAEuC3wBvasluTHANc1s7q6Kpa2YN+kiRpJToL+6o6cBXlBfz1GGWnAKd0US9JktY2foOeJEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc+tNdwUkaW2VpLN5V1Vn89bM45m9JE2Tqhr3a+6RX53Q+NIgw16SpJ4z7CVJ6jnDXpKknjPsJUnqOcNekqSeM+wlSeo5w16SpJ7rNOyT7J1kcZIlSRaOUj43yQVJrkpyUZI5A2V/l+SaJNcl+XC6/PYJSZJ6rLOwT7Iu8DHgRcB84MAk84dG+xBwelXtAhwNHNdO+wfAc4FdgKcBzwZ276qukiT1WZdn9rsCS6rqhqq6FzgD2HdonPnABW33hQPlBWwEbABsCKwP/LzDukqS1Ftdhv12wNKB/mXtsEFXAvu33fsBs5JsVVWX0IT/z9rXeVV13fACkhyaZFGSRcuXL5/yNzAZSTp7SZI0GV2G/WjpNPyFzW8Hdk9yBc1l+p8C9yd5ErATMIfmAOEFSf5whZlVnVRVC6pqwezZs6e29pPkd11LktY0Xf7q3TJg+4H+OcDNgyNU1c3AywGSbAbsX1W3JzkUuLSq7mzL/h3YDfhmh/WVJKmXugz7y4Adk+xAc8Z+APDqwRGSbA3cWlUPAu8ETmmLfgIckuQ4misEuwP/3GFdJWlKPP39X+f2u+/rZN7zFp4z5fPcfOP1ufJ9e035fLVm6Szsq+r+JIcD5wHrAqdU1TVJjgYWVdXZwB7AcUmK5qz9r9vJzwJeAFxNc+n/a1X1la7qKklT5fa77+PG418y3dUYty4OILTmSV/uBS9YsKAWLVrUyby7PFLvgkfq0vTZ+bSdp7sKE3b1QVdPdxU0SUm+X1ULVjVel5fxe8MjdUnjdcd1x7u/0BrHr8uVJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOn7gdh1k7LWTn0xZOdzXGbdZOADPnJzYlSd0y7MfB36eWJM1kXsaXJKnnDHtJknrOsJc0aUcccQQbbbQRSdhoo4044ogjprtK0sMk6ew1kxj2kibliCOO4MQTT+SDH/wgd911Fx/84Ac58cQTDXytUapq3K+5R351QuPPJIa9pEk5+eSTOeGEE3jrW9/KJptswlvf+lZOOOEETj755OmumqQhhr1mnC4vy820S3PT6Z577uGwww572LDDDjuMe+65Z5pqJGkshr1mnC4vy820S3PTacMNN+TEE0982LATTzyRDTfccJpqJGks/p+9pEk55JBDOPLII4HmjP7EE0/kyCOPXOFsX9L0M+wlTcpHPvIRAN71rnfxtre9jQ033JDDDjvsd8MlrTkMe0mT9pGPfMRwl2YAw16SpthM+srqzTdef7qroEeAYS9JU6ir39GYt/CcGfUbHVqzGPaSpBnl6e//OrfffV8n8+7qqszmG6/Ple/bq5N5j4dhL0maUW6/+74Zd5Vjum/t+H/2kiT1XKdhn2TvJIuTLEmycJTyuUkuSHJVkouSzBkoe3ySrye5Lsm1SeZ1WVdJkvqqs7BPsi7wMeBFwHzgwCTzh0b7EHB6Ve0CHA0cN1B2OvD3VbUTsCvwi67qKklSn3V5Zr8rsKSqbqiqe4EzgH2HxpkPXNB2XzhS3h4UrFdV5wNU1Z1V9dsO6ypJUm91GfbbAUsH+pe1wwZdCezfdu8HzEqyFfBk4LYkX0pyRZK/b68UPEySQ5MsSrJo+fLlHbwFSZJmvi7DfrSfDxv+lZG3A7snuQLYHfgpcD/Nfwk8vy1/NvAE4OAVZlZ1UlUtqKoFs2fPnsKqS5LUH12G/TJg+4H+OcDNgyNU1c1V9fKqeibw7nbY7e20V7S3AO4H/g14Vod1lSSpt7oM+8uAHZPskGQD4ADg7MERkmydZKQO7wROGZh2yyQjp+svAK7tsK6SJPVWZ2HfnpEfDpwHXAecWVXXJDk6yT7taHsAi5P8GNgGOLad9gGaS/gXJLma5pbAyV3VVZKkPuv0G/Sq6lzg3KFh7x3oPgs4a4xpzwd26bJ+kiStDfwGPUmSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrO37OXpGmSjPZFoysZ/4Txj1s1/IWlWpsZ9pI0TQxkPVK8jC9JUs8Z9pIk9ZxhL0lSz3nPXpI0o8zaaSE7n7ZwuqsxIbN2AnjJtC3fsJckzSh3XHc8Nx4/fcE5GfMWnjOty/cyviRJPWfYS5LUc4a9JEk95z37cRrv/ZabTvjTzuow98ivjmu8zTdev7M6SJJmHsN+HCb0IMjxfiOWJGnN4mV8SZJ6zrCXJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqec6DfskeydZnGRJkoWjlM9NckGSq5JclGTOUPmjkvw0yUe7rKckSX3WWdgnWRf4GPAiYD5wYJL5Q6N9CDi9qnYBjgaOGyo/Bri4qzpKkrQ26PLMfldgSVXdUFX3AmcA+w6NMx+4oO2+cLA8ye8D2wBf77COkiT1Xpdhvx2wdKB/WTts0JXA/m33fsCsJFslWQf4B+BvVraAJIcmWZRk0fLly6eo2pIk9UuXYZ9RhtVQ/9uB3ZNcAewO/BS4H/gr4NyqWspKVNVJVbWgqhbMnj17KuosSVLvrNfhvJcB2w/0zwFuHhyhqm4GXg6QZDNg/6q6Pcn/AJ6f5K+AzYANktxZVSs85CdJklauy7C/DNgxyQ40Z+wHAK8eHCHJ1sCtVfUg8E7gFICqes3AOAcDCwx6SZImp7PL+FV1P3A4cB5wHXBmVV2T5Ogk+7Sj7QEsTvJjmofxju2qPpIkra26PLOnqs4Fzh0a9t6B7rOAs1Yxj1OBUzuoniRJawW/QU+SpJ4z7CVJ6jnDXpKknjPsJUnqOcNekqSeM+wlSeq5VYZ9km2SfCrJv7f985O8sfuqSZKkqTCeM/tTab4YZ9u2/8fAW7qqkCRJmlrjCfutq+pM4EH43TfjPdBprSRJ0pQZT9jflWQr2l+sS7IbcHuntZIkSVNmPF+X+1bgbOCJSb4DzAZe0WmtJEnSlFll2FfV5Ul2B55C8xv1i6vqvs5rJkmSpsQqwz7J64cGPSsJVXV6R3WSJElTaDyX8Z890L0RsCdwOWDYS5I0A4znMv4Rg/1JNgc+01mNtFZ6+vu/zu13d3N3aN7CczqZ7+Ybr8+V79urk3lL0lSazO/Z/xbYcaororXb7Xffx43Hv2S6qzEhXR1ESNJUG889+6/Q/tsdzb/qzQfO7LJSkiRp6oznzP5DA933AzdV1bKO6iNJkqbYeO7ZX/xIVESSJHVjzLBPcgcPXb5/WBFQVfWozmolSZKmzJhhX1WzHsmKSJKkboz7afwkj6H5P3sAquonndRIkiRNqfH8nv0+Sa4H/gu4GLgR+PeO6yVJkqbIeH717hhgN+DHVbUDzTfofafTWkmSpCkznrC/r6p+BayTZJ2quhB4Rsf1kiRJU2Q89+xvS7IZ8C3gc0l+QfP/9pIkaQYY88w+yUeTPBfYl+Yrct8CfA34T+Clj0z1JEnS6lrZmf31NN+e9zjgC8Dnq+q0R6RWkiRpyox5Zl9V/1JV/wPYHbgV+HSS65L8bZInP2I1lCRJq2U8X5d7E3ACcEKSZwKnAEcB63ZbNUmSRjfTfnVy843Xn9blj+dX79YH9gYOoPm3u4uB93dcL0mSRtXVz2HPW3jOjPup7fFa2XfjvxA4EHgJ8D3gDODQqrrrEaqbJEmaAiv7P/t3AZcAO1XVS6vqcxMN+iR7J1mcZEmShaOUz01yQZKrklyUZE47/BlJLklyTVv2ZxN6V5Ik6XdW9kM4f7Q6M06yLvAx4IXAMuCyJGdX1bUDo30IOL2qTkvyAuA44HU0/+r3+qq6Psm2wPeTnFdVt61OnSRJWhuN+4dwJmFXYElV3QCQ5Aya/9kfDPv5wP9quy8E/g2gqn48MkJV3dx+kc9swLCXJI1bkomNf8L4x60a7Vfg10zj+brcydoOWDrQv6wdNuhKYP+2ez9gVpKtBkdIsiuwAc2X+UiSNG5V1dlrJuky7Ec7nBpunbcDuye5gub/+X/KwFfxJnkc8BngDVX14AoLSA5NsijJouXLl09dzSVJ6pEuw34ZsP1A/xzg5sERqurmqnp5VT0TeHc77HaAJI8CzgHeU1WXjraAqjqpqhZU1YLZs2d38R4kSZrxugz7y4Adk+yQZAOa/9M/e3CEJFsnGanDO2m+sId2/C/TPLz3xQ7rKElS73UW9lV1P3A4cB5wHXBmVV2T5Ogk+7Sj7QEsTvJjYBvg2Hb4q4A/BA5O8oP25c/qSpI0CV0+jU9VnQucOzTsvQPdZwFnjTLdZ4HPdlk3SZLWFl1expckSWsAw16SpJ4z7CVJ6jnDXpKknjPsJUnqOcNekqSeM+wlSeo5w16SpJ4z7CVJ6jnDXpKknjPsJUnqOcNekqSeM+wlSeo5w16SpJ4z7CVJ6jnDXpKknjPsJUnqufWmuwISwKydFrLzaQunuxoTMmsngJdMdzUkaZUMe60R7rjueG48fmYF57yF50x3FSRpXLyML0lSzxn2kiT1nGEvSVLPGfaSJPWcYS9JUs8Z9pIk9ZxhL0lSzxn2kiT1nGEvSVLPGfaSJPWcYS9JUs8Z9pIk9ZxhL0lSzxn2kiT1XKdhn2TvJIuTLEmywo+VJ5mb5IIkVyW5KMmcgbKDklzfvg7qsp6SJPVZZ2GfZF3gY8CLgPnAgUnmD432IeD0qtoFOBo4rp320cD7gOcAuwLvS7JlV3WVJKnPujyz3xVYUlU3VNW9wBnAvkPjzAcuaLsvHCj/E+D8qrq1qn4NnA/s3WFdJUnqrS7Dfjtg6UD/snbYoCuB/dvu/YBZSbYa57QkOTTJoiSLli9fPmUVlySpT7oM+4wyrIb63w7snuQKYHfgp8D945yWqjqpqhZU1YLZs2evbn0lSeql9Tqc9zJg+4H+OcDNgyNU1c3AywGSbAbsX1W3J1kG7DE07UUd1lWSpN7q8sz+MmDHJDsk2QA4ADh7cIQkWycZqcM7gVPa7vOAvZJs2T6Yt1c7TJIkTVBnYV9V9wOH04T0dcCZVXVNkqOT7NOOtgewOMmPgW2AY9tpbwWOoTlguAw4uh0mSZImqMvL+FTVucC5Q8PeO9B9FnDWGNOewkNn+pIkaZL8Bj1JknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknqmk5dAAAAM20lEQVTOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqecMe0mSes6wlySp5wx7SZJ6zrCXJKnnDHtJknrOsJckqec6DfskeydZnGRJkoWjlD8+yYVJrkhyVZIXt8PXT3JakquTXJfknV3WU5KkPuss7JOsC3wMeBEwHzgwyfyh0d4DnFlVzwQOAD7eDn8lsGFV7Qz8PvCXSeZ1VVdJkvqsyzP7XYElVXVDVd0LnAHsOzROAY9quzcHbh4YvmmS9YCNgXuB33RYV0mSeqvLsN8OWDrQv6wdNugo4LVJlgHnAke0w88C7gJ+BvwE+FBV3dphXSVJ6q0uwz6jDKuh/gOBU6tqDvBi4DNJ1qG5KvAAsC2wA/C2JE9YYQHJoUkWJVm0fPnyqa29JEk90WXYLwO2H+ifw0OX6Ue8ETgToKouATYCtgZeDXytqu6rql8A3wEWDC+gqk6qqgVVtWD27NkdvAVJkma+LsP+MmDHJDsk2YDmAbyzh8b5CbAnQJKdaMJ+eTv8BWlsCuwG/KjDukqS1FudhX1V3Q8cDpwHXEfz1P01SY5Osk872tuAQ5JcCXweOLiqiuYp/s2AH9IcNHy6qq7qqq6SJPXZel3OvKrOpXnwbnDYewe6rwWeO8p0d9L8+50kSVpNfoOeJEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9t950V0AaMW/hOdNdhQnZfOP1p7sKkjQuhr3WCDce/5JO5jtv4TmdzVuSZopOL+Mn2TvJ4iRLkiwcpfzxSS5MckWSq5K8eKBslySXJLkmydVJNuqyrpIk9VVnZ/ZJ1gU+BrwQWAZcluTsqrp2YLT3AGdW1SeSzAfOBeYlWQ/4LPC6qroyyVbAfV3VVZKkPuvyzH5XYElV3VBV9wJnAPsOjVPAo9ruzYGb2+69gKuq6kqAqvpVVT3QYV0lSeqtLsN+O2DpQP+ydtigo4DXJllGc1Z/RDv8yUAlOS/J5UneMdoCkhyaZFGSRcuXL5/a2kuS1BNdhn1GGVZD/QcCp1bVHODFwGeSrENze+F5wGvav/sl2XOFmVWdVFULqmrB7Nmzp7b2kiT1RJdhvwzYfqB/Dg9dph/xRuBMgKq6BNgI2Lqd9uKq+mVV/ZbmrP9ZHdZVkqTe6jLsLwN2TLJDkg2AA4Czh8b5CbAnQJKdaMJ+OXAesEuSTdqH9XYHrkWSJE1YZ0/jV9X9SQ6nCe51gVOq6pokRwOLqups4G3AyUn+F80l/oOrqoBfJ/lHmgOGAs6tqpn1jSuSJK0hOv1Snao6l+YS/OCw9w50Xws8d4xpP0vz73eSJGk1+A16mnGS0Z79XMn4J0xs/s3FJUnqD8NeM45hLEkT46/eSZLUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HOGvSRJPWfYS5LUc+nLL4glWQ7cNN31mKCtgV9OdyV6zjZ+ZNjO3bONuzcT23huVc1e1Ui9CfuZKMmiqlow3fXoM9v4kWE7d8827l6f29jL+JIk9ZxhL0lSzxn20+uk6a7AWsA2fmTYzt2zjbvX2zb2nr0kST3nmb0kST1n2I9Tkv2SVJKnTndd+irJA0l+kOSHSb6YZJMpmOeCJB9eSfm2Sc5a3eX0wVD7fyXJFlM8/4OTfLTtPirJ26dy/tNttH1Ekj2SfHVovFOTvKLtXj/J8Umub9v9e0leNMq8/zTJFUmuTHJtkr/s/h2t+QbW2ZHXvCRbJbkwyZ0j69sY065VbWrYj9+BwLeBA7paQJJ1u5r3DHF3VT2jqp4G3AscNliYxoTW2apaVFVvWkn5zVX1islVt3cG2/9W4K+nu0IzzGT2EccAjwOe1rb7S4FZgyMkWZ/mXvJLq+rpwDOBi1anopPZltZQI+vsyOtG4L+BvwXGPJhcG9t0ja3YmiTJZsBzgTcysCEneUeSq9sjw+PbYU9K8o122OVJnjh8dJ/ko0kObrtvTPLeJN8GXpnkkCSXtdP/68jZbZJtkny5HX5lkj9IckySNw/M99gkYwbbDPMt4Entkfp1ST4OXA5sn2SvJJe07fvF9vMhybOT/EfbPt9LMmuw7ZPsPnAGcEVbPi/JD9vyjZJ8uv1Mr0jyR+3wg5N8KcnX2jOwv5umNnkkXQJsN9KT5G/a9fKqJO8fGP76dtiVST7TDntpku+2bfiNJNtMQ/0fUWPtI1YxzSbAIcARVXUPQFX9vKrOHBp1FrAe8Kt2nHuqanE7jxX2C+3wt7ZXCn6Y5C3tsHFvSzNZVd1VVd+mCf2xrH1tWlW+VvECXgt8qu3+D+BZwIva7k3a4Y9u/34X2K/t3gjYBNgD+OrA/D4KHNx23wi8Y6Bsq4HuD9DsCAC+ALyl7V4X2ByYB1zeDlsH+M/B6WfaC7iz/bse8H+B/9m+xweB3dqyrYFvApu2/UcC7wU2AG4Ant0Of1Q7n9+1PfAV4Llt92Zt+Tzgh+2wtwGfbrufCvyk/QwPbue9edt/E7D9dLdXh+2/LvBFYO+2fy+as6C069lXgT8Efg9YDGw9tA1syUMP//4F8A9t98HAR9vuo4C3T/d7nsK2W2Ef0XY/bNtvh50KvALYBbhinPP/JPAL4PPAa4B12uGj7Rd+H7ga2LRdz6+hOXMd17Y03W05wXZ/APhB+/ryUNnv1jfbtFgPjceBwD+33We0/evQBMNvAarq1iSzgO2q6svtsP8GSLKq+X9hoPtpST4AbEGzUp3XDn8B8Pp2vg8AtwO3J/lVkmcC29DsOH61Om90mm2c5Adt97eATwHbAjdV1aXt8N2A+cB32nbdgOYs9CnAz6rqMoCq+g2s0PbfAf4xyeeAL1XVsqHy5wEfaaf/UZKbgCe3ZRdU1e3tPK8F5gJLp+h9rylG2n8e8H3g/Hb4Xu3rirZ/M2BH4OnAWVX1S2i2gbZ8DvCFJI+j+Xz+6xGp/fQabR9xOTDWvztN6N+gquovkuwM/DHN5ekX0oTZCvuFJM+jCb67AJJ8CXg+cDbj25Zmkrur6hmTmXBta1PDfhWSbEXz4T8tSdEc6RXwr6y4wY6V6vfz8FsmGw2V3zXQfSrwsqq6sr3Uv8cqqvhJmhX0scApqxh3TbfChttuMIPtE+D8qjpwaLxdWMUOtKqOT3IO8GLg0iR/zMMv9a3sqOyege4H6Oe2c3dVPSPJ5jRn738NfJimXY6rqv89OHJ7y2i0Nv8I8I9VdXaSPWjO4ntrrH1EknfQXCbecmiSR9N8//oS4PFJZlXVHataTlVdDVzd3i75L5rtftQqrWQ2q9yW1iZrU5t6z37VXgGcXlVzq2peVW1Ps1LcCvx5Hrqn/uj2bHJZkpe1wzZsy28C5rf9mwN7rmR5s4CfpXmA5DUDwy+guaxNknWTPKod/mVgb+DZPHQVoM8uBZ6b5EnQ3PdM8mTgR8C2SZ7dDp+V5GGBnOSJVXV1VZ0ALKK5VD/om7Rt3s7z8TSXqdcq7RWMNwFvb9fD82jW9ZFnI7ZL8hiadfJVbdiR5NHtLDYHftp2H/SIVn56jLWPeB5wPc16uRNAkrk0V0R+0F4V/BTw4SQbtOWPS/LawZkn2aw9aBrxDB760a/R9gvfBF7WbhubAvvRXCkbNta21HtrY5sa9qt2IE2gDvpXmsvLZwOL2kufI09+vg54U5KraO7dPbaqlgJnAlcBn+Ohy6Gj+Vua+/7n0wTYiDcDf5TkappLrL8HUFX3AhcCZ7aXnHqtqpbTHH1/vm3jS4Gntu3wZ8BHklxJ037DV1De0j5ccyVwN/DvQ+UfB9Zt2/gLNM9V3MNaqKquAK4EDqiqrwP/B7ikbZuzgFlVdQ1wLHBx26b/2E5+FPDFJN9i5v2C2GSMtY94dbv+vBb4dLufOAv4i5FbQsB7gOXAtWkeFP23tn9QgHckWdzO4/08dAa6wn6hqi6nuUL4PZp9ySfbz/NhxtqWJtUCa5gkN9KsjwcnWZZk/vAorGVt6jfozXBp/tXjcuCVVXX9dNdHkrTm8cx+BmuPVpfQPDxm0EuSRuWZvSRJPeeZvSRJPWfYS5LUc4a9JEk9Z9hLktRzhr0kST1n2EuS1HP/H6IovoYr5U6aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize = (8,6))\n",
    "result_df.drop(['Model'], axis = 1).boxplot(vert = 1, ax = ax)\n",
    "ax.set_title('Binary Claasification Model Performance on Test Data : All features')\n",
    "ax.grid(b=None)\n",
    "ax.set_ylabel('Value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model        5.500000\n",
       "Accuracy     0.965000\n",
       "Precision    0.901916\n",
       "Recall       0.993333\n",
       "AUC Score    0.973095\n",
       "F1 Score     0.944916\n",
       "dtype: float64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}