ET.General_All_GridSearch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Hyperparameter tuning of All classifiers for emotional transition detection\n",
    "* 6 fold cross validation with grid-search\n",
    "* Binary classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.ensemble.bagging module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.ensemble. Anything that cannot be imported from sklearn.ensemble is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.ensemble.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.ensemble. Anything that cannot be imported from sklearn.ensemble is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.ensemble.forest module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.ensemble. Anything that cannot be imported from sklearn.ensemble is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "Using TensorFlow backend.\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.utils.testing module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.utils. Anything that cannot be imported from sklearn.utils is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.metrics.classification module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.metrics. Anything that cannot be imported from sklearn.metrics is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import datetime\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from pprint import pprint\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn import metrics   \n",
    "from sklearn.feature_selection import SelectFromModel,RFECV\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import GridSearchCV \n",
    "from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score, PredefinedSplit\n",
    "from sklearn.feature_selection import SelectKBest, mutual_info_classif\n",
    "from sklearn.model_selection import GridSearchCV \n",
    "from sklearn.model_selection import KFold, StratifiedKFold, cross_val_score\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "\n",
    "\n",
    "from sklearn import metrics   \n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.over_sampling import SMOTENC\n",
    "from imblearn.over_sampling import ADASYN\n",
    "from imblearn.over_sampling import SVMSMOTE\n",
    "from imblearn.combine import SMOTEENN\n",
    "from imblearn.combine import SMOTETomek\n",
    "\n",
    "pd.options.mode.chained_assignment = None\n",
    "import re\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "#warnings.filterwarnings('always')\n",
    "import pickle\n",
    "\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from imblearn.pipeline import Pipeline\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from imblearn.metrics import specificity_score\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import make_scorer, f1_score, roc_auc_score, precision_score, recall_score, confusion_matrix\n",
    "from sklearn import metrics   \n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "from catboost import CatBoostClassifier, Pool, cv\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "\n",
    "#from pandas_ml import ConfusionMatrix\n",
    "\n",
    "#import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_input(p):\n",
    "    #Read input file of each person\n",
    "    filename='data/NonOverlap_w5_emoChange_SelFeat_data_p'+str(p)+'.csv'\n",
    "    \n",
    "    raw_df= pd.read_csv(filename)\n",
    "    print(\"The shape of the dataframe is \",raw_df.shape)\n",
    "\n",
    "    return raw_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace NANs with -999\n",
    "def prep_data(data):\n",
    "    return data.fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop columns\n",
    "def drop_cols(data, col_list):\n",
    "    return data.drop(col_list, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# normalize data with minmax\n",
    "def scale_data(trn_x, tst_x):\n",
    "    \n",
    "    sc= StandardScaler()\n",
    "    scaled_trn_x = sc.fit_transform(trn_x)\n",
    "    scaled_tst_x = sc.fit_transform(tst_x)\n",
    "    \n",
    "    return scaled_trn_x, scaled_tst_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# oversampling with SMOTE with 'minority' and 'not majority'\n",
    "def over_sample_SMOTE(X_train, y_train):\n",
    "    sm=SMOTE(sampling_strategy='not majority', random_state=10) # 'minority'\n",
    "    X_train_ovr, y_train_ovr=sm.fit_sample(X_train, y_train)\n",
    "\n",
    "    #print(X_train_ovr.shape, y_train_ovr.shape)\n",
    "    return X_train_ovr, y_train_ovr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# oversampling with SMOTENC with 'minority' and 'not majority'\n",
    "def over_sample_SMOTENC(X_train, y_train):\n",
    "    \n",
    "    sm = SMOTENC(sampling_strategy='not majority',random_state=10)\n",
    "    \n",
    "    #sm = SMOTENC(sampling_strategy='minority',random_state=10)\n",
    "    \n",
    "    X_train_ovr, y_train_ovr=sm.fit_sample(X_train, y_train)\n",
    "\n",
    "    #print(X_train_ovr.shape, y_train_ovr.shape)\n",
    "    return X_train_ovr, y_train_ovr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# oversampling with SVMSMOTE \n",
    "def over_sample_SVMSMOTE(X_train, y_train):\n",
    "    sm=SVMSMOTE(random_state=10)\n",
    "    \n",
    "    X_train_ovr, y_train_ovr=sm.fit_sample(X_train, y_train)\n",
    "\n",
    "    #print(X_train_ovr.shape, y_train_ovr.shape)\n",
    "    return X_train_ovr, y_train_ovr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_dataframes(p_list):\n",
    "    df = pd.DataFrame()\n",
    "    for p in p_list:\n",
    "        new_df = read_input(p)\n",
    "        df=df.append(new_df,ignore_index = True)\n",
    "        \n",
    "    #drop all variables that contain all NANs\n",
    "    df.dropna(axis=1,how='all', inplace=True)\n",
    "    #reset the index\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    #drop columns with all zeros in pandas dataframe\n",
    "    df=df.T[(df!=0).any()].T\n",
    "    \n",
    "    #keep columns with missing values < 30%  \n",
    "    df = df.loc[:, df.isnull().mean() < .3]\n",
    "    \n",
    "    print(\"The shape of the merged dataframe is \",df.shape)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop all columns that contain location information (if any)\n",
    "def drop_location(df):\n",
    "    print(df.shape)\n",
    "    df = df[df.columns.drop(list(df.filter(regex='location')))]\n",
    "    df = df[df.columns.drop(list(df.filter(regex='latitude')))]\n",
    "    df = df[df.columns.drop(list(df.filter(regex='lonitude')))]\n",
    "    print(df.shape)\n",
    "   \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_k_features(X_train_scaled,X_test_scaled,y_train,k):\n",
    "    selection = SelectKBest(mutual_info_classif, k)\n",
    "    X_train = selection.fit_transform(X_train_scaled,y_train)\n",
    "    X_test = selection.transform(X_test_scaled)\n",
    "    \n",
    "    return X_train, X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_results(accu, bl_accu, prec, rec_, spec_, roc_, f1_):   \n",
    "    print('.....................')\n",
    "    print(\"Average Accuracy: %.2f%% (%.2f)\" % (np.mean(accu), np.std(accu)))\n",
    "    print(\"Average Balanced_accuracy: %.2f%% (%.2f)\" % (np.mean(bl_accu),np.std(bl_accu)))\n",
    "    print(\"Average Precision: %.2f%% (%.2f)\" % (np.mean(prec),np.std(prec)))\n",
    "    print(\"Average Recall: %.2f%% (%.2f)\" % (np.mean(rec_),np.std(rec_)))\n",
    "    print(\"Average Specificity: %.2f%% (%.2f)\" % (np.mean(spec_),np.std(spec_)))\n",
    "    print(\"Average ROC AUC: %.2f%% (%.2f)\" % (np.mean(roc_),np.std(roc_)))\n",
    "    print(\"Average F1 score: %.2f%% (%.2f)\" % (np.mean(f1_),np.std(f1_)))\n",
    "    print('..................................................')\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = Pipeline([('scaler', StandardScaler()), # MinMaxScaler()\n",
    "                 ('selector', SelectKBest(mutual_info_classif, k=90)), #\n",
    "                 ('classifier', LogisticRegression())])\n",
    "\n",
    "search_space = [{'selector__k': [ 50, 70, 90]},\n",
    "                \n",
    "                {'classifier': [LogisticRegression(solver='lbfgs')],\n",
    "                 'classifier__C': [0.01, 0.1, 1.0],\n",
    "                 'classifier__penalty': ['l1', 'l2', None],\n",
    "                 'classifier__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],\n",
    "                 'classifier__max_iter':[100, 150, 200], \n",
    "                 'classifier__class_weight':[None, 'balanced']},\n",
    "                 \n",
    "                {'classifier': [RandomForestClassifier()],\n",
    "                 'classifier__max_depth': [5, 10, 30, None],\n",
    "                 'classifier__criterion':['gini','entropy'], \n",
    "                 'classifier__bootstrap': [True],\n",
    "                 'classifier__max_features':['log2', None],\n",
    "                 'classifier__n_estimators': [50, 100, 200, 300, 400]},\n",
    "                \n",
    "                {'classifier': [MLPClassifier(random_state=1, early_stopping=True)],\n",
    "                 'classifier__hidden_layer_sizes' : [(50, 50, 50), (50, 100, 50), (20, 20, 20), (30, ), (50,),(100,)], \n",
    "                 'classifier__activation' : ['tanh', 'relu', 'logistic'],\n",
    "                 'classifier__max_iter':[50, 100, 150, 200, 300],\n",
    "                 'classifier__solver': ['sgd', 'adam', 'lbfgs'],\n",
    "                 'classifier__alpha': [0.0001, 0.001, 0.05]},\n",
    "                \n",
    "                {'classifier': [CatBoostClassifier(random_seed=1)],\n",
    "                 'classifier__learning_rate': [0.05, 0.1, 0.15, 0.2]},\n",
    "                \n",
    "                {'classifier': [XGBClassifier(objective='binary:logistic', random_state=1)],\n",
    "                 'classifier__learning_rate': [0.05, 0.1, 0.15, 0.2],\n",
    "                 'classifier__colsample_bytree':[.5, .75, 1],\n",
    "                 'classifier__max_depth': np.arange(3, 6, 10),\n",
    "                 'classifier__n_estimators': [50, 100, 200, 300, 400]}]\n",
    "                  \n",
    "\n",
    "scorer = make_scorer(f1_score, average = 'binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR_pipe = Pipeline([('scaler', StandardScaler()), # MinMaxScaler()\n",
    "                 ('selector', SelectKBest(mutual_info_classif, k=90)), #\n",
    "                 ('classifier', LogisticRegression())])\n",
    "\n",
    "LR_search_space = [{'selector__k': [ 50, 70, 90, 110]},\n",
    "                \n",
    "                {'classifier': [LogisticRegression(solver='lbfgs')],\n",
    "                 'classifier__C': [0.01, 0.1, 1.0],\n",
    "                 'classifier__penalty': ['l1', 'l2', None],\n",
    "                 'classifier__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],\n",
    "                 'classifier__max_iter':[100, 150, 200], \n",
    "                 'classifier__class_weight':[None, 'balanced']}]\n",
    "                 \n",
    "################################################################################          \n",
    "\n",
    "RF_pipe = Pipeline([('scaler', StandardScaler()), # MinMaxScaler()\n",
    "                 ('selector', SelectKBest(mutual_info_classif, k=90)), #\n",
    "                 ('classifier', RandomForestClassifier())])\n",
    "\n",
    "RF_search_space = [{'selector__k': [ 50, 70, 90, 110]},\n",
    "                \n",
    "                {'classifier': [RandomForestClassifier()],\n",
    "                 'classifier__max_depth': [5, 10, 30, None],\n",
    "                 'classifier__criterion':['gini','entropy'], \n",
    "                 'classifier__bootstrap': [True],\n",
    "                 'classifier__max_features':['log2', None],\n",
    "                 'classifier__n_estimators': [50, 100, 200, 300, 400]}]\n",
    "                  \n",
    "################################################################################\n",
    "\n",
    "MLP_pipe = Pipeline([('scaler', StandardScaler()), # MinMaxScaler()\n",
    "                 ('selector', SelectKBest(mutual_info_classif, k=90)), #\n",
    "                 ('classifier', MLPClassifier(random_state=1, early_stopping=True))])\n",
    "\n",
    "MLP_search_space = [{'selector__k': [ 50, 70, 90, 110]},\n",
    "                \n",
    "                {'classifier': [MLPClassifier(random_state=1, early_stopping=True)],\n",
    "                 'classifier__hidden_layer_sizes' : [(50, 50, 50), (50, 100, 50), (20, 20, 20), (30, ), (50,),(100,)], \n",
    "                 'classifier__activation' : ['tanh', 'relu', 'logistic'],\n",
    "                 'classifier__max_iter':[50, 100, 150, 200, 300],\n",
    "                 'classifier__solver': ['sgd', 'adam', 'lbfgs'],\n",
    "                 'classifier__alpha': [0.0001, 0.001, 0.05]}]\n",
    "\n",
    "################################################################################\n",
    "\n",
    "CB_pipe = Pipeline([('scaler', StandardScaler()), # MinMaxScaler()\n",
    "                 ('selector', SelectKBest(mutual_info_classif, k=90)), #\n",
    "                 ('classifier', CatBoostClassifier(random_seed=1))])\n",
    "\n",
    "CB_search_space = [{'selector__k': [ 50, 70, 90, 110]},\n",
    "                \n",
    "                {'classifier': [CatBoostClassifier(random_seed=1, verbose=False)],\n",
    "                 'classifier__learning_rate': [0.05, 0.1, 0.15, 0.2]}]\n",
    "#'iterations': Integer(10, 1000),\n",
    " #                'depth': Integer(1, 8),\n",
    "  #               'learning_rate': Real(0.01, 1.0, 'log-uniform'),\n",
    "   #              'random_strength': Real(1e-9, 10, 'log-uniform'),\n",
    "    #             'bagging_temperature': Real(0.0, 1.0),\n",
    "     #            'border_count': Integer(1, 255),\n",
    "      #           'l2_leaf_reg': Integer(2, 30),\n",
    "       #          'scale_pos_weight':Real(0.01, 1.0, 'uniform')\n",
    "\n",
    "################################################################################\n",
    "\n",
    "XGB_pipe = Pipeline([('scaler', StandardScaler()), # MinMaxScaler()\n",
    "                 ('selector', SelectKBest(mutual_info_classif, k=90)), #\n",
    "                 ('classifier', XGBClassifier(objective='binary:logistic', random_state=1))])\n",
    "\n",
    "XGB_search_space = [{'selector__k': [ 50, 70, 90, 110]},\n",
    "                \n",
    "                {'classifier': [XGBClassifier(objective='binary:logistic', random_state=1)],\n",
    "                 'classifier__learning_rate': [0.05, 0.1, 0.15, 0.2],\n",
    "                 'classifier__colsample_bytree':[.5, .75, 1],\n",
    "                 'classifier__max_depth': np.arange(3, 6, 10),\n",
    "                 'classifier__n_estimators': [50, 100, 200, 300, 400]}]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_list=[8,10,12,13,15,20,21,25, 27, 33,35,40,46,48,49,52,54,55]\n",
    "nfolds = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of the dataframe is  (268, 274)\n",
      "The shape of the dataframe is  (1275, 274)\n",
      "The shape of the dataframe is  (539, 274)\n",
      "The shape of the merged dataframe is  (2082, 233)\n",
      "The shape of the dataframe is  (1245, 274)\n",
      "The shape of the dataframe is  (551, 274)\n",
      "The shape of the dataframe is  (623, 274)\n",
      "The shape of the merged dataframe is  (2419, 238)\n",
      "The shape of the dataframe is  (1268, 274)\n",
      "The shape of the dataframe is  (257, 274)\n",
      "The shape of the dataframe is  (317, 274)\n",
      "The shape of the merged dataframe is  (1842, 179)\n",
      "The shape of the dataframe is  (862, 274)\n",
      "The shape of the dataframe is  (911, 274)\n",
      "The shape of the dataframe is  (752, 274)\n",
      "The shape of the merged dataframe is  (2525, 237)\n",
      "The shape of the dataframe is  (755, 274)\n",
      "The shape of the dataframe is  (1164, 274)\n",
      "The shape of the dataframe is  (692, 274)\n",
      "The shape of the merged dataframe is  (2611, 186)\n",
      "The shape of the dataframe is  (868, 274)\n",
      "The shape of the dataframe is  (529, 274)\n",
      "The shape of the dataframe is  (707, 274)\n",
      "The shape of the merged dataframe is  (2104, 189)\n"
     ]
    }
   ],
   "source": [
    "# make a predifined CV split (test_fold)\n",
    "test_fold = []\n",
    "for i in range(nfolds):\n",
    "    p_test = p_list[i*3:i*3+3]\n",
    "    \n",
    "    df_test = merge_dataframes(p_test)\n",
    "    \n",
    "    tst = [i] * df_test.shape[0] \n",
    "    \n",
    "    test_fold= test_fold + tst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of the dataframe is  (268, 274)\n",
      "The shape of the dataframe is  (1275, 274)\n",
      "The shape of the dataframe is  (539, 274)\n",
      "The shape of the dataframe is  (1245, 274)\n",
      "The shape of the dataframe is  (551, 274)\n",
      "The shape of the dataframe is  (623, 274)\n",
      "The shape of the dataframe is  (1268, 274)\n",
      "The shape of the dataframe is  (257, 274)\n",
      "The shape of the dataframe is  (317, 274)\n",
      "The shape of the dataframe is  (862, 274)\n",
      "The shape of the dataframe is  (911, 274)\n",
      "The shape of the dataframe is  (752, 274)\n",
      "The shape of the dataframe is  (755, 274)\n",
      "The shape of the dataframe is  (1164, 274)\n",
      "The shape of the dataframe is  (692, 274)\n",
      "The shape of the dataframe is  (868, 274)\n",
      "The shape of the dataframe is  (529, 274)\n",
      "The shape of the dataframe is  (707, 274)\n",
      "The shape of the merged dataframe is  (13583, 204)\n",
      "(13583, 204)\n",
      "(13583, 191)\n"
     ]
    }
   ],
   "source": [
    "ps = PredefinedSplit(test_fold)\n",
    "\n",
    "# df contains all persons' data in one dataset\n",
    "df = merge_dataframes(p_list)\n",
    "df = prep_data(df)\n",
    "\n",
    "# remove day_of_month variable if present in data\n",
    "if 'day_of_month' in df.columns:\n",
    "    drop_col=['day_of_month']\n",
    "    df=drop_cols(df, drop_col)\n",
    "\n",
    "#drop all columns that contain location information \n",
    "df = drop_location(df)\n",
    "\n",
    "\n",
    "labels = list(df.columns)\n",
    "labels.remove('emotion_change')\n",
    "\n",
    "X = df[labels]\n",
    "y = df['emotion_change']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grid_search_wrapper(pipe = pipe, search_space = search_space, verbose= False,refit_score=scorer):\n",
    "    \"\"\"\n",
    "    fits a GridSearchCV classifiers using refit_score for optimization\n",
    "    prints classifier performance metrics\n",
    "    \"\"\"\n",
    "    #cross_validation = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)\n",
    "    cross_validation = ps\n",
    "    \n",
    "    grid_search = GridSearchCV(pipe, search_space, cv=cross_validation, verbose=verbose,  n_jobs = -1) #scoring=scorer, refit=scorer\n",
    "    \n",
    "    grid_search.fit(X, y)\n",
    "    \n",
    "    return grid_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 6 folds for each of 274 candidates, totalling 1644 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=-1)]: Done 349 tasks      | elapsed: 18.9min\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed: 58.4min\n",
      "[Parallel(n_jobs=-1)]: Done 997 tasks      | elapsed: 79.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1442 tasks      | elapsed: 105.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1644 out of 1644 | elapsed: 117.5min finished\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    }
   ],
   "source": [
    "# do gird search for best parameters\n",
    "pipeline_grid_search_LR = grid_search_wrapper(pipe = LR_pipe, search_space = LR_search_space, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 6 folds for each of 84 candidates, totalling 504 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed: 22.1min\n",
      "[Parallel(n_jobs=-1)]: Done 349 tasks      | elapsed: 83.1min\n",
      "[Parallel(n_jobs=-1)]: Done 504 out of 504 | elapsed: 127.8min finished\n"
     ]
    }
   ],
   "source": [
    "pipeline_grid_search_RF = grid_search_wrapper(pipe = RF_pipe, search_space = RF_search_space, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 6 folds for each of 814 candidates, totalling 4884 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed: 10.0min\n",
      "[Parallel(n_jobs=-1)]: Done 349 tasks      | elapsed: 23.5min\n",
      "[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed: 41.8min\n",
      "[Parallel(n_jobs=-1)]: Done 997 tasks      | elapsed: 65.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1442 tasks      | elapsed: 95.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1969 tasks      | elapsed: 133.4min\n",
      "[Parallel(n_jobs=-1)]: Done 2576 tasks      | elapsed: 171.1min\n",
      "[Parallel(n_jobs=-1)]: Done 3265 tasks      | elapsed: 217.3min\n",
      "[Parallel(n_jobs=-1)]: Done 4034 tasks      | elapsed: 269.3min\n",
      "[Parallel(n_jobs=-1)]: Done 4884 out of 4884 | elapsed: 322.7min finished\n"
     ]
    }
   ],
   "source": [
    "pipeline_grid_search_MLP = grid_search_wrapper(pipe = MLP_pipe, search_space = MLP_search_space, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 6 folds for each of 64 candidates, totalling 384 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed: 13.5min\n",
      "[Parallel(n_jobs=-1)]: Done 349 tasks      | elapsed: 35.5min\n",
      "[Parallel(n_jobs=-1)]: Done 384 out of 384 | elapsed: 39.1min finished\n"
     ]
    }
   ],
   "source": [
    "pipeline_grid_search_XGB = grid_search_wrapper(pipe = XGB_pipe, search_space = XGB_search_space, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_grid_search_CB = grid_search_wrapper(pipe = CB_pipe, search_space = CB_search_space, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "         steps=[('scaler',\n",
      "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
      "                ('selector',\n",
      "                 SelectKBest(k=90,\n",
      "                             score_func=<function mutual_info_classif at 0x1a216088c0>)),\n",
      "                ('classifier',\n",
      "                 RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
      "                                        class_weight=None, criterion='gini',\n",
      "                                        max_depth=5, max_features=None,\n",
      "                                        max_leaf_nodes=None, max_samples=None,\n",
      "                                        min_impurity_decrease=0.0,\n",
      "                                        min_impurity_split=None,\n",
      "                                        min_samples_leaf=1, min_samples_split=2,\n",
      "                                        min_weight_fraction_leaf=0.0,\n",
      "                                        n_estimators=200, n_jobs=None,\n",
      "                                        oob_score=False, random_state=None,\n",
      "                                        verbose=0, warm_start=False))],\n",
      "         verbose=False)\n",
      "0.9490926047485976\n"
     ]
    }
   ],
   "source": [
    "print(pipeline_grid_search_RF.best_estimator_)\n",
    "print(pipeline_grid_search_RF.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "         steps=[('scaler',\n",
      "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
      "                ('selector',\n",
      "                 SelectKBest(k=90,\n",
      "                             score_func=<function mutual_info_classif at 0x1a216088c0>)),\n",
      "                ('classifier',\n",
      "                 XGBClassifier(base_score=0.5, booster='gbtree',\n",
      "                               colsample_bylevel=1, colsample_bynode=1,\n",
      "                               colsample_bytree=0.5, gamma=0, learning_rate=0.1,\n",
      "                               max_delta_step=0, max_depth=3,\n",
      "                               min_child_weight=1, missing=None,\n",
      "                               n_estimators=400, n_jobs=1, nthread=None,\n",
      "                               objective='binary:logistic', random_state=1,\n",
      "                               reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
      "                               seed=None, silent=None, subsample=1,\n",
      "                               verbosity=1))],\n",
      "         verbose=False)\n",
      "0.9493380165556605\n"
     ]
    }
   ],
   "source": [
    "print(pipeline_grid_search_XGB.best_estimator_)\n",
    "print(pipeline_grid_search_XGB.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "         steps=[('scaler',\n",
      "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
      "                ('selector',\n",
      "                 SelectKBest(k=50,\n",
      "                             score_func=<function mutual_info_classif at 0x1a216088c0>)),\n",
      "                ('classifier',\n",
      "                 LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
      "                                    fit_intercept=True, intercept_scaling=1,\n",
      "                                    l1_ratio=None, max_iter=100,\n",
      "                                    multi_class='auto', n_jobs=None,\n",
      "                                    penalty='l2', random_state=None,\n",
      "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
      "                                    warm_start=False))],\n",
      "         verbose=False)\n",
      "0.947902068375397\n"
     ]
    }
   ],
   "source": [
    "print(pipeline_grid_search_LR.best_estimator_)\n",
    "print(pipeline_grid_search_LR.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "         steps=[('scaler',\n",
      "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
      "                ('selector',\n",
      "                 SelectKBest(k=90,\n",
      "                             score_func=<function mutual_info_classif at 0x1a216088c0>)),\n",
      "                ('classifier',\n",
      "                 <catboost.core.CatBoostClassifier object at 0x1a4c753050>)],\n",
      "         verbose=False)\n",
      "0.9491117417230509\n"
     ]
    }
   ],
   "source": [
    "print(pipeline_grid_search_CB.best_estimator_)\n",
    "print(pipeline_grid_search_CB.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "         steps=[('scaler',\n",
      "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
      "                ('selector',\n",
      "                 SelectKBest(k=90,\n",
      "                             score_func=<function mutual_info_classif at 0x1a216088c0>)),\n",
      "                ('classifier',\n",
      "                 MLPClassifier(activation='tanh', alpha=0.001,\n",
      "                               batch_size='auto', beta_1=0.9, beta_2=0.999,\n",
      "                               early_stopping=True, epsilon=1e-08,\n",
      "                               hidden_layer_sizes=(50, 50, 50),\n",
      "                               learning_rate='constant',\n",
      "                               learning_rate_init=0.001, max_fun=15000,\n",
      "                               max_iter=50, momentum=0.9, n_iter_no_change=10,\n",
      "                               nesterovs_momentum=True, power_t=0.5,\n",
      "                               random_state=1, shuffle=True, solver='sgd',\n",
      "                               tol=0.0001, validation_fraction=0.1,\n",
      "                               verbose=False, warm_start=False))],\n",
      "         verbose=False)\n",
      "0.9498000071749098\n"
     ]
    }
   ],
   "source": [
    "print(pipeline_grid_search_MLP.best_estimator_)\n",
    "print(pipeline_grid_search_MLP.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# best models\n",
    "\n",
    "LR_model = LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
    "                                    fit_intercept=True, intercept_scaling=1,\n",
    "                                    l1_ratio=None, max_iter=100,\n",
    "                                    multi_class='auto', n_jobs=None,\n",
    "                                    penalty='l2', random_state=None,\n",
    "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
    "                                    warm_start=False)\n",
    "\n",
    "RF_model = RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
    "                                        class_weight=None, criterion='gini',\n",
    "                                        max_depth=5, max_features=None,\n",
    "                                        max_leaf_nodes=None, max_samples=None,\n",
    "                                        min_impurity_decrease=0.0,\n",
    "                                        min_impurity_split=None,\n",
    "                                        min_samples_leaf=1, min_samples_split=2,\n",
    "                                        min_weight_fraction_leaf=0.0,\n",
    "                                        n_estimators=200, n_jobs=None,\n",
    "                                        oob_score=False, random_state=None,\n",
    "                                        verbose=0, warm_start=False)\n",
    "\n",
    "XGB_model = XGBClassifier(base_score=0.5, booster='gbtree',\n",
    "                               colsample_bylevel=1, colsample_bynode=1,\n",
    "                               colsample_bytree=0.5, gamma=0, learning_rate=0.1,\n",
    "                               max_delta_step=0, max_depth=3,\n",
    "                               min_child_weight=1, missing=None,\n",
    "                               n_estimators=400, n_jobs=1, nthread=None,\n",
    "                               objective='binary:logistic', random_state=1,\n",
    "                               reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
    "                               seed=None, silent=None, subsample=1,\n",
    "                               verbosity=1)\n",
    "\n",
    "MLP_model = MLPClassifier(activation='tanh', alpha=0.001,\n",
    "                               batch_size='auto', beta_1=0.9, beta_2=0.999,\n",
    "                               early_stopping=True, epsilon=1e-08,\n",
    "                               hidden_layer_sizes=(50, 50, 50),\n",
    "                               learning_rate='constant',\n",
    "                               learning_rate_init=0.001, max_fun=15000,\n",
    "                               max_iter=50, momentum=0.9, n_iter_no_change=10,\n",
    "                               nesterovs_momentum=True, power_t=0.5,\n",
    "                               random_state=1, shuffle=True, solver='sgd',\n",
    "                               tol=0.0001, validation_fraction=0.1,\n",
    "                               verbose=False, warm_start=False)\n",
    "\n",
    "\n",
    "CB_model = CatBoostClassifier(random_seed=1, verbose=False,learning_rate= 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_models = {} # dictionary of best models with best parameters\n",
    "\n",
    "best_models['Logistic Regression'] = LR_model\n",
    "best_models['RandomForest Classifier'] = RF_model\n",
    "best_models['MLP Classifier'] = MLP_model\n",
    "best_models['XGBoost Classifier'] = XGB_model\n",
    "best_models['CatBoost Classifier'] = CB_model\n",
    "\n",
    "n_features = [50, 90, 90, 90, 90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "nfolds = 6\n",
    "rnd_state=42\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restuls for:  Logistic Regression\n",
      ".....................\n",
      "Average Accuracy: 87.06% (1.46)\n",
      "Average Balanced_accuracy: 82.75% (4.03)\n",
      "Average Precision: 25.27% (5.79)\n",
      "Average Recall: 77.83% (8.18)\n",
      "Average Specificity: 87.68% (1.54)\n",
      "Average ROC AUC: 90.16% (2.77)\n",
      "Average F1 score: 37.51% (6.37)\n",
      "..................................................\n",
      "\n",
      "\n",
      "Restuls for:  RandomForest Classifier\n",
      ".....................\n",
      "Average Accuracy: 22.13% (22.16)\n",
      "Average Balanced_accuracy: 56.41% (10.04)\n",
      "Average Precision: 6.48% (3.01)\n",
      "Average Recall: 94.60% (3.76)\n",
      "Average Specificity: 18.22% (23.34)\n",
      "Average ROC AUC: 51.68% (20.99)\n",
      "Average F1 score: 11.97% (5.06)\n",
      "..................................................\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "/Users/majed_al-jefri/opt/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (50) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restuls for:  MLP Classifier\n",
      ".....................\n",
      "Average Accuracy: 84.15% (3.64)\n",
      "Average Balanced_accuracy: 81.96% (4.71)\n",
      "Average Precision: 22.07% (5.48)\n",
      "Average Recall: 79.40% (7.85)\n",
      "Average Specificity: 84.51% (3.67)\n",
      "Average ROC AUC: 89.52% (3.73)\n",
      "Average F1 score: 34.01% (6.82)\n",
      "..................................................\n",
      "\n",
      "\n",
      "Restuls for:  XGBoost Classifier\n",
      ".....................\n",
      "Average Accuracy: 9.91% (8.65)\n",
      "Average Balanced_accuracy: 51.66% (3.91)\n",
      "Average Precision: 5.32% (1.47)\n",
      "Average Recall: 98.30% (2.25)\n",
      "Average Specificity: 5.02% (9.56)\n",
      "Average ROC AUC: 66.23% (8.12)\n",
      "Average F1 score: 10.06% (2.62)\n",
      "..................................................\n",
      "\n",
      "\n",
      "Restuls for:  CatBoost Classifier\n",
      ".....................\n",
      "Average Accuracy: 55.23% (27.40)\n",
      "Average Balanced_accuracy: 60.06% (6.78)\n",
      "Average Precision: 8.90% (3.13)\n",
      "Average Recall: 65.44% (19.92)\n",
      "Average Specificity: 54.67% (29.26)\n",
      "Average ROC AUC: 67.69% (10.05)\n",
      "Average F1 score: 15.15% (4.70)\n",
      "..................................................\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# this is to get all the detailed performance meterics after selecting the best model parameters\n",
    "k_i = -1    \n",
    "for model_name, model in best_models.items(): \n",
    "    k_i = k_i + 1\n",
    "    accu = []\n",
    "    prec = []\n",
    "    rec_ = []\n",
    "    f1_ = []\n",
    "    bl_accu = []\n",
    "    roc_ = []\n",
    "    spec_ = []\n",
    "\n",
    "    i = 1\n",
    "    for train_index, test_index in ps.split():\n",
    "        #print(\"fold\", i)\n",
    "        i+=1\n",
    "\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index] \n",
    "        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        \n",
    "        #scale features\n",
    "        X_train_scaled, X_test_scaled= scale_data(X_train, X_test) \n",
    "        #feature selection\n",
    "        X_train, X_test = select_k_features(X_train_scaled,X_test_scaled,y_train,k=n_features[k_i])\n",
    "\n",
    "        #oversample training data\n",
    "        X_train_imb,y_train_imb=over_sample_SMOTE(X_train, y_train)\n",
    "        #X_train_imb,y_train_imb=over_sample_SMOTENC(X_train, y_train)\n",
    "        #X_train_imb,y_train_imb=over_sample_SVMSMOTE(X_train, y_train)\n",
    "\n",
    "\n",
    "        # train model on imbalance-handled data\n",
    "        model.fit(X_train_imb, y_train_imb)\n",
    "\n",
    "        #train model on imbalance data \n",
    "        #model.fit(X_train, y_train)\n",
    "\n",
    "        # test model, measure class label and probability score\n",
    "        y_pred = model.predict(X_test)\n",
    "        y_scores = model.predict_proba(X_test)[:,1]\n",
    "\n",
    "        #calculate metrices\n",
    "\n",
    "        accuracy = accuracy_score(y_test, y_pred)\n",
    "        bl_accuracy = balanced_accuracy_score(y_test, y_pred)\n",
    "        precision=precision_score(y_test, y_pred, labels=np.unique(y_pred))\n",
    "        recall=recall_score(y_test, y_pred, labels=np.unique(y_pred))\n",
    "        f1=f1_score(y_test, y_pred, labels=np.unique(y_pred))\n",
    "        roc=roc_auc_score(y_test, y_scores, labels=np.unique(y_pred))\n",
    "        spec=specificity_score(y_test, y_pred ,labels=np.unique(y_pred))\n",
    "        \n",
    "        ac=accuracy * 100.0\n",
    "        pr=precision*100\n",
    "        rc=recall*100\n",
    "        f1_p=f1*100\n",
    "        bl_ac=bl_accuracy*100\n",
    "        roc=roc*100\n",
    "        spec=spec*100\n",
    "    \n",
    "        accu.append(ac)\n",
    "        prec.append(pr)\n",
    "        rec_.append(rc)\n",
    "        f1_.append(f1_p)\n",
    "        bl_accu.append(bl_ac)\n",
    "        roc_.append(roc)\n",
    "        spec_.append(spec)\n",
    "    \n",
    "    print('Restuls for: ', model_name)\n",
    "    print_results(accu, bl_accu, prec, rec_, spec_, roc_, f1_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}