evidence_impact_detector_eid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/jorge/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "import numpy as np\n",
    "import nltk\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import re\n",
    "\n",
    "\n",
    "nltk.download(\"stopwords\")\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import joblib\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import *\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn import metrics\n",
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import cross_val_score, KFold, RandomizedSearchCV\n",
    "\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from xgboost import XGBClassifier "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import data\n",
    "impact_sentences = pd.read_csv(\"output/output_sentence_extractor.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Successful bids for European Union and Nationa...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The project has reached an international audie...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Research carried out by the University of Gree...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Moreover, the research has informed the govern...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The findings have provided policy-makers in ed...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence source          label\n",
       "0  Successful bids for European Union and Nationa...    ref  social_impact\n",
       "1  The project has reached an international audie...    ref  social_impact\n",
       "2  Research carried out by the University of Gree...    ref  social_impact\n",
       "3  Moreover, the research has informed the govern...    ref  social_impact\n",
       "4  The findings have provided policy-makers in ed...    ref  social_impact"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "impact_sentences.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1028, 3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "impact_sentences.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add a numeric column to represent the textual label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "impact_sentences['num_label'] = np.where(impact_sentences['label'] == 'social_impact', 1, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>source</th>\n",
       "      <th>label</th>\n",
       "      <th>num_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Successful bids for European Union and Nationa...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The project has reached an international audie...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Research carried out by the University of Gree...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Moreover, the research has informed the govern...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The findings have provided policy-makers in ed...</td>\n",
       "      <td>ref</td>\n",
       "      <td>social_impact</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence source          label  \\\n",
       "0  Successful bids for European Union and Nationa...    ref  social_impact   \n",
       "1  The project has reached an international audie...    ref  social_impact   \n",
       "2  Research carried out by the University of Gree...    ref  social_impact   \n",
       "3  Moreover, the research has informed the govern...    ref  social_impact   \n",
       "4  The findings have provided policy-makers in ed...    ref  social_impact   \n",
       "\n",
       "   num_label  \n",
       "0          1  \n",
       "1          1  \n",
       "2          1  \n",
       "3          1  \n",
       "4          1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "impact_sentences.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store sentences and labels into separate variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data, labels = impact_sentences['sentence'], impact_sentences['num_label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Processing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove any html formatting and any non-alpha numeric characters that may appear in the sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = PorterStemmer()\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def sentence_to_words(sentence, steeming=False, lemmatization=False):\n",
    "    text = BeautifulSoup(sentence, \"html.parser\").get_text() # Remove HTML tags\n",
    "    text = re.sub(r\"[^a-zA-Z0-9]\", \" \", sentence.lower()) # Convert to lower case\n",
    "    words = word_tokenize(text)   # Split string into words\n",
    "    words = [word for word in words if word not in stopwords.words(\"english\")] # Remove stopwords\n",
    "    if steeming:    \n",
    "        words = [stemmer.stem(word) for word in words]\n",
    "    if lemmatization:\n",
    "        words = [lemmatizer.lemmatize(word) for word in words]\n",
    "    \n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cache_dir = \"cache\"  # directory to store cache files\n",
    "os.makedirs(cache_dir, exist_ok=True)  # ensure cache directory exists\n",
    "\n",
    "def preprocess_data(sentences, labels, cache_dir=cache_dir, cache_file=\"preprocessed_data.pkl\"):\n",
    "    \"\"\"Convert each review to words; read from cache if available.\"\"\"\n",
    "\n",
    "    # Try to read data from cache first\n",
    "    cache_data = None\n",
    "    if cache_file is not None:\n",
    "        try:\n",
    "            with open(os.path.join(cache_dir, cache_file), \"rb\") as f:\n",
    "                cache_data = pickle.load(f)\n",
    "            print(\"Read preprocessed data from cache file:\", cache_file)\n",
    "        except:\n",
    "            pass  # unable to read from cache\n",
    "    else:\n",
    "        cache_file = 'preprocessed_data.pkl'\n",
    "    \n",
    "    # If cache is missing, then do the processing\n",
    "    if cache_data is None:\n",
    "        # Preprocess data to obtain words for each sentence\n",
    "        words = [sentence_to_words(sentence) for sentence in sentences]\n",
    "        # Write to cache file for future use\n",
    "        cache_data = dict(words=words, labels=labels)\n",
    "        with open(os.path.join(cache_dir, cache_file), \"wb\") as f:\n",
    "            pickle.dump(cache_data, f)\n",
    "        print(\"Wrote preprocessed data to cache file:\", cache_file)\n",
    "    else:\n",
    "        # Unpack data loaded from cache file\n",
    "        words, labels = (cache_data['words'], cache_data['labels'])\n",
    "    \n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read preprocessed data from cache file: preprocessed_data.pkl\n"
     ]
    }
   ],
   "source": [
    "# Preprocess data\n",
    "data, labels = preprocess_data(data, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['successful',\n",
       " 'bids',\n",
       " 'european',\n",
       " 'union',\n",
       " 'national',\n",
       " 'heritage',\n",
       " 'lottery',\n",
       " 'funding',\n",
       " 'ensured',\n",
       " 'impact',\n",
       " 'international',\n",
       " 'scope',\n",
       " 'results',\n",
       " 'disseminated',\n",
       " 'via',\n",
       " 'websites',\n",
       " 'print',\n",
       " 'publications',\n",
       " 'media',\n",
       " 'widest',\n",
       " 'constituencies']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform to Bag-of-Words Representation\n",
    "\n",
    "Transform each sentence into a Bag-of-Words feature representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_BoW_transformation(sentences, \n",
    "                          max_features=None,\n",
    "                          transformation='tc',  # it can be either 'tc' (term_count), 'tf', or 'tfidf'\n",
    "                          ngram_range=(1,1)):\n",
    "    if transformation == 'tc':\n",
    "        vectorizer = CountVectorizer(max_features=max_features, ngram_range=ngram_range, \n",
    "                                     preprocessor=lambda x: x, tokenizer=lambda x: x,\n",
    "                                     lowercase=False)\n",
    "    elif transformation == 'tf':\n",
    "        vectorizer = TfidfVectorizer(max_features=max_features, ngram_range=ngram_range, \n",
    "                                     preprocessor=lambda x: x, tokenizer=lambda x: x,\n",
    "                                     use_idf=False, lowercase=False)\n",
    "    elif transformation == 'tfidf':\n",
    "        vectorizer = TfidfVectorizer(max_features=max_features, ngram_range=ngram_range, \n",
    "                                     preprocessor=lambda x: x, tokenizer=lambda x: x,\n",
    "                                     lowercase=False)\n",
    "    bow_features = vectorizer.fit_transform(sentences).toarray()\n",
    "    \n",
    "    return bow_features, vectorizer.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_BoW_features(sentences, \n",
    "                         vocabulary_size=None,\n",
    "                         transformation='tc',  # it can be either 'tc' (term_count), 'tf', or 'tfidf'\n",
    "                         ngram_range=(1,1),\n",
    "                         cache_dir=cache_dir, \n",
    "                         cache_file=None):\n",
    "    \"\"\"Extract Bag-of-Words for a given set of documents, already preprocessed into words.\"\"\"\n",
    "    \n",
    "    # If cache_file is not None, try to read from it first\n",
    "    cache_data = None\n",
    "    if cache_file is not None:\n",
    "        try:\n",
    "            with open(os.path.join(cache_dir, cache_file), \"rb\") as f:\n",
    "                cache_data = joblib.load(f)\n",
    "            print(\"Read features from cache file:\", cache_file)\n",
    "        except:\n",
    "            pass\n",
    "    else:\n",
    "        cache_file = 'bow_features.pkl'\n",
    "    \n",
    "    # If cache is missing, then do the processing\n",
    "    if cache_data is None:\n",
    "        features, vocabulary = do_BoW_transformation(sentences, \n",
    "                                                     vocabulary_size, \n",
    "                                                     transformation, \n",
    "                                                     ngram_range)\n",
    "        cache_data = dict(features=features, vocabulary=vocabulary)\n",
    "        with open(os.path.join(cache_dir, cache_file), \"wb\") as f:\n",
    "            joblib.dump(cache_data, f)  # joblib is an enhanced version of pickle that is more efficient for storing NumPy arrays\n",
    "        print(\"Wrote features to cache file:\", cache_file)\n",
    "    else:\n",
    "        # Unpack data loaded from cache file\n",
    "        features, vocabulary = (cache_data['features'], cache_data['vocabulary'])\n",
    "    \n",
    "    # Return both the extracted features as well as the vocabulary\n",
    "    return features, vocabulary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Text transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read features from cache file: bow_features_tc_100.pkl\n",
      "Read features from cache file: bow_features_tc_200.pkl\n",
      "Read features from cache file: bow_features_tc_400.pkl\n",
      "Read features from cache file: bow_features_tc_500.pkl\n",
      "Read features from cache file: bow_features_tc_800.pkl\n",
      "Read features from cache file: bow_features_tf_100.pkl\n",
      "Read features from cache file: bow_features_tf_200.pkl\n",
      "Read features from cache file: bow_features_tf_400.pkl\n",
      "Read features from cache file: bow_features_tf_500.pkl\n",
      "Read features from cache file: bow_features_tf_800.pkl\n",
      "Read features from cache file: bow_features_tfidf_100.pkl\n",
      "Read features from cache file: bow_features_tfidf_200.pkl\n",
      "Read features from cache file: bow_features_tfidf_400.pkl\n",
      "Read features from cache file: bow_features_tfidf_500.pkl\n",
      "Read features from cache file: bow_features_tfidf_800.pkl\n"
     ]
    }
   ],
   "source": [
    "transformations = ['tc', 'tf', 'tfidf']\n",
    "transformed_txt = dict()\n",
    "max_features=[100, 200, 400, 500, 800]\n",
    "for transformation in transformations:\n",
    "    transformed_txt[transformation] = dict()\n",
    "    for max_feature in max_features:\n",
    "        cache_file_name = 'bow_features_{0}_{1}.pkl'.format(transformation, max_feature)\n",
    "        data_transformed, _ = extract_BoW_features(data, \n",
    "                                                   vocabulary_size=max_feature, \n",
    "                                                   cache_file=cache_file_name, \n",
    "                                                   transformation=transformation) \n",
    "        transformed_txt[transformation][str(max_feature)] = data_transformed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's do some sanity check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.40824829, 0.        ,\n",
       "       0.        , 0.        , 0.40824829, 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.40824829, 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.40824829, 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.40824829, 0.        , 0.        , 0.40824829,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "       0.        , 0.        , 0.        , 0.        , 0.        ])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed_txt['tf']['200'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train and test ML algorithms with the differente text transformations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trainining, testing, and validating machine learning algorithms that are reported to perform well on unbalanced, small, and textual datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_classifier(algorithm_name):\n",
    "    if algorithm_name == 'NB':\n",
    "        classifier = GaussianNB()\n",
    "    elif algorithm_name == 'SVMR':\n",
    "        classifier = SVC(kernel='rbf', gamma='auto')\n",
    "    elif algorithm_name == 'SVML':\n",
    "        classifier = SVC(kernel='linear', gamma='auto')\n",
    "    elif algorithm_name == 'LR1':\n",
    "        classifier = LogisticRegression(penalty='l1', solver='liblinear')\n",
    "    elif algorithm_name == 'LR2':\n",
    "        classifier = LogisticRegression(penalty='l2', solver='liblinear')\n",
    "    elif algorithm_name == 'RF':\n",
    "        params = {\n",
    "            'n_estimators': 600,\n",
    "            'min_samples_split': 5,\n",
    "            'min_samples_leaf': 1,\n",
    "            'max_features': 'auto',\n",
    "            'max_depth': 50,\n",
    "            'bootstrap': True\n",
    "        }\n",
    "        classifier = RandomForestClassifier(**params)\n",
    "    elif algorithm_name == 'XGB':\n",
    "        classifier = XGBClassifier(objective=\"binary:logistic\", eta=0.2, gamma=4, min_child_weight=6)\n",
    "    else:\n",
    "        print(\"Unknown algorithm: {0}\",format(algorithm_name))\n",
    "\n",
    "    return classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_cross_validation(kfold, classifier, data, labels):\n",
    "    scores = {\n",
    "        'balanced_accuracy': [],\n",
    "        'f1': [],\n",
    "        'recall': [],\n",
    "        'precision': []\n",
    "    }\n",
    "    for train_index, test_index in kfold.split(data):   \n",
    "        kf_X_train, kf_X_test = data[train_index], data[test_index]\n",
    "        kf_y_train, kf_y_test = labels[train_index], labels[test_index]        \n",
    "        classifier.fit(kf_X_train, kf_y_train)\n",
    "        y_pred = classifier.predict(kf_X_test)\n",
    "        scores['balanced_accuracy'].append(metrics.balanced_accuracy_score(kf_y_test, y_pred))\n",
    "        scores['f1'].append(metrics.f1_score(kf_y_test, y_pred))\n",
    "        scores['recall'].append(metrics.recall_score(kf_y_test, y_pred))\n",
    "        scores['precision'].append(metrics.precision_score(kf_y_test, y_pred))\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformation: tc, Max Features: 100, Algorithm: NB\n",
      "Transformation: tc, Max Features: 100, Algorithm: SVMR\n",
      "Transformation: tc, Max Features: 100, Algorithm: SVML\n",
      "Transformation: tc, Max Features: 100, Algorithm: LR1\n",
      "Transformation: tc, Max Features: 100, Algorithm: LR2\n",
      "Transformation: tc, Max Features: 100, Algorithm: RF\n",
      "Transformation: tc, Max Features: 100, Algorithm: XGB\n",
      "Transformation: tc, Max Features: 200, Algorithm: NB\n",
      "Transformation: tc, Max Features: 200, Algorithm: SVMR\n",
      "Transformation: tc, Max Features: 200, Algorithm: SVML\n",
      "Transformation: tc, Max Features: 200, Algorithm: LR1\n",
      "Transformation: tc, Max Features: 200, Algorithm: LR2\n",
      "Transformation: tc, Max Features: 200, Algorithm: RF\n",
      "Transformation: tc, Max Features: 200, Algorithm: XGB\n",
      "Transformation: tc, Max Features: 400, Algorithm: NB\n",
      "Transformation: tc, Max Features: 400, Algorithm: SVMR\n",
      "Transformation: tc, Max Features: 400, Algorithm: SVML\n",
      "Transformation: tc, Max Features: 400, Algorithm: LR1\n",
      "Transformation: tc, Max Features: 400, Algorithm: LR2\n",
      "Transformation: tc, Max Features: 400, Algorithm: RF\n",
      "Transformation: tc, Max Features: 400, Algorithm: XGB\n",
      "Transformation: tc, Max Features: 500, Algorithm: NB\n",
      "Transformation: tc, Max Features: 500, Algorithm: SVMR\n",
      "Transformation: tc, Max Features: 500, Algorithm: SVML\n",
      "Transformation: tc, Max Features: 500, Algorithm: LR1\n",
      "Transformation: tc, Max Features: 500, Algorithm: LR2\n",
      "Transformation: tc, Max Features: 500, Algorithm: RF\n",
      "Transformation: tc, Max Features: 500, Algorithm: XGB\n",
      "Transformation: tc, Max Features: 800, Algorithm: NB\n",
      "Transformation: tc, Max Features: 800, Algorithm: SVMR\n",
      "Transformation: tc, Max Features: 800, Algorithm: SVML\n",
      "Transformation: tc, Max Features: 800, Algorithm: LR1\n",
      "Transformation: tc, Max Features: 800, Algorithm: LR2\n",
      "Transformation: tc, Max Features: 800, Algorithm: RF\n",
      "Transformation: tc, Max Features: 800, Algorithm: XGB\n",
      "Transformation: tf, Max Features: 100, Algorithm: NB\n",
      "Transformation: tf, Max Features: 100, Algorithm: SVMR\n",
      "Transformation: tf, Max Features: 100, Algorithm: SVML\n",
      "Transformation: tf, Max Features: 100, Algorithm: LR1\n",
      "Transformation: tf, Max Features: 100, Algorithm: LR2\n",
      "Transformation: tf, Max Features: 100, Algorithm: RF\n",
      "Transformation: tf, Max Features: 100, Algorithm: XGB\n",
      "Transformation: tf, Max Features: 200, Algorithm: NB\n",
      "Transformation: tf, Max Features: 200, Algorithm: SVMR\n",
      "Transformation: tf, Max Features: 200, Algorithm: SVML\n",
      "Transformation: tf, Max Features: 200, Algorithm: LR1\n",
      "Transformation: tf, Max Features: 200, Algorithm: LR2\n",
      "Transformation: tf, Max Features: 200, Algorithm: RF\n",
      "Transformation: tf, Max Features: 200, Algorithm: XGB\n",
      "Transformation: tf, Max Features: 400, Algorithm: NB\n",
      "Transformation: tf, Max Features: 400, Algorithm: SVMR\n",
      "Transformation: tf, Max Features: 400, Algorithm: SVML\n",
      "Transformation: tf, Max Features: 400, Algorithm: LR1\n",
      "Transformation: tf, Max Features: 400, Algorithm: LR2\n",
      "Transformation: tf, Max Features: 400, Algorithm: RF\n",
      "Transformation: tf, Max Features: 400, Algorithm: XGB\n",
      "Transformation: tf, Max Features: 500, Algorithm: NB\n",
      "Transformation: tf, Max Features: 500, Algorithm: SVMR\n",
      "Transformation: tf, Max Features: 500, Algorithm: SVML\n",
      "Transformation: tf, Max Features: 500, Algorithm: LR1\n",
      "Transformation: tf, Max Features: 500, Algorithm: LR2\n",
      "Transformation: tf, Max Features: 500, Algorithm: RF\n",
      "Transformation: tf, Max Features: 500, Algorithm: XGB\n",
      "Transformation: tf, Max Features: 800, Algorithm: NB\n",
      "Transformation: tf, Max Features: 800, Algorithm: SVMR\n",
      "Transformation: tf, Max Features: 800, Algorithm: SVML\n",
      "Transformation: tf, Max Features: 800, Algorithm: LR1\n",
      "Transformation: tf, Max Features: 800, Algorithm: LR2\n",
      "Transformation: tf, Max Features: 800, Algorithm: RF\n",
      "Transformation: tf, Max Features: 800, Algorithm: XGB\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: NB\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: SVMR\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: SVML\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: LR1\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: LR2\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: RF\n",
      "Transformation: tfidf, Max Features: 100, Algorithm: XGB\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: NB\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: SVMR\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: SVML\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: LR1\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: LR2\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: RF\n",
      "Transformation: tfidf, Max Features: 200, Algorithm: XGB\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: NB\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: SVMR\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: SVML\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: LR1\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: LR2\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: RF\n",
      "Transformation: tfidf, Max Features: 400, Algorithm: XGB\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: NB\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: SVMR\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: SVML\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: LR1\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: LR2\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: RF\n",
      "Transformation: tfidf, Max Features: 500, Algorithm: XGB\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: NB\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: SVMR\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: SVML\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: LR1\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: LR2\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: RF\n",
      "Transformation: tfidf, Max Features: 800, Algorithm: XGB\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "kfold = KFold(n_splits=10, shuffle=True, random_state=42)\n",
    "algorithms = ['NB', 'SVMR', 'SVML', 'LR1', 'LR2', 'RF', 'XGB']\n",
    "outputs = [] \n",
    "\n",
    "for transformation in transformations:\n",
    "    for max_feature in max_features:\n",
    "        t_data = transformed_txt[transformation][str(max_feature)]\n",
    "        for algorithm in algorithms:\n",
    "            print('Transformation: {0}, Max Features: {1}, Algorithm: {2}'.format(transformation, max_feature, \n",
    "                                                                                  algorithm))\n",
    "            classifier = get_classifier(algorithm)\n",
    "            scores = do_cross_validation(kfold, classifier, t_data, labels)\n",
    "            outputs.append(\n",
    "                {\n",
    "                    'algorithm': algorithm,\n",
    "                    'transformation': transformation,\n",
    "                    'max_features': max_feature,\n",
    "                    'metrics': scores\n",
    "                }\n",
    "            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save results into a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_df = pd.DataFrame(columns=['algorithm', 'transformation', 'max_features', 'balanced_accuracy', 'f1'])\n",
    "for output in outputs:\n",
    "    row = {\n",
    "        'algorithm': output['algorithm'],\n",
    "        'transformation': output['transformation'],\n",
    "        'max_features': output['max_features'],\n",
    "        'mean_balanced_accuracy': round(np.array(output['metrics']['balanced_accuracy']).mean(), 2),\n",
    "        'mean_recall': round(np.array(output['metrics']['recall']).mean(), 2),\n",
    "        'mean_precision': round(np.array(output['metrics']['precision']).mean(), 2),\n",
    "        'mean_f1': round(np.array(output['metrics']['f1']).mean(), 2),\n",
    "    }\n",
    "    output_df = output_df.append(row, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>transformation</th>\n",
       "      <th>max_features</th>\n",
       "      <th>balanced_accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NB</td>\n",
       "      <td>tc</td>\n",
       "      <td>100</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SVMR</td>\n",
       "      <td>tc</td>\n",
       "      <td>100</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SVML</td>\n",
       "      <td>tc</td>\n",
       "      <td>100</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LR1</td>\n",
       "      <td>tc</td>\n",
       "      <td>100</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LR2</td>\n",
       "      <td>tc</td>\n",
       "      <td>100</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  algorithm transformation max_features  balanced_accuracy    f1  recall\n",
       "0        NB             tc          100               0.65  0.51    0.55\n",
       "1      SVMR             tc          100               0.51  0.04    0.02\n",
       "2      SVML             tc          100               0.63  0.45    0.36\n",
       "3       LR1             tc          100               0.63  0.45    0.35\n",
       "4       LR2             tc          100               0.63  0.45    0.36"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Look at the maximum Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>transformation</th>\n",
       "      <th>max_features</th>\n",
       "      <th>balanced_accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SVML</td>\n",
       "      <td>tc</td>\n",
       "      <td>200</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>LR1</td>\n",
       "      <td>tc</td>\n",
       "      <td>400</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>LR1</td>\n",
       "      <td>tc</td>\n",
       "      <td>500</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   algorithm transformation max_features  balanced_accuracy   f1  recall\n",
       "9       SVML             tc          200               0.66  0.5    0.45\n",
       "17       LR1             tc          400               0.66  0.5    0.43\n",
       "24       LR1             tc          500               0.66  0.5    0.43"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_balanced_accuracy = output_df['mean_balanced_accuracy'].max()\n",
    "output_df[output_df['mean_balanced_accuracy'] == max_balanced_accuracy]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspect the maximum Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>transformation</th>\n",
       "      <th>max_features</th>\n",
       "      <th>balanced_accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>NB</td>\n",
       "      <td>tc</td>\n",
       "      <td>500</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   algorithm transformation max_features  balanced_accuracy    f1  recall\n",
       "21        NB             tc          500               0.51  0.42    0.74"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_recall = output_df['mean_recall'].max()\n",
    "output_df[output_df['mean_recall'] == max_recall]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspect the maximum Precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>transformation</th>\n",
       "      <th>max_features</th>\n",
       "      <th>balanced_accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>NB</td>\n",
       "      <td>tc</td>\n",
       "      <td>500</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   algorithm transformation max_features  balanced_accuracy    f1  recall\n",
       "21        NB             tc          500               0.51  0.42    0.74"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_recall = output_df['mean_precision'].max()\n",
    "output_df[output_df['mean_precision'] == max_recall]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspect the maximun F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>transformation</th>\n",
       "      <th>max_features</th>\n",
       "      <th>balanced_accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NB</td>\n",
       "      <td>tc</td>\n",
       "      <td>100</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  algorithm transformation max_features  balanced_accuracy    f1  recall\n",
       "0        NB             tc          100               0.65  0.51    0.55"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_f1 = output_df['mean_f1'].max()\n",
    "output_df[output_df['mean_f1'] == max_f1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save results to a csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_df.to_csv('./experiments/e_06022020.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the data in train and test datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = model_selection.train_test_split(data_tc, labels, test_size=0.20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check size of datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 822 records\n"
     ]
    }
   ],
   "source": [
    "print(\"Train: {0} records\".format(len(X_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test: 206 records\n"
     ]
    }
   ],
   "source": [
    "print(\"Test: {0} records\".format(len(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the percentage of true cases in the datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of true cases in train: 29.0%\n"
     ]
    }
   ],
   "source": [
    "print(\"Percentage of true cases in train: {0}%\".format(round(100*len(Y_train[Y_train==1])/len(Y_train),0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of true cases in test: 30.0%\n"
     ]
    }
   ],
   "source": [
    "print(\"Percentage of true cases in test: {0}%\".format(round(100*len(Y_test[Y_test==1])/len(Y_test), 0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train, Test, and Validate ML Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trainining, testing, and validating machine learning algorithms that are reported to perform well on unbalanced, small, and textual datasets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive-Bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None, var_smoothing=1e-09)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_classifier = GaussianNB()\n",
    "nb_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = nb_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[85 65]\n",
      " [17 39]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48750000000000004\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.631547619047619\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine (RBF Kernel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_rbf_classifier = SVC(kernel='rbf', gamma='auto')\n",
    "svm_rbf_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = svm_rbf_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[150   0]\n",
      " [ 56   0]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine (Linear Kernel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_linear_classifier = SVC(kernel='linear', gamma='auto')\n",
    "svm_linear_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = svm_linear_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[131  19]\n",
      " [ 30  26]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5148514851485149\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6688095238095237\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, eta=0.2, gamma=4,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "              min_child_weight=6, missing=None, n_estimators=100, n_jobs=1,\n",
       "              nthread=None, objective='binary:logistic', random_state=0,\n",
       "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "              silent=None, subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_classifier = XGBClassifier(objective=\"binary:logistic\", eta=0.2, gamma=4, min_child_weight=6)\n",
    "xgb_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = xgb_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[147   3]\n",
      " [ 49   7]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.21212121212121213\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5525\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_classifier = RandomForestClassifier(n_estimators=1000)\n",
    "rf_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = rf_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[129  15]\n",
      " [ 38  24]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47524752475247517\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6414650537634409\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cohen-kappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3163431433938635\n"
     ]
    }
   ],
   "source": [
    "cohen_kappa = metrics.cohen_kappa_score(Y_test, pred)\n",
    "print(cohen_kappa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5698839662447257, 0.6281453867660765, 0.6195378151260504, 0.6764964788732395, 0.7157534246575342, 0.5307125307125308, 0.5481735159817351, 0.5743589743589743, 0.5970713273500237, 0.6589513462446859]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "kfold = KFold(n_splits=10, shuffle=True, random_state=42)\n",
    "scores = []\n",
    "\n",
    "for train_index, test_index in kfold.split(data_tc):   \n",
    "    kf_X_train, kf_X_test = data_tc[train_index], data_tc[test_index]\n",
    "    kf_y_train, kf_y_test = labels[train_index], labels[test_index]\n",
    "    rf_classifier = RandomForestClassifier(n_estimators=1000)\n",
    "    rf_classifier.fit(kf_X_train, kf_y_train)\n",
    "    y_pred = rf_classifier.predict(kf_X_test)\n",
    "    scores.append(metrics.balanced_accuracy_score(kf_y_test, y_pred))\n",
    "    \n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6119084766315577"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(scores).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.67853881 0.51392694 0.60205479 0.64908676 0.56187215 0.62853881\n",
      " 0.6109589  0.57945205 0.69027778 0.57222222]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "rf_classifier = RandomForestClassifier(n_estimators=1000)\n",
    "scores = cross_val_score(rf_classifier,data_tc, labels, scoring=\"balanced_accuracy\", cv=10)\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6086929223744293"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(scores).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, None], 'min_samples_split': [2, 5, 7, 10], 'min_samples_leaf': [1, 2, 3], 'bootstrap': [True, False]}\n"
     ]
    }
   ],
   "source": [
    "# Set the parameters\n",
    "n_estimators = [int(x) for x in np.linspace(start=100, stop=1000, num=10)]\n",
    "max_features = ['auto', 'sqrt']\n",
    "max_depth = [int(x) for x in np.linspace(10, 100, num = 10)]\n",
    "max_depth.append(None)\n",
    "min_samples_split = [2, 5, 7, 10]\n",
    "min_samples_leaf = [1, 2, 3]\n",
    "bootstrap = [True, False]\n",
    "# Set parameters grid \n",
    "param_grid = {'n_estimators': n_estimators,\n",
    "              'max_features': max_features,\n",
    "              'max_depth': max_depth,\n",
    "              'min_samples_split': min_samples_split,\n",
    "              'min_samples_leaf': min_samples_leaf,\n",
    "              'bootstrap': bootstrap}\n",
    "print(param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 50 candidates, totalling 250 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  25 tasks      | elapsed:    6.1s\n",
      "[Parallel(n_jobs=-1)]: Done 146 tasks      | elapsed:   41.1s\n",
      "[Parallel(n_jobs=-1)]: Done 250 out of 250 | elapsed:  1.1min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, error_score=nan,\n",
       "                   estimator=RandomForestClassifier(bootstrap=True,\n",
       "                                                    ccp_alpha=0.0,\n",
       "                                                    class_weight=None,\n",
       "                                                    criterion='gini',\n",
       "                                                    max_depth=None,\n",
       "                                                    max_features='auto',\n",
       "                                                    max_leaf_nodes=None,\n",
       "                                                    max_samples=None,\n",
       "                                                    min_impurity_decrease=0.0,\n",
       "                                                    min_impurity_split=None,\n",
       "                                                    min_samples_leaf=1,\n",
       "                                                    min_samples_split=2,\n",
       "                                                    min_weight_fraction_leaf=0.0,\n",
       "                                                    n_estimators=100,\n",
       "                                                    n_jobs...\n",
       "                   iid='deprecated', n_iter=50, n_jobs=-1,\n",
       "                   param_distributions={'bootstrap': [True, False],\n",
       "                                        'max_depth': [10, 20, 30, 40, 50, 60,\n",
       "                                                      70, 80, 90, 100, None],\n",
       "                                        'max_features': ['auto', 'sqrt'],\n",
       "                                        'min_samples_leaf': [1, 2, 3],\n",
       "                                        'min_samples_split': [2, 5, 7, 10],\n",
       "                                        'n_estimators': [100, 200, 300, 400,\n",
       "                                                         500, 600, 700, 800,\n",
       "                                                         900, 1000]},\n",
       "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
       "                   return_train_score=False, scoring=None, verbose=2)"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_classifier = RandomForestClassifier()\n",
    "rf_random = RandomizedSearchCV(estimator = rf_classifier, param_distributions = param_grid, \n",
    "                               n_iter = 50, cv = 5, verbose=2, random_state=42, n_jobs = -1)\n",
    "rf_random.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 600,\n",
       " 'min_samples_split': 5,\n",
       " 'min_samples_leaf': 1,\n",
       " 'max_features': 'auto',\n",
       " 'max_depth': 50,\n",
       " 'bootstrap': True}"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = rf_random.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[131  13]\n",
      " [ 38  24]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48484848484848475\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6484094982078853\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cohen-kappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3353157028976338\n"
     ]
    }
   ],
   "source": [
    "cohen_kappa = metrics.cohen_kappa_score(Y_test, pred)\n",
    "print(cohen_kappa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lor_classifier = LogisticRegression(penalty='l2')\n",
    "lor_classifier.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = lor_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[133  17]\n",
      " [ 32  24]]\n"
     ]
    }
   ],
   "source": [
    "conf_matrix = metrics.confusion_matrix(Y_test.values, pred)\n",
    "print(conf_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "F1-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4948453608247423\n"
     ]
    }
   ],
   "source": [
    "f1_score = metrics.f1_score(Y_test, pred)\n",
    "print(f1_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Balanced Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6576190476190477\n"
     ]
    }
   ],
   "source": [
    "b_accuracy = metrics.balanced_accuracy_score(Y_test, pred)\n",
    "print(b_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}