AutoGluon_Heart_Disease-Combined.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "249b9a3a",
   "metadata": {},
   "source": [
    "# SETUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d1b36d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import metrics\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "# Import AutoGluon Libraries\n",
    "from autogluon.tabular import TabularDataset, TabularPredictor\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "33c122f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: autogluon.tabular in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (0.7.0)\n",
      "Requirement already satisfied: autogluon.core==0.7.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (0.7.0)\n",
      "Requirement already satisfied: scipy<1.12,>=1.5.4 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (1.9.1)\n",
      "Requirement already satisfied: pandas<1.6,>=1.4.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (1.4.4)\n",
      "Requirement already satisfied: numpy<1.27,>=1.21 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (1.21.5)\n",
      "Requirement already satisfied: networkx<3.0,>=2.3 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (2.8.4)\n",
      "Requirement already satisfied: scikit-learn<1.3,>=1.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (1.0.2)\n",
      "Requirement already satisfied: autogluon.features==0.7.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.tabular) (0.7.0)\n",
      "Requirement already satisfied: matplotlib in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.core==0.7.0->autogluon.tabular) (3.5.2)\n",
      "Requirement already satisfied: autogluon.common==0.7.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.core==0.7.0->autogluon.tabular) (0.7.0)\n",
      "Requirement already satisfied: boto3<2,>=1.10 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.core==0.7.0->autogluon.tabular) (1.24.28)\n",
      "Requirement already satisfied: tqdm<5,>=4.38 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.core==0.7.0->autogluon.tabular) (4.64.1)\n",
      "Requirement already satisfied: requests in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.core==0.7.0->autogluon.tabular) (2.28.1)\n",
      "Requirement already satisfied: psutil<6,>=5.7.3 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.common==0.7.0->autogluon.core==0.7.0->autogluon.tabular) (5.9.0)\n",
      "Requirement already satisfied: setuptools in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from autogluon.common==0.7.0->autogluon.core==0.7.0->autogluon.tabular) (63.4.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from pandas<1.6,>=1.4.1->autogluon.tabular) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from pandas<1.6,>=1.4.1->autogluon.tabular) (2022.1)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from scikit-learn<1.3,>=1.0->autogluon.tabular) (1.1.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from scikit-learn<1.3,>=1.0->autogluon.tabular) (2.2.0)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from boto3<2,>=1.10->autogluon.core==0.7.0->autogluon.tabular) (0.10.0)\n",
      "Requirement already satisfied: botocore<1.28.0,>=1.27.28 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from boto3<2,>=1.10->autogluon.core==0.7.0->autogluon.tabular) (1.27.28)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from boto3<2,>=1.10->autogluon.core==0.7.0->autogluon.tabular) (0.6.0)\n",
      "Requirement already satisfied: six>=1.5 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from python-dateutil>=2.8.1->pandas<1.6,>=1.4.1->autogluon.tabular) (1.16.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from matplotlib->autogluon.core==0.7.0->autogluon.tabular) (0.11.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from matplotlib->autogluon.core==0.7.0->autogluon.tabular) (21.3)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from matplotlib->autogluon.core==0.7.0->autogluon.tabular) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from matplotlib->autogluon.core==0.7.0->autogluon.tabular) (3.0.9)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from matplotlib->autogluon.core==0.7.0->autogluon.tabular) (4.25.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from matplotlib->autogluon.core==0.7.0->autogluon.tabular) (1.4.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from requests->autogluon.core==0.7.0->autogluon.tabular) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from requests->autogluon.core==0.7.0->autogluon.tabular) (2022.9.24)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from requests->autogluon.core==0.7.0->autogluon.tabular) (2.0.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages (from requests->autogluon.core==0.7.0->autogluon.tabular) (1.26.11)\n"
     ]
    }
   ],
   "source": [
    "# Install AutoGluon\n",
    "!pip install autogluon.tabular"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fb8c3338",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load Cleveland datafile\n",
    "cleveland_csv_path = \"processed.cleveland.data\"\n",
    "cleveland_data = pd.read_csv(cleveland_csv_path, header = None)\n",
    "# assign column names\n",
    "cleveland_data.set_axis(['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs',\n",
    "                  'restecg', 'thalach', 'exang', 'oldpeak', 'slope',\n",
    "                  'ca', 'thal', 'num'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b10b475b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define label column\n",
    "label = 'num'\n",
    "heart_label = 'num'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99198924",
   "metadata": {},
   "source": [
    "# SIMPLEST OF MODELS\n",
    "zero data preprocessing performed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe4fa752",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  63.0  1.0  1.0     145.0  233.0  1.0      2.0    150.0    0.0      2.3   \n",
       "1  67.0  1.0  4.0     160.0  286.0  0.0      2.0    108.0    1.0      1.5   \n",
       "2  67.0  1.0  4.0     120.0  229.0  0.0      2.0    129.0    1.0      2.6   \n",
       "3  37.0  1.0  3.0     130.0  250.0  0.0      0.0    187.0    0.0      3.5   \n",
       "4  41.0  0.0  2.0     130.0  204.0  0.0      2.0    172.0    0.0      1.4   \n",
       "\n",
       "   slope   ca thal  num  \n",
       "0    3.0  0.0  6.0    0  \n",
       "1    2.0  3.0  3.0    2  \n",
       "2    2.0  2.0  7.0    1  \n",
       "3    3.0  0.0  3.0    0  \n",
       "4    1.0  0.0  3.0    0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c9ce854a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 303 entries, 0 to 302\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       303 non-null    float64\n",
      " 1   sex       303 non-null    float64\n",
      " 2   cp        303 non-null    float64\n",
      " 3   trestbps  303 non-null    float64\n",
      " 4   chol      303 non-null    float64\n",
      " 5   fbs       303 non-null    float64\n",
      " 6   restecg   303 non-null    float64\n",
      " 7   thalach   303 non-null    float64\n",
      " 8   exang     303 non-null    float64\n",
      " 9   oldpeak   303 non-null    float64\n",
      " 10  slope     303 non-null    float64\n",
      " 11  ca        303 non-null    object \n",
      " 12  thal      303 non-null    object \n",
      " 13  num       303 non-null    int64  \n",
      "dtypes: float64(11), int64(1), object(2)\n",
      "memory usage: 33.3+ KB\n"
     ]
    }
   ],
   "source": [
    "cleveland_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "559c4ba0",
   "metadata": {},
   "source": [
    "## BINARY MODELING\n",
    "(Note: \"Cleveland\" label here would need to be revised because used in Slightly Better Model below when split the data, if care.)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f298a4c",
   "metadata": {},
   "source": [
    "### PREPARE DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4b65f8ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/sg/9r6jl3rj2l96943q_2ph1y980000gn/T/ipykernel_40379/20290669.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cleveland_binary['num'][cleveland_binary['num'] > 0] = 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  63.0  1.0  1.0     145.0  233.0  1.0      2.0    150.0    0.0      2.3   \n",
       "1  67.0  1.0  4.0     160.0  286.0  0.0      2.0    108.0    1.0      1.5   \n",
       "2  67.0  1.0  4.0     120.0  229.0  0.0      2.0    129.0    1.0      2.6   \n",
       "3  37.0  1.0  3.0     130.0  250.0  0.0      0.0    187.0    0.0      3.5   \n",
       "4  41.0  0.0  2.0     130.0  204.0  0.0      2.0    172.0    0.0      1.4   \n",
       "\n",
       "   slope   ca thal  num  \n",
       "0    3.0  0.0  6.0    0  \n",
       "1    2.0  3.0  3.0    1  \n",
       "2    2.0  2.0  7.0    1  \n",
       "3    3.0  0.0  3.0    0  \n",
       "4    1.0  0.0  3.0    0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make copy for binary labeling\n",
    "cleveland_binary = cleveland_data.copy()\n",
    "cleveland_binary['num'][cleveland_binary['num'] > 0] = 1\n",
    "cleveland_binary.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b11cbefe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>45.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>59.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>61.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>68.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>66.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "100  45.0  1.0  4.0     115.0  260.0  0.0      2.0    185.0    0.0      0.0   \n",
       "68   59.0  1.0  4.0     170.0  326.0  0.0      2.0    140.0    1.0      3.4   \n",
       "108  61.0  1.0  4.0     120.0  260.0  0.0      0.0    140.0    1.0      3.6   \n",
       "299  68.0  1.0  4.0     144.0  193.0  1.0      0.0    141.0    0.0      3.4   \n",
       "229  66.0  1.0  4.0     112.0  212.0  0.0      2.0    132.0    1.0      0.1   \n",
       "\n",
       "     slope   ca thal  num  \n",
       "100    1.0  0.0  3.0    0  \n",
       "68     3.0  0.0  7.0    1  \n",
       "108    2.0  1.0  7.0    1  \n",
       "299    2.0  2.0  7.0    1  \n",
       "229    1.0  1.0  3.0    1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the binary data\n",
    "from sklearn.model_selection import train_test_split\n",
    "cleveland_train, cleveland_test = train_test_split(cleveland_binary, test_size=0.2, random_state=42)\n",
    "cleveland_test_labels = cleveland_test[heart_label]\n",
    "cleveland_test_nolabel = cleveland_test.drop([heart_label], axis=1)\n",
    "cleveland_train_labels = cleveland_train[heart_label]\n",
    "cleveland_train_nolabel = cleveland_train.drop([heart_label], axis=1)\n",
    "cleveland_train.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c63d073",
   "metadata": {},
   "source": [
    "### MODEL - TRAIN & EVALUATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa5160a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    242\n",
      "Train Data Columns: 13\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [0, 1]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    313.57 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.05 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 3 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', [])  : 11 | ['age', 'sex', 'cp', 'trestbps', 'chol', ...]\n",
      "\t\t('object', []) :  2 | ['ca', 'thal']\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', [])  : 2 | ['ca', 'thal']\n",
      "\t\t('float', [])     : 8 | ['age', 'cp', 'trestbps', 'chol', 'restecg', ...]\n",
      "\t\t('int', ['bool']) : 3 | ['sex', 'fbs', 'exang']\n",
      "\t0.0s = Fit runtime\n",
      "\t13 features in original data used to generate 13 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.05s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 193, Val Rows: 49\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.6735\t = Validation score   (accuracy)\n",
      "\t2.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.6531\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.8571\t = Validation score   (accuracy)\n",
      "\t0.3s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.8571\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8776\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.8367\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 310 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.8367\t = Validation score   (accuracy)\n",
      "\t0.08s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.09s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 3.53s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "cle_TabPre = TabularPredictor(label=heart_label, path='.').fit(cleveland_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "464722b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8852459016393442\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8852459016393442,\n",
      "    \"balanced_accuracy\": 0.8873922413793103,\n",
      "    \"mcc\": 0.7747844827586207,\n",
      "    \"f1\": 0.8852459016393444,\n",
      "    \"precision\": 0.9310344827586207,\n",
      "    \"recall\": 0.84375\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Check the performance using test data\n",
    "cle_pred = cle_TabPre.predict(cleveland_test_nolabel)\n",
    "cle_perf = cle_TabPre.evaluate_predictions(y_true=cleveland_test_labels, y_pred=cle_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "785cc012",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8852459016393442\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8852459016393442,\n",
      "    \"balanced_accuracy\": 0.8873922413793103,\n",
      "    \"mcc\": 0.7747844827586207,\n",
      "    \"roc_auc\": 0.9461206896551724,\n",
      "    \"f1\": 0.8852459016393444,\n",
      "    \"precision\": 0.9310344827586207,\n",
      "    \"recall\": 0.84375\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cle_perf2 = cle_TabPre.evaluate(data=cleveland_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7b8b33b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8852459016393442\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8852459016393442,\n",
      "    \"balanced_accuracy\": 0.8873922413793103,\n",
      "    \"mcc\": 0.7747844827586207,\n",
      "    \"roc_auc\": 0.9461206896551724,\n",
      "    \"f1\": 0.8852459016393444,\n",
      "    \"precision\": 0.9310344827586207,\n",
      "    \"recall\": 0.84375\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cle_perf3 = cle_TabPre.evaluate(data=cleveland_test, auxiliary_metrics=True, model='ExtraTreesGini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c53e2cbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8852459016393442\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8852459016393442,\n",
      "    \"balanced_accuracy\": 0.8873922413793103,\n",
      "    \"mcc\": 0.7747844827586207,\n",
      "    \"roc_auc\": 0.9558189655172414,\n",
      "    \"f1\": 0.8852459016393444,\n",
      "    \"precision\": 0.9310344827586207,\n",
      "    \"recall\": 0.84375\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cle_perf4 = cle_TabPre.evaluate(data=cleveland_test, auxiliary_metrics=True, model='ExtraTreesEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bc661f9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.885246</td>\n",
       "      <td>0.877551</td>\n",
       "      <td>0.027582</td>\n",
       "      <td>0.021327</td>\n",
       "      <td>0.255840</td>\n",
       "      <td>0.027582</td>\n",
       "      <td>0.021327</td>\n",
       "      <td>0.255840</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.885246</td>\n",
       "      <td>0.836735</td>\n",
       "      <td>0.028082</td>\n",
       "      <td>0.021282</td>\n",
       "      <td>0.254199</td>\n",
       "      <td>0.028082</td>\n",
       "      <td>0.021282</td>\n",
       "      <td>0.254199</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.885246</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.061902</td>\n",
       "      <td>0.046186</td>\n",
       "      <td>0.642596</td>\n",
       "      <td>0.001530</td>\n",
       "      <td>0.001469</td>\n",
       "      <td>0.091696</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.885246</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.125380</td>\n",
       "      <td>0.020789</td>\n",
       "      <td>0.255968</td>\n",
       "      <td>0.125380</td>\n",
       "      <td>0.020789</td>\n",
       "      <td>0.255968</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.868852</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.032790</td>\n",
       "      <td>0.023390</td>\n",
       "      <td>0.295060</td>\n",
       "      <td>0.032790</td>\n",
       "      <td>0.023390</td>\n",
       "      <td>0.295060</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.836066</td>\n",
       "      <td>0.836735</td>\n",
       "      <td>0.007129</td>\n",
       "      <td>0.003149</td>\n",
       "      <td>0.084084</td>\n",
       "      <td>0.007129</td>\n",
       "      <td>0.003149</td>\n",
       "      <td>0.084084</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.672131</td>\n",
       "      <td>0.673469</td>\n",
       "      <td>0.006293</td>\n",
       "      <td>0.002785</td>\n",
       "      <td>2.016339</td>\n",
       "      <td>0.006293</td>\n",
       "      <td>0.002785</td>\n",
       "      <td>2.016339</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.655738</td>\n",
       "      <td>0.653061</td>\n",
       "      <td>0.004014</td>\n",
       "      <td>0.001086</td>\n",
       "      <td>0.004252</td>\n",
       "      <td>0.004014</td>\n",
       "      <td>0.001086</td>\n",
       "      <td>0.004252</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0       ExtraTreesGini    0.885246   0.877551        0.027582       0.021327   \n",
       "1       ExtraTreesEntr    0.885246   0.836735        0.028082       0.021282   \n",
       "2  WeightedEnsemble_L2    0.885246   0.897959        0.061902       0.046186   \n",
       "3     RandomForestEntr    0.885246   0.857143        0.125380       0.020789   \n",
       "4     RandomForestGini    0.868852   0.857143        0.032790       0.023390   \n",
       "5              XGBoost    0.836066   0.836735        0.007129       0.003149   \n",
       "6       KNeighborsUnif    0.672131   0.673469        0.006293       0.002785   \n",
       "7       KNeighborsDist    0.655738   0.653061        0.004014       0.001086   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.255840                 0.027582                0.021327   \n",
       "1  0.254199                 0.028082                0.021282   \n",
       "2  0.642596                 0.001530                0.001469   \n",
       "3  0.255968                 0.125380                0.020789   \n",
       "4  0.295060                 0.032790                0.023390   \n",
       "5  0.084084                 0.007129                0.003149   \n",
       "6  2.016339                 0.006293                0.002785   \n",
       "7  0.004252                 0.004014                0.001086   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.255840            1       True          5  \n",
       "1           0.254199            1       True          6  \n",
       "2           0.091696            2       True          8  \n",
       "3           0.255968            1       True          4  \n",
       "4           0.295060            1       True          3  \n",
       "5           0.084084            1       True          7  \n",
       "6           2.016339            1       True          1  \n",
       "7           0.004252            1       True          2  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "cle_TabPre.leaderboard(cleveland_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "af2e56cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "cle_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64034ebf",
   "metadata": {},
   "source": [
    "## MULTICLASS MODELING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "384b82b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  63.0  1.0  1.0     145.0  233.0  1.0      2.0    150.0    0.0      2.3   \n",
       "1  67.0  1.0  4.0     160.0  286.0  0.0      2.0    108.0    1.0      1.5   \n",
       "2  67.0  1.0  4.0     120.0  229.0  0.0      2.0    129.0    1.0      2.6   \n",
       "3  37.0  1.0  3.0     130.0  250.0  0.0      0.0    187.0    0.0      3.5   \n",
       "4  41.0  0.0  2.0     130.0  204.0  0.0      2.0    172.0    0.0      1.4   \n",
       "\n",
       "   slope   ca thal  num  \n",
       "0    3.0  0.0  6.0    0  \n",
       "1    2.0  3.0  3.0    1  \n",
       "2    2.0  2.0  7.0    1  \n",
       "3    3.0  0.0  3.0    0  \n",
       "4    1.0  0.0  3.0    0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_binary.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "83e7b93f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  63.0  1.0  1.0     145.0  233.0  1.0      2.0    150.0    0.0      2.3   \n",
       "1  67.0  1.0  4.0     160.0  286.0  0.0      2.0    108.0    1.0      1.5   \n",
       "2  67.0  1.0  4.0     120.0  229.0  0.0      2.0    129.0    1.0      2.6   \n",
       "3  37.0  1.0  3.0     130.0  250.0  0.0      0.0    187.0    0.0      3.5   \n",
       "4  41.0  0.0  2.0     130.0  204.0  0.0      2.0    172.0    0.0      1.4   \n",
       "\n",
       "   slope   ca thal  num  \n",
       "0    3.0  0.0  6.0    0  \n",
       "1    2.0  3.0  3.0    2  \n",
       "2    2.0  2.0  7.0    1  \n",
       "3    3.0  0.0  3.0    0  \n",
       "4    1.0  0.0  3.0    0  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4720e3b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>205.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230</th>\n",
       "      <td>52.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>169.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>48.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>255.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>58.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>173.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>47.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>275.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "142  52.0  1.0  2.0     128.0  205.0  1.0      0.0    184.0    0.0      0.0   \n",
       "230  52.0  0.0  3.0     136.0  196.0  0.0      2.0    169.0    0.0      0.1   \n",
       "164  48.0  1.0  3.0     124.0  255.0  1.0      0.0    175.0    0.0      0.0   \n",
       "23   58.0  1.0  3.0     132.0  224.0  0.0      2.0    173.0    0.0      3.2   \n",
       "247  47.0  1.0  4.0     110.0  275.0  0.0      2.0    118.0    1.0      1.0   \n",
       "\n",
       "     slope   ca thal  num  \n",
       "142    1.0  0.0  3.0    0  \n",
       "230    2.0  0.0  3.0    0  \n",
       "164    1.0  2.0  3.0    0  \n",
       "23     1.0  2.0  7.0    3  \n",
       "247    2.0  1.0  3.0    1  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the multiclass data\n",
    "from sklearn.model_selection import train_test_split\n",
    "mccleveland_train, mccleveland_test = train_test_split(cleveland_data, test_size=0.2, random_state=42)\n",
    "mccleveland_test_labels = mccleveland_test[heart_label]\n",
    "mccleveland_test_nolabel = mccleveland_test.drop([heart_label], axis=1)\n",
    "mccleveland_train_labels = mccleveland_train[heart_label]\n",
    "mccleveland_train_nolabel = mccleveland_train.drop([heart_label], axis=1)\n",
    "mccleveland_train.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fd27bac",
   "metadata": {},
   "source": [
    "### MODEL - TRAIN & EVALUATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d854e610",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    242\n",
      "Train Data Columns: 13\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'multiclass' (because dtype of label-column == int, but few unique label-values observed).\n",
      "\t5 unique label values:  [0, 3, 1, 2, 4]\n",
      "\tIf 'multiclass' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Warning: Updated label_count_threshold from 10 to 9 to avoid cutting too many classes.\n",
      "Train Data Class Count: 5\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    327.68 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.05 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 3 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', [])  : 11 | ['age', 'sex', 'cp', 'trestbps', 'chol', ...]\n",
      "\t\t('object', []) :  2 | ['ca', 'thal']\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', [])  : 2 | ['ca', 'thal']\n",
      "\t\t('float', [])     : 8 | ['age', 'cp', 'trestbps', 'chol', 'restecg', ...]\n",
      "\t\t('int', ['bool']) : 3 | ['sex', 'fbs', 'exang']\n",
      "\t0.0s = Fit runtime\n",
      "\t13 features in original data used to generate 13 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.05s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 193, Val Rows: 49\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.5306\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.5306\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.6122\t = Validation score   (accuracy)\n",
      "\t0.29s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.6122\t = Validation score   (accuracy)\n",
      "\t0.27s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.6531\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.6327\t = Validation score   (accuracy)\n",
      "\t0.27s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 307 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.551\t = Validation score   (accuracy)\n",
      "\t0.12s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.6531\t = Validation score   (accuracy)\n",
      "\t0.08s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.55s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "mccle_TabPre = TabularPredictor(label=heart_label, path='.').fit(mccleveland_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6e36163d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.5081967213114754\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.5081967213114754,\n",
      "    \"balanced_accuracy\": 0.24865900383141773,\n",
      "    \"mcc\": 0.23660814133937144\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Check the performance using test data\n",
    "mccle_pred = mccle_TabPre.predict(mccleveland_test_nolabel)\n",
    "mccle_perf = mccle_TabPre.evaluate_predictions(y_true=mccleveland_test_labels, y_pred=mccle_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "51b78d4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.508197</td>\n",
       "      <td>0.653061</td>\n",
       "      <td>0.029400</td>\n",
       "      <td>0.021166</td>\n",
       "      <td>0.260073</td>\n",
       "      <td>0.029400</td>\n",
       "      <td>0.021166</td>\n",
       "      <td>0.260073</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.508197</td>\n",
       "      <td>0.653061</td>\n",
       "      <td>0.030734</td>\n",
       "      <td>0.021374</td>\n",
       "      <td>0.339908</td>\n",
       "      <td>0.001334</td>\n",
       "      <td>0.000208</td>\n",
       "      <td>0.079835</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.508197</td>\n",
       "      <td>0.612245</td>\n",
       "      <td>0.052223</td>\n",
       "      <td>0.021236</td>\n",
       "      <td>0.287482</td>\n",
       "      <td>0.052223</td>\n",
       "      <td>0.021236</td>\n",
       "      <td>0.287482</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.508197</td>\n",
       "      <td>0.632653</td>\n",
       "      <td>0.052574</td>\n",
       "      <td>0.021403</td>\n",
       "      <td>0.265118</td>\n",
       "      <td>0.052574</td>\n",
       "      <td>0.021403</td>\n",
       "      <td>0.265118</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.491803</td>\n",
       "      <td>0.551020</td>\n",
       "      <td>0.007419</td>\n",
       "      <td>0.002953</td>\n",
       "      <td>0.118258</td>\n",
       "      <td>0.007419</td>\n",
       "      <td>0.002953</td>\n",
       "      <td>0.118258</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.491803</td>\n",
       "      <td>0.612245</td>\n",
       "      <td>0.055530</td>\n",
       "      <td>0.021308</td>\n",
       "      <td>0.270204</td>\n",
       "      <td>0.055530</td>\n",
       "      <td>0.021308</td>\n",
       "      <td>0.270204</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.475410</td>\n",
       "      <td>0.530612</td>\n",
       "      <td>0.003148</td>\n",
       "      <td>0.001197</td>\n",
       "      <td>0.005485</td>\n",
       "      <td>0.003148</td>\n",
       "      <td>0.001197</td>\n",
       "      <td>0.005485</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.475410</td>\n",
       "      <td>0.530612</td>\n",
       "      <td>0.007408</td>\n",
       "      <td>0.002081</td>\n",
       "      <td>0.006328</td>\n",
       "      <td>0.007408</td>\n",
       "      <td>0.002081</td>\n",
       "      <td>0.006328</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0       ExtraTreesGini    0.508197   0.653061        0.029400       0.021166   \n",
       "1  WeightedEnsemble_L2    0.508197   0.653061        0.030734       0.021374   \n",
       "2     RandomForestGini    0.508197   0.612245        0.052223       0.021236   \n",
       "3       ExtraTreesEntr    0.508197   0.632653        0.052574       0.021403   \n",
       "4              XGBoost    0.491803   0.551020        0.007419       0.002953   \n",
       "5     RandomForestEntr    0.491803   0.612245        0.055530       0.021308   \n",
       "6       KNeighborsDist    0.475410   0.530612        0.003148       0.001197   \n",
       "7       KNeighborsUnif    0.475410   0.530612        0.007408       0.002081   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.260073                 0.029400                0.021166   \n",
       "1  0.339908                 0.001334                0.000208   \n",
       "2  0.287482                 0.052223                0.021236   \n",
       "3  0.265118                 0.052574                0.021403   \n",
       "4  0.118258                 0.007419                0.002953   \n",
       "5  0.270204                 0.055530                0.021308   \n",
       "6  0.005485                 0.003148                0.001197   \n",
       "7  0.006328                 0.007408                0.002081   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.260073            1       True          5  \n",
       "1           0.079835            2       True          8  \n",
       "2           0.287482            1       True          3  \n",
       "3           0.265118            1       True          6  \n",
       "4           0.118258            1       True          7  \n",
       "5           0.270204            1       True          4  \n",
       "6           0.005485            1       True          2  \n",
       "7           0.006328            1       True          1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "mccle_TabPre.leaderboard(mccleveland_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0f328f90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "mccle_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90de869a",
   "metadata": {},
   "source": [
    "# SLIGHTLY BETTER MODEL\n",
    "no missing values & stratified sampling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb169a8b",
   "metadata": {},
   "source": [
    "## DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afad248a",
   "metadata": {},
   "source": [
    "### MISSING VALUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4e09dc2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make copy of original dataframe\n",
    "cleveland = cleveland_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "01b04a92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 303 entries, 0 to 302\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       303 non-null    float64\n",
      " 1   sex       303 non-null    float64\n",
      " 2   cp        303 non-null    float64\n",
      " 3   trestbps  303 non-null    float64\n",
      " 4   chol      303 non-null    float64\n",
      " 5   fbs       303 non-null    float64\n",
      " 6   restecg   303 non-null    float64\n",
      " 7   thalach   303 non-null    float64\n",
      " 8   exang     303 non-null    float64\n",
      " 9   oldpeak   303 non-null    float64\n",
      " 10  slope     303 non-null    float64\n",
      " 11  ca        303 non-null    object \n",
      " 12  thal      303 non-null    object \n",
      " 13  num       303 non-null    int64  \n",
      "dtypes: float64(11), int64(1), object(2)\n",
      "memory usage: 33.3+ KB\n"
     ]
    }
   ],
   "source": [
    "# check for missing values\n",
    "cleveland.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5f8e366d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>45.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>264.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>68.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>236.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>174.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>173.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "298  45.0  1.0  1.0     110.0  264.0  0.0      0.0    132.0    0.0      1.2   \n",
       "299  68.0  1.0  4.0     144.0  193.0  1.0      0.0    141.0    0.0      3.4   \n",
       "300  57.0  1.0  4.0     130.0  131.0  0.0      0.0    115.0    1.0      1.2   \n",
       "301  57.0  0.0  2.0     130.0  236.0  0.0      2.0    174.0    0.0      0.0   \n",
       "302  38.0  1.0  3.0     138.0  175.0  0.0      0.0    173.0    0.0      0.0   \n",
       "\n",
       "     slope   ca  thal  num  \n",
       "298    2.0  0.0   7.0    1  \n",
       "299    2.0  2.0   7.0    2  \n",
       "300    2.0  1.0   7.0    3  \n",
       "301    2.0  1.0   3.0    1  \n",
       "302    1.0  NaN   3.0    0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace '?' values with NaN so can impute\n",
    "cleveland['thal'].replace('?', np.NaN, inplace=True)\n",
    "cleveland['thal'] = cleveland['thal'].astype(float)\n",
    "cleveland['ca'].replace('?', np.NaN, inplace=True)\n",
    "cleveland['ca'] = cleveland['ca'].astype(float)\n",
    "cleveland.tail() # to verify didn't mess up dataframe IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e01a9795",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 303 entries, 0 to 302\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       303 non-null    float64\n",
      " 1   sex       303 non-null    float64\n",
      " 2   cp        303 non-null    float64\n",
      " 3   trestbps  303 non-null    float64\n",
      " 4   chol      303 non-null    float64\n",
      " 5   fbs       303 non-null    float64\n",
      " 6   restecg   303 non-null    float64\n",
      " 7   thalach   303 non-null    float64\n",
      " 8   exang     303 non-null    float64\n",
      " 9   oldpeak   303 non-null    float64\n",
      " 10  slope     303 non-null    float64\n",
      " 11  ca        303 non-null    float64\n",
      " 12  thal      303 non-null    float64\n",
      " 13  num       303 non-null    float64\n",
      "dtypes: float64(14)\n",
      "memory usage: 33.3 KB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages/sklearn/impute/_base.py:49: FutureWarning: Unlike other reduction functions (e.g. `skew`, `kurtosis`), the default behavior of `mode` typically preserves the axis it acts along. In SciPy 1.11.0, this behavior will change: the default value of `keepdims` will become False, the `axis` over which the statistic is taken will be eliminated, and the value None will no longer be accepted. Set `keepdims` to True or False to avoid this warning.\n",
      "  mode = stats.mode(array)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "# impute with mode as 'thal' and 'ca' (attributes w/ missing values) are discrete\n",
    "imputeMode = SimpleImputer(strategy=\"most_frequent\") # create mode imputer\n",
    "imputeMode.fit(cleveland) # fit - learns the data\n",
    "imputed = imputeMode.transform(cleveland) # transform - imputes with chosen strategy\n",
    "cleveland = pd.DataFrame(imputed, columns=cleveland.columns, index=cleveland.index) # back to pandas DataFrame\n",
    "cleveland.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1274324f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>45.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>264.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>68.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>57.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>131.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>236.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>174.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>38.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>173.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "298  45.0  1.0  1.0     110.0  264.0  0.0      0.0    132.0    0.0      1.2   \n",
       "299  68.0  1.0  4.0     144.0  193.0  1.0      0.0    141.0    0.0      3.4   \n",
       "300  57.0  1.0  4.0     130.0  131.0  0.0      0.0    115.0    1.0      1.2   \n",
       "301  57.0  0.0  2.0     130.0  236.0  0.0      2.0    174.0    0.0      0.0   \n",
       "302  38.0  1.0  3.0     138.0  175.0  0.0      0.0    173.0    0.0      0.0   \n",
       "\n",
       "     slope   ca  thal  num  \n",
       "298    2.0  0.0   7.0  1.0  \n",
       "299    2.0  2.0   7.0  2.0  \n",
       "300    2.0  1.0   7.0  3.0  \n",
       "301    2.0  1.0   3.0  1.0  \n",
       "302    1.0  0.0   3.0  0.0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd33fbe8",
   "metadata": {},
   "source": [
    "### BINARY LABEL\n",
    "(multiclass for \"future research\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "afdebecf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>41.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  63.0  1.0  1.0     145.0  233.0  1.0      2.0    150.0    0.0      2.3   \n",
       "1  67.0  1.0  4.0     160.0  286.0  0.0      2.0    108.0    1.0      1.5   \n",
       "2  67.0  1.0  4.0     120.0  229.0  0.0      2.0    129.0    1.0      2.6   \n",
       "3  37.0  1.0  3.0     130.0  250.0  0.0      0.0    187.0    0.0      3.5   \n",
       "4  41.0  0.0  2.0     130.0  204.0  0.0      2.0    172.0    0.0      1.4   \n",
       "\n",
       "   slope   ca  thal  num  \n",
       "0    3.0  0.0   6.0  0.0  \n",
       "1    2.0  3.0   3.0  1.0  \n",
       "2    2.0  2.0   7.0  1.0  \n",
       "3    3.0  0.0   3.0  0.0  \n",
       "4    1.0  0.0   3.0  0.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland['num'][cleveland['num'] > 0] = 1\n",
    "cleveland.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "503c891d",
   "metadata": {},
   "source": [
    "### STRATIFIED SAMPLING\n",
    "\n",
    "(due to clear significance of sex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2e814a37",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleveland_strat = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in cleveland_strat.split(cleveland, cleveland[\"num\"]):\n",
    "  cleveland_strat_train = cleveland.loc[train_index] # cleveland_train,\n",
    "  cleveland_strat_test = cleveland.loc[test_index] # cleveland_test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d1cd59a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.540984\n",
       "1.0    0.459016\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_strat_test[\"num\"].value_counts()/len(cleveland_strat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e177bd08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.541322\n",
       "1.0    0.458678\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_strat_train[\"num\"].value_counts()/len(cleveland_strat_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07c81d67",
   "metadata": {},
   "source": [
    "Less significant for Cleveland and Statlog datasets, which have less dramatic difference in sex representation in data, but different for other three datasets. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "752a185b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>62.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>244.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>55.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>353.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>237.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "209  62.0  0.0  4.0     150.0  244.0  0.0      0.0    154.0    1.0      1.4   \n",
       "97   60.0  0.0  4.0     150.0  258.0  0.0      2.0    157.0    0.0      2.6   \n",
       "38   55.0  1.0  4.0     132.0  353.0  0.0      0.0    132.0    1.0      1.2   \n",
       "112  52.0  1.0  1.0     118.0  186.0  0.0      2.0    190.0    0.0      0.0   \n",
       "245  67.0  1.0  4.0     120.0  237.0  0.0      0.0     71.0    0.0      1.0   \n",
       "\n",
       "     slope   ca  thal  \n",
       "209    2.0  0.0   3.0  \n",
       "97     2.0  2.0   7.0  \n",
       "38     2.0  1.0   7.0  \n",
       "112    2.0  0.0   6.0  \n",
       "245    2.0  0.0   3.0  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split stratified data\n",
    "cleveland_strat_train_X = cleveland_strat_train.drop([heart_label], axis=1)#cleveland_train_nolabel\n",
    "cleveland_strat_train_y = cleveland_strat_train[heart_label]#cleveland_train_labels \n",
    "cleveland_strat_test_X = cleveland_strat_test.drop([heart_label], axis=1)#cleveland_test_nolabel \n",
    "cleveland_strat_test_y = cleveland_strat_test[heart_label]#cleveland_test_labels\n",
    "cleveland_strat_train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3a5cd68",
   "metadata": {},
   "source": [
    "## BINARY MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "de6986ef",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    242\n",
      "Train Data Columns: 13\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [1.0, 0.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    319.6 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.03 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 3 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', []) : 13 | ['age', 'sex', 'cp', 'trestbps', 'chol', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('float', [])     : 10 | ['age', 'cp', 'trestbps', 'chol', 'restecg', ...]\n",
      "\t\t('int', ['bool']) :  3 | ['sex', 'fbs', 'exang']\n",
      "\t0.0s = Fit runtime\n",
      "\t13 features in original data used to generate 13 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.04s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 193, Val Rows: 49\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.7143\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.7143\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8776\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 301 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.8163\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.09s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.37s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "ccle_TabPre = TabularPredictor(label=heart_label, path='.').fit(cleveland_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f7d1bc07",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8688524590163934\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8688524590163934,\n",
      "    \"balanced_accuracy\": 0.876082251082251,\n",
      "    \"mcc\": 0.7546185527373646,\n",
      "    \"f1\": 0.8709677419354839,\n",
      "    \"precision\": 0.7941176470588235,\n",
      "    \"recall\": 0.9642857142857143\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Check the performance using test data\n",
    "ccle_pred = ccle_TabPre.predict(cleveland_strat_test_X)\n",
    "ccle_perf = ccle_TabPre.evaluate_predictions(y_true=cleveland_strat_test_y, y_pred=ccle_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "1586ef61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8688524590163934\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8688524590163934,\n",
      "    \"balanced_accuracy\": 0.876082251082251,\n",
      "    \"mcc\": 0.7546185527373646,\n",
      "    \"roc_auc\": 0.9442640692640693,\n",
      "    \"f1\": 0.8709677419354839,\n",
      "    \"precision\": 0.7941176470588235,\n",
      "    \"recall\": 0.9642857142857143\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "ccle_perf2 = ccle_TabPre.evaluate(data=cleveland_strat_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "748ac1ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8688524590163934\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8688524590163934,\n",
      "    \"balanced_accuracy\": 0.876082251082251,\n",
      "    \"mcc\": 0.7546185527373646,\n",
      "    \"roc_auc\": 0.9442640692640693,\n",
      "    \"f1\": 0.8709677419354839,\n",
      "    \"precision\": 0.7941176470588235,\n",
      "    \"recall\": 0.9642857142857143\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "ccle_perf3 = ccle_TabPre.evaluate(data=cleveland_strat_test, auxiliary_metrics=True, model='RandomForestGini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "91955670",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8524590163934426\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8524590163934426,\n",
      "    \"balanced_accuracy\": 0.8582251082251082,\n",
      "    \"mcc\": 0.7164502164502164,\n",
      "    \"roc_auc\": 0.9458874458874459,\n",
      "    \"f1\": 0.8524590163934426,\n",
      "    \"precision\": 0.7878787878787878,\n",
      "    \"recall\": 0.9285714285714286\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "ccle_perf4 = ccle_TabPre.evaluate(data=cleveland_strat_test, auxiliary_metrics=True, model='RandomForestEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebf67b36",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2ede8c86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.868852</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.031337</td>\n",
       "      <td>0.019909</td>\n",
       "      <td>0.264666</td>\n",
       "      <td>0.031337</td>\n",
       "      <td>0.019909</td>\n",
       "      <td>0.264666</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.868852</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.033044</td>\n",
       "      <td>0.020168</td>\n",
       "      <td>0.353134</td>\n",
       "      <td>0.001707</td>\n",
       "      <td>0.000259</td>\n",
       "      <td>0.088468</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.027681</td>\n",
       "      <td>0.020012</td>\n",
       "      <td>0.253528</td>\n",
       "      <td>0.027681</td>\n",
       "      <td>0.020012</td>\n",
       "      <td>0.253528</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.836066</td>\n",
       "      <td>0.877551</td>\n",
       "      <td>0.046257</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>0.249304</td>\n",
       "      <td>0.046257</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>0.249304</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.819672</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.047483</td>\n",
       "      <td>0.020231</td>\n",
       "      <td>0.250673</td>\n",
       "      <td>0.047483</td>\n",
       "      <td>0.020231</td>\n",
       "      <td>0.250673</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.786885</td>\n",
       "      <td>0.816327</td>\n",
       "      <td>0.008872</td>\n",
       "      <td>0.002272</td>\n",
       "      <td>0.013639</td>\n",
       "      <td>0.008872</td>\n",
       "      <td>0.002272</td>\n",
       "      <td>0.013639</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.655738</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.006076</td>\n",
       "      <td>0.002227</td>\n",
       "      <td>0.007989</td>\n",
       "      <td>0.006076</td>\n",
       "      <td>0.002227</td>\n",
       "      <td>0.007989</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.622951</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.015360</td>\n",
       "      <td>0.002002</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>0.015360</td>\n",
       "      <td>0.002002</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0     RandomForestGini    0.868852   0.897959        0.031337       0.019909   \n",
       "1  WeightedEnsemble_L2    0.868852   0.897959        0.033044       0.020168   \n",
       "2     RandomForestEntr    0.852459   0.897959        0.027681       0.020012   \n",
       "3       ExtraTreesGini    0.836066   0.877551        0.046257       0.020055   \n",
       "4       ExtraTreesEntr    0.819672   0.897959        0.047483       0.020231   \n",
       "5              XGBoost    0.786885   0.816327        0.008872       0.002272   \n",
       "6       KNeighborsUnif    0.655738   0.714286        0.006076       0.002227   \n",
       "7       KNeighborsDist    0.622951   0.714286        0.015360       0.002002   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.264666                 0.031337                0.019909   \n",
       "1  0.353134                 0.001707                0.000259   \n",
       "2  0.253528                 0.027681                0.020012   \n",
       "3  0.249304                 0.046257                0.020055   \n",
       "4  0.250673                 0.047483                0.020231   \n",
       "5  0.013639                 0.008872                0.002272   \n",
       "6  0.007989                 0.006076                0.002227   \n",
       "7  0.004158                 0.015360                0.002002   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.264666            1       True          3  \n",
       "1           0.088468            2       True          8  \n",
       "2           0.253528            1       True          4  \n",
       "3           0.249304            1       True          5  \n",
       "4           0.250673            1       True          6  \n",
       "5           0.013639            1       True          7  \n",
       "6           0.007989            1       True          1  \n",
       "7           0.004158            1       True          2  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "ccle_TabPre.leaderboard(cleveland_strat_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e8ad1848",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.019909</td>\n",
       "      <td>0.264666</td>\n",
       "      <td>0.019909</td>\n",
       "      <td>0.264666</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.020012</td>\n",
       "      <td>0.253528</td>\n",
       "      <td>0.020012</td>\n",
       "      <td>0.253528</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.020168</td>\n",
       "      <td>0.353134</td>\n",
       "      <td>0.000259</td>\n",
       "      <td>0.088468</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.020231</td>\n",
       "      <td>0.250673</td>\n",
       "      <td>0.020231</td>\n",
       "      <td>0.250673</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.877551</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>0.249304</td>\n",
       "      <td>0.020055</td>\n",
       "      <td>0.249304</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.816327</td>\n",
       "      <td>0.002272</td>\n",
       "      <td>0.013639</td>\n",
       "      <td>0.002272</td>\n",
       "      <td>0.013639</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.002002</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>0.002002</td>\n",
       "      <td>0.004158</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.002227</td>\n",
       "      <td>0.007989</td>\n",
       "      <td>0.002227</td>\n",
       "      <td>0.007989</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_val  pred_time_val  fit_time  \\\n",
       "0     RandomForestGini   0.897959       0.019909  0.264666   \n",
       "1     RandomForestEntr   0.897959       0.020012  0.253528   \n",
       "2  WeightedEnsemble_L2   0.897959       0.020168  0.353134   \n",
       "3       ExtraTreesEntr   0.897959       0.020231  0.250673   \n",
       "4       ExtraTreesGini   0.877551       0.020055  0.249304   \n",
       "5              XGBoost   0.816327       0.002272  0.013639   \n",
       "6       KNeighborsDist   0.714286       0.002002  0.004158   \n",
       "7       KNeighborsUnif   0.714286       0.002227  0.007989   \n",
       "\n",
       "   pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                0.019909           0.264666            1       True   \n",
       "1                0.020012           0.253528            1       True   \n",
       "2                0.000259           0.088468            2       True   \n",
       "3                0.020231           0.250673            1       True   \n",
       "4                0.020055           0.249304            1       True   \n",
       "5                0.002272           0.013639            1       True   \n",
       "6                0.002002           0.004158            1       True   \n",
       "7                0.002227           0.007989            1       True   \n",
       "\n",
       "   fit_order  \n",
       "0          3  \n",
       "1          4  \n",
       "2          8  \n",
       "3          6  \n",
       "4          5  \n",
       "5          7  \n",
       "6          2  \n",
       "7          1  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "ccle_TabPre.leaderboard( silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b132bde3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "cle_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72875060",
   "metadata": {},
   "source": [
    "# MODEL USING ONLY CORRELATED ATTRIBUTES\n",
    "(incl. no missing values & stratified sampling)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "738d26e5",
   "metadata": {},
   "source": [
    "## DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "001b6465",
   "metadata": {},
   "source": [
    "### REMOVE UNCORRELATED ATTRIBUTES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "54ee1974",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "thalach    -0.417167\n",
       "fbs         0.025264\n",
       "chol        0.085164\n",
       "trestbps    0.150825\n",
       "restecg     0.169202\n",
       "age         0.223120\n",
       "sex         0.276816\n",
       "slope       0.339213\n",
       "cp          0.414446\n",
       "oldpeak     0.424510\n",
       "exang       0.431894\n",
       "ca          0.460033\n",
       "thal        0.522057\n",
       "num         1.000000\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check correlation\n",
    "cleveland_correlation = cleveland.corr()\n",
    "cleveland_correlation['num'].sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "089eb645",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 303 entries, 0 to 302\n",
      "Data columns (total 10 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   age      303 non-null    float64\n",
      " 1   sex      303 non-null    float64\n",
      " 2   cp       303 non-null    float64\n",
      " 3   thalach  303 non-null    float64\n",
      " 4   exang    303 non-null    float64\n",
      " 5   oldpeak  303 non-null    float64\n",
      " 6   slope    303 non-null    float64\n",
      " 7   ca       303 non-null    float64\n",
      " 8   thal     303 non-null    float64\n",
      " 9   num      303 non-null    float64\n",
      "dtypes: float64(10)\n",
      "memory usage: 23.8 KB\n"
     ]
    }
   ],
   "source": [
    "# eliminate columns with correlation less than abs(0.2)\n",
    "drop_columns = ['fbs', 'chol', 'trestbps', 'restecg']\n",
    "cleveland_downsized = cleveland.drop(columns = drop_columns)\n",
    "cleveland_downsized.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a7dce68",
   "metadata": {},
   "source": [
    "### STRATIFIED SPLIT OF DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "809aae2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleveland_downsized_strat = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in cleveland_downsized_strat.split(cleveland_downsized, cleveland_downsized[\"num\"]):\n",
    "  cleveland_downsized_strat_train = cleveland_downsized.loc[train_index]\n",
    "  cleveland_downsized_strat_test = cleveland_downsized.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "f3ecb3e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.540984\n",
       "1.0    0.459016\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_downsized_strat_test[\"num\"].value_counts()/len(cleveland_downsized_strat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "fb412991",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.541322\n",
       "1.0    0.458678\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleveland_downsized_strat_train[\"num\"].value_counts()/len(cleveland_downsized_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6d2c0890",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>67.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>42.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>178.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>65.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>70.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>51.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  thalach  exang  oldpeak  slope   ca  thal\n",
       "2    67.0  1.0  4.0    129.0    1.0      2.6    2.0  2.0   7.0\n",
       "35   42.0  1.0  4.0    178.0    0.0      0.0    1.0  0.0   3.0\n",
       "70   65.0  0.0  3.0    148.0    0.0      0.8    1.0  0.0   3.0\n",
       "170  70.0  1.0  3.0    112.0    1.0      2.9    2.0  1.0   7.0\n",
       "253  51.0  0.0  3.0    157.0    0.0      0.6    1.0  0.0   3.0"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split stratified data\n",
    "cleveland_downsized_strat_train_X = cleveland_downsized_strat_train.drop([heart_label], axis=1)\n",
    "cleveland_downsized_strat_train_y = cleveland_downsized_strat_train[heart_label]\n",
    "cleveland_downsized_strat_test_X = cleveland_downsized_strat_test.drop([heart_label], axis=1)\n",
    "cleveland_downsized_strat_test_y = cleveland_downsized_strat_test[heart_label]\n",
    "cleveland_downsized_strat_train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7545d63",
   "metadata": {},
   "source": [
    "## BINARY MODELING\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "950835fe",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    242\n",
      "Train Data Columns: 9\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [1.0, 0.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    335.46 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 2 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', []) : 9 | ['age', 'sex', 'cp', 'thalach', 'exang', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('float', [])     : 7 | ['age', 'cp', 'thalach', 'oldpeak', 'slope', ...]\n",
      "\t\t('int', ['bool']) : 2 | ['sex', 'exang']\n",
      "\t0.0s = Fit runtime\n",
      "\t9 features in original data used to generate 9 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.01 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.04s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 193, Val Rows: 49\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.7959\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.8367\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.27s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.898\t = Validation score   (accuracy)\n",
      "\t0.29s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8571\t = Validation score   (accuracy)\n",
      "\t0.34s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.8571\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 299 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.8367\t = Validation score   (accuracy)\n",
      "\t0.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.9388\t = Validation score   (accuracy)\n",
      "\t0.09s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.53s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "cccle_TabPre = TabularPredictor(label=heart_label, path='.').fit(cleveland_downsized_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "8f4aa1fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.7868852459016393\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.7868852459016393,\n",
      "    \"balanced_accuracy\": 0.7895021645021645,\n",
      "    \"mcc\": 0.5771335508376638,\n",
      "    \"f1\": 0.7796610169491526,\n",
      "    \"precision\": 0.7419354838709677,\n",
      "    \"recall\": 0.8214285714285714\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Check the performance using test data\n",
    "cccle_pred = cccle_TabPre.predict(cleveland_downsized_strat_test_X)\n",
    "cccle_perf = cccle_TabPre.evaluate_predictions(y_true=cleveland_downsized_strat_test_y, y_pred=cccle_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "6d78820d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.7868852459016393\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.7868852459016393,\n",
      "    \"balanced_accuracy\": 0.7895021645021645,\n",
      "    \"mcc\": 0.5771335508376638,\n",
      "    \"roc_auc\": 0.8874458874458875,\n",
      "    \"f1\": 0.7796610169491526,\n",
      "    \"precision\": 0.7419354838709677,\n",
      "    \"recall\": 0.8214285714285714\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cccle_perf2 = cccle_TabPre.evaluate(data=cleveland_downsized_strat_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "b49fd4c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8524590163934426\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8524590163934426,\n",
      "    \"balanced_accuracy\": 0.8555194805194806,\n",
      "    \"mcc\": 0.7087415755146639,\n",
      "    \"roc_auc\": 0.9377705627705628,\n",
      "    \"f1\": 0.8474576271186439,\n",
      "    \"precision\": 0.8064516129032258,\n",
      "    \"recall\": 0.8928571428571429\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cccle_perf3 = cccle_TabPre.evaluate(data=cleveland_downsized_strat_test, auxiliary_metrics=True, model='RandomForestGini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "2ca44a8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8360655737704918\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8360655737704918,\n",
      "    \"balanced_accuracy\": 0.8403679653679654,\n",
      "    \"mcc\": 0.6792672430902679,\n",
      "    \"roc_auc\": 0.9388528138528138,\n",
      "    \"f1\": 0.8333333333333334,\n",
      "    \"precision\": 0.78125,\n",
      "    \"recall\": 0.8928571428571429\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "cccle_perf4 = cccle_TabPre.evaluate(data=cleveland_downsized_strat_test, auxiliary_metrics=True, model='RandomForestEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "8a2f86f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.868852</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.061676</td>\n",
       "      <td>0.020115</td>\n",
       "      <td>0.254714</td>\n",
       "      <td>0.061676</td>\n",
       "      <td>0.020115</td>\n",
       "      <td>0.254714</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.041253</td>\n",
       "      <td>0.019756</td>\n",
       "      <td>0.267854</td>\n",
       "      <td>0.041253</td>\n",
       "      <td>0.019756</td>\n",
       "      <td>0.267854</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.852459</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.058943</td>\n",
       "      <td>0.020543</td>\n",
       "      <td>0.335050</td>\n",
       "      <td>0.058943</td>\n",
       "      <td>0.020543</td>\n",
       "      <td>0.335050</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.836066</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.040387</td>\n",
       "      <td>0.024979</td>\n",
       "      <td>0.288438</td>\n",
       "      <td>0.040387</td>\n",
       "      <td>0.024979</td>\n",
       "      <td>0.288438</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.786885</td>\n",
       "      <td>0.938776</td>\n",
       "      <td>0.056059</td>\n",
       "      <td>0.023411</td>\n",
       "      <td>0.374652</td>\n",
       "      <td>0.002884</td>\n",
       "      <td>0.000264</td>\n",
       "      <td>0.087513</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.737705</td>\n",
       "      <td>0.836735</td>\n",
       "      <td>0.010364</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.015928</td>\n",
       "      <td>0.010364</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.015928</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.672131</td>\n",
       "      <td>0.836735</td>\n",
       "      <td>0.004857</td>\n",
       "      <td>0.001393</td>\n",
       "      <td>0.004595</td>\n",
       "      <td>0.004857</td>\n",
       "      <td>0.001393</td>\n",
       "      <td>0.004595</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.655738</td>\n",
       "      <td>0.795918</td>\n",
       "      <td>0.007065</td>\n",
       "      <td>0.001998</td>\n",
       "      <td>0.014690</td>\n",
       "      <td>0.007065</td>\n",
       "      <td>0.001998</td>\n",
       "      <td>0.014690</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0       ExtraTreesEntr    0.868852   0.857143        0.061676       0.020115   \n",
       "1     RandomForestGini    0.852459   0.897959        0.041253       0.019756   \n",
       "2       ExtraTreesGini    0.852459   0.857143        0.058943       0.020543   \n",
       "3     RandomForestEntr    0.836066   0.897959        0.040387       0.024979   \n",
       "4  WeightedEnsemble_L2    0.786885   0.938776        0.056059       0.023411   \n",
       "5              XGBoost    0.737705   0.836735        0.010364       0.002206   \n",
       "6       KNeighborsDist    0.672131   0.836735        0.004857       0.001393   \n",
       "7       KNeighborsUnif    0.655738   0.795918        0.007065       0.001998   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.254714                 0.061676                0.020115   \n",
       "1  0.267854                 0.041253                0.019756   \n",
       "2  0.335050                 0.058943                0.020543   \n",
       "3  0.288438                 0.040387                0.024979   \n",
       "4  0.374652                 0.002884                0.000264   \n",
       "5  0.015928                 0.010364                0.002206   \n",
       "6  0.004595                 0.004857                0.001393   \n",
       "7  0.014690                 0.007065                0.001998   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.254714            1       True          6  \n",
       "1           0.267854            1       True          3  \n",
       "2           0.335050            1       True          5  \n",
       "3           0.288438            1       True          4  \n",
       "4           0.087513            2       True          8  \n",
       "5           0.015928            1       True          7  \n",
       "6           0.004595            1       True          2  \n",
       "7           0.014690            1       True          1  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "cccle_TabPre.leaderboard(cleveland_downsized_strat_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "00044466",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.938776</td>\n",
       "      <td>0.023411</td>\n",
       "      <td>0.374652</td>\n",
       "      <td>0.000264</td>\n",
       "      <td>0.087513</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.019756</td>\n",
       "      <td>0.267854</td>\n",
       "      <td>0.019756</td>\n",
       "      <td>0.267854</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.897959</td>\n",
       "      <td>0.024979</td>\n",
       "      <td>0.288438</td>\n",
       "      <td>0.024979</td>\n",
       "      <td>0.288438</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.020115</td>\n",
       "      <td>0.254714</td>\n",
       "      <td>0.020115</td>\n",
       "      <td>0.254714</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.020543</td>\n",
       "      <td>0.335050</td>\n",
       "      <td>0.020543</td>\n",
       "      <td>0.335050</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.836735</td>\n",
       "      <td>0.001393</td>\n",
       "      <td>0.004595</td>\n",
       "      <td>0.001393</td>\n",
       "      <td>0.004595</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.836735</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.015928</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.015928</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.795918</td>\n",
       "      <td>0.001998</td>\n",
       "      <td>0.014690</td>\n",
       "      <td>0.001998</td>\n",
       "      <td>0.014690</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_val  pred_time_val  fit_time  \\\n",
       "0  WeightedEnsemble_L2   0.938776       0.023411  0.374652   \n",
       "1     RandomForestGini   0.897959       0.019756  0.267854   \n",
       "2     RandomForestEntr   0.897959       0.024979  0.288438   \n",
       "3       ExtraTreesEntr   0.857143       0.020115  0.254714   \n",
       "4       ExtraTreesGini   0.857143       0.020543  0.335050   \n",
       "5       KNeighborsDist   0.836735       0.001393  0.004595   \n",
       "6              XGBoost   0.836735       0.002206  0.015928   \n",
       "7       KNeighborsUnif   0.795918       0.001998  0.014690   \n",
       "\n",
       "   pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                0.000264           0.087513            2       True   \n",
       "1                0.019756           0.267854            1       True   \n",
       "2                0.024979           0.288438            1       True   \n",
       "3                0.020115           0.254714            1       True   \n",
       "4                0.020543           0.335050            1       True   \n",
       "5                0.001393           0.004595            1       True   \n",
       "6                0.002206           0.015928            1       True   \n",
       "7                0.001998           0.014690            1       True   \n",
       "\n",
       "   fit_order  \n",
       "0          8  \n",
       "1          3  \n",
       "2          4  \n",
       "3          6  \n",
       "4          5  \n",
       "5          2  \n",
       "6          7  \n",
       "7          1  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cccle_TabPre.leaderboard(silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "0a5b49fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "cccle_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1abd4ff5",
   "metadata": {},
   "source": [
    "# Hungarian dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "0ad92f41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load Hungarian datafile\n",
    "hungarian_csv_path = \"reprocessed.hungarian.data\"\n",
    "hungarian_data = pd.read_csv(hungarian_csv_path, delimiter = ' ', header = None)\n",
    "# assign column names\n",
    "hungarian_data.set_axis(['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs',\n",
    "                  'restecg', 'thalach', 'exang', 'oldpeak', 'slope',\n",
    "                  'ca', 'thal', 'num'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "14caac3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define label column\n",
    "label = 'num'\n",
    "heart_label = 'num'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2efe9b2",
   "metadata": {},
   "source": [
    "# SIMPLEST OF MODELS\n",
    "zero data preprocessing performed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "7a26b77c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>289.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>283.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>214.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>-9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  40.0  1.0  2.0     140.0  289.0  0.0      0.0    172.0    0.0      0.0   \n",
       "1  49.0  0.0  3.0     160.0  180.0  0.0      0.0    156.0    0.0      1.0   \n",
       "2  37.0  1.0  2.0     130.0  283.0  0.0      1.0     98.0    0.0      0.0   \n",
       "3  48.0  0.0  4.0     138.0  214.0  0.0      0.0    108.0    1.0      1.5   \n",
       "4  54.0  1.0  3.0     150.0   -9.0  0.0      0.0    122.0    0.0      0.0   \n",
       "\n",
       "   slope   ca  thal  num  \n",
       "0   -9.0 -9.0  -9.0  0.0  \n",
       "1    2.0 -9.0  -9.0  1.0  \n",
       "2   -9.0 -9.0  -9.0  0.0  \n",
       "3    2.0 -9.0  -9.0  3.0  \n",
       "4   -9.0 -9.0  -9.0  0.0  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ca2e8b64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 295 entries, 0 to 294\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       294 non-null    float64\n",
      " 1   sex       294 non-null    float64\n",
      " 2   cp        294 non-null    float64\n",
      " 3   trestbps  294 non-null    float64\n",
      " 4   chol      294 non-null    float64\n",
      " 5   fbs       294 non-null    float64\n",
      " 6   restecg   294 non-null    float64\n",
      " 7   thalach   294 non-null    float64\n",
      " 8   exang     294 non-null    float64\n",
      " 9   oldpeak   294 non-null    float64\n",
      " 10  slope     294 non-null    float64\n",
      " 11  ca        294 non-null    float64\n",
      " 12  thal      294 non-null    float64\n",
      " 13  num       294 non-null    float64\n",
      "dtypes: float64(14)\n",
      "memory usage: 32.4 KB\n"
     ]
    }
   ],
   "source": [
    "hungarian_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3695406",
   "metadata": {},
   "source": [
    "## BINARY MODELING\n",
    "(Note: \"Cleveland\" label here would need to be revised because used in Slightly Better Model below when split the data, if care.)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01194c44",
   "metadata": {},
   "source": [
    "### PREPARE DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8c634158",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>289.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>283.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>214.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>108.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>54.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "0  40.0  1.0  2.0     140.0  289.0  0.0      0.0    172.0    0.0      0.0   \n",
       "1  49.0  0.0  3.0     160.0  180.0  0.0      0.0    156.0    0.0      1.0   \n",
       "2  37.0  1.0  2.0     130.0  283.0  0.0      1.0     98.0    0.0      0.0   \n",
       "3  48.0  0.0  4.0     138.0  214.0  0.0      0.0    108.0    1.0      1.5   \n",
       "4  54.0  1.0  3.0     150.0    NaN  0.0      0.0    122.0    0.0      0.0   \n",
       "\n",
       "   slope  ca  thal  num  \n",
       "0    NaN NaN   NaN  0.0  \n",
       "1    2.0 NaN   NaN  1.0  \n",
       "2    NaN NaN   NaN  0.0  \n",
       "3    2.0 NaN   NaN  1.0  \n",
       "4    NaN NaN   NaN  0.0  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make copy of original dataframe\n",
    "hungarian_basic = hungarian_data.copy()\n",
    "# convert unknowns (-9.0) to NaN\n",
    "hungarian_basic.replace(-9.0, np.NaN, inplace=True)\n",
    "# make copy for binary labeling\n",
    "hungarian_basic['num'][hungarian_basic['num'] > 0] = 1\n",
    "hungarian_basic.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4e5c3b96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# try dropping slope, ca, and thal\n",
    "hungarian_basic.drop(['slope', 'ca', 'thal', ], axis=1, inplace=True) # did not fix error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "ccd3256e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>45.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>58.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>213.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>404.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>52.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>246.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>56.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "194  45.0  1.0  3.0     135.0    NaN  0.0      0.0    110.0    0.0      0.0   \n",
       "57   58.0  1.0  3.0     130.0  213.0  0.0      1.0    140.0    0.0      0.0   \n",
       "183  52.0  1.0  4.0     140.0  404.0  0.0      0.0    124.0    1.0      2.0   \n",
       "68   52.0  1.0  4.0     160.0  246.0  0.0      1.0     82.0    1.0      4.0   \n",
       "205  56.0  1.0  2.0     130.0  184.0  0.0      0.0    100.0    0.0      0.0   \n",
       "\n",
       "     num  \n",
       "194  0.0  \n",
       "57   1.0  \n",
       "183  1.0  \n",
       "68   1.0  \n",
       "205  0.0  "
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the binary data\n",
    "from sklearn.model_selection import train_test_split\n",
    "hungarian_basic_train, hungarian_basic_test = train_test_split(hungarian_basic, test_size=0.2, random_state=42)\n",
    "hungarian_basic_test_labels = hungarian_basic_test[heart_label]\n",
    "hungarian_basic_test_nolabel = hungarian_basic_test.drop([heart_label], axis=1)\n",
    "hungarian_basic_train_labels = hungarian_basic_train[heart_label]\n",
    "hungarian_basic_train_nolabel = hungarian_basic_train.drop([heart_label], axis=1)\n",
    "hungarian_basic_train.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46a4852a",
   "metadata": {},
   "source": [
    "### MODEL - TRAIN & EVALUATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "831a21cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#hungarian_basic_train.to_csv('hungarian.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "de165c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(hungarian_basic_train.loc[[294]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b44a2e1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian_basic_train = hungarian_basic_train.drop([294]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "799d8a98",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    235\n",
      "Train Data Columns: 10\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [0.0, 1.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    337.19 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 1 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', []) : 10 | ['age', 'sex', 'cp', 'trestbps', 'chol', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('float', [])     : 9 | ['age', 'cp', 'trestbps', 'chol', 'fbs', ...]\n",
      "\t\t('int', ['bool']) : 1 | ['sex']\n",
      "\t0.0s = Fit runtime\n",
      "\t10 features in original data used to generate 10 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.04s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 188, Val Rows: 47\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.6596\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.6596\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8723\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 315 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.8723\t = Validation score   (accuracy)\n",
      "\t0.09s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.38s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "hun_TabPre = TabularPredictor(label=heart_label, path='.').fit(hungarian_basic_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e8c69219",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.7796610169491526\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.7796610169491526,\n",
      "    \"balanced_accuracy\": 0.7622377622377623,\n",
      "    \"mcc\": 0.5572653931830464,\n",
      "    \"f1\": 0.7111111111111111,\n",
      "    \"precision\": 0.8421052631578947,\n",
      "    \"recall\": 0.6153846153846154\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Check the performance using test data\n",
    "hun_pred = hun_TabPre.predict(hungarian_basic_test_nolabel)\n",
    "hun_perf = hun_TabPre.evaluate_predictions(y_true=hungarian_basic_test_labels, y_pred=hun_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "99ac898b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.796610</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.046586</td>\n",
       "      <td>0.020034</td>\n",
       "      <td>0.263037</td>\n",
       "      <td>0.046586</td>\n",
       "      <td>0.020034</td>\n",
       "      <td>0.263037</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.796610</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.059125</td>\n",
       "      <td>0.020119</td>\n",
       "      <td>0.259386</td>\n",
       "      <td>0.059125</td>\n",
       "      <td>0.020119</td>\n",
       "      <td>0.259386</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.779661</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.028520</td>\n",
       "      <td>0.019805</td>\n",
       "      <td>0.255893</td>\n",
       "      <td>0.028520</td>\n",
       "      <td>0.019805</td>\n",
       "      <td>0.255893</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.779661</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.030380</td>\n",
       "      <td>0.020058</td>\n",
       "      <td>0.344331</td>\n",
       "      <td>0.001860</td>\n",
       "      <td>0.000253</td>\n",
       "      <td>0.088438</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.779661</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.049789</td>\n",
       "      <td>0.019869</td>\n",
       "      <td>0.250142</td>\n",
       "      <td>0.049789</td>\n",
       "      <td>0.019869</td>\n",
       "      <td>0.250142</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.762712</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.027594</td>\n",
       "      <td>0.002241</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.027594</td>\n",
       "      <td>0.002241</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.677966</td>\n",
       "      <td>0.659574</td>\n",
       "      <td>0.002887</td>\n",
       "      <td>0.001100</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.002887</td>\n",
       "      <td>0.001100</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.677966</td>\n",
       "      <td>0.659574</td>\n",
       "      <td>0.007464</td>\n",
       "      <td>0.002212</td>\n",
       "      <td>0.009040</td>\n",
       "      <td>0.007464</td>\n",
       "      <td>0.002212</td>\n",
       "      <td>0.009040</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0     RandomForestGini    0.796610   0.851064        0.046586       0.020034   \n",
       "1     RandomForestEntr    0.796610   0.851064        0.059125       0.020119   \n",
       "2       ExtraTreesGini    0.779661   0.872340        0.028520       0.019805   \n",
       "3  WeightedEnsemble_L2    0.779661   0.872340        0.030380       0.020058   \n",
       "4       ExtraTreesEntr    0.779661   0.851064        0.049789       0.019869   \n",
       "5              XGBoost    0.762712   0.851064        0.027594       0.002241   \n",
       "6       KNeighborsDist    0.677966   0.659574        0.002887       0.001100   \n",
       "7       KNeighborsUnif    0.677966   0.659574        0.007464       0.002212   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.263037                 0.046586                0.020034   \n",
       "1  0.259386                 0.059125                0.020119   \n",
       "2  0.255893                 0.028520                0.019805   \n",
       "3  0.344331                 0.001860                0.000253   \n",
       "4  0.250142                 0.049789                0.019869   \n",
       "5  0.017864                 0.027594                0.002241   \n",
       "6  0.004320                 0.002887                0.001100   \n",
       "7  0.009040                 0.007464                0.002212   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.263037            1       True          3  \n",
       "1           0.259386            1       True          4  \n",
       "2           0.255893            1       True          5  \n",
       "3           0.088438            2       True          8  \n",
       "4           0.250142            1       True          6  \n",
       "5           0.017864            1       True          7  \n",
       "6           0.004320            1       True          2  \n",
       "7           0.009040            1       True          1  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "hun_TabPre.leaderboard(hungarian_basic_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "4a18eef7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "hun_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ae91b89",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af657953",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a06e9ed7",
   "metadata": {},
   "source": [
    "## SLIGHTLY BETTER MODEL\n",
    "no missing values & stratified sampling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8142fc11",
   "metadata": {},
   "source": [
    "### DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40d1c89c",
   "metadata": {},
   "source": [
    "#### MISSING VALUES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "ffcdde09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make copy of original dataframe\n",
    "hungarian = hungarian_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "34aa1db1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert unknowns (-9.0) to NaN\n",
    "hungarian.replace(-9.0, np.NaN, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "c1d03c01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 295 entries, 0 to 294\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       294 non-null    float64\n",
      " 1   sex       294 non-null    float64\n",
      " 2   cp        294 non-null    float64\n",
      " 3   trestbps  293 non-null    float64\n",
      " 4   chol      271 non-null    float64\n",
      " 5   fbs       286 non-null    float64\n",
      " 6   restecg   293 non-null    float64\n",
      " 7   thalach   293 non-null    float64\n",
      " 8   exang     293 non-null    float64\n",
      " 9   oldpeak   294 non-null    float64\n",
      " 10  slope     104 non-null    float64\n",
      " 11  ca        4 non-null      float64\n",
      " 12  thal      28 non-null     float64\n",
      " 13  num       294 non-null    float64\n",
      "dtypes: float64(14)\n",
      "memory usage: 32.4 KB\n"
     ]
    }
   ],
   "source": [
    "# check for missing values\n",
    "hungarian.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "d732af72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     age  sex  cp  trestbps  chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
      "294  NaN  NaN NaN       NaN   NaN  NaN      NaN      NaN    NaN      NaN   \n",
      "\n",
      "     slope  ca  thal  num  \n",
      "294    NaN NaN   NaN  NaN  \n"
     ]
    }
   ],
   "source": [
    "print(hungarian.loc[[294]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "0fcd766f",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian = hungarian.drop([294]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "014b6cb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop slope, ca, and thal due to >50% of values missing\n",
    "hungarian.drop(['slope', 'ca', 'thal'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ad5e8e50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>308.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>36.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>166.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>48.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>47.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>257.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>53.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol  fbs  restecg  thalach  exang  oldpeak  \\\n",
       "289  48.0  0.0  2.0       NaN  308.0  0.0      1.0      NaN    NaN      2.0   \n",
       "290  36.0  1.0  2.0     120.0  166.0  0.0      0.0    180.0    0.0      0.0   \n",
       "291  48.0  1.0  3.0     110.0  211.0  0.0      0.0    138.0    0.0      0.0   \n",
       "292  47.0  0.0  2.0     140.0  257.0  0.0      0.0    135.0    0.0      1.0   \n",
       "293  53.0  1.0  4.0     130.0  182.0  0.0      0.0    148.0    0.0      0.0   \n",
       "\n",
       "     num  \n",
       "289  0.0  \n",
       "290  0.0  \n",
       "291  0.0  \n",
       "292  0.0  \n",
       "293  0.0  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian.tail() # check if next code block is needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "ab148889",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 294 entries, 0 to 293\n",
      "Data columns (total 3 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   fbs      294 non-null    float64\n",
      " 1   restecg  294 non-null    float64\n",
      " 2   exang    294 non-null    float64\n",
      "dtypes: float64(3)\n",
      "memory usage: 7.0 KB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/alexanderhughes/opt/anaconda3/lib/python3.9/site-packages/sklearn/impute/_base.py:49: FutureWarning: Unlike other reduction functions (e.g. `skew`, `kurtosis`), the default behavior of `mode` typically preserves the axis it acts along. In SciPy 1.11.0, this behavior will change: the default value of `keepdims` will become False, the `axis` over which the statistic is taken will be eliminated, and the value None will no longer be accepted. Set `keepdims` to True or False to avoid this warning.\n",
      "  mode = stats.mode(array)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "# impute discrete values using mode ()\n",
    "imputeMode = SimpleImputer(strategy=\"most_frequent\") # create mode imputer\n",
    "hungarian_disc = pd.concat([hungarian.pop(x) for x in ['fbs', 'restecg', 'exang']], axis=1) # isolate discrete\n",
    "imputeMode.fit(hungarian_disc) # fit - learns the data\n",
    "imputed_disc = imputeMode.transform(hungarian_disc) # transform - imputes with chosen strategy\n",
    "hungarian_disc_imp = pd.DataFrame(imputed_disc, columns=hungarian_disc.columns, index=hungarian.index) # back to pandas DataFrame\n",
    "hungarian_disc_imp.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "d34cedb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 294 entries, 0 to 293\n",
      "Data columns (total 3 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   trestbps  294 non-null    float64\n",
      " 1   chol      294 non-null    float64\n",
      " 2   thalach   294 non-null    float64\n",
      "dtypes: float64(3)\n",
      "memory usage: 7.0 KB\n"
     ]
    }
   ],
   "source": [
    "# NEED TO COMBINE cat, cont, and remaining\n",
    "from sklearn.impute import SimpleImputer\n",
    "# impute continuous values using mean\n",
    "imputeMean = SimpleImputer(strategy=\"mean\") # create mode imputer\n",
    "hungarian_cont = pd.concat([hungarian.pop(x) for x in ['trestbps', 'chol', 'thalach']], axis=1) # isolate continuous\n",
    "imputeMean.fit(hungarian_cont) # fit - learns the data\n",
    "imputed_cont = imputeMean.transform(hungarian_cont) # transform - imputes with chosen strategy\n",
    "hungarian_cont_imp = pd.DataFrame(imputed_cont, columns=hungarian_cont.columns, index=hungarian.index) # back to pandas DataFrame\n",
    "hungarian_cont_imp.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "f9dfe9c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian_imp = pd.concat((hungarian_cont_imp, hungarian_disc_imp, hungarian), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "b70b93b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>thalach</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>exang</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>140.0</td>\n",
       "      <td>289.000000</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>160.0</td>\n",
       "      <td>180.000000</td>\n",
       "      <td>156.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>130.0</td>\n",
       "      <td>283.000000</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>138.0</td>\n",
       "      <td>214.000000</td>\n",
       "      <td>108.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>150.0</td>\n",
       "      <td>250.848708</td>\n",
       "      <td>122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   trestbps        chol  thalach  fbs  restecg  exang   age  sex   cp  \\\n",
       "0     140.0  289.000000    172.0  0.0      0.0    0.0  40.0  1.0  2.0   \n",
       "1     160.0  180.000000    156.0  0.0      0.0    0.0  49.0  0.0  3.0   \n",
       "2     130.0  283.000000     98.0  0.0      1.0    0.0  37.0  1.0  2.0   \n",
       "3     138.0  214.000000    108.0  0.0      0.0    1.0  48.0  0.0  4.0   \n",
       "4     150.0  250.848708    122.0  0.0      0.0    0.0  54.0  1.0  3.0   \n",
       "\n",
       "   oldpeak  num  \n",
       "0      0.0  0.0  \n",
       "1      1.0  1.0  \n",
       "2      0.0  0.0  \n",
       "3      1.5  3.0  \n",
       "4      0.0  0.0  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian_imp.head() # check for anything obviously wonky"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "f804f67f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 294 entries, 0 to 293\n",
      "Data columns (total 11 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   trestbps  294 non-null    float64\n",
      " 1   chol      294 non-null    float64\n",
      " 2   thalach   294 non-null    float64\n",
      " 3   fbs       294 non-null    float64\n",
      " 4   restecg   294 non-null    float64\n",
      " 5   exang     294 non-null    float64\n",
      " 6   age       294 non-null    float64\n",
      " 7   sex       294 non-null    float64\n",
      " 8   cp        294 non-null    float64\n",
      " 9   oldpeak   294 non-null    float64\n",
      " 10  num       294 non-null    float64\n",
      "dtypes: float64(11)\n",
      "memory usage: 25.4 KB\n"
     ]
    }
   ],
   "source": [
    "hungarian_imp.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d34ae52",
   "metadata": {},
   "source": [
    "#### BINARY LABEL\n",
    "(multiclass for \"future research\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "a241b4d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>thalach</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>exang</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>140.0</td>\n",
       "      <td>289.000000</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>160.0</td>\n",
       "      <td>180.000000</td>\n",
       "      <td>156.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>130.0</td>\n",
       "      <td>283.000000</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>138.0</td>\n",
       "      <td>214.000000</td>\n",
       "      <td>108.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>150.0</td>\n",
       "      <td>250.848708</td>\n",
       "      <td>122.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   trestbps        chol  thalach  fbs  restecg  exang   age  sex   cp  \\\n",
       "0     140.0  289.000000    172.0  0.0      0.0    0.0  40.0  1.0  2.0   \n",
       "1     160.0  180.000000    156.0  0.0      0.0    0.0  49.0  0.0  3.0   \n",
       "2     130.0  283.000000     98.0  0.0      1.0    0.0  37.0  1.0  2.0   \n",
       "3     138.0  214.000000    108.0  0.0      0.0    1.0  48.0  0.0  4.0   \n",
       "4     150.0  250.848708    122.0  0.0      0.0    0.0  54.0  1.0  3.0   \n",
       "\n",
       "   oldpeak  num  \n",
       "0      0.0  0.0  \n",
       "1      1.0  1.0  \n",
       "2      0.0  0.0  \n",
       "3      1.5  1.0  \n",
       "4      0.0  0.0  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian_imp['num'][hungarian_imp['num'] > 0] = 1\n",
    "hungarian_imp.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eb91718",
   "metadata": {},
   "source": [
    "#### STRATIFIED SAMPLING\n",
    "(due to difference in sex representation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "e33c2172",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian_imp_strat = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in hungarian_imp_strat.split(hungarian_imp, hungarian_imp[\"num\"]):\n",
    "  hungarian_imp_strat_train = hungarian_imp.loc[train_index]\n",
    "  hungarian_imp_strat_test = hungarian_imp.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "fe410983",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.644068\n",
       "1.0    0.355932\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian_imp_strat_test[\"num\"].value_counts()/len(hungarian_imp_strat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "bf42adbf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.638298\n",
       "1.0    0.361702\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian_imp_strat_train[\"num\"].value_counts()/len(hungarian_imp_strat_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47f18231",
   "metadata": {},
   "source": [
    "Less significant for Cleveland and Statlog datasets, which have less dramatic difference in sex representation in data, but different for other three datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "6d27ec36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>thalach</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>exang</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>oldpeak</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>135.0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>140.0</td>\n",
       "      <td>306.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>145.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>120.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>180.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     trestbps   chol  thalach  fbs  restecg  exang   age  sex   cp  oldpeak\n",
       "251     135.0  491.0    135.0  0.0      0.0    0.0  44.0  1.0  4.0      0.0\n",
       "36      140.0  306.0     87.0  1.0      0.0    1.0  65.0  1.0  4.0      1.5\n",
       "241     145.0  326.0    155.0  0.0      0.0    0.0  55.0  1.0  2.0      0.0\n",
       "106     120.0  254.0    110.0  0.0      1.0    0.0  48.0  0.0  4.0      0.0\n",
       "191     180.0  280.0    120.0  0.0      1.0    0.0  46.0  1.0  4.0      0.0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split stratified data\n",
    "hungarian_imp_strat_train_X = hungarian_imp_strat_train.drop([heart_label], axis=1)\n",
    "hungarian_imp_strat_train_y = hungarian_imp_strat_train[heart_label]\n",
    "hungarian_imp_strat_test_X = hungarian_imp_strat_test.drop([heart_label], axis=1)\n",
    "hungarian_imp_strat_test_y = hungarian_imp_strat_test[heart_label]\n",
    "hungarian_imp_strat_train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "403c49d5",
   "metadata": {},
   "source": [
    "### BINARY MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "f66f3276",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    235\n",
      "Train Data Columns: 10\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [1.0, 0.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    328.04 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.02 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 3 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', []) : 10 | ['trestbps', 'chol', 'thalach', 'fbs', 'restecg', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('float', [])     : 7 | ['trestbps', 'chol', 'thalach', 'restecg', 'age', ...]\n",
      "\t\t('int', ['bool']) : 3 | ['fbs', 'exang', 'sex']\n",
      "\t0.0s = Fit runtime\n",
      "\t10 features in original data used to generate 10 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.01 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.03s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 188, Val Rows: 47\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.5957\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.5957\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.8298\t = Validation score   (accuracy)\n",
      "\t0.28s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8298\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 299 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.7447\t = Validation score   (accuracy)\n",
      "\t0.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.09s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.4s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "hhun_TabPre = TabularPredictor(label=heart_label, path='.').fit(hungarian_imp_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "b397d439",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.847457627118644\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.847457627118644,\n",
      "    \"balanced_accuracy\": 0.8283208020050126,\n",
      "    \"mcc\": 0.6641750237558575,\n",
      "    \"f1\": 0.7804878048780488,\n",
      "    \"precision\": 0.8,\n",
      "    \"recall\": 0.7619047619047619\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhun_pred = hhun_TabPre.predict(hungarian_imp_strat_test_X)\n",
    "hhun_perf = hhun_TabPre.evaluate_predictions(y_true=hungarian_imp_strat_test_y, y_pred=hhun_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "1832f905",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.847457627118644\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.847457627118644,\n",
      "    \"balanced_accuracy\": 0.8283208020050126,\n",
      "    \"mcc\": 0.6641750237558575,\n",
      "    \"roc_auc\": 0.8671679197994987,\n",
      "    \"f1\": 0.7804878048780488,\n",
      "    \"precision\": 0.8,\n",
      "    \"recall\": 0.7619047619047619\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhun_perf2 =  hhun_TabPre.evaluate(data=hungarian_imp_strat_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "4b1eb506",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8305084745762712\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8305084745762712,\n",
      "    \"balanced_accuracy\": 0.8151629072681704,\n",
      "    \"mcc\": 0.6303258145363408,\n",
      "    \"roc_auc\": 0.869047619047619,\n",
      "    \"f1\": 0.7619047619047619,\n",
      "    \"precision\": 0.7619047619047619,\n",
      "    \"recall\": 0.7619047619047619\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhun_perf3 =  hhun_TabPre.evaluate(data=hungarian_imp_strat_test, auxiliary_metrics=True, model='ExtraTreesEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "ccd6c374",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.847457627118644\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.847457627118644,\n",
      "    \"balanced_accuracy\": 0.8283208020050126,\n",
      "    \"mcc\": 0.6641750237558575,\n",
      "    \"roc_auc\": 0.8671679197994987,\n",
      "    \"f1\": 0.7804878048780488,\n",
      "    \"precision\": 0.8,\n",
      "    \"recall\": 0.7619047619047619\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhun_perf4 =  hhun_TabPre.evaluate(data=hungarian_imp_strat_test, auxiliary_metrics=True, model='RandomForestEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "34337a92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.847458</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.026202</td>\n",
       "      <td>0.019715</td>\n",
       "      <td>0.250962</td>\n",
       "      <td>0.026202</td>\n",
       "      <td>0.019715</td>\n",
       "      <td>0.250962</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.847458</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.031569</td>\n",
       "      <td>0.019977</td>\n",
       "      <td>0.337367</td>\n",
       "      <td>0.005367</td>\n",
       "      <td>0.000262</td>\n",
       "      <td>0.086405</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.847458</td>\n",
       "      <td>0.829787</td>\n",
       "      <td>0.034598</td>\n",
       "      <td>0.019714</td>\n",
       "      <td>0.279578</td>\n",
       "      <td>0.034598</td>\n",
       "      <td>0.019714</td>\n",
       "      <td>0.279578</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.830508</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.030860</td>\n",
       "      <td>0.021339</td>\n",
       "      <td>0.260072</td>\n",
       "      <td>0.030860</td>\n",
       "      <td>0.021339</td>\n",
       "      <td>0.260072</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.813559</td>\n",
       "      <td>0.744681</td>\n",
       "      <td>0.023582</td>\n",
       "      <td>0.002040</td>\n",
       "      <td>0.021139</td>\n",
       "      <td>0.023582</td>\n",
       "      <td>0.002040</td>\n",
       "      <td>0.021139</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.813559</td>\n",
       "      <td>0.829787</td>\n",
       "      <td>0.028561</td>\n",
       "      <td>0.020302</td>\n",
       "      <td>0.249695</td>\n",
       "      <td>0.028561</td>\n",
       "      <td>0.020302</td>\n",
       "      <td>0.249695</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.694915</td>\n",
       "      <td>0.595745</td>\n",
       "      <td>0.002780</td>\n",
       "      <td>0.001166</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.002780</td>\n",
       "      <td>0.001166</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.644068</td>\n",
       "      <td>0.595745</td>\n",
       "      <td>0.027813</td>\n",
       "      <td>0.001669</td>\n",
       "      <td>0.006533</td>\n",
       "      <td>0.027813</td>\n",
       "      <td>0.001669</td>\n",
       "      <td>0.006533</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0     RandomForestEntr    0.847458   0.851064        0.026202       0.019715   \n",
       "1  WeightedEnsemble_L2    0.847458   0.851064        0.031569       0.019977   \n",
       "2     RandomForestGini    0.847458   0.829787        0.034598       0.019714   \n",
       "3       ExtraTreesEntr    0.830508   0.851064        0.030860       0.021339   \n",
       "4              XGBoost    0.813559   0.744681        0.023582       0.002040   \n",
       "5       ExtraTreesGini    0.813559   0.829787        0.028561       0.020302   \n",
       "6       KNeighborsDist    0.694915   0.595745        0.002780       0.001166   \n",
       "7       KNeighborsUnif    0.644068   0.595745        0.027813       0.001669   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.250962                 0.026202                0.019715   \n",
       "1  0.337367                 0.005367                0.000262   \n",
       "2  0.279578                 0.034598                0.019714   \n",
       "3  0.260072                 0.030860                0.021339   \n",
       "4  0.021139                 0.023582                0.002040   \n",
       "5  0.249695                 0.028561                0.020302   \n",
       "6  0.003935                 0.002780                0.001166   \n",
       "7  0.006533                 0.027813                0.001669   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.250962            1       True          4  \n",
       "1           0.086405            2       True          8  \n",
       "2           0.279578            1       True          3  \n",
       "3           0.260072            1       True          6  \n",
       "4           0.021139            1       True          7  \n",
       "5           0.249695            1       True          5  \n",
       "6           0.003935            1       True          2  \n",
       "7           0.006533            1       True          1  "
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "hhun_TabPre.leaderboard(hungarian_imp_strat_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "c097f17f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.019715</td>\n",
       "      <td>0.250962</td>\n",
       "      <td>0.019715</td>\n",
       "      <td>0.250962</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.019977</td>\n",
       "      <td>0.337367</td>\n",
       "      <td>0.000262</td>\n",
       "      <td>0.086405</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.021339</td>\n",
       "      <td>0.260072</td>\n",
       "      <td>0.021339</td>\n",
       "      <td>0.260072</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.829787</td>\n",
       "      <td>0.019714</td>\n",
       "      <td>0.279578</td>\n",
       "      <td>0.019714</td>\n",
       "      <td>0.279578</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.829787</td>\n",
       "      <td>0.020302</td>\n",
       "      <td>0.249695</td>\n",
       "      <td>0.020302</td>\n",
       "      <td>0.249695</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.744681</td>\n",
       "      <td>0.002040</td>\n",
       "      <td>0.021139</td>\n",
       "      <td>0.002040</td>\n",
       "      <td>0.021139</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.595745</td>\n",
       "      <td>0.001166</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.001166</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.595745</td>\n",
       "      <td>0.001669</td>\n",
       "      <td>0.006533</td>\n",
       "      <td>0.001669</td>\n",
       "      <td>0.006533</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_val  pred_time_val  fit_time  \\\n",
       "0     RandomForestEntr   0.851064       0.019715  0.250962   \n",
       "1  WeightedEnsemble_L2   0.851064       0.019977  0.337367   \n",
       "2       ExtraTreesEntr   0.851064       0.021339  0.260072   \n",
       "3     RandomForestGini   0.829787       0.019714  0.279578   \n",
       "4       ExtraTreesGini   0.829787       0.020302  0.249695   \n",
       "5              XGBoost   0.744681       0.002040  0.021139   \n",
       "6       KNeighborsDist   0.595745       0.001166  0.003935   \n",
       "7       KNeighborsUnif   0.595745       0.001669  0.006533   \n",
       "\n",
       "   pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                0.019715           0.250962            1       True   \n",
       "1                0.000262           0.086405            2       True   \n",
       "2                0.021339           0.260072            1       True   \n",
       "3                0.019714           0.279578            1       True   \n",
       "4                0.020302           0.249695            1       True   \n",
       "5                0.002040           0.021139            1       True   \n",
       "6                0.001166           0.003935            1       True   \n",
       "7                0.001669           0.006533            1       True   \n",
       "\n",
       "   fit_order  \n",
       "0          4  \n",
       "1          8  \n",
       "2          6  \n",
       "3          3  \n",
       "4          5  \n",
       "5          7  \n",
       "6          2  \n",
       "7          1  "
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hhun_TabPre.leaderboard(silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "f4f55aa7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "hhun_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56f453d3",
   "metadata": {},
   "source": [
    "## REDUCED FOR CORRELATION\n",
    "(incl. no missing values & stratified sampling)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "679ea993",
   "metadata": {},
   "source": [
    "### DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e484338",
   "metadata": {},
   "source": [
    "#### REMOVE UNCORRELATED ATTRIBUTES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "5c70b1f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "thalach    -0.331074\n",
       "restecg    -0.031988\n",
       "trestbps    0.139582\n",
       "age         0.159315\n",
       "fbs         0.162869\n",
       "chol        0.202372\n",
       "sex         0.272781\n",
       "cp          0.505864\n",
       "oldpeak     0.545700\n",
       "exang       0.584541\n",
       "num         1.000000\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check correlation\n",
    "hungarian_imp_correlation = hungarian_imp.corr()\n",
    "hungarian_imp_correlation['num'].sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "f9ddf221",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 294 entries, 0 to 293\n",
      "Data columns (total 7 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   chol     294 non-null    float64\n",
      " 1   thalach  294 non-null    float64\n",
      " 2   exang    294 non-null    float64\n",
      " 3   sex      294 non-null    float64\n",
      " 4   cp       294 non-null    float64\n",
      " 5   oldpeak  294 non-null    float64\n",
      " 6   num      294 non-null    float64\n",
      "dtypes: float64(7)\n",
      "memory usage: 16.2 KB\n"
     ]
    }
   ],
   "source": [
    "# eliminate columns with correlation less than abs(0.2)\n",
    "drop_columns = ['age', 'fbs', 'trestbps', 'restecg']\n",
    "hungarian_imp_downsized = hungarian_imp.drop(columns = drop_columns)\n",
    "hungarian_imp_downsized.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08596071",
   "metadata": {},
   "source": [
    "#### STRATIFIED SPLIT OF DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "db1555b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian_imp_downsized_strat = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in hungarian_imp_downsized_strat.split(hungarian_imp_downsized, hungarian_imp_downsized[\"num\"]):\n",
    "  hungarian_imp_downsized_strat_train = hungarian_imp_downsized.loc[train_index]\n",
    "  hungarian_imp_downsized_strat_test = hungarian_imp_downsized.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "c393e6da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    0.644068\n",
       "1.0    0.355932\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hungarian_imp_downsized_strat_test[\"num\"].value_counts()/len(hungarian_imp_downsized_strat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "5bfdcee7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chol</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>oldpeak</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>326.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>291.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>214.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>272.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231</th>\n",
       "      <td>315.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      chol  thalach  exang  sex   cp  oldpeak\n",
       "241  326.0    155.0    0.0  1.0  2.0      0.0\n",
       "154  291.0    160.0    0.0  1.0  2.0      0.0\n",
       "107  214.0    168.0    0.0  1.0  2.0      0.0\n",
       "121  272.0    139.0    0.0  0.0  3.0      0.0\n",
       "231  315.0    158.0    0.0  1.0  4.0      0.0"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split stratified data\n",
    "hungarian_imp_downsized_strat_train_X = hungarian_imp_downsized_strat_train.drop([heart_label], axis=1)\n",
    "hungarian_imp_downsized_strat_train_y = hungarian_imp_downsized_strat_train[heart_label]\n",
    "hungarian_imp_downsized_strat_test_X = hungarian_imp_downsized_strat_test.drop([heart_label], axis=1)\n",
    "hungarian_imp_downsized_strat_test_y = hungarian_imp_downsized_strat_test[heart_label]\n",
    "hungarian_imp_downsized_strat_train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "defbc5ef",
   "metadata": {},
   "source": [
    "### BINARY MODELING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "42b7e551",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    235\n",
      "Train Data Columns: 6\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [1.0, 0.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    341.52 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.01 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 2 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', []) : 6 | ['chol', 'thalach', 'exang', 'sex', 'cp', ...]\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('float', [])     : 4 | ['chol', 'thalach', 'cp', 'oldpeak']\n",
      "\t\t('int', ['bool']) : 2 | ['exang', 'sex']\n",
      "\t0.0s = Fit runtime\n",
      "\t6 features in original data used to generate 6 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.01 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.04s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 188, Val Rows: 47\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.617\t = Validation score   (accuracy)\n",
      "\t0.04s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.6596\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.26s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.8723\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.8723\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.8723\t = Validation score   (accuracy)\n",
      "\t0.25s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 314 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.8511\t = Validation score   (accuracy)\n",
      "\t0.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.8723\t = Validation score   (accuracy)\n",
      "\t0.09s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.41s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "hhhun_TabPre = TabularPredictor(label=heart_label, path='.').fit(hungarian_imp_downsized_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "632d3ae4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8983050847457628\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8983050847457628,\n",
      "    \"balanced_accuracy\": 0.8784461152882206,\n",
      "    \"mcc\": 0.7755837421854133,\n",
      "    \"f1\": 0.8500000000000001,\n",
      "    \"precision\": 0.8947368421052632,\n",
      "    \"recall\": 0.8095238095238095\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhhun_pred = hhhun_TabPre.predict(hungarian_imp_downsized_strat_test_X)\n",
    "hhhun_perf = hhhun_TabPre.evaluate_predictions(y_true=hungarian_imp_downsized_strat_test_y, y_pred=hhhun_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "ffc80bcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8983050847457628\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8983050847457628,\n",
      "    \"balanced_accuracy\": 0.8784461152882206,\n",
      "    \"mcc\": 0.7755837421854133,\n",
      "    \"roc_auc\": 0.8784461152882207,\n",
      "    \"f1\": 0.8500000000000001,\n",
      "    \"precision\": 0.8947368421052632,\n",
      "    \"recall\": 0.8095238095238095\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhhun_perf2 =  hhhun_TabPre.evaluate(data=hungarian_imp_downsized_strat_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "8873ca0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8983050847457628\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8983050847457628,\n",
      "    \"balanced_accuracy\": 0.8784461152882206,\n",
      "    \"mcc\": 0.7755837421854133,\n",
      "    \"roc_auc\": 0.8784461152882207,\n",
      "    \"f1\": 0.8500000000000001,\n",
      "    \"precision\": 0.8947368421052632,\n",
      "    \"recall\": 0.8095238095238095\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhhun_perf2 =  hhhun_TabPre.evaluate(data=hungarian_imp_downsized_strat_test, auxiliary_metrics=True, model='RandomForestEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "5db07753",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.864406779661017\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.864406779661017,\n",
      "    \"balanced_accuracy\": 0.8521303258145363,\n",
      "    \"mcc\": 0.7042606516290727,\n",
      "    \"roc_auc\": 0.880325814536341,\n",
      "    \"f1\": 0.8095238095238095,\n",
      "    \"precision\": 0.8095238095238095,\n",
      "    \"recall\": 0.8095238095238095\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "hhhun_perf2 =  hhhun_TabPre.evaluate(data=hungarian_imp_downsized_strat_test, auxiliary_metrics=True, model='ExtraTreesGini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "8c362524",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.898305</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.027316</td>\n",
       "      <td>0.019844</td>\n",
       "      <td>0.247765</td>\n",
       "      <td>0.027316</td>\n",
       "      <td>0.019844</td>\n",
       "      <td>0.247765</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.898305</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.028725</td>\n",
       "      <td>0.020872</td>\n",
       "      <td>0.337257</td>\n",
       "      <td>0.001409</td>\n",
       "      <td>0.001028</td>\n",
       "      <td>0.089492</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.898305</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.053830</td>\n",
       "      <td>0.019957</td>\n",
       "      <td>0.264865</td>\n",
       "      <td>0.053830</td>\n",
       "      <td>0.019957</td>\n",
       "      <td>0.264865</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.864407</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.027277</td>\n",
       "      <td>0.019821</td>\n",
       "      <td>0.247021</td>\n",
       "      <td>0.027277</td>\n",
       "      <td>0.019821</td>\n",
       "      <td>0.247021</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.864407</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.041440</td>\n",
       "      <td>0.019889</td>\n",
       "      <td>0.249101</td>\n",
       "      <td>0.041440</td>\n",
       "      <td>0.019889</td>\n",
       "      <td>0.249101</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.830508</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.006706</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>0.006706</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.644068</td>\n",
       "      <td>0.617021</td>\n",
       "      <td>0.008847</td>\n",
       "      <td>0.001958</td>\n",
       "      <td>0.040769</td>\n",
       "      <td>0.008847</td>\n",
       "      <td>0.001958</td>\n",
       "      <td>0.040769</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.627119</td>\n",
       "      <td>0.659574</td>\n",
       "      <td>0.004270</td>\n",
       "      <td>0.001859</td>\n",
       "      <td>0.005329</td>\n",
       "      <td>0.004270</td>\n",
       "      <td>0.001859</td>\n",
       "      <td>0.005329</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0     RandomForestEntr    0.898305   0.872340        0.027316       0.019844   \n",
       "1  WeightedEnsemble_L2    0.898305   0.872340        0.028725       0.020872   \n",
       "2     RandomForestGini    0.898305   0.851064        0.053830       0.019957   \n",
       "3       ExtraTreesGini    0.864407   0.872340        0.027277       0.019821   \n",
       "4       ExtraTreesEntr    0.864407   0.872340        0.041440       0.019889   \n",
       "5              XGBoost    0.830508   0.851064        0.006706       0.002896   \n",
       "6       KNeighborsUnif    0.644068   0.617021        0.008847       0.001958   \n",
       "7       KNeighborsDist    0.627119   0.659574        0.004270       0.001859   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.247765                 0.027316                0.019844   \n",
       "1  0.337257                 0.001409                0.001028   \n",
       "2  0.264865                 0.053830                0.019957   \n",
       "3  0.247021                 0.027277                0.019821   \n",
       "4  0.249101                 0.041440                0.019889   \n",
       "5  0.023464                 0.006706                0.002896   \n",
       "6  0.040769                 0.008847                0.001958   \n",
       "7  0.005329                 0.004270                0.001859   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.247765            1       True          4  \n",
       "1           0.089492            2       True          8  \n",
       "2           0.264865            1       True          3  \n",
       "3           0.247021            1       True          5  \n",
       "4           0.249101            1       True          6  \n",
       "5           0.023464            1       True          7  \n",
       "6           0.040769            1       True          1  \n",
       "7           0.005329            1       True          2  "
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "hhhun_TabPre.leaderboard(hungarian_imp_downsized_strat_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "ef360292",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.019821</td>\n",
       "      <td>0.247021</td>\n",
       "      <td>0.019821</td>\n",
       "      <td>0.247021</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.019844</td>\n",
       "      <td>0.247765</td>\n",
       "      <td>0.019844</td>\n",
       "      <td>0.247765</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.019889</td>\n",
       "      <td>0.249101</td>\n",
       "      <td>0.019889</td>\n",
       "      <td>0.249101</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.020872</td>\n",
       "      <td>0.337257</td>\n",
       "      <td>0.001028</td>\n",
       "      <td>0.089492</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>0.002896</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.851064</td>\n",
       "      <td>0.019957</td>\n",
       "      <td>0.264865</td>\n",
       "      <td>0.019957</td>\n",
       "      <td>0.264865</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.659574</td>\n",
       "      <td>0.001859</td>\n",
       "      <td>0.005329</td>\n",
       "      <td>0.001859</td>\n",
       "      <td>0.005329</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.617021</td>\n",
       "      <td>0.001958</td>\n",
       "      <td>0.040769</td>\n",
       "      <td>0.001958</td>\n",
       "      <td>0.040769</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_val  pred_time_val  fit_time  \\\n",
       "0       ExtraTreesGini   0.872340       0.019821  0.247021   \n",
       "1     RandomForestEntr   0.872340       0.019844  0.247765   \n",
       "2       ExtraTreesEntr   0.872340       0.019889  0.249101   \n",
       "3  WeightedEnsemble_L2   0.872340       0.020872  0.337257   \n",
       "4              XGBoost   0.851064       0.002896  0.023464   \n",
       "5     RandomForestGini   0.851064       0.019957  0.264865   \n",
       "6       KNeighborsDist   0.659574       0.001859  0.005329   \n",
       "7       KNeighborsUnif   0.617021       0.001958  0.040769   \n",
       "\n",
       "   pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                0.019821           0.247021            1       True   \n",
       "1                0.019844           0.247765            1       True   \n",
       "2                0.019889           0.249101            1       True   \n",
       "3                0.001028           0.089492            2       True   \n",
       "4                0.002896           0.023464            1       True   \n",
       "5                0.019957           0.264865            1       True   \n",
       "6                0.001859           0.005329            1       True   \n",
       "7                0.001958           0.040769            1       True   \n",
       "\n",
       "   fit_order  \n",
       "0          5  \n",
       "1          4  \n",
       "2          6  \n",
       "3          8  \n",
       "4          7  \n",
       "5          3  \n",
       "6          2  \n",
       "7          1  "
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hhhun_TabPre.leaderboard( silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "b0fefbfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "hhhun_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03c2c76f",
   "metadata": {},
   "source": [
    "# COMBINED"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42ace093",
   "metadata": {},
   "source": [
    "## TAKE A PEEK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "b3d60c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load Switzerland datafile\n",
    "switzerland_csv_path = \"processed.switzerland.data\"\n",
    "switzerland_data = pd.read_csv(switzerland_csv_path, header = None)\n",
    "# assign column names\n",
    "switzerland_data.set_axis(['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs',\n",
    "                  'restecg', 'thalach', 'exang', 'oldpeak', 'slope',\n",
    "                  'ca', 'thal', 'num'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "0e71309d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load VA datafile\n",
    "va_csv_path = \"processed.va.data\"\n",
    "va_data = pd.read_csv(va_csv_path, header = None)\n",
    "# assign column names\n",
    "va_data.set_axis(['age', 'sex', 'cp', 'trestbps', 'chol', 'fbs',\n",
    "                  'restecg', 'thalach', 'exang', 'oldpeak', 'slope',\n",
    "                  'ca', 'thal', 'num'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "d20520dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>95</td>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "      <td>.7</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>115</td>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>154</td>\n",
       "      <td>0</td>\n",
       "      <td>.2</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "      <td>130</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>110</td>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "      <td>125</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>105</td>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "      <td>166</td>\n",
       "      <td>0</td>\n",
       "      <td>2.8</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp trestbps  chol fbs restecg thalach exang oldpeak slope ca  \\\n",
       "0   32    1   1       95     0   ?       0     127     0      .7     1  ?   \n",
       "1   34    1   4      115     0   ?       ?     154     0      .2     1  ?   \n",
       "2   35    1   4        ?     0   ?       0     130     1       ?     ?  ?   \n",
       "3   36    1   4      110     0   ?       0     125     1       1     2  ?   \n",
       "4   38    0   4      105     0   ?       0     166     0     2.8     1  ?   \n",
       "\n",
       "  thal  num  \n",
       "0    ?    1  \n",
       "1    ?    1  \n",
       "2    7    3  \n",
       "3    6    1  \n",
       "4    ?    2  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "switzerland_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "7a0500ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>140</td>\n",
       "      <td>260</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>112</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>130</td>\n",
       "      <td>209</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>127</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>132</td>\n",
       "      <td>218</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>140</td>\n",
       "      <td>1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>142</td>\n",
       "      <td>228</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>149</td>\n",
       "      <td>1</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>66</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>110</td>\n",
       "      <td>213</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>99</td>\n",
       "      <td>1</td>\n",
       "      <td>1.3</td>\n",
       "      <td>2</td>\n",
       "      <td>?</td>\n",
       "      <td>?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  sex  cp trestbps chol fbs  restecg thalach exang oldpeak slope ca  \\\n",
       "0   63    1   4      140  260   0        1     112     1       3     2  ?   \n",
       "1   44    1   4      130  209   0        1     127     0       0     ?  ?   \n",
       "2   60    1   4      132  218   0        1     140     1     1.5     3  ?   \n",
       "3   55    1   4      142  228   0        1     149     1     2.5     1  ?   \n",
       "4   66    1   3      110  213   1        2      99     1     1.3     2  ?   \n",
       "\n",
       "  thal  num  \n",
       "0    ?    2  \n",
       "1    ?    0  \n",
       "2    ?    2  \n",
       "3    ?    1  \n",
       "4    ?    0  "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "va_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "685aadb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 123 entries, 0 to 122\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   age       123 non-null    int64 \n",
      " 1   sex       123 non-null    int64 \n",
      " 2   cp        123 non-null    int64 \n",
      " 3   trestbps  123 non-null    object\n",
      " 4   chol      123 non-null    int64 \n",
      " 5   fbs       123 non-null    object\n",
      " 6   restecg   123 non-null    object\n",
      " 7   thalach   123 non-null    object\n",
      " 8   exang     123 non-null    object\n",
      " 9   oldpeak   123 non-null    object\n",
      " 10  slope     123 non-null    object\n",
      " 11  ca        123 non-null    object\n",
      " 12  thal      123 non-null    object\n",
      " 13  num       123 non-null    int64 \n",
      "dtypes: int64(5), object(9)\n",
      "memory usage: 13.6+ KB\n"
     ]
    }
   ],
   "source": [
    "switzerland_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "b390fabe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200 entries, 0 to 199\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   age       200 non-null    int64 \n",
      " 1   sex       200 non-null    int64 \n",
      " 2   cp        200 non-null    int64 \n",
      " 3   trestbps  200 non-null    object\n",
      " 4   chol      200 non-null    object\n",
      " 5   fbs       200 non-null    object\n",
      " 6   restecg   200 non-null    int64 \n",
      " 7   thalach   200 non-null    object\n",
      " 8   exang     200 non-null    object\n",
      " 9   oldpeak   200 non-null    object\n",
      " 10  slope     200 non-null    object\n",
      " 11  ca        200 non-null    object\n",
      " 12  thal      200 non-null    object\n",
      " 13  num       200 non-null    int64 \n",
      "dtypes: int64(5), object(9)\n",
      "memory usage: 22.0+ KB\n"
     ]
    }
   ],
   "source": [
    "va_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b970348",
   "metadata": {},
   "source": [
    "## DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "213dbbd4",
   "metadata": {},
   "source": [
    "### COPY DATASETS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "56835843",
   "metadata": {},
   "outputs": [],
   "source": [
    "switzerland = switzerland_data.copy()\n",
    "va = va_data.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d2474c5",
   "metadata": {},
   "source": [
    "### MISSING VALUES"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c076622",
   "metadata": {},
   "source": [
    "#### CONVERT ? TO NAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "32f222c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>115</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>70</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>157</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>72</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>160</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>114</td>\n",
       "      <td>0</td>\n",
       "      <td>1.6</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>160</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>74</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>145</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>123</td>\n",
       "      <td>0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp trestbps  chol  fbs restecg thalach exang oldpeak slope  \\\n",
       "118   70    1   4      115     0    0       1      92     1       0     2   \n",
       "119   70    1   4      140     0    1       0     157     1       2     2   \n",
       "120   72    1   3      160     0  NaN       2     114     0     1.6     2   \n",
       "121   73    0   3      160     0    0       1     121     0       0     1   \n",
       "122   74    1   2      145     0  NaN       1     123     0     1.3     1   \n",
       "\n",
       "      ca thal  num  \n",
       "118  NaN    7    1  \n",
       "119  NaN    7    3  \n",
       "120    2  NaN    0  \n",
       "121  NaN    3    1  \n",
       "122  NaN  NaN    1  "
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "switzerland = switzerland.replace('?', np.NaN) # replace '?' values with NaN so can impute\n",
    "switzerland[['age', 'trestbps', 'chol', 'thalach', 'oldpeak']].astype(float) # convert dtype\n",
    "switzerland.tail() # to verify didn't mess up dataframe IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "d8acd487",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make dataset version for plotting purposes\n",
    "switzerland_for_plot = switzerland.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "c35914d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>fbs</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>slope</th>\n",
       "      <th>ca</th>\n",
       "      <th>thal</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>127</td>\n",
       "      <td>333</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>154</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>62</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>139</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>55</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>122</td>\n",
       "      <td>223</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>58</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>385</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>62</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>120</td>\n",
       "      <td>254</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>93</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     age  sex  cp trestbps chol fbs  restecg thalach exang oldpeak slope   ca  \\\n",
       "195   54    0   4      127  333   1        1     154     0       0   NaN  NaN   \n",
       "196   62    1   1      NaN  139   0        1     NaN   NaN     NaN   NaN  NaN   \n",
       "197   55    1   4      122  223   1        1     100     0       0   NaN  NaN   \n",
       "198   58    1   4      NaN  385   1        2     NaN   NaN     NaN   NaN  NaN   \n",
       "199   62    1   2      120  254   0        2      93     1       0   NaN  NaN   \n",
       "\n",
       "    thal  num  \n",
       "195  NaN    1  \n",
       "196  NaN    0  \n",
       "197    6    2  \n",
       "198  NaN    0  \n",
       "199  NaN    1  "
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "va = va.replace('?', np.NaN) # replace '?' values with NaN so can impute\n",
    "va[['age', 'trestbps', 'chol', 'thalach', 'oldpeak']].astype(float) # convert dtype\n",
    "va.tail() # to verify didn't mess up dataframe IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "08e57567",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make dataset version for plotting purposes\n",
    "va_for_plot = va.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd0cfb6b",
   "metadata": {},
   "source": [
    "#### DROP IF MISSING >50%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "7b28fb59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 123 entries, 0 to 122\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   age       123 non-null    int64 \n",
      " 1   sex       123 non-null    int64 \n",
      " 2   cp        123 non-null    int64 \n",
      " 3   trestbps  121 non-null    object\n",
      " 4   chol      123 non-null    int64 \n",
      " 5   fbs       48 non-null     object\n",
      " 6   restecg   122 non-null    object\n",
      " 7   thalach   122 non-null    object\n",
      " 8   exang     122 non-null    object\n",
      " 9   oldpeak   117 non-null    object\n",
      " 10  slope     106 non-null    object\n",
      " 11  ca        5 non-null      object\n",
      " 12  thal      71 non-null     object\n",
      " 13  num       123 non-null    int64 \n",
      "dtypes: int64(5), object(9)\n",
      "memory usage: 13.6+ KB\n"
     ]
    }
   ],
   "source": [
    "switzerland.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "2b0022f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200 entries, 0 to 199\n",
      "Data columns (total 14 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   age       200 non-null    int64 \n",
      " 1   sex       200 non-null    int64 \n",
      " 2   cp        200 non-null    int64 \n",
      " 3   trestbps  144 non-null    object\n",
      " 4   chol      193 non-null    object\n",
      " 5   fbs       193 non-null    object\n",
      " 6   restecg   200 non-null    int64 \n",
      " 7   thalach   147 non-null    object\n",
      " 8   exang     147 non-null    object\n",
      " 9   oldpeak   144 non-null    object\n",
      " 10  slope     98 non-null     object\n",
      " 11  ca        2 non-null      object\n",
      " 12  thal      34 non-null     object\n",
      " 13  num       200 non-null    int64 \n",
      "dtypes: int64(5), object(9)\n",
      "memory usage: 22.0+ KB\n"
     ]
    }
   ],
   "source": [
    "va.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "66ca9f1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Switzerland - drop fbs, ca, and thal due to >50% of values missing\n",
    "# VA - drop slope, ca, and thal due to >50% of values missing\n",
    "# as combined dataset - drop fbs, slope, ca, and thal\n",
    "switzerland_mod = switzerland.copy()\n",
    "switzerland_mod.drop(['fbs', 'slope', 'ca', 'thal'], axis=1, inplace=True)\n",
    "va_mod = va.copy()\n",
    "va_mod.drop(['fbs', 'slope', 'ca', 'thal'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6a6e47f",
   "metadata": {},
   "source": [
    "#### IMPUTE TO MODE OR MEAN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f564b79b",
   "metadata": {},
   "source": [
    "##### SWITZERLAND"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "990c8be0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 123 entries, 0 to 122\n",
      "Data columns (total 10 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   age       123 non-null    int64 \n",
      " 1   sex       123 non-null    int64 \n",
      " 2   cp        123 non-null    int64 \n",
      " 3   trestbps  121 non-null    object\n",
      " 4   chol      123 non-null    int64 \n",
      " 5   restecg   122 non-null    object\n",
      " 6   thalach   122 non-null    object\n",
      " 7   exang     122 non-null    object\n",
      " 8   oldpeak   117 non-null    object\n",
      " 9   num       123 non-null    int64 \n",
      "dtypes: int64(5), object(5)\n",
      "memory usage: 9.7+ KB\n"
     ]
    }
   ],
   "source": [
    "switzerland_mod.info()\n",
    "# to impute -\n",
    "#    to mode: restecg, exang\n",
    "#    to mean: trestbps, thalach, oldpeak"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "a9b2e6bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 123 entries, 0 to 122\n",
      "Data columns (total 2 columns):\n",
      " #   Column   Non-Null Count  Dtype \n",
      "---  ------   --------------  ----- \n",
      " 0   restecg  123 non-null    object\n",
      " 1   exang    123 non-null    object\n",
      "dtypes: object(2)\n",
      "memory usage: 2.0+ KB\n"
     ]
    }
   ],
   "source": [
    "# impute discrete values using mode ()\n",
    "imputeMode = SimpleImputer(strategy=\"most_frequent\") # create mode imputer\n",
    "switzerland_mod_disc = pd.concat([switzerland_mod.pop(x) for x in ['restecg', 'exang']], axis=1) # isolate discrete\n",
    "imputeMode.fit(switzerland_mod_disc) # fit - learns the data\n",
    "imputed_disc = imputeMode.transform(switzerland_mod_disc) # transform - imputes with chosen strategy\n",
    "switzerland_mod_disc_imp = pd.DataFrame(imputed_disc, columns=switzerland_mod_disc.columns, index=switzerland_mod.index) # back to pandas DataFrame\n",
    "switzerland_mod_disc_imp.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "f02579bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 123 entries, 0 to 122\n",
      "Data columns (total 3 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   trestbps  123 non-null    float64\n",
      " 1   oldpeak   123 non-null    float64\n",
      " 2   thalach   123 non-null    float64\n",
      "dtypes: float64(3)\n",
      "memory usage: 3.0 KB\n"
     ]
    }
   ],
   "source": [
    "# impute continuous values using mean\n",
    "imputeMean = SimpleImputer(strategy=\"mean\") # create mode imputer\n",
    "switzerland_mod_cont = pd.concat([switzerland_mod.pop(x) for x in ['trestbps', 'oldpeak', 'thalach']], axis=1) # isolate continuous\n",
    "imputeMean.fit(switzerland_mod_cont) # fit - learns the data\n",
    "imputed_cont = imputeMean.transform(switzerland_mod_cont) # transform - imputes with chosen strategy\n",
    "switzerland_mod_cont_imp = pd.DataFrame(imputed_cont, columns=switzerland_mod_cont.columns, index=switzerland_mod.index) # back to pandas DataFrame\n",
    "switzerland_mod_cont_imp.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "18b6f540",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine cat, cont, and remaining\n",
    "switzerland_mod_imp = pd.concat((switzerland_mod_cont_imp, switzerland_mod_disc_imp, switzerland_mod), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "c8f3fc24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trestbps</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>thalach</th>\n",
       "      <th>restecg</th>\n",
       "      <th>exang</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>chol</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>95.000000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>127.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>115.000000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>154.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>130.206612</td>\n",
       "      <td>0.653846</td>\n",
       "      <td>130.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>110.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>125.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>105.000000</td>\n",
       "      <td>2.800000</td>\n",
       "      <td>166.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     trestbps   oldpeak  thalach restecg exang  age  sex  cp  chol  num\n",
       "0   95.000000  0.700000    127.0       0     0   32    1   1     0    1\n",
       "1  115.000000  0.200000    154.0       0     0   34    1   4     0    1\n",
       "2  130.206612  0.653846    130.0       0     1   35    1   4     0    3\n",
       "3  110.000000  1.000000    125.0       0     1   36    1   4     0    1\n",
       "4  105.000000  2.800000    166.0       0     0   38    0   4     0    2"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "switzerland_mod_imp.head() # check for anything obviously wonky"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "2ecd67cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 123 entries, 0 to 122\n",
      "Data columns (total 10 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   trestbps  123 non-null    float64\n",
      " 1   oldpeak   123 non-null    float64\n",
      " 2   thalach   123 non-null    float64\n",
      " 3   restecg   123 non-null    object \n",
      " 4   exang     123 non-null    object \n",
      " 5   age       123 non-null    int64  \n",
      " 6   sex       123 non-null    int64  \n",
      " 7   cp        123 non-null    int64  \n",
      " 8   chol      123 non-null    int64  \n",
      " 9   num       123 non-null    int64  \n",
      "dtypes: float64(3), int64(5), object(2)\n",
      "memory usage: 9.7+ KB\n"
     ]
    }
   ],
   "source": [
    "switzerland_mod_imp.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3f2768d",
   "metadata": {},
   "source": [
    "##### VA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "a23d5ceb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200 entries, 0 to 199\n",
      "Data columns (total 10 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   age       200 non-null    int64 \n",
      " 1   sex       200 non-null    int64 \n",
      " 2   cp        200 non-null    int64 \n",
      " 3   trestbps  144 non-null    object\n",
      " 4   chol      193 non-null    object\n",
      " 5   restecg   200 non-null    int64 \n",
      " 6   thalach   147 non-null    object\n",
      " 7   exang     147 non-null    object\n",
      " 8   oldpeak   144 non-null    object\n",
      " 9   num       200 non-null    int64 \n",
      "dtypes: int64(5), object(5)\n",
      "memory usage: 15.8+ KB\n"
     ]
    }
   ],
   "source": [
    " va_mod.info()\n",
    "# to impute -\n",
    "#    to mode: exang\n",
    "#    to mean: trestbps, thalach, oldpeak, chol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "3b9f718c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200 entries, 0 to 199\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   exang   200 non-null    object\n",
      "dtypes: object(1)\n",
      "memory usage: 1.7+ KB\n"
     ]
    }
   ],
   "source": [
    "# impute discrete values using mode ()\n",
    "imputeMode = SimpleImputer(strategy=\"most_frequent\") # create mode imputer\n",
    "va_mod_disc = pd.concat([va_mod.pop(x) for x in ['exang']], axis=1) # isolate discrete\n",
    "imputeMode.fit(va_mod_disc) # fit - learns the data\n",
    "imputed_disc = imputeMode.transform(va_mod_disc) # transform - imputes with chosen strategy\n",
    "va_mod_disc_imp = pd.DataFrame(imputed_disc, columns=va_mod_disc.columns, index=va_mod.index) # back to pandas DataFrame\n",
    "va_mod_disc_imp.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "20dbd7c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200 entries, 0 to 199\n",
      "Data columns (total 4 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   trestbps  200 non-null    float64\n",
      " 1   chol      200 non-null    float64\n",
      " 2   oldpeak   200 non-null    float64\n",
      " 3   thalach   200 non-null    float64\n",
      "dtypes: float64(4)\n",
      "memory usage: 6.4 KB\n"
     ]
    }
   ],
   "source": [
    "# impute continuous values using mean\n",
    "imputeMean = SimpleImputer(strategy=\"mean\") # create mode imputer\n",
    "va_mod_cont = pd.concat([va_mod.pop(x) for x in ['trestbps', 'chol', 'oldpeak', 'thalach']], axis=1) # isolate continuous\n",
    "imputeMean.fit(va_mod_cont) # fit - learns the data\n",
    "imputed_cont = imputeMean.transform(va_mod_cont) # transform - imputes with chosen strategy\n",
    "va_mod_cont_imp = pd.DataFrame(imputed_cont, columns=va_mod_cont.columns, index=va_mod.index) # back to pandas DataFrame\n",
    "va_mod_cont_imp.info() # check for missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "9f7d4536",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine cat, cont, and remaining\n",
    "va_mod_imp = pd.concat((va_mod_cont_imp, va_mod_disc_imp, va_mod), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "64fdd679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>oldpeak</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>restecg</th>\n",
       "      <th>num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>140.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>130.0</td>\n",
       "      <td>209.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>127.0</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>132.0</td>\n",
       "      <td>218.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>140.0</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>142.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>149.0</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110.0</td>\n",
       "      <td>213.0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>99.0</td>\n",
       "      <td>1</td>\n",
       "      <td>66</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   trestbps   chol  oldpeak  thalach exang  age  sex  cp  restecg  num\n",
       "0     140.0  260.0      3.0    112.0     1   63    1   4        1    2\n",
       "1     130.0  209.0      0.0    127.0     0   44    1   4        1    0\n",
       "2     132.0  218.0      1.5    140.0     1   60    1   4        1    2\n",
       "3     142.0  228.0      2.5    149.0     1   55    1   4        1    1\n",
       "4     110.0  213.0      1.3     99.0     1   66    1   3        2    0"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "va_mod_imp.head() # check for anything obviously wonky"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "d6e6dabf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 200 entries, 0 to 199\n",
      "Data columns (total 10 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   trestbps  200 non-null    float64\n",
      " 1   chol      200 non-null    float64\n",
      " 2   oldpeak   200 non-null    float64\n",
      " 3   thalach   200 non-null    float64\n",
      " 4   exang     200 non-null    object \n",
      " 5   age       200 non-null    int64  \n",
      " 6   sex       200 non-null    int64  \n",
      " 7   cp        200 non-null    int64  \n",
      " 8   restecg   200 non-null    int64  \n",
      " 9   num       200 non-null    int64  \n",
      "dtypes: float64(4), int64(5), object(1)\n",
      "memory usage: 15.8+ KB\n"
     ]
    }
   ],
   "source": [
    "va_mod_imp.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "208a57e4",
   "metadata": {},
   "source": [
    "### BINARY LABEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "fdcea504",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/sg/9r6jl3rj2l96943q_2ph1y980000gn/T/ipykernel_40379/3411012929.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  switzerland_mod_imp['num'][switzerland_mod_imp['num'] > 0] = 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "num\n",
       "1    115\n",
       "0      8\n",
       "dtype: int64"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "switzerland_mod_imp['num'][switzerland_mod_imp['num'] > 0] = 1\n",
    "switzerland_mod_imp.value_counts('num')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "502339a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/sg/9r6jl3rj2l96943q_2ph1y980000gn/T/ipykernel_40379/2235763240.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  va_mod_imp['num'][va_mod_imp['num'] > 0] = 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "num\n",
       "1    149\n",
       "0     51\n",
       "dtype: int64"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "va_mod_imp['num'][va_mod_imp['num'] > 0] = 1\n",
    "va_mod_imp.value_counts('num')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5897add",
   "metadata": {},
   "source": [
    "### CREATE COMBINED DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "cf1a9146",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleveland.info()\n",
    "# for combined, drop - slope, ca, thal, fbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "0ef7fbbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleveland_comb = cleveland.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "0fc1f2d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleveland_comb.drop(['fbs', 'slope', 'ca', 'thal'], axis=1, inplace=True)\n",
    "#cleveland_comb.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "042f8076",
   "metadata": {},
   "outputs": [],
   "source": [
    "#hungarian_imp.info()\n",
    "# for combined, drop - fbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "721d1495",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian_comb = hungarian_imp.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "60f546f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "hungarian_comb.drop(['fbs'], axis=1, inplace=True)\n",
    "#hungarian_comb.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "7070fe49",
   "metadata": {},
   "outputs": [],
   "source": [
    "#switzerland_mod_imp.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "8a99bdde",
   "metadata": {},
   "outputs": [],
   "source": [
    "#va_mod_imp.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "a652df96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 920 entries, 0 to 919\n",
      "Data columns (total 10 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       920 non-null    float64\n",
      " 1   sex       920 non-null    float64\n",
      " 2   cp        920 non-null    float64\n",
      " 3   trestbps  920 non-null    float64\n",
      " 4   chol      920 non-null    float64\n",
      " 5   restecg   920 non-null    object \n",
      " 6   thalach   920 non-null    float64\n",
      " 7   exang     920 non-null    object \n",
      " 8   oldpeak   920 non-null    float64\n",
      " 9   num       920 non-null    float64\n",
      "dtypes: float64(8), object(2)\n",
      "memory usage: 72.0+ KB\n"
     ]
    }
   ],
   "source": [
    "combined = pd.concat((cleveland_comb, hungarian_comb, switzerland_mod_imp, va_mod_imp), axis=0, ignore_index=True)\n",
    "combined.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5b99868",
   "metadata": {},
   "source": [
    "### STRATIFIED SAMPLING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "2c7cabdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_strat = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in combined_strat.split(combined, combined[\"num\"]):\n",
    "  combined_strat_train = combined.loc[train_index]\n",
    "  combined_strat_test = combined.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "8041759d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    0.554348\n",
       "0.0    0.445652\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_strat_test[\"num\"].value_counts()/len(combined_strat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "d1599c47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    0.552989\n",
       "0.0    0.447011\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_strat_train[\"num\"].value_counts()/len(combined_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "5e3a4e73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>trestbps</th>\n",
       "      <th>chol</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>817</th>\n",
       "      <td>62.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>1</td>\n",
       "      <td>150.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.320833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580</th>\n",
       "      <td>51.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>62.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>394.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>59.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544</th>\n",
       "      <td>55.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp  trestbps   chol restecg  thalach exang   oldpeak\n",
       "817  62.0  1.0  1.0     112.0  258.0       1    150.0     1  1.320833\n",
       "580  51.0  1.0  3.0     135.0  160.0     0.0    150.0   0.0  2.000000\n",
       "173  62.0  0.0  4.0     140.0  394.0     2.0    157.0   0.0  1.200000\n",
       "766  59.0  1.0  4.0     122.0  233.0       0    117.0     1  1.300000\n",
       "544  55.0  1.0  2.0     145.0  326.0     0.0    155.0   0.0  0.000000"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split stratified data\n",
    "combined_strat_train_X = combined_strat_train.drop([heart_label], axis=1)\n",
    "combined_strat_train_y = combined_strat_train[heart_label]\n",
    "combined_strat_test_X = combined_strat_test.drop([heart_label], axis=1)\n",
    "combined_strat_test_y = combined_strat_test[heart_label]\n",
    "combined_strat_train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23ca3bc6",
   "metadata": {},
   "source": [
    "## BINARY MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76e42c33",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "42441f0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    736\n",
      "Train Data Columns: 9\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [1.0, 0.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    337.85 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.1 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 1 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', [])  : 7 | ['age', 'sex', 'cp', 'trestbps', 'chol', ...]\n",
      "\t\t('object', []) : 2 | ['restecg', 'exang']\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', [])  : 2 | ['restecg', 'exang']\n",
      "\t\t('float', [])     : 6 | ['age', 'cp', 'trestbps', 'chol', 'thalach', ...]\n",
      "\t\t('int', ['bool']) : 1 | ['sex']\n",
      "\t0.0s = Fit runtime\n",
      "\t9 features in original data used to generate 9 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.04 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.05s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 588, Val Rows: 148\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.6622\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.6689\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.7703\t = Validation score   (accuracy)\n",
      "\t0.3s\t = Training   runtime\n",
      "\t0.03s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.7973\t = Validation score   (accuracy)\n",
      "\t0.29s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.7838\t = Validation score   (accuracy)\n",
      "\t0.29s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.7703\t = Validation score   (accuracy)\n",
      "\t0.28s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 302 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.723\t = Validation score   (accuracy)\n",
      "\t0.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.8108\t = Validation score   (accuracy)\n",
      "\t0.11s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.59s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "comb_TabPre = TabularPredictor(label=heart_label, path='.').fit(combined_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "3c0efb62",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8478260869565217\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8478260869565217,\n",
      "    \"balanced_accuracy\": 0.8400286944045912,\n",
      "    \"mcc\": 0.6925917533073389,\n",
      "    \"f1\": 0.8691588785046729,\n",
      "    \"precision\": 0.8303571428571429,\n",
      "    \"recall\": 0.9117647058823529\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_pred = comb_TabPre.predict(combined_strat_test_X)\n",
    "comb_perf = comb_TabPre.evaluate_predictions(y_true=combined_strat_test_y, y_pred=comb_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "3785645a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8478260869565217\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8478260869565217,\n",
      "    \"balanced_accuracy\": 0.8400286944045912,\n",
      "    \"mcc\": 0.6925917533073389,\n",
      "    \"roc_auc\": 0.9184600669536107,\n",
      "    \"f1\": 0.8691588785046729,\n",
      "    \"precision\": 0.8303571428571429,\n",
      "    \"recall\": 0.9117647058823529\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_perf2 =  comb_TabPre.evaluate(data=combined_strat_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "25d7b2ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8478260869565217\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8478260869565217,\n",
      "    \"balanced_accuracy\": 0.8400286944045912,\n",
      "    \"mcc\": 0.6925917533073389,\n",
      "    \"roc_auc\": 0.9149330463892874,\n",
      "    \"f1\": 0.8691588785046729,\n",
      "    \"precision\": 0.8303571428571429,\n",
      "    \"recall\": 0.9117647058823529\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_perf3 =  comb_TabPre.evaluate(data=combined_strat_test, auxiliary_metrics=True, model='RandomForestEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "319a1d96",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.842391304347826\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.842391304347826,\n",
      "    \"balanced_accuracy\": 0.8339311334289814,\n",
      "    \"mcc\": 0.6819075880955596,\n",
      "    \"roc_auc\": 0.919296987087518,\n",
      "    \"f1\": 0.8651162790697673,\n",
      "    \"precision\": 0.8230088495575221,\n",
      "    \"recall\": 0.9117647058823529\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_perf4 =  comb_TabPre.evaluate(data=combined_strat_test, auxiliary_metrics=True, model='ExtraTreesGini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "ff2b33cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.847826</td>\n",
       "      <td>0.797297</td>\n",
       "      <td>0.036896</td>\n",
       "      <td>0.024798</td>\n",
       "      <td>0.294500</td>\n",
       "      <td>0.036896</td>\n",
       "      <td>0.024798</td>\n",
       "      <td>0.294500</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.847826</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.123012</td>\n",
       "      <td>0.074949</td>\n",
       "      <td>0.979291</td>\n",
       "      <td>0.001744</td>\n",
       "      <td>0.000401</td>\n",
       "      <td>0.105070</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.842391</td>\n",
       "      <td>0.770270</td>\n",
       "      <td>0.037669</td>\n",
       "      <td>0.025544</td>\n",
       "      <td>0.297175</td>\n",
       "      <td>0.037669</td>\n",
       "      <td>0.025544</td>\n",
       "      <td>0.297175</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.842391</td>\n",
       "      <td>0.783784</td>\n",
       "      <td>0.041508</td>\n",
       "      <td>0.024531</td>\n",
       "      <td>0.287530</td>\n",
       "      <td>0.041508</td>\n",
       "      <td>0.024531</td>\n",
       "      <td>0.287530</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.826087</td>\n",
       "      <td>0.770270</td>\n",
       "      <td>0.046702</td>\n",
       "      <td>0.024206</td>\n",
       "      <td>0.282546</td>\n",
       "      <td>0.046702</td>\n",
       "      <td>0.024206</td>\n",
       "      <td>0.282546</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.809783</td>\n",
       "      <td>0.722973</td>\n",
       "      <td>0.005913</td>\n",
       "      <td>0.003386</td>\n",
       "      <td>0.023183</td>\n",
       "      <td>0.005913</td>\n",
       "      <td>0.003386</td>\n",
       "      <td>0.023183</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.690217</td>\n",
       "      <td>0.668919</td>\n",
       "      <td>0.003670</td>\n",
       "      <td>0.001662</td>\n",
       "      <td>0.004442</td>\n",
       "      <td>0.003670</td>\n",
       "      <td>0.001662</td>\n",
       "      <td>0.004442</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.690217</td>\n",
       "      <td>0.662162</td>\n",
       "      <td>0.014753</td>\n",
       "      <td>0.002047</td>\n",
       "      <td>0.012023</td>\n",
       "      <td>0.014753</td>\n",
       "      <td>0.002047</td>\n",
       "      <td>0.012023</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0     RandomForestEntr    0.847826   0.797297        0.036896       0.024798   \n",
       "1  WeightedEnsemble_L2    0.847826   0.810811        0.123012       0.074949   \n",
       "2     RandomForestGini    0.842391   0.770270        0.037669       0.025544   \n",
       "3       ExtraTreesGini    0.842391   0.783784        0.041508       0.024531   \n",
       "4       ExtraTreesEntr    0.826087   0.770270        0.046702       0.024206   \n",
       "5              XGBoost    0.809783   0.722973        0.005913       0.003386   \n",
       "6       KNeighborsDist    0.690217   0.668919        0.003670       0.001662   \n",
       "7       KNeighborsUnif    0.690217   0.662162        0.014753       0.002047   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.294500                 0.036896                0.024798   \n",
       "1  0.979291                 0.001744                0.000401   \n",
       "2  0.297175                 0.037669                0.025544   \n",
       "3  0.287530                 0.041508                0.024531   \n",
       "4  0.282546                 0.046702                0.024206   \n",
       "5  0.023183                 0.005913                0.003386   \n",
       "6  0.004442                 0.003670                0.001662   \n",
       "7  0.012023                 0.014753                0.002047   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.294500            1       True          4  \n",
       "1           0.105070            2       True          8  \n",
       "2           0.297175            1       True          3  \n",
       "3           0.287530            1       True          5  \n",
       "4           0.282546            1       True          6  \n",
       "5           0.023183            1       True          7  \n",
       "6           0.004442            1       True          2  \n",
       "7           0.012023            1       True          1  "
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "comb_TabPre.leaderboard(combined_strat_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "07220e1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.074949</td>\n",
       "      <td>0.979291</td>\n",
       "      <td>0.000401</td>\n",
       "      <td>0.105070</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.797297</td>\n",
       "      <td>0.024798</td>\n",
       "      <td>0.294500</td>\n",
       "      <td>0.024798</td>\n",
       "      <td>0.294500</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.783784</td>\n",
       "      <td>0.024531</td>\n",
       "      <td>0.287530</td>\n",
       "      <td>0.024531</td>\n",
       "      <td>0.287530</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.770270</td>\n",
       "      <td>0.024206</td>\n",
       "      <td>0.282546</td>\n",
       "      <td>0.024206</td>\n",
       "      <td>0.282546</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.770270</td>\n",
       "      <td>0.025544</td>\n",
       "      <td>0.297175</td>\n",
       "      <td>0.025544</td>\n",
       "      <td>0.297175</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.722973</td>\n",
       "      <td>0.003386</td>\n",
       "      <td>0.023183</td>\n",
       "      <td>0.003386</td>\n",
       "      <td>0.023183</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.668919</td>\n",
       "      <td>0.001662</td>\n",
       "      <td>0.004442</td>\n",
       "      <td>0.001662</td>\n",
       "      <td>0.004442</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.662162</td>\n",
       "      <td>0.002047</td>\n",
       "      <td>0.012023</td>\n",
       "      <td>0.002047</td>\n",
       "      <td>0.012023</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_val  pred_time_val  fit_time  \\\n",
       "0  WeightedEnsemble_L2   0.810811       0.074949  0.979291   \n",
       "1     RandomForestEntr   0.797297       0.024798  0.294500   \n",
       "2       ExtraTreesGini   0.783784       0.024531  0.287530   \n",
       "3       ExtraTreesEntr   0.770270       0.024206  0.282546   \n",
       "4     RandomForestGini   0.770270       0.025544  0.297175   \n",
       "5              XGBoost   0.722973       0.003386  0.023183   \n",
       "6       KNeighborsDist   0.668919       0.001662  0.004442   \n",
       "7       KNeighborsUnif   0.662162       0.002047  0.012023   \n",
       "\n",
       "   pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                0.000401           0.105070            2       True   \n",
       "1                0.024798           0.294500            1       True   \n",
       "2                0.024531           0.287530            1       True   \n",
       "3                0.024206           0.282546            1       True   \n",
       "4                0.025544           0.297175            1       True   \n",
       "5                0.003386           0.023183            1       True   \n",
       "6                0.001662           0.004442            1       True   \n",
       "7                0.002047           0.012023            1       True   \n",
       "\n",
       "   fit_order  \n",
       "0          8  \n",
       "1          4  \n",
       "2          5  \n",
       "3          6  \n",
       "4          3  \n",
       "5          7  \n",
       "6          2  \n",
       "7          1  "
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "comb_TabPre.leaderboard( silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "86472178",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "comb_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c4c7e29",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a1054c9b",
   "metadata": {},
   "source": [
    "## CORRELATED"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf22ee9c",
   "metadata": {},
   "source": [
    "### ADDITIONAL DATA PREPROCESSING\n",
    "(starting after combined dataset created and before stratified sampling above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "c05c5a00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 920 entries, 0 to 919\n",
      "Data columns (total 10 columns):\n",
      " #   Column    Non-Null Count  Dtype  \n",
      "---  ------    --------------  -----  \n",
      " 0   age       920 non-null    float64\n",
      " 1   sex       920 non-null    float64\n",
      " 2   cp        920 non-null    float64\n",
      " 3   trestbps  920 non-null    float64\n",
      " 4   chol      920 non-null    float64\n",
      " 5   restecg   920 non-null    object \n",
      " 6   thalach   920 non-null    float64\n",
      " 7   exang     920 non-null    object \n",
      " 8   oldpeak   920 non-null    float64\n",
      " 9   num       920 non-null    float64\n",
      "dtypes: float64(8), object(2)\n",
      "memory usage: 72.0+ KB\n"
     ]
    }
   ],
   "source": [
    "combined.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "c5e7f4ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "thalach    -0.385972\n",
       "chol       -0.234679\n",
       "trestbps    0.103828\n",
       "age         0.282700\n",
       "sex         0.307284\n",
       "oldpeak     0.373382\n",
       "cp          0.471712\n",
       "num         1.000000\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check correlation\n",
    "combined_correlation = combined.corr()\n",
    "combined_correlation['num'].sort_values()\n",
    "# only trestbps has abs(corr) < 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "59a028e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 920 entries, 0 to 919\n",
      "Data columns (total 9 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   age      920 non-null    float64\n",
      " 1   sex      920 non-null    float64\n",
      " 2   cp       920 non-null    float64\n",
      " 3   chol     920 non-null    float64\n",
      " 4   restecg  920 non-null    object \n",
      " 5   thalach  920 non-null    float64\n",
      " 6   exang    920 non-null    object \n",
      " 7   oldpeak  920 non-null    float64\n",
      " 8   num      920 non-null    float64\n",
      "dtypes: float64(7), object(2)\n",
      "memory usage: 64.8+ KB\n"
     ]
    }
   ],
   "source": [
    "# eliminate columns with correlation less than abs(0.2)\n",
    "drop_columns = ['trestbps']\n",
    "combined_downsized = combined.drop(columns = drop_columns)\n",
    "combined_downsized.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "0ac0ee1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_downsized_strat = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in combined_downsized_strat.split(combined_downsized, combined[\"num\"]):\n",
    "  combined_downsized_strat_train = combined_downsized.loc[train_index]\n",
    "  combined_downsized_strat_test = combined_downsized.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "0596670c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    0.554348\n",
       "0.0    0.445652\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_downsized_strat_test[\"num\"].value_counts()/len(combined_downsized_strat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "4234bf35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0    0.552989\n",
       "0.0    0.447011\n",
       "Name: num, dtype: float64"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_downsized_strat_train[\"num\"].value_counts()/len(combined_downsized_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "9a9b7dbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>cp</th>\n",
       "      <th>chol</th>\n",
       "      <th>restecg</th>\n",
       "      <th>thalach</th>\n",
       "      <th>exang</th>\n",
       "      <th>oldpeak</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>47.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>120.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>53.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>197.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>152.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>153.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>831</th>\n",
       "      <td>61.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>283.0</td>\n",
       "      <td>0</td>\n",
       "      <td>122.795918</td>\n",
       "      <td>1</td>\n",
       "      <td>1.320833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  sex   cp   chol restecg     thalach exang   oldpeak\n",
       "617  47.0  1.0  3.0    0.0       0  120.000000     1  0.000000\n",
       "49   53.0  1.0  3.0  197.0     2.0  152.000000   0.0  1.200000\n",
       "52   44.0  1.0  4.0  290.0     2.0  153.000000   0.0  0.000000\n",
       "831  61.0  1.0  2.0  283.0       0  122.795918     1  1.320833\n",
       "151  42.0  0.0  4.0  265.0     2.0  122.000000   0.0  0.600000"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split stratified data\n",
    "combined_downsized_strat_train_X = combined_downsized_strat_train.drop([heart_label], axis=1)\n",
    "combined_downsized_strat_train_y = combined_downsized_strat_train[heart_label]\n",
    "combined_downsized_strat_test_X = combined_downsized_strat_test.drop([heart_label], axis=1)\n",
    "combined_downsized_strat_test_y = combined_downsized_strat_test[heart_label]\n",
    "combined_downsized_strat_train_X.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a499a28",
   "metadata": {},
   "source": [
    "### BINARY MODEL - correlated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "84879e32",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: path already exists! This predictor may overwrite an existing predictor! path=\".\"\n",
      "Beginning AutoGluon training ...\n",
      "AutoGluon will save models to \"./\"\n",
      "AutoGluon Version:  0.7.0\n",
      "Python Version:     3.9.13\n",
      "Operating System:   Darwin\n",
      "Platform Machine:   x86_64\n",
      "Platform Version:   Darwin Kernel Version 22.2.0: Fri Nov 11 02:04:44 PST 2022; root:xnu-8792.61.2~4/RELEASE_ARM64_T8103\n",
      "Train Data Rows:    736\n",
      "Train Data Columns: 8\n",
      "Label Column: num\n",
      "Preprocessing data ...\n",
      "AutoGluon infers your prediction problem is: 'binary' (because only two unique label-values observed).\n",
      "\t2 unique label values:  [1.0, 0.0]\n",
      "\tIf 'binary' is not the correct problem_type, please manually specify the problem_type parameter during predictor init (You may specify problem_type as one of: ['binary', 'multiclass', 'regression'])\n",
      "Selected class <--> label mapping:  class 1 = 1, class 0 = 0\n",
      "Using Feature Generators to preprocess the data ...\n",
      "Fitting AutoMLPipelineFeatureGenerator...\n",
      "\tAvailable Memory:                    349.51 MB\n",
      "\tTrain Data (Original)  Memory Usage: 0.09 MB (0.0% of available memory)\n",
      "\tInferring data type of each feature based on column values. Set feature_metadata_in to manually specify special dtypes of the features.\n",
      "\tStage 1 Generators:\n",
      "\t\tFitting AsTypeFeatureGenerator...\n",
      "\t\t\tNote: Converting 1 features to boolean dtype as they only contain 2 unique values.\n",
      "\tStage 2 Generators:\n",
      "\t\tFitting FillNaFeatureGenerator...\n",
      "\tStage 3 Generators:\n",
      "\t\tFitting IdentityFeatureGenerator...\n",
      "\t\tFitting CategoryFeatureGenerator...\n",
      "\t\t\tFitting CategoryMemoryMinimizeFeatureGenerator...\n",
      "\tStage 4 Generators:\n",
      "\t\tFitting DropUniqueFeatureGenerator...\n",
      "\tTypes of features in original data (raw dtype, special dtypes):\n",
      "\t\t('float', [])  : 6 | ['age', 'sex', 'cp', 'chol', 'thalach', ...]\n",
      "\t\t('object', []) : 2 | ['restecg', 'exang']\n",
      "\tTypes of features in processed data (raw dtype, special dtypes):\n",
      "\t\t('category', [])  : 2 | ['restecg', 'exang']\n",
      "\t\t('float', [])     : 5 | ['age', 'cp', 'chol', 'thalach', 'oldpeak']\n",
      "\t\t('int', ['bool']) : 1 | ['sex']\n",
      "\t0.0s = Fit runtime\n",
      "\t8 features in original data used to generate 8 features in processed data.\n",
      "\tTrain Data (Processed) Memory Usage: 0.03 MB (0.0% of available memory)\n",
      "Data preprocessing and feature engineering runtime = 0.04s ...\n",
      "AutoGluon will gauge predictive performance using evaluation metric: 'accuracy'\n",
      "\tTo change this, specify the eval_metric parameter of Predictor()\n",
      "Automatically generating train/validation split with holdout_frac=0.2, Train Rows: 588, Val Rows: 148\n",
      "Fitting 13 L1 models ...\n",
      "Fitting model: KNeighborsUnif ...\n",
      "\t0.6622\t = Validation score   (accuracy)\n",
      "\t0.01s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: KNeighborsDist ...\n",
      "\t0.6554\t = Validation score   (accuracy)\n",
      "\t0.0s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: LightGBMXT ...\n",
      "\tWarning: Exception caused LightGBMXT to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: LightGBM ...\n",
      "\tWarning: Exception caused LightGBM to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: RandomForestGini ...\n",
      "\t0.8041\t = Validation score   (accuracy)\n",
      "\t0.3s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: RandomForestEntr ...\n",
      "\t0.8108\t = Validation score   (accuracy)\n",
      "\t0.31s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: CatBoost ...\n",
      "\tWarning: Exception caused CatBoost to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import catboost` failed. A quick tip is to install via `pip install autogluon.tabular[catboost]==0.7.0`.\n",
      "Fitting model: ExtraTreesGini ...\n",
      "\t0.7973\t = Validation score   (accuracy)\n",
      "\t0.28s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: ExtraTreesEntr ...\n",
      "\t0.777\t = Validation score   (accuracy)\n",
      "\t0.29s\t = Training   runtime\n",
      "\t0.02s\t = Validation runtime\n",
      "Fitting model: NeuralNetFastAI ...\n",
      "\tWarning: Exception caused NeuralNetFastAI to fail during training (ImportError)... Skipping this model.\n",
      "\t\tImport fastai failed. A quick tip is to install via `pip install autogluon.tabular[fastai]==0.7.0`. \n",
      "Fitting model: XGBoost ...\n",
      "Warning: Large XGB model size may cause OOM error if training continues\n",
      "Available Memory: 311 MB\n",
      "Estimated XGB model size: 0 MB\n",
      "\t0.723\t = Validation score   (accuracy)\n",
      "\t0.02s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "Fitting model: NeuralNetTorch ...\n",
      "\tWarning: Exception caused NeuralNetTorch to fail during training (ImportError)... Skipping this model.\n",
      "\t\tUnable to import dependency torch\n",
      "A quick tip is to install via `pip install torch`.\n",
      "The minimum torch version is currently 1.6.\n",
      "Fitting model: LightGBMLarge ...\n",
      "\tWarning: Exception caused LightGBMLarge to fail during training (ImportError)... Skipping this model.\n",
      "\t\t`import lightgbm` failed. A quick tip is to install via `pip install autogluon.tabular[lightgbm]==0.7.0`.\n",
      "Fitting model: WeightedEnsemble_L2 ...\n",
      "\t0.8108\t = Validation score   (accuracy)\n",
      "\t0.1s\t = Training   runtime\n",
      "\t0.0s\t = Validation runtime\n",
      "AutoGluon training complete, total runtime = 1.6s ... Best model: \"WeightedEnsemble_L2\"\n",
      "TabularPredictor saved. To load, use: predictor = TabularPredictor.load(\"./\")\n"
     ]
    }
   ],
   "source": [
    "# Create an instance and then fit the train data\n",
    "comb_downsized_TabPre = TabularPredictor(label=heart_label, path='.').fit(combined_downsized_strat_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "9ba6d327",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.842391304347826\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.842391304347826,\n",
      "    \"balanced_accuracy\": 0.8351267336202775,\n",
      "    \"mcc\": 0.6809624459644162,\n",
      "    \"f1\": 0.863849765258216,\n",
      "    \"precision\": 0.8288288288288288,\n",
      "    \"recall\": 0.9019607843137255\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_downsized_pred = comb_downsized_TabPre.predict(combined_downsized_strat_test_X)\n",
    "comb_downsized_perf = comb_downsized_TabPre.evaluate_predictions(\n",
    "    y_true=combined_downsized_strat_test_y, y_pred=comb_downsized_pred, auxiliary_metrics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "ae249351",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.842391304347826\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.842391304347826,\n",
      "    \"balanced_accuracy\": 0.8351267336202775,\n",
      "    \"mcc\": 0.6809624459644162,\n",
      "    \"roc_auc\": 0.9149330463892874,\n",
      "    \"f1\": 0.863849765258216,\n",
      "    \"precision\": 0.8288288288288288,\n",
      "    \"recall\": 0.9019607843137255\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_downsized_perf2 =  comb_downsized_TabPre.evaluate(data=combined_downsized_strat_test, auxiliary_metrics=True, model='WeightedEnsemble_L2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "2c80641f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.842391304347826\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.842391304347826,\n",
      "    \"balanced_accuracy\": 0.8351267336202775,\n",
      "    \"mcc\": 0.6809624459644162,\n",
      "    \"roc_auc\": 0.9149330463892874,\n",
      "    \"f1\": 0.863849765258216,\n",
      "    \"precision\": 0.8288288288288288,\n",
      "    \"recall\": 0.9019607843137255\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_downsized_perf3 =  comb_downsized_TabPre.evaluate(data=combined_downsized_strat_test, auxiliary_metrics=True, model='RandomForestEntr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "0374fc0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Evaluation: accuracy on test data: 0.8369565217391305\n",
      "Evaluations on test data:\n",
      "{\n",
      "    \"accuracy\": 0.8369565217391305,\n",
      "    \"balanced_accuracy\": 0.8302247728359637,\n",
      "    \"mcc\": 0.669475140994274,\n",
      "    \"roc_auc\": 0.9206719273075084,\n",
      "    \"f1\": 0.8584905660377359,\n",
      "    \"precision\": 0.8272727272727273,\n",
      "    \"recall\": 0.8921568627450981\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "comb_downsized_perf4 =  comb_downsized_TabPre.evaluate(data=combined_downsized_strat_test, auxiliary_metrics=True, model='RandomForestGini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "ae3e801f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_test</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_test</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_test_marginal</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.853261</td>\n",
       "      <td>0.777027</td>\n",
       "      <td>0.054566</td>\n",
       "      <td>0.024395</td>\n",
       "      <td>0.287584</td>\n",
       "      <td>0.054566</td>\n",
       "      <td>0.024395</td>\n",
       "      <td>0.287584</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.842391</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.035884</td>\n",
       "      <td>0.024316</td>\n",
       "      <td>0.309347</td>\n",
       "      <td>0.035884</td>\n",
       "      <td>0.024316</td>\n",
       "      <td>0.309347</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.842391</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.037257</td>\n",
       "      <td>0.024707</td>\n",
       "      <td>0.413412</td>\n",
       "      <td>0.001373</td>\n",
       "      <td>0.000391</td>\n",
       "      <td>0.104065</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.836957</td>\n",
       "      <td>0.804054</td>\n",
       "      <td>0.055402</td>\n",
       "      <td>0.024281</td>\n",
       "      <td>0.297836</td>\n",
       "      <td>0.055402</td>\n",
       "      <td>0.024281</td>\n",
       "      <td>0.297836</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.836957</td>\n",
       "      <td>0.797297</td>\n",
       "      <td>0.070570</td>\n",
       "      <td>0.024703</td>\n",
       "      <td>0.284881</td>\n",
       "      <td>0.070570</td>\n",
       "      <td>0.024703</td>\n",
       "      <td>0.284881</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.809783</td>\n",
       "      <td>0.722973</td>\n",
       "      <td>0.007525</td>\n",
       "      <td>0.003518</td>\n",
       "      <td>0.022230</td>\n",
       "      <td>0.007525</td>\n",
       "      <td>0.003518</td>\n",
       "      <td>0.022230</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.668478</td>\n",
       "      <td>0.662162</td>\n",
       "      <td>0.014795</td>\n",
       "      <td>0.002446</td>\n",
       "      <td>0.009459</td>\n",
       "      <td>0.014795</td>\n",
       "      <td>0.002446</td>\n",
       "      <td>0.009459</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.657609</td>\n",
       "      <td>0.655405</td>\n",
       "      <td>0.004528</td>\n",
       "      <td>0.001539</td>\n",
       "      <td>0.004670</td>\n",
       "      <td>0.004528</td>\n",
       "      <td>0.001539</td>\n",
       "      <td>0.004670</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_test  score_val  pred_time_test  pred_time_val  \\\n",
       "0       ExtraTreesEntr    0.853261   0.777027        0.054566       0.024395   \n",
       "1     RandomForestEntr    0.842391   0.810811        0.035884       0.024316   \n",
       "2  WeightedEnsemble_L2    0.842391   0.810811        0.037257       0.024707   \n",
       "3     RandomForestGini    0.836957   0.804054        0.055402       0.024281   \n",
       "4       ExtraTreesGini    0.836957   0.797297        0.070570       0.024703   \n",
       "5              XGBoost    0.809783   0.722973        0.007525       0.003518   \n",
       "6       KNeighborsUnif    0.668478   0.662162        0.014795       0.002446   \n",
       "7       KNeighborsDist    0.657609   0.655405        0.004528       0.001539   \n",
       "\n",
       "   fit_time  pred_time_test_marginal  pred_time_val_marginal  \\\n",
       "0  0.287584                 0.054566                0.024395   \n",
       "1  0.309347                 0.035884                0.024316   \n",
       "2  0.413412                 0.001373                0.000391   \n",
       "3  0.297836                 0.055402                0.024281   \n",
       "4  0.284881                 0.070570                0.024703   \n",
       "5  0.022230                 0.007525                0.003518   \n",
       "6  0.009459                 0.014795                0.002446   \n",
       "7  0.004670                 0.004528                0.001539   \n",
       "\n",
       "   fit_time_marginal  stack_level  can_infer  fit_order  \n",
       "0           0.287584            1       True          6  \n",
       "1           0.309347            1       True          4  \n",
       "2           0.104065            2       True          8  \n",
       "3           0.297836            1       True          3  \n",
       "4           0.284881            1       True          5  \n",
       "5           0.022230            1       True          7  \n",
       "6           0.009459            1       True          1  \n",
       "7           0.004670            1       True          2  "
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the leaderboard\n",
    "comb_downsized_TabPre.leaderboard(combined_downsized_strat_test, silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "6b7e9c1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>score_val</th>\n",
       "      <th>pred_time_val</th>\n",
       "      <th>fit_time</th>\n",
       "      <th>pred_time_val_marginal</th>\n",
       "      <th>fit_time_marginal</th>\n",
       "      <th>stack_level</th>\n",
       "      <th>can_infer</th>\n",
       "      <th>fit_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomForestEntr</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.024316</td>\n",
       "      <td>0.309347</td>\n",
       "      <td>0.024316</td>\n",
       "      <td>0.309347</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WeightedEnsemble_L2</td>\n",
       "      <td>0.810811</td>\n",
       "      <td>0.024707</td>\n",
       "      <td>0.413412</td>\n",
       "      <td>0.000391</td>\n",
       "      <td>0.104065</td>\n",
       "      <td>2</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForestGini</td>\n",
       "      <td>0.804054</td>\n",
       "      <td>0.024281</td>\n",
       "      <td>0.297836</td>\n",
       "      <td>0.024281</td>\n",
       "      <td>0.297836</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ExtraTreesGini</td>\n",
       "      <td>0.797297</td>\n",
       "      <td>0.024703</td>\n",
       "      <td>0.284881</td>\n",
       "      <td>0.024703</td>\n",
       "      <td>0.284881</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ExtraTreesEntr</td>\n",
       "      <td>0.777027</td>\n",
       "      <td>0.024395</td>\n",
       "      <td>0.287584</td>\n",
       "      <td>0.024395</td>\n",
       "      <td>0.287584</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.722973</td>\n",
       "      <td>0.003518</td>\n",
       "      <td>0.022230</td>\n",
       "      <td>0.003518</td>\n",
       "      <td>0.022230</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNeighborsUnif</td>\n",
       "      <td>0.662162</td>\n",
       "      <td>0.002446</td>\n",
       "      <td>0.009459</td>\n",
       "      <td>0.002446</td>\n",
       "      <td>0.009459</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>KNeighborsDist</td>\n",
       "      <td>0.655405</td>\n",
       "      <td>0.001539</td>\n",
       "      <td>0.004670</td>\n",
       "      <td>0.001539</td>\n",
       "      <td>0.004670</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 model  score_val  pred_time_val  fit_time  \\\n",
       "0     RandomForestEntr   0.810811       0.024316  0.309347   \n",
       "1  WeightedEnsemble_L2   0.810811       0.024707  0.413412   \n",
       "2     RandomForestGini   0.804054       0.024281  0.297836   \n",
       "3       ExtraTreesGini   0.797297       0.024703  0.284881   \n",
       "4       ExtraTreesEntr   0.777027       0.024395  0.287584   \n",
       "5              XGBoost   0.722973       0.003518  0.022230   \n",
       "6       KNeighborsUnif   0.662162       0.002446  0.009459   \n",
       "7       KNeighborsDist   0.655405       0.001539  0.004670   \n",
       "\n",
       "   pred_time_val_marginal  fit_time_marginal  stack_level  can_infer  \\\n",
       "0                0.024316           0.309347            1       True   \n",
       "1                0.000391           0.104065            2       True   \n",
       "2                0.024281           0.297836            1       True   \n",
       "3                0.024703           0.284881            1       True   \n",
       "4                0.024395           0.287584            1       True   \n",
       "5                0.003518           0.022230            1       True   \n",
       "6                0.002446           0.009459            1       True   \n",
       "7                0.001539           0.004670            1       True   \n",
       "\n",
       "   fit_order  \n",
       "0          4  \n",
       "1          8  \n",
       "2          3  \n",
       "3          5  \n",
       "4          6  \n",
       "5          7  \n",
       "6          1  \n",
       "7          2  "
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_downsized_TabPre.leaderboard(silent=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "e32f8ea9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'WeightedEnsemble_L2'"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Show the best model\n",
    "comb_downsized_TabPre.get_model_best()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0585f2a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac9bd33e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce1f1987",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35bba57b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9f28498",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "134124b2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66c79d5f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed7b9212",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98f790af",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67ca361f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c270665e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "303f1716",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}