H2OAutoML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Install Python Dependencies, JDK, and H2O for Python\n",
    "# !pip install requests tabulate \"colorama>=0.3.8\" future mlflow\n",
    "# !apt-get update && apt install default-jdk\n",
    "# !pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# H2O AutoML w/MLFlow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "References:\n",
    "\n",
    "* http://docs.h2o.ai/h2o/latest-stable/h2o-docs/automl.html\n",
    "* https://github.com/h2oai/h2o-tutorials/tree/master/h2o-world-2017/automl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start H2O"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import H2O python modules first, then initialize a H2O instance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h2o\n",
    "from h2o.automl import H2OAutoML, get_leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321 ..... not found.\n",
      "Attempting to start a local H2O server...\n",
      "  Java Version: openjdk version \"11.0.6\" 2020-01-14; OpenJDK Runtime Environment (build 11.0.6+10-post-Ubuntu-1ubuntu118.04.1); OpenJDK 64-Bit Server VM (build 11.0.6+10-post-Ubuntu-1ubuntu118.04.1, mixed mode, sharing)\n",
      "  Starting server from /opt/conda/lib/python3.7/site-packages/h2o/backend/bin/h2o.jar\n",
      "  Ice root: /tmp/tmpxiq2hlxc\n",
      "  JVM stdout: /tmp/tmpxiq2hlxc/h2o_unknownUser_started_from_python.out\n",
      "  JVM stderr: /tmp/tmpxiq2hlxc/h2o_unknownUser_started_from_python.err\n",
      "  Server is running at http://127.0.0.1:54321\n",
      "Connecting to H2O server at http://127.0.0.1:54321 ... successful.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>02 secs</td></tr>\n",
       "<tr><td>H2O cluster timezone:</td>\n",
       "<td>Etc/UTC</td></tr>\n",
       "<tr><td>H2O data parsing timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.28.0.3</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>1 month and 12 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_unknownUser_5b5zax</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>498 Mb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>2</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>2</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>accepting new members, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://127.0.0.1:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>{'http': None, 'https': None}</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O API Extensions:</td>\n",
       "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.7.3 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ------------------------------------------------------------------\n",
       "H2O cluster uptime:         02 secs\n",
       "H2O cluster timezone:       Etc/UTC\n",
       "H2O data parsing timezone:  UTC\n",
       "H2O cluster version:        3.28.0.3\n",
       "H2O cluster version age:    1 month and 12 days\n",
       "H2O cluster name:           H2O_from_python_unknownUser_5b5zax\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    498 Mb\n",
       "H2O cluster total cores:    2\n",
       "H2O cluster allowed cores:  2\n",
       "H2O cluster status:         accepting new members, healthy\n",
       "H2O connection url:         http://127.0.0.1:54321\n",
       "H2O connection proxy:       {'http': None, 'https': None}\n",
       "H2O internal security:      False\n",
       "H2O API Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4\n",
       "Python version:             3.7.3 final\n",
       "--------------------------  ------------------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Start H2O instance\n",
    "h2o.init()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We take [Higgs Dataset (Higgs Boson classification dataset)](https://archive.ics.uci.edu/ml/datasets/HIGGS) as a demostration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# Import a sample binary outcome train/test set into H2O\n",
    "train = h2o.import_file(\"https://s3.amazonaws.com/erin-data/higgs/higgs_train_10k.csv\")\n",
    "test = h2o.import_file(\"https://s3.amazonaws.com/erin-data/higgs/higgs_test_5k.csv\")\n",
    "\n",
    "# Identify predictors and response\n",
    "x = train.columns\n",
    "y = \"response\"\n",
    "x.remove(y)\n",
    "\n",
    "# For binary classification, response should be a factor\n",
    "train[y] = train[y].asfactor()\n",
    "test[y] = test[y].asfactor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use H2O AutoML module to train models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoML progress: |████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# Run AutoML for 10 base models (limited to 1 hour max runtime by default)\n",
    "aml = H2OAutoML(max_models=10, seed=1337)\n",
    "aml.train(x=x, y=y, training_frame=train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observe models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get model metrics by getting the Leaderboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the top experiments by reading the Leaderboard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>model_id                                           </th><th style=\"text-align: right;\">     auc</th><th style=\"text-align: right;\">  logloss</th><th style=\"text-align: right;\">   aucpr</th><th style=\"text-align: right;\">  mean_per_class_error</th><th style=\"text-align: right;\">    rmse</th><th style=\"text-align: right;\">     mse</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>StackedEnsemble_AllModels_AutoML_20200318_085617   </td><td style=\"text-align: right;\">0.788494</td><td style=\"text-align: right;\"> 0.553656</td><td style=\"text-align: right;\">0.806794</td><td style=\"text-align: right;\">              0.319353</td><td style=\"text-align: right;\">0.433116</td><td style=\"text-align: right;\">0.18759 </td></tr>\n",
       "<tr><td>StackedEnsemble_BestOfFamily_AutoML_20200318_085617</td><td style=\"text-align: right;\">0.786347</td><td style=\"text-align: right;\"> 0.555761</td><td style=\"text-align: right;\">0.804659</td><td style=\"text-align: right;\">              0.315266</td><td style=\"text-align: right;\">0.434086</td><td style=\"text-align: right;\">0.188431</td></tr>\n",
       "<tr><td>XGBoost_1_AutoML_20200318_085617                   </td><td style=\"text-align: right;\">0.784169</td><td style=\"text-align: right;\"> 0.555429</td><td style=\"text-align: right;\">0.804717</td><td style=\"text-align: right;\">              0.318868</td><td style=\"text-align: right;\">0.434427</td><td style=\"text-align: right;\">0.188726</td></tr>\n",
       "<tr><td>XGBoost_3_AutoML_20200318_085617                   </td><td style=\"text-align: right;\">0.782974</td><td style=\"text-align: right;\"> 0.558924</td><td style=\"text-align: right;\">0.801905</td><td style=\"text-align: right;\">              0.311213</td><td style=\"text-align: right;\">0.435598</td><td style=\"text-align: right;\">0.189746</td></tr>\n",
       "<tr><td>XGBoost_2_AutoML_20200318_085617                   </td><td style=\"text-align: right;\">0.782565</td><td style=\"text-align: right;\"> 0.557239</td><td style=\"text-align: right;\">0.800632</td><td style=\"text-align: right;\">              0.335346</td><td style=\"text-align: right;\">0.435148</td><td style=\"text-align: right;\">0.189354</td></tr>\n",
       "<tr><td>GBM_5_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.780062</td><td style=\"text-align: right;\"> 0.559536</td><td style=\"text-align: right;\">0.799004</td><td style=\"text-align: right;\">              0.336324</td><td style=\"text-align: right;\">0.436322</td><td style=\"text-align: right;\">0.190377</td></tr>\n",
       "<tr><td>GBM_1_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.778358</td><td style=\"text-align: right;\"> 0.561238</td><td style=\"text-align: right;\">0.797036</td><td style=\"text-align: right;\">              0.338392</td><td style=\"text-align: right;\">0.437122</td><td style=\"text-align: right;\">0.191075</td></tr>\n",
       "<tr><td>GBM_2_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.778158</td><td style=\"text-align: right;\"> 0.562321</td><td style=\"text-align: right;\">0.796394</td><td style=\"text-align: right;\">              0.327346</td><td style=\"text-align: right;\">0.437378</td><td style=\"text-align: right;\">0.1913  </td></tr>\n",
       "<tr><td>GBM_3_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.777609</td><td style=\"text-align: right;\"> 0.563194</td><td style=\"text-align: right;\">0.797764</td><td style=\"text-align: right;\">              0.339052</td><td style=\"text-align: right;\">0.437837</td><td style=\"text-align: right;\">0.191701</td></tr>\n",
       "<tr><td>GBM_4_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.775928</td><td style=\"text-align: right;\"> 0.565782</td><td style=\"text-align: right;\">0.794377</td><td style=\"text-align: right;\">              0.347013</td><td style=\"text-align: right;\">0.439276</td><td style=\"text-align: right;\">0.192963</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get basic Leaderboard\n",
    "aml.leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>model_id                                           </th><th style=\"text-align: right;\">     auc</th><th style=\"text-align: right;\">  logloss</th><th style=\"text-align: right;\">   aucpr</th><th style=\"text-align: right;\">  mean_per_class_error</th><th style=\"text-align: right;\">    rmse</th><th style=\"text-align: right;\">     mse</th><th style=\"text-align: right;\">  training_time_ms</th><th style=\"text-align: right;\">  predict_time_per_row_ms</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>StackedEnsemble_AllModels_AutoML_20200318_085617   </td><td style=\"text-align: right;\">0.788494</td><td style=\"text-align: right;\"> 0.553656</td><td style=\"text-align: right;\">0.806794</td><td style=\"text-align: right;\">              0.319353</td><td style=\"text-align: right;\">0.433116</td><td style=\"text-align: right;\">0.18759 </td><td style=\"text-align: right;\">              3535</td><td style=\"text-align: right;\">                 0.231121</td></tr>\n",
       "<tr><td>StackedEnsemble_BestOfFamily_AutoML_20200318_085617</td><td style=\"text-align: right;\">0.786347</td><td style=\"text-align: right;\"> 0.555761</td><td style=\"text-align: right;\">0.804659</td><td style=\"text-align: right;\">              0.315266</td><td style=\"text-align: right;\">0.434086</td><td style=\"text-align: right;\">0.188431</td><td style=\"text-align: right;\">              1587</td><td style=\"text-align: right;\">                 0.08393 </td></tr>\n",
       "<tr><td>XGBoost_1_AutoML_20200318_085617                   </td><td style=\"text-align: right;\">0.784169</td><td style=\"text-align: right;\"> 0.555429</td><td style=\"text-align: right;\">0.804717</td><td style=\"text-align: right;\">              0.318868</td><td style=\"text-align: right;\">0.434427</td><td style=\"text-align: right;\">0.188726</td><td style=\"text-align: right;\">              7268</td><td style=\"text-align: right;\">                 0.010207</td></tr>\n",
       "<tr><td>XGBoost_3_AutoML_20200318_085617                   </td><td style=\"text-align: right;\">0.782974</td><td style=\"text-align: right;\"> 0.558924</td><td style=\"text-align: right;\">0.801905</td><td style=\"text-align: right;\">              0.311213</td><td style=\"text-align: right;\">0.435598</td><td style=\"text-align: right;\">0.189746</td><td style=\"text-align: right;\">              3623</td><td style=\"text-align: right;\">                 0.005727</td></tr>\n",
       "<tr><td>XGBoost_2_AutoML_20200318_085617                   </td><td style=\"text-align: right;\">0.782565</td><td style=\"text-align: right;\"> 0.557239</td><td style=\"text-align: right;\">0.800632</td><td style=\"text-align: right;\">              0.335346</td><td style=\"text-align: right;\">0.435148</td><td style=\"text-align: right;\">0.189354</td><td style=\"text-align: right;\">              7020</td><td style=\"text-align: right;\">                 0.008852</td></tr>\n",
       "<tr><td>GBM_5_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.780062</td><td style=\"text-align: right;\"> 0.559536</td><td style=\"text-align: right;\">0.799004</td><td style=\"text-align: right;\">              0.336324</td><td style=\"text-align: right;\">0.436322</td><td style=\"text-align: right;\">0.190377</td><td style=\"text-align: right;\">              2225</td><td style=\"text-align: right;\">                 0.037089</td></tr>\n",
       "<tr><td>GBM_1_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.778358</td><td style=\"text-align: right;\"> 0.561238</td><td style=\"text-align: right;\">0.797036</td><td style=\"text-align: right;\">              0.338392</td><td style=\"text-align: right;\">0.437122</td><td style=\"text-align: right;\">0.191075</td><td style=\"text-align: right;\">              2453</td><td style=\"text-align: right;\">                 0.035969</td></tr>\n",
       "<tr><td>GBM_2_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.778158</td><td style=\"text-align: right;\"> 0.562321</td><td style=\"text-align: right;\">0.796394</td><td style=\"text-align: right;\">              0.327346</td><td style=\"text-align: right;\">0.437378</td><td style=\"text-align: right;\">0.1913  </td><td style=\"text-align: right;\">              1686</td><td style=\"text-align: right;\">                 0.033426</td></tr>\n",
       "<tr><td>GBM_3_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.777609</td><td style=\"text-align: right;\"> 0.563194</td><td style=\"text-align: right;\">0.797764</td><td style=\"text-align: right;\">              0.339052</td><td style=\"text-align: right;\">0.437837</td><td style=\"text-align: right;\">0.191701</td><td style=\"text-align: right;\">              1736</td><td style=\"text-align: right;\">                 0.033174</td></tr>\n",
       "<tr><td>GBM_4_AutoML_20200318_085617                       </td><td style=\"text-align: right;\">0.775928</td><td style=\"text-align: right;\"> 0.565782</td><td style=\"text-align: right;\">0.794377</td><td style=\"text-align: right;\">              0.347013</td><td style=\"text-align: right;\">0.439276</td><td style=\"text-align: right;\">0.192963</td><td style=\"text-align: right;\">              2141</td><td style=\"text-align: right;\">                 0.040724</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get full Leaderboard by passing the \"extra_columns\" parameter\n",
    "lb = get_leaderboard(aml, extra_columns='ALL')\n",
    "lb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get model's full report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A full report contains results w/ & w/o Cross-validation and their metrics, Confusion Matrix, and Gains/Lift Table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OStackedEnsembleEstimator :  Stacked Ensemble\n",
      "Model Key:  StackedEnsemble_AllModels_AutoML_20200318_085617\n",
      "\n",
      "No model summary for this model\n",
      "\n",
      "ModelMetricsBinomialGLM: stackedensemble\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.1003049515043338\n",
      "RMSE: 0.31670956964438857\n",
      "LogLoss: 0.3524437087238197\n",
      "Null degrees of freedom: 9999\n",
      "Residual degrees of freedom: 9990\n",
      "Null deviance: 13828.113387424315\n",
      "Residual deviance: 7048.874174476394\n",
      "AIC: 7068.874174476394\n",
      "AUC: 0.9617137455482535\n",
      "AUCPR: 0.9630971969896885\n",
      "Gini: 0.923427491096507\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.46353024409337423: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3952.0</td>\n",
       "      <td>753.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>(753.0/4705.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>329.0</td>\n",
       "      <td>4966.0</td>\n",
       "      <td>0.0621</td>\n",
       "      <td>(329.0/5295.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Total</td>\n",
       "      <td>4281.0</td>\n",
       "      <td>5719.0</td>\n",
       "      <td>0.1082</td>\n",
       "      <td>(1082.0/10000.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0       1   Error               Rate\n",
       "0      0  3952.0   753.0    0.16     (753.0/4705.0)\n",
       "1      1   329.0  4966.0  0.0621     (329.0/5295.0)\n",
       "2  Total  4281.0  5719.0  0.1082   (1082.0/10000.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>max f1</td>\n",
       "      <td>0.463530</td>\n",
       "      <td>0.901761</td>\n",
       "      <td>219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>max f2</td>\n",
       "      <td>0.380601</td>\n",
       "      <td>0.939033</td>\n",
       "      <td>249.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.613619</td>\n",
       "      <td>0.910210</td>\n",
       "      <td>158.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.488694</td>\n",
       "      <td>0.892100</td>\n",
       "      <td>209.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>max precision</td>\n",
       "      <td>0.932010</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>max recall</td>\n",
       "      <td>0.179619</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>342.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.932010</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.463530</td>\n",
       "      <td>0.784623</td>\n",
       "      <td>219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.529581</td>\n",
       "      <td>0.888952</td>\n",
       "      <td>193.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.504941</td>\n",
       "      <td>0.890452</td>\n",
       "      <td>203.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>max tns</td>\n",
       "      <td>0.932010</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>max fns</td>\n",
       "      <td>0.932010</td>\n",
       "      <td>5286.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>max fps</td>\n",
       "      <td>0.060959</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>max tps</td>\n",
       "      <td>0.179619</td>\n",
       "      <td>5295.000000</td>\n",
       "      <td>342.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.932010</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.932010</td>\n",
       "      <td>0.998300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.060959</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.179619</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>342.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold        value    idx\n",
       "0                        max f1   0.463530     0.901761  219.0\n",
       "1                        max f2   0.380601     0.939033  249.0\n",
       "2                  max f0point5   0.613619     0.910210  158.0\n",
       "3                  max accuracy   0.488694     0.892100  209.0\n",
       "4                 max precision   0.932010     1.000000    0.0\n",
       "5                    max recall   0.179619     1.000000  342.0\n",
       "6               max specificity   0.932010     1.000000    0.0\n",
       "7              max absolute_mcc   0.463530     0.784623  219.0\n",
       "8    max min_per_class_accuracy   0.529581     0.888952  193.0\n",
       "9   max mean_per_class_accuracy   0.504941     0.890452  203.0\n",
       "10                      max tns   0.932010  4705.000000    0.0\n",
       "11                      max fns   0.932010  5286.000000    0.0\n",
       "12                      max fps   0.060959  4705.000000  399.0\n",
       "13                      max tps   0.179619  5295.000000  342.0\n",
       "14                      max tnr   0.932010     1.000000    0.0\n",
       "15                      max fnr   0.932010     0.998300    0.0\n",
       "16                      max fpr   0.060959     1.000000  399.0\n",
       "17                      max tpr   0.179619     1.000000  342.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate: 52.95 %, avg score: 52.97 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.912744</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.919510</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.919510</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>88.857413</td>\n",
       "      <td>88.857413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.905638</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.908824</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.914167</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>0.037771</td>\n",
       "      <td>88.857413</td>\n",
       "      <td>88.857413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.900851</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.903052</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.910462</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>0.056657</td>\n",
       "      <td>88.857413</td>\n",
       "      <td>88.857413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.896947</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.898792</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.907544</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>0.075543</td>\n",
       "      <td>88.857413</td>\n",
       "      <td>88.857413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.892187</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.894617</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.904959</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>0.094429</td>\n",
       "      <td>88.857413</td>\n",
       "      <td>88.857413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.870910</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.888574</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.881658</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.893309</td>\n",
       "      <td>0.094429</td>\n",
       "      <td>0.188857</td>\n",
       "      <td>88.857413</td>\n",
       "      <td>88.857413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.849066</td>\n",
       "      <td>1.884797</td>\n",
       "      <td>1.887315</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.860096</td>\n",
       "      <td>0.999333</td>\n",
       "      <td>0.882238</td>\n",
       "      <td>0.094240</td>\n",
       "      <td>0.283097</td>\n",
       "      <td>88.479698</td>\n",
       "      <td>88.731508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.823820</td>\n",
       "      <td>1.858357</td>\n",
       "      <td>1.880076</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.836712</td>\n",
       "      <td>0.995500</td>\n",
       "      <td>0.870856</td>\n",
       "      <td>0.092918</td>\n",
       "      <td>0.376015</td>\n",
       "      <td>85.835694</td>\n",
       "      <td>88.007554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.761017</td>\n",
       "      <td>1.801700</td>\n",
       "      <td>1.853950</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.793907</td>\n",
       "      <td>0.981667</td>\n",
       "      <td>0.845206</td>\n",
       "      <td>0.180170</td>\n",
       "      <td>0.556185</td>\n",
       "      <td>80.169972</td>\n",
       "      <td>85.395027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.676403</td>\n",
       "      <td>1.675165</td>\n",
       "      <td>1.809254</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.720008</td>\n",
       "      <td>0.958000</td>\n",
       "      <td>0.813907</td>\n",
       "      <td>0.167517</td>\n",
       "      <td>0.723702</td>\n",
       "      <td>67.516525</td>\n",
       "      <td>80.925401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.557127</td>\n",
       "      <td>1.405099</td>\n",
       "      <td>1.728423</td>\n",
       "      <td>0.744</td>\n",
       "      <td>0.619505</td>\n",
       "      <td>0.915200</td>\n",
       "      <td>0.775027</td>\n",
       "      <td>0.140510</td>\n",
       "      <td>0.864212</td>\n",
       "      <td>40.509915</td>\n",
       "      <td>72.842304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.425438</td>\n",
       "      <td>0.942398</td>\n",
       "      <td>1.597419</td>\n",
       "      <td>0.499</td>\n",
       "      <td>0.491709</td>\n",
       "      <td>0.845833</td>\n",
       "      <td>0.727807</td>\n",
       "      <td>0.094240</td>\n",
       "      <td>0.958451</td>\n",
       "      <td>-5.760151</td>\n",
       "      <td>59.741895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.308881</td>\n",
       "      <td>0.330500</td>\n",
       "      <td>1.416431</td>\n",
       "      <td>0.175</td>\n",
       "      <td>0.362237</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.675583</td>\n",
       "      <td>0.033050</td>\n",
       "      <td>0.991501</td>\n",
       "      <td>-66.949953</td>\n",
       "      <td>41.643059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.223638</td>\n",
       "      <td>0.064212</td>\n",
       "      <td>1.247403</td>\n",
       "      <td>0.034</td>\n",
       "      <td>0.262851</td>\n",
       "      <td>0.660500</td>\n",
       "      <td>0.623991</td>\n",
       "      <td>0.006421</td>\n",
       "      <td>0.997923</td>\n",
       "      <td>-93.578848</td>\n",
       "      <td>24.740321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.154792</td>\n",
       "      <td>0.020774</td>\n",
       "      <td>1.111111</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.189151</td>\n",
       "      <td>0.588333</td>\n",
       "      <td>0.575676</td>\n",
       "      <td>0.002077</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-97.922568</td>\n",
       "      <td>11.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.060959</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.115847</td>\n",
       "      <td>0.529500</td>\n",
       "      <td>0.529693</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-100.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                      0.01         0.912744  1.888574   \n",
       "1         2                      0.02         0.905638  1.888574   \n",
       "2         3                      0.03         0.900851  1.888574   \n",
       "3         4                      0.04         0.896947  1.888574   \n",
       "4         5                      0.05         0.892187  1.888574   \n",
       "5         6                      0.10         0.870910  1.888574   \n",
       "6         7                      0.15         0.849066  1.884797   \n",
       "7         8                      0.20         0.823820  1.858357   \n",
       "8         9                      0.30         0.761017  1.801700   \n",
       "9        10                      0.40         0.676403  1.675165   \n",
       "10       11                      0.50         0.557127  1.405099   \n",
       "11       12                      0.60         0.425438  0.942398   \n",
       "12       13                      0.70         0.308881  0.330500   \n",
       "13       14                      0.80         0.223638  0.064212   \n",
       "14       15                      0.90         0.154792  0.020774   \n",
       "15       16                      1.00         0.060959  0.000000   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          1.888574          1.000  0.919510                  1.000000   \n",
       "1          1.888574          1.000  0.908824                  1.000000   \n",
       "2          1.888574          1.000  0.903052                  1.000000   \n",
       "3          1.888574          1.000  0.898792                  1.000000   \n",
       "4          1.888574          1.000  0.894617                  1.000000   \n",
       "5          1.888574          1.000  0.881658                  1.000000   \n",
       "6          1.887315          0.998  0.860096                  0.999333   \n",
       "7          1.880076          0.984  0.836712                  0.995500   \n",
       "8          1.853950          0.954  0.793907                  0.981667   \n",
       "9          1.809254          0.887  0.720008                  0.958000   \n",
       "10         1.728423          0.744  0.619505                  0.915200   \n",
       "11         1.597419          0.499  0.491709                  0.845833   \n",
       "12         1.416431          0.175  0.362237                  0.750000   \n",
       "13         1.247403          0.034  0.262851                  0.660500   \n",
       "14         1.111111          0.011  0.189151                  0.588333   \n",
       "15         1.000000          0.000  0.115847                  0.529500   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.919510      0.018886                 0.018886   88.857413   \n",
       "1           0.914167      0.018886                 0.037771   88.857413   \n",
       "2           0.910462      0.018886                 0.056657   88.857413   \n",
       "3           0.907544      0.018886                 0.075543   88.857413   \n",
       "4           0.904959      0.018886                 0.094429   88.857413   \n",
       "5           0.893309      0.094429                 0.188857   88.857413   \n",
       "6           0.882238      0.094240                 0.283097   88.479698   \n",
       "7           0.870856      0.092918                 0.376015   85.835694   \n",
       "8           0.845206      0.180170                 0.556185   80.169972   \n",
       "9           0.813907      0.167517                 0.723702   67.516525   \n",
       "10          0.775027      0.140510                 0.864212   40.509915   \n",
       "11          0.727807      0.094240                 0.958451   -5.760151   \n",
       "12          0.675583      0.033050                 0.991501  -66.949953   \n",
       "13          0.623991      0.006421                 0.997923  -93.578848   \n",
       "14          0.575676      0.002077                 1.000000  -97.922568   \n",
       "15          0.529693      0.000000                 1.000000 -100.000000   \n",
       "\n",
       "    cumulative_gain  \n",
       "0         88.857413  \n",
       "1         88.857413  \n",
       "2         88.857413  \n",
       "3         88.857413  \n",
       "4         88.857413  \n",
       "5         88.857413  \n",
       "6         88.731508  \n",
       "7         88.007554  \n",
       "8         85.395027  \n",
       "9         80.925401  \n",
       "10        72.842304  \n",
       "11        59.741895  \n",
       "12        41.643059  \n",
       "13        24.740321  \n",
       "14        11.111111  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomialGLM: stackedensemble\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.1875896800249936\n",
      "RMSE: 0.4331162430860722\n",
      "LogLoss: 0.5536564863051258\n",
      "Null degrees of freedom: 9999\n",
      "Residual degrees of freedom: 9989\n",
      "Null deviance: 13829.717313823485\n",
      "Residual deviance: 11073.129726102514\n",
      "AIC: 11095.129726102514\n",
      "AUC: 0.7884943086885449\n",
      "AUCPR: 0.8067942794870006\n",
      "Gini: 0.5769886173770897\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.35213494797673345: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2329.0</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>0.505</td>\n",
       "      <td>(2376.0/4705.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>708.0</td>\n",
       "      <td>4587.0</td>\n",
       "      <td>0.1337</td>\n",
       "      <td>(708.0/5295.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Total</td>\n",
       "      <td>3037.0</td>\n",
       "      <td>6963.0</td>\n",
       "      <td>0.3084</td>\n",
       "      <td>(3084.0/10000.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0       1   Error               Rate\n",
       "0      0  2329.0  2376.0   0.505    (2376.0/4705.0)\n",
       "1      1   708.0  4587.0  0.1337     (708.0/5295.0)\n",
       "2  Total  3037.0  6963.0  0.3084   (3084.0/10000.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>max f1</td>\n",
       "      <td>0.352135</td>\n",
       "      <td>0.748409</td>\n",
       "      <td>271.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>max f2</td>\n",
       "      <td>0.182968</td>\n",
       "      <td>0.861183</td>\n",
       "      <td>350.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.632797</td>\n",
       "      <td>0.739389</td>\n",
       "      <td>145.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.551310</td>\n",
       "      <td>0.712500</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>max precision</td>\n",
       "      <td>0.931652</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>max recall</td>\n",
       "      <td>0.083776</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>393.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.931652</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.551310</td>\n",
       "      <td>0.427063</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.528765</td>\n",
       "      <td>0.709458</td>\n",
       "      <td>193.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.551310</td>\n",
       "      <td>0.713858</td>\n",
       "      <td>183.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>max tns</td>\n",
       "      <td>0.931652</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>max fns</td>\n",
       "      <td>0.931652</td>\n",
       "      <td>5293.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>max fps</td>\n",
       "      <td>0.067840</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>max tps</td>\n",
       "      <td>0.083776</td>\n",
       "      <td>5295.000000</td>\n",
       "      <td>393.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.931652</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.931652</td>\n",
       "      <td>0.999622</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.067840</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.083776</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>393.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold        value    idx\n",
       "0                        max f1   0.352135     0.748409  271.0\n",
       "1                        max f2   0.182968     0.861183  350.0\n",
       "2                  max f0point5   0.632797     0.739389  145.0\n",
       "3                  max accuracy   0.551310     0.712500  183.0\n",
       "4                 max precision   0.931652     1.000000    0.0\n",
       "5                    max recall   0.083776     1.000000  393.0\n",
       "6               max specificity   0.931652     1.000000    0.0\n",
       "7              max absolute_mcc   0.551310     0.427063  183.0\n",
       "8    max min_per_class_accuracy   0.528765     0.709458  193.0\n",
       "9   max mean_per_class_accuracy   0.551310     0.713858  183.0\n",
       "10                      max tns   0.931652  4705.000000    0.0\n",
       "11                      max fns   0.931652  5293.000000    0.0\n",
       "12                      max fps   0.067840  4705.000000  399.0\n",
       "13                      max tps   0.083776  5295.000000  393.0\n",
       "14                      max tnr   0.931652     1.000000    0.0\n",
       "15                      max fnr   0.931652     0.999622    0.0\n",
       "16                      max fpr   0.067840     1.000000  399.0\n",
       "17                      max tpr   0.083776     1.000000  393.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate: 52.95 %, avg score: 52.94 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.908602</td>\n",
       "      <td>1.850803</td>\n",
       "      <td>1.850803</td>\n",
       "      <td>0.980</td>\n",
       "      <td>0.916511</td>\n",
       "      <td>0.980000</td>\n",
       "      <td>0.916511</td>\n",
       "      <td>0.018508</td>\n",
       "      <td>0.018508</td>\n",
       "      <td>85.080264</td>\n",
       "      <td>85.080264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.900933</td>\n",
       "      <td>1.869688</td>\n",
       "      <td>1.860246</td>\n",
       "      <td>0.990</td>\n",
       "      <td>0.904651</td>\n",
       "      <td>0.985000</td>\n",
       "      <td>0.910581</td>\n",
       "      <td>0.018697</td>\n",
       "      <td>0.037205</td>\n",
       "      <td>86.968839</td>\n",
       "      <td>86.024551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.895174</td>\n",
       "      <td>1.794145</td>\n",
       "      <td>1.838212</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.898125</td>\n",
       "      <td>0.973333</td>\n",
       "      <td>0.906429</td>\n",
       "      <td>0.017941</td>\n",
       "      <td>0.055146</td>\n",
       "      <td>79.414542</td>\n",
       "      <td>83.821215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.890219</td>\n",
       "      <td>1.756374</td>\n",
       "      <td>1.817753</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.892544</td>\n",
       "      <td>0.962500</td>\n",
       "      <td>0.902958</td>\n",
       "      <td>0.017564</td>\n",
       "      <td>0.072710</td>\n",
       "      <td>75.637394</td>\n",
       "      <td>81.775260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.885024</td>\n",
       "      <td>1.794145</td>\n",
       "      <td>1.813031</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.887340</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.899834</td>\n",
       "      <td>0.017941</td>\n",
       "      <td>0.090652</td>\n",
       "      <td>79.414542</td>\n",
       "      <td>81.303116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.859835</td>\n",
       "      <td>1.677054</td>\n",
       "      <td>1.745042</td>\n",
       "      <td>0.888</td>\n",
       "      <td>0.872648</td>\n",
       "      <td>0.924000</td>\n",
       "      <td>0.886241</td>\n",
       "      <td>0.083853</td>\n",
       "      <td>0.174504</td>\n",
       "      <td>67.705382</td>\n",
       "      <td>74.504249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.832008</td>\n",
       "      <td>1.593957</td>\n",
       "      <td>1.694681</td>\n",
       "      <td>0.844</td>\n",
       "      <td>0.845384</td>\n",
       "      <td>0.897333</td>\n",
       "      <td>0.872622</td>\n",
       "      <td>0.079698</td>\n",
       "      <td>0.254202</td>\n",
       "      <td>59.395656</td>\n",
       "      <td>69.468052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.800869</td>\n",
       "      <td>1.593957</td>\n",
       "      <td>1.669500</td>\n",
       "      <td>0.844</td>\n",
       "      <td>0.816310</td>\n",
       "      <td>0.884000</td>\n",
       "      <td>0.858544</td>\n",
       "      <td>0.079698</td>\n",
       "      <td>0.333900</td>\n",
       "      <td>59.395656</td>\n",
       "      <td>66.949953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.725254</td>\n",
       "      <td>1.372993</td>\n",
       "      <td>1.570664</td>\n",
       "      <td>0.727</td>\n",
       "      <td>0.765085</td>\n",
       "      <td>0.831667</td>\n",
       "      <td>0.827391</td>\n",
       "      <td>0.137299</td>\n",
       "      <td>0.471199</td>\n",
       "      <td>37.299339</td>\n",
       "      <td>57.066415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.637655</td>\n",
       "      <td>1.225685</td>\n",
       "      <td>1.484419</td>\n",
       "      <td>0.649</td>\n",
       "      <td>0.681409</td>\n",
       "      <td>0.786000</td>\n",
       "      <td>0.790895</td>\n",
       "      <td>0.122568</td>\n",
       "      <td>0.593768</td>\n",
       "      <td>22.568461</td>\n",
       "      <td>48.441926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.540005</td>\n",
       "      <td>1.063267</td>\n",
       "      <td>1.400189</td>\n",
       "      <td>0.563</td>\n",
       "      <td>0.589719</td>\n",
       "      <td>0.741400</td>\n",
       "      <td>0.750660</td>\n",
       "      <td>0.106327</td>\n",
       "      <td>0.700094</td>\n",
       "      <td>6.326723</td>\n",
       "      <td>40.018886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.443067</td>\n",
       "      <td>0.885741</td>\n",
       "      <td>1.314448</td>\n",
       "      <td>0.469</td>\n",
       "      <td>0.491673</td>\n",
       "      <td>0.696000</td>\n",
       "      <td>0.707495</td>\n",
       "      <td>0.088574</td>\n",
       "      <td>0.788669</td>\n",
       "      <td>-11.425873</td>\n",
       "      <td>31.444759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.347714</td>\n",
       "      <td>0.800755</td>\n",
       "      <td>1.241063</td>\n",
       "      <td>0.424</td>\n",
       "      <td>0.394752</td>\n",
       "      <td>0.657143</td>\n",
       "      <td>0.662818</td>\n",
       "      <td>0.080076</td>\n",
       "      <td>0.868744</td>\n",
       "      <td>-19.924457</td>\n",
       "      <td>24.106300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.260794</td>\n",
       "      <td>0.638338</td>\n",
       "      <td>1.165722</td>\n",
       "      <td>0.338</td>\n",
       "      <td>0.304411</td>\n",
       "      <td>0.617250</td>\n",
       "      <td>0.618017</td>\n",
       "      <td>0.063834</td>\n",
       "      <td>0.932578</td>\n",
       "      <td>-36.166195</td>\n",
       "      <td>16.572238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.179929</td>\n",
       "      <td>0.491029</td>\n",
       "      <td>1.090756</td>\n",
       "      <td>0.260</td>\n",
       "      <td>0.221221</td>\n",
       "      <td>0.577556</td>\n",
       "      <td>0.573928</td>\n",
       "      <td>0.049103</td>\n",
       "      <td>0.981681</td>\n",
       "      <td>-50.897073</td>\n",
       "      <td>9.075648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.065722</td>\n",
       "      <td>0.183192</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.097</td>\n",
       "      <td>0.129024</td>\n",
       "      <td>0.529500</td>\n",
       "      <td>0.529438</td>\n",
       "      <td>0.018319</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-81.680831</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                      0.01         0.908602  1.850803   \n",
       "1         2                      0.02         0.900933  1.869688   \n",
       "2         3                      0.03         0.895174  1.794145   \n",
       "3         4                      0.04         0.890219  1.756374   \n",
       "4         5                      0.05         0.885024  1.794145   \n",
       "5         6                      0.10         0.859835  1.677054   \n",
       "6         7                      0.15         0.832008  1.593957   \n",
       "7         8                      0.20         0.800869  1.593957   \n",
       "8         9                      0.30         0.725254  1.372993   \n",
       "9        10                      0.40         0.637655  1.225685   \n",
       "10       11                      0.50         0.540005  1.063267   \n",
       "11       12                      0.60         0.443067  0.885741   \n",
       "12       13                      0.70         0.347714  0.800755   \n",
       "13       14                      0.80         0.260794  0.638338   \n",
       "14       15                      0.90         0.179929  0.491029   \n",
       "15       16                      1.00         0.065722  0.183192   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          1.850803          0.980  0.916511                  0.980000   \n",
       "1          1.860246          0.990  0.904651                  0.985000   \n",
       "2          1.838212          0.950  0.898125                  0.973333   \n",
       "3          1.817753          0.930  0.892544                  0.962500   \n",
       "4          1.813031          0.950  0.887340                  0.960000   \n",
       "5          1.745042          0.888  0.872648                  0.924000   \n",
       "6          1.694681          0.844  0.845384                  0.897333   \n",
       "7          1.669500          0.844  0.816310                  0.884000   \n",
       "8          1.570664          0.727  0.765085                  0.831667   \n",
       "9          1.484419          0.649  0.681409                  0.786000   \n",
       "10         1.400189          0.563  0.589719                  0.741400   \n",
       "11         1.314448          0.469  0.491673                  0.696000   \n",
       "12         1.241063          0.424  0.394752                  0.657143   \n",
       "13         1.165722          0.338  0.304411                  0.617250   \n",
       "14         1.090756          0.260  0.221221                  0.577556   \n",
       "15         1.000000          0.097  0.129024                  0.529500   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate       gain  \\\n",
       "0           0.916511      0.018508                 0.018508  85.080264   \n",
       "1           0.910581      0.018697                 0.037205  86.968839   \n",
       "2           0.906429      0.017941                 0.055146  79.414542   \n",
       "3           0.902958      0.017564                 0.072710  75.637394   \n",
       "4           0.899834      0.017941                 0.090652  79.414542   \n",
       "5           0.886241      0.083853                 0.174504  67.705382   \n",
       "6           0.872622      0.079698                 0.254202  59.395656   \n",
       "7           0.858544      0.079698                 0.333900  59.395656   \n",
       "8           0.827391      0.137299                 0.471199  37.299339   \n",
       "9           0.790895      0.122568                 0.593768  22.568461   \n",
       "10          0.750660      0.106327                 0.700094   6.326723   \n",
       "11          0.707495      0.088574                 0.788669 -11.425873   \n",
       "12          0.662818      0.080076                 0.868744 -19.924457   \n",
       "13          0.618017      0.063834                 0.932578 -36.166195   \n",
       "14          0.573928      0.049103                 0.981681 -50.897073   \n",
       "15          0.529438      0.018319                 1.000000 -81.680831   \n",
       "\n",
       "    cumulative_gain  \n",
       "0         85.080264  \n",
       "1         86.024551  \n",
       "2         83.821215  \n",
       "3         81.775260  \n",
       "4         81.303116  \n",
       "5         74.504249  \n",
       "6         69.468052  \n",
       "7         66.949953  \n",
       "8         57.066415  \n",
       "9         48.441926  \n",
       "10        40.018886  \n",
       "11        31.444759  \n",
       "12        24.106300  \n",
       "13        16.572238  \n",
       "14         9.075648  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Simply get the best model among the results by call the model directly\n",
    "aml.leader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['StackedEnsemble_AllModels_AutoML_20200318_085617',\n",
       " 'StackedEnsemble_BestOfFamily_AutoML_20200318_085617',\n",
       " 'XGBoost_1_AutoML_20200318_085617',\n",
       " 'XGBoost_3_AutoML_20200318_085617',\n",
       " 'XGBoost_2_AutoML_20200318_085617',\n",
       " 'GBM_5_AutoML_20200318_085617',\n",
       " 'GBM_1_AutoML_20200318_085617',\n",
       " 'GBM_2_AutoML_20200318_085617',\n",
       " 'GBM_3_AutoML_20200318_085617',\n",
       " 'GBM_4_AutoML_20200318_085617',\n",
       " 'DRF_1_AutoML_20200318_085617',\n",
       " 'GLM_1_AutoML_20200318_085617']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Or get a specific model by using \"h2o.get_model()\" function\n",
    "\n",
    "# Get Model ID first\n",
    "model_ids = lb.as_data_frame()['model_id'].tolist()\n",
    "model_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OGeneralizedLinearEstimator :  Generalized Linear Modeling\n",
      "Model Key:  GLM_1_AutoML_20200318_085617\n",
      "\n",
      "\n",
      "GLM Model: summary\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>family</th>\n",
       "      <th>link</th>\n",
       "      <th>regularization</th>\n",
       "      <th>lambda_search</th>\n",
       "      <th>number_of_predictors_total</th>\n",
       "      <th>number_of_active_predictors</th>\n",
       "      <th>number_of_iterations</th>\n",
       "      <th>training_frame</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>binomial</td>\n",
       "      <td>logit</td>\n",
       "      <td>Ridge ( lambda = 3.465E-4 )</td>\n",
       "      <td>nlambda = 30, lambda.max = 7.6643, lambda.min = 3.465E-4, lambda.1...</td>\n",
       "      <td>28</td>\n",
       "      <td>28</td>\n",
       "      <td>43</td>\n",
       "      <td>automl_training_py_1_sid_af52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       family   link               regularization  \\\n",
       "0    binomial  logit  Ridge ( lambda = 3.465E-4 )   \n",
       "\n",
       "                                                           lambda_search  \\\n",
       "0  nlambda = 30, lambda.max = 7.6643, lambda.min = 3.465E-4, lambda.1...   \n",
       "\n",
       "   number_of_predictors_total number_of_active_predictors  \\\n",
       "0                          28                          28   \n",
       "\n",
       "   number_of_iterations                 training_frame  \n",
       "0                    43  automl_training_py_1_sid_af52  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.22196178360326163\n",
      "RMSE: 0.47112820293765223\n",
      "LogLoss: 0.6352093336032741\n",
      "Null degrees of freedom: 9999\n",
      "Residual degrees of freedom: 9971\n",
      "Null deviance: 13828.113387424315\n",
      "Residual deviance: 12704.186672065483\n",
      "AIC: 12762.186672065483\n",
      "AUC: 0.688363413040795\n",
      "AUCPR: 0.6860796446366819\n",
      "Gini: 0.3767268260815899\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.40010771077268487: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1439.0</td>\n",
       "      <td>3266.0</td>\n",
       "      <td>0.6942</td>\n",
       "      <td>(3266.0/4705.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>465.0</td>\n",
       "      <td>4830.0</td>\n",
       "      <td>0.0878</td>\n",
       "      <td>(465.0/5295.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Total</td>\n",
       "      <td>1904.0</td>\n",
       "      <td>8096.0</td>\n",
       "      <td>0.3731</td>\n",
       "      <td>(3731.0/10000.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0       1   Error               Rate\n",
       "0      0  1439.0  3266.0  0.6942    (3266.0/4705.0)\n",
       "1      1   465.0  4830.0  0.0878     (465.0/5295.0)\n",
       "2  Total  1904.0  8096.0  0.3731   (3731.0/10000.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>max f1</td>\n",
       "      <td>0.400108</td>\n",
       "      <td>0.721380</td>\n",
       "      <td>273.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>max f2</td>\n",
       "      <td>0.241779</td>\n",
       "      <td>0.850883</td>\n",
       "      <td>341.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.532460</td>\n",
       "      <td>0.662667</td>\n",
       "      <td>189.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.496042</td>\n",
       "      <td>0.646700</td>\n",
       "      <td>213.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>max precision</td>\n",
       "      <td>0.983163</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>max recall</td>\n",
       "      <td>0.001815</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.983163</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.483011</td>\n",
       "      <td>0.289385</td>\n",
       "      <td>222.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.537966</td>\n",
       "      <td>0.637960</td>\n",
       "      <td>185.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.532460</td>\n",
       "      <td>0.641506</td>\n",
       "      <td>189.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>max tns</td>\n",
       "      <td>0.983163</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>max fns</td>\n",
       "      <td>0.983163</td>\n",
       "      <td>5293.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>max fps</td>\n",
       "      <td>0.001815</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>max tps</td>\n",
       "      <td>0.001815</td>\n",
       "      <td>5295.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.983163</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.983163</td>\n",
       "      <td>0.999622</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.001815</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.001815</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold        value    idx\n",
       "0                        max f1   0.400108     0.721380  273.0\n",
       "1                        max f2   0.241779     0.850883  341.0\n",
       "2                  max f0point5   0.532460     0.662667  189.0\n",
       "3                  max accuracy   0.496042     0.646700  213.0\n",
       "4                 max precision   0.983163     1.000000    0.0\n",
       "5                    max recall   0.001815     1.000000  399.0\n",
       "6               max specificity   0.983163     1.000000    0.0\n",
       "7              max absolute_mcc   0.483011     0.289385  222.0\n",
       "8    max min_per_class_accuracy   0.537966     0.637960  185.0\n",
       "9   max mean_per_class_accuracy   0.532460     0.641506  189.0\n",
       "10                      max tns   0.983163  4705.000000    0.0\n",
       "11                      max fns   0.983163  5293.000000    0.0\n",
       "12                      max fps   0.001815  4705.000000  399.0\n",
       "13                      max tps   0.001815  5295.000000  399.0\n",
       "14                      max tnr   0.983163     1.000000    0.0\n",
       "15                      max fnr   0.983163     0.999622    0.0\n",
       "16                      max fpr   0.001815     1.000000  399.0\n",
       "17                      max tpr   0.001815     1.000000  399.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate: 52.95 %, avg score: 52.95 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.885186</td>\n",
       "      <td>1.680831</td>\n",
       "      <td>1.680831</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.917471</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.917471</td>\n",
       "      <td>0.016808</td>\n",
       "      <td>0.016808</td>\n",
       "      <td>68.083097</td>\n",
       "      <td>68.083097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.848571</td>\n",
       "      <td>1.567517</td>\n",
       "      <td>1.624174</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.867235</td>\n",
       "      <td>0.860000</td>\n",
       "      <td>0.892353</td>\n",
       "      <td>0.015675</td>\n",
       "      <td>0.032483</td>\n",
       "      <td>56.751653</td>\n",
       "      <td>62.417375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.820187</td>\n",
       "      <td>1.416431</td>\n",
       "      <td>1.554926</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.833397</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>0.872701</td>\n",
       "      <td>0.014164</td>\n",
       "      <td>0.046648</td>\n",
       "      <td>41.643059</td>\n",
       "      <td>55.492603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.798783</td>\n",
       "      <td>1.473088</td>\n",
       "      <td>1.534466</td>\n",
       "      <td>0.780</td>\n",
       "      <td>0.808362</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.856616</td>\n",
       "      <td>0.014731</td>\n",
       "      <td>0.061379</td>\n",
       "      <td>47.308782</td>\n",
       "      <td>53.446648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.779171</td>\n",
       "      <td>1.416431</td>\n",
       "      <td>1.510859</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.788570</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.843007</td>\n",
       "      <td>0.014164</td>\n",
       "      <td>0.075543</td>\n",
       "      <td>41.643059</td>\n",
       "      <td>51.085930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.718805</td>\n",
       "      <td>1.272899</td>\n",
       "      <td>1.391879</td>\n",
       "      <td>0.674</td>\n",
       "      <td>0.747137</td>\n",
       "      <td>0.737000</td>\n",
       "      <td>0.795072</td>\n",
       "      <td>0.063645</td>\n",
       "      <td>0.139188</td>\n",
       "      <td>27.289896</td>\n",
       "      <td>39.187913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.686092</td>\n",
       "      <td>1.322002</td>\n",
       "      <td>1.368587</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.701787</td>\n",
       "      <td>0.724667</td>\n",
       "      <td>0.763977</td>\n",
       "      <td>0.066100</td>\n",
       "      <td>0.205288</td>\n",
       "      <td>32.200189</td>\n",
       "      <td>36.858672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.658315</td>\n",
       "      <td>1.318225</td>\n",
       "      <td>1.355996</td>\n",
       "      <td>0.698</td>\n",
       "      <td>0.671529</td>\n",
       "      <td>0.718000</td>\n",
       "      <td>0.740865</td>\n",
       "      <td>0.065911</td>\n",
       "      <td>0.271199</td>\n",
       "      <td>31.822474</td>\n",
       "      <td>35.599622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.614692</td>\n",
       "      <td>1.246459</td>\n",
       "      <td>1.319484</td>\n",
       "      <td>0.660</td>\n",
       "      <td>0.635436</td>\n",
       "      <td>0.698667</td>\n",
       "      <td>0.705722</td>\n",
       "      <td>0.124646</td>\n",
       "      <td>0.395845</td>\n",
       "      <td>24.645892</td>\n",
       "      <td>31.948379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.576628</td>\n",
       "      <td>1.199245</td>\n",
       "      <td>1.289424</td>\n",
       "      <td>0.635</td>\n",
       "      <td>0.595520</td>\n",
       "      <td>0.682750</td>\n",
       "      <td>0.678171</td>\n",
       "      <td>0.119924</td>\n",
       "      <td>0.515770</td>\n",
       "      <td>19.924457</td>\n",
       "      <td>28.942398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.539815</td>\n",
       "      <td>1.159585</td>\n",
       "      <td>1.263456</td>\n",
       "      <td>0.614</td>\n",
       "      <td>0.558300</td>\n",
       "      <td>0.669000</td>\n",
       "      <td>0.654197</td>\n",
       "      <td>0.115958</td>\n",
       "      <td>0.631728</td>\n",
       "      <td>15.958451</td>\n",
       "      <td>26.345609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.501302</td>\n",
       "      <td>1.002833</td>\n",
       "      <td>1.220019</td>\n",
       "      <td>0.531</td>\n",
       "      <td>0.521629</td>\n",
       "      <td>0.646000</td>\n",
       "      <td>0.632102</td>\n",
       "      <td>0.100283</td>\n",
       "      <td>0.732011</td>\n",
       "      <td>0.283286</td>\n",
       "      <td>22.001889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.458743</td>\n",
       "      <td>0.898961</td>\n",
       "      <td>1.174154</td>\n",
       "      <td>0.476</td>\n",
       "      <td>0.481273</td>\n",
       "      <td>0.621714</td>\n",
       "      <td>0.610555</td>\n",
       "      <td>0.089896</td>\n",
       "      <td>0.821907</td>\n",
       "      <td>-10.103872</td>\n",
       "      <td>17.415351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.405718</td>\n",
       "      <td>0.815864</td>\n",
       "      <td>1.129367</td>\n",
       "      <td>0.432</td>\n",
       "      <td>0.434293</td>\n",
       "      <td>0.598000</td>\n",
       "      <td>0.588522</td>\n",
       "      <td>0.081586</td>\n",
       "      <td>0.903494</td>\n",
       "      <td>-18.413598</td>\n",
       "      <td>12.936733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.321457</td>\n",
       "      <td>0.585458</td>\n",
       "      <td>1.068933</td>\n",
       "      <td>0.310</td>\n",
       "      <td>0.367919</td>\n",
       "      <td>0.566000</td>\n",
       "      <td>0.564011</td>\n",
       "      <td>0.058546</td>\n",
       "      <td>0.962040</td>\n",
       "      <td>-41.454202</td>\n",
       "      <td>6.893296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.000143</td>\n",
       "      <td>0.379603</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.201</td>\n",
       "      <td>0.218905</td>\n",
       "      <td>0.529500</td>\n",
       "      <td>0.529500</td>\n",
       "      <td>0.037960</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-62.039660</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                      0.01         0.885186  1.680831   \n",
       "1         2                      0.02         0.848571  1.567517   \n",
       "2         3                      0.03         0.820187  1.416431   \n",
       "3         4                      0.04         0.798783  1.473088   \n",
       "4         5                      0.05         0.779171  1.416431   \n",
       "5         6                      0.10         0.718805  1.272899   \n",
       "6         7                      0.15         0.686092  1.322002   \n",
       "7         8                      0.20         0.658315  1.318225   \n",
       "8         9                      0.30         0.614692  1.246459   \n",
       "9        10                      0.40         0.576628  1.199245   \n",
       "10       11                      0.50         0.539815  1.159585   \n",
       "11       12                      0.60         0.501302  1.002833   \n",
       "12       13                      0.70         0.458743  0.898961   \n",
       "13       14                      0.80         0.405718  0.815864   \n",
       "14       15                      0.90         0.321457  0.585458   \n",
       "15       16                      1.00         0.000143  0.379603   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          1.680831          0.890  0.917471                  0.890000   \n",
       "1          1.624174          0.830  0.867235                  0.860000   \n",
       "2          1.554926          0.750  0.833397                  0.823333   \n",
       "3          1.534466          0.780  0.808362                  0.812500   \n",
       "4          1.510859          0.750  0.788570                  0.800000   \n",
       "5          1.391879          0.674  0.747137                  0.737000   \n",
       "6          1.368587          0.700  0.701787                  0.724667   \n",
       "7          1.355996          0.698  0.671529                  0.718000   \n",
       "8          1.319484          0.660  0.635436                  0.698667   \n",
       "9          1.289424          0.635  0.595520                  0.682750   \n",
       "10         1.263456          0.614  0.558300                  0.669000   \n",
       "11         1.220019          0.531  0.521629                  0.646000   \n",
       "12         1.174154          0.476  0.481273                  0.621714   \n",
       "13         1.129367          0.432  0.434293                  0.598000   \n",
       "14         1.068933          0.310  0.367919                  0.566000   \n",
       "15         1.000000          0.201  0.218905                  0.529500   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate       gain  \\\n",
       "0           0.917471      0.016808                 0.016808  68.083097   \n",
       "1           0.892353      0.015675                 0.032483  56.751653   \n",
       "2           0.872701      0.014164                 0.046648  41.643059   \n",
       "3           0.856616      0.014731                 0.061379  47.308782   \n",
       "4           0.843007      0.014164                 0.075543  41.643059   \n",
       "5           0.795072      0.063645                 0.139188  27.289896   \n",
       "6           0.763977      0.066100                 0.205288  32.200189   \n",
       "7           0.740865      0.065911                 0.271199  31.822474   \n",
       "8           0.705722      0.124646                 0.395845  24.645892   \n",
       "9           0.678171      0.119924                 0.515770  19.924457   \n",
       "10          0.654197      0.115958                 0.631728  15.958451   \n",
       "11          0.632102      0.100283                 0.732011   0.283286   \n",
       "12          0.610555      0.089896                 0.821907 -10.103872   \n",
       "13          0.588522      0.081586                 0.903494 -18.413598   \n",
       "14          0.564011      0.058546                 0.962040 -41.454202   \n",
       "15          0.529500      0.037960                 1.000000 -62.039660   \n",
       "\n",
       "    cumulative_gain  \n",
       "0         68.083097  \n",
       "1         62.417375  \n",
       "2         55.492603  \n",
       "3         53.446648  \n",
       "4         51.085930  \n",
       "5         39.187913  \n",
       "6         36.858672  \n",
       "7         35.599622  \n",
       "8         31.948379  \n",
       "9         28.942398  \n",
       "10        26.345609  \n",
       "11        22.001889  \n",
       "12        17.415351  \n",
       "13        12.936733  \n",
       "14         6.893296  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomialGLM: glm\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.22342897708759782\n",
      "RMSE: 0.4726827446476101\n",
      "LogLoss: 0.6385204577838607\n",
      "Null degrees of freedom: 9999\n",
      "Residual degrees of freedom: 9971\n",
      "Null deviance: 13832.373764615928\n",
      "Residual deviance: 12770.409155677213\n",
      "AIC: 12828.409155677213\n",
      "AUC: 0.6826480980292398\n",
      "AUCPR: 0.6803442006197458\n",
      "Gini: 0.3652961960584795\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.39216075368199954: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1358.0</td>\n",
       "      <td>3347.0</td>\n",
       "      <td>0.7114</td>\n",
       "      <td>(3347.0/4705.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>440.0</td>\n",
       "      <td>4855.0</td>\n",
       "      <td>0.0831</td>\n",
       "      <td>(440.0/5295.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Total</td>\n",
       "      <td>1798.0</td>\n",
       "      <td>8202.0</td>\n",
       "      <td>0.3787</td>\n",
       "      <td>(3787.0/10000.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0       1   Error               Rate\n",
       "0      0  1358.0  3347.0  0.7114    (3347.0/4705.0)\n",
       "1      1   440.0  4855.0  0.0831     (440.0/5295.0)\n",
       "2  Total  1798.0  8202.0  0.3787   (3787.0/10000.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>max f1</td>\n",
       "      <td>0.392161</td>\n",
       "      <td>0.719419</td>\n",
       "      <td>275.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>max f2</td>\n",
       "      <td>0.231964</td>\n",
       "      <td>0.851022</td>\n",
       "      <td>342.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.524765</td>\n",
       "      <td>0.659999</td>\n",
       "      <td>193.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.485376</td>\n",
       "      <td>0.643900</td>\n",
       "      <td>218.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>max precision</td>\n",
       "      <td>0.980375</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>max recall</td>\n",
       "      <td>0.001193</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.980375</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.485376</td>\n",
       "      <td>0.283305</td>\n",
       "      <td>218.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.536628</td>\n",
       "      <td>0.634006</td>\n",
       "      <td>186.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.524765</td>\n",
       "      <td>0.638646</td>\n",
       "      <td>193.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>max tns</td>\n",
       "      <td>0.980375</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>max fns</td>\n",
       "      <td>0.980375</td>\n",
       "      <td>5293.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>max fps</td>\n",
       "      <td>0.001193</td>\n",
       "      <td>4705.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>max tps</td>\n",
       "      <td>0.001193</td>\n",
       "      <td>5295.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.980375</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.980375</td>\n",
       "      <td>0.999622</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.001193</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.001193</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold        value    idx\n",
       "0                        max f1   0.392161     0.719419  275.0\n",
       "1                        max f2   0.231964     0.851022  342.0\n",
       "2                  max f0point5   0.524765     0.659999  193.0\n",
       "3                  max accuracy   0.485376     0.643900  218.0\n",
       "4                 max precision   0.980375     1.000000    0.0\n",
       "5                    max recall   0.001193     1.000000  399.0\n",
       "6               max specificity   0.980375     1.000000    0.0\n",
       "7              max absolute_mcc   0.485376     0.283305  218.0\n",
       "8    max min_per_class_accuracy   0.536628     0.634006  186.0\n",
       "9   max mean_per_class_accuracy   0.524765     0.638646  193.0\n",
       "10                      max tns   0.980375  4705.000000    0.0\n",
       "11                      max fns   0.980375  5293.000000    0.0\n",
       "12                      max fps   0.001193  4705.000000  399.0\n",
       "13                      max tps   0.001193  5295.000000  399.0\n",
       "14                      max tnr   0.980375     1.000000    0.0\n",
       "15                      max fnr   0.980375     0.999622    0.0\n",
       "16                      max fpr   0.001193     1.000000  399.0\n",
       "17                      max tpr   0.001193     1.000000  399.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate: 52.95 %, avg score: 52.97 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>1.680831</td>\n",
       "      <td>1.680831</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.917839</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.917839</td>\n",
       "      <td>0.016808</td>\n",
       "      <td>0.016808</td>\n",
       "      <td>68.083097</td>\n",
       "      <td>68.083097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.848960</td>\n",
       "      <td>1.529745</td>\n",
       "      <td>1.605288</td>\n",
       "      <td>0.810</td>\n",
       "      <td>0.867101</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>0.892470</td>\n",
       "      <td>0.015297</td>\n",
       "      <td>0.032106</td>\n",
       "      <td>52.974504</td>\n",
       "      <td>60.528801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.819737</td>\n",
       "      <td>1.340888</td>\n",
       "      <td>1.517155</td>\n",
       "      <td>0.710</td>\n",
       "      <td>0.834464</td>\n",
       "      <td>0.803333</td>\n",
       "      <td>0.873135</td>\n",
       "      <td>0.013409</td>\n",
       "      <td>0.045515</td>\n",
       "      <td>34.088763</td>\n",
       "      <td>51.715455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.797959</td>\n",
       "      <td>1.586402</td>\n",
       "      <td>1.534466</td>\n",
       "      <td>0.840</td>\n",
       "      <td>0.808654</td>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.857014</td>\n",
       "      <td>0.015864</td>\n",
       "      <td>0.061379</td>\n",
       "      <td>58.640227</td>\n",
       "      <td>53.446648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.782480</td>\n",
       "      <td>1.397545</td>\n",
       "      <td>1.507082</td>\n",
       "      <td>0.740</td>\n",
       "      <td>0.789732</td>\n",
       "      <td>0.798000</td>\n",
       "      <td>0.843558</td>\n",
       "      <td>0.013975</td>\n",
       "      <td>0.075354</td>\n",
       "      <td>39.754485</td>\n",
       "      <td>50.708215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.719365</td>\n",
       "      <td>1.254013</td>\n",
       "      <td>1.380548</td>\n",
       "      <td>0.664</td>\n",
       "      <td>0.747891</td>\n",
       "      <td>0.731000</td>\n",
       "      <td>0.795724</td>\n",
       "      <td>0.062701</td>\n",
       "      <td>0.138055</td>\n",
       "      <td>25.401322</td>\n",
       "      <td>38.054769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.686822</td>\n",
       "      <td>1.269122</td>\n",
       "      <td>1.343406</td>\n",
       "      <td>0.672</td>\n",
       "      <td>0.702821</td>\n",
       "      <td>0.711333</td>\n",
       "      <td>0.764757</td>\n",
       "      <td>0.063456</td>\n",
       "      <td>0.201511</td>\n",
       "      <td>26.912181</td>\n",
       "      <td>34.340573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.658616</td>\n",
       "      <td>1.371105</td>\n",
       "      <td>1.350331</td>\n",
       "      <td>0.726</td>\n",
       "      <td>0.672192</td>\n",
       "      <td>0.715000</td>\n",
       "      <td>0.741615</td>\n",
       "      <td>0.068555</td>\n",
       "      <td>0.270066</td>\n",
       "      <td>37.110482</td>\n",
       "      <td>35.033050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.615667</td>\n",
       "      <td>1.197356</td>\n",
       "      <td>1.299339</td>\n",
       "      <td>0.634</td>\n",
       "      <td>0.636107</td>\n",
       "      <td>0.688000</td>\n",
       "      <td>0.706446</td>\n",
       "      <td>0.119736</td>\n",
       "      <td>0.389802</td>\n",
       "      <td>19.735600</td>\n",
       "      <td>29.933900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.577442</td>\n",
       "      <td>1.214353</td>\n",
       "      <td>1.278093</td>\n",
       "      <td>0.643</td>\n",
       "      <td>0.596139</td>\n",
       "      <td>0.676750</td>\n",
       "      <td>0.678869</td>\n",
       "      <td>0.121435</td>\n",
       "      <td>0.511237</td>\n",
       "      <td>21.435316</td>\n",
       "      <td>27.809254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.540044</td>\n",
       "      <td>1.155807</td>\n",
       "      <td>1.253636</td>\n",
       "      <td>0.612</td>\n",
       "      <td>0.558871</td>\n",
       "      <td>0.663800</td>\n",
       "      <td>0.654870</td>\n",
       "      <td>0.115581</td>\n",
       "      <td>0.626818</td>\n",
       "      <td>15.580737</td>\n",
       "      <td>25.363551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.502124</td>\n",
       "      <td>1.016053</td>\n",
       "      <td>1.214038</td>\n",
       "      <td>0.538</td>\n",
       "      <td>0.521088</td>\n",
       "      <td>0.642833</td>\n",
       "      <td>0.632573</td>\n",
       "      <td>0.101605</td>\n",
       "      <td>0.728423</td>\n",
       "      <td>1.605288</td>\n",
       "      <td>21.403840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.459702</td>\n",
       "      <td>0.900850</td>\n",
       "      <td>1.169297</td>\n",
       "      <td>0.477</td>\n",
       "      <td>0.481823</td>\n",
       "      <td>0.619143</td>\n",
       "      <td>0.611037</td>\n",
       "      <td>0.090085</td>\n",
       "      <td>0.818508</td>\n",
       "      <td>-9.915014</td>\n",
       "      <td>16.929718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.405991</td>\n",
       "      <td>0.821530</td>\n",
       "      <td>1.125826</td>\n",
       "      <td>0.435</td>\n",
       "      <td>0.433775</td>\n",
       "      <td>0.596125</td>\n",
       "      <td>0.588879</td>\n",
       "      <td>0.082153</td>\n",
       "      <td>0.900661</td>\n",
       "      <td>-17.847025</td>\n",
       "      <td>12.582625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.322165</td>\n",
       "      <td>0.604344</td>\n",
       "      <td>1.067884</td>\n",
       "      <td>0.320</td>\n",
       "      <td>0.367952</td>\n",
       "      <td>0.565444</td>\n",
       "      <td>0.564332</td>\n",
       "      <td>0.060434</td>\n",
       "      <td>0.961095</td>\n",
       "      <td>-39.565628</td>\n",
       "      <td>6.788375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.000124</td>\n",
       "      <td>0.389046</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.206</td>\n",
       "      <td>0.218346</td>\n",
       "      <td>0.529500</td>\n",
       "      <td>0.529733</td>\n",
       "      <td>0.038905</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-61.095373</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                      0.01         0.884615  1.680831   \n",
       "1         2                      0.02         0.848960  1.529745   \n",
       "2         3                      0.03         0.819737  1.340888   \n",
       "3         4                      0.04         0.797959  1.586402   \n",
       "4         5                      0.05         0.782480  1.397545   \n",
       "5         6                      0.10         0.719365  1.254013   \n",
       "6         7                      0.15         0.686822  1.269122   \n",
       "7         8                      0.20         0.658616  1.371105   \n",
       "8         9                      0.30         0.615667  1.197356   \n",
       "9        10                      0.40         0.577442  1.214353   \n",
       "10       11                      0.50         0.540044  1.155807   \n",
       "11       12                      0.60         0.502124  1.016053   \n",
       "12       13                      0.70         0.459702  0.900850   \n",
       "13       14                      0.80         0.405991  0.821530   \n",
       "14       15                      0.90         0.322165  0.604344   \n",
       "15       16                      1.00         0.000124  0.389046   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          1.680831          0.890  0.917839                  0.890000   \n",
       "1          1.605288          0.810  0.867101                  0.850000   \n",
       "2          1.517155          0.710  0.834464                  0.803333   \n",
       "3          1.534466          0.840  0.808654                  0.812500   \n",
       "4          1.507082          0.740  0.789732                  0.798000   \n",
       "5          1.380548          0.664  0.747891                  0.731000   \n",
       "6          1.343406          0.672  0.702821                  0.711333   \n",
       "7          1.350331          0.726  0.672192                  0.715000   \n",
       "8          1.299339          0.634  0.636107                  0.688000   \n",
       "9          1.278093          0.643  0.596139                  0.676750   \n",
       "10         1.253636          0.612  0.558871                  0.663800   \n",
       "11         1.214038          0.538  0.521088                  0.642833   \n",
       "12         1.169297          0.477  0.481823                  0.619143   \n",
       "13         1.125826          0.435  0.433775                  0.596125   \n",
       "14         1.067884          0.320  0.367952                  0.565444   \n",
       "15         1.000000          0.206  0.218346                  0.529500   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate       gain  \\\n",
       "0           0.917839      0.016808                 0.016808  68.083097   \n",
       "1           0.892470      0.015297                 0.032106  52.974504   \n",
       "2           0.873135      0.013409                 0.045515  34.088763   \n",
       "3           0.857014      0.015864                 0.061379  58.640227   \n",
       "4           0.843558      0.013975                 0.075354  39.754485   \n",
       "5           0.795724      0.062701                 0.138055  25.401322   \n",
       "6           0.764757      0.063456                 0.201511  26.912181   \n",
       "7           0.741615      0.068555                 0.270066  37.110482   \n",
       "8           0.706446      0.119736                 0.389802  19.735600   \n",
       "9           0.678869      0.121435                 0.511237  21.435316   \n",
       "10          0.654870      0.115581                 0.626818  15.580737   \n",
       "11          0.632573      0.101605                 0.728423   1.605288   \n",
       "12          0.611037      0.090085                 0.818508  -9.915014   \n",
       "13          0.588879      0.082153                 0.900661 -17.847025   \n",
       "14          0.564332      0.060434                 0.961095 -39.565628   \n",
       "15          0.529733      0.038905                 1.000000 -61.095373   \n",
       "\n",
       "    cumulative_gain  \n",
       "0         68.083097  \n",
       "1         60.528801  \n",
       "2         51.715455  \n",
       "3         53.446648  \n",
       "4         50.708215  \n",
       "5         38.054769  \n",
       "6         34.340573  \n",
       "7         35.033050  \n",
       "8         29.933900  \n",
       "9         27.809254  \n",
       "10        25.363551  \n",
       "11        21.403840  \n",
       "12        16.929718  \n",
       "13        12.582625  \n",
       "14         6.788375  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Cross-Validation Metrics Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sd</th>\n",
       "      <th>cv_1_valid</th>\n",
       "      <th>cv_2_valid</th>\n",
       "      <th>cv_3_valid</th>\n",
       "      <th>cv_4_valid</th>\n",
       "      <th>cv_5_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>accuracy</td>\n",
       "      <td>0.6256</td>\n",
       "      <td>0.01552981</td>\n",
       "      <td>0.637</td>\n",
       "      <td>0.6255</td>\n",
       "      <td>0.631</td>\n",
       "      <td>0.6355</td>\n",
       "      <td>0.599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>auc</td>\n",
       "      <td>0.6826591</td>\n",
       "      <td>0.016180484</td>\n",
       "      <td>0.69881976</td>\n",
       "      <td>0.68099487</td>\n",
       "      <td>0.67526984</td>\n",
       "      <td>0.6978596</td>\n",
       "      <td>0.6603513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>aucpr</td>\n",
       "      <td>0.6804682</td>\n",
       "      <td>0.024902431</td>\n",
       "      <td>0.6903477</td>\n",
       "      <td>0.6809779</td>\n",
       "      <td>0.6769227</td>\n",
       "      <td>0.7113003</td>\n",
       "      <td>0.64279234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>err</td>\n",
       "      <td>0.3744</td>\n",
       "      <td>0.01552981</td>\n",
       "      <td>0.363</td>\n",
       "      <td>0.3745</td>\n",
       "      <td>0.369</td>\n",
       "      <td>0.3645</td>\n",
       "      <td>0.401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>err_count</td>\n",
       "      <td>748.8</td>\n",
       "      <td>31.05962</td>\n",
       "      <td>726.0</td>\n",
       "      <td>749.0</td>\n",
       "      <td>738.0</td>\n",
       "      <td>729.0</td>\n",
       "      <td>802.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>f0point5</td>\n",
       "      <td>0.64003783</td>\n",
       "      <td>0.017270906</td>\n",
       "      <td>0.6469149</td>\n",
       "      <td>0.64221364</td>\n",
       "      <td>0.64877856</td>\n",
       "      <td>0.65242946</td>\n",
       "      <td>0.6098525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>f1</td>\n",
       "      <td>0.72128296</td>\n",
       "      <td>0.013551233</td>\n",
       "      <td>0.7239544</td>\n",
       "      <td>0.72714025</td>\n",
       "      <td>0.72483224</td>\n",
       "      <td>0.7326733</td>\n",
       "      <td>0.69781464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>f2</td>\n",
       "      <td>0.8263422</td>\n",
       "      <td>0.009803491</td>\n",
       "      <td>0.8218232</td>\n",
       "      <td>0.8379513</td>\n",
       "      <td>0.8210846</td>\n",
       "      <td>0.835424</td>\n",
       "      <td>0.81542796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>lift_top_group</td>\n",
       "      <td>1.6823573</td>\n",
       "      <td>0.076186374</td>\n",
       "      <td>1.7077799</td>\n",
       "      <td>1.682243</td>\n",
       "      <td>1.5755329</td>\n",
       "      <td>1.6605166</td>\n",
       "      <td>1.7857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>logloss</td>\n",
       "      <td>0.6382205</td>\n",
       "      <td>0.009399038</td>\n",
       "      <td>0.6335576</td>\n",
       "      <td>0.6357881</td>\n",
       "      <td>0.64216685</td>\n",
       "      <td>0.62744224</td>\n",
       "      <td>0.6521476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>max_per_class_error</td>\n",
       "      <td>0.7003129</td>\n",
       "      <td>0.02875882</td>\n",
       "      <td>0.65961945</td>\n",
       "      <td>0.727957</td>\n",
       "      <td>0.6851249</td>\n",
       "      <td>0.70305675</td>\n",
       "      <td>0.7258065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>mcc</td>\n",
       "      <td>0.27618256</td>\n",
       "      <td>0.016757142</td>\n",
       "      <td>0.29757294</td>\n",
       "      <td>0.2768313</td>\n",
       "      <td>0.2695549</td>\n",
       "      <td>0.28432855</td>\n",
       "      <td>0.25262508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>mean_per_class_accuracy</td>\n",
       "      <td>0.6075443</td>\n",
       "      <td>0.009444297</td>\n",
       "      <td>0.62180316</td>\n",
       "      <td>0.60237664</td>\n",
       "      <td>0.6078546</td>\n",
       "      <td>0.60926497</td>\n",
       "      <td>0.5964222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>mean_per_class_error</td>\n",
       "      <td>0.3924557</td>\n",
       "      <td>0.009444297</td>\n",
       "      <td>0.37819684</td>\n",
       "      <td>0.39762336</td>\n",
       "      <td>0.39214537</td>\n",
       "      <td>0.39073503</td>\n",
       "      <td>0.40357783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>mse</td>\n",
       "      <td>0.22341162</td>\n",
       "      <td>0.0043744603</td>\n",
       "      <td>0.21988294</td>\n",
       "      <td>0.22265317</td>\n",
       "      <td>0.22547193</td>\n",
       "      <td>0.2191962</td>\n",
       "      <td>0.22985382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>null_deviance</td>\n",
       "      <td>2766.4749</td>\n",
       "      <td>8.274729</td>\n",
       "      <td>2766.8323</td>\n",
       "      <td>2763.1602</td>\n",
       "      <td>2761.3486</td>\n",
       "      <td>2760.4211</td>\n",
       "      <td>2780.6116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>pr_auc</td>\n",
       "      <td>0.6804682</td>\n",
       "      <td>0.024902431</td>\n",
       "      <td>0.6903477</td>\n",
       "      <td>0.6809779</td>\n",
       "      <td>0.6769227</td>\n",
       "      <td>0.7113003</td>\n",
       "      <td>0.64279234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>precision</td>\n",
       "      <td>0.595371</td>\n",
       "      <td>0.01892361</td>\n",
       "      <td>0.6040609</td>\n",
       "      <td>0.5958209</td>\n",
       "      <td>0.60636306</td>\n",
       "      <td>0.6080341</td>\n",
       "      <td>0.56257594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>r2</td>\n",
       "      <td>0.102571316</td>\n",
       "      <td>0.01610108</td>\n",
       "      <td>0.11789605</td>\n",
       "      <td>0.10500182</td>\n",
       "      <td>0.09244823</td>\n",
       "      <td>0.11698464</td>\n",
       "      <td>0.08052583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>recall</td>\n",
       "      <td>0.9154015</td>\n",
       "      <td>0.013312261</td>\n",
       "      <td>0.9032258</td>\n",
       "      <td>0.9327103</td>\n",
       "      <td>0.9008341</td>\n",
       "      <td>0.9215867</td>\n",
       "      <td>0.9186508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    mean            sd  cv_1_valid  \\\n",
       "0                  accuracy       0.6256    0.01552981       0.637   \n",
       "1                       auc    0.6826591   0.016180484  0.69881976   \n",
       "2                     aucpr    0.6804682   0.024902431   0.6903477   \n",
       "3                       err       0.3744    0.01552981       0.363   \n",
       "4                 err_count        748.8      31.05962       726.0   \n",
       "5                  f0point5   0.64003783   0.017270906   0.6469149   \n",
       "6                        f1   0.72128296   0.013551233   0.7239544   \n",
       "7                        f2    0.8263422   0.009803491   0.8218232   \n",
       "8            lift_top_group    1.6823573   0.076186374   1.7077799   \n",
       "9                   logloss    0.6382205   0.009399038   0.6335576   \n",
       "10      max_per_class_error    0.7003129    0.02875882  0.65961945   \n",
       "11                      mcc   0.27618256   0.016757142  0.29757294   \n",
       "12  mean_per_class_accuracy    0.6075443   0.009444297  0.62180316   \n",
       "13     mean_per_class_error    0.3924557   0.009444297  0.37819684   \n",
       "14                      mse   0.22341162  0.0043744603  0.21988294   \n",
       "15            null_deviance    2766.4749      8.274729   2766.8323   \n",
       "16                   pr_auc    0.6804682   0.024902431   0.6903477   \n",
       "17                precision     0.595371    0.01892361   0.6040609   \n",
       "18                       r2  0.102571316    0.01610108  0.11789605   \n",
       "19                   recall    0.9154015   0.013312261   0.9032258   \n",
       "\n",
       "    cv_2_valid  cv_3_valid  cv_4_valid  cv_5_valid  \n",
       "0       0.6255       0.631      0.6355       0.599  \n",
       "1   0.68099487  0.67526984   0.6978596   0.6603513  \n",
       "2    0.6809779   0.6769227   0.7113003  0.64279234  \n",
       "3       0.3745       0.369      0.3645       0.401  \n",
       "4        749.0       738.0       729.0       802.0  \n",
       "5   0.64221364  0.64877856  0.65242946   0.6098525  \n",
       "6   0.72714025  0.72483224   0.7326733  0.69781464  \n",
       "7    0.8379513   0.8210846    0.835424  0.81542796  \n",
       "8     1.682243   1.5755329   1.6605166   1.7857143  \n",
       "9    0.6357881  0.64216685  0.62744224   0.6521476  \n",
       "10    0.727957   0.6851249  0.70305675   0.7258065  \n",
       "11   0.2768313   0.2695549  0.28432855  0.25262508  \n",
       "12  0.60237664   0.6078546  0.60926497   0.5964222  \n",
       "13  0.39762336  0.39214537  0.39073503  0.40357783  \n",
       "14  0.22265317  0.22547193   0.2191962  0.22985382  \n",
       "15   2763.1602   2761.3486   2760.4211   2780.6116  \n",
       "16   0.6809779   0.6769227   0.7113003  0.64279234  \n",
       "17   0.5958209  0.60636306   0.6080341  0.56257594  \n",
       "18  0.10500182  0.09244823  0.11698464  0.08052583  \n",
       "19   0.9327103   0.9008341   0.9215867   0.9186508  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>iteration</th>\n",
       "      <th>lambda</th>\n",
       "      <th>predictors</th>\n",
       "      <th>deviance_train</th>\n",
       "      <th>deviance_test</th>\n",
       "      <th>deviance_xval</th>\n",
       "      <th>deviance_se</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.000 sec</td>\n",
       "      <td>2</td>\n",
       "      <td>.77E1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.378643</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.379976</td>\n",
       "      <td>0.001881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.018 sec</td>\n",
       "      <td>4</td>\n",
       "      <td>.48E1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.376337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.378142</td>\n",
       "      <td>0.001905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.035 sec</td>\n",
       "      <td>6</td>\n",
       "      <td>.3E1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.372925</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.375392</td>\n",
       "      <td>0.001950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.060 sec</td>\n",
       "      <td>8</td>\n",
       "      <td>.18E1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.368055</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.371402</td>\n",
       "      <td>0.002034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.081 sec</td>\n",
       "      <td>10</td>\n",
       "      <td>.11E1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.361409</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.365846</td>\n",
       "      <td>0.002189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.100 sec</td>\n",
       "      <td>12</td>\n",
       "      <td>.71E0</td>\n",
       "      <td>29</td>\n",
       "      <td>1.352820</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.358487</td>\n",
       "      <td>0.002459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.116 sec</td>\n",
       "      <td>14</td>\n",
       "      <td>.44E0</td>\n",
       "      <td>29</td>\n",
       "      <td>1.342413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.349303</td>\n",
       "      <td>0.002890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.134 sec</td>\n",
       "      <td>16</td>\n",
       "      <td>.27E0</td>\n",
       "      <td>29</td>\n",
       "      <td>1.330709</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.338619</td>\n",
       "      <td>0.003508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.150 sec</td>\n",
       "      <td>18</td>\n",
       "      <td>.17E0</td>\n",
       "      <td>29</td>\n",
       "      <td>1.318619</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.327139</td>\n",
       "      <td>0.004299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.179 sec</td>\n",
       "      <td>20</td>\n",
       "      <td>.11E0</td>\n",
       "      <td>29</td>\n",
       "      <td>1.307164</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.315843</td>\n",
       "      <td>0.005194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.201 sec</td>\n",
       "      <td>22</td>\n",
       "      <td>.65E-1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.297171</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.305649</td>\n",
       "      <td>0.006094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.223 sec</td>\n",
       "      <td>24</td>\n",
       "      <td>.41E-1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.289012</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.297132</td>\n",
       "      <td>0.006894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:26</td>\n",
       "      <td>0.240 sec</td>\n",
       "      <td>26</td>\n",
       "      <td>.25E-1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.282689</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.290430</td>\n",
       "      <td>0.007518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.257 sec</td>\n",
       "      <td>28</td>\n",
       "      <td>.16E-1</td>\n",
       "      <td>29</td>\n",
       "      <td>1.278021</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.285406</td>\n",
       "      <td>0.007947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.280 sec</td>\n",
       "      <td>30</td>\n",
       "      <td>.97E-2</td>\n",
       "      <td>29</td>\n",
       "      <td>1.274788</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.281856</td>\n",
       "      <td>0.008204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.298 sec</td>\n",
       "      <td>32</td>\n",
       "      <td>.6E-2</td>\n",
       "      <td>29</td>\n",
       "      <td>1.272731</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.279551</td>\n",
       "      <td>0.008344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.315 sec</td>\n",
       "      <td>34</td>\n",
       "      <td>.38E-2</td>\n",
       "      <td>29</td>\n",
       "      <td>1.271548</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.278205</td>\n",
       "      <td>0.008419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.332 sec</td>\n",
       "      <td>36</td>\n",
       "      <td>.23E-2</td>\n",
       "      <td>29</td>\n",
       "      <td>1.270931</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.277509</td>\n",
       "      <td>0.008461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.344 sec</td>\n",
       "      <td>37</td>\n",
       "      <td>.14E-2</td>\n",
       "      <td>29</td>\n",
       "      <td>1.270633</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.277197</td>\n",
       "      <td>0.008488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td></td>\n",
       "      <td>2020-03-18 08:58:27</td>\n",
       "      <td>0.353 sec</td>\n",
       "      <td>38</td>\n",
       "      <td>.9E-3</td>\n",
       "      <td>29</td>\n",
       "      <td>1.270499</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.277078</td>\n",
       "      <td>0.008508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  iteration  lambda  predictors  \\\n",
       "0     2020-03-18 08:58:26   0.000 sec          2   .77E1          29   \n",
       "1     2020-03-18 08:58:26   0.018 sec          4   .48E1          29   \n",
       "2     2020-03-18 08:58:26   0.035 sec          6    .3E1          29   \n",
       "3     2020-03-18 08:58:26   0.060 sec          8   .18E1          29   \n",
       "4     2020-03-18 08:58:26   0.081 sec         10   .11E1          29   \n",
       "5     2020-03-18 08:58:26   0.100 sec         12   .71E0          29   \n",
       "6     2020-03-18 08:58:26   0.116 sec         14   .44E0          29   \n",
       "7     2020-03-18 08:58:26   0.134 sec         16   .27E0          29   \n",
       "8     2020-03-18 08:58:26   0.150 sec         18   .17E0          29   \n",
       "9     2020-03-18 08:58:26   0.179 sec         20   .11E0          29   \n",
       "10    2020-03-18 08:58:26   0.201 sec         22  .65E-1          29   \n",
       "11    2020-03-18 08:58:26   0.223 sec         24  .41E-1          29   \n",
       "12    2020-03-18 08:58:26   0.240 sec         26  .25E-1          29   \n",
       "13    2020-03-18 08:58:27   0.257 sec         28  .16E-1          29   \n",
       "14    2020-03-18 08:58:27   0.280 sec         30  .97E-2          29   \n",
       "15    2020-03-18 08:58:27   0.298 sec         32   .6E-2          29   \n",
       "16    2020-03-18 08:58:27   0.315 sec         34  .38E-2          29   \n",
       "17    2020-03-18 08:58:27   0.332 sec         36  .23E-2          29   \n",
       "18    2020-03-18 08:58:27   0.344 sec         37  .14E-2          29   \n",
       "19    2020-03-18 08:58:27   0.353 sec         38   .9E-3          29   \n",
       "\n",
       "    deviance_train  deviance_test  deviance_xval  deviance_se  \n",
       "0         1.378643            NaN       1.379976     0.001881  \n",
       "1         1.376337            NaN       1.378142     0.001905  \n",
       "2         1.372925            NaN       1.375392     0.001950  \n",
       "3         1.368055            NaN       1.371402     0.002034  \n",
       "4         1.361409            NaN       1.365846     0.002189  \n",
       "5         1.352820            NaN       1.358487     0.002459  \n",
       "6         1.342413            NaN       1.349303     0.002890  \n",
       "7         1.330709            NaN       1.338619     0.003508  \n",
       "8         1.318619            NaN       1.327139     0.004299  \n",
       "9         1.307164            NaN       1.315843     0.005194  \n",
       "10        1.297171            NaN       1.305649     0.006094  \n",
       "11        1.289012            NaN       1.297132     0.006894  \n",
       "12        1.282689            NaN       1.290430     0.007518  \n",
       "13        1.278021            NaN       1.285406     0.007947  \n",
       "14        1.274788            NaN       1.281856     0.008204  \n",
       "15        1.272731            NaN       1.279551     0.008344  \n",
       "16        1.271548            NaN       1.278205     0.008419  \n",
       "17        1.270931            NaN       1.277509     0.008461  \n",
       "18        1.270633            NaN       1.277197     0.008488  \n",
       "19        1.270499            NaN       1.277078     0.008508  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n"
     ]
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get specific model\n",
    "h2o.get_model(model_ids[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make ensemble exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the \"All Models\" Stacked Ensemble model\n",
    "se = h2o.get_model(model_ids[0])\n",
    "# Get the Stacked Ensemble metalearner model\n",
    "metalearner = h2o.get_model(se.metalearner()['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Intercept': 0.14802018905233716,\n",
       " 'XGBoost_1_AutoML_20200318_085617': 0.3110710984103567,\n",
       " 'XGBoost_3_AutoML_20200318_085617': 0.1765994850963209,\n",
       " 'XGBoost_2_AutoML_20200318_085617': 0.181074588310919,\n",
       " 'GBM_5_AutoML_20200318_085617': 0.04924611254697128,\n",
       " 'GBM_1_AutoML_20200318_085617': 0.09991303580168046,\n",
       " 'GBM_2_AutoML_20200318_085617': 0.08325612594184778,\n",
       " 'GBM_3_AutoML_20200318_085617': 0.12360941807445644,\n",
       " 'GBM_4_AutoML_20200318_085617': 0.17464997889789913,\n",
       " 'DRF_1_AutoML_20200318_085617': 0.0,\n",
       " 'GLM_1_AutoML_20200318_085617': 0.10720856814124097}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metalearner.coef_norm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+YAAAJTCAYAAACFEAxVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde/x19Zz//8dTV+VQ0sEhFYUohwqVKDohhAqpawzVFD+mocyMKcZg8P2Kvg5DDtOQYmqqKSk6ITlEapKrkxhRKqWDSAmpXr8/3u/t2u325/PZ19VVu/K43277tvd+r/d6r/dae+3PZ7/W+7BSVUiSJEmSpOm437QrIEmSJEnSXzIDc0mSJEmSpsjAXJIkSZKkKTIwlyRJkiRpigzMJUmSJEmaIgNzSZIkSZKmyMBcknSfl2TNJJXk4CnX4+BejzWH0u4RdRtI8q5eny2mXZclIcn8JD9IckPfr49Mu073NEnm9WPztSnX4z97PVafZj0kaRoMzCVJd5BkqSSvTfLNJNcl+VOSq5Ocm+TTSV46kn/X/oN61ylVWVOWZJkkuyc5PsmVSf7Yg+EFST6SZL0p1OmZwKHA8sAngX8FTlrMsh7Xz/FK8tskD5oh3/2SXDKUd7PF3oEpS/Lee/s+LGlD58FFs+QZXOi4ZSR9lf539YtJLkry+yS/SfLtJLslySxlJsmOSY4b+n5d29fdO8kD7sQ+bZTk35NckOT6/vf+ml72vyZZe8w6g4sofz1B+YPzqJJ8ZpZ8Ww/lm/H4SvdV86ZdAUnSPUuSpYAvAy8AfgMcD1wOrAQ8FvgrYB3guGnV8T7mF8C6wPXTrsjiSvJ44Iu0/bgW+CpwKbAM8ETg9cCbkmxfVXfnebMtEOA1VfXdJVTmLbRAfyfgoDHLnw88uue7V/zOqqpbkqwL/G7adbmP2xn4GHAFcCpwGfAI4GW0c+kFtPPqdpKsCPw3sDUL/yZfSvub/ALgw8Abk7y4qi6ctDJJlu31eS1wG/Bd4OvAb4EVgacDbwfenuQlVXXCou/y7dwC7JRk76q6Yczy13Iv+t5IS5onviRp1Hzaj71zgM2r6nYBY5IHAs+YRsXui6rqT8CPpl2PxZXk4cApwOrAR4C3VdXvR/I8DHgn7cf+3emR/fmKJVjmmcDjaEHEuMD8tcDvgW8B2yzB7d6lqupeew7ei/wIeDFwYlXdNkhM8s+08+qVSQ6rqmOHli0FHA1sCZwA/HVV/Xpo+dLAe4F/Ar6S5GlVdc2E9fk08Ne0v/XzxwX1fdjNPwMPWYT9nMmXge1pF3f/fWQ7qwA7AF/qz9JfHLuyS5JGPas/HzwalANU1U1VdergfZJvAJ/tbz871BXxz2OpkzwyyTuSfCfJL5PcnOSKJIf1lrrbydC46/768N5t8w9Jzkry4nEVT7J8kg8lubzn/VGSv2eG/3dJHp9kv17mNb176M+THJgx41yTbNHr9a4kG6d1274udxw3/tzeDfR3ffkXk6wzQx3uMMY8C4cGzPZYc6ScZyQ5auj4Xta7pz6SMZI8PclJad3Nf5vka2ldvxfVe2lB+X9V1ZtHg3KAqrq6qvYEDh+pw6pJPp7W9fvm/hl8IcnTZ9pY2pjxU5P8un/GFyZ5e2/9G+TZNUkBu/Wki2c6bovhT8AhwCZJnjxSt4cDL6G1bv5mhvpvnTYc5MJ+3G9Kcn6Sfxneh5F1HpnkkH58fp82Zv6v+3lWSd4+kv+0JLckWbofm4v6uX1pkvf1YG44/x3GmCe5nBaQAXx76PjdMrqdGeq8R2bo6pxkm7S/BYPvxzFpvS5mlOSZSY4eOb8/lWTVMXkf24/xT/vx+lWS85J8Mq31eSqq6mtVdfxwUN7TrwAO7G+3GFnt1bSg/CfAK4aD8r7un6pqH+Ao2vfw3ZPUJcnWtKD8GuD5M7W0V9UlVfVa4MhJyp3D8cCVtItXo15D62HzH0tgO9K9ki3mkqRRv+rPs/5QHnIwLQjZDjgWWDC0bBCcPAfYl9Z982jgRmBt4BXAS5NsWlXnjCn70bSWpJ8Bn6d13dwJODbJc0cuECxLa7ndiNYCdCitledfgM1nqPvLaN2sT6V147wZeBKwB/CSJBtW1S/GrPdM4K3AabRW01X6uiR5BXBEf38E7YfoZsDpwLkz1GPUAtp46FErAHsBBfxhkJhkN9oP2j/ShhhcRju+g/3YpKouHcr/LOBrtB/CXwAuAjYAvkHryjqRtHGtr+5vx9X3dqrqj0PrrkU7fo/s2/wvYA1gR2DbJC+vqi+PbO8zwN/QhlZ8gXZ+bQK8B9g6yfOq6hYWHr/tgfWBf2PhuTg2YF5EnwbeQju+ew+l7wosTfss/m6Gdd8KPAb4Hq118IHAprSAavMk21TVrYPMSR5BO3ceRft8vgesSgvkTp6jnofTztWTgBtoXfv3pZ2v44KjYR+iHb9n0y68Dc6f22ZcYwJJdgIOo52rRwC/pP19OB344QzrvBb4FK0nwnG0z//xfR9enOQZg+9pktWA/wGWo7UwHwU8AFiLFvz9GzDc4nw5sBqwRlVdfmf27U76U38evdAx+Jz2H3fRa8h7aH9Pd0myV1XdPMf29ujPn6yqq+eqXP9e3Vm30M6ltyXZoKqG/1fsQfs79I0lsB3p3qmqfPjw4cOHjz8/gKfSgsrbaMHwy4BHz7HOrrRgcdcZlj8MWH5M+vq0IP3EkfQ1e3kFvHNk2TY9/YSR9Lf19KOB+w2lrwVc15cdPLLOasCyY+r1fOBW2o/W4fQthur1/41ZbznahY0/ARuOLPvw0LprjtnXg0fLG1l/aVowXcBeQ+mP75/XRcBqI+ts1ffjmKG00LrUFrDdSP69huq4xQTnyrN73ssX4zw7ua/7zyPpz6L9gP8VsNyYc+wLwANG1nnX6HHp6QePHu878b14XC/rG/39N3odlx06rj8BLuzvD+/5Nxsp5zFAxpT/vp7/5SPph/T0/zOS/rT+uRfw9pFlp/X0M4EVR87Pn/Xj+9Ch9Hk9/9dGynnvuH0Y2c4tMyzbo6/710NpD6YFxTcDTx3J/7Ghc2/1ofR1e/4fA6vO8D3976G0N/cy9pzh+3n/kbTLR7c54XlwXT/vxj3e3fOMPTZjylyadlGigK2H0peh/S0pYK0Jyrmq591kgryX9rybL+b34T9HP99Z8g7Oo11pf49vAz4+tHyzvnwf4P799UWLUy8fPu7ND7uyS5Jup6p+QOvieFV/Phq4pHcHPSbJSxajzKtrzGQ/1VrJvw5sOdq9tvs57Ufd8Don035UbjySdzfaD75/qqGuolV1MfDRGer1ixpqxR1K/wpwATOPEV5QVf8+Jn07Wqv+YVV11siyd3HnJnj7FG3yp49V1b8Npb+B9sN+rxpp3a+qr9NaGF+SZPme/CzgCcC3amgsa3cA8NNFqNOgG/EitTSmDRN4Pu1z/MBInb9Laz1fiXZRaGAvWkD5N3XHlsP30ILkVy1KPe6k/6DV8eX9/Ra0oG3WrrhV9bOqqjGLPtyf/3zOJbk/rYfIr4H/O1LO2bReIbP5pxrq+lxVN9KO7VK0ib3ubjvQerF8vv+dGfYOWqv+qL+lnd9vqqorhxf07+kJwPa54yz544ZU3FhVfxhJ3pwW/P9y4r1oVqTNmzDu8S+LWNb+vQ7HVdUpQ+mrsLB362UTlDPIM3b4yohH9Oc79AhK8rS04TrDj9dMUOac+t/jU4BXpc1XAgsnfTt4SWxDureyK7sk6Q6q6sgkx9DGNm5Ga0XfjNa1dfskn6O1jo8LMMZKsi2t2/iG3P4H58AqtG7fwxbUULfeIZfRuugOyl6eFhRdVlXjAstv0H4wj9YptGBuV1rr/Yq0oGVgpu6gZ86Q/rT+/M3RBVV1fZIFzNytfkZpk0P9Da3r894jiwfHYfMkG41Z/WG0fXo88P056nhrktNos+9PVLXBqhPmH3hqf/52tcnvRn2ddlHoqcDn+g/49Wkzvu+d8XeV+iMtuLm7HE274PNaWtfs19HOl8/NtlKS5Wif4fa0z2Q5Fh5HaL04BtYFlgW+W1XjZkw/jXbuzmT04hAsDN6mMdZ6tnPv10nOpXXrHzY4v7fM+DkQBn9LHkcbwnIs7ULNp5K8iNYz4zu0ngx3OE9n+HsxiZ9W1ePGLUgyj4Vd02eVNgfGXrQLgbuOLl7EOi3O93Fc3qdxx7+XpzDHub0I/gN4LrBjkmNpw1eOq6qr+sUo6S+SgbkkaaweMH2lPwazA7+cNqb6NcAxtFtkzSnJm1g4tnNwK62baD8KB+OAx018NdN44Fu4/YRuK/Tnq2bIP1Nr2IdoQdKVtB/wv2BhS9uutDHui1Le4tZjRknm0wKN79NmTh4d47tyf37LHEUtdxfUcTDb+R0mypvDoA6jF2IYSR/MBL0iLeh4KGMusExDVf0hyX/SblO1Ca01+JiqunamdZIsQ7tI9HTgPFp392toQdz9aC2tw9+DuT6rmdIBbu0t5KMGY4WXGrPsrrY4597g/N5njrKXg9YjIckzaOfJNizs0XBpkv2r6oBFqO9dKslewAeB82ld2H89kuUaFt4+bA3g4jmKHHwPZ/peDftlL3M1RnrJVNWnafMokDZp5cS3YJvQF2n7tgft1oMPwEnfJANzSdJkesv1kUmeQru37VZMEJj31qN/pf0QfNpod9QZWsEW1aCL+MNnWP6I0YS0W3i9ifaj+FmjXe17QDyTmVqkFrkes0kymHjrMuAlM7SaDra5QlX9doJil2Qdz6K1VK+e5AlV9eMJ1xvUYaZtrTqSb/D8g6p62pj803Ig7Rz6b1pAfeDs2XkZLSj/TFXtMbwgyRrcsQv04POc6bOaKf3uchut48n9xlwwGnd7rcU59wbrPKiqbpqkUlV1Ae3WY/NoF/2eD7wR+FiSG6rqkEnKuSsl+UdaF/ZzgOeOu6BTVTcn+R9ar4HnMkvw2v8uP4x2YXF0mMA436HdV31r2q397jZ9vw4B/pE25vzn9AvA0l8yx5hLkhbVIIAd7mY56G4+rhVuFdqP9O+OCcqXY2H31sXWg+qLgNWSjOuGvcWYtMfQ/g9+ZUxQvnpfvqjO7s936K6eZAXazOcTSbI2rVfCH4FtR4/dkO/152cvgTouRRuyMJE+1vvz/e2c42qz8HZgg8Bhsx48jdpyuK695fcC4ElJVpq0fne1HgCeTmup/Cltdv/ZDLo+Hz1m2bghDj+kff4bjBlDDYvwWS2m2b7X0HrA3I/bd78f2HBM2mzn3orAemPWWdTz+8+q6paq+n5VvY+F8w9sv6jlLGl9aMr+tOOx1Wy9LOgt18A/zNHNe3DLvEPGzZsxS7mvT/LQCfIvaYPtr0a7UHWnZvuX7gsMzCVJt5N2n+jnJbnD/4h+66bB7XuGW1kGt1h71Jgir6Z1W396D8QHZS1N696+yhKpeGtZvh/w/uG699tyvWlM/kv682Y9IB3kX47WMrU4vcqOpQUrf5VkNDB5Fwu78s4qySq0Sa1WoN27+PxZsh9A6wr94XH3gk6yTG95H/gubYbr5yTZbiT73zH5+PKBt9Mmf3tVkv37LdRG67BKko/SWuiodluqr9JmpN97JO8zgL+iHcdjhhZ9iDZL9UFJ7tAam2TFJBNf5Em7h/o6SR486Toz2J3Wjf0VE8y5cEl/3mKkLo+lzcp+O32isv+mdeV/28g6T+Wun+xutu81LJxr4Xa3XkvyfNq44VHH0FrAX93rP+zdtG7Noz5G6879b0nuMKa7n9+bDb3fuPeGGTVopb9dq3vaPc/XmeEC0RKX5F20CS3PpHVfv26OVT5H+1v7BFqPpdud+2n3oP+/wCtpQ3EmGurRJ5n7T1or+8m9y/o443o+3Gm9d80LaN+dj98V25DubezKLkka9QzaZES/7BOBDcY1rkW7D/IDaAHoUUPrnE77wbt3b9EcjCH9WJ/07KO0+yef1yf7WYbWKroSrZVxS+68D9Jaw14OnJ3kZFpguxPth+1LhzNX1S+THE4LFhck+UrP/zzaPcIXsAgt3L3MG5O8jnZ/5m8nGb6P+ZN7PZ4zQVHvprWung1smmR0QiyAj1TVb6rqR0n+hjb2/4IkJwH/S5vJ+lG0lsZrgHV6HSvJ7rTA+Ogkg/uYr0/rLnsS7QfzpPt8VZKtacMa/pF2H+XBPALL0CYw24LW1Xu4tfL1tO60+/dA7iwW3sf8NmC34Z4MVXVQkqfTZun+af98L6WdQ2vRjutne7mT2J8W2L6aFqAslqq6kMnH4B5L+z79U5L1ad2YHw28GPgy7Vwd9U+04/e2tPvPn07r6r8TcDztmN5VrY1fpw3beH+v72+A26pqMEP8Z4B/AP6lB9oX0s6zF9CC8JcPF1ZVv03yetpked/p34/BfczXpU1mt9nIOhck2YN2seyHSU6k3ZZuWRae31fQvl/Q5r94XZJv0s7r39C+Sy+hfa+H72gAbSK61Wjn3l16H/P+vXsnrSfCdxg/keHPqurPk6xV1S1JXkbrZfES4GdJjmfhuf8C2gWun9GGu8x5T/Ihe9B6ZOxO+9vxHdrfvRt62Y+nnXu39fqO87okz51h2edHZpm/nX6HDUkDS+q+az58+PDh477xoP1A3ZP2w/rHtHGuN9MCzBNos2Xfb8x6L6AFDTcycr9u2oXgv6d1zf097cf452lBycHDeXv+NZnl3t60CbRqTPqDaS2rv6D9CP8RLXB4zLjygAcC/4f2A/4PtLHcH6dNOHWHbbDwPubvmuMYPo8WZNxEa/k9lhawTLSvQ/lme6w5ss2n9PV+TvuxfR1t/Py/07rLjtbx6bQg/Ib++BptLOu7mPA+5iPlLUP7gX9CP1du7uWeR5u9/Clj1lkN+GSv8820Wde/CGw0y3YGQezVfZ1f0lof3wusM5L3Dsd7aNnE92Hu+W93H/MJ8s90H/NH0QLTK2jfhQtoFzSWZcy9xPs6q9NaTq9l4RjiV9MuKhXwdyP5F/X+4mPvY96X7UK7gPB7xtybu593J/bP+kbahbZnj9vO0Drb0AK9m/p5+kVaEDj4TO5wT3HahaNDxpzfnxw+V/s5/Cng3J7n97Tv90HAE8eUu7j3MZ/xPttDx3P0WA3u5z3b4w6fQV/3fv3z/nI/52/u+3ca7d7tD1yU7+tI2RvT5ke4sH+Of6JdzDut13ntWb4/sz3+bmS/d52gLt7H3Mdf7CNVi3qHE0mSJE1bkvfTWtSfW7O0TEqS7vkMzCVJku7Bkjyyqq4YSVuf1ur8e1pr7yQTfkmS7qEcYy5JknTPtiDJhbSu2zfRun2/iNa9eXeDckm697PFXJIk6R4sybtpkxc+GliONqHZ94D9q+puvQe1JOmuYWAuSZIkSdIU2ZVdWgIOOeSQ2mWXXaZdDUmSJEn3XHe4R+LA/e7OWkj3Vb/73e+mXQVJkiRJ91IG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVNkYC5JkiRJ0hQZmEuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVNkYC5JkiRJ0hTNm3YFpPuC835xPWvue/y0qyFJkiQJuGS/baddhUVii7kkSZIkSVNkYC5JkiRJ0hQZmEuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVNkYC5JkiRJ0hQZmEuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJUzRrYJ5kjSQXJ1mpv1+xv390krWTfDnJT5N8P8mpSZ7T8+2a5JokC5JckOSoJA9cUpVOskGSF82RZ50kpyf5Y5J/nLDcHZJUknUmzL/3JPuV5JIk3x5JW5Dk/P56iyRfnnCbhyb5cZLzkxyUZOmeniQfTXJRknOTPK2nr9E/mwv7Z7HXUFkrJflqkp/05xWHlr21l/XjJNsMpZ+U5Jxe1qeSLNXTn5Pk7CS3JHnFSJ0/0PNf2OuYWfZvrSRn9DodkWSZnr5Cki8NbXu3keN7Xj+mZ42U98a+Dxck+UBPW7kfkxuTHDCUd/lexuBxbZKPTPK5SJIkSdLimjUwr6rLgE8C+/Wk/YADgauA44EDq+qxVfV04I3AY4ZWP6KqNqiqJwE3AzstwXpvAMwamAPXAW8C/t8ilDsfOA3YecL8ewOTXnBYPskaAEnWXYQ6jToUWAd4CvAAYI+e/kJg7f54He1zA7gF+IeqWhfYBNgzyRP7sn2BU6pqbeCU/p6+fGfgScALgE8MAnDglVW1PvBk4KHAjj39UmBX4LDhyiZ5FrApsF5fZyNg81n27/3Ah3udfg3s3tP3BH7Yt70F8MFB0N5t2c+3DYe2vSWwHbBePw8H58IfgH8BbnfBpqpu6GVsUFUbAD8HvjBLXSVJkiTpTpukK/uHgU2S7A1sBnwQeBVwelUdN8hUVedX1cGjKyeZBzyIFmTRW9tP6a26pyR51BzpO/bW4XOSfKsHY+8GduqtmmMD/qq6uqr+B/jTJAciyXK0AHJ3hgLz0dbsJAf0HgFvAh4JnJrk1L5sfm+5PT/J+0c2cSQLL07MB/5rknqN2a8TqgPOBFbvi7YDPtcXfQ94SJJVq+rKqjq7r3sDcCGw2tA6h/TXhwDbD6UfXlV/rKqLgYuAjXsZv+155gHLANXTL6mqc4HbRqsM3L/nXRZYmnZh5w56S/pWwFFj6lS0ixsBlqNdeLlljsP1BmC/qvpjr+PV/fl3VXUaLUAfK8nawMOAb8+S53VJzkpy1q03XT9HVSRJkiRpvDkD86r6E/AWWoC+d1XdTGtJPXuOVXdKsgD4BbAS8KWefgAtgFyP1vr70TnS3wFs01tKX9q3/w4WtsgfMdmuzml74KSq+l/gukFX8JlU1UeBK2gttVsmeSSttXcrWov+Rkm2H1rlKOBl/fVLWHg8Fkvvwv5q4KSetBpw2VCWy1kYgA/WWRN4KnBGT3p4VV3Z9+dKWiA6Z1lJTgauBm5gYRA9VlWdDpwKXNkfJ1fVhTNkXxn4TVUNAu7h7R4ArEs75ucBe1XV4CJAAV9JG1LxuqHyHg88u3eN/2aSjWar64j5tHOsZtm3A6tqw6racKkHrrAIRUuSJEnSQpNO/vZCWlD15HELkxzTW4mHu/0e0bsDP4IWSL2lpz+Thd2dP09rhZ8t/TvAwUleCwy6U98V5gOH99eH9/eLYiPgG1V1TQ8sDwWeM7T8OuDXSXamtVrfdCfr+wngW1U1aNEdN277z0Fl7xFwNO3iym/H5B02a1lVtQ2wKq0FfKtZC0oeRwuoV6cF2Vulz0WwiNvdBlhA66WwAXBAkgf3ZZtW1dNo5+meQ+XPA1akdeF/C3DkbOPbR+zMYvZqkCRJkqRFMWdgnmQD4Hm04ObNSVYFLgD+3KJcVTvQxhevNLp+b3H8ErcPUm+XZbb0qno98HZgDWBBkpXnqvOi6mVuBXw6ySW0IG6nHsTdwu2P0/1nKmaCTR0BfJw7GfAleSdtfPffDyVfTjtGA6vTWpcHretHA4dW1fDFk6v650l/vnqusgaq6g/AcbRu77PZAfheVd1YVTcCJ9LOpXGupXXBnzdmu7sBX+hd9S8CLqaNtaeqrujPVwPH0Lvd9/0YrHMmrZv9KnPUlyTrA/Oq6vtz5ZUkSZKkO2uuWdlDm0Rs76q6FNifNoHWYcCmSV46lH22SdA2A37aX3+XhWO4X0WbbG3G9CSPraozquodtMBtDVoX6uXn3LvJvYLWjf7RVbVmVa1BC/w2o00A9sQkyyZZAdh6aL3hepwBbJ5klT5R2nzgmyPbOQb4AHDy4lY0yR601uP5Q125oQXJr0mzCXB9VV3ZP8PPABdW1YdGijsO2KW/3gU4dih9577Pa9EmlDszyXJDgfw82gR8P5qjypfSjsu8foFgc1qPgTvoF3FOpX0eo3W6lH7skzwceALwsyQPSrJ8T38Q8Hzg/L7OF+kt+kkeTxvnfu0c9YU7MQeAJEmSJC2qeXMsfy1waVV9tb//BK1lfGPgxcCH+u2krqIFqe8dWnenJJvRgv/L+3rQZko/KMlbgGtoLaGzpe/fJ+IKbebwc2hB2r59DPv7xo0zT/II4CzgwcBtffK6J87QjXs+C2eeHzga+KuqekOSI4FzgZ8APxjKcyBwYpIr+zjzt9ICywAnVNWxwwX2ydfe3+s3Woetk1w+9H7HPj571KdoFwtO72V8oareDZxAC5QvonWTHxy/TWlj0c/rxwvgbVV1Qt/nI5PsTjumO/Z6XtD3+Ye0HgN7VtWtPfA9LsmytGEFX+/1oY/fPobWdfwlSf61z4R+FC04Po/WC+KkqpptfP0+wOFJ3ks71p/p6e+hDWk4j3Z896mqa5M8BjimH4t5wGFVNRh3fxDtnDqfdmeAXQZjxnvPiAcDy/S5AJ5fVT/s672SuWf9lyRJkqQlIrPMbSVpQm/45/fVibeuN+1qSJIkSQIu2W/baVdhnBmHP086+ZskSZIkSboLzNWV/R4vyW7AXiPJ36mqPcfkXZnWHX7U1lX1q7uifosryTHAWiPJ+1TVYo9Pvye5r++fJEmSJE3qXh+YV9Vngc9OmPdXtFtt3eP1me7vs+7r+ydJkiRJk7IruyRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVNkYC5JkiRJ0hTNm3YFpPuCp6y2Ap/8222nXQ1JkiRJ90K2mEuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRF86ZdAem+4LxfXM+a+x4/7WpIkrTILtlv22lXQZL+4tliLkmSJEnSFBmYS5IkSZI0RZ6viKQAACAASURBVAbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVNkYC5JkiRJ0hQZmEuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUzRqYJ1kjycVJVurvV+zvH51k7SRfTvLTJN9PcmqS5/R8uya5JsmCJBckOSrJA5dUpZNskORFc+R5VZJz++O7SdafoNwdklSSdSasx96T7FeSS5J8eyRtQZLz++stknx5wm0emuTHSc5PclCSpXt6knw0yUV9n5/W09fon82F/bPYa6islZJ8NclP+vOKQ8ve2sv6cZJthtJPSnJOL+tTSZbq6c9JcnaSW5K8YqTOH+j5L+x1zCz7t1aSM3qdjkiyTE9fIcmXhra928jxPa8f07NGyntj34cLknygp63cj8mNSQ4Yyrt8L2PwuDbJRyb5XCRJkiRpcc0amFfVZcAngf160n7AgcBVwPHAgVX12Kp6OvBG4DFDqx9RVRtU1ZOAm4GdlmC9NwBmDcyBi4HNq2o94D293nOZD5wG7DxhPfYGJr3gsHySNQCSrDvhOuMcCqwDPAV4ALBHT38hsHZ/vI72uQHcAvxDVa0LbALsmeSJfdm+wClVtTZwSn9PX74z8CTgBcAnBgE48MqqWh94MvBQYMeefimwK3DYcGWTPAvYFFivr7MRsPks+/d+4MO9Tr8Gdu/pewI/7NveAvjgIGjvtuzn24ZD294S2A5Yr5+H/68v+gPwL8A/Dm+4qm7oZWxQVRsAPwe+MEtdJUmSJOlOm6Qr+4eBTZLsDWwGfBB4FXB6VR03yFRV51fVwaMrJ5kHPIgWZNFb20/prbqnJHnUHOk79tbhc5J8qwdj7wZ26q2aYwP+qvpuVf26v/0esPpsO5lkOVoAuTtDgfloa3aSA3qPgDcBjwROTXJqXza/t9yen+T9I5s4koUXJ+YD/zVbfWZSVSdUB5w5tF/bAZ/ri74HPCTJqlV1ZVWd3de9AbgQWG1onUP660OA7YfSD6+qP1bVxcBFwMa9jN/2PPOAZYDq6ZdU1bnAbaNVBu7f8y4LLE27sHMHvSV9K+CoMXUq2sWNAMsB19EuOszmDcB+VfXHXser+/Pvquo0WoA+VpK1gYcB354pjyRJkiQtCXMG5lX1J+AttAB976q6mdaSevYcq+6UZAHwC2Al4Es9/QBaALkerfX3o3OkvwPYpreUvrRv/x0sbJE/YoL93B04cY482wMnVdX/AtcNuoLPpKo+ClxBa6ndMskjaa29W9Fa9DdKsv3QKkcBL+uvX8LC47FYehf2VwMn9aTVgMuGslzOwgB8sM6awFOBM3rSw6vqyr4/V9IC0TnLSnIycDVwAwuD6LGq6nTgVODK/ji5qi6cIfvKwG+qahBwD2/3AGBd2jE/D9irqgYXAQr4StqQitcNlfd44Nm9a/w3k2w0W11HzKedYzVThiSvS3JWkrNuven6RShakiRJkhaadPK3F9KCqiePW5jkmN5KPNzt94jeHfgRtEDqLT39mSzs7vx5Wiv8bOnfAQ5O8lpg0J16Yr078+7APnNknQ8c3l8f3t8vio2Ab1TVNT2wPBR4ztDy64BfJ9mZ1mp90yKWP+oTwLeqatCiO27c9p+Dyt4j4GjaxZXfjsk7bNayqmobYFVaC/hWsxaUPI4WUK9OC7K3Sp+LYBG3uw2wgNZLYQPggCQP7ss2raqn0c7TPYfKnwesSOvC/xbgyNnGt4/YmTl6NVTVgVW1YVVtuNQDV5iwWEmSJEm6vTkD8yQbAM+jBTdvTrIqcAHw5xblqtqBNr54pdH1e4vjl7h9kHq7LLOlV9XrgbcDawALkqw8V52H6r4e8Glgu6r61Sz5VqYFmJ9OcgktiNupB3G3cPvjdP+ZipmgSkcAH2cxu7H/eUPJO2nju/9+KPly2jEaWJ3WujxoXT8aOLSqhi+eXNU/T/rz1XOVNVBVfwCOo3V7n80OwPeq6saqupHWc2GTGfJeS+uCP2/MdncDvtC76l9Em0NgnV6XK/rz1cAx9G73fT8G65xJ62a/yhz1JW2iwHlV9f258kqSJEnSnTXXrOyhTSK2d1VdCuxPm0DrMGDTJC8dyj7bJGibAT/tr7/LwjHcr6JNtjZjepLHVtUZVfUOWuC2Bq0L9fJz1P1RtIm7Xt27p8/mFbRu9I+uqjWrag1a4LcZbQKwJyZZNskKwNZD6w3X4wxg8ySr9InS5gPfHNnOMcAHgJPnqM9s+7UHrfV4/lBXbmhB8mvSbAJcX1VX9s/wM8CFVfWhkeKOA3bpr3cBjh1K37nv81q0CeXOTLLcUCA/jzYB34/mqPKltOMyr18g2JzWY+AO+kWcU2mfx2idLqUf+yQPB54A/CzJg5Is39MfBDwfOL+v80V6i36Sx9PGuV87R33hTswBIEmSJEmLat4cy18LXFpVX+3vP0FrGd8YeDHwoX47qatoQep7h9bdKclmtOD/8r4ewJuAg5K8BbiG1hI6W/r+fSKu0GYOP4cWpO3bx7C/b4Zx5u+gjVn+RO+9fMvwjN0j5rNw5vmBo4G/qqo3JDkSOBf4CfCDoTwHAicmubKPM38rLbAMcEJVHTtcYJ987f0AY3pUb53k8qH3O/bx2aM+RbtYcHov4wtV9W7gBFqgfBGtm/zg+G1KG4t+Xj9eAG+rqhP6Ph+ZZHfaMd2x1/OCvs8/pPUY2LOqbu2B73FJlqUNK/h6rw99/PYxtK7jL0nyr30m9KNowfF5tF4QJ1XVbOPr9wEOT/Je2rH+TE9/D21Iw3m047tPVV2b5DHAMf1YzAMOq6rBuPuDaOfU+bQ7A+wyGDPee0Y8GFimzwXw/Kr6YV/vlcw9678kSZIkLRGZZW4rSRN6wz+/r068db1pV0OSpEV2yX7bTrsKkvSXYsbhz5NO/iZJkiRJku4Cc3Vlv8dLshuw10jyd6pqzzF5V6Z1hx+19WyTw01DkmOAtUaS96mqxR6ffk9yX98/SZIkSZrUvT4wr6rPAp+dMO+vaLfausfrM93fZ93X90+SJEmSJmVXdkmSJEmSpsjAXJIkSZKkKTIwlyRJkiRpigzMJUmSJEmaIgNzSZIkSZKmyMBckiRJkqQpMjCXJEmSJGmKDMwlSZIkSZoiA3NJkiRJkqbIwFySJEmSpCkyMJckSZIkaYrmTbsC0n3BU1ZbgU/+7bbTroYkSZKkeyFbzCVJkiRJmiIDc0mSJEmSpsjAXJIkSZKkKTIwlyRJkiRpigzMJUmSJEmaIgNzSZIkSZKmyMBckiRJkqQpMjCXJEmSJGmKDMwlSZIkSZqiedOugHRfcN4vrmfNfY+fdjUkSQLgkv22nXYVJEmLwBZzSZIkSZKmyMBckiRJkqQpMjCXJEmSJGmKDMwlSZIkSZoiA3NJkiRJkqbIwFySJEmSpCkyMJckSZIkaYoMzCVJkiRJmiIDc0mSJEmSpsjAXJIkSZKkKTIwlyRJkiRpigzMJUmSJEmaIgNzSZIkSZKmyMBckiRJkqQpMjCXJEmSJGmKDMwlSZIkSZoiA3NJkiRJkqZo1sA8yRpJLk6yUn+/Yn//6CRrJ/lykp8m+X6SU5M8p+fbNck1SRYkuSDJUUkeuKQqnWSDJC+aI892Sc7tdTgryWYTlLtDkkqyzoT12HuS/UpySZJvj6QtSHJ+f71Fki9PuM1Dk/w4yflJDkqydE9Pko8muajv99N6+hr9s7mwfxZ7DZW1UpKvJvlJf15xaNlbe1k/TrLNUPpJSc7pZX0qyVI9/TlJzk5yS5JXjNT5Az3/hb2OmWX/1kpyRq/TEUmW6ekrJPnS0LZ3Gzm+5w0+65Hy3tj34YIkH+hpK/djcmOSA4byLt/LGDyuTfKRST4XSZIkSVpcswbmVXUZ8Elgv560H3AgcBVwPHBgVT22qp4OvBF4zNDqR1TVBlX1JOBmYKclWO8NgFkDc+AUYP2q2gD4G+DTE5Q7HzgN2HnCeuwNTHrBYfkkawAkWXfCdcY5FFgHeArwAGCPnv5CYO3+eB3tcwO4BfiHqloX2ATYM8kT+7J9gVOqam3a8dq31++JtGPwJOAFwCcGATjwyqpaH3gy8FBgx55+KbArcNhwZZM8C9gUWK+vsxGw+Sz7937gw71OvwZ27+l7Aj/s294C+OAgaO+27OfbhkPb3hLYDlivn4f/ry/6A/AvwD8Ob7iqbuhlbNDPm58DX5ilrpIkSZJ0p03Slf3DwCZJ9gY2Az4IvAo4vaqOG2SqqvOr6uDRlZPMAx5EC7Lore2n9FbdU5I8ao70HXvr8DlJvtWDsXcDO/VWzbEBf1XdWFXV3z4IqHH5huq5HC2A3J2hwHy0NTvJAb1HwJuARwKnJjm1L5vfW27PT/L+kU0cycKLE/OB/5qtPjOpqhOqA84EVu+LtgM+1xd9D3hIklWr6sqqOruvewNwIbDa0DqH9NeHANsPpR9eVX+sqouBi4CNexm/7XnmAcvQj2tVXVJV5wK3jVYZuH/PuyywNO3Czh30lvStgKPG1KloFzcCLAdcR7voMJs3APtV1R97Ha/uz7+rqtNoAfpYSdYGHgZ8e5Y8r0vrjXHWrTddP0dVJEmSJGm8OQPzqvoT8BZagL53Vd1Ma0k9e45Vd0qyAPgFsBLwpZ5+AC2AXI/W+vvROdLfAWzTW0pf2rf/Dha2yB8xUwXSuqb/iNa6/zdz1Hd74KSq+l/gukFX8JlU1UeBK2gttVsmeSSttXcrWov+Rkm2H1rlKOBl/fVLho7HYuld2F8NnNSTVgMuG8pyOQsD8ME6awJPBc7oSQ+vqiv7/lxJC0TnLCvJycDVwA0sDKLHqqrTgVOBK/vj5Kq6cIbsKwO/qapBwD283QOAdWnH/Dxgr6oaXAQo4CtpQypeN1Te44Fn967x30yy0Wx1HTGfdo7NeEGnqg6sqg2rasOlHrjCIhQtSZIkSQtNOvnbC2lB1ZPHLUxyTG8lHu72e0TvDvwIWiD1lp7+TBZ2d/48rRV+tvTvAAcneS0w6E49kao6pqrWoQXd75kj+3zg8P768P5+UWwEfKOqrumB5aHAc4aWXwf8OsnOtFbrmxax/FGfAL5VVYMW3XHjtv8cVPYeAUfTLq78dkzeYbOWVVXbAKvSWsC3mrWg5HG0gHp1WpC9VfpcBIu43W2ABbReChsAByR5cF+2aVU9jXae7jlU/jxgRVoX/rcAR842vn3EzixmrwZJkiRJWhRzBuZJNgCeRwtu3pxkVeAC4M8tylW1A2188Uqj6/cWxy9x+yD1dllmS6+q1wNvB9YAFiRZea46j6nDt4DHJlll3PJe5lbAp5NcQgvidupB3C3c/jjdf4bNTBLwHQF8nDsZ8CV5J218998PJV9OO0YDq9Nalwet60cDh1bV8MWTq/rnSX++eq6yBqrqD8BxtG7vs9kB+F4fWnAjcCLtXBrnWloX/Hljtrsb8IXeVf8i4GLaWHuq6or+fDVwDL3bfd+PwTpn0rrZjz0HhiVZH5hXVd+fK68kSZIk3Vlzzcoe2iRie1fVpcD+tAm0DgM2TfLSoeyzTYK2GfDT/vq7LBzD/SraZGszpid5bFWdUVXvoAVua9C6UC8/R90fN2gd7d3SlwF+NUP2V9C60T+6qtasqjVogd9mtAnAnphk2SQrAFsPrTdcjzOAzZOs0idKmw98c2Q7xwAfAE6ere5z7NcetNbj+UNduaEFya9JswlwfVVd2Y/BZ4ALq+pDI8UdB+zSX+8CHDuUvnPf57VoE8qdmWS5oUB+Hm0Cvh/NUeVLacdlXr9AsDmtx8Ad9Is4p9I+j9E6XUo/9kkeDjwB+FmSByVZvqc/CHg+cH5f54v0Fv0kj6edA9fOUV+4E3MASJIkSdKimjfH8tcCl1bVV/v7T9BaxjcGXgx8qN9O6ipakPreoXV3SrtF2f1oLZe79vQ3AQcleQtwDa0ldLb0/ftEXKHNHH4OLUjbt49hf98M48xfTgtU/wT8HthplvHC81k48/zA0cBfVdUbkhwJnAv8BPjBUJ4DgROTXNnHmb+VFlgGOKGqjh0usE++9n6AMT2qt05y+dD7Hfv47FGfol0sOL2X8YWqejdwAi1QvojWTX5w/DaljUU/rx8vgLdV1Ql9n49MsjvtmO7Y63lB3+cf0noM7FlVt/bA97gky9KGFXy914c+fvsYWtfxlyT51z4T+lG04Pg8Wi+Ik6pqtvH1+wCHJ3kv7Vh/pqe/hzak4Tza8d2nqq5N8hjgmH4s5gGHVdVg3P1BtHPqfNqdAXYZnAO9Z8SDgWX6XADPr6of9vVeydyz/kuSJEnSEpFZ5raSNKE3/PP76sRb15t2NSRJAuCS/baddhUkSXc04/DnSSd/kyRJkiRJd4G5urLf4yXZDdhrJPk7VbXnmLwr07rDj9q6qmYafz4VSY4B1hpJ3qeqFnt8+j3JfX3/JEmSJGlS9/rAvKo+C3x2wry/ot1q6x6vz3R/n3Vf3z9JkiRJmpRd2SVJkiRJmiIDc0mSJEmSpsjAXJIkSZKkKTIwlyRJkiRpigzMJUmSJEmaIgNzSZIkSZKmyMBckiRJkqQpMjCXJEmSJGmKDMwlSZIkSZoiA3NJkiRJkqZo3rQrIN0XPGW1Ffjk32477WpIkiRJuheyxVySJEmSpCkyMJckSZIkaYoMzCVJkiRJmiIDc0mSJEmSpsjAXJIkSZKkKTIwlyRJkiRpigzMJUmSJEmaIgNzSZIkSZKmyMBckiRJkqQpmjftCkj3Bef94nrW3Pf4aVdDkvQX6JL9tp12FSRJd5It5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVNkYC5JkiRJ0hQZmEuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRN0d0emCd5eJLDkvwsyfeTnJ5khyRbJLk+yYIk5yb5WpKH9XV2TVJJth4qZ4ee9ooJtvmxJDdOWL9jk5w+Yd41k/zVBPm26HXdfSjtqT3tH/v7gyfclw36MbugH6edhpatleSMJD9JckSSZXr6q3rec5N8N8n6Q+u8IMmPk1yUZN+h9JWSfLWX9dUkK/b0jftntCDJOUl2GFrn/yS5bPRYJ3lUklOT/KDX4UVz7OMufbs/SbLLUPrWSc7u2z4tyeOGju/1Q/V6x9A6D0lyVJIfJbkwyTN7+o79GN6WZMOh/K8aKmdBX77BXJ+LJEmSJC2uuzUwTxLgi8C3quoxVfV0YGdg9Z7l21W1QVWtB/wPsOfQ6ucB84fe7wycM8E2NwQeMmH9HgI8DXhIkrUmWGVNYM7AvDsP2Gno/UT1H+Mm4DVV9STgBcBHer0B3g98uKrWBn4NDC4EXAxs3o/re4ADAZIsBXwceCHwRGB+kif2dfYFTullndLfA5wPbFhVG/Tt/3uSeX3Zl4CNx9T57cCRVfXUvt+fmGnnkqwEvBN4Ri/rnYOLAsAngVf1bR/Wyx0YnDsbVNW7h9L/DTipqtYB1gcuHNqPlwHfGt5+VR06KAd4NXBJVS2Yqb6SJEmSdGfd3S3mWwE3V9WnBglV9fOq+thwph7AL08LLge+DWycZOkkywGPA2YNmHrguT/wTxPW7+W04PJwWgA5KOd2rdlDLcL7Ac/uLatvTnL/JJ9Ncl5vHd5yqOxLgfv3HgOhBbUnTlivP6uq/62qn/TXVwBXAw/tZW4FHNWzHgJs3/N9t6oGx/J7LLwQsjFwUVX9rKpu7vu9XV+2XS9jtKybquqWnn5/oIbq9r2qunJctYEH99crAFfMsovbAF+tqut6nb9KO1aLWg5JHgw8B/hMr9/NVfWb/vrCqvrxbOvTLgT91yzlvy7JWUnOuvWm6+coSpIkSZLGu7sD8ycBZ8+y/NlJFtCC2OcCBw0tK+BrtMBtO+C4Cbb3d8BxMwSL4wwCsf/i9q3zM9mXhS21H6a38FfVU/r6hyS5/1D+o4AdgWfRjsMfJ6zXWEk2BpYBfgqsDPxmKGi+HFhtzGq7s/CCwGrAZUPLhtd5+OC49eeHDW33GUkuoPUCeP3QNmfyLuCvk1wOnAC8cZa8s9VpD+CEXs6raRdGBp7Zu9afmORJPe0xwDXAZ/uFkk8nedAcdR22E7ME5lV1YFVtWFUbLvXAFRahWEmSJElaaKqTvyX5eA+m/qcnDYLcNYDPAh8YWWXQkr0zswRMvexH0oLgj82Wbyj/w2mt8KdV1f8CtyR58uR7A8BmwOcBqupHwM+Bxw8tP7LXadaW2Anru2rf1m5VdRuQMdlqZJ0taYH5PoOkudYZp6rO6F3pNwLeOnLxYZz5wMFVtTrwIuDzSWY692ar05uBF/VyPgt8qKefDTy6qtanfd5f7OnzaEMTPtm70f+OhV3yZ5XkGcBNVXX+JPklSZIkaXHd3YH5BbRACYCq2hPYGnjomLzH0bohM5T/TODJwCo9eJ7NU2mB9kVJLgEemOSiWfLvBKwIXNzzr8nC7uy30I9V7zK+zAxljAsqh+v/S+BPwPNo47YXS++ifTzw9qr6Xk++ljY2fjDee3WGunonWQ/4NLBdVf2qJ18OrDFU9PA6V/Xgf3AR4Oox+3MhLdid6wLG7rSLElTV6bQu8KvMkHdsnZI8FFi/qs7o6UfQeh5QVb+tqhv76xOApZOs0su6fGidoxg6/+Yw58UfSZIkSVoS7u7A/Ou0cdZvGEp74Ax5N6N10R71VuBtc22oqo6vqkdU1ZpVtSat9fNxs6wyH3jBUP7BxHQAl/T30LrRL91f30AbCz/wLeBVAEkeDzwKGB3H/A5gn6q6da59GCdtpvVjgM9V1X8P0quqgFOBwVj4XYBj+zqPAr4AvHrkgsb/AGunzea+TN/fwRCB43oZo2WtNQj+kzwaeALt+MzmUtoFGJKsSwvMr5kh78nA85Os2Cd9e35P+zWwQj+u0C5uXNjLfES/YDLo3n8/4Ff9QshlSZ7Q19ka+OEcdaW35u9I66EhSZIkSXepeXNnWXKqqpJsD3w4yT/RgrPfsbBr9WCMeYDraWOKR8tY5AnT5pJkTVoQPWh9pqouTvLb3qX5P4Bjk5xJa+n+Xc92Lq3L+znAwbTZxj+V5DxaK/uuVfXHHjMOyv3uLFX59yQf6a8vq6pnjsnzSlpPgpWT7NrTdu0zh+8DHJ7kvcAP6JOe0S4GrAx8otfllj42+pYkf0cLfJcCDqqqC/o6+wFHpt3i7VJaoArtgsm+Sf4E3Ab8bVVd24/jB2iz1D+wjwP/dFW9C/gH4D+SvJnWLX3XfiHhDqrquiTvoV00AHh3VV3Xy38tcHSS22iB+t/0PK8A3pDkFuD3wM5D5b8ROLRfePgZsFsvawdat/eHAscnWVBV2/R1nkNraf/ZuDpKkiRJ0pKUGeIjSYvgDf/8vjrx1vWmXQ1J0l+gS/bbdtpVkCRNZsahz1Od/E2SJEmSpL90d2tX9rtKkmOAtUaS96mqk8fk3Q3YayT5O30iunuMJE+hz/A+5I9V9Yxp1GdJu6/vnyRJkiRN6j4RmFfVDouQ97O0W23do1XVecAG067HXeW+vn+SJEmSNCm7skuSJEmSNEUG5pIkSZIkTZGBuSRJkiRJU2RgLkmSJEnSFBmYS5IkSZI0RQbmkiRJkiRNkYG5JEmSJElTZGAuSZIkSdIUGZhLkiRJkjRFBuaSJEmSJE2RgbkkSZIkSVM0b9oVkO4LnrLaCnzyb/9/9u4//vO5zvf/7b6GtSq/ZVvU6KQfhFFitZEoWc5ZplhmrbDaTvqxe2x7MrZOOtm+qfakThsqhboopESLHMl3KT8ihjFNRUxIJakpdDA8zh+v57t5+ezn14wxL0236+Uyl3m/n6/n8/l6vl6fzz/35/P5en32HnoYkiRJkn4HuWIuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oBmDD0AaVUw/0eLmTn3/KGHIUmPsei4vYcegiRJmgZXzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRrQSg/mSTZO8rkktyb5dpIrk8xOsmuSxUnmJbkxydeSPL21OTRJJdm918/sVrbfJOf6VJIbWn9nJ3nqNMZ3bpIrp3ktM5P81TTq7drGenivbLtW9o/t+6mTXUuv3ax2zxa06zqgd2zzJFcnuTnJmUnWaOUHtbo3Jrkiyba9Nnsm+V6SW5LM7ZWvn+Ti1tfFSdZr5Tu0n9G8dm9n99q8N8kdSe4bM+ZnJrk0yfVtDHtNcY2HtPPenOSQXvnuSa5r5/5Gkuf07u/i3rje1WuzbvvZfzfJwiQ7tfL92z18NMn2vfoH9fqZ147PmurnIkmSJEnLa6UG8yQBvgxcVlXPrqoXAwcCm7Yql1fVrKraBrgGeHOv+XxgTu/7gcANU5zyyKratvV3O/CWKca3LvAiYN0km0/jkmYCUwbzZj5wQO/7dMY/ngeA11XVVsCewIfbuAHeDxxfVVsAvwBGEwG3AS9v9+FY4BMASVYDPgb8ObAlMCfJlq3NXOCS1tcl7TvATcD2VTWrnf/jSWa0Y18BdhhnzO8Ezqqq7dp1nzDRxSVZHzgG2LH1dcxoUgA4ETionftzrd+R0e/OrKp6T6/8I8BXq+r5SqvjywAAIABJREFUwLbAwt51vAa4rH/+qjp91A9wMLCoquZNNF5JkiRJerxW9or5bsBDVXXSqKCqflhVH+1XagH+aXThcuRyYIckq7eV7+cAkwamqvpVr78/AmqK8b2WLlyeQRcgR+N5zGp2b0X4OGDntrJ6ZJI1k5ySZH5bHX5Fr+/bgTXbjoHQhdoLpxjPeNf0/aq6uX2+C7gb2Kj1uRtwdqt6GrBvq3dFVY3u5VUsnQjZAbilqm6tqofade/Tju3T+hjb1wNVtaSVr0nvnlbVVVX14/GGDazdPq8D3DXJJb4auLiq7m1jvpjuXi1rPyRZG9gF+FQb30NV9cv2eWFVfW+y9nQTQZ+fpP83JLk2ybWPPLB4iq4kSZIkaXwrO5hvBVw3yfGdk8yjC7GvBD7dO1bA1+iC2z7AedM5YZJTgJ8Azwc+OkX1URD7PI9dnZ/IXJau1B5PW+Gvqq1b+9OSrNmrfzawP/BSuvvw4HSuYSJJdgDWAH4AbAD8shea7wQ2GafZ4SydENgEuKN3rN9m41HIbv8/vXfeHZMsoNsF8MbeOSfybuCvk9wJXAC8dZK6k43p9cAFrZ+D6SZGRnZqW+svTLJVK3s28DPglDZRcnKSp0wx1r4DmCSYV9Unqmr7qtp+tbXWWYZuJUmSJGmpQV/+luRjLUxd04pGIXcz4BTgA2OajFayD2SSwNRXVYcBf0K3hfmAieol2ZhuFf4bVfV9YEmSFy7TBcHLgM+2834X+CHw3N7xs+iC+aQrsdOR5BntXIdV1aNAxqlWY9q8gi6YHzUqmqrNeKrq6raV/iXA0WMmH8YzBzi1qjYF9gI+m2Si373JxnQksFfr5xTgQ638OuBZVbUt3eTLl1v5DLpHE05s2+jvZ+mW/Ekl2RF4oKpumk59SZIkSVpeKzuYL6ALSgBU1ZuB3YGNxql7Ht02ZHr1vwW8ENiwhedpqapHgDPptqpP5ABgPeC2JIvonh8fbWdfQrtXbcv4GhP0MV6o7I/jJ8DDwKvontteLm2L9vnAO6vqqlZ8D92z8aPnvTelt9U7yTbAycA+VfXzVnwnsFmv636bn7bwP5oEuHuc61lIF3anmsA4nG5Sgqq6km4L/IYT1B13TEk2Aratqqtb+Zl0Ow+oql9V1X3t8wXA6kk2bH3d2WtzNr3fvylMe/JHkiRJkh6PlR3Mv073nPURvbK1Jqj7Mrot2mMdDfzTVCdKZ/TW7gD/BfjuJE3mAHtW1cyqmgmMXkwHsKh9h24b/ert86/pnoUfuQw4qJ3zucAzgbHPMb8LOKpNFiyzdG9aPwf4TFV9YVReVQVcCoyehT8EOLe1eSbwJeDgMRMa1wBbpHub+xrtekePCJzX+hjb1+aj8J/kWcDz6O7PZG6nm4AhyQvogvnPJqh7EbBHkvXaS9/2aGW/ANZp9xW6yY2Frc8/bj/j0fb+PwB+3iZC7kjyvNZmd+A7U4yVtpq/P90ODUmSJEl6Qs2YusqKU1WVZF/g+CRvpwtn97N0a/XoGfMAi+meKR7bx3RfmBa6Z7zXbp9vAI4Yt2Iyky5Ej1afqarbkvyqbWn+JHBukm/RrXTf36rdSLfl/QbgVLq3jZ+UZD7dKvuhVfVgy4yjfq+YZMwfT/Lh9vmOqtppnDp/SbeTYIMkh7ayQ9ubw48Czkjyz8D1tJee0U0GbACc0MaypD0bvSTJW+iC72rAp6tqQWtzHHBWuj/xdjtdUIVuwmRukoeBR4E3VdU97T5+gO4t9Wu158BPrqp3A28DPpnkSLpt6Ye2iYT/oKruTXIs3aQBwHuq6t7W/98CX0zyKF1Q/5tWZz/giCRLgN8AB/b6fytwept4uBU4rPU1m27b+0bA+UnmVdWrW5td6Fbabx1vjJIkSZK0ImWCfCRpGRzxjvfVhY9sM/QwJOkxFh2399BDkCRJS0346POgL3+TJEmSJOn33Urdyv5ESXIOsPmY4qOq6qJx6h4G/P2Y4m+2F9E9aSTZmvaG954Hq2rHIcazoq3q1ydJkiRJ07VKBPOqmr0MdU+h+1NbT2pVNR+YNfQ4niir+vVJkiRJ0nS5lV2SJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEka0IyhByCtCrbeZB1OfNPeQw9DkiRJ0u8gV8wlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRqQwVySJEmSpAEZzCVJkiRJGtCMoQcgrQrm/2gxM+eeP/QwJA1g0XF7Dz0ESZL0O84Vc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgY0SDBPsnGSzyW5Ncm3k1yZZHaSXZP82zj1//8ktydJr+zLSe6b4jxfTfLL8fqcoP5GSR5O8l+nWX/fJFtOo96pSR5I8rRe2UeSVJIN2/dJr6XX7h+SfCfJjUkuSfKs3rFDktzc/h3SKz89yfeS3JTk00lWb+VJ8r+T3NL6e1GvzZ6tzS1J5vbKj2115yX5P0n+pJVvkOTSJPcl+dcxY56TZH5r99XRNU9wfX+Y5Mx23quTzOwd+0CSBUkWtnGnd39va2Oal2RWr82urWxBkn/vlX86yd1Jbhpz/jN7/SxKMm86PxdJkiRJWl4rPZi3MPVl4LKqenZVvRg4ENh0iqa/BP6s9bEu8IxpnO6DwMHLMLz9gauAOdOsvy8wZTBvbgH2AUjyB8ArgB8tw9hGrge2r6ptgLOBD7Q+1weOAXYEdgCOSbJea3M68Hxga+CPgNe38j8Htmj/3gCc2PpaDfhYO74lMKc3AfHBqtqmqmYB/wa8q5X/X+B/AP/YH2ySGcBHgFe0Md8IvGWS6zsc+EVVPQc4Hnh/6+eldD//bYAXAi8BXt5r99+ralb7N6+1WRc4AfiLqtqK7uc7ciqw59iTV9UBo36ALwJfmmSskiRJkvS4DbFivhvwUFWdNCqoqh9W1UenaHcGXYAHeA3TCExVdQnw62UY2xzgbcCmSTYZFfZXs5Ps11ZoXwr8BfDBtrr6n5LMSnJVWxk+pxeMAT4PHNA+7wp8E1iyDGMbXdOlVfVA+3oVSyc0Xg1cXFX3VtUvgItpwbOqLqgG+FavzT7AZ9qhq4B1kzyDLtjfUlW3VtVDdPd+n9bXr3rDeQpQrfz+qvoGXUDvS/v3lDYpszZw1ySXuA9wWvt8NrB7a1fAmsAawB8CqwM/neJ2/RXwpaq6vY3x7tGBqroMuHeihu2cf0n3c5uozhuSXJvk2kceWDzFUCRJkiRpfEME862A65aj3SXALm0190DgzBU5qCSbAX9cVd8CzmJpiB5XVV0BnMfSldofAJ8Bjmorw/PpVrBHbgY2amF9Dl3YfbwOBy5snzcB7ugdu7OV/Vbbwn4w8NUp2kzaV5L3JrkDOIilK+bjqqqHgSPo7sdddCvwn5qkyW/PXVVLgMXABlV1JXAp8OP276KqWthr9942IXJ8kj9sZc8F1kv3KMS3k7xusrGOsTPw06q6eZJr+0RVbV9V26+21jrL0LUkSZIkLTX4y9+SfCzJDUmumaLqI8A36ALzH1XVohU8lAPpAjl0oXm629kBSLIOsG5VjZ5jPg3YZUy1L7Xz7AhcvvxDhSR/DWxPt10fulXpsWrM9xPoHiEYnXuiNpP2VVXvqKrN6LbIT7YtfTQZcASwHfAndFvZj56syXjnTvIc4AV0q/2bALslGd3fo+m26r8EWB84qpXPAF4M7E23o+B/JHnuZOPtmcMkq+WSJEmStKIMEcwXAL99yVhVvRnYHdhoGm3PAD7K0gC9Is0BDk2yiG4lfNskW4yG2au35uM4xxnAsXRbzh9d3k6SvBJ4B92z0w+24juBzXrVNqW3ZTzJMXT3+B96dSZqM2lfPZ8DXjvFcGcBVNUP2lb6s4CXTlL/t+duz6evQ7flfDZwVVXdV1X30e0U+NPW94/bdvwHgVPotuKP+vpq22Z/D3AZsO0U4x2d9zWs4F0ZkiRJkjSeIYL514E1kxzRK1trmm0vB97HCl7JTPI84ClVtUlVzayqme08o2faf5rkBe2lbbN7TX8NPA2gqhYDv0iyczt2MPDvvbq0Z53fQbdyvbxj3Q74OF0ov7t36CJgjyTrte3ye7QykryebsV4zpgJgfOA17W3s/8psLiqfgxcA2yRZPMka7T7cF7ra4te+78AvjvFkH8EbJlkNPHyKmDhJPXPA0ZvlN8P+HoL9LcDL08yo63Cv3zUT3sufvRc+L7A6E3r5wI7tzZr0e1UmOzcI68EvltVd06jriRJkiQ9LjNW9gmrqpLsCxyf5O3Az4D7Wbr9ePck/UC0f78t8C/TPVeSy+m2OD+19Xl4VV00TtU5wDljyr7I0hXuuXRvIL+DLvQ9tdU5A/hkkr+jC5GHACe1EHgrcNjYE1XVxycY7lpjrvtDVfWhcep9sJ3/C+2vhd1eVX9RVfcmOZYuVAO8p6pGLzc7CfghcGVr86Wqeg9wAbAX3RvjHxiNt6qWJHkLXbBfDfh0VS1ofR3XJjIebX2+cTSwtttgbWCN9jPeo6q+k+R/Apclebi1OXSCewDd8+efTXIL3Ur5aHLkbLoXB86n28Hw1ar6Sjt2egv+AeaNxlRVC5N8lW77/KPAyVV1Uxvr5+lewrdhu+/HVNXo2fcDcRu7JEmSpJUkXdaV9Hgc8Y731YWPbDP0MCQNYNFxew89BEmS9LthvPdpAU+Cl79JkiRJkvT7bKVvZV/RkmwNfHZM8YNVteME9c8BNh9TfNQEW9wHk+Qd9LbxN1+oqvcOMZ4VbVW/PkmSJEmart/5YF5V82lv/p5m/dlT1xpeC6irbEhd1a9PkiRJkqbLreySJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgGYMPQBpVbD1Jutw4pv2HnoYkiRJkn4HuWIuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oBmDD0AaVUw/0eLmTn3/KGHIWkZLTpu76GHIEmS5Iq5JEmSJElDMphLkiRJkjQgg7kkSZIkSQMymEuSJEmSNCCDuSRJkiRJAzKYS5IkSZI0IIO5JEmSJEkDMphLkiRJkjQgg7kkSZIkSQMymEuSJEmSNCCDuSRJkiRJAzKYS5IkSZI0IIO5JEmSJEkDMphLkiRJkjQgg7kkSZIkSQMymEuSJEmSNCCDuSRJkiRJAzKYS5IkSZI0oJUezJNsnORzSW5N8u0kVyaZnWTXJIuTzEtyY5KvJXl6a3Nokkqye6+f2a1sv0nO9ZYkt7R6G05zfOcmuXKadWcm+atp1Nu1jeHwXtl2rewf2/dTJ7uWXrtZ7Z4taPfpgN6xzZNcneTmJGcmWaOVH9Tq3pjkiiTb9trsmeR77T7N7ZWvn+Ti1tfFSdZr5Tu0n9G8JDckmd1r894kdyS5b8yYn5nk0iTXtzHsNcU1HtLOe3OSQ3rluye5rp37G0me07u/i3vjelevzbpJzk7y3SQLk+zUyvdv9/DRJNv36h/U62deOz5rqp+LJEmSJC2vlRrMkwT4MnBZVT27ql4MHAhs2qpcXlWzqmob4Brgzb3m84E5ve8HAjdMccpvAq8EfjjN8a0LvAhYN8nm02gyE5gymDfzgQN636cz/vE8ALyuqrYC9gQ+3MYN8H7g+KraAvgFMJoIuA14ebuvxwKfAEiyGvAx4M+BLYE5SbZsbeYCl7S+LmnfAW4Ctq+qWe38H08yox37CrDDOGN+J3BWVW3XrvuEiS4uyfrAMcCOra9jRpMCwInAQe3cn2v9jox+d2ZV1Xt65R8BvlpVzwe2BRb2ruM1wGX981fV6aN+gIOBRVU1b6LxSpIkSdLjtbJXzHcDHqqqk0YFVfXDqvpov1IL8E+jC5cjlwM7JFk9yVOB5wCTBqaqur6qFi3D+F5LFy7PoAuQo/E8ZjW7tyJ8HLBzW1k9MsmaSU5JMr+tDr+i1/ftwJptx0DoQu2FyzC20TV9v6pubp/vAu4GNmp97gac3aqeBuzb6l1RVaN7eRVLJ0J2AG6pqlur6qF23fu0Y/u0Psb29UBVLWnlawLVG9tVVfXj8YYNrN0+rwPcNcklvhq4uKrubWO+mO5eLWs/JFkb2AX4VBvfQ1X1y/Z5YVV9b7L2dBNBn5+k/zckuTbJtY88sHiKriRJkiRpfCs7mG8FXDfJ8Z2TzKMLsa8EPt07VsDX6ILbPsB5T8D4RkHs8zx2dX4ic1m6Uns8bYW/qrZu7U9Lsmav/tnA/sBL6e7Dg49nsEl2ANYAfgBsAPyyF5rvBDYZp9nhLJ0Q2AS4o3es32bjUchu/z+9d94dkyyg2wXwxt45J/Ju4K+T3AlcALx1krqTjen1wAWtn4PpJkZGdmpb6y9MslUrezbwM+CUNlFycpKnTDHWvgOYJJhX1Seqavuq2n61tdZZhm4lSZIkaalBX/6W5GMtTF3TikYhdzPgFOADY5qMVrIPZJLAtJxj2ZhuFf4bVfV9YEmSFy5jNy8DPgtQVd+l20L/3N7xs+iC+aQrsdMc7zPauQ6rqkeBjFOtxrR5BV0wP2pUNFWb8VTV1W0r/UuAo8dMPoxnDnBqVW0K7AV8NslEv3uTjelIYK/WzynAh1r5dcCzqmpb4KN0j0sAzKB7NOHEto3+fpZuyZ9Ukh2BB6rqpunUlyRJkqTltbKD+QK6oARAVb0Z2B3YaJy659FtQ6ZX/1vAC4ENW3hekQ4A1gNuS7KI7vnx0Xb2JbR71baMrzFBH+OFyt+qqp8ADwOvontue7m0LdrnA++sqqta8T10z8aPnvfelN5W7yTbACcD+1TVz1vxncBmva77bX7awv9oEuDuca5nIV3YnWoC43C6SQmq6kq6LfATvYxv3DEl2QjYtqqubuVn0u08oKp+VVX3tc8XAKune9nfncCdvTZn0/v9m8IKn/yRJEmSpPGs7GD+dbrnrI/ola01Qd2X0W3RHuto4J9W9MDoVnX3rKqZVTUTGL2YDmBR+w7dNvrV2+df0z0LP3IZcBBAkucCzwTGPsf8LuCoqnpkeQbZ3rR+DvCZqvrCqLyqCrgUGD0LfwhwbmvzTOBLwMFjJjSuAbZI9zb3Ndr1jh4ROK/1MbavzUfhP8mzgOfR3Z/J3E43AUOSF9AF859NUPciYI8k67WXvu3Ryn4BrNPuK3STGwtbn3/cJkxG2/v/APh5mwi5I8nzWpvdge9MMVbaav7+dDs0JEmSJOkJNWPqKitOVVWSfYHjk7ydLpzdz9Kt1aNnzAMspnumeGwf035hWpK/A94O/DFwY5ILquo/9JlkJl2IHq0+U1W3JflV29L8SeDcJN+iW+m+v1W7kW7L+w3AqXRvGz8pyXy6VfZDq+rBlhlH/V4xyZA/nuTD7fMdVbXTOHX+km4nwQZJDm1lh7Y3hx8FnJHkn4HraS89o5sM2AA4oY1lSXs2ekmSt9AF39WAT1fVgtbmOOCsdH/i7Xa6oArdhMncJA8DjwJvqqp72n38AN1b6tdqz4GfXFXvBt4GfDLJkXTb0g9tEwn/QVXdm+RYukkDgPdU1b2t/78FvpjkUbqg/jetzn7AEUmWAL8BDuz1/1bg9DbxcCtwWOtrNt22942A85PMq6pXtza70K203zreGCVJkiRpRcoE+UjSMjjiHe+rCx/ZZuhhSFpGi47be+ghSJKk3x8TPvo86MvfJEmSJEn6fbdSt7I/UZKcA2w+pvioqrponLqHAX8/pvib7UV0TxpJtqa94b3nwaracYjxrGir+vVJkiRJ0nStEsG8qmYvQ91T6P7U1pNaVc0HZg09jifKqn59kiRJkjRdbmWXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBjRj6AFIq4KtN1mHE9+099DDkCRJkvQ7yBVzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgY0Y+gBSKuC+T9azMy55w89DOlJZ9Fxew89BEmSpCc9V8wlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEkakMFckiRJkqQBGcwlSZIkSRqQwVySJEmSpAEZzCVJkiRJGpDBXJIkSZKkARnMJUmSJEka0EoP5kk2TvK5JLcm+XaSK5PMTrJrksVJ5iW5McnXkjy9tTk0SSXZvdfP7Fa23yTnOj3J95LclOTTSVafxvjOTXLlNK9lZpK/mka9XdtYD++VbdfK/rF9P3Wya+m1m9Xu2YJ2nw7oHds8ydVJbk5yZpI1WvlBre6NSa5Ism2vzZ7tHt2SZG6vfP0kF7e+Lk6yXivfof2M5iW5IcnsXpv3JrkjyX1jxvzMJJcmub6NYa8prvGQdt6bkxzSK989yXXt3N9I8pze/V3cG9e7em3WTXJ2ku8mWZhkp1a+f7uHjybZvlf/oF4/89rxWVP9XCRJkiRpea3UYJ4kwJeBy6rq2VX1YuBAYNNW5fKqmlVV2wDXAG/uNZ8PzOl9PxC4YYpTng48H9ga+CPg9VOMb13gRcC6STafxiXNBKYM5s184IDe9+mMfzwPAK+rqq2APYEPt3EDvB84vqq2AH4BjCYCbgNe3u7rscAnAJKsBnwM+HNgS2BOki1bm7nAJa2vS9p3gJuA7atqVjv/x5PMaMe+AuwwzpjfCZxVVdu16z5hootLsj5wDLBj6+uY0aQAcCJwUDv351q/I6PfnVlV9Z5e+UeAr1bV84FtgYW963gNcFn//FV1+qgf4GBgUVXNm2i8kiRJkvR4rewV892Ah6rqpFFBVf2wqj7ar9QC/NPowuXI5cAOSVZP8lTgOcCkgamqLqgG+BZLJwAm8lq6cHkGXYAcjecxq9m9FeHjgJ3byuqRSdZMckqS+W11+BW9vm8H1mw7BkIXai+cYjzjXdP3q+rm9vku4G5go9bnbsDZreppwL6t3hVVNbqXV/Xuww7ALVV1a1U91K57n3Zsn9bH2L4eqKolrXxNoHpju6qqfjzesIG12+d1gLsmucRXAxdX1b1tzBfT3atl7YckawO7AJ9q43uoqn7ZPi+squ9N1p5uIujzk/T/hiTXJrn2kQcWT9GVJEmSJI1vZQfzrYDrJjm+c5J5dCH2lcCne8cK+BpdcNsHOG+6J21b2A8GvjpF1VEQ+zyPXZ2fyFyWrtQeT1vhr6qtW/vTkqzZq382sD/wUrr78OB0r2E8SXYA1gB+AGwA/LIXmu8ENhmn2eEsnRDYBLijd6zfZuNRyG7/P7133h2TLKDbBfDG3jkn8m7gr5PcCVwAvHWSupON6fXABa2fg+kmRkZ2alvrL0yyVSt7NvAz4JQ2UXJykqdMMda+A5gkmFfVJ6pq+6rafrW11lmGbiVJkiRpqUFf/pbkYy1MXdOKRiF3M+AU4ANjmoxWsg9kksA0jhPots9fPslYNqZbhf9GVX0fWJLkhctwDoCXAZ8FqKrvAj8Ents7fhZdMJ90JXY6kjyjneuwqnoUyDjVakybV9AF86NGRVO1GU9VXd220r8EOHrM5MN45gCnVtWmwF7AZ5NM9Ls32ZiOBPZq/ZwCfKiVXwc8q6q2BT5K97gEwAy6RxNObNvo72fplvxJJdkReKCqbppOfUmSJElaXis7mC+gC0oAVNWbgd2Bjcapex7dNmR69b8FvBDYsIXnKSU5pvX/D1NUPQBYD7gtySK658dH29mX0O5V2zK+xkSnm+wEVfUT4GHgVXTPbS+XtkX7fOCdVXVVK76H7tn40fPem9Lb6p1kG+BkYJ+q+nkrvhPYrNd1v81PW/gfTQLcPc71LKQLu1NNYBxONylBVV1JtwV+wwnqjjumJBsB21bV1a38TLqdB1TVr6rqvvb5AmD1JBu2vu7stTmb3u/fFJZ18keSJEmSlsvKDuZfp3vO+ohe2VoT1H0Z3RbtsY4G/mk6J0vyerqt73PaqvJk5gB7VtXMqpoJjF5MB7CofYduG/3o7e6/pnsWfuQy4KB27ucCzwTGPsf8LuCoqnpkOtcwVro3rZ8DfKaqvjAqb8/RXwqMnoU/BDi3tXkm8CXg4DETGtcAW6R7m/sa7XpHjwic1/oY29fmo/Cf5FnA8+juz2Rup5uAIckL6IL5zyaoexGwR5L12kvf9mhlvwDWafcVusmNha3PP24TJqPt/X8A/LxNhNyR5Hmtze7Ad6YYK201f3+6HRqSJEmS9ISaMXWVFaeqKsm+wPFJ3k4Xzu5n6dbq0TPmARYzzlvUq2pZXph2Et128itbbvvSmDd2A92fPaML0aPVZ6rqtiS/aluaPwmcm+RbdCvd97dqN9Jteb8BOJVuy/xJSebTrbIfWlUPtnOP+r1ikvF+PMmH2+c7qmqncer8Jd1Ogg2SHNrKDm1vDj8KOCPJPwPX0156RjcZsAFwQhvLkvZs9JIkb6ELvqsBn66qBa1weH0EAAAcr0lEQVTNccBZ6f7E2+10QRW6CZO5SR4GHgXeVFX3tPv4Abq31K/VngM/uareDbwN+GSSI+m2pR/aJhL+g6q6N8mxdJMGAO+pqntb/38LfDHJo3RB/W9anf2AI5IsAX4DHNjr/63A6W3i4VbgsNbXbLpt7xsB5yeZV1Wvbm12oVtpv3W8MUqSJEnSipQJ8pGkZXDEO95XFz6yzdDDkJ50Fh2399BDkCRJerKY8NHnQV/+JkmSJEnS77uVupX9iZLkHGDzMcVHVdVF49Q9DPj7McXfbC+ie9JIsjXtDe89D1bVjkOMZ0Vb1a9PkiRJkqZrlQjmVTV7GeqeQventp7Uqmo+MGvocTxRVvXrkyRJkqTpciu7JEmSJEkDMphLkiRJkjQgg7kkSZIkSQMymEuSJEmSNCCDuSRJkiRJAzKYS5IkSZI0IIO5JEmSJEkDMphLkiRJkjQgg7kkSZIkSQMymEuSJEmSNKAZQw9AWhVsvck6nPimvYcehiRJkqTfQa6YS5IkSZI0IIO5JEmSJEkDMphLkiRJkjQgg7kkSZIkSQMymEuSJEmSNCCDuSRJkiRJAzKYS5IkSZI0IIO5JEmSJEkDMphLkiRJkjSgGUMPQFoVzP/RYmbOPX/oYWgFWnTc3kMPQZIkSb8nXDGXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQAZzSZIkSZIGZDCXJEmSJGlABnNJkiRJkgZkMJckSZIkaUAGc0mSJEmSBmQwlyRJkiRpQCs9mCfZOMnnktya5NtJrkwyO8muSRYnmZfkxiRfS/L01ubQJJVk914/s1vZfpOc69Qkt7U+5yWZNY3xnZvkymley8wkfzWNeru2sR7eK9uulf1jb6wTXkuv3ax2zxa0+3RA79jmSa5OcnOSM5Os0coPanVvTHJFkm17bfZM8r0ktySZ2ytfP8nFra+Lk6zXynfo3c8bkszutXlvkjuS3DdmzM9McmmS69sY9priGg9p5705ySG98t2TXNfO/Y0kz+nd38W9cb2r12bdJGcn+W6ShUl2auX7t3v4aJLte/UP6vUzrx2f8vdGkiRJkpbXSg3mSQJ8Gbisqp5dVS8GDgQ2bVUur6pZVbUNcA3w5l7z+cCc3vcDgRumcdr/3vqcVVXzphjfusCLgHWTbD6NvmcCUwbzZj5wQO/7dMc/1gPA66pqK2BP4MNt3ADvB46vqi2AXwCjiYDbgJe3+3os8AmAJKsBHwP+HNgSmJNky9ZmLnBJ6+uS9h3gJmD7qprVzv/xJDPasa8AO4wz5ncCZ1XVdu26T5jo4pKsDxwD7Nj6OmY0KQCcCBzUzv251u/I5b2f83t65R8BvlpVzwe2BRb2ruM1wGX981fV6aN+gIOBRVP93kiSJEnS47GyV8x3Ax6qqpNGBVX1w6r6aL9SC/BPowuXI5cDOyRZPclTgecAKzowvZYuXJ5BFyBH43nManZvRfg4YOe2snpkkjWTnJJkflsdfkWv79uBNduOgdCF2guXdYBV9f2qurl9vgu4G9io9bkbcHarehqwb6t3RVWN7uVVLJ0I2QG4papuraqH2nXv047t0/oY29cDVbWkla8JVG9sV1XVj8cbNrB2+7wOcNckl/hq4OKqureN+WK6e7Ws/ZBkbWAX4FNtfA9V1S/b54VV9b3J2tNNBH1+kv7fkOTaJNc+8sDiKbqSJEmSpPGt7GC+FXDdJMd3TjKPLsS+Evh071gBX6MLbvsA503znO9t26ePT/KHU9QdBbHP89jV+YnMZelK7fG0Ff6q2rq1Py3Jmr36ZwP7Ay+luw8PTvMaxpVkB2AN4AfABsAve6H5TmCTcZodztIJgU2AO3rH+m02HoXs9v/Te+fdMckCul0Ab+ydcyLvBv46yZ3ABcBbJ6k72ZheD1zQ+jmYbmJkZKe2tf7CJFu1smcDPwNOaRMlJyd5yhRj7TuASYJ5VX2iqravqu1XW2udZehWkiRJkpYa9OVvST7WwtQ1rWgUcjcDTgE+MKbJaCX7QCYJTD1HA88HXgKsDxw1yVg2pluF/0ZVfR9YkuSFy3RB8DLgswBV9V3gh8Bze8fPogvmk67ETkeSZ7RzHVZVjwIZp1qNafMKumA+ug9TthlPVV3dttK/BDh6zOTDeOYAp1bVpsBewGeTTPS7N9mYjgT2av2cAnyolV8HPKuqtgU+Sve4BMAMukcTTmzb6O9n6Zb8SSXZEXigqm6aTn1JkiRJWl4rO5gvoAtKAFTVm4HdgY3GqXse3TZkevW/BbwQ2LCF50lV1Y+r8yBdkBvv+eeRA4D1gNuSLKJ7fny0nX0J7V61LeNrTNDHeKGyP56fAA8Dr6J7bnu5tC3a5wPvrKqrWvE9dM/Gj5733pTeVu8k2wAnA/tU1c9b8Z3AZr2u+21+2sL/aBLg7nGuZyFd2J1qAuNwukkJqupKui3wG05Qd9wxJdkI2Laqrm7lZ9LtPKCqflVV97XPFwCrJ9mw9XVnr83Z9H7/pjDdyR9JkiRJelxWdjD/Ot1z1kf0ytaaoO7L6LZoj3U08E/TOVkvWIbuGenJVj/nAHtW1cyqmgmMXkwHsKh9h24b/ert86/pnoUfuQw4qJ3zucAzgbHPMb8LOKqqHpnONYyV7k3r5wCfqaovjMqrqoBLgdGz8IcA57Y2zwS+BBw8ZkLjGmCLdG9zX6Nd7+gRgfNaH2P72nwU/pM8C3ge3f2ZzO10EzAkeQFdMP/ZBHUvAvZIsl576dserewXwDrtvkI3ubGw9fnH7Wc82t7/B8DP20TIHUme19rsDnxnirHSVvP3p9uhIUmSJElPqBlTV1lxqqqS7Ascn+TtdOHsfpZurR49Yx5gMd0zxWP7WJYXpp3eVlpD96K4N45XKclMuhA9Wn2mqm5L8qu2pfmTwLlJvkW30n1/q3Yj3Zb3G4BT6d42flKS+XSr7IdW1YMtM476vWKS8X48yYfb5zuqaqdx6vwl3U6CDZIc2soObW8OPwo4I8k/A9fTXnpGNxmwAXBCG8uS9mz0kiRvoQu+qwGfrqoFrc1xwFnp/sTb7XRBFboJk7lJHgYeBd5UVfe0+/gBurfUr9WeAz+5qt4NvA34ZJIj6balH9omEv6Dqro3ybF0kwYA76mqe1v/fwt8McmjdEH9b1qd/YAjkiwBfgMc2Ov/rXS/B2sAtwKHtb5m02173wg4P8m8qnp1a7ML3Ur7reONUZIkSZJWpEyQjyQtgyPe8b668JFthh6GVqBFx+099BAkSZK0apnw0edBX/4mSZIkSdLvu5W6lf2JkuQcYPMxxUdV1UXj1D0M+Psxxd9sL6J70kiyNe0N7z0PVtWOQ4xnRVvVr0+SJEmSpmuVCOZVNXsZ6p5C94b2J7Wqmg/MGnocT5RV/fokSZIkabrcyi5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKAZgw9AGlVsPUm63Dim/YeehiSJEmSfge5Yi5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAYL5kkeSTIvyYIkNyT5hyR/0I7tmmRxkuuTfDfJv/TaHZrkZ63tvCSfmeQc+7f+H02y/TTH9ZEkPxqNZYq66yZ50zTqzUxSSY7tlW2Y5OEk/9q+vzvJP06jr82SXJpkYbu2v+8dWz/JxUlubv+v18pfleTbSea3/3frtXlxK78lyf9Oklb+h0nObOVXJ5nZyp/V+hj97N7Y6+strX4l2bBXvk6Sr7Sf84Ikh01xjXsm+V7ra26vfFaSq9q5r02yQ+/+/qb3O3FSr80aST6R5Pvtd+m1rXyXJNclWZJkv179V/T6mZfk/ybZd6qfiyRJkiQtryFXzH9TVbOqaivgVcBewDG945dX1XbAdsB/TvJnvWNntrazqup1k5zjJuA1wGXTGVAL47OBO4BdptFkXWDKYN7cCvzn3vf9gQXTbNu3BHhbVb0A+FPgzUm2bMfmApdU1RbAJe07wD3Af6mqrYFDgM/2+jsReAOwRfu3Zys/HPhFVT0HOB54fyv/MfDSqpoF7AjMTfIn7dg3gVcCPxwz5jcD36mqbYFdgf+VZI3xLi7JasDHgD8HtgTm9K7vA8D/bOd+V/s+8oPe78Qbe+XvAO6uque2/v69ld8OHAp8rn/+qrp01A+wG/AA8H/GG6skSZIkrQhPiq3sVXU3XTh8y2jFtnfsN8A8YJPl6HdhVX1vGZq8gi7MnwjMGRWOXc1OclNbQT4O+E9tZfWD6XywHZ+f5IBe378BFvZW7g8AzlqOa/pxVV3XPv8aWMjSe7MPcFr7fBqwb6t3fVXd1coXAGu2FfFnAGtX1ZVVVcBnRm3G9HU2sHuSVNVDVfVgK/9Der9D7TyLxhs28LT2s30qcC/dBMN4dgBuqapbq+oh4Iw2llE/a7fP6wB3jdN+rL8B3tfG92hV3dM+L6qqG4FHJ2m7H3BhVT0w3sEkb2gr99fec8890xiKJEmSJP1HT4pgDlBVt9KN5+n98rYdewseu+p9QG+r8aTbopfRHODzwDl0q/SrT1F/LktXav873er8LGBbupXjD7bwO3IGcGCSTYFHmF6wnFCbHNgOuLoVbVxVP4YuwDPmXjavBa5v4XoT4M7esTtZGvI3ods5QFUtARYDG7Tzbpbkxnb8/b3QP5F/BV5Ad73zgb+vqokC8W/PO86Y/hvdPb0D+Bfg6F69zdM9+vDvSXZu41y3HTu2bVv/QpKNpxhr34F0vw/jqqpPVNX2VbX9hhtuOFE1SZIkSZrUkyaYN/3V8p1b+PsJ8G9V9ZPesf5W9lNWyIm7rdV7AV+uql/Rhd09lrGblwGfr6pHquqndNumX9I7/lW6bftzgDMf53ifCnwR+G9tvNNpsxXdlvT/Oioap1pNdayq7qiqbYDnAIdMI+y+mm7Xw5/QTVz8a5K1J6g72ZiOAI6sqs2AI4FPtfIfA89sjz78A/C51v8MYFPgm1X1IuBKukA/pTahsjVw0XTqS5IkSdLyetIE8yTPpltFvrsVXd7C39bAEUlmPcFD2JNue/T8JIvoQvZoO/sSHnuv1pygj/FC5W+1rdnfBt5GF6qXS1vJ/yJwelV9qXfop6MV+vb/3b02m9LtBHhdVf2gFd9JF1xHNmXpKv6dwGat7Qy6e3PvmOu5i25r/M5TDPkw4EvVuQW4DXj+BHV/e95xxnQIMLreL9Bte6eqHqyqn7fP3wZ+ADwX+DndM+Ln9Nq8aIqxjvwlcE5VPTzN+pIkSZK0XJ4UwTzJRsBJwL+2Z51/q6q+T/eM8FFP8DDmAK+vqplVNRPYHNgjyVrAIlqgS/Kidgzg18DTen1cRrfNfrV2TbsA3xpznv8FHDUKksuqPaf9KWBhVX1ozOHz6MIr7f9zW5t1gfOBo6vqm6PKbbv7r5P8aev3daM2Y/raD/h6VVWSTZP8Uet3PeDPgKme478d2L212Rh4Ht3L8MZzDbBFks3bLoYD21igC+gvb593A25ufW7UXho3muDZAri1/S59he6Fc7QxfGeKsY6MHmuQJEmSpCfUkMH8j9oz4guAr9G9+fp/TlD3JGCXJJtPcHxcSWYnuRPYCTg/ybjbklv4fjVdeAWgqu4HvgH8F7rV6fWTzKPbTv39VufnwDfby94+SLcyeyNwA/B14O1jtuBTVQuq6jTG984kd47+TVDnz4CDgd16z9nv1Y4dB7wqyc10W+aPa+Vvodt2/j96bUbPnx8BnAzcQrfSfGEr/xSwQZJb6LaHj97w/gLg6iQ30G3V/5eqmt/u49+1cW8K3Jjk5NbmWOClSebTvS3+qNFL2MZqz7O/hW4L+ULgrKoavb3+b+ne6H4D8P/RvTAQugmQG1v52cAbq2q0un8U8O72WMTBdLsVSPKSNtb9gY+330PasZl0q/ajN7hLkiRJ0hMmYxaoJS2HE044od70pun+5TxJkiRJv4cmfPT5SbGVXZIkSZKk31czhh7AipDkY3RbvPs+Mt4b25O8mu7N5H23VdXsJ2p8yyPJBnTbvsfafXmfT38yWdWvT5IkSZKma5UI5lX15mWoexG/A38Cq4XTJ/pN9INZ1a9PkiRJkqbLreySJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNyGAuSZIkSdKADOaSJEmSJA3IYC5JkiRJ0oAM5pIkSZIkDchgLkmSJEnSgAzmkiRJkiQNaMbQA5D0WNv/88Xcc99DU9bb8KlrcO07X7USRrT8bv9/7d1tbJXlHcfx7x9KaWyBTk94BpWnLL6QSLTqWGYxA9wMNroXjizTFBNCyKImkriEYdQVEqPxxYiELIupJC6+YQaVyYMvdBGEAAnTmSi4BrDpkgo+cVLLg1x70YpPFO72lN6Hw/eT3GnPua/r9LrPL/+e/Hvuc/fIEa677joOHDjAxIkTBzxGkiRJqmS+Yy6VmSxNeX/GZdXY2MjIkSOpq6tjzJgx3HDDDWzcuLGkx5w6dSrFYvFsw93a2sqMGTPOO0aSJEm63NiYSzpr1apVFItFjh07xuLFi7n33ns5cOBA3suSJEmSKpqNuaQfqaqqYvny5Xz99de89957HD58mKamJgqFAlOmTOHhhx/mq6++AiClxMqVK5k4cSKjRo3immuuYe3atQAcOnSIiKC9vZ133nmHZcuW0dbWRl1dHXV1dbz55pvfG/Ppp59SU1PD/v37v7ee2267jSeffBKA06dPs2bNGmbNmkV9fT1z585l3759Q/sESZIkSYPIxlzSj5w8eZLnnnuOESNGMHv2bO68807Gjx/P4cOH2bVrFzt27GDFihUAbN++nRdeeIHdu3dz/Phxdu/ezdy5c3/0mLfeeivr169n2rRpFItFisUijY2N3xtz5ZVXctddd9Ha2nr2vra2Nnbs2MH9998PwGOPPcamTZvYsmULx44dY8mSJSxcuJDPPvvsoj0fkiRJ0sVkYy7prNWrV1NfX8/kyZPZtGkTGzdupLOzk4MHD/Lss89SW1vLpEmTaGlp4fnnnyelRHV1Nd3d3bz//vt0d3czbtw45syZM+A1NDc38+KLL3Lq1Cmg53Pp8+bN4+qrryalxNq1a3n66aeZNm0aw4cP54EHHmDChAls3rx5sJ4GSZIkaUjZmEs6a+XKlXz++ed0dnayc+dOFi1axMcff8zYsWOpra09O2769Ol0d3fzySef0NjYyJo1a2hpaWHs2LEsXLiQvXv3DngNCxYsoLq6mldffZWUEhs2bGDJkiUAHD16lGKxyKJFi6ivrz+7tbW10d7eXvLxS5IkSXnw36VJOq8pU6bQ2dlJV1cXV1xxBdBzenlNTQ2FQgGApUuXsnTpUrq6unj88ce55557OHLkyI8ea9iwC/8tcPjw4dx33320trYyZswYvvjiC+6++24ACoUCtbW1vPHGG9x0002DeJSSJElSfnzHXNJ5NTQ0MGPGDB555BG6urro6Ohg1apVNDc3M2zYMPbs2cPbb7/NiRMnGDlyJKNGjaKq6tx/8xs/fjydnZ18+eWX5/2Zzc3NvP766zz11FMsXryYmpoaACKChx56iBUrVnDw4EEAisUiW7dupaOjY3APXJIkSRoiNuaSzquqqorXXnuN9vZ2pk6dSkNDAzfffDPPPPMMAMePH+fBBx+kUChw1VVXsW3bNl566aVzPtbtt9/O/Pnzufbaa6mvr+ett94657hZs2bR0NDA9u3bz57G/o0nnniCpqYmmpqaGD16NDNnzmT9+vWcOXNmcA9ckiRJGiKRUsp7DdIlb926dWn58uWD8lg3tmznaPHkBccV6qrZ+6f5g/IzJUmSJF100dcOP2MulRmbbUmSJOny4qnskiRJkiTlyMZckiRJkqQc2ZhLkiRJkpQjG3NJkiRJknJkYy5JkiRJUo5szCVJkiRJypGNuSRJkiRJObIxlyRJkiQpRzbmkiRJkiTlyMZckiRJkqQc2ZhLkiRJkpQjG3NJkiRJknIUKaW81yBd8h599NHjI0aM+DDvdWjwFYvFQl1d3dG816GLw3wrl9lWNvOtbOZbucyWoy0tLXeca4eNuTQIImJvSunGvNehwWe2lc18K5fZVjbzrWzmW7nMtm+eyi5JkiRJUo5szCVJkiRJypGNuTQ4/pr3AnTRmG1lM9/KZbaVzXwrm/lWLrPtg58xlyRJkiQpR75jLkmSJElSjmzMJUmSJEnKkY25dB4RcUdEfBgRH0XEH8+xPyLiL737342IOVnnKn8l5nsoIt6LiP0RsXdoV64LyZDtTyPinYg4EREr+jNX+SsxX2u3jGXI9ne9v4/fjYidETE761zlr8R8rd0ylyHfpt5s90fE3oj4eda5l4WUkpub2zk2YDjwX2AaUA38G7juB2N+DbwOBHALsDvrXLdLN9/efYeAQt7H4TbgbMcCNwGrgRX9met26ebbu8/aLdMtY7Y/A37S+/2vfN29dLZS8u29be2W8ZYx3zq+vcbZ9cAHWedeDpvvmEt9awA+Sim1pZROAi8BTT8Y0wRsSD12AfURMSHjXOWrlHxV3i6YbUqpM6W0BzjV37nKXSn5qrxlyXZnSumz3pu7gMlZ5yp3peSr8pcl32Lq7cSBWiBlnXs5sDGX+jYJ+Pg7t9t778syJstc5auUfKHnxWRbROyLiKUXbZUaiFLqz9otf6VmZO2Wr/5m+wA9ZzUNZK6GXin5grVb7jLlGxF3R8QHwGZgSX/mVrqqvBcglbE4x30//P+CfY3JMlf5KiVfgLkppY6IGAtsj4gPUkr/GtQVaqBKqT9rt/yVmpG1W74yZxsR8+hp3L75jKq1W/5KyRes3XKXKd+U0svAyxHxC+DPwC+zzq10vmMu9a0dmPKd25OBjoxjssxVvkrJl5TSN187gZfpOQ1L5aGU+rN2y19JGVm7ZS1TthFxPfA3oCmldKw/c5WrUvK1dstfv2qw948q0yOi0N+5lcrGXOrbHmBmRFwbEdXAb4FXfjDmFeC+3qt33wJ8kVL6X8a5yteA842I2ogYBRARtcAC4D9DuXidVyn1Z+2WvwFnZO2WvQtmGxFTgX8Av08pHejPXOVuwPlau5eELPnOiIjo/X4OPRd6O5Zl7uXAU9mlPqSUTkfEH4Ct9Fwt8vmU0vsRsax3/3rgn/RcufsjoAtoPt/cHA5DfSglX2AcPadhQc/v0b+nlLYM8SGoD1myjYjxwF5gNHAmIh6m5wqwX1q75a2UfIEC1m7Zyvh7+THgKmBdb46nU0o3+rpb/krJF193y17GfH9Dzxsep4CvgHt7LwZn/fLt5eolSZIkSVIOPJVdkiRJkqQc2ZhLkiRJkpQjG3NJkiRJknJkYy5JkiRJUo5szCVJkiRJypGNuSRJkiRJObIxlyRJkiQpR/8HNd796mXI6iQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "metalearner.std_coef_plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/notebooks/StackedEnsemble_AllModels_AutoML_20200318_085617'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the Best Model\n",
    "h2o.save_model(model=aml.leader, path='./')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/notebooks/GLM_1_AutoML_20200318_085617'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save specific model\n",
    "h2o.save_model(model=h2o.get_model(model_ids[-1]),\n",
    "               path='./')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Log with MLFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import MLFLow dependencies\n",
    "import mlflow\n",
    "import mlflow.h2o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['auc',\n",
       " 'logloss',\n",
       " 'aucpr',\n",
       " 'mean_per_class_error',\n",
       " 'rmse',\n",
       " 'mse',\n",
       " 'training_time_ms',\n",
       " 'predict_time_per_row_ms']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get Metrics Names before MLFlow logs them\n",
    "metrics_names = lb.columns[1:]\n",
    "metrics_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlflow.set_tracking_uri('./mlruns/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StackedEnsemble_AllModels_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7884943086885449\n",
      "  logloss = 0.5536564863051258\n",
      "  aucpr = 0.8067942794870006\n",
      "  mean_per_class_error = 0.31935286733117985\n",
      "  rmse = 0.4331162430860722\n",
      "  mse = 0.1875896800249936\n",
      "  training_time_ms = 3535.0\n",
      "  predict_time_per_row_ms = 0.231121\n",
      "StackedEnsemble_BestOfFamily_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7863473752131169\n",
      "  logloss = 0.5557610797952202\n",
      "  aucpr = 0.8046586350385025\n",
      "  mean_per_class_error = 0.3152660410890309\n",
      "  rmse = 0.4340863858887624\n",
      "  mse = 0.18843099041396755\n",
      "  training_time_ms = 1587.0\n",
      "  predict_time_per_row_ms = 0.08393\n",
      "XGBoost_1_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7841691327511066\n",
      "  logloss = 0.5554293837832908\n",
      "  aucpr = 0.8047173111044984\n",
      "  mean_per_class_error = 0.3188679794364182\n",
      "  rmse = 0.43442654477923154\n",
      "  mse = 0.18872642280882168\n",
      "  training_time_ms = 7268.0\n",
      "  predict_time_per_row_ms = 0.010207\n",
      "XGBoost_3_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7829742734458651\n",
      "  logloss = 0.558924039075108\n",
      "  aucpr = 0.8019045651457449\n",
      "  mean_per_class_error = 0.31121263116909964\n",
      "  rmse = 0.4355983637422664\n",
      "  mse = 0.18974593449493984\n",
      "  training_time_ms = 3623.0\n",
      "  predict_time_per_row_ms = 0.005727\n",
      "XGBoost_2_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7825651292147967\n",
      "  logloss = 0.5572385578855694\n",
      "  aucpr = 0.8006318998279536\n",
      "  mean_per_class_error = 0.3353455378171415\n",
      "  rmse = 0.43514844976368255\n",
      "  mse = 0.18935417333173615\n",
      "  training_time_ms = 7020.0\n",
      "  predict_time_per_row_ms = 0.008852\n",
      "GBM_5_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7800618553183633\n",
      "  logloss = 0.5595364841567618\n",
      "  aucpr = 0.7990035025307236\n",
      "  mean_per_class_error = 0.33632444539441797\n",
      "  rmse = 0.43632182726649177\n",
      "  mse = 0.19037673694917023\n",
      "  training_time_ms = 2225.0\n",
      "  predict_time_per_row_ms = 0.037089\n",
      "GBM_1_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7783580242825274\n",
      "  logloss = 0.5612380248379343\n",
      "  aucpr = 0.797036210228986\n",
      "  mean_per_class_error = 0.338391842002009\n",
      "  rmse = 0.4371215501327334\n",
      "  mse = 0.19107524959044375\n",
      "  training_time_ms = 2453.0\n",
      "  predict_time_per_row_ms = 0.035969\n",
      "GBM_2_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7781581485149807\n",
      "  logloss = 0.5623205260286717\n",
      "  aucpr = 0.7963935853876395\n",
      "  mean_per_class_error = 0.3273464931426295\n",
      "  rmse = 0.4373780100332224\n",
      "  mse = 0.1912995236606216\n",
      "  training_time_ms = 1686.0\n",
      "  predict_time_per_row_ms = 0.033426\n",
      "GBM_3_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7776094986648523\n",
      "  logloss = 0.5631938416550872\n",
      "  aucpr = 0.7977636367294679\n",
      "  mean_per_class_error = 0.33905163875450445\n",
      "  rmse = 0.4378368809064991\n",
      "  mse = 0.19170113428193183\n",
      "  training_time_ms = 1736.0\n",
      "  predict_time_per_row_ms = 0.033174\n",
      "GBM_4_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7759284469237415\n",
      "  logloss = 0.5657821566617467\n",
      "  aucpr = 0.7943769678196057\n",
      "  mean_per_class_error = 0.347013353483476\n",
      "  rmse = 0.43927570068343336\n",
      "  mse = 0.1929631412109213\n",
      "  training_time_ms = 2141.0\n",
      "  predict_time_per_row_ms = 0.040724\n",
      "DRF_1_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.7602945854519583\n",
      "  logloss = 0.5851385367225719\n",
      "  aucpr = 0.7764269891472034\n",
      "  mean_per_class_error = 0.3501510758951912\n",
      "  rmse = 0.4474553753224053\n",
      "  mse = 0.20021631290491465\n",
      "  training_time_ms = 3390.0\n",
      "  predict_time_per_row_ms = 0.037955\n",
      "GLM_1_AutoML_20200318_085617 model metrics\n",
      "  auc = 0.6826480980292398\n",
      "  logloss = 0.6385204577838607\n",
      "  aucpr = 0.6803442006197458\n",
      "  mean_per_class_error = 0.39723407180394954\n",
      "  rmse = 0.4726827446476101\n",
      "  mse = 0.22342897708759785\n",
      "  training_time_ms = 501.0\n",
      "  predict_time_per_row_ms = 0.002039\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(lb)):\n",
    "    model_id = lb[i, 'model_id']\n",
    "    \n",
    "    with mlflow.start_run(run_name=model_id):\n",
    "        print(\"{model_id} model metrics\".format(model_id=model_id))\n",
    "        \n",
    "        for metrics_name in metrics_names:\n",
    "            metric = lb[i, metrics_name]\n",
    "            print(\"  {metrics_name} = {metric}\".format(metrics_name=metrics_name,\n",
    "                                                       metric=metric))\n",
    "            mlflow.log_metric(metrics_name, metric)\n",
    "        \n",
    "        mlflow.h2o.log_model(h2o.get_model(model_id), \"model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shutdown H2O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "H2O session _sid_af52 closed.\n"
     ]
    }
   ],
   "source": [
    "h2o.cluster().shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}