ML Modelling and Hyperparameter Tuning .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xgboost in c:\\users\\avina\\anaconda3\\envs\\tf-gpu\\lib\\site-packages (1.1.1)\n",
      "Requirement already satisfied: numpy in c:\\users\\avina\\anaconda3\\envs\\tf-gpu\\lib\\site-packages (from xgboost) (1.18.5)\n",
      "Requirement already satisfied: scipy in c:\\users\\avina\\anaconda3\\envs\\tf-gpu\\lib\\site-packages (from xgboost) (1.5.0)\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting scikit-optimize\n",
      "  Using cached scikit_optimize-0.7.4-py2.py3-none-any.whl (80 kB)\n",
      "Collecting pyaml>=16.9\n",
      "  Using cached pyaml-20.4.0-py2.py3-none-any.whl (17 kB)\n",
      "Requirement already satisfied: scikit-learn>=0.19.1 in c:\\users\\avina\\appdata\\roaming\\python\\python37\\site-packages (from scikit-optimize) (0.22.2.post1)\n",
      "Requirement already satisfied: numpy>=1.11.0 in c:\\users\\avina\\anaconda3\\envs\\tf-gpu\\lib\\site-packages (from scikit-optimize) (1.18.5)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\avina\\appdata\\roaming\\python\\python37\\site-packages (from scikit-optimize) (0.15.1)\n",
      "Requirement already satisfied: scipy>=0.18.0 in c:\\users\\avina\\anaconda3\\envs\\tf-gpu\\lib\\site-packages (from scikit-optimize) (1.5.0)\n",
      "Collecting PyYAML\n",
      "  Using cached PyYAML-5.3.1-cp37-cp37m-win_amd64.whl (216 kB)\n",
      "Installing collected packages: PyYAML, pyaml, scikit-optimize\n",
      "Successfully installed PyYAML-5.3.1 pyaml-20.4.0 scikit-optimize-0.7.4\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install scikit-optimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "import xgboost \n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "from sklearn import preprocessing\n",
    "from functools import partial\n",
    "from skopt import space\n",
    "from skopt import gp_minimize\n",
    "from hyperopt import hp,fmin,tpe,Trials\n",
    "from hyperopt.pyll import scope as ho_scope\n",
    "from hyperopt.pyll.stochastic import sample as ho_sample\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    X_train = np.loadtxt(open(\"X_train.csv\",\"rb\"),delimiter=\",\")\n",
    "    X_test = np.loadtxt(open(\"X_test.csv\",\"rb\"),delimiter=\",\")\n",
    "    labels = pd.read_csv(\"Training set labels.csv\")\n",
    "    submission_fr = pd.read_csv(\"SubmissionFormat.csv\")\n",
    "    \n",
    "    y_train = labels.status_group.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Creating some baseline model without any kind hyperparameter tuning that predicts only one class**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BaslineModel(BaseEstimator):\n",
    "    def fit(self,X,y=None):\n",
    "        pass\n",
    "    def predict(self,X):\n",
    "        return ['functional'] * len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waste Model\n",
      "CV accuracy: 0.543 +/- 0.000\n",
      "----------------\n",
      "SGD Classifier\n",
      "CV accuracy: 0.731 +/- 0.006\n",
      "----------------\n",
      "Knn Classifier\n",
      "CV accuracy: 0.777 +/- 0.005\n",
      "----------------\n",
      "Decision Tree Classifier\n",
      "CV accuracy: 0.727 +/- 0.005\n",
      "----------------\n",
      "Random Forest Classifier\n",
      "CV accuracy: 0.781 +/- 0.007\n",
      "----------------\n",
      "Ada Boost Classifier\n",
      "CV accuracy: 0.697 +/- 0.004\n",
      "----------------\n",
      "Softmax Classifier\n",
      "CV accuracy: 0.736 +/- 0.007\n",
      "----------------\n",
      "Gradient Boost Classifier\n",
      "CV accuracy: 0.752 +/- 0.006\n",
      "----------------\n"
     ]
    }
   ],
   "source": [
    "models = { \"Waste Model\" : BaslineModel(),\n",
    "           \"SGD Classifier\" : SGDClassifier(),\n",
    "           \"Knn Classifier\" : KNeighborsClassifier(),\n",
    "           \"Decision Tree Classifier\" : DecisionTreeClassifier(),\n",
    "           \"Random Forest Classifier\" : RandomForestClassifier(),\n",
    "           \"Ada Boost Classifier\" : AdaBoostClassifier(algorithm=\"SAMME.R\"),\n",
    "           \"Softmax Classifier\" : LogisticRegression(multi_class=\"multinomial\",solver=\"lbfgs\"),\n",
    "           \"Gradient Boost Classifier\" : GradientBoostingClassifier()\n",
    "}\n",
    "\n",
    "cv = StratifiedKFold(n_splits=10)\n",
    "results = []\n",
    "names = []\n",
    "\n",
    "for key,value in models.items():\n",
    "    cv_scores = cross_val_score(estimator=value,X=X_train,y=y_train,cv = cv, n_jobs=1,scoring='accuracy')\n",
    "    \n",
    "    results.append(cv_scores)\n",
    "    names.append(key)\n",
    "    \n",
    "    print(key)\n",
    "    print('CV accuracy: %.3f +/- %.3f' % (np.mean(cv_scores), np.std(cv_scores)))\n",
    "    print('----------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBgAAAMCCAYAAADDC3UaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdf7Dld13f8de7dxMjgcS7JciEJIIVdXGn0OEO6AiFlKLBEVH/UBYsKWwnpS0ZtK0UXceE1nQsHa0pP0qppJSpLFoFQSu/2q7iMmCz66AkBGyaAtmuQOheBaKBTfj0j3sWDjd3d8/mvdnv3b2Px8ydved8v9/7fZ+7ZNn73O/3c2qMEQAAAICOvzL1AAAAAMDZT2AAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAOAVV9caq+rkH6Ws/v6rec4LtT6+qQw/GubeKqnpqVX1s6jkA4FwkMADABqrqd6tqtaq+7kydc4zxK2OM75mbYVTVt5yp8292DySwrP8ejjF+f4zxbad/OgBAYACAdarq0UmemmQk+YEzdM5tZ+I8AAAPFoEBAO7vBUk+mOSNSa4+0Y5V9bKq+tOqOlxVf2/+X8yr6uKqelNV3VVVn6iqn6mqvzLb9ner6v1V9W+q6kiS62fP7Z9tf9/sFH9UVV+oqh+dO+c/qarPzM77wrnn31hVr62qd86OeX9VPbKqfml2NcZHq+pvnOC1fEdVvbeqjlTVp6vqp2fPf93saxyeffzSsSs7jl1VMPs+HJvpB6vq+6rqT2Zf66fnznF9Vf16Vf1qVX2+qv6wqh4/t/1rrjg4dktKVV2Y5J1JLp29ti9U1aVV9aSq+kBV/dns3K+uqvOP9z1cfxVEVe2YXa3yZ1V1a1X9wLpzv6aq/uts1j+oqr92ov89AMBWJjAAwP29IMmvzD6+t6q+caOdquqqJP84yd9O8i1JnrZul1cluTjJN8+2vSDJC+e2PznJHUkekeSG+QPHGH9z9unjxxgPHWP86uzxI2df81FJdid5TVUtzx36I0l+JsnDk3wxyQeS/OHs8a8n+cXjvJaHJflvSd6V5NLZ6/nvs817knxnkickeXySJ83Occwjk1wwm+lnk/yHJD+W5IlZuxLkZ6vqm+f2f06S/5Jke5I3J/nNqjpvo7nmvh93J3lWksOz78dDxxiHk9yX5Cdmr++7kjwjyT+cHXO87+Gx13xekt9K8p6s/R5cm+RXqmr+FopdSV6RZDnJ7Vn3+wQAfJXAAABzquopSb4pya+NMQ4m+d9Jnnec3X8kyX8cY9w6xviLrP0geuzrLCX50SQ/Ncb4/Bjj40l+IcnfmTv+8BjjVWOMe8cYf7ngiEeT/PMxxtExxu8k+UKS+R+I3zbGODjGuCfJ25LcM8Z40xjjviS/muR4VzB8f5JPjTF+YYxxz2zmP5hte/7snJ8ZY9w1e53zr+NokhvGGEeTvCVrP+zfOPsatya5Nclfn9v/4Bjj12f7/2LW4sR3Lvj6v8bstX5w9j38eJJ/n/uHnuP5ziQPTfLzY4wvjTH+R5LfzlpUOOatY4z/Oca4N2vB6QkPZE4A2AoEBgD4Wlcnec8Y47Ozx2/O8W+TuDTJnXOP5z9/eJLzk3xi7rlPZO1f+Tfaf1H/b/bD7jF/kbUfko/59Nznf7nB4/l9512etZiykUtz/9dx6bqZ7ps7x0ZzzJ/3K697jPHlJIfWfb2FVdW3VtVvV9WnqupzSf5l1r73i7g0yZ2zGY5Z/3v0qbnP13+vAYA5AgMAzFTV12ftqoSnzX5g/VTWLr9//Pw6AXP+NMllc48vn/v8s1n7l/1vmnvuiiT/d+7xOC2Dnx53Jjne+gKHc//Xcbhxrq98n2ZrUlw29/X+IslD5vZ95NznG32//l2SjyZ57BjjoiQ/naQWnONwksuPrYsxs/73CABYkMAAAF/1g1m7p/9xWbsU/glJdiT5/aytn7DeryV54WyhwIdkbf2BJMnsX/R/LckNVfWwqvqmrK3X8J9PYZ5PZ239hjPht5M8sqp+fLao48Oq6smzbXuT/ExVXVJVD8/a6zyV17HeE6vqh2fvnPHjWVsr4oOzbR9K8ryqWpqtcTF/u8Onk/zVqrp47rmHJflcki9U1bcn+QfrznWi7+EfJLk7ycuq6ryqenqSZ2ftNg8A4BQJDADwVVdnbU2FT44xPnXsI8mrkzy/1r2V5BjjnUn+bZJ9WVsA8AOzTV+c/Xpt1n6AvSPJ/qzdbnHTKcxzfZL/NHuHgx95gK9pIWOMzyd5ZtZ+wP5Ukv+V5MrZ5p9LciDJHyf5cNYWjfy5xunenrX1KVaztpbDD8/WY0iSl85m+LOsrf3wm3MzfjRrseOO2ffk0iT/NGtrZHw+a4tLfs1CjjnB93CM8aWsvQ3ps7J2xclrk7xgdh4A4BTVGJvp6kwAOHtV1Y4ktyT5unXrJDBTVdcn+ZYxxo9NPQsAcHq5ggEAGqrqh6rq/NlbRf6rJL8lLgAAW5HAAAA9fz/JXVl7B4b7cv81AAAAtgS3SAAAAABtrmAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgbdvUA2zk4Q9/+Hj0ox899RgAAADAnIMHD352jHHJRts2ZWB49KMfnQMHDkw9BgAAADCnqj5xvG1ukQAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgbaHAUFVXVdXHqur2qnr5Btsvrqrfqqo/qqpbq+qFix4LAAAAnP1OGhiqainJa5I8K8njkuyqqset2+0fJfnIGOPxSZ6e5Beq6vwFjwUAAADOcotcwfCkJLePMe4YY3wpyVuSPGfdPiPJw6qqkjw0yZEk9y54LAAAAHCWWyQwPCrJnXOPD82em/fqJDuSHE7y4SQvHWN8ecFjAQAAgLPcIoGhNnhurHv8vUk+lOTSJE9I8uqqumjBY9dOUnVNVR2oqgN33XXXAmMBAAAAm8UigeFQksvnHl+WtSsV5r0wyVvHmtuT/J8k377gsUmSMcbrxxgrY4yVSy65ZNH5AQAAgE1gkcBwc5LHVtVjqur8JM9N8o51+3wyyTOSpKq+Mcm3JbljwWMBAACAs9y2k+0wxri3ql6S5N1JlpLcNMa4tapePNv+uiT/Iskbq+rDWbst4p+NMT6bJBsd++C8FAAAAGAqNcaGSyJMamVlZRw4cGDqMQAAAIA5VXVwjLGy0bZFbpEAAAAAOCGBAQAAAGgTGAAATrO9e/dm586dWVpays6dO7N3796pRwKAB91JF3kEAGBxe/fuzZ49e/KGN7whT3nKU7J///7s3r07SbJr166JpwOAB49FHgEATqOdO3fmVa96Va688sqvPLdv375ce+21ueWWWyacDAD6TrTIo8AAAHAaLS0t5Z577sl55533leeOHj2aCy64IPfdd9+EkwFAn3eRAAA4Q3bs2JH9+/d/zXP79+/Pjh07JpoIAM4MgQEA4DTas2dPdu/enX379uXo0aPZt29fdu/enT179kw9GgA8qCzyCABwGh1byPHaa6/Nbbfdlh07duSGG26wwCMA5zxrMAAAAAALsQYDAAAA8KASGAAAAIA2gQEAAABoExgAAACANoEBAAAAaBMYAAAAgDaBAQAAAGgTGAAAAIA2gQEAAABoExgAAACANoEBAAAAaBMYAAAAgDaBAQAAAGgTGAAAAIA2gQEAAABoExgAAACANoEBAAAAaBMYAAAAgDaBAQAAAGgTGAAAAIC2bVMPAHA2qqpJzz/GmPT8AACwnsAA8AB0fsCvKoEAzhJiIgAszi0SAMA5a/v27amqB/wxtc7s27dvn3p8ALYYVzAAAOes1dXVLXsVwGYIJABsLa5gAAAAANoEBgAAAKDNLRIAwDlrXHdRcv3FU48xiXHdRVOPAMAWIzAAAOesesXntvQaDOP6qacAYCtxiwQAAADQJjAAAAAAbQIDAAAA0CYwAAAAAG0CAwAAANAmMAAAAABt3qYS2JK2b9+e1dXVyc5fVZOde3l5OUeOHJns/AAAnJsEBmBLWl1dzRhj6jEmMWXcAADg3OUWCQAAAKBNYAAAAADaBAYAAACgzRoMAMA5bauuO7K8vDz1CABsMQIDsCWN6y5Krr946jEmMa67aOoR4IyZejHXqpp8BgA4UwQGYEuqV3xuy/6lv6oyrp96CgAAzjXWYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgbdvUAwBMpaqmHmESy8vLU48AAMA5SGAAtqQxxmTnrqpJzw8AAA8GgQHgAehe/dA9XqCAM+N0XOnU+Rr+WwfgbCIwADwA/tIPW4P/1gFgcRZ5BAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADatk09AAAAAJxpVTXp+ccYk57/wSAwAAAAsOV0fsCvqnMyEHS5RQIAAABoExgAAACANoEBAAAAaBMYAAAAgDaBAQAAAGgTGAAAAIA2gQEAAABoExgAAACANoEBAAAAaBMYAAAAgDaBAQAAAGgTGAAAAIA2gQEAAABoExgAAACANoEBAAAAaBMYAAAAgLZtUw8AAAAAp2r79u1ZXV2d7PxVNdm5l5eXc+TIkcnOfzwCAwAAAGed1dXVjDGmHmMSU8aNE3GLBAAAANAmMAAAAABtAgMAAADQttAaDFV1VZIbkywl+eUxxs+v2/6TSZ4/9zV3JLlkjHGkqj6e5PNJ7kty7xhj5TTNDgAAwBY1rrsouf7iqceYxLjuoqlH2FCdbFGMqlpK8idJnpnkUJKbk+waY3zkOPs/O8lPjDH+1uzxx5OsjDE+u+hQKysr48CBA4vuDgAAwBZTVVt6kcepXntVHTzehQOL3CLxpCS3jzHuGGN8KclbkjznBPvvSrL31McEAAAAzlaLBIZHJblz7vGh2XP3U1UPSXJVkt+Ye3okeU9VHayqax7ooAAAAMDmtcgaDBu9webxrsV4dpL3jzGOzD333WOMw1X1iCTvraqPjjHed7+TrMWHa5LkiiuuWGAsAAAAYLNY5AqGQ0kun3t8WZLDx9n3uVl3e8QY4/Ds188keVvWbrm4nzHG68cYK2OMlUsuuWSBsQAAAIDNYpHAcHOSx1bVY6rq/KxFhHes36mqLk7ytCRvn3vuwqp62LHPk3xPkltOx+AAAADA5nHSWyTGGPdW1UuSvDtrb1N50xjj1qp68Wz762a7/lCS94wx7p47/BuTvK2qjp3rzWOMd53OFwAAAABM76RvUzkFb1MJAADAiXibys33NpWLLPIIAAAAm87savktZ3l5eeoRNiQwAAAAcNaZ8uqFrXz1xIkIDAAAwJY29b+C+0GVc4XAAAAAbGndH/D9azasWeRtKgEAAABOSGAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2rZNPQAAAACcaVU16fFjjNbxm5HAAE3dP1hOh3PxDycAAHgw+Tv06ScwQFP3D6aq8ocbAABw1rMGAwAAANAmMAAAAABtAgMAAADQZg0GAADgrLZ9+/asrq5OOsNUC38vLy/nyJEjk5wb1hMYAACAs9rq6uqWXTR7M7yjGRwjMADAAjbDX+C26l+eAYCzg8AAAAvwlrQAACdmkUcAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2rZNPQAAAEDHuO6i5PqLpx5jEuO6i6YeAb5CYAAAAM5q9YrPZYwx9RiTqKqM66eeAtYIDJBk+/btWV1dnez8VTXJeZeXl3PkyJFJzg0AAJxbBAZIsrq6uiWr91RhAwAAOPdY5BEAAABoExgAAACANrdIQLbuysNWHQYAzhVb9dbP5eXlqUeArxAYIFt35WGrDgMA54Kp/x5XVZPPAJuBWyQAAACANoEBAAAAaBMYAAAAgDZrMACwZWzfvj2rq6uTnX+qBciWl5dz5MiRSc4NAGwdAgMAW8bq6uqWXIRrq66sDgCcWW6RAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANq2TT0AbBZb8X3il5eXpx4BAAA4RwgMkGSMMdm5q2rS8wMAAJwObpEAAAAA2lzBAMCWMa67KLn+4qnHOOPGdRdNPQIAsAUIDABsGfWKz23JW5KqKuP6qacA2LxOx1pcna+xFf+/iXOTwAAAAGxpfsCH08MaDAAAAECbwAAAAAC0uUUCgC3ldNxne7ZZXl6eegQAYAsQGADYMqa8x7aq3OMLAJzT3CIBAAAAtAkMAAAAQJtbJKBp6vdNTry1EgAAMD2BAZr8cA8AACAwAMBCXK0EAHBiAgMALMAP9wAAJ2aRRwAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAABo2rt3b3bu3JmlpaXs3Lkze/funXokgDNu29QDAADA2Wzv3r3Zs2dP3vCGN+QpT3lK9u/fn927dydJdu3aNfF0AGdOjTGmnuF+VlZWxoEDB6YeAwAATmrnzp151atelSuvvPIrz+3bty/XXnttbrnllgknAzj9qurgGGNlw20CAwAAPHBLS0u55557ct55533luaNHj+aCCy7IfffdN+FkAKffiQKDNRgAAKBhx44d2b9//9c8t3///uzYsWOiiQCmITAAAEDDnj17snv37uzbty9Hjx7Nvn37snv37uzZs2fq0QDOKIs8AgBAw7GFHK+99trcdttt2bFjR2644QYLPAJbjjUYAAAAgIVYgwEAAAB4UC0UGKrqqqr6WFXdXlUv32D7T1bVh2Yft1TVfVW1fZFjAQAAgLPfSQNDVS0leU2SZyV5XJJdVfW4+X3GGP96jPGEMcYTkvxUkt8bYxxZ5FgAAADg7LfIFQxPSnL7GOOOMcaXkrwlyXNOsP+uJHsf4LEAAADAWWiRwPCoJHfOPT40e+5+quohSa5K8huneiwAAABw9lokMNQGzx3vrSeeneT9Y4wjp3psVV1TVQeq6sBdd921wFgAAADAZrFtgX0OJbl87vFlSQ4fZ9/n5qu3R5zSsWOM1yd5fbL2NpULzAUAAKdV1Ub/PnZmbca3kQdYxCJXMNyc5LFV9ZiqOj9rEeEd63eqqouTPC3J20/1WAAAOF22b9+eqnpAH5vBA519+/btU48ObHEnvYJhjHFvVb0kybuTLCW5aYxxa1W9eLb9dbNdfyjJe8YYd5/s2NP9IgAA4JjV1dUteRXAZgkkwNZVm/EP35WVlXHgwIGpxwAA4Gx0/cVTTzCd6/986gmAc1xVHRxjrGy0bZE1GAAA4KxRr/jclr2CYVw/9RTAViYwAABwztmKtwssLy9PPQKwxQkMAACcU7bi1QsAm8Ei7yIBAAAAcEICAwAAANAmMAAAAABtAgMAAADQJjAAAAAAbQIDAAAA0CYwAAAAAG0CAwAAANAmMAAAQNPevXuzc+fOLC0tZefOndm7d+/UIwGccQIDAAA07N27Ny996Utz9913Z4yRu+++Oy996UtFBmDLERgAAKDhZS97WZaWlnLTTTfli1/8Ym666aYsLS3lZS972dSjAZxRAgMAADQcOnQob3rTm3LllVfmvPPOy5VXXpk3velNOXTo0NSjAZxRAgMAAADQJjAAAEDDZZddlquvvjr79u3L0aNHs2/fvlx99dW57LLLph4N4IwSGAAAoOGVr3xl7r333rzoRS/KBRdckBe96EW5995788pXvnLq0QDOKIEBAAAadu3alRtvvDEXXnhhkuTCCy/MjTfemF27dk08GcCZVWOMqWe4n5WVlXHgwIGpxwAAAADmVNXBMcbKRttcwQAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQJvAAAAAALQJDAAAAECbwAAAAAC0CQwAAABAm8AAAAAAtAkMAAAAQNtCgaGqrqqqj1XV7VX18uPs8/Sq+lBV3VpVvzf3/Mer6sOzbQdO1+AAAADA5rHtZDtU1VKS1yR5ZpJDSW6uqneMMT4yt883JHltkqvGGJ+sqkes+zJXjjE+exrnBgAAADaRRa5geFKS28cYd4wxvpTkLUmes26f5yV56xjjk0kyxvjM6R0TAAAA2MwWCQyPSnLn3ONDs+fmfWuS5ar63ao6WFUvmNs2krxn9vw1vXEBAACAzeikt0gkqQ2eGxt8nScmeUaSr0/ygar64BjjT5J89xjj8Oy2ifdW1UfHGO+730nW4sM1SXLFFVecymsAAAAAJrbIFQyHklw+9/iyJIc32OddY4y7Z2stvC/J45NkjHF49utnkrwta7dc3M8Y4/VjjJUxxsoll1xyaq8CAAAAmNQigeHmJI+tqsdU1flJnpvkHev2eXuSp1bVtqp6SJInJ7mtqi6sqoclSVVdmOR7ktxy+sYHAAAANoOT3iIxxri3ql6S5N1JlpLcNMa4tapePNv+ujHGbVX1riR/nOTLSX55jHFLVX1zkrdV1bFzvXmM8a4H68UAAAAA06gx1i+nML2VlZVx4MCBqccAAAAA5lTVwTHGykbbFrlFAgAAAOCEBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAI/TvPMAABxrSURBVAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBtocBQVVdV1ceq6vaqevlx9nl6VX2oqm6tqt87lWMBAACAs9u2k+1QVUtJXpPkmUkOJbm5qt4xxvjI3D7fkOS1Sa4aY3yyqh6x6LEAAADA2W+RKxielOT2McYdY4wvJXlLkues2+d5Sd46xvhkkowxPnMKxwIAAABnuUUCw6OS3Dn3+NDsuXnfmmS5qn63qg5W1QtO4VgAAADgLHfSWySS1AbPjQ2+zhOTPCPJ1yf5QFV9cMFj105SdU2Sa5LkiiuuWGAsAAAAYLNY5AqGQ0kun3t8WZLDG+zzrjHG3WOMzyZ5X5LHL3hskmSM8foxxsoYY+WSSy5ZdH4AAABgE1gkMNyc5LFV9ZiqOj/Jc5O8Y90+b0/y1KraVlUPSfLkJLcteCwAAABwljvpLRJjjHur6iVJ3p1kKclNY4xbq+rFs+2vG2PcVlXvSvLHSb6c5JfHGLckyUbHPkivBQAAAJhIjbHhkgiTWllZGQcOHJh6DAAAAGBOVR0cY6xstG2RWyQAAAAATkhgAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoE1gAAAAANoEBgAAAKBNYAAAAADaBAYAAACgTWAAAAAA2gQGAAAAoG2hwFBVV1XVx6rq9qp6+Qbbn15Vf15VH5p9/Ozcto9X1Ydnzx84ncMDAAAAm8O2k+1QVUtJXpPkmUkOJbm5qt4xxvjIul1/f4zx/cf5MleOMT7bGxUAAADYrBa5guFJSW4fY9wxxvhSkrckec6DOxYAAABwNlkkMDwqyZ1zjw/Nnlvvu6rqj6rqnVX1HXPPjyTvqaqDVXVNY1YAAABgkzrpLRJJaoPnxrrHf5jkm8YYX6iq70vym0keO9v23WOMw1X1iCTvraqPjjHed7+TrMWHa5LkiiuuWPgFAAAAANNb5AqGQ0kun3t8WZLD8zuMMT43xvjC7PPfSXJeVT189vjw7NfPJHlb1m65uJ8xxuvH/2/vzsPlqOo0jn9fEiBASBAI4AIEGUEkYNSIA4KGRR5RHFFBiKBG4oKjQWQAfSYMRpFFUGFYhJEtgCYoKA5hkD1hBwmQlUUdFnHcWCSIoAT4zR/nFLfSqerb99Y1N6Tfz/Pc51afrq46XefU75w6tXTEuIgYN2rUqD5/ETMzMzMzMzMbPJ0MMNwJvEHSZpJWA/YDLivPIGkjScrT2+XlPiFpLUlr5/S1gN2BhQP5BczMzMzMzMxs8PV6i0REvCDpi8BVwBDg3IhYJOmg/P6ZwN7A5yW9ADwH7BcRIWlD4NI89jAUmB4RV/6DvouZmZmZmZmZDRJFtD5OYfCNGzcu5syZM9jZMDMzMzMzM7MSSXdFxLiq9zq5RcLMzMzMzMzMrC0PMJiZmZmZmZlZYx5gMDMzMzMzM7PGPMBgZmZmZmZmZo15gMHMzMzMzMzMGvMAg5mZmZmZmZk15gEGMzMzMzMzM2vMAwxmZmZmZmZm1pgHGFYAM2bMYMyYMQwZMoQxY8YwY8aMwc6SmZmZmZmZWZ8MHewMdLsZM2YwZcoUzjnnHHbccUduvvlmJk2aBMCECRMGOXdmZmZmZmZmnVFEDHYeljFu3LiYM2fOYGdjuRgzZgynnnoqO++888tps2bNYvLkySxcuHAQc2ZmZmZmZma2NEl3RcS4yvc8wNBi6sjBWe+KYuriwc6BmZmZmZmZraDaDTD4FolWy/kA21cwmJmZmZmZ2crAD3kcZFOmTGHSpEnMmjWLJUuWMGvWLCZNmsSUKVMGO2tmZmZmZmZmHfMVDIOseJDj5MmTue+++9hqq6045phj/IBHMzMzMzMze0XxMxjMzMzMzMzMrCPtnsHgWyTMzMzMzMzMrDEPMJiZmZmZmZlZYx5gMDMzMzMzM7PGPMBgZmZmZmZmZo15gMHMzMzMzMzMGvMAg5mZmZmZmZk15gEGMzMzMzMzM2vMAwxmZmZmZmZm1pgHGMzMzMzMzMysMQ8wmJmZmZmZmVljHmAwMzMzMzMzs8Y8wGBmZmZmZmZmjXmAwczMzMzMzMwa8wCDmZmZmZmZmTXmAQYzMzMzMzMza8wDDGZmZmZmZmbWmAcYzMzMzMzMzKwxDzCYmZmZmZmZWWMeYDAzMzMzMzOzxjzAYGZmZmZmZmaNeYDBzMzMzMzMzBrzAIOZmZmZmZmZNeYBBjMzMzMzMzNrzAMMZmZmZmZmZtaYBxjMzMzMzMzMrDFFxGDnYRmSHgMeGex8DIL1gccHOxO23Lncu4/LvDu53LuTy737uMy7k8u9+3RzmW8aEaOq3lghBxi6laQ5ETFusPNhy5fLvfu4zLuTy707udy7j8u8O7ncu4/LvJpvkTAzMzMzMzOzxjzAYGZmZmZmZmaNeYBhxfL9wc6ADQqXe/dxmXcnl3t3crl3H5d5d3K5dx+XeQU/g8HMzMzMzMzMGvMVDGZmZmZmZmbWmAcYSiSdJOmQ0uurJJ1dev0dSYf2cZnjJe3Qj8+EpEmltLfktMP6sJzRkhY2nWdlJGmKpEWS5kuaK+kdOX2opGMl/Sqnz5U0pfS5F3PaIknzJB0qqXI/krSFpCsk/VrSfZJ+LGnDXL6XD+B3OVvSm/L0PnldsySNk3TKQK1nZSHpmdL0+3JZbzJAyz5M0v2SFub68YmcPlvSgDxluFyuklaXdG2uk/uW68IrTV/2rQ6W9Q1Ju7V5/6CibPpL0jalGPGkpIfy9LVNlluxnj0kzcn79f2Svp3Tp/alPehgPbeWpk/M5XDiQGyrlvUU5bxQ0kxJ6wzQcidKOm0gltWy3NmSHiiV9d4DvY68ntGSPtbm/a5qTyR9KPd53thmnj7FVUnTSvvp/ZK+NjC5fXn5e7WLv5I+kev9Ikn3FvtvzteA1CtJr5F0Sen1DKV+zpd7i4srGtX009rMv1Oef66k7SW9b3nltZSHru4D5Jg0XdKDku6SdJukDzVc5sttXZM6LGlsXZ3IcXRx3o7z8zbdoEm+W5bfdfF96EAubCVwK7APcLJSx3Z9YETp/R2AQ6o+2MZ44Jm87L5YAOwLnJNf7wfM6+MyrIKk7YE9gbdGxN8lrQ+slt/+JrARsE1E/E3S2sC/lT7+XESMzcvZAJgOjASW6qhIGgb8D3BoRMzMaTsDlb8X20REfLr0chLwrxExK7+e0+lyJA2NiBcGNHMrMEm7AqcCu0fEbwZgeQcB7wG2i4inJY0E9mq63FYRMYeecn0LsGpRJ4Ef9WVZkoZExIsDmb8GOtq3OhERR/Xy/pn9yuHSy1gAFPmdBlweEZeU52m6T0kaA5wGvD8i7pc0FPhs/3NdLyLKA+GfA0ZFxN/7upwOvnO5nM8HvgAc09f1LGf75/2uY/0o+9HAx0j1vnVZ3dieTABuJvV9pjZYTqvDI+KSvE3vlXRBRDw0QMveC7gcuLf1DUl7kPqPu0fE7/L6Pz5A631ZRPwO2DuvcyNgh4jYtD/LGsw+QS/9tDr7A9+OiPMkTQTGAVf8Y3Pao9v7AJIE/Aw4PyI+ltM2Bf6lYt5+1a3e2vZejKV9nbgpIvbM+TuO1DYN1CDkaLotvkeE//If8Brgt3l6G+B84GrgVcDqwFOkAHcUcCewkPRwj+JZFgeTGpb5wEWkCvUH4P+AucBOpArzk/z5O4F3VuRjPKmRuhHYEBBpcOE44LA8z1jg9ryuS4FX5fS35XlvA04EFub0Ifn1nfkzn8vpo4t5uuUP+DAwsyJ9TeAJYO02n32m5fXr82fUkn4gcEHNMsaTDkYAtiMNPt2T/2+Z07cGfpHrzXzgDcBapCA0L9e9ffO8s0lB8yjSYNYDuazL61kLODeX/z3AB3P6ROBiYCZw/WCXzXIq/2fyvvgg8MZS+jTglFwODwJ7l8prNnAJcD/ww9byzvP9Bti8Zp2zgXF5+gxSIF8EfL00z/H0xI9v57R9clnPA24s1x9gA+DXwOJcTzZvWc/upDhwdy7j4Tn94VxXbgb2G+zy6GTfoiZ+5fmOIA3IzgOOL5VlUX5V23UqvcfS2cC3SPvhL4Gd2uS9vL7ZwLHADaTBybfl6buAq4BX5/k2B67M6TeV62JpuRcAB9ass/wdPpO3zTxS+7Jmm/qzTGwpb3/gMuDF/P6+LeupzHP+/t8FZgHf6bScgYOA7+Xpulg4EfhpXu+vgBNKn/9ULpsbgLOA03L6psB1+ftdB2xSyucZOZ8PAu8mxcX7gGm97bultHVJHen5ue5sWyqT75P6DdOpae/zeufmv3uAtfNyin35yy3r66r2BBhO6jdtAdxfSl+D1LeaTzqQuoNe4mqb/XSdXAc2yK93zd9lQf5uq/eSvlRcIZ2AehJ4KG/nzVvWfSOwSwf56qh/2aYejaan3zcfeI6e/md5PXVxaTal+NU0rjeoA5X9tLoyAT5d2v4zSO3xYywdx4o+/cN5+SfkZVxJOkiv3P6kk7F3AuPzPMcBx1Tkq6v7ALlcbmjz/kRK8YG0n1+X87eAHEfyvFNIsefaXJ5FG9RpHV6q7SYduy1VJ1ryNp6e+CbSwP4h+XVdvK9Ld3yP8ABDRWE+DGxCOoNzEHA08D7gnfTs3OuW5r8Q+ECe/h09jc86+f/UYsfIr6cDO+bpTYD76ioUqUH5Yl73eSzd0ZsPvDtPfwM4uSK9PMDwWeDIPL06KbhtRncOMAzPO+Ivge+Vtte2wD29fPaZirQ/Axu2pH0X+FLNMso78ghgaJ7eDfhJnj6VdNYMUmBcA/gIcFZpOSPz/9n0NCjl6fJ6jgUOKOpm/u5r5YDx23KdXtn/gCWkjsi2LenTSMFzFeBNwK9L23Ex8Lr83m3FPlz67NrAn9uss1wu6+b/Q3L6tqSG6gF6OpNF/FgAvLYlrVyuL0+X10O6+upGYK2c/hXgqDz9MHDEYJdDxTaq3beoj197kBra4oC62LbTSGfx6rbrVHqPpbPJB8ukNuDaNnmfxtIDDMVB86o5f6Py632Bc/P0dfQc3L+Digab1PF6c806y99hvVL6N4HJberPMrGldfu3TJfXU5nn/P0vB4Z0Ws6k+n8x8N78ui4WTiQdCI4EhgGPABsDryZ1GEfl73ELPQMMM4FP5ukDgZ+V8nkRqQP5QeBp0smEVUid1LE1++4D9HQY18vb8Gv5/V2AuaVtdVdpm1a29zl/xWDDcNIBzHhK+3JLHrqqPQEOAM7J07eSzmIDHErP/rMt8AJt4mrNfvpQLsdngGNz+jDgUWCL/PoC0tUGdel1cWUaOQ5UrPvJYhv3Ej/60r+sqkej6en3vTxdXg/t49JscvwazD/q+2mVZVKxHSeS40Fp37w5f/c3A88Ce+T3LgX26mX7b00aiHwP6aBrtZb8dn0fgHTMclKb9ydSig+5vo7I0+uTBktEGjhYQDrpNyKnLzXA0EEdXqbtbq0TLXkbT88AwKOkk0lF3urifV2643uEn8FQ4RbSSPQOpAOJ20qvi9scdpZ0h6QFpEq1dU6fD/xQ0gGkhq/KbsBpkuaSzhSNyJfhV/kxafRyAmkED4B82dU6EXFDTjofeFdF+oWlZe0OfCKv9w5SJ+kNbbfESioiniEFsM+SRjN/lC+nW4qkT+X7sR6VtHGbRapBdkYCFys9B+MkeurSbcC/S/oKsGlEPEcKuLtJ+paknSJicR/Wszvw1Vz+s0mNdPHcgWsi4skG3+GVZglpX55U8d7PIuKliLiXdGBb+EVE/DYiXiI1QKNbPicgOlz/RyXdTeqkbE0azHga+BtwtqQPkzo/kOLRNEmfIXVGOvXPebm35DL/JOmsbqFPl1EOomLfqotfuwHnRcSzABX1uG67poXXxNLSLD/N/+9i2TJvp9i+WwJjgGty3o8EXidpOKlNuTin/xfpgLm/xki6KbdJ+9MTR6rqT1Vs6VUHeb44OrvUdo38+SdIneprcnpdLAS4LiIWR8TfSGf4NiUNcMyOiMci4nmWrtPb03Mp6oXAjqX3ZkbqOS0A/hgRC/J+vYj6Mt4/Isbmvyfy8i4EiIjrgfVyXQK4rLRN69r7W4DvSjqYVP8G6jL0laU9mUAaCCL/n5Cn3wX8ACAi5pP6XIWquFrl8EiXk28E7Kr0jKwtgYci4pd5niIO1KW3jSsN9aV/2d96VBmXSu8PevvQpp9WVyad+HlELCHV/SGkKxfIr0fn6crtHxGLSPv8TNJVZc+3LNt9gBaSTld6DsWdpeRyfBBwrKT5pCsVXkvqd+0EXBoRz0bE06TY2aq3OtyftvumHOM3Jp3UPSGn18X7unTHd/yQxyq3kjpR25AuK7md1FnZgbSjDiONpu4dEduQLssclj/7fuB0UlC8K98v22oVYPtSZ+W1EfGXqoxExB9IB0PvIZ056k27ACfSWa1ivZtFxNUdLHOlFBEvRsTsiPga6SqRj5BGSTcpBnwi4rzcEVlMTWCX9HrS5cR/anlrEake9OZoYFZEjAE+QK5LETGddN/ac8BVknbJDWoxsnucpL7ciybgI6Xy3yQi7svv/bUPy1kZvAR8FHi7pH9vea98z7lq0l+k5fk1uRH8a64PtSRtBhwG7BoR25IuYRuWG6DtSJdT70Xu+ETEQaSGc2NgrqT1OvqGKe/XlMr7TRFRHlBZ4cu8Zd+qi19tO3V127UPinJfpsx7UWxfAYtK+d4mInYntQNPldLHRsRWFcvpNI5MA76Y26Sv0xNHlqk/VbGlw+/UW547rVPFMxg2JZ1t+UJOr4yFWd3+12mHvjxfsayXWpb7Ep2XcdWgcrGO8naobO8j4njSJd1rALerzYMMs65pT3KM24V0oPUwcDiwb76/GyrKvC6utltPPoCdTTpIqDtJUJnez7jSaxn2tX/Zj3r08qqojkuFFaJ9qOmnNTmh8/e83JeAJXmgEfK+38v2h3Rc8BRLn3wo8uo+QKrjby1eRMQXSLdNlJ8lUF7v/vm9t+U24Y/0bO/eYntvdbi/bXfhMnoGrurifWW643viAYZl3UJ6sMyTObg9SbpEZHvSKFFR+R/PZ3SKh+msAmwc6WEZR+TPDAf+Qrp0qnA1KVCSPzeW9o4CvhKlM0N5JOrPknbKSR8n3ff0FLBYUnG2Zv/Scq4CPi9p1bzeLSSt1evWWAlJ2lJS+eqNscAjkc6CnkM64zQszzuEmgcLSRoFnEm65Ko1GE4HdpD0/tL875W0Tct8I0n3mkK6/KiY9/XAgxFxCinQbSvpNcCzEfED0j2fb6VzVwGTi06apLf04bMrnVzWewL7q/RrLQ0dB5wuaQSApBGSWh/IN4IUoBdL2pB0iX9xdnhkRFxBugS3eAje5hFxR6QHGz1O6mR04nbgnZL+KS9nTUlbNPt6y0/FvlUXv64GDpS0Zk5ft2U5ldu1UBdLB/CrPACMUnpgGZJWlbR17ow+JGmfnC5Jb674/ImkMxNb5PlWUfUvGa0N/D5vn5fjflX9qYotnXyRPuS5I3nbHwwclvNdGQvbuAMYL2m9/Pl9Su/dSno4IKTtcXN/81njxrxcJI0HHs/bp1Vle5/LZUFEfIt0u88bWbavUNZN7cnepPuRN42I0ZHOJj5EGggob/cx9NTdyrjaTj4B9A7gf0mXQ48u4iU9caAyvU1caVeGxwEnKD14sXj6/8Et8/Spf1lTjzpRGZc6/OxyUddPo76sWrUrizqV2z/n58OkK+feBZyi6l+/6fY+wPXAMEmfL6Wt2Wb+kcCfImKJ0kMNiyssbgQ+JGkNpRN+H6j4bH/qcF/qxI6k2FDkpyreV6Y7vif+FYllLSDdCzS9JW14RDwOIOmsnPYw6SEYkM5w/0Dp8hiR7kN6StJM4BJJHwQmkzpUpytdEjSUVEEPqstMRNT9+sQngTNzx/pB0sOuyP/PlfQsqZIUziZdJnR3rjSP8Q94uu0rxHDg1NxAvEC6cqFoBKaQRgkXSvoLaUTwfNL9j9Bzee+q+bMXku6fWkpEPCdpT9IvkpxMuhJlPvAlUiNVOAE4Px80XF9K3xc4QNIS0oNCvwG8HThR0kt5eeUg3pujgZOB+bn8HyYdYHetiHhS0nuBGyU9PgCLPINUt+7M5bYE+E7LOudJuoc0Yv0gaUATUsPz30oDWwK+nNNPzJ0ska5imkd6gFBbEfGY0uWkMyStnpOPJN1Lt6Jqt29Vxq+IuDIftM2R9Dzp6dDlq1LqtmtZXSxtLCKeV/r5uVNy2zCUtB8uInVMzpB0ZP7OF9HyS0ERMV/pp5Nn5PwF6YxXq/8gHXA/Qmqbio5MVf35KsvGlk71mue+iIh7JM0jDQbUxcK6z/5e0lTSwP/vSc+rKK40O5jUDh5OqisDVqbZVOC83I4/S6pDVera+0Nyh/pF0i0fPyedRX0hb49pEXFSsZAua08mkB52V/YT0hPYD6Vnu88lPdisXVytcmKuv6uR9omfRkRI+hTp8uPigX5nRvr1gmXSSbf2VMWVi4Cz8sDB3hFRHKAQEVfkA8pr8zYL0oPUKM3zVB/7l0dX1KNeb7XqJS6tKCr7aZF+3auqTFrNoucy7+M6WWHd9lf6BYvjSVcdPKr0c7j/ybL7fVf3AfJ+tBdwkqQjSLH3r6TnP1T5ITBT0hzS/nx/Xs7dkn6U0x4hPVC4dV39qcNL1YmIaL1NZKf8nkhXLhe/uDCV6nhfl+74Ts/DRMzMzMzMzMzM+s23SJiZmZmZmZlZYx5gMDMzMzMzM7PGPMBgZmZmZmZmZo15gMHMzMzMzMzMGvMAg5mZmZmZmZk15gEGMzMzMzMzM2vMAwxmZmZmZmZm1pgHGMzMzMzMzMyssf8HQQ+iq2upS1oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1296x864 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "fig = plt.figure(figsize=(18,12))\n",
    "fig.suptitle(\"Algorithm computation\")\n",
    "ax = fig.add_subplot(111)\n",
    "plt.boxplot(results)\n",
    "ax.set_xticklabels(names)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.77606061, 0.77919192, 0.77575758])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = RandomForestClassifier()\n",
    "cross_val_score(rf, X_train, y_train, cv=StratifiedKFold(n_splits=3), scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(rf, X_train, y_train, cv=StratifiedKFold(n_splits=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAI4CAYAAACSixhGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZyO5eKA8esZIx2FkpSdZMsyGIwlOkhU9hRa7GlfT0qdDuoXqegcLVqVraJOZcuSEC2yk7WoFBI5ZR3M4vn9wZmaQzXKmDxzfT8fn2ae5Z37rtt0zfO+zztBGIZIkiSd6GKyegCSJEnHglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjJqIC4KgWRAEnwVBsC4Igt5ZPR4pI4IgeCkIgq1BEKzI6rFIGRUEQbEgCGYFQbA6CIKVQRDcltVjym4C36cmuoIgyAF8DjQBNgILgI5hGK7K0oFJvyEIggbAbmBkGIaVsno8UkYEQVAIKBSG4eIgCPIAi4DWfs89frxSE221gHVhGH4ZhmESMAZolcVjkn5TGIZzgB+yehzS0QjDcHMYhosPfbwLWA0UydpRZS9GTbQVATb87PON+BdMkjJdEAQlgWrAvKwdSfZi1ERbcIRtPt8oSZkoCIJTgTeB28Mw3JnV48lOjJpo2wgU+9nnRYFvs2gskhR5QRDk5GDQvBKG4VtZPZ7sxqiJtgVAmSAISgVBcBLQAZiQxWOSpEgKgiAAhgGrwzB8PKvHkx0ZNREWhmEKcDMwjYMvWHs9DMOVWTsq6bcFQfAaMBcoFwTBxiAIumf1mKQMqAdcAzQKgmDpoT+XZPWgshNv6ZYkSZHglRpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqMkmgiDomdVjkI6W61YnKtdu1jBqsg//gulE5LrVicq1mwWMGkmSFAl/qvepyXfa6WHBswtn9TAiacf2H8l32ulZPYxIynvqX7J6CJG17fvvKXDmmVk9jMj6E337j5xt276nQAHXbmZYsfzTnUlJSfmOtC/2eA/m1xQ8uzBDnh+T1cOQjspF9Spl9RCk3yU59UBWD0E6aoXOOnPrL+3z6SdJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGpOIN9v/Y7et3XnumtacUPnNoz/92gABvbrxc3dL+fm7pfTtX0zbu5+OQCzpr+Ttv3m7pfT/K9xfLF2TbrHfODeW7ixS5u0z7du2Uzv27pzS/cruKnrZSz45IPjN0FlCz16dKNwobOoGlc5bdsPP/xAs6YXUaF8WZo1vYgff/wRgPemT6dWrRpUrVqFWrVqMGvmzLRzFi1aRNWqVShfrgy3334rYRge97ko+9i3bx/169UmoUZ14qtW4f8e7Jdu/78eH0zuXLFs27YNgKSkJHpe252a1auSUKM6c2a/n3Zs3z73U6Z0Sc7Mn+84ziB7MGpOIDly5KDHTX/juVHjGfzMaCa9PZZv1n9B736P8dSwN3hq2BvUa3Ahdes3BqBhk0vTtt91X38Knl2Y0mXKpz3eR3Pe4y9/yZ3ua4wZ+Tz1G17Ek8Ne556+jzL0n/2P6xwVfZ07dWHSO1PSbXv0kYE0atSI1Ws+p1GjRjz6yEAAzihQgHHjJrB06ae89NJwunTplHbOzTfdyDPPPMfqNZ+zbu06pk2delznoewlV65cTJn2HvMWLuaTBYuY/u405s/7BICNGzYwc8Z7FCtePO34l4a9CMCCxUuZOHkqve+5mwMHDgBw6aXNmfPh3OM/iWzAqDmB5D/jTM4tex4AuXOfQrESpfjP91vT9odhyAezpnHBhRcfdu7sGVO4oPFP2/cmJjLu9VF06NQz3XFBEJCYuAeAPbt3k/+MMzNjKsrG6jdoQP78+dNtmzhxAtd06gzANZ06M2HCeACqVatG4cKFAahYsSL79u1j//79bN68mV27dlKnTh2CIODqa65h/IRxx3ciylaCIODUU08FIDk5meTkFAgCAO7u9TceenggwaHPAdasXk3Dho0AKFiwIKfly8eiRQsBqJVQm0KFCh3nGWQPRs0JasvmTXy5dg3lzvvpEv7KTxdxWv4zKFK0xGHHz5k1LV3UjHrpKdpc0YlcuU5Od9xVXW9g1ruT6NTuQvrecyPX33Zv5k1COmTLli1p3+QLFSrE1q1bDzvmrbfepGrVauTKlYtNmzZRpEjRtH1FixTl203fHrfxKntKTU0loWY8JYoWonHjxtSqlcCkiRMpXLgIVarEpTu2cpUqTJo4gZSUFNZ/9RVLlixm08aNWTTy7CM2qwego7c3MZH+fe7k2lvuJvcpp6Ztn/1e+qsx/7Vm1afkynUyJc8pA8AXa9eweeM39Lz5brZs3pTu2NnvTeHCi1vRtn1nVq9YxuD+9zF0+FvExNi/yjorV67kvnt7M3nKNIAjvn7m5z8lS5khR44czFuwiO3bt9PhistYvvxTHn1kABPfOfypz85duvLZmtXUq5NA8eLFSahdhxw5/F9uZvP/VCeYlJRkBvS5k4YXXkq9BhembU9NSeHjD2bQoGHTw86ZM3NquthZs3IZ6z5fTdf2zeh1S2c2bfia3rd1A+DdyW9T/9BjVKgUR1LSfnbu+DGTZ6Xs7qyzzmLz5s0AbN68mYIFC6bt27hxI5e3a8tLL4+gdOnSABQtWpRNm376qXfjpo0UKuzlfB0fp512GvUbXMCkiRP4ev16EmpWp3zZ0mzauJG6tWvy3XffERsby6ODHmfegkW88ebb7NixnXPLnJvVQ488o+YEEoYhQx7pS7ESpWjTvlO6fUsWfULR4qUoUPDsdNsPHDjAh++/S4OfRc2lrdsz6q0ZvDx2Ko89OYIixUowcMhLAJxZ8GyWLpoHwDfrvyQ5KYl8p6V//YN0rDVv3oJRI0cAMGrkCFq0aAnA9u3badmyOQ/1H0C9evXSji9UqBCn5snDJ598QhiGjB41ipYtWmXJ2JU9fP/992zfvh2AvXv3MmvmDOLiqvL1xs2s+fwL1nz+BUWKFuXjTxZw9tlnk5iYyJ49B1+fOOO96cTGxlKhwnlZOYVswWthJ5BVy5cw891JlDynTNpt252vvZWatesfdjXmv1YsW0SBM8+iUOGih+07kh433cUTjz3A+DdGQRBwx73/52V9HVNXX3Uls2e/z7Zt2yhZohh9+vbj7nt607FDe15++SWKFSvOmLGvAzD06af4Yt06+vd/iP79HwJgypRpFCxYkKeeGkqP7l3Zu3cvTZs1o9nFh69/6Vj57rvNXNu9GwdSUzlw4ABt27Xjkkub/+Lx32/dSsvmlxATE0PhwoUZ9tKItH1/v/cexo4dQ2JiIueeU4IuXbtx/z/6Ho9pRF7wZ3pvhzLlK4ZDnh+T1cOQjspF9Spl9RCk3yU59UBWD0E6aoXOOnPd9h9/KHOkfT79JEmSIsGokSRJkWDUSJKkSMjUqAmCoFkQBJ8FQbAuCILemfm1JElS9pZpURMEQQ7gaeBi4DygYxAE3s8mSZIyRWZeqakFrAvD8MswDJOAMYBvJCFJkjJFZkZNEWDDzz7feGibJEnSMZeZUXOkd2w77E1xgiDoGQTBwiAIFu7Y7tvxS5Kk3yczo2YjUOxnnxcFDvs1umEYPh+GYY0wDGvkO+30TByOJEmKssyMmgVAmSAISgVBcBLQAZiQiV9PkiRlY5n2u5/CMEwJguBmYBqQA3gpDMOVmfX1JElS9papv9AyDMPJwOTM/BqSJEngOwpLkqSIMGokSVIkGDUngNTUVG7pfgX9et+ctm3Cm6/S8+oW3NC5DS898/hh52z85itu7n552p92F9dh3BujAPhg1rvc0LkNzf8ax9o1P73MadXyJdzU9TJu79mRbzd+A8DuXTv5x13XE4aH3Y0vZViPHt0oXOgsqsZVPuL+wYMeIz6+GvHx1agaV5lcJ8Xyww8/sGHDBi5s3IjKlc4jrkolnnhiSNo59/a+h2rV4ujSpXPattGjR6U7RvqjypctTc3qVUmoGU+9OgmH7d+xYweXtWlFQo3qxFetwsgRw9P2PTnkX8RXrUKNanF0vuYq9u3bB8D99/WmVnw1enTrknbsq6+M5uknn8js6USeUXMCmPDvVyhWolTa58sWz+eTj2bx9Etv8syIt2nbofNh5xQtXoqnhr3BU8PeYMjzY8h18snUrd8YgBKlzuXv//c4leLi053z1tiR3Pd/j9P52luZPP51AMaMfJ4rru5BEBzpbYekjOncqQuT3pnyi/v/dlcvFi1awqJFS3jooQE0aHAB+fPnJzY2lkcfG8TyFav48KO5PPvMUFatWsWOHTuYO3cuS5YsIzU1leXLl7N3715GjhjBDTfceBxnpuxgyrvvMW/BIj6aO++wfc89O5QKFSowb+Fipk6fwb339CIpKYlNmzYx9Omn+HDuPBYeWqdvvD6WHTt28Mknc5m/aAmpqamsWHFw7Y4eOYKe19+QBbOLFqPmT27b1u9Y8MkcmjZvm7Zt8vjXufzK7uQ86SQATjv9jF99jGWL51GocDEKnl0YgOIlz6Fo8VKHHRcbG0vS/v3s27+PHLGxbN60gf9s20rlqjWO4YyUHdVv0ID8+fNn6NixY8fQvkMHAAoVKkT16tUByJMnD+XLV+DbTZuIiYkhKSmJMAzZt3cvOXPmZPCgx7j5llvImTNnps1D+l9BELBr127CMGTP7t2cfvrBGAdISU1h7969pKSkkJiYSKFChdKt3b1795IzNif/fHwQN9zk2j0WjJo/ueefepSu199JEPz0n2rTxq9Z+eki7rj+Su65tSufr17xq48xZ8ZULmh88W9+rcuv6s6Tgx5g/BujadGmAyNffJKru9/0h+cgZVRiYiLTpk2lbdvLDtu3fv16li5dQq2EBPLkyUPbtm2pUaM6JUuWJF++fCxcuJCWLf31cjq2AgJaXHoxdWvXYtiLLxy2//obbuKzz1ZzTsli1IyvymODHycmJoYiRYpw++13Uu7cUpxToij58uXjwiYXkSdPHlq3bkvtWjUoWbIkefPlY9HChbRo2TILZhc9mXpLt/6Y+R/PJt9p+SlT7jw+XbIgbfuB1BR279rF48+8wudrVjCw310MGzPliE8RJScnM+/j9+nc87bf/Hqly5Tn8WdeAWDFsoXkP+NMCEMG9utFjthYetx4F6fn//WrQtIfMWnSROrWrXfYVZ3du3dzxRXtGPz4P8mbNy8Ad/W6m7t63Q1Az5496NvvAYYNe5H3pk+ncuXK3Pf3+4/7+BU9M96fQ+HChdm6dSstLmlGuXLlOL9+g7T9701/lypV4pgy7T2+/OILml/SjHrn1yc1NZVJkyaw6rN1nHbaaVzVsT2vvfoKHa+8ijvv6sWdd/UC4Ibre/KPvv14+aVhzHhvOpUqV6b3vX/Pqume8LxS8ye2asVS5n38Pl3bN+ORB+/m08XzeeyheznjzLOo26AxQRBQrkJlgpgYdu448u/NWjjvQ0qXqXBUMRKGIWNGvkCHztfx6vBnuarrjTRscikT3nzlGM1MOrLXx45Ne+rpv5KTk7ni8nZ07Hglbdq0PeycJUuWAFC2bFlGjxrFa2PGsnLlStauXXtcxqxoK1z44NP2BQsWpEWrVixcsCDd/pEjhtOqdRuCIKD0uedSslRJPvtsDbNmzqBEyVKceeaZ5MyZk1at2/DJ3Lnpzl269ODaLVOmLK++MorRr45h1cqVrHPt/m5GzZ9Yl563MfLf7/Hy2Knc0+dRqlSvRa/7H6bO+Y1Ytng+AJs2rCclOZm8+Y78e7PmzJiSoaeefu69qROoWac+efLkZf++fcTEBMTExLB//74/PCfpl+zYsYM5c2anewopDEOuvbYH5SuU54477jzief369qFfvwdJTk4mNTUVgJiYGBITE4/LuBVde/bsYdeuXWkfz3hvOudVrJjumGLFijNr1kwAtmzZwueff06pUudQtFgxFsybR2JiImEY8v6smZQvXz7duQ/268s/+vQ7fO3ude3+XkbNCajJJW34bvNGbuzShkceuJs773uIIAj4z7at9L37pzs/9u3by5KFc6nboHG68z+eM4NO7S5k9cpl9Ot9E/+46/p058yYOoFLW7cHoPUV19D/H3cy/PkhXNrqiuMzQUXO1VddSf3z6/LZZ59RskQxXnppGM899yzPPfds2jHjxr1NkyYXccopp6Rt++ijj3hl9ChmzZqVdsv3lMk/vUn5+PHjqFGjBoULF+a0006jdu3aVK1aBYKAuLi44zpHRc/WLVu4sOEFJNSoToN6dWh28SVc1LQZLzz/HC88/xwAve/7O/PmzqVm9apc2uwiHur/MAUKFKBWrQRat21L3YSa1KxelQMHDtCtx7Vpjz1h/Hjif7Z2ExJqU7N6VYIgoEoV1+7vFfyZ3n+kTPmK4ZDnx2T1MKSjclG9Slk9BOl3SU49kNVDkI5aobPOXLf9xx/KHGmfV2okSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiITarB/BzeU/5C43rVszqYUhHJczqAUi/U85Yf67ViScm+JV9x28YkiRJmceokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZEQ+0s7giDYBYT//fTQP8NDH4dhGObN5LFJkiRl2C9GTRiGeY7nQCRJkv6IDD39FATB+UEQdD30cYEgCEpl7rAkSZKOzm9GTRAEfYF7gHsPbToJGJ2Zg5IkSTpaGblS0wZoCewBCMPwW8CnpiRJ0p9KRqImKQzDkEMvGg6C4JTMHZIyYsOGDVx0YWPiKlekWlxlnnriCQDuveduqlQ6jxrVqnJFu7Zs37497Zzln37KBefXo1pcZeKrxrFv3z4AkpKSuPH666h0XnmqVDqPt996M0vmpOzh2h7dKFLoLKrGVU7b1vvuXlSqWIHq1eJod9lP63b9+vXkPTU3NeKrUSO+GjfdeH3aOc0vuZj46lWJq1KJm268ntTU1OM+F2VPn332GfHVq6X9yX9aPoYM+RfLli3j/Hp1qRpXhdYtW7Jz504AkpOT6dqlC1XjqlC54nk8MvDhLJ5BdGUkal4PguA54LQgCK4F3gNeyNxh6bfExsbyyKOPsWz5SuZ8+DHPPjuU1atW0ejCC1m89FMWLllKmTJleeyRgQCkpKTQtXMnnnx6KEuWLefdGTPJmTMnAAMfHsCZBQuyYtUaln66gvoNLsjKqSniOnXqwqR3pqTb1vjCJixdtpzFS5ZRpkyZdN/0zyldmoWLlrBw0RKeHvps2vZXx4xl0eKlLF22nO+/38a///3GcZuDsrdy5cqxaPESFi1ewvwFC8mdOzetW7fhup7XMmDAwyxd9imtWrdm8KDHAPj3G2+QtH8/S5d9yrwFC3nh+edZv3591k4ion4zasIwHAT8G3gTKAv0CcPwycwemH5doUKFqFa9OgB58uShfPnybPp2E02aXERs7MGb2molJLBx40YA3pv+LpUqV6ZKXBwAZ5xxBjly5ABgxPCXufue3gDExMRQoECB4z0dZSP1GzTg9Pz5021rctFP6zahdm02bdr0m4+TN+/Bd5VISUkhKSmJIAh+4wzp2Js5YwbnlC5NiRIl+Pyzz6jfoAEAFzZpwttvvQVAEATs2bOHlJQU9u7dy0knnZS2fnVsZfTN95YDHwBzDn2sP5H169ezdOlSatVKSLd9xPCXadqsGQBrP19LEAQ0v6QZtWvWSPsJ4r+X+R/o24faNWtwZYcr2LJly/GdgPQzw1/+ad0CrP/qK2rWqE7jhn/lww8+SHfspRc3o0ihs8iTJw+XXdbueA9VYuzYMbTv0AGAipUqMXHCBAD+/e832LBhAwCXtWvHKaecQrEihTmnZAnuuPNv5P+fsNexkZG7n3oA84G2QDvgkyAIumX2wJQxu3fvpuMVlzNo8OPpyn/gwwOIjY2l45VXAZCSmsLHH3/E8JGjmTl7DhPGjWPmzBmkpKSwaeNG6tStyycLFpKQUIfed/fKqukom3t4QH9iY2O58tC6LVSoEF989TULFi7msUGD6XTNVWmvUwB4Z8pUvtn4Lfv372fWzJlZNWxlU0lJSUyaOJF27S4H4IUXh/HM0KHUqlmD3bt2cdJJJwEwf/58YnLk4JuNm1j7xZf865+P8+WXX2bl0CMrI1dqegHVwjDsEoZhZyCeg7d4K4slJyfT4Yp2dOh4Ja3btE3bPmrkCKa88w7DR45OuyRfpEgR6tdvQIECBcidOzdNL76YpUuWcMYZZ5A7d25atW4DQNt27Vi6dEmWzEfZ28iRI5j8zjuMHPXTus2VKxdnnHEGANXj4znnnNKs/fzzdOedfPLJNG/RgokTxx/3MSt7mzplCtWqVeess84CoHz58kyZNo35CxbSvkNHzildGoAxr71K06ZNyZkzJwULFqRO3bosWrgwK4ceWRmJmo3Arp99vgvYkDnDUUaFYch11/agfPkK3HbHHWnb3502lcGDHuPfb48jd+7cadubXNSUFcuXk5iYSEpKCh/MmUOFChUIgoBLmzdn9uz3AZg1cwYVKlQ43tNRNjdt6lQGPfYob40bn27dfv/992l3NX355ZesW7eWUuecw+7du9m8eTNw8DU1U6dMoVy58lkydmVfY8f89NQTwNatWwE4cOAAA/r3p2fP6wAoXrw4s2bNIgxD9uzZw/x58yhX3vWaGYKDd2sfYUcQ3Hnow6pAZWA8B2/rbgXMD8Pw+iOe+AfEx9cIP543/1g/bCR99OGHNG54AZUqVSYm5mCbPvjQQ9x5x+3s37+fM/If/Om2VkICTw19BoBXXxnNY48+QhAENGt2MQMGPgLA119/TbcundmxfTsFzjyT518cRvHixbNmYiegGF+gelSuvupK5sx+n23btnHWWWfRp28/Hn1kIPv37yf/oasyCQkJPD30Wd56600e6NeX2NhYcuTIQZ8+/WjeogVbtmyhdasW7N+/n9TUVBo2bMigwf9Me7GxMsal+/slJiZSqkRxPl/3Bfny5QPgiSeG8OzQoQC0btOG/gMeJggCdu/eTY9u3Vi9ehVhGNK5Sxf+dpdP8/9eBQucse6HH34oc6R9vxY1fX/tQcMwfOAYjC0do0YnIqNGJyqXrk5EvxY1v/YLLY95tEiSJGWW37xWGwTBmcDdQEXg5P9uD8OwUSaOS5Ik6ahk5IXCrwBrgFLAA8B6YEEmjkmSJOmoZSRqzgjDcBiQHIbh7DAMuwG1M3lckiRJRyUjUZN86J+bgyC4NAiCakDR3zopCIKXgiDYGgTBij80QkmSpAzISNQ8FARBPuBvwF3Ai8Adv34KAMOBZr91kCRJ0rHwmy8UDsNw0qEPdwANM/rAYRjOCYKg5O8bliRJ0tH5xagJguBJDr7Z3hGFYXjrsRhAEAQ9gZ4AxXzDN0mS9Dv92pWa4/KLKcIwfB54Hg6++d7x+JqSJCl6fu3N90Ycz4FIkiT9ERl5obAkSdKfXqZFTRAErwFzgXJBEGwMgqB7Zn0tSZKkTPuVtmEYdsysx5YkSfpfv3mlJgiCskEQzPjvm+gFQVAlCIL7M39okiRJGZeRp59eAO7l0DsLh2H4KdAhMwelI9u3bx/n16lNzerVqBZXmQcf6HfYMTt27KBt6w+mtisAACAASURBVJZpx4wY/nLavp49ulOs8NlUr1ol3Tl/v7c3NapVpVuXzmnbXhk9iqeeeCLT5qLsY8OGDTRp3IjKlc4jrkolnnxiyGHHrFmzhvr16nJq7pN5fPCgdPuu7dGNIoXOompc5XTb7+19D9WrxdH1Z+t29OhRR3x86ffo0b0bhc8+i6pVKh9x/4Tx46lWNY746tVIqFWTDz/8MN3+1NRUasRXp1WLFmnb7u19D9WqxtGl88/W7ahRPOG6PSYyEjW5wzCc/z/bUjJjMPp1uXLlYur091iweAnzFy5m+rRpzPvkk3THPPvMUCpUOI8Fi5fw7nsz6X13L5KSkgC4pnNnJkyanO74HTt2MHfuXBYuWUpqaiorli9n7969jBo5kutuuOG4zU3RFRsby6OPDWL5ilV8+NFcnnlmKKtWrUp3TP78+fnnv4Zwx51/O+z8Tp26MOmdKem27dixg0/mzmXxkmWkpqay/L/rdsQIrr/hxkydj7KPzp27MGnylF/c36hxYxYvWcqixUt44cVhXN/z2nT7n3hiCBXKV0j7/L/fb5csTb9uR44cwQ2u22MiI1GzLQiC0hx6I74gCNoBmzN1VDqiIAg49dRTAUhOTiY5OZkgCA47ZteuXYRhyO7duzk9f35iYw++dKp+/Qacnj9/uuNjYmJITkoiDEP27dtLzpw5eXzwIG66+WZy5sx5fCamSCtUqBDVqlcHIE+ePJQvX4FvN21Kd0zBggWpUbPmEddc/QZHXrdJh9bt3r0H1+3gQY9x0y23uG51zNRv0ID8/7P2fu7UU09N+x68Z8+edN+PN27cyJTJk+nW/ad7ZH6+bvf9bN3efLPr9ljJSNTcBDwHlA+CYBNwO+CP8FkkNTWVWvHVKVb4bBpfeCG1EhLS7b/hxptYs2YNpYoXpUa1OAY//k9iYn75P3OePHlo3bYtCTXiKVmyFHnz5WPRwgW0aNkqs6eibGj9+vUsW7rksHV7tPLkyUObtm2pWaM6pUqWJF++fCxcuJCWrlsdZ+PefptK51WgVYvmPP/isLTtf7vjDh4e+Ei677958uShbdu21IivTslSh9btgoW0bOW6PVZ+M2rCMPwyDMMLgTOB8mEYnh+G4fpMH5mOKEeOHMxftJgv1n/DggULWLki/S9Bn/7uNOLi4vjqm43MX7iY22+7lZ07d/7qY/7trl7MX7SYRx4bxAN9+9Cn7wO8NOxFrurYnocH9M/M6Sgb2b17N+2vaMegx/9J3rx5//Dj3dXrbhYuWsKjgwbTr+8/6Nvv4Lrt2KE9A/o/dAxGLP221m3asGLVat5862369e0DwDuTJnFmwTOJj48/7Pi7et3NosVLeGzQYPr2+Qd9H3iAYS++SMf2rttjISN3P/UJgqAPB39L9x0/+1xZ6LTTTqPBBRfw7rvT0m0fOWI4rdq0IQgCSp97LiVLluKzNWsy9JhLlywBoEzZsrwyejSvvDaWVStXsG7t2mM+fmUvycnJtL+8HR07XkmbNm2P6WMvObRuy5Yty+hRo3htzFhWrlzJWtetjqP6DRrw5RdfsG3bNj7++CMmTZzIueeU4qorOzJr1kw6XXNNuuPTrdvRo3ht7FhWrnDd/lEZefppz8/+pAIXAyUzcUz6Bd9//z3bt28HYO/evcycMYNy5cqlO6ZYseLMmjkTgC1btrD2888odc45GXr8B/r1oU+/B0hOTiY1NRWAICaGxMTEYzgLZTdhGNLz2h6Ur1Ce2++485g//gN9+9C334Pp1m2M61bHwbp16wjDg7+ycPHixSQlJXHGGWfQf8DDrP9mA+u+/IpXXn2Nhg0bMXLUqHTn9uvTh34PuG6Ptd98870wDAf//PMgCAYBEzJtRPpF323eTI9uXUlNTeVAeIDL2l3OJZc254XnngXg2uuu596/38+13bsSXzWOkJCHBjxMgQIFALjm6iv5YPZstm3bRumSxbm/T1+6djv4IrYJ48cRX6MmhQsXBqB27drEV42jUuXKVImLy5oJKxI+/ugjXhk9ikqVK1MjvhoA//d//dmw4RsAel53Pd999x11Emqyc+dOYmJiePKJISxbvpK8efNy9VVXMmf2+2zbto1SJYrRp2+/tHU7fvw44mvUSLduq1WtQuXKVYhz3eoPuvrKK5l9aO2VLH5w7SUnJwNw3fXX8/ZbbzJ61Chic+bkL3/5C6+8NuawmzeOZPy4cdSomX7dVo1z3R4LwX8rM8MnBMHpwPwwDMsc68HEx9cIP573v3ePS39uMRn4Jib9Gbl0dSIqWOCMdT/88MMRG+Q3r9QEQbCcQ7dzAzk4+ILhB4/d8CRJkv64jPzup+Y/+zgF2BKGoW++J0mS/lR+NWqCIIgB3gnDsNJxGo8kSdLv8qt3P4VheABYFgRB8eM0HkmSpN8lI08/FQJWBkEwn4O3dQMQhmHLTBuVJEnSUcpI1DyQ6aOQJEn6gzISNZeEYXjPzzcEQfAIMDtzhiRJknT0MvKOwk2OsO3iYz0QSZKkP+IXr9QEQXADcCNwThAEn/5sVx7go8wemCRJ0tH4taefXgWmAA8DvX+2fVcYhj9k6qgkSZKO0i9GTRiGO4AdQMfjNxxJkqTfJyOvqZEkSfrTM2okSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAmxWT2AnwvDkKTkA1k9DOmo5M71p/prJGXYgm9+zOohSEdt1/6UX9znlRpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkhQJRo0kSYoEo0aSJEWCUSNJkiLBqJEkSZFg1EiSpEgwaiRJUiQYNZIkKRKMGkmSFAlGjSRJigSjRpIkRYJRI0mSIsGokSRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDUnsErlz6V2zarUS4jngnoJANx/3z3EV61EnVrVuLJ9O7Zv3w5AUlISN/TsTu2aVambUJ0P5sxOe5w2LS+lbkJ1asXHcfstN5Kampol81H2lJqaSo34arRs0RyAH374gaYXNaF8uTI0vagJP/74I3BwDXfv1pWqcZWpXi2O999/PwtHrezgobtv5pIaZbmqad20bfff3I1OlzSg0yUNaHN+HJ0uaQDAjh9/4KaOLWlUsRiD+tyddvy+vYn8rVt72jdO4MqL6jD0kQfS9r324tN0bFKbq5udz81XtWbzxg3Hb3IRZdSc4N6Z8h4fzVvE7I/mAdCw0YXMW7iUufOXcG6ZMjw+6BEAhr/0IgCfLFjK+IlT+XvvXhw4cACAEaNf4+N5i5m3cCnbtm3j7bf+nTWTUbb0xBNDKF++QtrnjzwykEaNG7Pms7U0atyYRx4ZCMCLL74AwNJly5k6bTp39/pb2hqWMsOll13JP4e/kW7bQ0+9xMjJcxg5eQ4Nm7XggmYHY/ykXLnoeed93Hzfg4c9zpXX3szYGfMYMWk2ny6cx9z3pwNQtmIVXp4wk9FTP6TRxS15emDfzJ9UxBk1EdP4wibExsYCULNmAps2bQRgzZrVXNCwEQBnFixIvtNOY/GihQDkzZsXgJSUFJKSkgiCIAtGruxo48aNTJ78Dt2690jbNnHCeDp16gxAp06dmTB+HACrV62iUaPGABQ8tIYXLlx4/AetbKNaQl3ynnb6EfeFYciMyeO4qMVlAPwl9ynE1axNrly50h138l9yE1+nPgA5TzqJcpWqsHXztwDE16nPyX/JDUDFajXY+t23mTWVbMOoOYEFQUDrFhfToG4tXh72wmH7R40cTpOLmgFQuXIVJk+aSEpKCuvXf8XSJYvTggegdctLKF2iMKfmyUPrNpcdtzkoe7vzjtsZOPBRYmJ++la0ZcsWChUqBEChQoXYunUrAFWqxDFhwnhSUlL46quvWLxoERs3eLleWWPp/LnkL1CQYqVKZ/icXTt38OGMadSod8Fh+yaOHU2dCy48lkPMlmKzegD6/d6dMZtChQvz/dattGrRjLLlylPv/IM/ETz2yMPExsbSvsOVAFzTuSuffbaGC+olUKx4CWol1CE2x0//+cdNmMy+ffvo0bUTs9+fRaPG/uVS5po0aRIFCxYkPj4+Q6+P6dqtG6vXrCahVg2KFy9BnTp1065KSsfb9Ilv0qRF2wwfn5KSQp9be3B5l54UKV4y3b6pb7/OmuVLGDpm0jEeZfbjd4QTWKHChYGDTyc1b9GaRQsXUO/8+rwyeiRTp7zDxMnvpj2VFBsby8BHB6ede2HD+pQ+99x0j3fyySdz8aXNeWfSBKNGme7jjz9i4sQJTJlyMKh37txJp2uu5qyzzmLz5s0UKlSIzZs3U7BgQeDgGn788X+mnX/++XU5t0yZrBq+srGUlBTenzqJ4RNnZvicgffdTrGSpenQ7YZ02+d/+D7Dnx7M0DGTOOl/nrrS0fPppxPUnj172LVrV9rHM2dMp8J5FZn+7jT+9fggxr7xNrlz5047PjExkT179gAwc8Z7xMbGUr7CeezevZvvNm8GDv5FnT5tKmXLljv+E1K2M2DAw3z9zUa++HI9r7w6hoYNGzFy1Giat2jJyJEjABg5cgQtWrYC0q/h6dOnExsby3nnnZdl41f2teCj9ylRugwFCxXJ0PHPDerPnl07ub3PgHTbP1v5KY/+/U4ee+FV8hc4MzOGmu14peYEtXXrFq7q0A6AlJRULr+iA00uakpcpfIk7d9Pq+YHX0tTs1YC/3pyKN9/v5U2LS8lJiaGwoUL8/yw4QAk7tlD+8vbkJS0n9TUAzS44K90v/a6rJqWxD339KZDhyt4+aVhFCtenLFjD959snXrVi65uOnBNVykCCNGjMrikSrq+tzag8WffMT2H/9DyzoV6XF7b1q2v4b3Jr5Nk5aHv/awzflx7Nm9i5TkZOZMf4chI9/klFPzMPzpwZQoXYYuzf8KQLtOPWjZoRNPPdyXxD17+PtNXQE4q3BRHnvx1eM5xcgJwjDM6jGkqV49PvzvrcnSiSJ3Ln820IlpwTc/ZvUQpKNWv2KJdcl7dh7xuWeffpIkSZFg1EiSpEgwaiRJUiRkWtQEQVAsCIJZQRCsDoJgZRAEt2XW15IkScrMVzimAH8Lw3BxEAR5gEVBEEwPw3BVJn5NSZKUTWXalZowDDeHYbj40Me7gNVAxm7qlyRJOkrH5TU1QRCUBKoB3q8tSZIyRaZHTRAEpwJvAreHYbjzCPt7BkGwMAiChdu2bcvs4UiSpIjK1KgJgiAnB4PmlTAM3zrSMWEYPh+GYY0wDGsUKFAgM4cjSZIiLDPvfgqAYcDqMAwfz6yvI0mSBJl7paYecA3QKAiCpYf+XJKJX0+SJGVjmXZLdxiGHwJBZj2+JEnSz/mOwpIkKRKMGkmSFAlGzQnmqSf/Ra34OBJqVKVr56vZt29fuv0//vgjV7ZvR51a1fhr/TqsWrkibd+N1/XgnBKFSahRNd05fe6/lzq1qtGzR5e0ba+9OpqhTz+RqXNR9jF16lTOq1COcmXP5ZFHBh7xmPfff5/46lWpUrkiDRtekLb9iSeGEFelElUqV2TIkH+lbe/d+x6qVa1Cl86d0raNHjWKJ54YknkTUaRt+XYjN3VsSYcLE7jyojqMfflZAD5ftZwebZrQ6ZIGdG3ZiJVLFx3x/NeGDeXKi+pwVdO69Lm1B/v3H/z+/OSAPrRvnMDVzc7nnuuuYdfOHQAsW/gJVzc7n26tGrNh/ZcA7Nq5g9s7XUYYhsdhxtFj1JxAvt20ieeGPs3sDz9h3sKlHEhN5c03xqY7ZvBjA6lcJY6585fw/Isvc0+vO9P2XXVNZ94aNynd8Tt27GDeJ3OZO38JqamprFyxnL179/Lq6JFc2/OG4zIvRVtqaiq33nITk96ZwvIVqxg75jVWrUr/21K2b9/OLTffyNvjJvDp8pWMHfsGACtWrGDYiy8w95P5LF6yjHfemcTatWvZsWMHc+d+zJKln5Kamsry5QfX7YiRw7nhhhuzYpqKgByxsdz69/9jzHvzeOGtd3lz5DC+WruGpx/uS/fb7mbk5Dlce8e9PD2w32Hnbv3uW94Y/jwvTZjJK9M+JjU1lfcmHnwnk1rn/5VXpn3E6KkfUrxUaUYO/ScAr704lIefGcH1d93P26NfAuDlJx6j0413cvAGYh0to+YEk5KSwt69e0lJSSExMZGzCxVOt3/N6tX8tWFDAMqWK8/XX3/N1i1bAKh3fn1Oz58/3fExMTEkJSURhiH79u4jZ86cDPnnYK6/4WZy5sx5fCalSJs/fz6lS5/LOeecw0knncQV7TswYcL4dMe89tqrtG7TluLFiwNQsGBB4OB6TkioTe7cuYmNjaVBgwsYN+7tdOt279695MyZk0GDHuOWm2913ep3K1DwbMpVigPglFPzUPLcsnz/3WaCIGDP7l0A7N61kwJnnX3E81NTU9i/bx8pKSns27eXAgUPHpfQoBGxsQfvy6lYrQZbv/sWgNjYWPbv28e+vXuJzZmTjV9/xfdbNlO9dr3MnmpkGTUnkMJFinDL7XdQsdw5lDmnGHnz5aXxhU3SHVO5chUmjB8HwMIF89nwzdds2rTxFx8zT548tGrdlvNr16BEyZLkzZuPxYsWcmmLlpk6F2Uf327aRLFixdI+L1qkKN9u2pTumLWff872H3+kUaO/UqtmPKNGjgSgYqVKfPDBHP7zn/+QmJjIlCmT2bhhA3ny5KFt28uoEV+NkqVKkS9fPhYuWEDLVq2O48wUZZs3fsPnqz6lYtV4bu8zgKce7kurupV4ckAfbujV57DjC55dmCuvvZk29arQIqECp+bJS0KDRocdN+n1V6hzwYUAdLrxDgbedztjX36Wdp168Nygh+h5532ZPrcoM2pOID/++COTJ01k+aq1fP7FNyTuSWTMa6+kO+aOu+5m+48/Ui8hnueefZoqcVXTfkL4JbffeRcfzVvEgIGP8dCDffn7P/oy4uVhdL66I48OHJCZU1I2cKTXBvzvpfWUlBQWLV7ExInvMHnKNPr3/z8+//xzKlSoQK9e99CsaRMuuaQZcVXiyHFoPffqdTeLFi9l0KDB9O3zD/o98CDDXnyRDu2voH//h47L3BRNiXt2c+8Nnbn9HwM4JU9e3hr9Mrfd35/xH6/gtvsfYkDvWw87Z+eO7XwwfQpvzlnCxE9WsS8xkalvv57umOFPDSZHbCxNW18OQNnzKvPi29N5+rUJbPrmawqcdTZhGHL/zd3od/t1/PD91uMy3ygxak4g78+aQYkSJSlw5pnkzJmTFq1aM++TuemOyZs3L888P4yP5i3i+ReH859t2yhRslSGHn/Z0iUAnFumLK+9OpoRo19j1aqVrFu39pjPRdlHkaJF2bBhQ9rnGzdtpFDhwocd07RpM0455RQKFChA/foN+HTZMgC6de/OgoWLef/9OZyePz9lypRJd+6SJQfXbdmyZRk1eiRjxr7OyhUrWLvWdaujl5KczH03dKZpq3b8tVkLACa/9Vrax40vbc2qZYe/UHjBh+9TqFhxTj+jALE5c3JB0+YsXzw/bf87b77GRzOn8cC/njss6sMwZPhTg+h6Sy+GPfEoPe7oTdM2l/P68OczcabRZNScQIoWLcaCBfNJTEwkDENmvz+TcuXLpztm+/btJCUlATDi5WHUPf988ubNm6HHf+jBfvz9H/1ITk4mNTUVOPiam72Jicd2IspWatasybp1a/nqq69ISkri9bFjaPE/T2+2bNmKDz/8IO21YvPnz6N8hQoAbN168KfVb775hnFvv0WHDh3TnfvfqzT/u24TXbc6SmEY0v+eWylxblk69rgpbXuBgmezZN5HACz8eA7FSpY+7NyzCxdl5ZKF7Nt78Pvzwo/nULJ0WQDmzn6P0c8O4dEXXuXkv+Q+7NzJb75G3UYXkTffaezbu5eYIIaYIIZ9+1zDRyvT3lFYx17NWgm0at2W+nVrERsbS5W4OLp2u5ZhLzwHQPdrr+Ozz1ZzXY9u5MiRg/LlK/DUMz+VftfOV/PhnNn85z/bKH9uSe67vw+dunQDYNKE8VSPr5H2E3SthNrUrlmVipUqU7lK3PGfrCIjNjaWIU88xSUXNyU1NZUuXbtRsWJFnnv24O2y111/PRUqVKBp02ZUq1qFmJgYunXvQaVKlQC4/PLL+OE//yFnzpw88eTTnH766WmPPX7cOGrUrEnhQ+u2du06VI2rTOXKVYiLc93q6Hy6cB5T3x5L6XLn0emSBgBc3+sf3PvwEP754L2kpqRwUq5c9B5w8O6l77ds5uHet/H4y69TsVoNGl7cks7NGxIbm4Oy51WhVcfOAAzuew/JSfu57Zq2wMEXC9/T/+CvRNy3N5HJb45hyMg3AejY/UbuvbEzOXOexINDXjje/wpOeMGf6V746tXjw9kfzcvqYUhHJXcufzbQiWnBNz9m9RCko1a/Yol1yXt2ljnSPp9+kiRJkWDUSJKkSDBqJElSJBg1kiQpEowaSZIUCUaNJEmKBKNGkiRFglEjSZIiwaiRJEmRYNRIkqRIMGokSVIkGDWSJCkSjBpJkv6/vXsLtawu4Dj++zUVZU5ZdIOgxiyVEpu0ie5ZyBB0oSIhCWpIrAy6EAlCFwqCBHuKoLQeDKSoNEGM8NKDqYyRTWqWN7rRQw/Wg0XhS/17OEvZncbxzISznb+fDxzO3mf99/r/12Gd4ctae85hCqIGAJiCqAEApiBqAIApiBoAYAqiBgCYgqgBAKYgagCAKYgaAGAKogYAmIKoAQCmIGoAgCmIGgBgCqIGAJiCqAEApiBqAIApiBoAYAqiBgCYgqgBAKYgagCAKYgaAGAKogYAmIKoAQCmIGoAgCmIGgBgCqIGAJiCqAEApiBqAIApiBoAYAqiBgCYgqgBAKYgagCAKYgaAGAKogYAmIKoAQCmIGoAgCmIGgBgCqIGAJiCqAEApiBqAIApiBoAYAqiBgCYgqgBAKYgagCAKYgaAGAKogYAmIKoAQCmIGoAgCmIGgBgCqIGAJiCqAEApiBqAIApiBoAYAqiBgCYgqgBAKYgagCAKYgaAGAKogYAmIKoAQCm0DHGutfwoLb3JvnjutcxqWcm+cu6FwEHyXnLkcq5+8h5wRjjWfvb8KiKGh45bW8eY7xi3euAg+G85Ujl3F0Pt58AgCmIGgBgCqLmseOidS8ADoHzliOVc3cNRM1jxBjDDxiHVdvT2l65PH5H2/MOMPaYth/d/PWHO2/bfqHtp7f69U1jLm77ngON2TR+R9vbtzqexzb/5q6HqAEOStttB/uaMcYVY4zzDzDkmCT/EzUAB0PUAEkevBJxZ9tvt72t7aVtj1q2/aHt59vekOSMtrvb7m27r+0P2h69jHvLso8bkrx7Zd972n5tefyctpe3vXX5eE2S85Mc1/aWthcs485t+/NlLV9c2ddn2t7V9tokJ2zhuM5e9nNr28seOKbF6W2vb3t327ct47e1vWBl7g//v99b4PAQNcCqE5JcNMY4Ocnf8t9XT+4fY7wuybVJPpvk9DHGKUluTvKptk9K8s0kb0/y+iTPfYg5vprkujHGy5KckuTXSc5L8tsxxs4xxrltdyd5cZJXJtmZ5NS2b2h7apL3Jnl5NqJp1xaO6YdjjF3LfHckOWtl244kb0zy1iTfWI7hrCT3jTF2Lfs/u+2xW5gHWLPHr3sBwKPKn8YYNy6PL0ny8SRfWZ5/b/n8qiQvSXJj2yR5YpK9SU5M8vsxxj1J0vaSJB/azxxvTvL+JBlj/CvJfW2fvmnM7uXjl8vzo7MROduTXD7G+OcyxxVbOKaT2n4pG7e4jk5y1cq2748x/p3knra/W45hd5KTV95v87Rl7ru3MBewRqIGWLX5t3GuPv/H8rlJrhljnLk6sO3O/bz+UDXJl8cYF26a45OHMMfFSd45xri17Z4kp61s29/xNsnHxhir8ZO2Ow5yXuAwc/sJWPX8tq9eHp+Z5Ib9jLkpyWvbvihJ2h7V9vgkdyY5tu1xK6/fn58kOWd57ba2T03y92xchXnAVUk+uPJenee1fXaSnyZ5V9snt92ejVtdD2d7kj+3fUKS923adkbbxy1rfmGSu5a5z1nGp+3xbZ+yhXmANRM1wKo7knyg7W1JnpHk65sHjDHuTbInyXeXcTclOXGMcX82bjf9aHmj8EP9HbdPJHlTjij6qgAAAI5JREFU218l+UWSl44x/pqN21m3t71gjHF1ku8k2buMuzTJ9jHGvmzcBrslyWVJrt/CMX0uyc+SXJON8Fp1V5Lrkvw4yUeWY/hWkt8k2bf8F+4L46o2HBH87ScgyYO3V64cY5y05qUAHBJXagCAKbhSAwBMwZUaAGAKogYAmIKoAQCmIGoAgCmIGgBgCv8BxzU958UvB38AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Print the Confusion Matrix\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "ax.matshow(conf_mx, cmap=plt.cm.Blues, alpha=0.3)\n",
    "for i in range(conf_mx.shape[0]):\n",
    "    for j in range(conf_mx.shape[1]):\n",
    "        perc = str(round((conf_mx[i, j]/conf_mx.sum())*100,2)) + \"%\"\n",
    "        ax.text(x=j, y=i, s=str(conf_mx[i, j])+\"\\n\\n\"+perc, va='center', ha='center')\n",
    "\n",
    "plt.xlabel('predicted label')\n",
    "plt.ylabel('true label')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Trying to Optimize the Random Forest Model with Baeysian Optimization with gaussian process**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a optimize callable function to be fed to sklearn.gpminimize and it takes a single list of parameters and it returns a value\n",
    "def optimize(params,param_names,x,y):\n",
    "    params = dict(zip(param_names,params)) \n",
    "    model = RandomForestClassifier(**params) #It takes the params as parameter\n",
    "    kf = StratifiedKFold(n_splits=5)\n",
    "    accuracies = []\n",
    "    \n",
    "    return -np.mean(cross_val_score(model, X_train, y_train, cv=StratifiedKFold(n_splits=3),n_jobs=-1, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_space = [\n",
    "    space.Integer(30,60,name=\"max_depth\"),\n",
    "    space.Integer(100,300,name=\"n_estimators\"),\n",
    "    space.Categorical([\"gini\",\"entropy\"],name=\"criterion\"),\n",
    "    space.Real(0.01,1,prior=\"uniform\",name=\"max_features\"),\n",
    "    space.Integer(1,10,name=\"min_samples_leaf\")\n",
    "]\n",
    "param_names = [\"max_depth\",\"n_estimators\",\"criterion\",\"max_features\"]\n",
    "optimization_function = partial(optimize,param_names= param_names,x = X_train,y = y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at random point.\n",
      "Iteration No: 1 ended. Evaluation done at random point.\n",
      "Time taken: 508.4996\n",
      "Function value obtained: -0.7774\n",
      "Current minimum: -0.7774\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "Iteration No: 2 ended. Evaluation done at random point.\n",
      "Time taken: 1003.3217\n",
      "Function value obtained: -0.7773\n",
      "Current minimum: -0.7774\n",
      "Iteration No: 3 started. Evaluating function at random point.\n",
      "Iteration No: 3 ended. Evaluation done at random point.\n",
      "Time taken: 430.9926\n",
      "Function value obtained: -0.7781\n",
      "Current minimum: -0.7781\n",
      "Iteration No: 4 started. Evaluating function at random point.\n",
      "Iteration No: 4 ended. Evaluation done at random point.\n",
      "Time taken: 370.7828\n",
      "Function value obtained: -0.7755\n",
      "Current minimum: -0.7781\n",
      "Iteration No: 5 started. Evaluating function at random point.\n",
      "Iteration No: 5 ended. Evaluation done at random point.\n",
      "Time taken: 482.4592\n",
      "Function value obtained: -0.7748\n",
      "Current minimum: -0.7781\n",
      "Iteration No: 6 started. Evaluating function at random point.\n",
      "Iteration No: 6 ended. Evaluation done at random point.\n",
      "Time taken: 736.1530\n",
      "Function value obtained: -0.7788\n",
      "Current minimum: -0.7788\n",
      "Iteration No: 7 started. Evaluating function at random point.\n",
      "Iteration No: 7 ended. Evaluation done at random point.\n",
      "Time taken: 162.5046\n",
      "Function value obtained: -0.7778\n",
      "Current minimum: -0.7788\n",
      "Iteration No: 8 started. Evaluating function at random point.\n",
      "Iteration No: 8 ended. Evaluation done at random point.\n",
      "Time taken: 730.7393\n",
      "Function value obtained: -0.7785\n",
      "Current minimum: -0.7788\n",
      "Iteration No: 9 started. Evaluating function at random point.\n",
      "Iteration No: 9 ended. Evaluation done at random point.\n",
      "Time taken: 6.9430\n",
      "Function value obtained: -0.7724\n",
      "Current minimum: -0.7788\n",
      "Iteration No: 10 started. Evaluating function at random point.\n",
      "Iteration No: 10 ended. Evaluation done at random point.\n",
      "Time taken: 725.1311\n",
      "Function value obtained: -0.7766\n",
      "Current minimum: -0.7788\n"
     ]
    }
   ],
   "source": [
    "result = gp_minimize(optimization_function,dimensions=param_space,n_calls=10,verbose=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score = -0.7788\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score = %.4f\" % result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 52,\n",
       " 'n_estimators': 271,\n",
       " 'criterion': 'gini',\n",
       " 'max_features': 0.5783538396142335}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_params = dict(zip(param_names,result.x))\n",
    "rf_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(result, open('bayessian_optimization_rf.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[52, 271, 'gini', 0.5783538396142335, 7]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SGD Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.73186869, 0.73141414, 0.73282828])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd = SGDClassifier()\n",
    "cross_val_score(sgd, X_train, y_train, cv=StratifiedKFold(n_splits=3), scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(sgd, X_train, y_train, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAI4CAYAAACSixhGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3QV1cJA8T0hoCJVEanSpEkPBFARuyCKVCk2pGNv2BVR7KIPsT4UbIiAlWbHipRQFcQCKj46KB2CkDDfH/HlmQ9UUEPgZP/WYpk75eYcvcR9Z+5MojiOkSRJ2t8l5PQAJEmS/glGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglETuCiKmkdR9E0URQujKLohp8cj7Y4oioZFUbQqiqJ5OT0WaXdFUVQ2iqIPoyj6KoqiL6MouiKnx5TbRN6nJlxRFOUBvgVOBZYA04HOcRzPz9GBSX8iiqKmwCbg+TiOa+b0eKTdEUVRSaBkHMezoigqCMwEWvszd+/xSE3YGgIL4zj+Po7jbcBIoFUOj0n6U3EcfwKsyelxSHsijuPlcRzP+vXrjcBXQOmcHVXuYtSErTSw+DePl+BfMEnKdlEUlQfqAdNydiS5i1ETtmgXyzzfKEnZKIqiAsCrwJVxHG/I6fHkJkZN2JYAZX/zuAywLIfGIknBi6IoLxlB82Icx6/l9HhyG6MmbNOBylEUVYiiKB/QCRibw2OSpCBFURQBQ4Gv4jh+KKfHkxsZNQGL4zgNuBR4h4wPrI2O4/jLnB2V9OeiKHoJmAJUjaJoSRRF3XN6TNJuOBY4HzgpiqI5v/5pkdODyk28pFuSJAXBIzWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNblEFEW9cnoM0p7ydav9la/dnGHU5B7+BdP+yNet9le+dnOAUSNJkoKwT92npnCRonHxEqVyehhBWr9uLYWLFM3pYQSpcIGDcnoIwVq9ejWHHXZYTg8jWPvQj//grP5pNYcV87WbHebO/WLDtm3bCu9qXeLeHswfKV6iFA8PGZnTw5D2SLMmtXJ6CNJfsj19R04PQdpjJYsXW/V76zz9JEmSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAk5vQAtPtWr1rBg3fdzNo1P5GQkEDzlu1o1f48vl/4DY89OIDU1C0cXqIU1956L/kPLgDAD999y6MD72DLls1EUcSgf79EvgMOYPv27Twx6G7mzplBQkLEBT0u49jjT2XVimUMuq8f69etpWChwvS9+W6KFS+RwzNXblGpYnkKFixInjx5SExMZFrKDDp36si3334DwLp16yhSpAgzZ83J4ZEqt1m8eDE9ul3IyhUrSEhIoFuPnlx62eXcecftDBv2NIcVOwyA2wfcSfPTWzB9egqXXtQHgDiOufnWfrRq3QaAs848nRXLV5CWlsaxTZowaPCj5MmTJ8fmFpIojuOcHkOmytVqxA8PGZnTw9hnrfl5NWt+Xs2RVY5iy5bNXNGzE7feNYiH7r6F7hdfQ626DXh3wuusXLGU87tfSnpaGpf37Mg1N99NxSOrsmH9Og4ukPE/jOHDHmPHjh1c0OMyduzYwcYN6ylcpCh397uGhsc05ZTmrfh81jTee3MMfW+5O6envk9r1qRWTg8hGJUqlmdaygyKFSu2y/V9+15D4cKFufXWfnt5ZGHanr4jp4ew31i+fDkrViynXr0kNm7cyDGNkhn9ymu8+srLHFygAFddfU2W7bds2UK+fPlITExk+fLlNGpQj+9/XEJiYiIbNmygUKFCxHFM545n07Zdezp07JRDM9v/lCxebOHatWsq72qdp5/2I4ccehhHVjkKgPz5D6ZsuQr8vHoVSxYvomad+gDUSz6azz5+H4BZM6ZQvlIVKh5ZFYBChYtkvht478036HBudwASEhIoXKQoAIt//J66SY0AqF2vIVM/+3DvTVD6A3Ec88rLo+nUqXNOD0W5UMmSJalXLwmAggULUq1aNZYtW/q72+fPn5/ExIyTIb9s3UoURZnrChUqBEBaWhrbt23Lsk5/j1Gzn1q5fCnfL/iaqkfVolyFI5n62UcATPrwXX5atQKApYsXERFxa98+XN6jA6+MGAbApo0bAHhh6GNc3qMDd/e7hrVrfgagQqUqfPZJRhRN/nQiqVs2s2H9ur08O+VWURRxevPTaJhcn6eGDMmy7tNPP+Xwww+ncuVdvkGT9pofFy1izudzSG6Y8QbwySceIzmpLr17dmft2rWZ26WkTCOpTi0aJNVh8KOPZ0YOQMszmnNE6RIUKFiQtu3a7/U5hMqo2Q+lbtnCXf2upudl15H/4AJcef0dTHh9JJf37Ehq6mYS8+YFID09nflzZ9H3lnu4/9HnmPLpB8yZOZX09HR+Wr2So2rVZfDTo6leow5DH38QgO4XX8PcOTO5rHsH5s2ZwaGHFfdcr/aaTz79jOkzZjF+wls88cRjfPLJJ5nrRo18iY4epVEO27RpE507ns0DAx+iUKFC9Ozdh/lfL2DajFmUKFGSG67rm7ltw4aNmPX5XCZNnsYD99/H1q1bM9eNm/A2P/xnKb/88gsfffhBTkwlSEbNfiYtbTt397uaE085g2ObngJA2XIVuPPBfzP4qVEcf/LplCxVFoBihx1OzboNKFykKAceeBANGh/Hd99+RaHCRTjgwAM5+riTAWhy4ml8t+ArAA4tVpxb7vwXjwwdzQU9Lgfg4AIFc2Cmyo1KlSoFQPHixWnVug3Tp6cAGYfpX3/9NTp06JiTw1Mut337djp3bE/HzufQuk1bAA4//HDy5MmT8eHh7j2YMX36TvtVq16dgw8+mC+/nJdl+YEHHsiZZ7Zk3Lixe2X8uYFRsx+J45iH77uNsuUq0KbjBZnL163NOHW0Y8cORj4/hNPPOhuApIbHsui7b9m6NZX0tDTmfj6DsuUrEUURjY45gblzMv7yzZk5jbLlKgKwft1aduzI+PDg6Bef5tTT2+zNKSoX27x5Mxs3bsz8+r333qVGjZoAvP/++1StVo0yZcrk5BCVi8VxTJ9ePaharTpXXHlV5vLly5dnfj1mzBscVaMGAIt++IG0tDQAfvzxR7799hvKlSvPpk2bMvdJS0vj7bffomrVantxJmHzku79yPy5s/ng3fGUr1iZS7tnhEuXnpezbMmPjH99FADHND2ZU1u0BqBgwUK07nABV/U+hyiCBo2Oo+HRTQHo2vtKBt51E0MeuZ/CRYpy5Q0DAJg7ZzrPDRkMUUTNOklcfOXNOTBT5UYrV66kfbuMiE5LS6NT53No3rw5AKNHjaRTR089KedMnvwZI14cTs2atWjUIOMDw7cPuJPRo0byxeefE0UR5cqV45HHn8zY/rNJDHzgfvLmzUtCQgIPD36UYsWKZbzO27Zm2y+/kJ6ezvEnnkjPXr1zcmpB8ZJu6W/ykm7tr7ykW/sjL+mWJEnBM2okSVIQjBpJkhSEbI2aKIqaR1H0TRRFC6MouiE7v5ckScrdsi1qoijKAzwGnA4cBXSOouio7Pp+kiQpd8vOIzUNgYVxHH8fx/E2YCTQKhu/nyRJysWyM2pKA4t/83jJr8skSZL+cdkZNbv6taM73RQniqJeURTNiKJoxvp1a3exiyRJ0p/LzqhZApT9zeMywLL/v1Ecx0PiOG4Qx3GDwkWKZuNwJElSyLIzaqYDlaMoqhBFUT6gE+Bv7ZIkSdki2373UxzHaVEUXQq8A+QBhsVx/GV2fT9JkpS7ZesvtIzj+E3gzez8HpIkSeAdhSVJUiCMGkmSFASjZj+Qnp7OZd070P+GSwH4fuE3XHPReVx8YVtuv+FStmzetMv9xrwynIsvbMNFXdrwxssvZC5/8ZnHuaDdKVza/Wwu7X4206d+CsD8ubO5pGs7ruzVmWVL/gPApo0buLVvH+J4p6vxpb/k7bff5qjqVala5Ujuu+/endZ//fXXHHvs0eQ/6AAefHBg5vLVq1fTtGkT6tSuyZg33shc3qZ1K5Yt2+nCSukfkZ6eTuPk+rRt3XKndS+NeJHkpLokJ9XlhKZN+OLzzzPXrVu3js4dz6ZOzaOoW6sGU6dOAeDmG28gOaku3bt2ydx2xPAXePSRwdk/mVzAqNkPjH3lRcqWq5D5ePD9/bmw95U8/uxrHH3cybw68tmd9ln0/QLeGf8qDz05gkeHvkzKlE9YuuTHzPWtzj6PR4e+zKNDXya58XEAvDbqeW4a8BBdel7Om2NGAzDy+SF0OK8HUbSr2w5JeyY9PZ3LL7uE8RPeYu68+Ywa+RLz58/Pss0hhxzCoEGDufqavlmWjxz5Ehec34VJn01h4IMPADBu3DjqJSVRqlSpvTYH5S6PPjKYqtWq7XJd+QoVeHfih0yfNYcbb7qZSy7uk7mu79VXclqzZnw+bz4pM2dTrVp11q9fz9SpU5g+aw7p6enMmzuX1NRUXnjheXr3uWhvTSloRs0+7qdVK5g+9ROandk2c9mSxYuoWac+APWSj+azj9/fab/FP/5A1aNqc+CBB5EnMZFadRow5ZOJf/i9EhMT2fbLL2z9ZSt5EhNZvnQxP/+0ilp1G/yzk1KulZKSQqVKR1KxYkXy5ctHh46dGDt2TJZtihcvTnJyMnnz5s2yPG9iXlK3pvLLL7+QkJBAWloagwcPom/fa/fmFJSLLFmyhLffepOu3brvcv3RRx9D0aIZ91dr2KgxS5cuAWDDhg1MmvQpF3bN2C9fvnwUKVKEhIQEtm3bRhzHpKamkjdvXv714EAuvuTSnV7v+muMmn3ckEfvp2ufq4mi//2nKlfhSKZ+9hEAkz58l59Wrdhpv3IVjmTe57PYsH4dW7emMmPqp6xetTJz/fjXR3JJ13YMurcfGzduAODsc7vzyMDbGfPycFq26cTzTz/Ced0vyd4JKldZtnQpZcv+756cZUqXYdnSpbu1b+dzzuHdd97hjBbN6devP0888Tjnn3cB+fPnz67hKpe79pqruOuee0lI+PP/VT77zDCaNWsOwA/ff0+xYofRq0c3GifX56LePdm8eTMFCxakdZu2NE6uT/kKFShUuDAzZ0yn5Vn+WsR/ilGzD0uZ/DGFixxC5apZf7n5ldffwYTXR3J5z46kpm4mcReFf0T5irQ/pyu3XNOLftdeRIUjq5InMQ8ALVp15OkRE3hk6MsUPbQYQx/L+NxCpcrVeOiJF7n34aGsWL6EQw49DOKYe/tfywN33sjaNT9n/6QVtF19Nmt3T20WLlyYceMnMC1lBklJSUwYP5627drRu1dPOpzdnilTpvzTw1Uu9uaE8RQvXpykpPp/uu3HH33Ic88M4867Mz4jlpaexpzZs+jZuw9Tp88k/8EHM/D++wC4pu+1TJsxi/vuH8gd/ftx622388ywpzm3c0fuvfuubJ1TbmDU7MPmz5vDtMkf0bVjc+674zq+mJXCA3feSNlyFbjzwX8z+KlRHH/y6ZQsVXaX+zc7oy2Dnx7N/Y88S8GChShV+ggAih5yKHny5CEhIYHmZ7bj26/nZtkvjmNGPv8Unbr0ZsSzT3Ju14s58dQzGPvqi9k9ZQWudJkyLF78v99zu2TpEkr+hc/DDBhwBzfedDMjX3qJpKT6PD10GLfcctM/OVTlclMmT2b8+HFUrVyRC847h48+/JCuXc7fabu5X3zBRX168fKrr3PooYcCULp0GUqXKUPDho0AaNO2HXPmzMqy35zZswGoXKUKLw4fzosvjeLLL+excMGCbJ5Z2IyafdiFva7g+Vfe55lRb3N9v/upndSQa2+5h3VrM46Y7Nixg5HPD+H0s87e5f7/3W7VyuVM/nQix5/SAoA1P6/O3Gbypx9QrkLlLPu9//ZYko8+joIFC/HL1q0kJEQkJCTwyy9bs2OaykWSk5NZuHABP/zwA9u2bWP0qJG0bHnWHj3HggULWL5sGccffzxbUreQkJBAFEVs3errU/+cAXfdzXc//IdvFnzP88NHcMKJJ/LMcy9k2eY///kPnTq2Z+gzz1G5SpXM5SVKlKBMmbJ8+803AHz0wQdUq571iPsdt2ccpdm+fTvp6ekAJCQksGXLlmyeWdiy9Y7Cyh4fT3yL8a+PAuCYpidzaovWAPz80yoG39+f2+9/HIC7b72aDRvWk5iYyEVX3kTBgoUAGPbEv/h+4ddEUUTxEqW4rG+/zOfeujWViW+P5c4HnwSgdYfzuevWq0nMm5fr+923N6epACUmJvLw4EdpcXoz0tPTubBrN2rUqMG/n8x4vfXu04cVK1bQqGEDNmzYQEJCAoMfHsTcefMpVCjj9XvrLTcz4M6Mw/SdOnWmbdvWPPLIw/Tvf0eOzUu5x1NDMl6rPXv14Z67BrDm55+58rKM220kJiby2dQUAB7618N07XI+27Zto3yFCgx5eljmc4wd8wb16ydnXrXXqHFjGtSrQ81atahdp85enlFYon3p/iOVq9WIHx4yMqeHIe2RZk1q5fQQpL9ke/qOnB6CtMdKFi+2cO3aNZV3tc7TT5IkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQpCYk4P4LcKHnwQJzQ+KqeHIUm5QmJClNNDkPZY9AcvW4/USJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIib+3IoqijUD834e//jP+9es4juNC2Tw2SZKk3fa7URPHccG9ORBJkqS/Y7dOP0VR1CSKoq6/fl0siqIK2TssSZKkPfOnURNF0W3A9cCNvy7KBwzPzkFJkiTtqd05UtMGOAvYDBDH8TLAU1OSJGmfsjtRsy2O45hfPzQcRdHB2Tsk7Y6tW7fS9NjGNGqQRIO6tbnzjv5Z1g966EEOPiCRn376CYAfFy3i0MIFaJxcn8bJ9bn8koszt+3f7xaqVCpP8UMK780pSDupVLE8devUon5SXRo1bADAnDlzOOaYxpnLUlJScniUEgwa9C/q1K5J3Tq1OO/cc9i6dStr1qyhebPTqF6tCs2bncbatWsB2L59O127XkjdurWpVfMo7rv3nhwefbh2J2pGR1H0b6BIFEU9gfeBp7J3WPozBxxwAG++8z7TZsxiyvSZvPfuO6RMmwrAksWL+WDi+5Q94ogs+1SoWImp02cydfpMBj/2eObyFmecyceTpuzV8Uu/5/2JHzJz1hympcwA4Ibrr+PWW29j5qw53Nb/Dm644bocHqFyu6VLl/LYo48wddp05nw+l/T0dEaNGsn9993LSSedxFdff8tJJ53E/ffdC8Arr7zMtl9+Yc6cL5iWMoOnnhrCokWLcnYSgfrTqInjeCDwCvAqUAXoF8fxI9k9MP2xKIooUKAAkPEuYPv2NKIo48r766+9hjvvuTfz8Z9p2KgxJUuWzLaxSn9HFEVs3LABgA3r11OqZKkcHpEEaWlppKamkpaWxpYtWyhVshTjxo3l/Au6AHD+BV0YO3YMkPEa3rx5c+Y++fLlo1Ah74qSHXb35ntzgU+BT379WvuA9PR0GifXp3yZkpx08skkN2zEhHHjKFmqNLVr19lp+x8X/cDRDRvQ7JQT+WzSpzkwYumPRVHE6c1Po2FyfZ4aMgSAh/41iOuvv5by5cpy3XV9uetuD90rZ5UuXZqrrr6GihXKUbZMKQoVLsypp53GypUrM98glixZklWrVgHQrl17Dj74YMqWKUXFCuW46uprOOSQQ3JyCsH63fvU/FcURT2AfsAHZNx475Eoiu6I43hYdg9OfyxPnjxMnT6TdevW0blDO+bO/YL777ubsRPe3mnbEiVL8vXCHzj00EOZPWsmHc9ux4zZX/huQfuUTz79jFKlSrFq1SqaNzuVqtWq8dqrr/Dgg/+ibbt2vDx6ND17dufdd9/P6aEqF1u7di3jxo5lwcLvKVKkCJ06duDFF3//ouCUlBQS8uThP4uXsnbtWk48oSknn3wKFStW3Iujzh1250jNtUC9OI4vjOO4C1CfjEu8tY8oUqQIxzU9ngnjxrJo0SIaJydRvUolli5ZwrGNk1mxYgUHHHAAhx56KAD1kupTsWJFFi74NodHLmVVqlTGqaXixYvTqnUbpk9P4fnnn6NN27YAtD/7bKb7QWHlsIkT36d8hfIcdthh5M2bl9Zt2jBlymQOP/xwli9fDsDy5cspXrw4ACNHjqBZs2bkzZuX4sWLc/QxxzBz5owcnEG4didqlgAbf/N4I7A4e4aj3bV69WrWrVsHQGpqKh9+MJE6dery45LlfPXtd3z17XeULlOGz6ZOp0SJEqxevZr09HQAfvj+exYuXEj5Cr5L0L5j8+bNbNy4MfPr9957lxo1alKqVCk+/vhjAD744AMqV66ck8OUKFv2CFKmTWPLli3EccwHH3xAtWrVOfPMlrzw/HMAvPD8c7RseRYAR5Q9gg8//JA4jtm8eTMp06ZRtWq1nJxCsP7odz9d/euXS4FpURSNIeOy7laAb5Vy2IoVy+nVvRvp6ens2LGDdu3bc/oZZ/7u9p9N+pQ7b+9PnsRE8uTJw+BHHss8p3vzjdczetRItmzZQuWK5biwazduvvW2vTUVCYCVK1fSvl0bIONDmJ06n0Pz5s0pUKAAV191BWlpaRxw4IE88eSQHB6pcrtGjRrRtm07GibXJzExkTp169GzZy82bdpE504deeaZYZQtewQjR40G4KKLL6FH927UrVOLOI7p0uVCateuncOzCFOUcQuaXazIuJPw74rj+PZ/ejBJ9RvEk6ZM+6efVspWByTmyekhSH/J7/38l/ZlxQ87dOGaNWt2ecj2j36h5T8eLZIkSdlld65+Ogy4DqgBHPjf5XEcn5SN45IkSdoju/NB4ReBr4EKwO3AImB6No5JkiRpj+1O1Bwax/FQYHscxx/HcdwNaJzN45IkSdojuxM123/95/Iois6IoqgeUObPdoqiaFgURauiKJr3t0YoSZK0G3Ynau6MoqgwcA3QF3gauGo39nsWaP7XhyZJkrT7/vSDwnEcj//1y/XAibv7xHEcfxJFUfm/NixJkqQ980c333uEjJvt7VIcx5f/EwOIoqgX0Aug7BFH/BNPKUmScqE/OlKzV34xRRzHQ4AhkHHzvb3xPSVJUnj+6OZ7z+3NgUiSJP0du/NBYUmSpH1etkVNFEUvAVOAqlEULYmiqHt2fS9JkqQ/vfrpr4rjuHN2PbckSdL/96dHaqIoqhJF0cT/3kQviqLaURTdkv1DkyRJ2n27c/rpKeBGfr2zcBzHXwCdsnNQ+n3Vq1QiOakujZPr0+ToRjutX79+Pe3btKJRgyQa1K3N8889m7muT68elCtTkgb16mTZ55abbqBh/Xr06HZh5rIRLw7nsUcGZ9c0lIu9/fbbHFW9KlWrHMl999270/qvv/6aY489mvwHHcCDDw7MXL569WqaNm1Cndo1GfPGG5nL27RuxbJly/bK2JW79OjRjVIlD6dunVq7XD927Bjq1atD/fr1aNQomUmTJmWuW7duHR07nE3NGtWpVfMopkyZAsCNN1xPvXp1uPDCLpnbDh/+AoMHP5y9k8kldidq8sdxnPL/lqVlx2C0e956932mTp/JpCnTdlo35MnHqVa9OtNmzOKt9yZy0/XXsm3bNgDOO/8C3hg3Icv269evZ9rUKaTMnE16ejrz5s0lNTWV4c8/R68+F+2V+WjZSLUAAB6nSURBVCj3SE9P5/LLLmH8hLeYO28+o0a+xPz587Nsc8ghhzBo0GCuvqZvluUjR77EBed3YdJnUxj44AMAjBs3jnpJSZQqVWqvzUG5R5cLLmT8hLd+d/1JJ53MrFlzmDlzNk89NZQ+vXtmrrvqqis5rVkz5n35FTNnzaF69eqsX7+eKVOmMHv256SnpzN3bsbP2+efe46LLrp4b0wpeLsTNT9FUVSJX2/EF0VRe2B5to5Kf10UsWnjJuI4ZvOmTRQtegiJiRkfnWpyXFMOKXpIls0TEhLYtm0bcRyzNTWVvIl5GfTQQC6+5DLy5s2bEzNQwFJSUqhU6UgqVqxIvnz56NCxE2PHjsmyTfHixUlOTt7p9Zc3MS+pW1P55ZdfSEhIIC0tjcGDB9G377V7cwrKRY5r2pRDDjnkd9cXKFCAKIoA2Lx5c+bXGzZsYNKnn9CtW8b1Mfny5aNIkSI7/7zNm5cHBz7ApZf58/afsjtRcwnwb6BaFEVLgSsB38LnkIiIs844nWMbN2TY00/ttL7PRZfwzTdfUal8WRrWr8sDDz5EQsLv/2cuWLAgrVq35eiGDShXvjyFChdm5owZnHnWWdk5DeVSy5YupWzZspmPy5Quw7KlS3dr387nnMO777zDGS2a069ff5544nHOP+8C8ufPn13Dlf7UG2+8Ts0a1Wl11pkMeWooAN9//z3Fih1G9+7daNAgiV69erB582YKFixI27ZtadAgifLly1O4cGFmzJjBWWe1yuFZhONPoyaO4+/jOD4FOAyoFsdxkziOF2X7yLRLEz/6hMnTpvP62PH8+8knmPTpJ1nWv//eu9SqXYfvFi1mSspMrr7yCjZs2PCHz3l132uZOn0m994/kAG338Ytt/Xn2WFDOf+cTtx3z13ZOR3lMnG8803D//vu9s8ULlyYceMnMC1lBklJSUwYP5627drRu1dPOpzdPvMzC9Le1Lp1G+Z9+RWvvvo6/W/rB0BaWhqzZ8+id+8+zJgxi4MPPpj7f/38WN9rr2PmzNk8MPBBbrvtVm7rfztDhz5N504dufuuO3NyKkHYnauf+kVR1I+M39J91W8eKweU/PWzA8WLF+esVq2YMX16lvUvPPcsrVq3IYoiKh15JOUqlOfbb77ereeeM2c2AJUrV2HEiy/wwoiRzP/ySxYuWPCPzkG5V+kyZVi8eHHm4yVLl2S+pvfEgAF3cONNNzPypZdISqrP00OHccstN/2TQ5X2yHFNm/L999/x008/UaZMGcqUKUOjRhkXc7Rr257Zs2dn2f6/j6tUqcLwF17gpZGj+PLLL1ngz9u/ZXdOP23+zZ904HSgfDaOSb9j8+bNbNy4MfPrie+/x1E1amTZpmzZI/joww8AWLlyJQu+/ZbyFSru1vMP6H8bt/brz/bt20lPTwcgSkhgS+qWf3AWys2Sk5NZuHABP/zwA9u2bWP0qJG0bLlnpzoXLFjA8mXLOP7449mSuoWEhASiKGLr1q3ZNGpp1xYuXJh59HHWrFls27aNQw89lBIlSlCmTFm++eYbAD74YCLVq1fPsm//2/rRv/8dWX7eJiQksGWLP2//jj+9+V4cxw/+9nEURQOBsdk2Iv2uVStX0qlDewDS09Lo0KkTpzVrztND/g1Aj169ueGmm+nVoxvJSXWJ45gBd91DsWLFAOhy/rl8+snH/PzTT1SuWI5bbr2NLl27ATBuzBjqN2iQ+a65YaPGJCfVpWatWtSuXWcXo5H2XGJiIg8PfpQWpzcjPT2dC7t2o0aNGvz7yScB6N2nDytWrKBRwwZs2LCBhIQEBj88iLnz5lOoUCEAbr3lZgbcmXFatFOnzrRt25pHHnmY/v3vyLF5KUznnXsOH3/8ET/99BPly5Wl320Zb/oAevfuw+uvvcrw4S+QmDcvBx14EC+OGJl5OnXQw4O54ILz2LZtGxUrVOTpocMyn3fMmDdo0KBB5lV7jRs3pm7d2tSqVZs6dfx5+3dEuzrH/Yc7RFFRICWO48r/9GCS6jeId3WZsrQvOyAxT04PQfpL9vTnv7QvKH7YoQvXrFmzywb50yM1URTN5dfLuYE8ZHxg2LdEkiRpn7I7v/vpzN98nQasjOPYm+9JkqR9yh9GTRRFCcCEOI5r7qXxSJIk/SV/ePVTHMc7gM+jKDpiL41HkiTpL9md008lgS+jKEoh47JuAOI49pazkiRpn7E7UXN7to9CkiTpb9qdqGkRx/H1v10QRdF9wMfZMyRJkqQ9tzt3FD51F8tO/6cHIkmS9Hf87pGaKIouAi4GKkZR9MVvVhUEPsvugUmSJO2JPzr9NAJ4C7gHuOE3yzfGcbwmW0clSZK0h343auI4Xg+sBzrvveFIkiT9NbvzmRpJkqR9nlEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCIk5PYDfSt8Rs3FrWk4PQ9ojBxTIk9NDkP6ShT9vzukhSHssdfuO313nkRpJkhQEo0aSJAXBqJEkSUEwaiRJUhCMGkmSFASjRpIkBcGokSRJQTBqJElSEIwaSZIUBKNGkiQFwaiRJElBMGokSVIQjBpJkhQEo0aSJAXBqJEkSUEwaiRJUhCMGkmSFASjRpIkBcGokSRJQTBqJElSEIwaSZIUBKNGkiQFwaiRJElBMGokSVIQjBpJkhQEo0aSJAXBqJEkSUEwaiRJUhCMGkmSFASjRpIkBcGokSRJQTBqJElSEIwaSZIUBKNGkiQFwaiRJElBMGokSVIQjBpJkhQEo0aSJAXBqJEkSUEwaiRJUhCMGkmSFASjRpIkBcGokSRJQTBqJElSEIwaSZIUBKNGkiQFwaiRJElBMGokSVIQjBpJkhQEo0aSJAXBqJEkSUEwaiRJUhCMGkmSFASjRpIkBcGokSRJQTBqJElSEIwaSZIUBKNGkiQFwaiRJElBMGokSVIQjBpJkhQEo0aSJAXBqJEkSUEwaiRJUhCMGkmSFASjRpIkBcGokSRJQTBqJElSEIwaSZIUBKNGkiQFwaiRJElBMGr2Y0898SjHN06iaaN6DHn8EQDWrllDh1YtOLpeDTq0asG6tWsBeHX0S5zcpGHmn5JFDmLeF58D0OaMUzm2fq3MdatXr8qxOSn3Wrx4MSeffCI1a1Sndq0aDB78MACff/45xx57NHXr1KLVWS3ZsGFDDo9UucXNV1zEsUdVoGXThpnLHr53AK1OaEybk46he4dWrFqxHIBxr4yizUnHZP45qkQhvpr3BalbttD73Ha0ODaJM5sm8+CAfpnPte2XX7iqZxeaNapDx+YnsvQ/P+71OYYmiuM4p8eQqU69+vG7H0/O6WHsF76a/yV9up3PWx9MIl++fHRu25L7/vUIw58dRtGiRbns6mt55KEHWLduHbfecVfWfb+cR5fO7Un54msgI2puG3AvdZPq58RU9nvFChyQ00MIwvLly1m+fDlJSUls3LiRhsn1efW1N+jWtQv33T+Q448/nmeGDeOHRT9wxx0Dcnq4QVjw06acHsI+bfqUSeQ/uAA3XNqLcZ+kALBp4wYKFCwEwAtPPcF3335N/wcezrLft/O/5JIunXhv+lxSt2zhi1kzaNSkKdu2baNb+zPpdUVfmp58GiOeeYpv58+j/wMPM+H1V3j/zXH866nn9vo89zf1KpVZmLpxXeVdrfNIzX5qwTdfU79BQ/Lnz09iYiJHNzmON8eN4Z03x9HhnPMA6HDOebw9YexO+77+yijatO+wt4cs/aGSJUuSlJQEQMGCBalWrTpLly7lm2++oWnTpgCccuqpvP7aqzk5TOUiyUc3oUiRolmW/TdoAFK3bIYo2mm/Ca+/zBlt2gNwUP78NGqS8frNly8fR9Wqy4plSwH44O0JtOpwDgDNWrZm6qSP2JcONOyPjJr9VLWjajB18iTWrPmZLVu2MPHdd1i2dAmrV6/i8BIlATi8REl+Wr16p33HvPYKrdt3zLLsykt6cXKThjx0/93+pVKOW7RoEXPmzKZRo0bUqFmTcWMz4vyVV15m8eLFOTw65XaD7r6dE+tVY9yro7n8upt3Wv/WmNdo0ebsnZZvWL+OD999i6OPOwGAlcuXUbJ0GQASExMpWLAw69b8nK1jD51Rs5+qUrUal155DR1bncE57VpSo2YtEhMT/3S/WTNSOCh/fqofVSNz2eNPPctHU2Yy5q2JTJv8GS+PfDE7hy79oU2bNtHh7HY89NAgChUqxNNPD+Pxxx+jYXJ9Nm7cSL58+XJ6iMrlrrzpNj6c/TUt23XgxWFDsqz7fOZ0DjzoIKpUPyrL8rS0NPr26cZ5PfpQtnwFAGJ28QZyF0d+tPuMmv3YORd05b1Pp/LGWxMpUrQoFSoeyWGHFWflrx9cW7liOcUOOyzLPm+8+jJt2mU99VSyVGkAChQsSJuzOzJ75oy9MwHp/9m+fTtnt29H53POpU3btgBUq1aNt995l5TpM+nUqTMVK1XK4VFKGc5o24F3x4/JsuzNN17NPPX0W7ddcxnlKlSiS+9LMpeVKFma5UuXABnRs3HjeooUPSR7Bx04o2Y/9t+rlJYs/g9vjhtDm/YdOO30Mxk9YjgAo0cMp1mLlpnb79ixg3FvvEbrdv87LJqWlsbPP/8EZPwP5b2336Ja9RpIe1scx/Ts0Z3q1atz1VVXZy5ftSrjdb5jxw7uvutOevfqk1NDlFj0/cLMrz98500qVq6S+XjHjh28M+51WrTOGjWD7rmDjRs3cOOd92VZfmKzFowZPQKAd8a9QeMmxxN5pOZv+fPzFdpn9Ti/E2vWrCFv3rzcM3AQRYoW5bKr+9Kry7mMeOFZSpcpy1PPjcjcfspnn1KyVGnKVaiYueyXX36hc5uWbE/bTnp6Ok1POInzLuyWE9NRLvfZZ58xfPgL1KpVi/pJdQEYcOfdLFy4gCcefwyA1m3acmHXrjk5TOUi1/TuSsrkT1m35mdOqFuVS6+9iU8mvssPCxeQkJBAqTJls1z5NGPKZxxeslTm6SWAFcuW8u9BD1CxchXandIEgHO69eLs8y6k/TkXcP2lPWnWqA6FixTlwX8/s9fnGBov6Zb+Ji/p1v7KS7q1P/KSbkmSFDyjRpIkBcGokSRJQci2qImiqGwURR9GUfRVFEVfRlF0RXZ9L0mSpOy8+ikNuCaO41lRFBUEZkZR9F4cx/Oz8XtKkqRcKtuO1MRxvDyO41m/fr0R+AoonV3fT5Ik5W575TM1URSVB+oB0/bG95MkSblPtkdNFEUFgFeBK+M43rCL9b2iKJoRRdGMNT/v/MsXJUmSdke2Rk0URXnJCJoX4zh+bVfbxHE8JI7jBnEcNzjk0MN2tYkkSdKfys6rnyJgKPBVHMcPZdf3kSRJguw9UnMscD5wUhRFc3790yIbv58kScrFsu2S7jiOJwH+ulFJkrRXeEdhSZIUBKNGkiQFwajZx115SS9qVCrL8Y2TMpc9cM8A6laryMlNGnJyk4a8/+7bO+23dMli2p55Gscl16Fpo3o89cSjmevmffE5LU5uyslNGnLa8ccwa+Z0AFKmTubEYxrQ7IRj+eG77wBYv24dndqcSRzH2TxTheztt9/mqOpVqVrlSO67796d1sdxzJVXXE7VKkdSr25tZs2albmuUsXy1K1Ti/pJdWnUsEHm8htuuJ56dWtzYZcLMpcNf+EFBg9+OHsno1xj+dIldGnTgjOa1OfMpsk8P+RxAB6+dwCtTmhMm5OOoXuHVqxasXyX+998xUUce1QFWjZtmGX5wAG30uqExlx/aa/MZWNefinz+fXXGTX7uI7nnM9Lr47daXmviy9j4qQUJk5K4ZTTmu+0PjExkf533sen0z/nzfc/4ZmnnuSbr78CYEC/m7jmhpuZOCmF627ux4B+NwHw5CMPM/SFkdzU73aeHTYEgIceuIfLr7mOjIvZpD2Xnp7O5ZddwvgJbzF33nxGjXyJ+fOz/raUt956iwULFvD1Nwt44skhXHLJRVnWvz/xQ2bOmsO0lBkArF+/nilTJjN7zhekp6czd+5cUlNTee75Z7nooov32twUtjyJiVx3+91MmDSTUW9+wIhnhrDwm6/pfskVjPloKq9/MJkTTm3O4w/uHOoArTudy5CRr2dZtnHDeuZMn8aYj6ayIz2db+d/ydbUVN4Y+SKdu/bcG9MKmlGzjzv62OMoUrToHu93eImS1K5bD4ACBQtSuWo1VixbCkAURWzckHEfxI0b1lOiREkAEvPmZWtqKqmpqeRNzMui779jxbJlHNOk6T80G+VGKSkpVKp0JBUrViRfvnx06NiJsWPHZNlm3NgxnH/+BURRROPGjVm/bh3Ll+/63S9AQkIC27ZtI47jjNdr3rwMHPgAl116OXnz5s3uKSmXKH54CWrUrgvAwQUKUqlyVVauWEaBgoUyt0ndshl+501f8tFNKFIk68/vhIQEtm/bThzHbN2aSmLeRIY+/jDn9ejja/cfYNTsp4Y99QQnHtOAKy/pxbq1a/9w2//8uIh5X8whqUHGIdA77h3IgH43knRUJW6/5UZuum0AAJdffS19r7iEIU88QrdefbhnQH+uv+W2bJ+LwrZs6VLKli2b+bhM6TIsW7o0yzZLly6lzG+2KV2mDEuX/i/CT29+Gg2T6/PUkIwjiAULFqRt23Y0qF+P8hUqULhwYWZMn85ZrVrthRkpN1r6nx/5at4X1EnKOAU66O7bObFeNca9OprLr7t5t5/n4AIFOfXMs2h78rGUOaIcBQoVZt7smZx8+pnZNfRcxajZD13YvRfT5nzFxEkpHH54Cfrfcv3vbrt50yZ6nN+ZO+4ZSMFCGe8unhs6hNvvfoBZ87/j9rvv5+pL+wBQs3Yd3pz4Ca+Nf5cfF/1AiZIlieOYXheexyU9L2T1qpV7ZX4Ky64+j/X/T2f+0TaffPoZ02fMYvyEt3jiicf45JNPALj22uuYOWsOAwc+yG39bqX/7Xcw9Omn6dSxA3fddWc2zES51ebNm7i8+3ncMODezKM0V950Gx/O/pqW7Trw4q+n63dXj0uv4vUPJnP97fcw+N4BXHb9Lbw8/Fmu6nkBTzx0f3ZMIdcwavZDhxU/nDx58pCQkMC5Xboxe+aMXW63fft2up/fibYdOnHGWa0zl49+aXjm47PatGP2rKz7x3HMoAfu5aprb+TBe+/k2ptupV3Hc3j6yceyb1IKVukyZVi8eHHm4yVLl1CyVKks25QpU4Ylv9lm6ZIllPp1m//+s3jx4rRq3Ybp01Oy7Dt79mwAqlSpwgvDn2fkqNF8OW8eCxYsyJb5KHfZvn07V3Q7j5btOnDaGTsfCTyjbQfeHT9mF3v+uflzPwegfMUjGfPyS/zrqedZ8PV8Fn2/8G+NOTczavZDK3/zSfu3xo+lWvUaO20TxzFXXdqbylWr0efSK7KsK1GiJJMnZbzbnfTxh1SseGSW9aNGvMApzU6nSNGipKamkpCQQEIUkZqamg2zUeiSk5NZuHABP/zwA9u2bWP0qJG0bHlWlm3ObHkWL7zwPHEcM3XqVAoVLkzJkiXZvHkzGzduBGDz5s2899671KhRM8u+/z1Ks337dtLT04GMzy1s2bJl70xQwYrjmFuuuoSKlatyYZ/LMpf/Njo+fOdNKlau8peef/C9A7j8+ltIS9vOjt+8drf6s/Yvy7Y7Cuuf0afb+Uye9Clrfv6JetUrce2NtzB50ifMm/sFURRR9ohyPDAo43LtFcuXcfVlFzHilTGkTJ3MKyNHUL1GTU5ukvFZmhv73cEppzVn4ODHufX6vqSlp3HAAQfywMP/OwKzZcsWRr80nFGvTwCg9yWX0/38TuTLm48nhj2/9/8FaL+XmJjIw4MfpcXpzUhPT+fCrt2oUaMG/37ySQB69+lDixYtePutN6la5Ujy58/P00OfAWDlypW0b9cGgLS0NDp1Pofmzf93td+YN96gQXJy5tGcxo2Ppm6dWtSqVZs6ders5ZkqNLNSpjD25ZeoUr0GbU46Bsg47fTqiOf5YeECEhISKFWmLP0fyLiNwKoVy7nl6ksZMuJVAK7p3ZWUyZ+ybs3PnFC3KpdeexPtz+0CwPtvjqNWvfoU//VCjboNGnLW8Y2oelRNqtWolQOzDUO0L91/pE69+vG7H0/O6WFIe6RYgQNyegjSX7Lgp005PQRpj9WrVGZh6sZ1lXe1ztNPkiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkIRo0kSQqCUSNJkoJg1EiSpCAYNZIkKQhGjSRJCoJRI0mSgmDUSJKkIBg1kiQpCEaNJEkKglEjSZKCYNRIkqQgGDWSJCkI/9fevYVYVcVxHP/+sqLMKYtuEJRmqZSYaUb3LGQIulCRkAQliZVBFyJB6EJBkGBPEZTWg4EUlSaIEV56MBWNzNQsb3Sjhx6sB4vCl/r1cJayO416ZsLZzvb3gcPZ56z/3v+1hnWGP2vtmZOiJiIiIhohRU1EREQ0QoqaiIiIaIQUNREREdEIKWoiIiKiEVLURERERCPIdt19OEDSHuDHuvvRUGcCv9TdiYheyryNgSpz98i5wPZZPTUcVUVNHDmSNtq+ou5+RPRG5m0MVJm79cj2U0RERDRCipqIiIhohBQ1x475dXcgog8yb2OgytytQYqaY4TtfMCiX0maJGlZOb5D0uxDxA6V9Gj7+4ebt5JekPR0p++3xSyQdM+hYtrih0na1ml8HNvyO7ceKWoiolckDertObaX2p5ziJChwH+KmoiI3khRExHAgZWIHZLelrRV0iJJg0vbD5Kel7QWmCKpW9J6SZskfSBpSIm7pVxjLXB35drTJL1Wjs+RtETSlvK4BpgDjJC0WdLcEjdL0uelLy9WrvWMpJ2SVgGjOhjXjHKdLZIW7x9TMVnSGkm7JN1W4gdJmlvJ/fD//dlGRP9IURMRVaOA+bbHAr/x79WTfbavA1YBzwKTbY8HNgJPSToJeBO4HbgeOPcgOV4FVtu+DBgPfA3MBr61Pc72LEndwMXAlcA4YIKkGyRNAO4FLqdVNE3sYEwf2p5Y8m0HplfahgE3ArcCb5QxTAf22p5Yrj9D0vAO8kREzY6vuwMRcVT5yfa6crwQeBx4pbx+rzxfBVwCrJMEcCKwHhgNfG97N4CkhcBDPeS4GbgfwPZfwF5Jp7fFdJfHl+X1EFpFThewxPafJcfSDsY0RtJLtLa4hgDLK23v2/4b2C3puzKGbmBs5X6b00ruXR3kiogapaiJiKr2/8ZZff1HeRaw0vbUaqCkcT2c31cCXrY9ry3Hk33IsQC40/YWSdOASZW2nsYr4DHb1eIHScN6mTci+lm2nyKi6nxJV5fjqcDaHmI2ANdKughA0mBJI4EdwHBJIyrn9+QTYGY5d5CkU4Hfaa3C7LcceLByr855ks4GPgXuknSypC5aW12H0wX8LOkE4L62timSjit9vhDYWXLPLPFIGinplA7yRETNUtRERNV24AFJW4EzgNfbA2zvAaYB75a4DcBo2/tobTd9VG4UPtj3uD0B3CTpK+AL4FLbv9Laztomaa7tFcA7wPoStwjosr2J1jbYZmAxsKaDMT0HfAaspFV4Ve0EVgMfA4+UMbwFfANsKn/CPY+sakcMCPnup4gADmyvLLM9puauRET0SVZqIiIiohGyUhMRERGNkJWaiIiIaIQUNREREdEIKWoiIiKiEVLURERERCOkqImIiIhG+Ac9BU3c8lXLogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "ax.matshow(conf_mx, cmap=plt.cm.Blues, alpha=0.3)\n",
    "for i in range(conf_mx.shape[0]):\n",
    "    for j in range(conf_mx.shape[1]):\n",
    "        perc = str(round((conf_mx[i, j]/conf_mx.sum())*100,2)) + \"%\"\n",
    "        ax.text(x=j, y=i, s=str(conf_mx[i, j])+\"\\n\\n\"+perc, va='center', ha='center')\n",
    "\n",
    "plt.xlabel('predicted label')\n",
    "plt.ylabel('true label')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trying to Optimize the SGD Model with Baeysian Optimization with gaussian process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_sgd(params,param_names,x,y):\n",
    "    params = dict(zip(param_names,params)) \n",
    "    model = SGDClassifier(**params) #It takes the params as parameter\n",
    "    kf = StratifiedKFold(n_splits=5)\n",
    "    accuracies = []\n",
    "    \n",
    "    return -np.mean(cross_val_score(model, X_train, y_train, cv=StratifiedKFold(n_splits=3),n_jobs=-1, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_space_sgd = [\n",
    "    space.Categorical([\"none\",\"l2\",\"l1\",\"elasticnet\"],name=\"penalty\"),\n",
    "    space.Real(0.00001,0.01,prior=\"log-uniform\",name=\"alpha\"),\n",
    "    space.Categorical([\"log\"],name=\"loss\")\n",
    "]\n",
    "param_names_sgd = [\"penalty\",\"alpha\",\"loss\"]\n",
    "optimization_function = partial(optimize_sgd,param_names= param_names_sgd,x = X_train,y = y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at random point.\n",
      "Iteration No: 1 ended. Evaluation done at random point.\n",
      "Time taken: 1.8025\n",
      "Function value obtained: -0.7352\n",
      "Current minimum: -0.7352\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "Iteration No: 2 ended. Evaluation done at random point.\n",
      "Time taken: 1.4575\n",
      "Function value obtained: -0.7357\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 3 started. Evaluating function at random point.\n",
      "Iteration No: 3 ended. Evaluation done at random point.\n",
      "Time taken: 1.8462\n",
      "Function value obtained: -0.7343\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 4 started. Evaluating function at random point.\n",
      "Iteration No: 4 ended. Evaluation done at random point.\n",
      "Time taken: 2.8818\n",
      "Function value obtained: -0.7292\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 5 started. Evaluating function at random point.\n",
      "Iteration No: 5 ended. Evaluation done at random point.\n",
      "Time taken: 0.9397\n",
      "Function value obtained: -0.7355\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 6 started. Evaluating function at random point.\n",
      "Iteration No: 6 ended. Evaluation done at random point.\n",
      "Time taken: 0.9830\n",
      "Function value obtained: -0.7355\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 7 started. Evaluating function at random point.\n",
      "Iteration No: 7 ended. Evaluation done at random point.\n",
      "Time taken: 0.8605\n",
      "Function value obtained: -0.7310\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 8 started. Evaluating function at random point.\n",
      "Iteration No: 8 ended. Evaluation done at random point.\n",
      "Time taken: 0.7217\n",
      "Function value obtained: -0.7278\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 9 started. Evaluating function at random point.\n",
      "Iteration No: 9 ended. Evaluation done at random point.\n",
      "Time taken: 0.9960\n",
      "Function value obtained: -0.7352\n",
      "Current minimum: -0.7357\n",
      "Iteration No: 10 started. Evaluating function at random point.\n",
      "Iteration No: 10 ended. Evaluation done at random point.\n",
      "Time taken: 1.0120\n",
      "Function value obtained: -0.7332\n",
      "Current minimum: -0.7357\n"
     ]
    }
   ],
   "source": [
    "result = gp_minimize(optimization_function,dimensions=param_space_sgd,n_calls=10,verbose=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score = -0.7357\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score = %.4f\" % result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'penalty': 'none', 'alpha': 0.0034706435880870164, 'loss': 'log'}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_params = dict(zip(param_names_sgd,result.x))\n",
    "sgd_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(result, open('bayessian_optimization_sgd.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### KNN Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.76717172, 0.76868687, 0.76641414])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "cross_val_score(knn, X_train, y_train, cv=StratifiedKFold(n_splits=3), scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(knn, X_train, y_train, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAI4CAYAAACSixhGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZiNZQPH8d89ZlT2MIMx9t0YszIiSmTfk61CEkILZemtLL3ZQkUlyRKmUCpbJFvIvowwlSWRQZbsM5jF8/4xOpnX0pAz09zz/VyXqznPcua+L7fTd855zhnjOI4AAADSO4+0HgAAAMCdQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNZYzxtQzxuwyxuw1xvRP6/EAKWGMmWyMOWaM2ZnWYwFSyhhTyBizwhjzkzEmyhjzfFqPKaMxfE6NvYwxmSTtlvSwpGhJmyS1dRznxzQdGPA3jDE1JJ2XNM1xnAppPR4gJYwxBSQVcBxnqzEmu6QtkprxmJt6eKbGbpUl7XUcZ5/jOHGSZkpqmsZjAv6W4zirJJ1M63EAt8JxnCOO42y98vU5ST9JKpi2o8pYiBq7FZR08Krb0eIfGAC4nTGmqKRgSRvSdiQZC1FjN3OdbbzeCABuZIzJJukLSS84jnM2rceTkRA1douWVOiq236SDqfRWADAesYYLyUFzSeO43yZ1uPJaIgau22SVMoYU8wYk1lSG0nz0nhMAGAlY4yRNEnST47jvJXW48mIiBqLOY6TIKmnpMVKumDtM8dxotJ2VMDfM8bMkLROUhljTLQx5qm0HhOQAtUkPSHpIWPMtit/GqT1oDIS3tINAACswDM1AADACkQNAACwAlEDAACsQNQAAAArEDUZhDGmS1qPAbhVrFukV6zdtEHUZBz8A0N6xLpFesXaTQNEDQAAsMK/6nNqcua61/HJ75vWw7DSmdOnlDPXvWk9DCvlyHZPWg/BWieOH1deb++0Hoa1/kUP/9Y5ceK48uZl7brDzh3bz8bFxeW83j7P1B7Mzfjk99WYCTPTehjALalzf4W0HgJwW+ITLqf1EIBbViCf97Eb7ePlJwAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFjBM60HgJQ7fux3jR7yik6dPCEPDw/Va/yImrZ8XMMH9VH0wf2SpJjz55Q1W3a9N+lzxcfH671Rr2vPrih5eHioy7P9VDG4kmJjY9T32Y6u+/3j+FHVfLihujzbT8eOHtFbQ19VzPlzunw5UR27vqBKVaqnzYRhvXfeeVtTJk2SMUYVKgRo4uTJ2rVrl3p0f0bnz59X0SJFNS0iQjly5ND+/fsV4F9epcuUkSSFh4dr3Afj03gGyCguXryoh2s9qLhLcUpISFCzFi302oBBGjxogL6eP1/Gw0M+3t76cOJk+fr6Kj4+Xt27ddG2yEglJCSo3eOPq0/f/pKkuLg49Xr+Oa1etVIeHh4a9Pp/1ax5izSeoR2ImnQkU6ZM6tzjRZUsXV6xsTF6/uk2Cg67T/0HjXQdM/H9UcqSNZskafGCLyRJ4z7+UqdP/aEBfbvrnQ9nKEuWrHpv0ueuc557urWq1qglSZo5bYKq16yjhs1a67f9v2hgvx6aMuubVJwlMopDhw7p/Xff1fadUbrnnnvUtnVrzZo5Ux98ME5vvjlSNR54QFMmT9boUSM1+PX/SpJKlCihLVsj03jkyIjuuusuLVq8VNmyZVN8fLxq1ayhunXrqVfvlzRw0OuSpHHvvathQ97Qu++P05dfzNalS5e0aes2xcbGKiQoQK1atVGRokU1YvhQeft4a3vUT7p8+bJOnjyZxrOzBy8/pSO583irZOnykqQsWbKqUJFi+uP4Mdd+x3G0esViPVC7viTpt/2/KDA0XJKU6948ypYtu/bsikp2n4eiD+jMqZPyrxgqSTLGKDY2RpIUc/68cufxdvu8kHElJCTowoULSkhIUGxsrHx9fbV71y5Vr1FDklT74Yf11ZdfpvEogaTHxmzZkn5gjI+PV3x8gmSMcuTI4TomJjZGxhjX8TExMa41ntkrs7JfOXba1I9dz9p4eHgob968qTwbexE16dTRI4e0b8/PKlM+wLUtavsW5cqdRwX9ikiSipUoo/Xfr1BiQoJ+PxKtvbt/0oljvye7n5VLF6n6Q3Vd/xAfe/IZrfh2gdq3rK2B/bqr2/Mvp96kkKEULFhQvV58UcWLFlGhgr7KkTOnHq5TR/4VKmj+vHmSpNmzP9fBgwdd5/z6668KCw3RQzUf1PerV6fV0JFBJSYmKrxSqIr4FVCtWrVUuXLSD40DB7yqUiWKataMGXpt4CBJUvMWjyhr1qwqXsRPZUoW0/O9eit37tw6ffq0JOn1QQN0X3glPda2tY4ePZpWU7IOUZMOXYiN1ZABvfX0s31dLzVJSYHyQK36rtt1GjRTXp98er5rW014902V8w+UR6bkrziuWv6NHqjVINl91K7fVNNmL9XgEeM0esh/dPnyZfdPChnOqVOnNH/ePO35ZZ9+iz6k2JgYfRIRoY8mTtIH48apcqUwnT93TpkzZ5YkFShQQPv2H9DmLVs1ctRoPfH4Yzp79mwazwIZSaZMmbRh0xbt2XdAmzdvUlTUTknS4Nff0J5f9qt127Ya/8H7kqTNmzYqU6ZM+mX/Qf24a6/GvvO2ft23TwkJCToUHa37qlbTug2bFB5eRf/p3zctp2UVoiadSUiI19ABvVWzdkNVq1HbtT0xIUFrVy9TjZp1XdsyeXqqS8++em/S5xowdKzOnz+ngn6FXfv37d2lxMRElSpT3rXt24VfqfqV+yhXIVBxcZd09sypVJgZMpplS5eqaNGi8vb2lpeXl5o1b65169aqbNmyWrR4sTZu2qzWbdqqeIkSkpKuaciTJ48kKTQ0VMVLlNDu3bvTcgrIoHLlyqXqNR7QksWLk21v3bqt5n71lSRp1syZerhOXXl5ecnHx0dVqlbV1q1blCdPHmXJkkVNmjaTJLV4pKW2RXKd2J1C1KQjjuNozIiBKlSkmJq3bp9sX+SW9fIrXEx5ffK7tl28eEEXL8Qm7d+0TpkyZVLhoiVc+1cuW6QHatVLdj/ePvm1bcsGSdJv+/cpPi5OOXPldteUkIEVKlxYGzdsUGxsrBzH0fLly1W2XDkdO5Z0ndjly5c1dMgQdenSVZJ0/PhxJSYmSpL27dunvXv2qHjx4mk2fmQsx48fd710dOHCBa1Yvkyly5TR3j17XMd8vWC+6915hQoX0nffrZDjOIqJidGmDRtUukwZGWPUoGEjrVr5nSRpxYqkdY87g3c/pSM/7ojU8m8XqGjxUur51KOSpA5PP6dKVapfeRmpfrLjz5w6qdf6dJMxHsrj7aOXXhmabP/qFYs1eMS4ZNs693hJY0cO1tzPp0vGqNfL/3VdbwPcSeHh4WrxyCOqHBYqT09PBQYF6+mnu+jDD8dr/LikddmseXN1fPJJSdLqVas0eNBAZfL0VKZMmfT+uA+UOzfBjdTx++9H9PRTnXQ5MVGXL19Wi5Yt1aBhI7Vt/aj27N4tDw8PFSpcWGPfS1q7Xbt1V9enn1JYcKAcx9ET7TsoIKCiJOmNIcP0VKcO6vvSi8qbN68+/GhSWk7NKsZxnLQeg0upsv7OmAkz03oYwC2pc3+FtB4CcFviE7heDulPgXzee0+fOlnqevt4+QkAAFiBqAEAAFYgagAAgBXcGjXGmHrGmF3GmL3GmP7u/F4AACBjc1vUGGMySXpfUn1J5SW1NcaUv/lZAAAAt8edz9RUlrTXcZx9juPESZopqakbvx8AAMjA3Bk1BSUdvOp29JVtAAAAd5w7o+Z6n9h2zYfiGGO6GGM2G2M2nznNx/EDAIDb486oiZZU6KrbfpIO//9BjuNMcBwnzHGcsJy57nXjcAAAgM3cGTWbJJUyxhQzxmSW1EbSPDd+PwAAkIG57Xc/OY6TYIzpKWmxpEySJjuOE+Wu7wcAADI2t/5CS8dxFkpa6M7vAQAAIPGJwgAAwBJEDQAAsAJRk84kJibq2adaaVD/npKk4YP6qOdTj6rnU4/qydb11POpR6973jvDB6hd0wfUvWPzZNsnj39bPZ58RKOH/Me1bfni+Zo7O8J9k0CG0/mpTvLNn09BFQOuu3/ld98pz725FBoSrNCQYL3x39f/9tyX+/dTcFCgOnbo4NoWMX26xo4d455JIMMrW7qEKoUEKbxSqKrdF37N/rdHj1J4pVCFVwpVWHCgst2TWSdPntTx48dVq2YNhQUHat7cua7jH32kuQ4fvuZNwfgHiJp0Zt7sT1SoSDHX7f6DRuq9SZ/rvUmfq1qN2qpavdZ1z6tdv4leH/lBsm0x58/p553b9P6UL3T58mXt/2W3Ll26qKXfzFXDZq3dOg9kLB06dNSChYtuesz991fXlq2R2rI1Uq++NuCm5545c0br1q1T5LYflJiYqB07dujChQuaNm2qnnmmu1vmAEjSom+XasOmLVqzbsM1+3q9+JI2bNqiDZu2aPB/31D1GjWUO3dufT5rph57vL1WrPpe77w9SpL09YL5CgoOlq+vb2pPwWpETTpy4tjv2rR+leo2anHNPsdxtHrFYj1Qu/51z60QGKbs2XMm22Y8PBSfEC/HcXTp0iVl8vTSFzM+VuNHHpOnp5db5oCM6c8H9zt1roeHh+Li4uQ4ji5euCAvLy+NHjVSPXs+Ky8v1i7S3uefzdKjrdpIkjy9vHTxwgVdunRJHh4eSkhI0PvvjlWv3i+l8SjtQ9SkIxPee1NPdustY679a4vavkW5cudRQb8iKb6/LFmyqlqN2nq2cyvlL1BQWbNl056fd+q++2veyWEDKbJ+/TqFBAepUYMGioq6+ac/ZM+eXS1atFBYaIiKFiuqnDlzavOmzWrSlF8vB/cxMmrcsL6qVqmsSRM/uuFxsbGxWvLtYjVrnvQDaOs2bbVkybdq2rihXnl1gCaM/0DtHn9CWbJkSa2hZxhufUs37pyNa1cqZ67cKlWmvLZHbrpm/8qli/RAres/S3MzLdt1Ust2nSRJY94cqMc79dDiBV9o66Z1KlaitNq07/KPxw78neCQEP3y635ly5ZNixYuVMsWzfXTrt03PeelPn31Up++kqQuT3fWwMGDNWniRC1dskQBFQP0n1deTY2hIwNZ9t0q+fr66tixY2rcoJ7KlCmj+6vXuOa4hV8vUJX7qrqeYcyZM6e+mjtfknTq1Cm9NWqkZnw2W92f6arTp07p+Rd6KbzKfak6F1vxTE068ePObdqw9js92bqeRrzeV9u3btTIN16WJCUmJGjt6mWqUbPubd//L7t/kiQVLFREyxbP18uDR+nAr3t1KPrAnRg+cFM5cuRQtmzZJEn1GzRQfHy8Tpw4kaJzIyMjJUmlS5dWRMR0zZg1S1E7o7Rnzx63jRcZ05/Xv/j4+Khx06bavOnaHzClpJeeWrVuc919w4b8V337v6zPZs1UcHCIxk+YqIEDCPA7hahJJzp2eV7TZi/VlFnfqN+AN1UxpLL6vDpMkhS5Zb38ChdTXp/8t33/0ye/r8c79VBCQoIuX74sSTLG6NLFi3dk/MDN/P7773KcpN93u3HjRl2+fFl58uRJ0bmDBgzQoMGvKz4+XomJiZKSrrmJjY1123iR8cTExOjcuXOur5ctXaLy/v7XHHfmzBl9v3qVGjVucs2+vXv26MiRI6pe4wFdiI2Vh4eHjDG6yOPsHUPUWGDV8m+ueenpjxPHNLDvX+8CGTG4r17s/oSifzug9i1ra/HXX7r2rVu9XKXL+itPXh9ly55DZf0rqnvHFjLGqHjJMqk2D9jr8XbtVL1aVe3atUtFCxfS5EmT9OH48fpw/HhJ0hdfzFZQxQCFBAep1wvPK+LTGTLG3PDcP82dM0dhlcLk6+urXLlyqUqVKgoKrCgZo8DAwDSZK+x07OhR1a75gMLDQlSj2n2qV7+B6tStp48mfKiPJnzoOm7e3DmqVfthZc2a9Zr7GDTwNQ0clPRxBY+2bqOI6VP1YPVqeqHXi6k2D9uZP386+jcoVdbfGTNhZloPA7glde6vkNZDAG5LfMLltB4CcMsK5PPee/rUyVLX28czNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALCCZ1oP4Go5st6jWlX903oYwK1x0noAwO3J7MnPtUh/PMxN9qXeMAAAANyHqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAVvC80Q5jzDlJzp83r/zXufK14zhODjePDQAAIMVuGDWO42RPzYEAAAD8Eyl6+ckYc78x5skrX+c1xhRz77AAAABuzd9GjTFmoKR+kl6+simzpAh3DgoAAOBWpeSZmuaSmkiKkSTHcQ5L4qUpAADwr5KSqIlzHMfRlYuGjTFZ3TskpMTBgwdVp3YtBQb4KzgwQO+NHZts/9tvjdbdXpl04sQJSdIff/yhOrVrKU+uHHrhuWeTHRsXF6fu3bqqQvmyqlihvL768otUmwcyns6dO8m3QD4FBQa4tvXr20cV/MspODhQLR9podOnT0uSli5ZosqVwxQUVFGVK4dpxfLlrnNqPVRT/uXLKjQ0WKGhwTp27FiqzwUZ1+nTp9Xq0UflX76cKviX17p169S2TRuFhgQrNCRYJYoXU2hIsOv47du3q1q1qqoYUEFBgRV18eLFNBy9vW54ofBVPjPGfCgplzHmaUmdJH3k3mHh73h6emrEmyMVHBKic+fO6b7wSqpVu7bKlS+vgwcPatnSJSpUuLDr+LvvvlsDBw3Wj1E7FRUVley+hg8bKm8fH+388WddvnxZJ0+eTO3pIAPp0L6junfvqU5PdnBtq137YQ0ZOkyenp56uX8/jRg+TMOGj1CevHk1Z848+fr6aufOnWrYoJ4O/BbtOm/qtAiFhYWlxTSQwfV64QXVrVtXn33+ueLi4hQbG6sZM2e69r/00ovKmTOnJCkhIUEd2j+hj6dOU2BgoP744w95eXml1dCt9rfP1DiOM0rSbElfSCotaYDjOO+6e2C4uQIFCig4JESSlD17dpUtW1aHDh+SJPV9qbeGDhshY4zr+KxZs6ra/ffrrrvvvua+pn48RX379ZckeXh4KG/evKkwA2RU1WvUUO7cuZNte7hOHXl6Jv2MFV6liqIPJa3l4OBg+fr6SpL8/f118eJFXbp0KXUHDPyfs2fPavXqVer01FOSpMyZMytXrlyu/Y7jaPbnn6tNm7aSpG+//VYBARUVGBgoScqTJ48yZcqU+gPPAFL64Xs7JK2WtOrK1/gX2b9/v7Zt26bKlcO1YP48+foWVMUr/3j+zp9P8w8eOEBVKoWpXZtWOnr0qDuHC9zUx1OmqF69etds//LLLxQUFKy77rrLta1z504KDQ3WkDf+q6RXyQH327dvn/J6e+upTp0UFhqiLk93VkxMjGv/6tWrlS9fPpUqVUqStGfPbhljVL9ePVUKC9XIkW+m1dCtl5J3P3WWtFFSC0ktJa03xnRy98CQMufPn1fbVo9q1Oi3kl6SGjZMAwYNTvH5CQkJOhQdrfuqVtX6TZsVHn6f+vft48YRAzc2bOgQeXp6ql27x5Jtj4qK0n9e7q9xH4x3bZs2PULbtm3Xd9+t0vfff6+IiOmpPVxkUAkJCYrculVdu3XT5i1blTVrVo0YMdy1f9bMGWrdpk2y49es+V7TIyK0ctVqzZkzR8uWLUuLoVsvJc/U9JEU7DhOR8dxOkgKVdJbvJHG4uPj1aZVS7Vp207NmrfQvl9+0f79v6pSaLBKlyyuQ9HRqlI5TL///vsN7yNPnjzKkiWLmjZrLklq0bKltm2LTK0pAC7Tpk3V119/rWnTI5K9dBodHa1HW7bQ5ClTVaJECdf2ggULSkp6+bVN27batGljqo8ZGZOfn5/8/PwUHh4uSWrxSEtFbk163ExISNBXX32lVq1a/3V8QT/VqPGA8ubNqyxZsqh+/fqKjNyaJmO3XUqiJlrSuatun5N00D3DQUo5jqOuT3dW2bLl9HyvXpKkCgEBOnj4d+3eu0+79+5TQT8/rd+4Wfnz57/h/Rhj1LBRI61c+Z0kacXyZSpXrlxqTAFwWfzNNxo18k19NWeusmTJ4tp++vRpNWnSSG8MGapq1aq5tickJLje2RcfH6+FX38tf/8KqT5uZEz58+eXX6FC2rVrlyRp+fJlKlc+6XFz6dKlKlO2rPz8/FzH16lbVzt2bFdsbKwSEhK0atUqlStXPk3Gbjtzo9ehjTG9r3wZJClA0lwlva27qaSNjuN0u9ODCQ0Nc9Zu4KetlFjz/feqVfMBVagQIA+PpDZ9/Y03VK9+A9cxpUsW19r1G10X/pYuWVznzp5VXFyccuXKpQULv1G58uV14MABderYQWdOn1Zeb29NmDhJha965xRuLtNVzyrg7z3+WDutXPmdTpw4oXz58mnAwEF6c8RwXbp0Sbnz5JEkhYeHa9y48Ro65A2NGDFcJa9cmyBJixYtVtasWVWz5gOKj4/X5cREPVSrlkaNeouLL28RS/f2bdu2TV27PK24uDgVK1ZckyZP1r333qtOTz6p8PBwde2W/H+Rn0REaMSI4TLGqF79+hoxgutqbpd33jx7T548Wep6+24WNQNvdqeO46T8wo0UImqQHhE1SK9YukiPbhY1N/uFlnc8WgAAANzlbz98zxjjLamvJH9Jrg85cRznITeOCwAA4Jak5ELhTyT9LKmYpMGS9kva5MYxAQAA3LKURE0ex3EmSYp3HGel4zidJFVx87gAAABuSUqiJv7Kf48YYxoaY4Il+d3sBEkyxkw2xhwzxuz8RyMEAABIgZREzRvGmJySXpT0kqSJknql4LyPJV37WecAAABu8LcXCjuOs+DKl2ck1UzpHTuOs8oYU/T2hgUAAHBrbhg1xph3lfRhe9flOM5zd2IAxpgukrpIUiE+8A0AANymmz1Tszk1BuA4zgRJE6SkD99Lje8JAADsc7MP35uamgMBAAD4J1JyoTAAAMC/ntuixhgzQ9I6SWWMMdHGmKfc9b0AAAD+9t1Pt8txnLbuum8AAID/97fP1BhjShtjlv35IXrGmIrGmFfdPzQAAICUS8nLTx9JellXPlnYcZztktq4c1C4scTERIWHhap508bX7HMcR71feF7ly5ZWWHCQIrdulSTt3rVLlUNDXH+8c+fSu2PGSJJeebm/woKD1KljB9f9fBIxXe+NHZs6E4L1Dh48qNq1HlJAhfIKrFhBY8eOueYYx3H0wgvPqWyZUgoODtTWK2tXkk6fPq3WrR5VBf9yCqhQXuvWrZMkvdy/n4KDA9XxqrUbETH9uvcP3I7OT3VSgfz5FFgx4Lr7f/75Z1WrVlVZ7rlbo0ePumZ/YmKiwkJD1KTxX4/X/fv3U3BQoDp2uGrdTmfd3ikpiZosjuNs/L9tCe4YDP7ee2PHqky5stfdt/ibRdq7d4+iftql9z8Yr+d69pAklS5TRhu3bNXGLVu1buMmZcmSRU2aNdOZM2e0bt06bY7cpsTERO3csUMXLlzQ9GnT1PWZZ1JzWrCYp6en3hw5Sjt2/qjv16zT+A/G6ccff0x2zDeLFmnvnr366efd+uCDD9WzR3fXvl69XlCdunW1M+onbdm6TeXKlXOt3cjIH9MLM6QAAB9iSURBVJSYmKgdV9butKlT9cwz3f9/CMBtad+ho75euOiG+3Pnzq133hmj3i++eN39Y8eOUdmy5Vy3Xet2W/J1O3Ua6/ZOSUnUnDDGlNCVD+IzxrSUdMSto8J1RUdHa9GihXqy0/WvuZ4/b54ee/wJGWMUXqWKTp85rSNHkv9VLV++TMWKl1CRIkXk4eGh+Lg4OY6jixcvyMvLS2+NHqUePXvKy8srNaaEDKBAgQIKCQmRJGXPnl1ly5bT4UOHkh0zb/5cPf5E0tqtUqWKzlxZu2fPntX3q1ep05U1nzlzZuXKlUseHh6K+3PtXkhau6NHjVTPZ59l7eKOqVGjhnLnzn3D/T4+PqpUqdJ111x0dLQWLlyoTk/99Xh99bq9cGXdjho1Us/2ZN3eKSmJmh6SPpRU1hhzSNILkvgxPg30ebGXhg4bLg+P6/+1HT58SH5+hVy3Cxb0u+Z/Hp/PmqXWrZNePcyePbuatWih8LBQFS1aTDly5tSWzZvUuElT900CGdr+/fu1bVukKoeHJ9t++NDha9buoUOHtG/fPuXN662nnuqksLAQdenSWTExMcqePbtatGihsLAQFS1aVDlz5tTmzZvVhLWLf4nevXpp+PARyR6vXes2NERFi11Zt5s2q0lT1u2d8rdR4zjOPsdxakvyllTWcZz7HcfZ7/aRIZmFXy+Qt7ePQkJDb3iM41z7gczGGNfXcXFx+nrBfLVo2dK17cWX+mjjlq0aMXKUBg8coAEDB2vypIl6rG1rDRs65M5OAhna+fPn1apVS41+623lyJEj2b4brd2EhARFRm5V167dtHnzVmXNmlVvjhguSXqpT19t2RKpkaNGa+DA1zRw0GBNmjRRbdu01tAhb6TKnIDrWbBggXx8vBV6ncfrPn36asvWSI0aNVoDB7ymQYMHa9LEiWrTurWGsG7/sZS8+2mAMWaAkn5Ld6+rbiMVrV27Vl8vmK/SJYur/WPt9N2KFerY/olkxxQs6Kfo6IOu24cORauAr6/r9uJvFikoOFj58uW75v63RUZKkkqVLq1PIiL0yYxZ+jFqp/bu2eOmGSEjiY+PV6tHW6pt23Zq3rzFNfsL+hW8Zu36+vrKz89Pfn5+Cr/yzM4jLVoq8spa/dOft0uXLq2I6dM1Y+YsRUVFaQ9rF2lk7do1mj9/vkoUL6bH2rXVihXL1f6J5I/XV6/b6RHTNXPWLEXtZN3+Uyl5+Snmqj+JkupLKurGMeE63hgyVL/s/0279+7TtE8+1YM1a+rjadOTHdOocWN9EjFdjuNow/r1ypkjpwoUKODa/9msmWrV+vpvXBs8aIAGDBqs+Ph4JSYmSpKMh4diY2PdNylkCI7j6OmnO6tsubLq1av3dY9p3KiJIqYnrd3169crx5W1mz9/fvn5FdKuXbskJV0TVq5cuWTnDho4QIMGvZ5s7XqwdpGGhg4dpgO/HdQv+37VJ5/OUM2aD2na9OSP1wMHDNCgwazbO+1vP3zPcZzRV982xoySNM9tI8It+ejD8ZKkp7t2U736DfTNokUqX7a0styTRRMmTnIdFxsbq2VLl+q9ceOvuY95c+coNKySfK88q1OlShWFBgWqQkCAKgYGps5EYK01a9bok4jpqhAQoNDQYEnSG/8dot8O/iZJ6tq1m+o3aKBF3yxU2TKldE+WLJo4cbLr/HfGjFX79o8rLi5OxYsV18RJf+2bO3eOwsLCkq3doKCKCgioqEDWLv6hx9q108qV3+nEiRMqUriQBg4cpPj4eElS127d9Pvvvyu8ciWdPXtWHh4eGjtmjHbsjLrm5dX/N3fOHIVV+r91G8i6vRPM9V7LvukJxtwraaPjOKXu9GBCQ8OctRv+/93jwL9bpquuWwLSE5Yu0iPvvHn2njx58roN8rfP1BhjdujK27klZVLSBcOv37nhAQAA/HMp+d1Pja76OkHSUcdx+PA9AADwr3LTqDHGeEj62nGcCqk0HgAAgNty03c/OY5zWdIPxpjCqTQeAACA25KSl58KSIoyxmxU0tu6JUmO4zRx26gAAABuUUqiZrDbRwEAAPAPpSRqGjiO0+/qDcaYEZJWumdIAAAAty4lnyj88HW21b/TAwEAAPgnbvhMjTHmGUndJRU3xmy/ald2SWvcPTAAAIBbcbOXnz6VtEjSMEn9r9p+znGck24dFQAAwC26YdQ4jnNG0hlJbVNvOAAAALcnJdfUAAAA/OsRNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKzgmdYDuNplx9Gl+MtpPQzglmS961/1zwhIscjo02k9BOCWnb+UeMN9PFMDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkRNOhZQtqTuqxSk+8ND9UC1cEnSju0/qPaD9+u+SkFq/UgznT17VpK0ZdNG3R8eqvvDQ1UtPETz585x3c8Xsz9T1crBCg8N1Guv9E+TuSDjKlG8qIICAxQaEqTwymGSpJMnT6punYdVtkwp1a3zsE6dOiVJ2rhxo0JDghQaEqSQ4EDN+eqrtBw6LPf6Sz1VJ6SUWj98X7Lts6ZM0CM1K6lV7fs0dugASVLUti1qV7960p9692vFNwtcxy+eO1tt6lRV27rV9Gz7ljp98g9J0icfva9Wtaqobd1qeqZtUx2J/i31Jmcp4zhOWo/BJTgk1Fm5ZkNaDyPdCChbUt99v1558uZ1bXvw/ip6Y9ibur96DU2fOkUH9u/XqwMHKzY2VpkzZ5anp6d+P3JE1aqEatcvv+nsmTOqfl8lrVyzQXm9vdXt6SfVpt0TerDmQ2k4s/Ql612eaT2EdK1E8aLasHGz8l61jvv166vcuXOrX7/+GjFiuE6dOqXhw0ckW8dHjhxRSHCgDkYflqcnfwe3IzL6dFoP4V9t64Y1ypIlmwb27qZZS9ZJkjavXa3J743WO1NmKfNdd+nkiePKnddbFy/EytMraW2eOPq72tWvroUbf5IkNahcTp8tXa9cufNo7NABuvueLOrSq782r12tCsGhuvueLJo9fZK2rF+jYe9PTssppwtVyxXZGxdzptT19vFMjWX27tmtavdXlyTVrFVb8+Ym/SSbJUsW1wP/xUsXZYyRJP366z6VKFVKeb29JUkP1qyleXO+TIORA3+ZP2+u2rfvIElq376D5l15ZjHZOr741zoG3CEkvJpy5Lo32bYvIiarQ/cXlPmuuyRJufMmPXbefc9fa/PSpUt/rU3HkeM4uhAbI8dxFHP+nPLmyy9JCqtaXXffk0WSFBBcSceOHEqNaVmNqEnPjFGzxvVVo2plTZn0kSSpXHl/LVwwX5I058vZOhR90HX45o0bFB4aqKqVgvX2mPfl6emp4iVKas+uXTpwYL8SEhK0YP48HYqOTpPpIGMyxqh+vTqqXClUH02YIEk6evSoChQoIEkqUKCAjh075jp+w4YNqhjgr6DAAI0bN55naZCqDvy6V9s2rlPHprXVpVVDRf2w1bVvZ+Rmtap9n9rWrab+Q96Sp6enPL281P+N0Wpb937Vr1ROv+7Zpaatn7jmfufOmq6qDz6cmlOxElGTjn27bKVWr9ukL+Ys0MQJH2jN96v1/viP9NGED1SjamWdP3deXpkzu44PqxyuDVt+0IrV6/TWqBG6ePGi7r33Xr015j09+UQ71av9oIoUKcL/JJCqVq1eo02bt2rB14v0wQfva9WqVTc9Pjw8XNt3RGn9hk0aPmKYLl68mEojBaTEhASdO3NaU+Ys0fP/eV3/6f6k/ryMo0JwmD5buk5T5y3Tx+Pe1qWLF5UQH6/ZEZMVsXClFm36SSXL+uvj999Odp8Lv5yln3Zs0xNdn02LKVmFqEnHCvj6SpK8fXzUqHEzbdm8SaXLlNWc+Yu0au1GtWzVWsWKFb/mvDJlyylr1qz6MWqnJKl+w0Zavmqtln73vUqWKq3iJUum6jyQsfleWcc+Pj5q2qy5Nm3aqHz58unIkSOSpCNHjsjHx+ea88qVS1rHO3fuTNXxImPzKVBQNes1ljFG/kGhMh4ergt//1SsVBndc08W/bL7J+36cYckya9IMRljVLtRM23f8te1oxu+/05T3ntLoyd+6npJC7ePqEmnYmJidO7cOdfXy5ctUfny/jp+5Wn6y5cva+SIoerUuYskaf/+X5WQkCBJ+u23A9qze7eKFCkqSa5zTp06pUkTxqtDx06pPBtkVP+/jpcs+Vb+/hXUqHETTZs2VZI0bdpUNW7SVJL0669/reMDBw5o965dKlq0aJqMHRnTg3UaaNPapGcTD+zbq/j4OOXKnUeHfjvgWptHon/TgX175etXWD75C+jXPbt06o8TkqQNq79T0ZJlJEm7dm7XsJd7afSkT13X5uCf4XWGdOrYsaN6vE1LSVJCQqJatmqj2nXq6oP3x+qjD8dLkho3babH23eUJK1fu0Zvjx4pL09PGQ8PjX7nXde7pvr16a2dO7ZLkvq+/IpKliqd+hNChnT06FG1fKS5JCkhIUFt2rZTvXr1VKlSJbVp00pTJk9SocKFNWvW55KkNd9/rzffHC4vLy95eHjovffGJXvXFHAnvfLsU9qybo1On/pDDcP91aVXfzVp9bhe79NTrR++T15emTVo9AcyxuiHzev08bgx8vTylIfxUL83RilX7jySpKdf6KsujzaUp5en8hcspIGjx0mSxgwdoAuxMerfvaMkKb+vn96aNCOtpmsF3tIN/EO8pRvpFW/pRnrEW7oBAID1iBoAAGAFogYAAFjBbVFjjClkjFlhjPnJGBNljHneXd8LAADAnVc4Jkh60XGcrcaY7JK2GGOWOI7zoxu/JwAAyKDc9kyN4zhHHMfZeuXrc5J+klTQXd8PAABkbKlyTY0xpqikYEm8XxsAALiF26PGGJNN0heSXnAc5+x19ncxxmw2xmz+48QJdw8HAABYyq1RY4zxUlLQfOI4zpfXO8ZxnAmO44Q5jhOWh08GBQAAt8md734ykiZJ+slxnLfc9X0AAAAk9z5TU03SE5IeMsZsu/KngRu/HwAAyMDc9pZux3G+l2Tcdf8AAABX4xOFAQCAFYgaAABgBaImHenRtbNKFPFVlbAg17aTJ0+qaaN6Cg4op6aN6unUqVPXPXfpt4sVGuivoApl9daoN13bB7z6sqpWDlbXzh1d22Z+GqEP3h/rtnkg4/nmm29UvlwZlSldUiNGDL9m/5kzZ9S0SWOFBAeqYoC/Pp4yxbWvRPGiCgoMUGhIkMIrh7m29+/fT8FBFdWxQ3vXtojp0zV27Bj3TgbW+v1wtLq1bqxHHwpXq9r3acbk8ZKkl3t0Urv61dWufnU1qVZR7epXv+75TapVVJs6VdWufnW1b1TTtX1X1A492exh1/aobVskST9sWq+2daupfeOHdHD/PknSuTNn9OwTj8hxHDfP1k5ETTrS7okO+mLOgmTb3h79ph548CFF7vhJDzz4kN4e/eY15yUmJurFXs9p9pz52rh1u774fKZ+/ulHnTlzRhvXr9PajZFKTExU1M4dunDhgj6JmKbOXZ5JrWnBcomJiXru2R5a8PUi7dj5o2bNnKEff0z+21LGjXtf5cqX19bIH7Rs+Xfq0+dFxcXFufYvXbZCW7Zu04aNmyUlRdC6dWsVuW27EhMTtWNH0tqdOu1jPfNM91SdH+zhmclTL7z6hj5fvkFT5nyr2dMmat/unzXs/cn6dNFqfbpotWrWa6Ka9Rrf8D7Gz5yvTxet1rQFK1zb3h02UJ2f76tPF61W194va+ywgZKkiI/e14jx09Sj72uaPX2yJGni2JHq2KO3kt5AjFtF1KQj1e6vrntz5062beGC+Wr32BOSpHaPPaGv58+75rwtmzeqeIkSKlasuDJnzqwWLVvr6wXz5eHhobi4ODmOowsXLsrLy0tj3x6tbs/0lJeXV6rMCfbbuHGjSpQoqeLFk9Zfq9ZtNG/e3GTHGGN0/tw5OY6j8+fPK3fu3PL0vPH7GJKv3Qvy8vLSqFEj9WzP51i7uG158+VX2YBASVLWbNlVtGRpHT96xLXfcRwt/for1W3yyC3drzFGMefPSZLOnzsrb5/8kiRPLy9dunhBFy9ckKeXp6IP/KrjRw8rtEq1OzSjjIeoSeeOHzuq/AUKSJLyFyig48ePXXPM4cOHVbCgn+t2wYIFdeTwIWXPnl1NmrVQ9SphKlq0qHLkyKmtWzarYeMmqTZ+2O/woUMqVKiQ67ZfQT8dPnQo2TE9evTUTz//pEJ+vgoKDNBbb4+Rh0fSw5MxRvXr1VHlSqH6aMIESVL27NnVosUjCgsNVtFixZQzZ05t3rRJTZo2Tb2JwWqHD/6mXVHb5R8U6toWuXGt8uT1UeFiJa57jpFRz8db6ImGD+rLTz92be89YKjGDh2ghlX8NWbIAPXoN0CS1LF7Lw19uZdmTPpArTo8rXEj/6tuL77i1nnZzp2/pRv/Etd7bfbPpzZf6P2SXuj9kiSp5zNd9J/XBmrqlElavmypKlQIUJ/+/0nVscI+N1t/f/p28WIFBgZp6dLl+uWXX1Sv7sOqXr26cuTIoVWr18jX11fHjh1TvboPq0zZsqpRo4b69OmrPn36SpK6PN1Zgwa/rkkTJ2rJkm8VULGiXnnl1VSZH+wTG3Ne/bq1V+8Bw5Qtew7X9m/nfaE6N3mWZuKX38g7XwGdPHFcPR9vrqIlSikkvJq+iJis3q8N1UMNmmjJgq/0377Padync1TGP0BT5iyRJG3dsEbePgXkOI5e7tFJnp5JL4Xl8fZx+3xtwjM16Zy3Tz79fiTp6dHfjxyR93X+ARQsWFCHDkW7bh86dEj5C/gmO+aHbZGSpJKlSmvmpxGaGjFDP/4YpV/27nHj6JERFPTz08GDB123ow9Fq4Bv8vX38cdT1Lx5CxljVLJkSRUtVkw///yzJMn3yrE+Pj5q2qy5Nm3amOzcyMiktVu6dGlNj5immbM+U9TOndqzh7WLW5cQH69+3TqoXrNH9VD9v66dSUhI0IpvFujhxs1veK53vqRnzXPn9daDdRspattWSdKCL2ao5pX7qt2wmX78YWuy8xzH0eR3R+up5/voo3dGqGuv/qrfvJVmTfnwTk/PekRNOle/YSN9+sl0SdKnn0xXg0bXXsAWElpJv+zdq/37f1VcXJy+nD1LDRo2SnbMkNcH6ZXXBik+Pl6JiYmSkq5biI2Ndf8kYLVKlSpp7949+vXXpPX32ayZavx/L3EWLlxYy5cvkyQdPXpUu3ftUvHixRUTE6Nz55KuRYiJidGSJd/K379CsnMHDnhNgwa/ztrFP+Y4jv7b91kVLVlajz3dI9m+jd9/pyIlSilfgYLXPfdCbIzrupkLsTFav2q5SpQpJ0ny9imgrevXSJI2rVmlQkWLJzt3wewZqvZQHeXImUuXLl6Q8fCQh4eHLl68cKenaD1efkpHOnV4XN+vWqk//jihciWL6uVXB6j3i33V4Ym2mj51ivwKFdLUiJmSpCOHD+vZ7l01e858eXp6atRbY9SiSUMlJibq8fYdVa68v+t+F8ybq5DQMNdPz5XCq+i+SkHyrxCggIqBaTJX2MPT01Njxr6nBvXrKjExUR2f7CR/f399OD7p7bJdu3XTK6++pk5PdlRQYIAcx9GwYSOUN29e7du3Ty0fSfrJOCEhQW3atlO9evVc9z13zhyFVarkejanSpX7FBQYoICAigoMZO3i1vyweb0WfjlLJcuWd71tu0ef11TtoTr6dv6X11wgfPzoEb3R9zmNmfq5/jhxXH27PC5JSkhIVL2mj6jqg7UlSa+MeEejB72sxMQEZb7rbv1n+Duu+7h4IVZfz56h9yKSfudzu87d1a9be3l5ZdYb705MjWlbxfyb3gsfHBLqrFyzIa2HAdySrHfxswHSp8jo02k9BOCWVS1XZG9czJlS19vHy08AAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1+F979xZq6RjHcfz7M8hhhiGnUozTzIQYh5HzKU3KIUSZFBM5lkOilEOUosaVlOMFNZFzieR0wcw0I4wZ5zE55cIFLhC54e9iPabXto21RzPLvPP91Gq/a73/932eZ/fs3a/nWXsvSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6wVAjSZJ6IVU16j6sluRb4KtR96OndgC+G3UnpAly3mpD5dxdd3avqh3HO/G/CjVad5K8XVWHjrof0kQ4b7Whcu6OhttPkiSpFww1kiSpFww1G48HRt0BaS04b7Whcu6OgKFmI1FV/oBpvUpyfJLn2/HpSW5YQ+3UJFeMff3f5m2SW5NcN+zrY2oeTnL2mmrG1E9L8sGw9dq4+Tt3NAw1kiYkyaSJXlNVz1XVnWsomQr8LdRI0kQYaiQBq1ciPknySJL3kjyVZKt27ssktyRZBJyTZE6SJUmWJXkyyeRWd3K7xyLgrM695yW5px3vnOTZJCva40jgTmCvJMuTzG911yd5q/Xlts69bkyyMsmrwIwhxnVxu8+KJE//OabmpCQLk3ya5NRWPynJ/E7bl/7X762k9cNQI6lrBvBAVR0A/MhfV09+raqjgVeBm4CTqupg4G3g2iRbAA8CpwHHALv8Qxt3A69X1YHAwcCHwA3AZ1U1q6quTzIH2Ac4DJgFHJLk2CSHAOcCBzEITbOHGNMzVTW7tfcxcFHn3DTgOOAU4L42houAH6pqdrv/xUn2GKIdSSO26ag7IOl/5euqWtyOFwBXAXe154+3r4cD+wKLkwBsDiwBZgJfVNUqgCQLgEvGaeNE4HyAqvoN+CHJdmNq5rTHu+35ZAYhZwrwbFX90tp4bogx7Z/kdgZbXJOBlzrnnqiq34FVST5vY5gDHNB5v822re1Ph2hL0ggZaiR1jf1vnN3nP7evAV6pqrndwiSzxrl+bQW4o6ruH9PGNWvRxsPAGVW1Isk84PjOufHGG+DKquqGH5JMm2C7ktYzt58kde2W5Ih2PBdYNE7NUuCoJHsDJNkqyXTgE2CPJHt1rh/Pa8Dl7dpJSbYBfmKwCvOnl4ALO+/V2TXJTsAbwJlJtkwyhcFW17+ZAnyTZDPgvDHnzkmySevznsDK1vblrZ4k05NsPUQ7kkbMUCOp62PggiTvAdsD944tqKpvgXnAY61uKTCzqn5lsN30Qnuj8D99jtvVwAlJ3gfeAfarqu8ZbGd9kGR+Vb0MPAosaXVPAVOqahmDbbDlwNPAwiHGdDPwJvAKg+DVtRJ4HXgRuKyN4SHgI2BZ+xPu+3FVW9og+NlPkoDV2yvPV9X+I+6KJK0VV2okSVIvuFIjSZJ6wZUaSZLUC4YaSZLUC4YaSZLUC4YaSZLUC4YaSZLUC38AZC6n1yQPwVgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "ax.matshow(conf_mx, cmap=plt.cm.Blues, alpha=0.3)\n",
    "for i in range(conf_mx.shape[0]):\n",
    "    for j in range(conf_mx.shape[1]):\n",
    "        perc = str(round((conf_mx[i, j]/conf_mx.sum())*100,2)) + \"%\"\n",
    "        ax.text(x=j, y=i, s=str(conf_mx[i, j])+\"\\n\\n\"+perc, va='center', ha='center')\n",
    "\n",
    "plt.xlabel('predicted label')\n",
    "plt.ylabel('true label')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trying to Optimize the KNN Model with Baeysian Optimization with gaussian process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_knn(params,param_names,x,y):\n",
    "    params = dict(zip(param_names,params)) \n",
    "    model = KNeighborsClassifier(**params) #It takes the params as parameter\n",
    "    return -np.mean(cross_val_score(model, X_train, y_train, cv=StratifiedKFold(n_splits=3),n_jobs=-1, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_space_knn = [\n",
    "    space.Categorical(['uniform', 'distance'],name=\"weights\"),\n",
    "    space.Integer(1,10,name=\"n_neighbors\")\n",
    "]\n",
    "param_names_knn = [\"weights\",\"n_neighbors\"]\n",
    "optimization_function = partial(optimize_knn,param_names= param_names_knn,x = X_train,y = y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at random point.\n",
      "Iteration No: 1 ended. Evaluation done at random point.\n",
      "Time taken: 62.1422\n",
      "Function value obtained: -0.7715\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "Iteration No: 2 ended. Evaluation done at random point.\n",
      "Time taken: 54.1438\n",
      "Function value obtained: -0.7697\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 3 started. Evaluating function at random point.\n",
      "Iteration No: 3 ended. Evaluation done at random point.\n",
      "Time taken: 32.1412\n",
      "Function value obtained: -0.7585\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 4 started. Evaluating function at random point.\n",
      "Iteration No: 4 ended. Evaluation done at random point.\n",
      "Time taken: 58.0561\n",
      "Function value obtained: -0.7703\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 5 started. Evaluating function at random point.\n",
      "Iteration No: 5 ended. Evaluation done at random point.\n",
      "Time taken: 58.2534\n",
      "Function value obtained: -0.7678\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 6 started. Evaluating function at random point.\n",
      "Iteration No: 6 ended. Evaluation done at random point.\n",
      "Time taken: 31.3748\n",
      "Function value obtained: -0.7585\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 7 started. Evaluating function at random point.\n",
      "Iteration No: 7 ended. Evaluation done at random point.\n",
      "Time taken: 67.6096\n",
      "Function value obtained: -0.7662\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 8 started. Evaluating function at random point.\n",
      "Iteration No: 8 ended. Evaluation done at random point.\n",
      "Time taken: 53.6644\n",
      "Function value obtained: -0.7697\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 9 started. Evaluating function at random point.\n",
      "Iteration No: 9 ended. Evaluation done at random point.\n",
      "Time taken: 57.4628\n",
      "Function value obtained: -0.7703\n",
      "Current minimum: -0.7715\n",
      "Iteration No: 10 started. Evaluating function at random point.\n",
      "Iteration No: 10 ended. Evaluation done at random point.\n",
      "Time taken: 49.8410\n",
      "Function value obtained: -0.7674\n",
      "Current minimum: -0.7715\n"
     ]
    }
   ],
   "source": [
    "result = gp_minimize(optimization_function,dimensions=param_space_knn,n_calls=10,verbose=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score = -0.7715\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score = %.4f\" % result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'weights': 'distance', 'n_neighbors': 8}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_params = dict(zip(param_names_knn,result.x))\n",
    "knn_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(result, open('bayessian_optimization_knn.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78439394, 0.78813131, 0.78242424])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg = xgboost.XGBClassifier()\n",
    "cross_val_score(xg, X_train, y_train, cv=StratifiedKFold(n_splits=3), scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_pred = cross_val_predict(xg, X_train, y_train, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAI4CAYAAACSixhGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3RU1cKG8XeHBL1gQAJEQ+89hBQMSvGDq4IovaP0XiwgIFgoCgJKkaJ4lSKdIHhpAkrvndClCShBpGqEBEzhfH+EOxJACcpkZOf5rcUyc8pkH93EJ3POmTGO4wgAAOB+5+XpAQAAANwLRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRYzljTDVjzCFjzFFjTG9PjwdICWPMRGPMWWPMPk+PBUgpY0xuY8wqY8y3xpj9xphXPD2mtMbwPjX2Msakk3RY0tOSoiRtk9TEcZwDHh0YcAfGmEqSLkua4jhOKU+PB0gJY0yApADHcXYaY3wl7ZBUm5+5qYdXauz2mKSjjuMccxwnTtIsSbU8PCbgjhzHWSvpoqfHAdwNx3FOO46z8/rXlyR9KymnZ0eVthA1dssp6eQNj6PEXzAAcDtjTD5JwZK2eHYkaQtRYzdzm2WcbwQANzLGPCRprqRXHcf51dPjSUuIGrtFScp9w+Nckn700FgAwHrGGB8lBc10x3G+9PR40hqixm7bJBU2xuQ3xqSX1FjSAg+PCQCsZIwxkiZI+tZxnBGeHk9aRNRYzHGcBEldJX2tpAvWZjuOs9+zowLuzBgzU9ImSUWNMVHGmDaeHhOQAuUlNZNUxRiz6/qf6p4eVFrCLd0AAMAKvFIDAACsQNQAAAArEDUAAMAKRA0AALACUZNGGGPae3oMwN1i3uJ+xdz1DKIm7eAvGO5HzFvcr5i7HkDUAAAAK/yj3qcm88NZHP9Hc3h6GFaK/uVnZX44i6eHYaXMD/3L00Ow1rlz55Q9e3ZPD8Na1/45P/6tc/78OWXLxtx1h3179/waFxeX+XbrvFN7MH/G/9EcGvXpLE8PA7grVSuU8vQQgL8kLpGqwf0nh3+2s3+0jtNPAADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsIK3pweAlDt39icNH/Smfr54Xl5eXqpWo55q1X9R3x05qI9GvKu4uDilS5dOnbu9qaLFA7Vq2VeaO+tz1/4nvjusUZ9FKCBnbvV6qaVr+YVzZ1T56efU/qXXtWzJfE0cN0JZs/tLkmrUaayqz9dL5SNFWvHLL7+ofbt22r9/n4wx+mz8BBUtWlRNGjfW99+fUN68+TQrIkJZsmTR1q1b1aljB0mS4zjq27efatep4+EjQFoRdfKk2rZuqTM//SQvLy+1bttOXV56Wbt37dLLXTvr6tWr8vb21odjxqps2ccUHx+vzh3aaVdkpBISEtT0xWbq+XpvSVK/t9/SjOlT9cvPP+vcz796+MjsYhzH8fQYXAoXK+mM+nSWp4fxj3XxwjldvHBOhYqUUGxsjF5p11hvD/pQn455X7UbvKiwchW1bfM6zZ05SUNGTUy274nvDuudN1/RxFlLbnnel9s1UvuuPVUqKEzLlszX0UP71enVN1LrsO57VSuU8vQQ7lutWrZUhQoV1KZtW8XFxSk2NlaDB78nPz8/vf56bw0dOkQ///yzhgwZqtjYWKVPn17e3t46ffq0QoLL6GTUKXl787vZXxWX+M/5+f9Pd/r0af3002kFB4fo0qVLKh9eVhFzvlSvHt3U9eVXVbXas1q6ZLFGDh+mr5evVMTMGfpq0UJNmT5TsbGxCgkqpa+XrVTefPm0dctm5c6TV6VLFCVq/oIc/tmO/vzzxcK3W8dPg/uIX9bs8suaXZKUIUNG5c6bXxfOnZUxRrGxMZKkmMuXXNvcaM2KJXry38/esvxU1PeK/vmiSpYOde/ggZv8+uuvWrdurSZOmiRJSp8+vdKnT6+FCxZoxcpVkqTmzVvo31Uqa8iQocqQIYNr36tXr8oY45FxI20KCAhQQECAJMnX11dFixXTjz+ekjFGl35NCpNfo6Nd2xhjFBMTo4SEBF25ckXpfdLLN1MmSdJj4eU8cxBpAFFznzpz+pSOHTmooiUC1a5rL/Xt2VETPh4ux3E07KMpt2y/dtXXenvQqFuWr1m+RBWrVE32P4gNa5Zr3+4dypk7r9p17aXs/o+69ViQNh07dkzZsmdXm9attWfPboWEhGjkh6N05swZ1/8YAgICdPbsWdc+W7ZsUbu2bfT9999r8uQpvEoDj/j+xAnt3r1LZR8L1/vDRqrm88+qT+9eunbtmlatWS9JqlOvvhYtXKACeXIqNjZWQ4cNl5+fn4dHbj8uFL4PXYmN1aC+3dXupV7KkPEhLZ4/W+269tTkOcvUrktPffh+v2TbHzywRw888KDyFbj11bq1K5fqyX9Xdz0Of+JJTYpYqo8mzVWZ0HIa8d6bbj8epE0JCQmK3LlTHTp21PYdO5UxY0YNHTrkT/cJDw/Xnr37tHnLVg0ZOkRXr15NpdECSS5fvqwmjRro/WEjlClTJn326Sd6/4PhOnLse73/wXB16tBOkrR921alS5dO330fpQOHv9PokSN1/NgxD4/efkTNfSYhIV7v9e2uyk89p/KVnpIkrfh6gZ64/nWFys/o8Lf7ku2TFC63nno6dvSQEhMTVbhoCdeyTJkflk/69JKkqs/X09HD37rrUJDG5cqVS7ly5VJ4eLgkqW69+orcGalHHnlEp0+flpR0HYO/v/8t+xYvXlwZM2bUvn37blkHuEt8fLyaNqqvxk2aqnadupKk6VOnqNb1r+vWb6Dt27ZKkiJmzdTTz1SVj4+P/P39Ve6JJ7Rzx3aPjT2tIGruI47jaNTQfsqdN7/qNGruWu6XNbv27kr6y7J75xblyJXHte7atWtav/obVbpN1CRdZ1Mt2bKLF865vt6yYbVy581/rw8DkCQ9+uijypU7tw4dOiRJWrlyhYqXKK7na9TQlCmTJUlTpkxWjZo1JUnHjx9XQkKCJOn777/X4UOHlC9fPo+MHWmP4zjq1L6tihYrrpdf7eZaHhCQQ+vWrpEkrV61UgULJb0injt3Hq1evUqO4ygmJkbbtmxRkaLFPDL2tIQT0veRA3sjtfKbRcpXoLC6tmkgSWrR7mW93LOf/jNmqK4lJsonfXq91OP300/7du9QtuyPKCBHrlueb92qrzVg6MfJli2YO0NbNqxWunTp9JBvZnXrPdC9B4U0bdSo0Wre7EXFxcUpf/4CmjBxoq5du6bGjRtp0sSJyp0njyIiZkuSNqxfr/ffHyofHx95eXlp7NiPlC1bNg8fAdKKTRs3aMb0aSpVKlDhYSGSpAHvDtRHn/xHPbp3U2JCgh548EGNHfeJJKlDp87q0La1wsqUluM4ataipQJLl5Ykvdn7dUVEJN0VVSh/HrVs1UZv9e33h98bKcct3cDfxC3duF9xSzfuR392SzennwAAgBWIGgAAYAWiBgAAWMGtUWOMqWaMOWSMOWqM6e3O7wUAANI2t0WNMSadpI8kPSuphKQmxpgSf74XAADAX+POV2oek3TUcZxjjuPESZolqZYbvx8AAEjD3Bk1OSWdvOFx1PVlAAAA95w7o+Z2H6F7y5siGGPaG2O2G2O2R//ysxuHAwAAbObOqImSlPuGx7kk/XjzRo7jfOo4TpjjOGGZH87ixuEAAACbuTNqtkkqbIzJb4xJL6mxpAVu/H4AACANc9tnPzmOk2CM6Srpa0npJE10HGe/u74fAABI29z6gZaO4yyWtNid3wMAAEDiHYUBAIAliBoAAGAFouY+kJiYqJfaNFT/3l0lSd8dOajunV5Q1zYN9Er7xjr07d4U73ujubM+13NPltb/bqU/sDdSXVrV06vtm+jHqB8kSZcv/aq3e3SU49xyNz6QYm3btFbAo48oqHTgbdc7jqNXX3lZRYsUVnCZIO3cuVOSdPLkSf3731VUqmQJlQ4spdGjR7n26d37dQWXCVLLFi1cy6ZNnZpsG+DvSkxMVLmyoapbu8Yt66Kjo1Wvdk2FhwYrNChQUyZPcq0rVriAygYHKTwsROXLPeZa/laf3nospIzatvp93s6YNlUfjRnt3gNJI4ia+8CCOdOVO29+1+NJn4xU0xYdNXbCF3qxdRdN+mRkivf9n3Nnf9Ku7ZuV/ZEA17IvI6bojXdHqEW7l7V4/mxJ0qwpn6rhi21lzO3edghImeYtWuqrxUv+cP2SJUt05MhRHTx0WOM++Y+6dOksSfL29tYHHwzTvv0HtGHjJo37+GMdOHBA0dHR2rRpkyJ37VZiYqL27t2rK1euaPKUyerUqXNqHRbSgI/GjFaxYsVuu+4/4z5W8eIltGVHpJYuX6k+vXoqLi7OtX7JshXasn2nNmzeKikpgjZv3qStO3cpMTFR+67P22lTp6h9x06pcjy2I2r+4c6f/UnbNq9V1efrupYZYxQbGyNJirl8SX5Zs6d43//5bOz7atWxW7JY8fb2Vtxvv+nqb1eVzttbp0+d1IXzZxVYJuweHxXSmkqVKsnPz+8P1y9cMF/NmjWTMUblypVT9C+/6PTp0woICFBISIgkydfXV8WKFdepU6fk5eWluLg4OY6jK1euyMfHR8OGfaCXur4kHx+f1DosWC4qKkpLlyxWy9ZtbrveGKNLly/JcRzFXL6sLH5+8vb+4/tvbjdvRw4fpk5dujJv7xGi5h/u07Hvq1XH7jLm9/9U7br20sRxI9Si/tOaOG6EWrZ/JcX7StLmDauUNZu/ChQqmmx5gxfaaMywAZr/xTTVqNNYU8aP0Yttutz7gwJucurUj8qV+/f36syZK5dOnTqVbJsTJ05o165IhYeHy9fXV3Xr1lVYaIjy5c+nzJkza/u27apZi4+Xw73T67VuGjh4iLy8bv+/yo6du+jQwYMqkDeXyoYE6YPhI13bGmNUo3o1PRFeVhPGfyopKcxr16mrcmVDlS9/fmXKnFk7tm9TjZrM23vFrbd04+/ZunGNMj/sp8JFS2hP5DbX8sXzZ6td154q/+TTWrfya334fj+9N+KzFO179eoVRUz9TAOH/eeW71ewcDGNGDddkrRv9/akV4AcR0P691Q6b2+17dxDWfyyuulokZbd7pqtG19FvHz5sho2qK8RI0YqU6ZMkqSePXupZ89ekqT27dqq/4ABmjB+vJYtW6bA0oF68823UmfwsNLirxYpu7+/QkJCtXbN6ttus/ybr1U6KEhLvlmuY999p+erV1X5ChWVKVMmrVi9Tjly5NDZs2dV49mqKlq0mCpUrKTuPXqqe4+ekqROHdrp7X4DNGnieK1YtkylAkur9xtvpuJR2odXav7BDuzbpS0bV6tVo2oa+k4v7dm5VR8M7KMVXy/QE5WekiRVqPyMDn+7L8X7/nTqpM6cPqWubRqoVaNqOn/ujF5p10gXL5x37es4jmZN+UyNW3TQjM8/0QutOqvy089pwdzpqXTkSGty5cqpqJO/f/7tqago5ciRQ5IUHx+vBvXrq0nTpqpT99ZTqZGRkZKkIkWKaOq0qZoVEaH9+/bryJEjqTN4WGnzxo36atFCFStcQM1fbKo1q1apdYtmybaZMuVz1apdR8YYFSxUSPny5dehQwclyTV//f39VaNWbW3fti3Zvruuz9vCRYpoxrRpmjYzQgf279NR5u3fQtT8g7Vs/4qmzFmuSRFL9Xrf91U65DH1fGuw/LJm195d2yVJu3duUY5ceVK8b76CRTRj/hpNiliqSRFLlS37Ixr1WYT8smZz7bt86QKVfbyifH0z6berV+XlZeTl5aXffruaaseOtOX5GjU1depUOY6jzZs3K1PmzAoICJDjOGrXtq2KFy+mbt2633bffn37qv+AdxQfH6/ExERJSdcuxMbGpuYhwDLvDHpPR4//oINHjmnKtBl6snJlTZw8Ndk2uXPn0aqVKyVJZ86c0eHDh5Q/fwHFxMTo0qVLkqSYmBitWL5MJUqWTP78A/rq7X4DmLf3GKef7kMv9+yn/4wZqmuJifJJn14v9egnSbpw/qxGv99fA97/+C8/99WrV7Ri6QINHP6JJKl2w2Ya9HZ3efv46PW+Q+/J+JH2vNC0qdasWa3z588rb57c6tevv+Lj4yVJHTp2VPXq1bV0yWIVLVJYGTJk0PgJEyVJGzZs0LRpUxUYGKjQkGBJ0rsDB6l69eqSpPnz5imsbJjrt+Jy5cqpTFBpBQaWVlBQkAeOFLb77NOkn43t2ndU7zfeUoe2rVQ2OEiO42jgoMHKli2bjh87psYN6kmSEhIS1LBxEz1TtZrrORbMn6fQ0LKueRterpzKBgepVGCgSjNv/xbzT3r/kcLFSjqjPp3l6WEAd6VqhVKeHgLwl8Ql/nN+/gMplcM/29Gff75Y+HbrOP0EAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAKxA1AADACkQNAACwAlEDAACsQNQAAAArEDUAAMAKRA0AALACUQMAAKxA1AAAACsQNQAAwApEDQAAsAJRAwAArEDUAAAAK3h7egA3ypTxX/r34yU9PQzgrlxzPD0C4K/x8TKeHgJw18yfTFteqQEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFbz/aIUx5pIk538Pr//Tuf614zhOJjePDQAAIMX+MGocx/FNzYEAAAD8HSk6/WSMqWCMaXX962zGmPzuHRYAAMDduWPUGGP6SXpdUp/ri9JLmubOQQEAANytlLxSU0dSTUkxkuQ4zo+SODUFAAD+UVISNXGO4zi6ftGwMSaje4eElDh58qSeefrfCgosqeCgQI0dMzrZ+pEjhuvB9Ol0/vx5SVJ8fLzatG6p0OAgBQWW1PtDh0iSLl26pMfCQlx/cgb4q8dr3VL9eJB2tGvbWjkDHlGZoEDXsosXL+rZqs+oRLEierbqM/r5558lSXFxcWrbprWCy5RWaEgZrVm92rXP22+9qQL58ihLZn7HQuo6dOiQQkODXX/8smTWqFEfql/ftxUcHKTQ0GA9W62qfvzxR0lJ87hNm9YqU6a0Qm6ax7i3UhI1s40x/5H0sDGmnaTlkj5z77BwJ97e3hr6/gfavXe/1q7fqE/GfaxvDxyQlBQ8K1YsU+48eVzbz53zheJ++007Indr05ZtGj/+U504cUK+vr7aun2n60+ePHlVq3YdTx0W0oDmzVtq0VdLki17f+gQVa5SRQcOHlblKlVc0T1hfNKPmshde7Rk6Tfq1auHrl27Jkl6/vka2rBpS+oOHpBUtGhR7dgRqR07IrV163ZlyJBBtWvX0Ws9eioycrd27IhU9eee08CB70iSxl+fx7t27dHSpd+o5w3zGPfWHaPGcZxhkuZImiupiKS+juOMcffA8OcCAgIUHBwiSfL19VWxYsV06sdTkqRePbrrvfeGyhjj2t4Yo5iYGCUkJOjKlStK75NemTIlvyv/6JEjOnvurCpUqJh6B4I0p2KlSsri55ds2cKFC9SseQtJUrPmLbRgwXxJ0rffHlDlKlUkSf7+/no488PasX27JCm8XDkFBASk4siBW61csUIFChRU3rx5k/1MjY2Jcf0M/vbbA6py0zzefn0e495K6Zvv7ZW0TtLa61/jH+TEiRPatXuXHnssXIsWLlCOnDlVOigo2TZ169VXxowZlS9PThUumE+vdu8uv5v+xxIRMUsNGjRMFkNAajh75owrUAICAnTu7FlJUunSQVq4YIESEhJ0/Phx7dy5QyejTnpyqEAyEbNnqVHjxq7Hb7/1pvLny6OZM2eof/+kV2puN4+jmMdukZK7n9pK2iqprqT6kjYbY1q7e2BImcuXL6tJowYaNmxE0impIYPVt9+AW7bbtm2rvNKl0/Hvo3Tw8HcaNXKkjh07lmybL2ZHqGGjxrfsC3hKy1atlStnTpULL6vXunfT448/IW/vP3x7LSBVxcXFadHChapfv4Fr2bsDB+n4iR/UpElTffzRWElSq1atlTNnToUzj90uJf9We0oKdhzngiQZY7JK2ihpojsHhjuLj49X40b11bhJU9WuU1f79u7ViRPHVTYsWJJ0KipK5cLDtH7DZkXMmqlnnqkqHx8f+fv76/EnntDOHdtVoEABSdKe3buVkJCgkJBQTx4S0ij/Rx7R6dOnFRAQoNOnTyu7v7+kpGvHho0Y6dquUoXyKlSosKeGCSSzdOkSBQeH6JFHHrllXeMmTVWr5vPq13+AvL29NfyGeVyReew2KTn9FCXp0g2PL0nidTMPcxxHHdq3VbFixfXKq0l3K5UKDNTJUz/p8JFjOnzkmHLmyqXNW7br0UcfVe7cebR69So5jqOYmBht3bJFRYsWcz3f7IhZvEoDj6nxfA1NnTJZkjR1ymTVqFFTkhQbG6uYmBhJ0vJly+Tt7a0SJUp4bJzAjSJmJT/1dOTIEdfXCxcucP2MZR6nnj/77Kfu1788JWmLMWa+km7rrqWk01HwoI0bN2jG9GkqVSpQj4UlXTD8zrsDVe3Z6rfdvmOnzmrftrVCypSW4zhq3qKlAkuXdq2fM/cLzZ+/KFXGjrTtxReaau2a1Tp//rzy582tvv36q+frvdW0cSN9PmmicufOo5kRsyVJZ8+e1XPVq8nLy0s5c+TUpMlTXM/T+/Veipg1U7GxscqfN7datW6jvv36e+iokNbExsZq+fJl+njcJ65lb77RR4cPH5Lx8lLePHn10cfjJCWfxzly5NTnN8xj3Fsm6S1obrMi6Z2E/5DjOLdeuPE3hYaGORs300u4v3h5cWE17k/MXNyP/LNnPXrx4sXbnr/7sw+0vOfRAgAA4C53vFDYGJNdUi9JJSU9+L/ljuNUceO4AAAA7kpKLhSeLumgpPySBkg6IWmbG8cEAABw11ISNVkdx5kgKd5xnDWO47SWVM7N4wIAALgrKYma+Ov/PG2Mec4YEywp1512MsZMNMacNcbs+1sjBAAASIGURM1AY0xmSa9J6iFpvKSUfIzz55Kq/fWhAQAApNwdLxR2HOd/b14SLalySp/YcZy1xph8f21YAAAAd+fP3nxvjJLebO+2HMd5+V4MwBjTXlJ7ScqdJ8+9eEoAAJAG/dkrNanyueiO43wq6VMp6c33UuN7AgAA+/zZm+9NTs2BAAAA/B0puVAYAADgH89tUWOMmSlpk6SixpgoY0wbd30vAACAO9799Fc5jtPEXc8NAABwszu+UmOMKWKMWfG/N9EzxpQ2xrzl/qEBAACkXEpOP30mqY+uv7Ow4zh7JDV256DwxxITExVeNlR1ate4Zd2hgwf1ZMXyyvTQvzRyxPBk68aOGa2QMqUVHBSoMaNHuZa/2ae3wkLKqHWrFq5l06dN1dgxo913EEhz2rVtrZwBj6hMUOBt10dHR6t2rZoKDSmjoNKlNPnzScnWJyYmqmxYiGrX/H3e9+n9ukKCg9Sq5e9zd9q0qcnmN/B3tG3bWjn+ZN7OmDFdwcFBCg4OUsUK5bV79+5k6xMTExUWFqJaN83b4OAgtbxp3o5m3t4TKYmaDI7jbL1pWYI7BoM7GztmtIoWK3bbdVn8/DR85Id6tdtryZbv37dPEyeM1/qNm7VtR6QWL/5KR48cUXR0tDZt3qTtO3cpMTFR+/bu1ZUrVzR16hR16NgpNQ4HaUTz5i216Kslf7h+3McfqXjx4tqxc5eWr1ilXj17KC4uzrV+zOhRKlasuOtxdHS0Nm/apJ2Ru5WYmOXoZiwAAB5JSURBVKi9/5u7kyerY6fObj0WpB0t7jBv8+XLr5UrVysycrfefPMtderYIdn60aNHqfhN83bTpk2KvGneTpk8WZ2Yt/dESqLmvDGmoK6/EZ8xpr6k024dFW4rKipKS5YsVqvWt7/m2t/fX2FhZeXj45Ns+cGD3+qx8HBlyJBB3t7eqlixkubPnycvLy/Fx8XJcRxdvXJFPj4+GjF8mLp06XrLcwB/R8VKlZTFz+8P1xtjdPnyJTmOo8uXL8vPz0/e3kmX/EVFRWnJ4sVqfcO89/LyUtz1uXvl+twdPuwDdXnpJeYu7pmKlSrJ70/m7RNPPKEsWbJIksLLldOpU1GudXeat1dvmLddmbf3TEqipouk/0gqZow5JelVSfwa7wE9X+um9wYPkZfX3d20VrJkKa1ft04XLlxQbGysvl66RFFRJ+Xr66vadeoqvGyo8uXPr0yZM2vH9m2qUbOWm44AuL3OXbrq4LcHlTd3ToWUKa3hIz50zfPXunfT4CFDk817X19f1albV2XDQpQ/Xz5lzpxZ27dvV03mLjxk0sQJqlrt9487/KN5W7duXYWFhSgf89YtUvLZT8ckPWWMySjJy3GcS+4fFm62+KtFyu7vr5CQUK1Zs/qu9i1WvLhe69lTzz1bVRkfekiBpUu7fgt+rUdPvdajpySpY4d26ttvgCZOHK8Vy5apVGBp9XnjzXt9KMAtvvnmawUFBemb5Sv03XffqXq1Z1ShYkWtW7tW/v7ZFRIaqjWrVyfbp0fPXurRs5ckqUP7turXf4AmThivZcuWKTAwUG+8yf0MSB2rV63SpEkTtXrNOknSV4sWKbt/doXeYd62vz5vJ0wYr+XM23siJXc/9TXG9FXSp3R3u+ExUtHGjRv11aKFKlK4gJq/2FSrV61SyxbNUrx/q1ZttHnrdq1YuVpZsvipUKHCydbvioyUJBUuUkTTp03T9JkROrB/n44eOXJPjwO4nSmff67aderKGKNChQopX778OnTwoDZu3KBFCxeqcMH8evGFJlq1aqVaNE8+7yOvz90iRYpo2tSpmjkrQvv379cR5i5SwZ49e9ShQzvN/XKesmbNKkmueVuoYH69cH3eNmfepoqUnMeIueFPoqRnJeVz45hwGwMHvafvjv+gw0eOacq0Gfq/ypX1+eSpKd7/7NmzkqQffvhB8+f9Vw0bJb+BbcCAvurbb4Di4+OVmJgoSTJeXoqNjb13BwH8gdx5cmvlyhWSpDNnzujw4UPKX6CABr03WMe/P6kj3x3XtOkzVblyFU2eknzeD+jXV/36v5Ns7noxd5EKfvjhBzVsUE+TPp+iIkWKuJYPem+wTnx/Uke/O67p1+ftlJvmbf9+fdWfeXvPpeT0U7J7g40xwyQtcNuIcFc++/QTSVK79h31008/qfzjj+nXX3+Vl5eXxo4Zpcjd+5QpUyY1btRAFy9ckI+Pjz4cPcZ1cZskLZg/T6GhZZUjRw5JUrly5RQaHKRSgYEqHRTkkeOCXV58oanWrlmt8+fPK3/e3Orbr7/i4+MlSe07dNQbb76ttq1bKbhMaTmOo0GDhyhbtmx3fN758+cpNCws2dwNLlNagYGlFcTcxd/04gtNteb6vM1307zt0KGjBg58RxcuXNBLL3WRJHl7e2vLlm13fN758+cp7KZ5W4Z5e08Yx7m7D8Y2xmSRtNVxnMJ33PguhYaGORs333z3OPDP5uVlPD0E4C9h5uJ+5J8969GLFy/etkHu+EqNMWavrt/OLSmdpOyS3rl3wwMAAPj7UvLZT8/f8HWCpDOO4/DmewAA4B/lT6PGGOMl6SvHcUql0ngAAAD+kj+9+8lxnGuSdhtj8qTSeAAAAP6SlJx+CpC03xizVUm3dUuSHMep6bZRAQAA3KWURM0At48CAADgb0pJ1FR3HOf1GxcYY4ZKWuOeIQEAANy9lLyj8NO3WfbsvR4IAADA3/GHr9QYYzpJ6iypgDFmzw2rfCVtcPfAAAAA7safnX6aIWmJpMGSet+w/JLjOBfdOioAAIC79IdR4zhOtKRoSU1SbzgAAAB/TUquqQEAAPjHI2oAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYwdvTA7jRNcfRbwnXPD0M4K5kfOAf9dcISLGdUb94egjAXbv8W+IfruOVGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFbw9vQA8NcFFiukh3wfUjqvdErn7a01G7aoZbOmOnr4kCQpOjpamTNn1votOxQfH6+XOrfX7l2RSkhIVOOmL+q1nq/r0qVLevap/3M956kfT6lR46Ya8sEIDx0V0oqTJ0+qZcvmOvPTT/Ly8lLbdu318suvqEnjRjp8fQ7/8ssvevjhh7Vj5y6dOHFCpUoWV9GiRSVJ4eHl9PG4Tzx5CLDcuz26av3Kr5UlazbNWrbJtTxi0qf6YspnSpfOW+WrPK2X33hHS/87W1M/HePa5ui3+zX1qzUqUjJQ8XFx+qBvL+3YvF5eXl7q1OMtValeU9M/+0gLZk1VOu90etgvm97+YIwCcuXxxKFag6i5zy1aslxZs2VzPf586gzX12/27qlMmTJLkuZ9OUe//RanTdt2KTY2VuEhpVW/YSPlzZtP67fscO1T6YnHVKNWndQ7AKRZ3t7e+uCD4QoJCdGlS5f0WNlQPfXU05o5K8K1TY8erylz5syuxwULFtSOnbs8MVykQc81aKIGLdqpf/eOrmXbN67T2mWLNWPpeqV/4AFdPH9OklStTkNVq9NQknT04H71aPuCipQMlCRNGjtcWbJm09zV23Xt2jX9+svPkqSiJUtr8qKVevBfGTRn6gSNGdxf7300MZWP0i6cfrKU4zj679w5qt+wkSTJGKPYmBglJCTo6pUr8kmfXr6+mZLt893RIzp/7pyeKF/BE0NGGhMQEKCQkBBJkq+vr4oVK65Tp0651juOozlfzFbjxk08NUSkcSHh5ZXp4SzJls2dNlEtOr+q9A88IEnyy5b9lv2+WTBXz9Ss53q8YPY0tezSTZLk5eWlh/2ySpLCnqioB/+VQZIUGFxWZ0+fuuW5cHeImvuZMapd41lVeuIxTZrwWbJVGzesV3Z/fxUsVFiSVKtOPWXImFFFCuRWyaIF9NIr3eTn55dsnzmzI1SnfgMZY1LtEABJOnHihHbtilR4eLhr2bp16/TII4+ocOHCrmXHjx9XWGiwKld+UuvWrfPEUJHG/XD8qHZt3aRWtZ5Sh4bP6cDunbdss2zhf1W1VlLUXIqOliR9Muw9Nav+pHp3aqkL587ess+CiKl6/P+edu/g0wCi5j72zYo1Wrdpm+bOW6Txn47ThvW//5CfM3uW6jds7Hq8Y/tWpUvnpUPf/aA9B45o7OgPdfz4sWTPN3fObNVv0CjVxg9I0uXLl9WwQT2NGPGhMmX6/dXDiFkz1eiGV2kCAgJ0/MQP2r4jUsOGjVCzF5vq119/9cSQkYYlJiTo1+hfNHHeMr38xjvq07mVHMdxrd8XuV0P/utfKli0RNL2iQk6e/pHBYWFa+riNQoMKavRg95O9pxLvozQt3t3qVmHl1L1WGxE1NzHAnLkkCRl9/fX8zVqa8f2bZKkhIQELVwwT3XrNXBt+0XELD31dFX5+Pgou7+/ypV7XJE7f7+WZu+e3UpISFBwSGjqHgTStPj4eDWoX09Nmr6gOnXrupYnJCTov//9Ug0b/h7ZDzzwgLJmTXrZPjQ0VAUKFtThw4dTfcxI2/wDcqpytRoyxqhkmVB5eXnpl4sXXOu/WfhlslNPmbP46cF/ZdD/VXtekvTUc7V0cN8e1/qt61dr0tgRGjZ+huuUFv46ouY+FRMTo0uXLrm+XrlimUqUKClJWr1yhYoUKaqcuXK5ts+VO4/Wrl4lx3EUExOjbdu2qkiRoq71c76I4FUapCrHcdSubRsVL15c3bp1T7Zu+fLlKlqsmHLdMIfPnTunxMRESdKxY8d09MgRFShQIFXHDDz5THVt37hWkvT9saOKj49zXSNz7do1rfxqfrKoMcao4lNVtWPTeknStg1rlb9w0s/eQ/v2aHCfbho2YcZtr83B3ePup/vU2bNn9GLj+pKkhIRE1W/YWE89U1WSNHdOhOrdFCjtOnRS5w5tVS6sjBzH0QvNWqhUYGnX+v/OnaM5/12QegeANG/Dhg2aNm2qAgMDFRpSRpL07sD3VL16dc2OmKXGjZJfILxu7Vr1799X3t7eSpcunT76+JNbrgsD7qW3XmqjHZs26JefL+j58JJq1623ajZ8Ue/27KrGTz8uH5/06jd8nOs6xMgtG+UfkEM58+RL9jxde/dXv24dNfKdPnrYL5v6DhsrSRr9Xl9diY1Rn84tJUmP5sil4RNmpuYhWsfceC7Q04JDQp01G7Z4ehjAXcn4AL8b4P60M+oXTw8BuGvli+c9GhcTXfh26zj9BAAArEDUAAAAKxA1AADACm6LGmNMbmPMKmPMt8aY/caYV9z1vQAAANx5hWOCpNccx9lpjPGVtMMYs8xxnANu/J4AACCNctsrNY7jnHYcZ+f1ry9J+lZSTnd9PwAAkLalyjU1xph8koIlcb82AABwC7dHjTHmIUlzJb3qOM4tH9RijGlvjNlujNl+4fx5dw8HAABYyq1RY4zxUVLQTHcc58vbbeM4zqeO44Q5jhOWNVs2dw4HAABYzJ13PxlJEyR96zjOCHd9HwAAAMm9r9SUl9RMUhVjzK7rf6q78fsBAIA0zG23dDuOs16ScdfzAwAA3Ih3FAYAAFYgagAAgBWImvvMuI9Gq1xYGYWHBunjsaNuWT9q5HBVCA9VhfBQlQsroywPPaCLFy+61icmJqpCuTA1rFvLtazvW330xGPB6tC2pWvZrBnTNO6j0W49FqQdS5cuVYniRVW0SCENHTrkD7fbtm2b0vuk09w5c1zLfvnlFzVsUF8lSxRTqZLFtWnTJklS796vK7hMabVs0dy17bSpUzV69K1/L4CUOPNjlDo1qqGGVcLV6KnHNWviJ5Kkw/v3qnXtp/XCsxXV/PnK2r9rR4r3laTRg95WgyqPqWnV8urZ/kVdio6WJO3etllNq5ZXixpVdPLEMUnSpehovdSsnhzHSYUjtg9Rcx85sH+fJk+aqJVrN2rDlh1aumSxvjt6JNk2r3R7Teu37ND6LTvUb8BAla9YSX5+fq714z4araLFirseR0dHa+vmTdq4NVKJiYnav2+vrly5ounTpqht+06pdmywV2Jiol5+qYsWfbVEe/cdUMSsmTpw4NZPS0lMTFSfPq/rmWeqJlve7dVXVLVqNe0/cFA7I3erePHiio6O1qZNGxW5a48SExO1d2/SvJ085XN16tQ5tQ4NlkmXzluvvDVQs1du0cR53+iLKeN17PBBjRncT21f6aXpS9apQ/c+GjO4X4r3laTHKlbWzG82asbXG5Qnf0F9/nHSDcHTP/tIQz6Zos693tbcqRMlSRNGf6BWXbor6QZi3C2i5j5y6NBBhZV9TBkyZJC3t7cqVKikhQvm/+H2c76IUP0GjVyPT0VF6eulS9S8ZWvXMi8vL8XFxclxHF25clU+Pj4aPXK4OnbqKh8fH7ceD9KGrVu3qmDBQipQoIDSp0+vho0aa8Ft5u3YsWNUt249+fv7u5b9+uuvWrdurVq3aSNJSp8+vR5++OGb5u0V+fj4aNiwD/RS15eZt/jLsj3yqIoFBkmSMj7kq/yFiujcmdOSMYq5fEmSdPnSr8rm/2jK95VUrlIVeXsn3ZdTKriszp7+UZLk7eOj365e0dUrV+Tt462o74/r3JkfFVKuvNuP1VZEzX2kRImS2rhhvS5euKDY2Fh98/USnYo6edttY2NjtXzZ16pZu65rWe9er+mdgYPl5fX7f3ZfX1/VrF1XFcuFKV++fMqUKbN27tiu52rUdPvxIG348dQp5c6d2/U4V85c+vHUqWTbnDp1SvPm/VcdOnRMtvzYsWPKlj272rRupbDQYLVv11YxMTHy9fVV3br1FBYarHz58ytz5szavm2bataqJeBe+PHkDzq0f49KlglV977vafR7ffV8uZIaPaivurzeN8X73mzh7Gl64v+ekiS17NxNg/t006wJ49SgRTuN++BddXjtTbccT1pB1NxHihYrrle791Ct56upXq3nVCqwtKv+b7Zk8SKVK/eE69TT0sVfKXv27AoOufUv2avde2j9lh0aNOQDDXynn954u58mT5qgFi820QdD3nPrMcF+t7s24OaX1rt3e1WDBw9VunTpki1PSEhQ5M6d6tCxk7bviFTGjBld1+T07NlLO3bu0rBhw9Wv79vqP+AdTRg/Xo0bNdSgQQPdd0CwXmzMZfXu2Fzd+w7WQ76ZNHfaRHV7+z0t2rxfr/YdpIG9Xk7xvjeaOGaY0nl7q1qdhpKkIiUDNXHeMo2LWKhTP5xQNv8AOY6jN7q0Vt9X2uvCubNuPU4bETX3meYtW2vdpm1asmyVsmTxU4GChW673ZdfzFb9hr+fetq8eaOWfLVIgcUKqXXzF7R2zSq1a9082T67d0VKkgoVLqJZM6Zp8rSZOnBg/y3X7QB3I2euXDp58vdXFKNORSkgR45k2+zYsV0vNG2sggXyae7cOeratbPmz5unXLlyKVeuXAoPD5ck1a1XX5E7dybbNzIyad4WKVJEU6dN0ayI2dq/b5+OHGHe4u4lxMfr9Y4tVLV2A1V+toYk6au5M11fP/VcbR3YvTPF+/7PojkztX7FN3p31Ke3RL3jOJo0ZrjavNJT4z8cqvbdeuvZOg0VMek/bjhCuxE195lzZ5PK/eTJH7RwwTzVb9j4lm2io6O1fv1aVX/+91NI/d8ZpG+PntDeg0c1ccp0VXqysj6bOCXZfoPe6a833+6v+Ph4JSYmSkq65iY2NtaNRwTblS1bVkePHtHx48cVFxen2RGzVOOm05tHvzuu746d0HfHTqhevfoaO/Zj1apdW48++qhy5c6tQ4cOSZJWrlyh4iVKJNv3f6/SMG/xdzmOo3d7vaT8hYrohXZdXMuz+wdo5+YNkqRtG9Yqd74CKd5XkjatXq6p40Zp+IQZevBfGW7Z96s5M1W+yjPKlPlhXb16RcbLS8bLS1evXrnHR2g/t72jMNyjWdOGunjxonx8vDVs5GhlyZJFEz5Lqvk27TpIkhYtmKcq/35aGTNmTPHzLlowXyGhYa7foMuGl9PjZcuoZKlABZYOuvcHgjTD29tbo0aPVfVnqyoxMVEtW7VWyZIl9Z9Pkm557dCx45/uP2rUGDVv9oLi4uKUP38BTZg4ybVu/rx5CitbVjmuz9ty5R5XmaBABQaWVlAQ8xZ3Z/f2zVryZYQKFSuhF56tKEnq3PNtvTH0Q43o30cJiQl64IEH1WfIh5Kkc2dOa1Cvl/Xh5C/+cN/yVZ7RB317KS7uN3V9sY4kqVRwmPq8N1KSdPVKrL6aM1NjpiV95nPTtp3Vu2Nz+fik17tjxqf2v4L7nvkn3QsfHBLqrNmwxdPDAO5Kxgf43QD3p51Rv3h6CMBdK18879G4mOjCt1vH6ScAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWIGoAAIAViBoAAGAFogYAAFiBqAEAAFYgagAAgBWIGgAAYAWiBgAAWIGoAQAAViBqAACAFYgaAABgBaIGAABYgagBAABWMI7jeHoMLsaYc5K+9/Q4LJVN0nlPDwK4S8xb3K+Yu+6T13Gc7Ldb8Y+KGriPMWa74zhhnh4HcDeYt7hfMXc9g9NPAADACkQNAACwAlGTdnzq6QEAfwHzFvcr5q4HEDVphOM4/AVDqjLG/J8xZtH1r2saY3r/ybYPG2M637z8TvPWGNPfGNMjpctv2uZzY0z9P9vmpu3zmf9v715CraqjOI5/v1lR5u01qCAIzTIpMVOM3i/kElRQgZAEJYmVgx4EQtADgiDBRhGU1aBAikoTpAjTBqaiUZma5Yse0KBBNLAonNRqcP7K7nato4G3br8PHM7e/Nf+r73O4LD4//e9R7f3Gx//b/nOHRlpaiLikKhjDvWaqlpZVYv+IuRk4E9NTUTEoUhTExHAgZWIneor6jZ1mTq2jX2jPq6uB2arg+pGdbP6pjquxV3f5lgP3NqZe676bDs+XV2hbm2vy4BFwER1i7q4xS1UP2r38kRnrkfUXeoa4Lw+6prf5tmqLt9fUzNLXafuVm9s8WPUxZ3c9/zTzzYijow0NRHRdR7wQlVNBX7kj6sn+6rqCmAN8Cgwq6qmAx8DD6nHAS8CNwFXAmccJMczwNqquhCYDnwOPAx8WVXTqmqhOgicC1wMTANmqFepM4DbgIvoNU0z+6jpraqa2fLtAOZ1xsYDVwM3AM+3GuYBe6tqZpt/vjqhjzwRMcKOHukbiIh/lW+rakM7XgrcDzzdzl9v75cA5wMbVIBjgY3AZODrqtoDoC4F7h4mx3XAHQBV9SuwVz1lSMxge33azsfRa3IGgBVV9UvLsbKPmqaoT9Lb4hoHrOqMvVFVvwF71K9aDYPA1M7zNie13Lv7yBURIyhNTUR0Df1vnN3zn9u7wOqqmtMNVKcNc/3hEniqqpYMyfHgYeR4Gbi5qraqc4FrOmPD1StwX1V1mx/U8YeYNyKOsGw/RUTXWeql7XgOsH6YmE3A5eo5AOpYdRKwE5igTuxcP5z3gQXt2jHqicBP9FZh9lsF3NV5VudM9TTgA+AW9Xh1gN5W198ZAL5TjwFuHzI2Wz2q3fPZwK6We0GLR52kntBHnogYYWlqIqJrB3Cnug04FXhuaEBVfQ/MBV5rcZuAyVW1j9520zvtQeGD/Y7bA8C16mfAJ8AFVfUDve2s7eriqnoPeBXY2OKWAQNVtZneNtgWYDmwro+aHgM+BFbTa7y6dgFrgXeBe1sNLwFfAJvbn3AvIavaEf8J+e2niAAObK+8XVVTRvhWIiIOS1ZqIiIiYlTISk1ERESMClmpiYiIiFEhTU1ERESMCmlqIiIiYlRIUxMRERGjQpqaiIiIGBV+B2ej+GkTIZtkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "ax.matshow(conf_mx, cmap=plt.cm.Blues, alpha=0.3)\n",
    "for i in range(conf_mx.shape[0]):\n",
    "    for j in range(conf_mx.shape[1]):\n",
    "        perc = str(round((conf_mx[i, j]/conf_mx.sum())*100,2)) + \"%\"\n",
    "        ax.text(x=j, y=i, s=str(conf_mx[i, j])+\"\\n\\n\"+perc, va='center', ha='center')\n",
    "\n",
    "plt.xlabel('predicted label')\n",
    "plt.ylabel('true label')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trying to Optimize the XGboost Model with Baeysian Optimization with gaussian process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_xg(params,param_names,x,y):\n",
    "    params = dict(zip(param_names,params)) \n",
    "    model = xgboost.XGBClassifier(**params) #It takes the params as parameter\n",
    "    return -np.mean(cross_val_score(model, X_train, y_train, cv=StratifiedKFold(n_splits=3),n_jobs=-1, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_space_xg = [\n",
    "    space.Integer(50,300,name=\"n_estimators\"),\n",
    "    space.Integer(1,10,name=\"max_depth\"),\n",
    "    space.Real(0.0001,0.1,prior=\"log-uniform\",name=\"learning_rate\")\n",
    "]\n",
    "param_names_xg = [\"n_estimators\",\"max_depth\",\"learning_rate\"]\n",
    "optimization_function = partial(optimize_xg,param_names= param_names_xg,x = X_train,y = y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at random point.\n",
      "Iteration No: 1 ended. Evaluation done at random point.\n",
      "Time taken: 149.4969\n",
      "Function value obtained: -0.6625\n",
      "Current minimum: -0.6625\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "Iteration No: 2 ended. Evaluation done at random point.\n",
      "Time taken: 69.4157\n",
      "Function value obtained: -0.6708\n",
      "Current minimum: -0.6708\n",
      "Iteration No: 3 started. Evaluating function at random point.\n",
      "Iteration No: 3 ended. Evaluation done at random point.\n",
      "Time taken: 408.8701\n",
      "Function value obtained: -0.7678\n",
      "Current minimum: -0.7678\n",
      "Iteration No: 4 started. Evaluating function at random point.\n",
      "Iteration No: 4 ended. Evaluation done at random point.\n",
      "Time taken: 356.7607\n",
      "Function value obtained: -0.6957\n",
      "Current minimum: -0.7678\n",
      "Iteration No: 5 started. Evaluating function at random point.\n",
      "Iteration No: 5 ended. Evaluation done at random point.\n",
      "Time taken: 352.9683\n",
      "Function value obtained: -0.7362\n",
      "Current minimum: -0.7678\n",
      "Iteration No: 6 started. Evaluating function at random point.\n",
      "Iteration No: 6 ended. Evaluation done at random point.\n",
      "Time taken: 291.6071\n",
      "Function value obtained: -0.6965\n",
      "Current minimum: -0.7678\n",
      "Iteration No: 7 started. Evaluating function at random point.\n",
      "Iteration No: 7 ended. Evaluation done at random point.\n",
      "Time taken: 406.5352\n",
      "Function value obtained: -0.7052\n",
      "Current minimum: -0.7678\n",
      "Iteration No: 8 started. Evaluating function at random point.\n",
      "Iteration No: 8 ended. Evaluation done at random point.\n",
      "Time taken: 361.6681\n",
      "Function value obtained: -0.7723\n",
      "Current minimum: -0.7723\n",
      "Iteration No: 9 started. Evaluating function at random point.\n",
      "Iteration No: 9 ended. Evaluation done at random point.\n",
      "Time taken: 412.0397\n",
      "Function value obtained: -0.7244\n",
      "Current minimum: -0.7723\n",
      "Iteration No: 10 started. Evaluating function at random point.\n",
      "Iteration No: 10 ended. Evaluation done at random point.\n",
      "Time taken: 77.6999\n",
      "Function value obtained: -0.6578\n",
      "Current minimum: -0.7723\n"
     ]
    }
   ],
   "source": [
    "result = gp_minimize(optimization_function,dimensions=param_space_xg,n_calls=10,verbose=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score = -0.7723\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score = %.4f\" % result.fun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 247, 'max_depth': 5, 'learning_rate': 0.046119524684849804}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_params = dict(zip(param_names_xg,result.x))\n",
    "xgb_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(result, open('bayessian_optimization_xgb.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XG perfomed better than Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Putting all the models in an ensemble model and run a voting classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_classifier = SGDClassifier(**sgd_params)\n",
    "rnd_classifier = RandomForestClassifier(**rf_params)\n",
    "knn_classifier = KNeighborsClassifier(**knn_params)\n",
    "log_classifier = LogisticRegression(multi_class=\"multinomial\",solver=\"lbfgs\",C= 30, n_jobs=-1)\n",
    "dtr_classifier = DecisionTreeClassifier(max_depth= 20, min_samples_split= 10)\n",
    "ada_classifier = AdaBoostClassifier(DecisionTreeClassifier(max_depth=5), n_estimators=200, algorithm=\"SAMME.R\", \n",
    "                             learning_rate=0.5)\n",
    "gbrt_classifier = GradientBoostingClassifier(max_depth=5, n_estimators=500, learning_rate=0.5)\n",
    "xgb_classifier = xgboost.XGBClassifier(**xgb_params)\n",
    "\n",
    "clf_list = [sgd_classifier,rnd_classifier,knn_classifier,log_classifier,dtr_classifier,ada_classifier,gbrt_classifier,xgb_classifier]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('SGD Classifier',\n",
       "                              SGDClassifier(alpha=0.0034706435880870164,\n",
       "                                            average=False, class_weight=None,\n",
       "                                            early_stopping=False, epsilon=0.1,\n",
       "                                            eta0=0.0, fit_intercept=True,\n",
       "                                            l1_ratio=0.15,\n",
       "                                            learning_rate='optimal', loss='log',\n",
       "                                            max_iter=1000, n_iter_no_change=5,\n",
       "                                            n_jobs=None, penalty='none',\n",
       "                                            power_t=0.5, random_state=None,\n",
       "                                            shuffle=True, tol=0.001,\n",
       "                                            validat...\n",
       "                                            max_delta_step=None, max_depth=5,\n",
       "                                            min_child_weight=None, missing=nan,\n",
       "                                            monotone_constraints=None,\n",
       "                                            n_estimators=247, n_jobs=None,\n",
       "                                            num_parallel_tree=None,\n",
       "                                            objective='binary:logistic',\n",
       "                                            random_state=None, reg_alpha=None,\n",
       "                                            reg_lambda=None,\n",
       "                                            scale_pos_weight=None,\n",
       "                                            subsample=None, tree_method=None,\n",
       "                                            validate_parameters=None,\n",
       "                                            verbosity=None))],\n",
       "                 flatten_transform=True, n_jobs=-1, voting='hard',\n",
       "                 weights=None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_cls = VotingClassifier(\n",
    "    estimators=[('SGD Classifier', clf_list[0]), \n",
    "                ('Random Forests', clf_list[1]),\n",
    "                ('k-Nearest Neighbors', clf_list[2]), \n",
    "                ('Softmax Regression', clf_list[3]),\n",
    "                ('Decission Trees', clf_list[4]), \n",
    "                ('AdaBoost', clf_list[5]),\n",
    "                ('Gradient Boost', clf_list[6]), \n",
    "                ('XGBoost', clf_list[7])], \n",
    "                voting='hard', n_jobs=-1)\n",
    "voting_cls.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7847474747474749"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results = cross_validate(voting_cls, X_train, y_train, cv = 3, scoring=\"accuracy\")\n",
    "cv_results['test_score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(voting_cls, open('voting_cls.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stacking "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59400, 100)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((33000, 100), (13200, 100), (13200, 100))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shuf_idx = np.random.permutation(len(X_train))\n",
    "\n",
    "#Splitting into train,test,validation\n",
    "Xtr = X_train[shuf_idx[:33000]]\n",
    "ytr = y_train[shuf_idx[:33000]]\n",
    "# validation set\n",
    "Xvl = X_train[shuf_idx[33000:46200]]\n",
    "yvl = y_train[shuf_idx[33000:46200]]\n",
    "# test set\n",
    "Xts = X_train[shuf_idx[46200:]]\n",
    "yts = y_train[shuf_idx[46200:]]\n",
    "\n",
    "Xtr.shape, Xvl.shape, Xts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitted 1/8\n",
      "Fitted 2/8\n",
      "Fitted 3/8\n",
      "Fitted 4/8\n",
      "Fitted 5/8\n",
      "Fitted 6/8\n",
      "Fitted 7/8\n",
      "Fitted 8/8\n"
     ]
    }
   ],
   "source": [
    "#Fitting the training set to all the classifiers in the clf_list\n",
    "for i,clf in enumerate(clf_list):\n",
    "    clf.fit(Xtr,ytr)\n",
    "    print(\"Fitted {}/{}\".format(i+1,len(clf_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted 1/8\n",
      "Predicted 2/8\n",
      "Predicted 3/8\n",
      "Predicted 4/8\n",
      "Predicted 5/8\n",
      "Predicted 6/8\n",
      "Predicted 7/8\n",
      "Predicted 8/8\n"
     ]
    }
   ],
   "source": [
    "# run individual classifiers on val set\n",
    "yhat = {}\n",
    "for i, clf in enumerate(clf_list):\n",
    "    yhat[i] = clf.predict(Xvl)\n",
    "    print(\"Predicted {}/{}\".format(i+1,len(clf_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new training set from predictions\n",
    "# combine the predictions into vectors using a horizontal stacking\n",
    "Xblend = np.c_[[preds for preds in yhat.values()]].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transform labels into codes\n",
    "le = preprocessing.LabelEncoder()\n",
    "Xblend = le.fit_transform(Xblend.reshape(13200*8)).reshape(13200,8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train a random forest classifier on Xblend using yvl for target labels\n",
    "rf_blend = RandomForestClassifier(n_estimators=100, n_jobs=-1)\n",
    "rf_blend.fit(Xblend, yvl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7805303030303031"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results = cross_validate(rf_blend, Xblend, yvl, cv = 3, scoring=\"accuracy\")\n",
    "\n",
    "pickle.dump(rf_blend, open('rf_blend.sav', 'wb'))\n",
    "\n",
    "cv_results['test_score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted 1/8\n",
      "Predicted 2/8\n",
      "Predicted 3/8\n",
      "Predicted 4/8\n",
      "Predicted 5/8\n",
      "Predicted 6/8\n",
      "Predicted 7/8\n",
      "Predicted 8/8\n"
     ]
    }
   ],
   "source": [
    "# run individual classifiers on test set\n",
    "yhatts = {}\n",
    "for i, clf in enumerate(clf_list):\n",
    "    yhatts[i] = clf.predict(Xts)\n",
    "    print(\"Predicted {}/{}\".format(i+1,len(clf_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7788636363636364"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xblendts = np.c_[[preds for preds in yhatts.values()]].T\n",
    "\n",
    "Xblendts = le.transform(Xblendts.reshape(13200*8)).reshape(13200,8)\n",
    "\n",
    "cv_results = cross_validate(rf_blend, Xblendts, yts, cv = 3, scoring=\"accuracy\")\n",
    "cv_results['test_score'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensembling Random Forest and XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "rnd_clf = RandomForestClassifier(**rf_params)\n",
    "xgb_clf = xgboost.XGBClassifier(**xgb_params)\n",
    "\n",
    "clf_rfx = [rnd_clf,xgb_clf]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('Random Forests',\n",
       "                              RandomForestClassifier(bootstrap=True,\n",
       "                                                     ccp_alpha=0.0,\n",
       "                                                     class_weight=None,\n",
       "                                                     criterion='gini',\n",
       "                                                     max_depth=52,\n",
       "                                                     max_features=0.5783538396142335,\n",
       "                                                     max_leaf_nodes=None,\n",
       "                                                     max_samples=None,\n",
       "                                                     min_impurity_decrease=0.0,\n",
       "                                                     min_impurity_split=None,\n",
       "                                                     min_samples_leaf=1,\n",
       "                                                     min_samples_split=2,\n",
       "                                                     min_weight_fraction_leaf=0.0,\n",
       "                                                     n_estimators=271,...\n",
       "                                            max_delta_step=None, max_depth=5,\n",
       "                                            min_child_weight=None, missing=nan,\n",
       "                                            monotone_constraints=None,\n",
       "                                            n_estimators=247, n_jobs=None,\n",
       "                                            num_parallel_tree=None,\n",
       "                                            objective='binary:logistic',\n",
       "                                            random_state=None, reg_alpha=None,\n",
       "                                            reg_lambda=None,\n",
       "                                            scale_pos_weight=None,\n",
       "                                            subsample=None, tree_method=None,\n",
       "                                            validate_parameters=None,\n",
       "                                            verbosity=None))],\n",
       "                 flatten_transform=True, n_jobs=-1, voting='soft',\n",
       "                 weights=None)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_clf_ens_rfxgb = VotingClassifier(\n",
    "    estimators=[('Random Forests', clf_rfx[0]),\n",
    "                ('XGBoost', clf_rfx[1])], \n",
    "                voting='soft', n_jobs=-1)\n",
    "voting_clf_ens_rfxgb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7873737373737373"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results = cross_validate(voting_clf_ens_rfxgb, X_train, y_train, cv = 3, scoring=\"accuracy\")\n",
    "cv_results['test_score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(voting_clf_ens_rfxgb, open('voting_clf_ens_rfxgb.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stacking Random Forest and XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((33000, 100), (13200, 100), (13200, 100))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.random.permutation(len(X_train))  # create shuffle index\n",
    "\n",
    "## split into three sets\n",
    "# training set\n",
    "Xtr = X_train[idx[:33000]]\n",
    "ytr = y_train[idx[:33000]]\n",
    "# validation set\n",
    "Xvl = X_train[idx[33000:46200]]\n",
    "yvl = y_train[idx[33000:46200]]\n",
    "# test set\n",
    "Xts = X_train[idx[46200:]]\n",
    "yts = y_train[idx[46200:]]\n",
    "\n",
    "Xtr.shape, Xvl.shape, Xts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitted 1/2\n",
      "Fitted 2/2\n"
     ]
    }
   ],
   "source": [
    "for i, clf in enumerate(clf_rfx):\n",
    "    clf.fit(Xtr, ytr)\n",
    "    print(\"Fitted {}/{}\".format(i+1,len(clf_rfx )))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n",
       "              importance_type='gain', interaction_constraints='',\n",
       "              learning_rate=0.046119524684849804, max_delta_step=0, max_depth=5,\n",
       "              min_child_weight=1, missing=nan, monotone_constraints='()',\n",
       "              n_estimators=247, n_jobs=0, num_parallel_tree=1,\n",
       "              objective='multi:softprob', random_state=0, reg_alpha=0,\n",
       "              reg_lambda=1, scale_pos_weight=None, subsample=1,\n",
       "              tree_method='exact', validate_parameters=1, verbosity=None)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted 1/2\n",
      "Predicted 2/2\n"
     ]
    }
   ],
   "source": [
    "# run individual classifiers on val set\n",
    "yhat = {}\n",
    "for i, clf in enumerate(clf_rfx):\n",
    "    yhat[i] = clf.predict(Xvl)\n",
    "    print(\"Predicted {}/{}\".format(i+1,len(clf_rfx)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array(['functional', 'functional', 'non functional', ..., 'functional',\n",
       "        'non functional', 'non functional'], dtype=object),\n",
       " 1: array(['functional', 'functional', 'non functional', ..., 'functional',\n",
       "        'non functional', 'non functional'], dtype=object)}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create new training set from predictions\n",
    "# combine the predictions into vectors using a horizontal stacking\n",
    "Xblend = np.c_[[preds for preds in yhat.values()]].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transform labels into codes\n",
    "le = preprocessing.LabelEncoder()\n",
    "Xblend = le.fit_transform(Xblend.reshape(13200*2)).reshape(13200,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=300,\n",
       "                       n_jobs=-1, oob_score=False, random_state=None, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train a random forest classifier on Xblend using yvl for target labels\n",
    "rf_blend = RandomForestClassifier(n_estimators=300, n_jobs=-1)\n",
    "rf_blend.fit(Xblend, yvl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7731060606060606"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results = cross_validate(rf_blend, Xblend, yvl, cv = 3, scoring=\"accuracy\")\n",
    "\n",
    "pickle.dump(rf_blend, open('rf_blend.sav', 'wb'))\n",
    "\n",
    "cv_results['test_score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted 1/2\n",
      "Predicted 2/2\n"
     ]
    }
   ],
   "source": [
    "#Lets see how it performs on unseen data\n",
    "# run individual classifiers on test set\n",
    "yhatts = {}\n",
    "for i, clf in enumerate(clf_rfx):\n",
    "    yhatts[i] = clf.predict(Xts)\n",
    "    print(\"Predicted {}/{}\".format(i+1,len(clf_rfx)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7784848484848484"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xblendts = np.c_[[preds for preds in yhatts.values()]].T\n",
    "\n",
    "Xblendts = le.transform(Xblendts.reshape(13200*2)).reshape(13200,2)\n",
    "\n",
    "cv_results = cross_validate(rf_blend, Xblendts, yts, cv = 3, scoring=\"accuracy\")\n",
    "cv_results['test_score'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Submisssion "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>status_group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50785</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>51630</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17168</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>45559</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49871</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>52449</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>24806</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>28965</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>36301</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>54122</td>\n",
       "      <td>predicted label</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id     status_group\n",
       "0  50785  predicted label\n",
       "1  51630  predicted label\n",
       "2  17168  predicted label\n",
       "3  45559  predicted label\n",
       "4  49871  predicted label\n",
       "5  52449  predicted label\n",
       "6  24806  predicted label\n",
       "7  28965  predicted label\n",
       "8  36301  predicted label\n",
       "9  54122  predicted label"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_fr.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = voting_clf_ens_rfxgb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14850,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({'id': submission_fr[\"id\"],'status_group': preds})\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorFlow-GPU-2.2",
   "language": "python",
   "name": "tf-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}