5.0 - train_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from fastai.structured import *\n",
    "from fastai.column_data import *\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from metrics import print_metrics_binary\n",
    "#from tqdm import TqdmSynchronisationWarning\n",
    "np.set_printoptions(threshold=50, edgeitems=20)\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(torch.cuda.current_device())\n",
    "torch.cuda.set_device(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#warnings.filterwarnings(\"ignore\", category=TqdmSynchronisationWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./datasets/ihm/train_height.csv')\n",
    "test = pd.read_csv('./datasets/ihm/test_height.csv')\n",
    "val = pd.read_csv('./datasets/ihm/val_heigth.csv')\n",
    "train = train.drop('Unnamed: 0', axis=1)\n",
    "test = test.drop('Unnamed: 0', axis=1)\n",
    "val = val.drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "180592\n",
      "1014121\n"
     ]
    }
   ],
   "source": [
    "print(len(train[train['Mortality'] == 1]))\n",
    "print(len(train[train['Mortality'] == 0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def balance_dataset(df):\n",
    "    split_set = []\n",
    "    num_survived = 0\n",
    "    num_died = 0\n",
    "    for episode, df_episode in df.groupby('Episode'):\n",
    "        split_set.append(df_episode)\n",
    "        if df_episode.iloc[0]['Mortality'] < 1:\n",
    "            num_survived = num_survived +1\n",
    "        else:\n",
    "            num_died = num_died +1\n",
    "    counter = 0\n",
    "    for patient in split_set:\n",
    "        if patient.iloc[0]['Mortality'] == 1 and num_died*4 > counter:\n",
    "            new_patient = patient.copy()\n",
    "            new_patient[\"Episode\"] = (\"e\" + str(counter))\n",
    "            split_set.append(patient)\n",
    "            counter = counter +1\n",
    "\n",
    "    print(num_died+counter, num_survived)\n",
    "    complete_data =  pd.DataFrame(columns=df.columns)\n",
    "    complete_data = pd.concat(split_set)\n",
    "    return complete_data\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "train_balanced = train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "180592\n",
      "1014121\n",
      "38996\n",
      "224337\n"
     ]
    }
   ],
   "source": [
    "print(len(train_balanced[train_balanced['Mortality'] == 1]))\n",
    "print(len(train_balanced[train_balanced['Mortality'] == 0]))\n",
    "\n",
    "\n",
    "print(len(val[val['Mortality'] == 1]))\n",
    "print(len(val[val['Mortality'] == 0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9935 12694\n",
      "1917081\n",
      "2180414\n"
     ]
    }
   ],
   "source": [
    "train_balanced =  balance_dataset(train)\n",
    "train_size = len(train_balanced)\n",
    "train_balanced = train_balanced.append(val)\n",
    "\n",
    "\n",
    "print(train_size)\n",
    "print(len(train_balanced))\n",
    "val_idx = list(range(train_size, len(train_balanced)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make categorical variables and embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Capillary refill rate', 3), ('Glascow coma scale eye opening', 6), ('Glascow coma scale motor response', 7), ('Glascow coma scale total', 14), ('Glascow coma scale verbal response', 6), ('Episode', 17904), ('Length', 221)]\n"
     ]
    }
   ],
   "source": [
    "cat_vars = ['Capillary refill rate', 'Glascow coma scale eye opening' ,'Glascow coma scale motor response','Glascow coma scale total', 'Glascow coma scale verbal response','Episode', 'Length' ]\n",
    "\n",
    "\n",
    "for v in cat_vars: train_balanced[v] = train_balanced[v].astype('category').cat.as_ordered()\n",
    "for v in cat_vars: test[v] = test[v].astype('category').cat.as_ordered()\n",
    "\n",
    "cat_sz = [(c, len(train_balanced[c].cat.categories)+1) for c in cat_vars]\n",
    "\n",
    "print(cat_sz)\n",
    "\n",
    "emb_szs = [(c, min(50, (c+1)//2)) for _,c in cat_sz]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "941956\n",
      "1238458\n"
     ]
    }
   ],
   "source": [
    "print(len(train_balanced[train_balanced['Mortality'] == 1]))\n",
    "print(len(train_balanced[train_balanced['Mortality'] == 0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, y, nas, mapper = proc_df(train_balanced, 'Mortality', do_scale=True)\n",
    "df_test, y_test, nas_test, mapper_test = proc_df(test, do_scale=True, mapper=mapper, na_dict=nas) \n",
    "\n",
    "md = ColumnarModelData.from_data_frame('./datasets/ihm/', val_idx, df, y.astype(np.float32), cat_flds=cat_vars, bs=128,\n",
    "                                       test_df=df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = md.get_learner(emb_szs, len(df.columns)-len(cat_vars),\n",
    "                   0.9, 1, [ 3000, 1500,750,300], [0.6,0.6,0.6,0.6], y_range=[0,1], crit = torch.nn.functional.binary_cross_entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find and set learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ff816c90ee64e44850e505615f0a50f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 6837/14978 [00:55<01:05, 124.01it/s, loss=0.211] "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEOCAYAAABmVAtTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4VPXZ//H3nYQQlhBAArInIIgoiBhQhLpLcal08WflqVUrFm3rU2tbLbbU+qBWW1tbtdqKXWy11qVWi4KiUndFCDuCIAWEsMi+BEggyf37Y4YxhCFMYM6cSfJ5XVcuzvKdc+7JIfOZs32PuTsiIiIAGWEXICIi6UOhICIiMQoFERGJUSiIiEiMQkFERGIUCiIiEqNQEBGRGIWCiIjEKBRERCRGoSAiIjFZYRdQV+3atfOCgoKwyxARqVdmzpy50d3zD9Wu3oVCQUEBxcXFYZchIlKvmNknibTT4SMREYlRKIiISIxCQUREYhQKIiISo1AQEZEYhYKIiMQ0mlAo2bKLSfPWhl2GiEhaazSh8OK8tXzniVls3rkn7FJERNJWowmF4zq2AmDg7a9SMHYSSz7dEXJFIiLpp9GEQr/OefuNXzZhWkiViIikr0YTCm1bZO83rsNIIiIHajShAPCjEX3CLkFEJK3Vuw7xjsR1Z/RgUEEbXpi7hmdnrQ67HBGRtNOo9hTMjKKCtjRvmkVpeQWrNu8KuyQRkbTSqEJhn0wzAB58fWnIlYiIpJdGGQrfPL0HAE/OWBVyJSIi6aVRhkJesyax4b2VVSFWIiKSXhplKAB0bdsMgN++tiTkSkRE0kejDYUnrjkVgNKyipArERFJH4GFgpn92czWm9mCg8w3M7vfzJaa2TwzGxhULfF0bdscgL++n9BjS0VEGoUg9xQeBUbUMv98oFf0Zwzw+wBrqdXuPZVhrVpEJK0EFgru/hawuZYmI4G/ecQ0oLWZdQyqnni+OKATADc8OTuVqxURSVthnlPoDFS/JrQkOi1lrvlc5NLUVxZ+yrRlm1K5ahGRtBRmKFicaR63odkYMys2s+INGzYkrYBjj86NDavXVBGRcEOhBOhabbwLsCZeQ3ef4O5F7l6Un5+ftAKaZGbw359fEBvfU6F7FkSkcQszFCYCV0SvQjoV2ObuKX9eZmaG0T63KQC9x73E/VM/TnUJIiJpI8hLUv8BvA8ca2YlZjbazK4zs+uiTSYDy4ClwCPAt4Oq5VB+89UBseF7X9XNbCLSeAXWdba7jzrEfAe+E9T66+LErq3DLkFEJC002juaq2vZNIsrhnSPje8s113OItI4KRSixo88ITb8l3eXh1iJiEh4FArVXDOsEIBfvaLzCiLSOCkUqvnh548FoElmvFsoREQaPoVCNTlNMsnNyWJvpVNZFfc+OhGRBk2hUMO+k8wzVtTWbZOISMOkUKjh8WtOAeCv760ItxARkRAoFGo4sUvknoWXFqwLuRIRkdRTKNTQomkWR7fKAaBg7CR2lO0NuSIRkdRRKMQxqLBtbHjkg+9y7WPFrNq8K8SKRERSQ6EQx2+r9YW0bMNOpnz4Kef95s0QKxIRSQ2FQhyZGcai8fs/SbRt8+yQqhERSR2FwkE0y85k7q3DadO8CQBrtpXp/IKINHgKhVrkNW/C7FuHx8b73fYK5RWVIVYkIhIshUICpv/knNjwseNeJtLrt4hIw6NQSED73Bx+elHf2Phf3l0RXjEiIgFSKCRo9LBCnr52CADjX1zILf+az9ptu0OuSkQkuQJ78lpDNLja/Qv/mL6Sf0xfScFRzXnjprNCrEpEJHm0p1BHPxrRZ7/xFZt2cd9rH4dUjYhIcikU6ui6M3ow/7bhLL3zfL44oBMAv3ltCe//d1PIlYmIHDmFQh2ZGbk5TcjKzOC3l50Umz7qkWkhViUikhwKhSO04u4LY8OT56/lrsmLQqxGROTIKBSSYNyFxwHw7b/P4uG3ltH31pcp26ub3ESk/lEoJMEF/TruN75rTyWzVm4JqRoRkcOnUEiCTq2bxYZzm0au8n1zyYawyhEROWy6TyFJ9p1bqKpyBt35Gg+/uYybhh9LVqZyV0TqD31iJVlGhnF2n/YAHPOTl0KuRkSkbhQKAfjlJf1jwwtWbwuxEhGRulEoBMDM+NkXIh3oXfTAOzw9YxWl5RVsLC0PuTIRkdoFGgpmNsLMFpvZUjMbG2d+NzN73cxmm9k8M7sgyHpS6YohBbHhm5+dxwk/m0LRHa+p220RSWuBhYKZZQIPAucDfYFRZta3RrNxwNPufhJwGfBQUPWkWmaG8dy3Tztg+gk/mxJCNSIiiQlyT2EwsNTdl7n7HuBJYGSNNg60ig7nAWsCrCflTurWhuV37b/zs3NPJQVjJ7F1156QqhIRObggQ6EzsKraeEl0WnW3AZebWQkwGfjfAOsJhZnx+OhTeOSKIv5Vbc9hwPhXQ6xKRCS+IEPB4kyreUB9FPCou3cBLgAeM7MDajKzMWZWbGbFGzbUv5vChvVqx3l9OzCwWxveHXt2bPoTH6wMsSoRkQMFGQolQNdq41048PDQaOBpAHd/H8gB2tVckLtPcPcidy/Kz88PqNzU6Ny6WWyP4cfPzef52avV7baIpI0gQ2EG0MvMCs0sm8iJ5Ik12qwEzgEws+OIhEL92xWoo4Hd2sSGv/fUHEY9Mo3nZpeEWJGISERgoeDuFcD1wBRgEZGrjD40s/FmdnG02Q+Ab5rZXOAfwFXeSK7ZXH7XBZzYJS82fuNTc1m/vSzEikREwOrbZ3BRUZEXFxeHXUZSFYydBMCxHXKZ9N1hzFixhbXbdnNe3w7k5jQJuToRaQjMbKa7Fx2ynUIhfO7OseNeZk9lVdz5S+44n+dml/Clk7qQnaWb0EWk7hINBX3CpAEz46lrTz3o/N7jXuJHz86n97iXWLdNh5hEJDgKhTRxUrc2/O3qwXzhxE4svmMEK+6+kIXjP39Au1PvmqpzDyISGB0+SnObSstZsWkXC1Zv42cTPwTgqtMK+MHw3vS77RUAbh95PGcf14HO1R72IyJSnc4pNFBXPzqD/3y0Pu68H5zXm/89p1eKKxKR+kDnFBqoK4Z0P+i8X7+6RM9vEJEjosdx1jNn9M7n9N75LF63nee+PZROrZuxctMu/vLecv7y7goueuCd/dq/f8vZ5LdsSoYZGRnxeh4REfmMDh81IH1vfZldeyoPOn/5XRdgpmAQaYx0+KgRmnPrcH55SX+uP+sY5t82nC5t9j/xXHjLZD5at51z732TBau3seTTHRSMnUTB2ElUVtWvLwciEgztKTRgpeUVvDh3DU2bZHDjU3MTes0z1w1hUEHbgCsTkVTT1Ueyn/Xbyxj886kHTL+of0denLf2gOk92rVgWK92HJ2Xw9cGdycjA3W5IVKPKRTkABWVVTw/Zw0X9e9ITpPM2PSVm3Yx5rFi1m0vY+uuvQd9fbMmmUy75RzymiscROobhYIcFnfnvqkf89vXPq61nRlcM6yQR95ezh8uH8iIEzqmqEIRORwKBUkad+cf01fx/OzVTF+xOW6bDq2a0rtDLjec04tnZ63me+f2okOrnBRXKiIHo1CQQOwsr2BPRRVf+N07lGzZzU2fP5Z7piw+5Ov+cPlAzu7TAQDHGfO3mby5JPI8pT9dWcSPn5tP7w65/PHKIppmZbKzvIIJby3jvqkf84uv9OOrg7oF+r5EGjqFgqRM2d5KRv91Bu8uDfaxoo+PPoWubZtRWeX0yG8Z6LpEGhqFgoRuZ3kF23bv5co/T+fj9aUHzL/qtAIefW8FAP275FGyZTebd+7Zr83XT+3OY9M+ibv8V248nZ75LVm0djs/fGYu3z2nFxf0i39uo2xvJfNKtjG4UJfbSuOkUJB6aeWmXRR/spke+S05sUte7A7sZ2eW8INnDn2vxQ3n9OLsPu35zhOz6NW+JYMK21JR6Ux4axml5RUAtGvZlI2l5Zx7XAe+cGJH3lyygTu+eALNs7N4c8kGdpVXUF5RxR2TFrGxtJzcnCy+dWZP2jbP5uTubWjbIpvlG3dycvc2VDlkZhhVVa5uRCStKRSkwdlbWUVWhmFm/Omd5dz+4sLYvO+efQz3/2fpES0/NyeLHWUVR7SMZ781hJO7a29E0o9CQRq8sr2VVLnTPDvSr+OuPRX0vXUKACOOP5qXP1xH/y55LN+4k7xmTXj1xjO4/olZdGrdjG8MLeDsX78JwCmFbflg+YFXVY0eVsg3hhbw3KzV/PrVJQnXNf3H57B7byWL1+2gZ/uWdGiVw8Q5a+jXOY+lG3bw6sJPWb21jLmrtu73ur9fcwpDj2m33/v7dHsZ3Y9qUeffjUhNCgWROnjig5WUV1Ry5ZCChA4DLV1fyj9nlnDjeb1omhW5EfDB15cmdCVWXWVlGBVVzqjB3fjm5wp5qngVr3z4KdmZGfz60hN5d+lGHvjPUgYXtmXchcdRWl5B/y6tk16H1G8KBZEQTF++mUsffj+hts99+zT2Vjqbd5Yzef46Js5dk/R6vjG0gK+d0p0OrZqqm5JGTqEgkgbcnVWbd9PtqOYJtf90exm/eOkjzurTnnOOa0/z7CzmrNrKv+esZtHa7UxbFjnMdfeX+/GLlz9iS7RbkuvPOobfvV77OZWrTivg5hHH0qxJprpQb4QUCiKNWGl5BbNXbuHrf5oed/64C4/jms/1SHFVEiaFgojErN66m6F3/2e/aW/88EwK2ukkdmOhUBCRuB6f9gnjnl8AQL/OeUy8fqgOJzUCevKaiMR1+and+d3/nATA/NXbKLxlMp9s2hlyVZIuFAoijdBF/TvxwY/PiY2fcc8b4RUjaUWhINJIdWiVw8vf+1xsfPSjM0KsRtKFQkGkEetzdCtm//Q8AKZ+tJ5NpeUhVyRhCzQUzGyEmS02s6VmNvYgbS41s4Vm9qGZPRFkPSJyoDYtsnl89CkAnHzHa5TtrQy5IglTYKFgZpnAg8D5QF9glJn1rdGmF3ALMNTdjwe+F1Q9InJww3p91udSn5++HGIlErYg9xQGA0vdfZm77wGeBEbWaPNN4EF33wLg7usDrEdEarFo/IjYcFVV/bpUXZInoVAwsxvMrJVF/MnMZpnZ8EO8rDOwqtp4SXRadb2B3mb2rplNM7MRiEgommVn8p2zegLwwrzk98Mk9UOiewpXu/t2YDiQD3wDuPsQr4l3N0zNrx9ZQC/gTGAU8EczO6B7RzMbY2bFZla8YcOGBEsWkbr69pnHAPDOxxtDrkTCkmgo7PuAvwD4i7vPJf6HfnUlQNdq412Aml8/SoB/u/ted18OLCYSEvtx9wnuXuTuRfn5+QmWLCJ11aJpFu1aZvPMzBIqKqvCLkdCkGgozDSzV4iEwhQzywUO9T9mBtDLzArNLBu4DJhYo83zwFkAZtaOyOGkZYkWLyLJd2L0WQxH+iQ7qZ8SDYXRwFhgkLvvApoQOYR0UO5eAVwPTAEWAU+7+4dmNt7MLo42mwJsMrOFwOvATe6+6TDeh4gkyQPRLjDun/oxe7W30OgkGgpDgMXuvtXMLgfGAdsO9SJ3n+zuvd29p7vfGZ12q7tPjA67u3/f3fu6ez93f/Jw34iIJEfz7Cwu6t8RgNc/0gWBjU2iofB7YJeZnQjcDHwC/C2wqkQkVD/7wvEAjHlsZsiVSKolGgoVHuljeyRwn7vfB+QGV5aIhCk/tyn5uU0B2F62N+RqJJUSDYUdZnYL8HVgUvRuZT3wVaQB+/3XBgLw1hJdBt6YJBoKXwXKidyvsI7ITWj3BFaViIRuQNfWNM3KYPbKrWGXIimUUChEg+DvQJ6ZXQSUubvOKYg0YFmZGRzXsRXzVx/ymhJpQBLt5uJSYDrw/4BLgQ/M7JIgCxOR8PXrnMfCNdvVF1Ijkujho58QuUfhSne/gkhndz8NriwRSQf9OudRWl7Bcj2us9FINBQyavRguqkOrxWReuqEznkALNAhpEYj0Q/2l81sipldZWZXAZOAycGVJSLpoFeHlmRnZTC/RKHQWGQl0sjdbzKzrwBDiXSEN8Hdnwu0MhEJXROdbG50EgoFAHd/Fng2wFpEJA3169yK52evobLKycw4VOfIUt/VevjIzHaY2fY4PzvMbHuqihSR8Azs1obS8gpmfrIl7FIkBWrdU3B3dWUh0sgN7NYGgH/NKmFwYduQq5Gg6QoiEalV96OaA1CsPYVGQaEgIrUyMy7s15HdeyrDLkVSQKEgIofUr0seq7fuZsvOPWGXIgFTKIjIIfXvErmJbfqKzSFXIkFTKIjIIQ0qiJxgvlYP3WnwFAoickhNMjMo6h65CmljaXnI1UiQFAoikpCbPn8sgLq8aOAUCiKSkD4dWwHwo2fnhVyJBEmhICIJyWsWeQLv+h3l7KmoCrkaCYpCQUQS9uD/RJ7b/OEaHUJqqBQKIpKwQQWRk82vL94QciUSFIWCiCSsfascAB55a1nIlUhQFAoiUic98luwe2+lntvcQCkURKRO+kavQhr54LshVyJBUCiISJ1868yeAHoaWwOlUBCROjm+Ux6n9TwKgMXrdoRcjSRboKFgZiPMbLGZLTWzsbW0u8TM3MyKgqxHRJLj+rOPAWDFpp0hVyLJFlgomFkm8CBwPtAXGGVmfeO0ywW+C3wQVC0iklwDurYmOzNDj+hsgILcUxgMLHX3Ze6+B3gSGBmn3e3AL4GyAGsRkSRqnp3FKT3a8urCT8MuRZIsyFDoDKyqNl4SnRZjZicBXd39xQDrEJEAnHlse5Zv3Mm6bfo+15AEGQoWZ1rswmYzywB+A/zgkAsyG2NmxWZWvGGD7qQUSQcDukYevDOvZGvIlUgyBRkKJUDXauNdgDXVxnOBE4A3zGwFcCowMd7JZnef4O5F7l6Un58fYMkikqjjO+XRJNOYuVLnFRqSIENhBtDLzArNLBu4DJi4b6a7b3P3du5e4O4FwDTgYncvDrAmEUmSnCaZDOzWhreXbAy7FEmiwELB3SuA64EpwCLgaXf/0MzGm9nFQa1XRFLn+E55LFy7XVchNSCB3qfg7pPdvbe793T3O6PTbnX3iXHanqm9BJH65csDI9eOPD97dciVSLLojmYROWwndM7DDB6b9knYpUiSKBRE5Ih49JpCXYXUMCgUROSIPPS1yNPYXtONbA2CQkFEjsgF/TrSrmU2a3UTW4OgUBCRI3Zcx1Y8M7OEB6Z+HHYpcoQUCiJyxMae3weAX7+6hOUb1XNqfaZQEJEjdnynvNjwC3PX1NJS0p1CQUSS4r2xZwNw76tL2FtZFXI1crgUCiKSFJ1aN+PyU7sBMHWRrkSqrxQKIpI0t150PADXPT6Lyio/RGtJRwoFEUma7KzPPlL+8u7yECuRw6VQEJGkmnfbcADumLQo5ErkcCgURCSpWuU04by+HQDYVFoecjVSVwoFEUm60cMKAbhTewv1jkJBRJKuf5fIfQv/Upfa9Y5CQUSSrnl2FmNO70GTTGP3nsqwy5E6UCiISCCGHtOOvZXOSwvWhl2K1IFCQUQCMaTHUTTPzuRxPYCnXlEoiEggsrMy6NAqh1krt1K2V4eQ6guFgogE5oJ+RwOweN2OkCuRRCkURCQwXxnYBdDdzfWJQkFEAtMjvyUAy/SMhXpDoSAigRpzeg/mlWxjZ3lF2KVIAhQKIhKogqNaAHD8z6bw7zmrcVfvqelMoSAigfrywM6x4RuenMNf31sRXjFySAoFEQlUTpNMVtx9YWz8thcW6lkLaUyhICIpMe2Wc2LDoyZMC7ESqY1CQURS4ui8HF658XQApq/YzNL1unchHSkURCRlenfI5dzj2gNw7r1vhVyNxKNQEJGUeuSKInKbZgFQvGJzyNVITYGGgpmNMLPFZrbUzMbGmf99M1toZvPMbKqZdQ+yHhEJn5nx8BUnA3DJH94PuRqpKbBQMLNM4EHgfKAvMMrM+tZoNhsocvf+wD+BXwZVj4ikj9N6tqNdy6YAuqktzQS5pzAYWOruy9x9D/AkMLJ6A3d/3d13RUenAV0CrEdE0shDXxsIRG5qm7Nqa8jVyD5BhkJnYFW18ZLotIMZDbwUYD0ikkaKureJDX/xwXdDrESqCzIULM60uHesmNnlQBFwz0HmjzGzYjMr3rBhQxJLFJGwZGQY//7O0Nh4n5/qO2E6CDIUSoCu1ca7AGtqNjKzc4GfABe7e3m8Bbn7BHcvcvei/Pz8QIoVkdQ7sWtrXrrhcwCU7a1i5idbQq4oPS1dX8qlf3ifisqqwNcVZCjMAHqZWaGZZQOXAROrNzCzk4CHiQTC+gBrEZE0dVzHVvzfxccD8JXfvxdyNenp3HvfZPqKzdz49NzA1xVYKLh7BXA9MAVYBDzt7h+a2Xgzuzja7B6gJfCMmc0xs4kHWZyINGBXnlYQG167bXd4haS5wnYtAl9HoPcpuPtkd+/t7j3d/c7otFvdfWJ0+Fx37+DuA6I/F9e+RBFpqO67bAAA90/9OORK0svWXXtiwzee2yvw9emOZhFJC+ef0BGAf0xfxSeb9KS2fTbv/CwUzOJdv5NcCgURSQvZWZ99HJ1xzxvhFZJmtkT3FP569eCUrE+hICJp4+M7z48N97ttChWVVVRVeUquuklX23bvBSCvWZOUrE+hICJpo0lmBi9cPwyAHWUVHPOTl+jx48mccc8bjfYxntt3R7oBadk0MyXrUyiISFrp1yWPx0efst+01Vt3U3jLZD5YtimkqsLzvafmANAi2rNs0BQKIpJ2hvVqx2vfjzyQ5+ITO8Wmf3XCNLZUO/HamLTK0eEjEWnEjmmfy4q7L+T+USex7OcXxKbfPmlhiFWFo3eHltpTEBHZJyPDWHH3hfRq35I5K7dy8z/nUjB2EgVjJ4VdWqD2nWTu1rZ5ytapUBCReuPqYYUs27iTp4tLYtO+8/dZIVYUrGUbSgHYtacyZetUKIhIvXH+CUfHhn84vDcAk+avje01PD7tk7BKS7qKyiq+9FCkL6hvDC1M2XpTc5BKRCQJWjfPZuoPzqB9blNyc5owuPAoLn34s0d6jnt+AT3zWzKk51EhVpkc+w4dAeypSN19GtpTEJF6pWd+S3KjV+IMKmjDl0/qzOeP7xCbP+qRaQ3inoat1UKhc5tmKVuv9hREpN4yM+796oDY+L4Tz/dMWczNI/qEVVZS7OsY8NrTezCga+uUrVd7CiLSYCwaPwKAh974LwVjJ1F0x6shV3R49lRU8e85kWeSXVGtW/FUUCiISIPRLDuTV288PTa+sXQPBWMn8fKCtUxfvpnKKmfR2u2s317G0vU7ovPWhVjxgT5YtokL7387Nt65deoOHQFYfTv2VlRU5MXFxWGXISJp7N9zVvP+fzfx5IxVCbX/w+UDGRHtujts1e+9uPvL/bhscLekLNfMZrp70aHaaU9BRBqckQM6c/dX+lM87tyE2l/3+CwKxk5ixG/fYmPpZ4+K353C+wMAnineP8TO7tM+pesHnWgWkQasXcumzL11OKV7KmjZNIsnp69k9LBCxr+4kC+e1Jm+HVvx88mL+Nv7kfsbPlq3g6I7XmNQQRtmrNgCwF+uGsTReTkUtmtBTpNMdpTtjV39lGw3/XMeANee0YPtu/fSvlVOIOupjQ4fiUij9+7SjSxet4N7pixm997E9g6eHHMqp/ZI7v0Q+w4dLb/rgqQ/ZU2Hj0REEjT0mHZcPayQRbeP4HO92gEwelgh157e46CvuWzCNIbcNZWRv3uHtdt217r89dvLWLB6G8BB76H41ZTFAPRo1yIlj908GO0piIgkoGxvJQ+9vpQxZ/Rk2C/+w9Zdew9oU9iuBTd//li+9fdZdG7djOe/M5RFa7dzxZ+n79fu7ZvPYumGUo7Jb0nXts3ZVFrOyXe8BsB9lw1g5IDOSa8/0T0FhYKIyGHYtnsvFz3wNqs2176XcCitcrLYXhZ5utrVQwu59Qt9k1HeAXT4SEQkQHnNmvD2zWez5I7Ic6Vb5Xx23c7J3dvs13ZwYVtmjjuXV248nQv77X/p675AAPjpRccFWHFitKcgIhKQfY8PPaXGCelVm3fRoVUO901dwoOv/5eff6kf/3NKcu5HOBgdPhIRkRgdPhIRkTpTKIiISIxCQUREYhQKIiISE2gomNkIM1tsZkvNbGyc+U3N7Kno/A/MrCDIekREpHaBhYKZZQIPAucDfYFRZlbzrozRwBZ3Pwb4DfCLoOoREZFDC3JPYTCw1N2Xufse4ElgZI02I4G/Rof/CZxjYXb6ISLSyAUZCp2B6p2Dl0SnxW3j7hXANiC53Q6KiEjCgnyeQrxv/DXvlEukDWY2BhgTHS0zsw/jvC6PSKhU1w7YeIg6UyFebWEsry6vS6RtbW3qOu9g7RviNjySZSX62iPdfrXNr8v0hrj9jmR5qfwbrDm/e0JrdfdAfoAhwJRq47cAt9RoMwUYEh3OIvKfxw6x3AmJTgeKg3p/dfxdxK051cury+sSaVtbm7rOq2W7NrhteCTLSvS1R7r9DrFN9DdYD/4GD7fOIA8fzQB6mVmhmWUDlwETa7SZCFwZHb4E+I9H30ktXqjj9HSQ7NoOd3l1eV0ibWtrU9d56bz9ILn1HcmyEn3tkW6/2ubrb7B+/A3WdX1AwH0fmdkFwG+BTODP7n6nmY0n8u1hopnlAI8BJwGbgcvcfVkS11/sCfT1IelL27B+0/arf+pdh3h1YWZj3H1C2HXI4dM2rN+0/eqfBh0KIiJSN+rmQkREYhQKIiISo1AQEZGYRhsKZnammb1tZn8wszPDrkfqzsxamNlMM7so7Fqk7szsuOjf3z/N7Fth1yMR9TIUzOzPZrbezBbUmF5rr6w1OFAK5BDpgkNSJEnbD+BHwNPBVCm1ScY2dPdF7n4dcCmgy1bTRL28+sjMTifygf43dz8hOi0TWAKcR+RDfgYwisg9EnfVWMTVwEZ3rzKzDsC97v61VNXf2CVp+/Un0oVCDpFt+WJqqhdIzjZ09/VmdjEwFviduz+Rqvrl4ILs+ygw7v5WnGcvxHplBTCzJ4GR7n4XUNvhhS1A0yDqlPiSsf3M7CygBZFu2Xeb2WR3rwq0cIlJ1t+gu08EJprZJEChkAbqZSgcRLxeWU85WGMz+zLweaAAdfFsAAAFw0lEQVQ18LtgS5ME1Gn7uftPAMzsKqJ7fYFWJ4mo69/gmcCXiXwpmxxoZZKwhhQKCfW4Gpvh/i/gX8GVI3VUp+0Xa+D+aPJLkcNU17/BN4A3gipGDk+9PNF8ECVA12rjXYA1IdUidaftV/9pGzYADSkUEumVVdKXtl/9p23YANTLUDCzfwDvA8eaWYmZjfbIk9uuJ/KMhkXA0+4e72E8EjJtv/pP27DhqpeXpIqISDDq5Z6CiIgEQ6EgIiIxCgUREYlRKIiISIxCQUREYhQKIiISo1CQwJlZaQrWcXGC3W0nc51nmtlph/G6k8zsj9Hhq8wsLfreMrOCml1hx2mTb2Yvp6omST2FgtQb0a6Z43L3ie5+dwDrrK1/sDOBOocC8GPggcMqKGTuvgFYa2ZDw65FgqFQkJQys5vMbIaZzTOz/6s2/fnoU9Q+NLMx1aaXmtl4M/sAGGJmK8zs/8xslpnNN7M+0Xaxb9xm9qiZ3W9m75nZMjO7JDo9w8weiq7jRTObvG9ejRrfMLOfm9mbwA1m9gUz+8DMZpvZa2bWIdpt9HXAjWY2x8w+F/0W/Wz0/c2I98FpZrlAf3efG2dedzObGv3dTDWzbtHpPc1sWnSZ4+PteVnkKXSTzGyumS0ws69Gpw+K/h7mmtl0M8uN7hG8Hf0dzoq3t2NmmWZ2T7VtdW212c8Dev5IQ+Xu+tFPoD9AafTf4cAEIr1pZgAvAqdH57WN/tsMWAAcFR134NJqy1oB/G90+NvAH6PDVxF5UAvAo8Az0XX0JdLHP8AlRLpozgCOJvIsjUvi1PsG8FC18TZ8dvf/NcCvo8O3AT+s1u4JYFh0uBuwKM6yzwKerTZeve4XgCujw1cDz0eHXwRGRYev2/f7rLHcrwCPVBvPA7KBZcCg6LRWRHpGbg7kRKf1AoqjwwXAgujwGGBcdLgpUAwURsc7A/PD/n+ln2B+GlLX2ZL+hkd/ZkfHWxL5UHoL+K6ZfSk6vWt0+iagEni2xnL2dXk+k0h//PE875FnLCy0yNP1AIYBz0SnrzOz12up9alqw12Ap8ysI5EP2uUHec25QF+zWA/Srcws1913VGvTEdhwkNcPqfZ+HgN+WW36F6PDTwC/ivPa+cCvzOwXwIvu/raZ9QPWuvsMAHffDpG9CuB3ZjaAyO+3d5zlDQf6V9uTyiOyTZYD64FOB3kPUs8pFCSVDLjL3R/eb2LkYSvnAkPcfZeZvUHkMZsAZe5eWWM55dF/Kzn4/+HyasNW499E7Kw2/ACRR7ZOjNZ620Fek0HkPeyuZbm7+ey9HUrCHZO5+xIzOxm4ALjLzF4hcpgn3jJuBD4FTozWXBanjRHZI5sSZ14OkfchDZDOKUgqTQGuNrOWAGbW2czaE/kWuiUaCH2AUwNa/zvAV6LnFjoQOVGciDxgdXT4ymrTdwC51cZfIdJLKADRb+I1LQKOOch63iPS3TREjtm/Ex2eRuTwENXm78fMOgG73P1xInsSA4GPgE5mNijaJjd64jyPyB5EFfB1Is9QrmkK8C0zaxJ9be/oHgZE9ixqvUpJ6i+FgqSMu79C5PDH+2Y2H/gnkQ/Vl4EsM5sH3E7kQzAIzxJ5EMwC4GHgA2BbAq+7DXjGzN4GNlab/gLwpX0nmoHvAkXRE7MLiRz/34+7fwTkRU841/Rd4BvR38PXgRui078HfN/MphM5/BSv5n7AdDObA/wEuMPd9wBfBR4ws7nAq0S+5T8EXGlm04h8wO+Ms7w/AguBWdHLVB/ms72ys4BJcV4jDYC6zpZGxcxaunupmR0FTAeGuvu6FNdwI7DD3f+YYPvmwG53dzO7jMhJ55GBFll7PW8BI919S1g1SHB0TkEamxfNrDWRE8a3pzoQon4P/L86tD+ZyIlhA7YSuTIpFGaWT+T8igKhgdKegoiIxOicgoiIxCgUREQkRqEgIiIxCgUREYlRKIiISIxCQUREYv4/xVEDhbg9UKEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f669a9795f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "m.lr_find()\n",
    "m.sched.plot(100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.0001\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrs = [lr/18, lr/9, lr/6]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d66e5d8417254ea2b50189a0a44a3464",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 513/14978 [00:04<01:59, 120.58it/s, loss=1.1]   \n",
      "  4%|▎         | 539/14978 [00:04<01:59, 120.90it/s, loss=1.1] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-6:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/andreas/anaconda3/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/andreas/anaconda3/lib/python3.6/site-packages/tqdm/_monitor.py\", line 62, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"/home/andreas/anaconda3/lib/python3.6/_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                    \n",
      "    0      0.688505   0.672246  \n",
      "    1      0.665538   0.636648                                    \n",
      "    2      0.658122   0.626507                                    \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.62651])]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.fit(lrs, 2, cycle_len=1, cycle_mult=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make predictions on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_test_set(df):\n",
    "    split_set = []\n",
    "    for episode, df_episode in df.groupby('Episode'):\n",
    "        split_set.append(df_episode)\n",
    "    return split_set\n",
    "\n",
    "test_split = split_test_set(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500/3236\n",
      "1000/3236\n",
      "1500/3236\n",
      "2000/3236\n",
      "2500/3236\n",
      "3000/3236\n"
     ]
    }
   ],
   "source": [
    "\n",
    "m.Precompute = False\n",
    "y_true = []\n",
    "counter = 1\n",
    "y_predicted = []\n",
    "for patient in test_split:\n",
    "    if counter % 500 == 0:\n",
    "        print(str(counter) + '/' + str(len(test_split)))\n",
    "    counter=counter+1\n",
    "    y = patient['Mortality'].iloc[0]\n",
    "    y_true.append(y)\n",
    "    p = ColumnarDataset.from_data_frame(patient.drop('Mortality', axis=1),cat_flds=cat_vars)\n",
    "    y_predicted.append(m.predict_dl(DataLoader(p)).max())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = []\n",
    "for pred in y_predicted:\n",
    "    if pred > 0.5: \n",
    "        preds.append(1)\n",
    "    else: \n",
    "        preds.append(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Present results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "[[1379 1483]\n",
      " [  74  300]]\n",
      "accuracy = 0.51885045\n",
      "precision class 0 = 0.9490709\n",
      "precision class 1 = 0.16825575\n",
      "recall class 0 = 0.4818309\n",
      "recall calss 1 = 0.80213904\n",
      "AUC of ROC = 0.64198496246221\n",
      "AUC of PRC = 0.4966312620596605\n",
      "min(+P, Se) = 0.16825574873808188\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'acc': 0.51885045,\n",
       " 'auprc': 0.4966312620596605,\n",
       " 'auroc': 0.64198496246221,\n",
       " 'minpse': 0.16825574873808188,\n",
       " 'prec0': 0.9490709,\n",
       " 'prec1': 0.16825575,\n",
       " 'rec0': 0.4818309,\n",
       " 'rec1': 0.80213904}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print_metrics_binary(y_true, preds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae2ac5a304f34c3a835cc9d00798d8eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                     \n",
      "    0      0.073836   0.540672  \n",
      "    1      0.02395    0.60556                                      \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.60556])]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.fit(lr, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500/3236\n",
      "1000/3236\n",
      "1500/3236\n",
      "2000/3236\n",
      "2500/3236\n",
      "3000/3236\n",
      "confusion matrix:\n",
      "[[2469  393]\n",
      " [ 313   61]]\n",
      "accuracy = 0.7818294\n",
      "precision class 0 = 0.887491\n",
      "precision class 1 = 0.13436124\n",
      "recall class 0 = 0.8626834\n",
      "recall calss 1 = 0.1631016\n",
      "AUC of ROC = 0.5128925212166056\n",
      "AUC of PRC = 0.19709359440446547\n",
      "min(+P, Se) = 0.1343612334801762\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'acc': 0.7818294,\n",
       " 'auprc': 0.19709359440446547,\n",
       " 'auroc': 0.5128925212166056,\n",
       " 'minpse': 0.1343612334801762,\n",
       " 'prec0': 0.887491,\n",
       " 'prec1': 0.13436124,\n",
       " 'rec0': 0.8626834,\n",
       " 'rec1': 0.1631016}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true = []\n",
    "counter = 1\n",
    "y_predicted = []\n",
    "for patient in test_split:\n",
    "    if counter % 500 == 0:\n",
    "        print(str(counter) + '/' + str(len(test_split)))\n",
    "    counter=counter+1\n",
    "    y = patient['Mortality'].iloc[0]\n",
    "    y_true.append(y)\n",
    "    p = ColumnarDataset.from_data_frame(patient.drop('Mortality', axis=1),cat_flds=cat_vars)\n",
    "    y_predicted.append(m.predict_dl(DataLoader(p)).max())\n",
    "\n",
    "preds = []\n",
    "for pred in y_predicted:\n",
    "    if pred > 0.5: \n",
    "        preds.append(1)\n",
    "    else: \n",
    "        preds.append(0)\n",
    "        \n",
    "print_metrics_binary(y_true, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ac858ff1adb469db7b2b2df82eaa88a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                     \n",
      "    0      0.012831   0.610768  \n",
      "    1      0.010353   0.68061                                       \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.68061])]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.fit(lr, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500/3236\n",
      "1000/3236\n",
      "1500/3236\n",
      "2000/3236\n",
      "2500/3236\n",
      "3000/3236\n",
      "confusion matrix:\n",
      "[[2428  434]\n",
      " [ 312   62]]\n",
      "accuracy = 0.7694685\n",
      "precision class 0 = 0.8861314\n",
      "precision class 1 = 0.125\n",
      "recall class 0 = 0.8483578\n",
      "recall calss 1 = 0.1657754\n",
      "AUC of ROC = 0.5070665964117684\n",
      "AUC of PRC = 0.1935953643172068\n",
      "min(+P, Se) = 0.125\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'acc': 0.7694685,\n",
       " 'auprc': 0.1935953643172068,\n",
       " 'auroc': 0.5070665964117684,\n",
       " 'minpse': 0.125,\n",
       " 'prec0': 0.8861314,\n",
       " 'prec1': 0.125,\n",
       " 'rec0': 0.8483578,\n",
       " 'rec1': 0.1657754}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true = []\n",
    "counter = 1\n",
    "y_predicted = []\n",
    "for patient in test_split:\n",
    "    if counter % 500 == 0:\n",
    "        print(str(counter) + '/' + str(len(test_split)))\n",
    "    counter=counter+1\n",
    "    y = patient['Mortality'].iloc[0]\n",
    "    y_true.append(y)\n",
    "    p = ColumnarDataset.from_data_frame(patient.drop('Mortality', axis=1),cat_flds=cat_vars)\n",
    "    y_predicted.append(m.predict_dl(DataLoader(p)).max())\n",
    "\n",
    "preds = []\n",
    "for pred in y_predicted:\n",
    "    if pred > 0.5: \n",
    "        preds.append(1)\n",
    "    else: \n",
    "        preds.append(0)\n",
    "        \n",
    "print_metrics_binary(y_true, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a8eafef0b634cd0a16050b92d380a55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 2578/14978 [00:21<01:42, 120.53it/s, loss=0.012]  "
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-964d0fdc0cde>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcycle_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcycle_mult\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Documents/HospitalMortalityRate/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    285\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/HospitalMortalityRate/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/HospitalMortalityRate/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    127\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/HospitalMortalityRate/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_scale\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_scale\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxtra\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mraw_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mupdate_fp32_grads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp32_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_scale\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/autograd/variable.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, retain_variables)\u001b[0m\n\u001b[1;32m    165\u001b[0m                 \u001b[0mVariable\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    166\u001b[0m         \"\"\"\n\u001b[0;32m--> 167\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_variables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    168\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    169\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(variables, grad_variables, retain_graph, create_graph, retain_variables)\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m---> 99\u001b[0;31m         variables, grad_variables, retain_graph)\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 2578/14978 [00:40<03:12, 64.45it/s, loss=0.012] "
     ]
    }
   ],
   "source": [
    "m.fit(lrs, 2, cycle_len=1, cycle_mult=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = []\n",
    "counter = 1\n",
    "y_predicted = []\n",
    "for patient in test_split:\n",
    "    if counter % 500 == 0:\n",
    "        print(str(counter) + '/' + str(len(test_split)))\n",
    "    counter=counter+1\n",
    "    y = patient['Mortality'].iloc[0]\n",
    "    y_true.append(y)\n",
    "    p = ColumnarDataset.from_data_frame(patient.drop('Mortality', axis=1),cat_flds=cat_vars)\n",
    "    y_predicted.append(m.predict_dl(DataLoader(p)).max())\n",
    "\n",
    "preds = []\n",
    "for pred in y_predicted:\n",
    "    if pred > 0.5: \n",
    "        preds.append(1)\n",
    "    else: \n",
    "        preds.append(0)\n",
    "        \n",
    "print_metrics_binary(y_true, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}