dl-project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TO DO\n",
    "\n",
    "At the cell that starts with train_ds = RetinaDataset(pathTo250s, X_train, y_train), output image is really weird looking. Making sure the image is more human-interpretable on that cell might improve model score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35126,\n",
       " [PosixPath('/home/ubuntu/data/train/23104_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/42184_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/23287_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/26100_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/18101_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/20499_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/7544_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/20255_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/19949_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/40220_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/25871_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/30058_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/33535_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/33531_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/40622_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/10988_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/20979_left.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/11182_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/760_right.jpeg'),\n",
       "  PosixPath('/home/ubuntu/data/train/4140_right.jpeg')])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import models\n",
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import metrics\n",
    "\n",
    "#!sudo pip3 install opencv-python\n",
    "import cv2\n",
    "\n",
    "\n",
    "\n",
    "PATH = Path(\"/home/ubuntu/data\")\n",
    "X = list((PATH/\"train\").iterdir())\n",
    "len(X), X[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Show an image and label here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auxiliary Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IMAGE OPERATIONS\n",
    "\n",
    "\n",
    "\n",
    "import math\n",
    "def crop(im, r, c, target_r, target_c): return im[r:r+target_r, c:c+target_c]\n",
    "\n",
    "def center_crop_square(im):\n",
    "    \"\"\" Returns a center crop of an image\"\"\"\n",
    "    r,c,*_ = im.shape\n",
    "    min_sz = min(r,c)\n",
    "    start_r = math.ceil((r-min_sz)/2)\n",
    "    start_c = math.ceil((c-min_sz)/2)\n",
    "    return crop(im, start_r, start_c, min_sz, min_sz)\n",
    "\n",
    "def get_a_crop(path, sz=250):\n",
    "    im = cv2.imread(str(path))\n",
    "    return cv2.resize(center_crop_square(im), (sz, sz))\n",
    "\n",
    "# random crop to the original size\n",
    "def random_crop(x, r_pix=8):\n",
    "    \"\"\" Returns a random crop\"\"\"\n",
    "    r, c,*_ = x.shape\n",
    "    r, c,*_ = x.shape\n",
    "    c_pix = round(r_pix*c/r)\n",
    "    rand_r = random.uniform(0, 1)\n",
    "    rand_c = random.uniform(0, 1)\n",
    "    start_r = np.floor(2*rand_r*r_pix).astype(int)\n",
    "    start_c = np.floor(2*rand_c*c_pix).astype(int)\n",
    "    return crop(x, start_r, start_c, r-2*r_pix, c-2*c_pix)\n",
    "\n",
    "def center_crop(x, r_pix=8):\n",
    "    r, c,*_ = x.shape\n",
    "    c_pix = round(r_pix*c/r)\n",
    "    return crop(x, r_pix, c_pix, r-2*r_pix, c-2*c_pix)\n",
    "\n",
    "\n",
    "def rotate_cv(im, deg, mode=cv2.BORDER_REFLECT, interpolation=cv2.INTER_AREA):\n",
    "    \"\"\" Rotates an image by deg degrees\"\"\"\n",
    "    r,c,*_ = im.shape\n",
    "    M = cv2.getRotationMatrix2D((c/2,r/2),deg,1)\n",
    "    return cv2.warpAffine(im,M,(c,r), borderMode=mode, \n",
    "                          flags=cv2.WARP_FILL_OUTLIERS+interpolation)\n",
    "\n",
    "def normalize(im):  #bunu zaten /255 işleminden geçmiş image'lara yapıyor.\n",
    "    \"\"\"Normalizes images with Imagenet stats.\"\"\"\n",
    "    imagenet_stats = np.array([[0.485, 0.456, 0.406], [0.229, 0.224, 0.225]])\n",
    "    return (im - imagenet_stats[0])/imagenet_stats[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resizing Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Create path for resized images given size such as dataset-450 or valid-250.'''\n",
    "\n",
    "def createResizedImagesFolder(pathToImages, pathToFolder, size):  #Takes str of PosixPath, returns PosixPath\n",
    "    pathToImages = Path(pathToImages)\n",
    "    Path(pathToFolder).mkdir(parents=True, exist_ok=True)\n",
    "    pathToFolder = Path(pathToFolder)\n",
    "    \n",
    "    if len(list(pathToFolder.iterdir())) == 0:  # Means we havent created any image in this folder yet\n",
    "        for p in pathToImages.iterdir():\n",
    "            img = get_a_crop(p, size)\n",
    "            fileName = str(p).split('/')[-1]\n",
    "            cv2.imwrite(str(pathToFolder/fileName), img)\n",
    "    else:  #Means we have been here before.\n",
    "        print(str(pathToFolder) + ' already exists. No images resized.')\n",
    "        \n",
    "    return pathToFolder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/data/images-250 already exists. No images resized.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/ubuntu/data/images-250/23104_left.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/42184_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/23287_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/26100_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/18101_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/20499_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/7544_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/20255_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/19949_right.jpeg'),\n",
       " PosixPath('/home/ubuntu/data/images-250/40220_left.jpeg')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pathTo250s = createResizedImagesFolder(PATH/'train', PATH/'images-250', 250)\n",
    "list(pathTo250s.iterdir())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing X and y, train/valid split (Find a better name for this heading)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['10_left.jpeg',\n",
       "  '10_right.jpeg',\n",
       "  '13_left.jpeg',\n",
       "  '13_right.jpeg',\n",
       "  '15_left.jpeg',\n",
       "  '15_right.jpeg',\n",
       "  '16_left.jpeg',\n",
       "  '16_right.jpeg',\n",
       "  '17_left.jpeg',\n",
       "  '17_right.jpeg'],\n",
       " 35126)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#PATH/\"trainLabels.csv\"\n",
    "#X must consist of full names of images only.\n",
    "#path before filename will be added on RetinaDataset.\n",
    "df = pd.read_csv(PATH/\"trainLabels.csv\")\n",
    "X = list(df['image'])\n",
    "X = [filename + '.jpeg' for filename in X]\n",
    "X[:10], len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 0, 0, 0, 1, 2, 4, 4, 0, 1], 35126)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = list(df['level'])\n",
    "y[:10], len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28100, 7026, 28100, 7026)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "#Beware, the line below returns pandas.Series objects. Convert them to lists first.\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=.2, random_state=0)\n",
    "X_train = list(X_train)\n",
    "X_valid = list(X_valid)\n",
    "y_train = list(y_train)\n",
    "y_valid = list(y_valid)\n",
    "\n",
    "#Checking out this line. It should look okay\n",
    "len(X_train), len(X_valid), len(y_train), len(y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Print an image and label here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Dataset / DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RetinaDataset(Dataset):\n",
    "    def __init__(self, path, X, y=None, transforms=False):\n",
    "        self.X = X\n",
    "        self.y = y  #Burada self.y=None yaptığından emin olman lazım çünkü if self.y case'leri mevcut.\n",
    "        self.transforms = transforms\n",
    "        self.path = path\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        path = str(self.path) + \"/\" + str(self.X[idx])\n",
    "        x = cv2.imread(str(path)).astype(np.float32)\n",
    "        x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)/255\n",
    "        if self.transforms:  #Brings a random augmentation.\n",
    "            rdeg = (np.random.random()-.50)*20\n",
    "            x = rotate_cv(x, rdeg)\n",
    "            x = random_crop(x)\n",
    "            if np.random.random() > 0.5: x = np.fliplr(x).copy()\n",
    "        else:  #Here, she center crops. However, don't do that for the fishes. You'll lose your fish.\n",
    "            x = center_crop(x)\n",
    "        x = normalize(x)\n",
    "        \n",
    "        if self.y:\n",
    "            return np.rollaxis(x, 2), self.y[idx]\n",
    "        else:\n",
    "            return np.rollaxis(x, 2)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 234, 234)\n",
      "1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fcbc0e890b8>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD8CAYAAAB+fLH0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFklJREFUeJzt3X+Uk9Wdx/H3VxMh6IwQNCwTYFBgO8Vhy7CAiIVW2SKiPdgfdtHTlWq72G091bPbdrE9q3bP2Z7arrbraWtLW1bUFmurVo61Wou2dKtUUJAfOsigBmaGTtCgEyVqInf/uM/UuQjOCDOTRD6vc3KS3DyZfJ/M5DP3PvcmMeccIiJdjip3ASJSWRQKIhJQKIhIQKEgIgGFgogEFAoiEui3UDCzeWa21cxazGxJfz2OiPQt6491CmZ2NPA08CGgFVgLXOCce7LPH0xE+lR/9RSmAy3OuWecc68DtwEL+umxRKQPxfrp56aBnd2utwKnHmxjM9OySpH+97xz7sSeNuqvULADtAUvfDNbDCzup8cXkbfK9Gaj/gqFVmB0t+ujgPbuGzjnlgJLQT0FkUrSX8cU1gITzOwkMzsGWAis7KfHEpE+1C89BedcycwuA+4HjgaWOee29MdjiUjf6pcpyXdchIYPIgPhMefc1J420opGEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkEDucO5vZc0AeeAMoOeemmlkS+DkwFngO+IRzbs/hlSkiA6UvegpnOOcmO+emRteXAKuccxOAVdF1EakS/TF8WAAsjy4vB87rh8cQkX5yuKHggN+a2WNmtjhqG+Gc2wUQnacOdEczW2xm68xs3WHWICJ96LCOKQCnO+fazSwFPGBmzb29o3NuKbAUwMzcYdYhIn3ksHoKzrn26DwL3AVMBzrMbCRAdJ493CJFZOAcciiY2bFmVtN1GZgLbAZWAouizRYBdx9ukSIycA5n+DACuMvMun7Oz5xz95nZWuB2M/s0sAM4//DLFJGBYs6VfzivYwoiA+KxbksHDkorGkUkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFeVcbUe4CqpBCQd7VOspdQBXqMRTMbJmZZc1sc7e2pJk9YGbbovNhUbuZ2Q1m1mJmG81sSn8WLyJ9rzc9hZuAefu1LQFWOecmAKui6wBnAxOi02Lgxr4pU+SdGwKku123chVSZXoMBefcaiC3X/MCYHl0eTlwXrf2m523BhhqZiP7qliRt1OLD4GG6HosOk3CH1sYU6a6qk3sEO83wjm3C8A5t8vMUlF7GtjZbbvWqG3XoZco0rPxwGnHAQU4Yxzkn4afR7d95mTYU4DMLtgK3F++MqvCoYbCwRyoh+YOuKHZYvwQQ+SQ1QKzBkHjeJg1aQilzhKTU/XUz0sTu+H37AamJaFx5ulsXt/MI398gUei++Y5yB/nEe5QQ6HDzEZGvYSRQDZqbwVGd9tuFNB+oB/gnFsKLAUwM/1u5JB8bAz84znH0zhpHOn0WD/QbekEEnxq6hB+sG4vD66DManN1CUSANQAP3zfUWzI7mPDLvhlGeuvRIc6JbkSWBRdXgTc3a39omgWYgbwUtcwQ6SvJYF/XTyRZKJAOpWARNGfSlnIZhgys4kaIAHUlmIMKxTZDjQC8z8ym89cdCpzZ8Dssu5F5emxp2BmK4APAieYWStwNfAN4HYz+zSwAzg/2vxeYD7QAuwFLu6HmkVoAE47BRrPPQ3GL4RNzZAcAZn1UGiDdD20tDML2Ayw6QUK0X3HAflChkR6GKc0Decr6QJn3r+XO16GTeXaoQpizpW/567hg7wT44GvfGE4F196LtSlgBLki9C8HQoxKAL3rAJG0HLTs7wKNM6YwI/WbGM7cAqw8N+Pp5CoIRaL82qmk1ghyT/cuo21ZdyvAfCYc25qTxtpRaNUlXpg7ly4+PJLYeIFUAAKRUinIFGAZBySSZg+E9Y+SxY/XGDNNtqB9wD/NPV44rkitR15dt//LIPb8lx66zY+Btw39RgWlm/3KkJfzz6I9BsD5p4B3/nmZ+Hks4AOGJkECvDKHhg3DvJxyHZCIU9uC5Si+zYDJwKTAWY2QtsjUBOHNvjBM68zC/jcF06FSQ2kZ2fYcP3vaS7LXpafegpSNaYOhe9860Li77scP82wB3+EIAmZNqAEiRjU1EJ9isHArOi+W/GzDk2fmgLFPZDZx+abXuLXz8Dc42D23wKTkjAYGj/QyCd77GS/e6mnIFXh4lnwna99lCF/fwV+PiGNn39oAVIwsRH2vQrEIFGCAgw5Edjt77/gRGDxqZCugWSCh298knGD4HNXTvQ/rtjJ+j/8hqZLPkzb+tXMP3cC7ZltfH93WXa3rNRTkIpm+Jf/GXP+htqmcUAdkMEfTSzieww18Hw7dHZCoQAnpGDGbLj+o9C1yL7hKBibhumTIJZg5okwYvmF/lhEMgbJOI0NQyCXJT1pHPXJGs6YM6Qs+1xuCgWpaA4YdzKkkwkYejl+nVwCf7Qgh+8t5OGERhg6HY5titpK8Mnz3nwjRGMjNE2DXAHyJRgG5PfA4Bg0jIdSkfj4BnzQQDJVy5kzm4gP8P5WAoWCVLQhwHvqYeq0afDXlQad+E9KSAMJ2PlL/Ks/FZ2GAacBk+HOm+kEiks3wl33QKLO/5jvfgka6mHeHIjVQLEE6bE+a2IlivkszU81KxREKk0MqK8/itoZc6KWBP4dD2PxPYU0jF6In1+oA+L490UWgCQMHUftNacTTwEr/gSzroIv3+lnHqZNg0IWxqfh/I9DLgeFVyEeIz6tkZkXnsdVR+ABR4WCVLQYMGt6I3ABftlSgTePJyTxvYbx+F5DHL+SIYEPhri/rbMAV34CvvYl/0Nfxs9SDKqBmmGQGAaFEtTUwfhxkEpBwg9RJjeNOuI+h0GhIBVt3IkwuakJGBy1xPAv/Gh9AgmgDZiG7znU44cQROc5IA519TDnLDjj7+A4oD4Kj6FjoSMLbVlIxKEmBW17IJuDQoFUKnXEvZNSoSAVrbEBalNJ/Jttoxc4BfwMRDw6Jbvdtr8UNDXCuef5F/pDG/3d10YLmnd2QKkG0g3+9kwG4gnfu+gsUEOc2v7fzYqidQpS0bIt8OAtN3PmJQUYncIHQA0+FJLAOfjeQhZ/gHE//3cdvHcS5PKQa8cB9gaQGgc785BLQE0SmtdCIgmFnF82XQLyJfZsz5AfoH2tFAoFqWjbd8HgRAJGfxyYif9olEfxoVAHO78Po+vglQIcOxOeWOpf0HX1fuoxkYL6BmjJwD2rsOOg7WVIb2qGFVdDOg3FAi7bSjYO8To/EbGhGXZk4Lc7jrwPYlEoSEWLA/liCX8wMYHv+6fxn7r4HIxe4tuObYE7boFsFiY1QnsOqIXnclDM+v/++QLUDSf/9AuwejVccSnUFCGXw2rjjOjI+MVMLS00ppqZVhrL2mufLNeul41CQSpaBniVIn7IkMN3AxrwBxdr8H/CCXglC80t0DAJkikoxv1ttSlYtsxvlq6DOXMoPH07zJwJH5oNr7RAZxKa18OIJMQLMLOJdDJNZtVmaoYCL5Zn38tFBxqlonUCmcwL+GMGBXwYZPF9iBx/naI8tgS5dn+wkBEwvgmSaahPwZLPwdg6mr/3M1hxu/98wGQN7MpAIQ/Nm6E24TsisQSUShQLeQan67njCAsEUChIFchkwfcZwH/NSDM+GJLAWuBBoAClAqzfDLlO6Cj49Qbja/22cybR8OOroGGUP3C4ajXkOyGbB4p+SjJWA4kayOXIZjvZnS/89VGPJBo+SMW79x647vmH4YQsMD1q7YzOY/hZhyKkkpCNwYcvI/Mi1I8BbroKyEG2HVra4PzZLKx9hPtv2MhZj26E6aNwhTyWqodNnTCpAQoJss2vcuENj5dhb8tPoSAVL/8G3P3DZSz46k/xC5KKwHaieUN//loOpk+DZAPMPosxn/kWa3dAzZn/ScNti/yBxtoa2LQWcjniQOd2+PaaVgrAdjbyqRPhnEtqyeSyrNvUog9ZEalUbfihv5+KzOOHDrnoPAN0QCn6gJX7fgGJPPbjz1LEDy7WLlzuP4Tl8c1++XI6wRig9pwpXLVwFF+eO4QxwBd3w6hrf8/aR1v4rzWvl2VfK4FCQarCjd9rhX0JfBh0LSfKwdOb4Jl2aM5AZ97PVJbaINvCzE8OZw/4L3+5b0M0TVmCWJEdAIkE2VicduDqf57I1UN9AJ3/xOtH5LGELgoFqQqZl6HtZ7+C1x7GTxPk/Ckfh0wOfrIC7rnHr4uuKUGhHcbXcya+P3HLmn0QS/qDkAV/ALEzm2F3oZNsYS/ZQoEx44fzjUHl3MvKoFCQqpABvv7fv4NC1wKmWngxB7kirPgVXHgBzPuAn5Zc/yhk2yCX5ZRBfjXDM8CD9/6F5tWtdOb2sh1IkKCzbQ/DYkPINnewPbuHla+Vcy8rg0JBqsZtT8CD3/gfeOgW3xBPwKoH4dyzoK3Frz0oFYEE5F6i7Vet2AeGsxAfI7cBpU4oFI6iDoiVoJTfR/v2vTy0bi+X7djHw2Xbu8qhUJCqkQMuufYvNK9tgZ2b4NgUzJwG6STECrBpPWzKQLYAHEM+BsXiHupnDSGGX9XwzRchWRrMBqB9fYbtW2D7DriVNyc5j3QKBakqGeDnv9jI3oeb4ZlmaGrwvYTaWojHIZFgb8sLQIxM9CbIzR17mc+bH/d665a97AD+Y8fr3Ij/yvojdfrxQBQKUnWuWQc/vvl3dK5v9m91bkhBMQfFAnvXb2NIcjir79/LrDnHsyWzjzagNAgW4D+C5V78wcc6/Kc0aMgQUihIVfr6vXDhv/zJr4FubgdqIFsgXzqKh//wAnWNsO65lygmoRSH7a/54QfABqJvmytf+RVNKxqlKnUAv94N/3vz/dSUYpxWN55CNsbWTfuYPH047fkXyJZ8R4Ks7xEMww8hYvhvjGrGL2uQkL51WqpaLXDmIOA1PxyYPwNqksewveV18nnI7vIBkMVPTW7AB8QR+gfXq2+dVijIu8oI/Ee8dq1ITONXKY7HB8MRPsPQq1DQ8EHeFQz/379jv/a26LxlYMupajrQKO8K6mr2HYWCiAQUCiISUCiISEChICIBhYKIBHoMBTNbZmZZM9vcre0aM2szsw3RaX632640sxYz22pmZ/VX4SLSP3rTU7gJmHeA9m875yZHp3sBzGwisBA4JbrP983s6L4qVkT6X4+h4JxbzZvvJenJAuA259xrzrln8WtGpvdwHxGpIIdzTOEyM9sYDS+6vu43Dezstk1r1PYWZrbYzNaZ2brDqEFE+tihhsKNwDhgMrALuC5qtwNse8DFZs65pc65qb1Ziy0iA+eQQsE51+Gce8M5tw/4EW8OEVqB0d02HYV/U5qIVIlDCgUzG9nt6keArpmJlcBCMxtkZicBE/Df4CEiVaLHd0ma2Qrgg8AJZtYKXA180Mwm44cGzwGXAjjntpjZ7cCT+O/0+rxz7o3+KV1E+oM+T0HkyNGrz1PQikYRCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCfQYCmY22sweMrOnzGyLmV0etSfN7AEz2xadD4vazcxuMLMWM9toZlP6eydEpO/0pqdQAv7NOfdeYAbweTObCCwBVjnnJgCrousAZwMTotNi4MY+r1pE+k2PoeCc2+Wcezy6nAeeAtLAAmB5tNly4Lzo8gLgZuetAYaa2cg+r1xE+sU7OqZgZmOBJuDPwAjn3C7wwQGkos3SwM5ud2uN2kSkCsR6u6GZHQfcAVzhnOs0s4NueoA2d4Cftxg/vBCRCtKrnoKZxfGB8FPn3J1Rc0fXsCA6z0btrcDobncfBbTv/zOdc0udc1Odc1MPtXgR6Xu9mX0w4CfAU86567vdtBJYFF1eBNzdrf2iaBZiBvBS1zBDRCqfOfeWnn24gdn7gT8Cm4B9UfNX8McVbgfGADuA851zuShEvgvMA/YCFzvn1vXwGG9fhIj0hcd60zPvMRQGgkJBZED0KhS0olFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCSgUBCRgEJBRAIKBREJKBREJKBQEJGAQkFEAgoFEQn0+qvo+9nzwCvRebU5geqsG6q39mqtG8pbe31vNqqI75IEMLN11fi19NVaN1Rv7dVaN1RH7Ro+iEhAoSAigUoKhaXlLuAQVWvdUL21V2vdUAW1V8wxBRGpDJXUUxCRClD2UDCzeWa21cxazGxJuevpiZk9Z2abzGyDma2L2pJm9oCZbYvOh1VAncvMLGtmm7u1HbBO826IfgcbzWxK+So/aO3XmFlb9LxvMLP53W67Mqp9q5mdVZ6qwcxGm9lDZvaUmW0xs8uj9qp43v/KOVe2E3A0sB04GTgGeAKYWM6aelHzc8AJ+7V9E1gSXV4CXFsBdc4GpgCbe6oTmA/8BjBgBvDnCqz9GuCLB9h2YvR3Mwg4Kfp7OrpMdY8EpkSXa4Cno/qq4nnvOpW7pzAdaHHOPeOcex24DVhQ5poOxQJgeXR5OXBeGWsBwDm3Gsjt13ywOhcANztvDTDUzEYOTKVvdZDaD2YBcJtz7jXn3LNAC/7vasA553Y55x6PLueBp4A0VfK8dyl3KKSBnd2ut0ZtlcwBvzWzx8xscdQ2wjm3C/wfBpAqW3Vv72B1Vsvv4bKom72s2xCtIms3s7FAE/Bnqux5L3co2AHaKn065HTn3BTgbODzZja73AX1gWr4PdwIjAMmA7uA66L2iqvdzI4D7gCucM51vt2mB2gr+/Ne7lBoBUZ3uz4KaC9TLb3inGuPzrPAXfiuakdXty86z5avwrd1sDor/vfgnOtwzr3hnNsH/Ig3hwgVVbuZxfGB8FPn3J1Rc1U97+UOhbXABDM7ycyOARYCK8tc00GZ2bFmVtN1GZgLbMbXvCjabBFwd3kq7NHB6lwJXBQdDZ8BvNTV3a0U+421P4J/3sHXvtDMBpnZScAE4NGBrg/8bALwE+Ap59z13W6qrue93Ec68Udgn8YfNf5quevpodaT8Ue6nwC2dNULDAdWAdui82QF1LoC380u4v8jffpgdeK7sd+LfgebgKkVWPstUW0b8S+mkd22/2pU+1bg7DLW/X58938jsCE6za+W573rpBWNIhIo9/BBRCqMQkFEAgoFEQkoFEQkoFAQkYBCQUQCCgURCSgURCTw/4jCia4RvlCgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_ds = RetinaDataset(pathTo250s, X_train, y_train)\n",
    "valid_ds = RetinaDataset(pathTo250s, X_valid, y_valid)\n",
    "\n",
    "#Check if Dataset works properly\n",
    "x, y = train_ds[np.random.randint(len(train_ds))]\n",
    "print(x.shape), print(y)\n",
    "plt.imshow(np.rollaxis(np.rollaxis(x, 2), 2), cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28100, 7026)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ds), len(valid_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64\n",
    "train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)\n",
    "valid_dl = DataLoader(valid_ds, batch_size=bs)\n",
    "CUDA_LAUNCH_BLOCKING=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 3, 234, 234]), torch.Size([64]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = next(iter(train_dl))\n",
    "x = x.cuda().float()\n",
    "y = y.cuda().float()\n",
    "x.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "resnet = models.resnet34(pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[AdaptiveAvgPool2d(output_size=(1, 1)),\n",
       " Linear(in_features=512, out_features=1000, bias=True)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layers = list(resnet.children())\n",
    "print(len(layers))\n",
    "layers[-2:]  #Last two layers out of 10 layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This model will be right after the 8th layer of resnet.\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        resnet = models.resnet34(pretrained=True)\n",
    "        # freezing parameters\n",
    "        for param in resnet.parameters():  #resnet.parameters() is a generator object.\n",
    "            param.requires_grad = False\n",
    "        # convolutional layers of resnet34\n",
    "        layers = list(resnet.children())[:8]\n",
    "        #burada top modeldan kasti image'a yakin olan taraf.\n",
    "        self.top_model = nn.Sequential(*layers).cuda()\n",
    "        self.fc = nn.Linear(512, 5)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.top_model(x))\n",
    "        x = nn.AdaptiveAvgPool2d((1,1))(x)\n",
    "        x = x.view(x.size(0), -1) # flattening \n",
    "        x = self.fc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([64, 3, 234, 234]), torch.Size([64]), torch.Tensor, torch.Tensor)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = next(iter(train_dl))\n",
    "x = x.cuda().float()\n",
    "y = y.cuda().float()\n",
    "x.shape, y.shape, type(x), type(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## val_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "def val_metrics(model, valid_dl, explanatoryMetrics=True):\n",
    "    model.eval()\n",
    "    \n",
    "    #add more metrics here.\n",
    "    #Do the 1x overestimation 1x underestimation ,2x over etc etc...\n",
    "    total = 0\n",
    "    sum_loss = 0\n",
    "    correct = 0\n",
    "    \n",
    "    labels = []\n",
    "    preds = []\n",
    "    \n",
    "    for x, y in valid_dl:\n",
    "        x = x.cuda().float()\n",
    "        y = y.cuda()  #bunu .unsqueeze(1) yaptigin an error veriyor.\n",
    "        out = model(x)  #Here, out has a shape of bs x 5. Gotta convert that into one variable.\n",
    "        \n",
    "        _, pred = torch.max(out, 1)\n",
    "        correct += pred.eq(y).sum().item()\n",
    "        #if above doesnt work use this line\n",
    "        #correct += pred.eq(y.data).sum().item()\n",
    "        loss = F.cross_entropy(out, y)\n",
    "        \n",
    "        #labels.append(list(y.detach().cpu().numpy()))\n",
    "        #preds.append(list(pred.detach().cpu().numpy()))\n",
    "        \n",
    "        labels += list(y.detach().cpu().numpy())\n",
    "        preds += list(pred.detach().cpu().numpy())\n",
    "        \n",
    "        batch = y.shape[0]\n",
    "        sum_loss += batch*(loss.item())\n",
    "        total += batch\n",
    "        \n",
    "    \n",
    "    #Get Quadratic Kappa\n",
    "    qkappa = metrics.cohen_kappa_score(labels,\n",
    "                                   preds,\n",
    "                                   weights=\"quadratic\")\n",
    "    \n",
    "    #Get Explanatory Metrics\n",
    "    #predDifferences = preds-labels\n",
    "    \n",
    "    return sum_loss/total, correct/total, qkappa #Returns loss, accuracy, quadratic weighted kappa\n",
    "    #Also return quadratic weighted kappa here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2, 0]\n",
      "[1, 1, 1]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2.0065630698794488, 0.0677483632223171, 0.0005831640416987804)"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_metrics(model, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7026"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Rate Finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here, what filter does is that, he runs the lambda function for each element in model.paramterers()\n",
    "#If it turns out true, it gets into the parameters variable.\n",
    "#In other words, parameters has the non-frozen parameters only.\n",
    "def get_optimizer(model, lr = 0.01, wd = 0.0):  #Brings non-frozen parameters.\n",
    "    parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "    optim = torch.optim.Adam(parameters, lr=lr, weight_decay=wd)\n",
    "    return optim\n",
    "\n",
    "def update_optimizer(optimizer, lr):  #Updates the learning rate.\n",
    "    for i, param_group in enumerate(optimizer.param_groups):\n",
    "        param_group[\"lr\"] = lr\n",
    "        \n",
    "def save_model(m, p): torch.save(m.state_dict(), p)\n",
    "    \n",
    "def load_model(m, p): m.load_state_dict(torch.load(p))\n",
    "    \n",
    "#Here, this guy increases learning rate bit by bit. Outputs a graph of loss function.\n",
    "#In the graph, loss function will go down until a point where it will start to go crazy up and down.\n",
    "#In other words, after a value of learning rate, it won't be able to converge. It'll go crazy.\n",
    "#You want to pick a learning rate before that point.\n",
    "#So that you can converge smoothly.\n",
    "#You keep updating the parameters of your model. However, you change learning rate every iteration.\n",
    "#number of iterations: number of epochs * batch size\n",
    "def LR_range_finder(model, train_dl, lr_low=1e-5, lr_high=0.1, epochs=2):\n",
    "    losses = []\n",
    "    p = PATH/\"mode_tmp.pth\"\n",
    "    save_model(model, str(p))\n",
    "    iterations = epochs * len(train_dl)\n",
    "    delta = (lr_high - lr_low)/iterations\n",
    "    lrs = [lr_low + i*delta for i in range(iterations)]\n",
    "    model.train()\n",
    "    ind = 0\n",
    "    optimizer = get_optimizer(model, lr = lrs[0], wd = 0.0)\n",
    "    for i in range(epochs):\n",
    "        for x,y in train_dl:\n",
    "            update_optimizer(optimizer, lr=lrs[ind])\n",
    "            x = x.cuda().float()\n",
    "            y = y.cuda()\n",
    "            out = model(x)\n",
    "            loss = F.cross_entropy(out, y)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            losses.append(loss.item())\n",
    "            ind +=1\n",
    "            \n",
    "    load_model(model, str(p))\n",
    "    return lrs, losses \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net().cuda()  #Bu line ile sanirim modelini GPU'da initialize ediyorsun.\n",
    "lrs, losses = LR_range_finder(model, train_dl, lr_low=1e-10, lr_high=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXeYJFd57t/TFTp3T06bRtok7SprlcBKgIQAG8EF2xIGW1ywrmzDtU1wwmCufJ2uE74myAILES7ZBAkLCSQLJJBW0irsSrtarTbP7Ozk6ekcqvvcP6rOqeqeTrPTPd0z8/2eZ5+d6a7pOlXd/dZX7/nO9zHOOQiCIIjVhavVAyAIgiAaD4k7QRDEKoTEnSAIYhVC4k4QBLEKIXEnCIJYhZC4EwRBrEJI3AmCIFYhJO4EQRCrEBJ3giCIVYjaqh339PTw4eHhVu2eIAhiRfLss89Oc857a23XMnEfHh7Gnj17WrV7giCIFQlj7EQ925EtQxAEsQohcScIgliFkLgTBEGsQkjcCYIgViEk7gRBEKsQEneCIIhVCIk7QRDEKmTVifuBsSieOT7b6mEQBEG0lFUn7n/34EF87HsvtnoYBEEQLaVlK1SbxehcEpFkrtXDIAiCaCmrStw55zg9n0bGKKBQ4HC5WKuHRBAE0RJWtC1zbDqBmz/9c0zG0gCA+VQOyWwe+QJHLG20eHQEQRCtY0WL+3eeHcHe0XnsG5kHAJyKpORzs8lsq4ZFEATRcla0uD+0fwIAMB41I/exSFo+N5sgcScIYu2yYsX9yFQchyfjAIAJKe525D5XQdwPjkeRyuabP0CCIIgWsmLF/aH94wAAn65gfN4S9/nqtsx8Modf+def46u76yqHTBAEsWJZsdkyD700jgvWh8EYK7JlegJuTMcziJQR91cmYsjluYz4CYIgVisrMnI/Pp3A3tF5vOX8QQyE3HbkHklha18AuuLCbGJhrvsr41EAwMhccsFzD750GocnY80dOEEQxDKxIsX9vr1jAIBfuXAIAyGPI3JPYajDi06/VtZzPzhuinepuHPO8QfffAF3P3a0ySMnCIJYHlacuHPO8YMXTuHys7ow1OFFf9iDWNrAfCqHiWga6zo86PTpZT33QxOmuI9F0jDyBfn4VDyDdK5QlG1DEASxkllx4n7gdBRHphK4+aIhAMBg2AMA2DcaQYHDjNx9+oLInXOOg+Mx+HUF+YK5klUwMmtOxDonZAmCIFYyK07cp2IZnN3rx5vPGwQA9IdMcf/pK1MAgI3dPnT5dcyVRO7j0TRiaQPXbOsFAIzM2tbMqGXTjEVS4Jw3/RgIgiCazYoT9+u29+G/PnwdOv06AGDAEvdv7xlB0K1i16Yu03MvKR4m/PY3nNsPoNh3H50zI/Z0rkBFxwiCWBXUFHfG2D2MsUnG2EsVng8zxu5njO1ljO1njL238cOszIBly0TTBq4/pw+66kKXT0ckmUW+YEfhL582M2Wu294LxcWkFQMUR/FkzRAEsRqoJ3K/F8BNVZ7/PQAHOOcXArgOwD8yxvSlD60+fLqKkMdM179xpxmVd/p1FDjw8MsT+Myjh/Hj/eP47KNHcP66MLoDbgx1eHDSIegjc0noqnkqTtOkKkEQq4Cai5g4548xxoarbQIgyBhjAAIAZgEsa0nGgbAH6VwS123vAwB0WZbNH3zjBaRyZqmBTd0+fP43dwEANnT6FtgyF2/owFPHZnGaIneCIFYBjVih+mkA9wEYAxAE8Ouc80K5DRljtwO4HQA2btzYgF2bXL21F7uG8wi4zcPp8Jninsrl8c+/fiFm4lm85YJBaeFs6PThkYMTKBQ4OMyJ1JvOG8BzJ+cwNk+RO0EQK59GiPsbAbwA4HUANgP4CWPscc55tHRDzvndAO4GgF27djUsLeXjv7yj6PcuS9yv2daLt1+8fsH2u4Y78c09I/jY91/E712/Bbk8x8YuH/pDHpyOUOROEMTKpxHi/l4Af8vNHMLDjLFjAM4B8HQDXvuM2NofwNsuGsIHX7+17PPvvHQ9Tswk8elHD+PAaTOLZkOnD0NhL0XuBEGsChoh7icBvB7A44yxfgDbAbR0Hb9HU/CpWy6u+DxjDB9543Zoigv//PAhAMD6Ti8GOzx47uQcjkzF0eHV0B1wL9eQCYIgGkpNcWeMfR1mFkwPY2wUwF8A0ACAc34XgL8EcC9j7EUADMAfc86nmzbiBvL7b9gKt+bCD14Yw/pOHwbDXozOjeHGf34Mb71wCP/86xfhkZcn4NNVXLW5u9XDJQiCqJt6smVurfH8GIAbGzaiZeaOazfjjms3AzAzajgHGLPz4j/xg/0IuFU89IfXtHKYBEEQi2LF1nNvBu+4ZD3WdXjxs0NT+OruE4ilc7Iv66lICus6vC0eIUEQRH2suPIDzURXXbhmWy+29AWQMQp47JDtLj16cLKFIyMIglgcJO5l2NwbAAD86KXTAAC/rpC4EwSxoiBxL8PmXj8AM1rXVRfefsk6/OLINNLWatdXxmNUPZIgiLaGxL0MXX4dHT4NiWwem3sDuG5bH9K5AvaNzuOlU/N446cew88OTbV6mARBEBUhcS8DYwxn95jR+9a+ADb3mTbNydmkbK793Im5lo2PIAiiFiTuFRC++7b+AIY6PGDMbOohygO/eGq+lcMjCIKoCqVCVkBE61v7g3CrCvqDHozMpqBYl8MXT0XBOYdZDJMgCKK9oMi9Alee3Y2egBsXb+gAAGzo8mJkLim7Nk3HM5iIZlo5RIIgiIqQuFfgog0d2PPnb0BfyC4TfGouhZG5JNZ3mouZhDVTKHAUCpQ9QxBE+0DiXifrO704PZ/CWCSNG3cMwMVscf+bH72Md9z1RItHSBAEYUOee52s7/KhwAFwjq39AWzpC+ClU/PgnOOH+05jJp5FocDhcpEHTxBE66HIvU6EFQOYFs2u4S7sPjqDvaPzOD2fRjZfwHSCPHiCINoDEvc62dDps3/u8uK/XbwOyWweH//+S/LxMWquTRBEm0DiXieDYQ8UF4OLAUMdXly6qRNn9fjx4ql5uFXzNI5Riz6CINoEEvc6URUXBsMeDIa90BQXGGN456Vmf9Y3nz8IgMSdIIj2gcR9EZw3FMbOoZD8/Z2Xrsdwtw/vvnIT/LpCtgxBEG0DZcssgk/dclHR7/0hD3760esBmFYNRe4EQbQLFLkvAo+mwKMpZZ8b7PBibL5Y3BMZA69OxAAAnHPE0rmmj5EgCAIgcW8Y6zo8CyL3f/vZEbz107+AkS/g28+O4vK/egTHphMtGiFBEGsJEvcGMRT2YjqelQ09AHMFayqXx2wyiyOTcaRyedx5//4WjpIgiLVCTXFnjN3DGJtkjL1UZZvrGGMvMMb2M8Z+1tghrgyGrObZL4xEsG80AgA4NGHWfp+KZTAVMxc4PfrKFP7r4ERrBkkQxJqhnsj9XgA3VXqSMdYB4LMA3so53wngVxsztJXFYIdZYOw9//4Ufu3fnsRULINTlk0zFctgKp7BeetC6A268YMXxlo5VIIg1gA1s2U4548xxoarbPIuAN/lnJ+0tl+TnaTXd5grWF2MIZ0r4BtPn5TPich9Q5cPXX637OZEEATRLBrhuW8D0MkY+ylj7FnG2G824DVXHBu6vLjz5p34wQdeC8XF8JXdJ+Rz0/EspmIZ9Abd2NoXwJGpOJUIJgiiqTQiz10FcCmA1wPwAniSMbabc36odEPG2O0AbgeAjRs3NmDX7QNjDL951TAA4ML1YTx3MgK/bqZNnp5PYTaZRW/AjcGwB+lcAaciKWzo8lV5RYIgiDOnEZH7KIAHOecJzvk0gMcAXFhuQ8753ZzzXZzzXb29vQ3YdXvyms09AIBtA0H0hTw4OB4D5zAj936zfd+rk7FWDpEgiFVOI8T9BwCuZoypjDEfgCsAvNyA112xvGZLNwDgnIEgegI6Xh6LAgD6gm5s6Q0CAF6dIN+dIIjmUdOWYYx9HcB1AHoYY6MA/gKABgCc87s45y8zxh4EsA9AAcAXOOcV0ybXApds7MT2/iCu3daL+VQOzxyfA2BG7mGfht4gTaoSBNFc6smWubWObf4ewN83ZESrAI+m4KE/vAYA8OSRGfl4b9ANANjaF8CrJO4EQTQRWqHaZISgA0BPwPx5S18Ahyfj4JwyZgiCaA4k7k1GiHvIo8qiY1v6AohnDEzGqC0fQRDNgcS9yQhxd0bwQ2GzVMHpear/ThBEcyBxbzLCinGKe3/ILFUwESVxJwiiOZC4Nxk7cvfIx/rD5mOTJO4EQTQJEvcm0+23xD3gLnpMcTFMRMlzJwiiOVCbvSajqy789dvPx+VndcrHFBdDb8CNcYrcCYJoEiTuy8C7rlhYR6c/7CHPnSCIpkG2TIvoD7oxSbYMQRBNgsS9RfSHPJiIpTEWSeGPvrOXmmcTBNFQSNxbxEDYg0gyh28+M4Jv7RnFl588UfuPAOQLHJ+8bz9OzFCjbYIgKkPi3iL6rBTJH+4zW+79+8+PIZXNV/sTAMDoXBL3PnEcD7+8JhteEQRRJyTuLUIsZDoylcCOwRBmE1l83dGarxKziSwAkI1DEERVSNxbxEDYXtT0vl86C+etC+HB/eMAgM/99Ag+9fCCRlYAgLmkKe7RlNH8QRIEsWIhcW8R/Y4Vq5ef1YXz14Xx6kQMnHN8/emT+OyjR2SU7mQ2YUbsFLkTBFENEvcWEfKqcKsuDIQ8WN/pxda+IOaSOZyYSeLkbBLZfAHffW50wd/NJsz0ySiJO0EQVSBxbxGMMWzpC+CabT1gjGFbv9l+7/695gSrW3XhG8+MLKj5bkfuZMsQBFEZEvcW8rX3X4n/9dbzAEA2zr7PEvf/ce1mHJ6M48mjM0V/M2dZNSJyzxoFavpBEMQCSNxbSNinwaubDTz6gm6EPCpenYwj6FHxO9duxkDIg7/70UEUCrZ4zyZFtoyBrFHAVX/zCL73/KmWjJ8giPaFxL1NcFoz5w6E4NUVfPSN27F3dB73W7nwgB25x9IGpuIZzCSyODZNC5oIgiiGxL2N2GqJ+zmD5v9vv3gddgyG8LmfHpHbzMpUyBymrTZ9iUztxU8EQawtSNzbiG2W737OQAgA4HIxvOWCQRwcj8mIXfxvFDhORVIAgESGJlcJgiimprgzxu5hjE0yxl6qsd1ljLE8Y+ydjRve2uKy4S7oiquo9vtlw10AgGdPzCFf4IikcrJ1n7Bj4lkSd4Igiqkncr8XwE3VNmCMKQD+DsBDDRjTmuW8dWHsv/ON2NIXlI9dsD4MXXHhmeOzmE/lwDmwqdsHADg6ZYo7Re4EQZRSU9w5548BmK2x2QcB/AcAqma1RDSl+C3xaAouWB/GM8dn5YpVIe7HpuMASNwJgljIkj13xtg6AG8HcFcd297OGNvDGNszNTW11F2vGXYNd+HFU/M4PW967Ju6/ACA4zNJADShShDEQhoxofopAH/MOa+pMJzzuznnuzjnu3p7exuw67XBZcOdyOU5HrHK/A73mJG7iOQT5LkTBFFCI3qo7gLwDcYYAPQAeDNjzOCcf78Br03ALCzm1xV8e88IAGBTt7/oebJlCIIoZcmRO+f8LM75MOd8GMB3APwuCXtjCXo0/OquDUhYzTyGLc9dEF+EuH/xF8fwvecXFiQjCGJ1UU8q5NcBPAlgO2NslDH2PsbYHYyxO5o/PELw3tcOgzHApysIezVoCgMAMAakcwUY+UJdr/ONp0fw3eeoXAFBrHZq2jKc81vrfTHO+W1LGg1RkU3dfrzpvAEcmUyAMYagR8NsIovBkAdj82kkc3mElNo3YomsgWS2EW4cQRDtDH3LVxD/+KsXIWlNnoY8KmYTWWzs9mFsPo1ExkDIo9V8jUTGII+eINYAVH5gBeHVFXRbq1ODlpAPW5Or9Qp2IptHKkepkwSx2iFxX6GEvOZN10ZrcjVeR657Ll9A1ihQXjxBrAFI3FcoQbcZuYsFTeUi94PjUfz2l/cgbUXqyaz4n2wZgljtkLivUEJeFYwB6zu9AMqL++OHpvGTAxOyBo0Q9WQ2X9QAhCCI1QeJ+wrlog2duOrsboS8ZgRfbpXqZCwNALJsgdOOSRtkzRDEaoayZVYo77piI951xUYp4OU890mrmceYVffdacckMnn4dHr7CWK1QpH7CifgNgW6nC0zGbXEfT5tbWNfAMh3J4jVDYn7CserKXCxCuJuRfWVIneCIFYvJO4rHMYY/Lpatr6MsGVOR4R1Y2+Tyhn45H378fHvV22wRRDECoXEfRXgcytIZvL4yu4TODQRAwCkc3nE0qaYn5KRux2tJzJ5PHtiDs+PzMnHZuIZXPHXD+O5k3MgCGJlQ+K+CvC7VYzMJfHx77+Eu352BIDttw+EPJiIppEv8CLrJpk1EElli+yZg+MxTEQz2DcSWd4DIAii4ZC4rwICbhV7TpjR9p7j5v/Cb79wQxhGgWMqllkQuUcSORndA8DIbNL628yi9p/LF3BgLLqkYyAIorGQuK8C/LqKrGGW/D05m8RENC0F+qINnQBMa8aZCx9N5xArKSI2MmeLe77A8TcPvIyTViu/avzopXH88r8+Li8oBEG0HhL3VYDfSof06QoA4Jnjs5iM2pE7YC5kSmbycKvmWz5upUemcnlZC35k1vTmJ2MZHJtO4N8eO4r79tau/R5JZlHgthVEEETrIXFfBfjdpqi/9cIh+HQFe47PYTKWgepi2DlkivuYFbl3+3UA9iQrYKdFysg9mpbPH51O1Ny/uGsQPV0Jgmg9JO6rABG5X7qpExdv7MDTx2YxGcugN+hG2Ksh6FYxFkkjmckj4FHh0xWZ+w4AccuuEZH7VCyDUUvoRV2aamQscZ9LtlbcnzgyjV8cnm7pGAiiXaD156sAsUr14o2dOBVJ4V8eeRXJrIG+oFn7fbDDg7FICqmcWXLAp6s4PW/74/G0gVQ2j+l4Bh7NhZlEFsetiP3oVBycc1gN0MuSs2yduRZH7v/440NgAF67pael4yCIdoAi91XAZcNduGZbL87u8eM9V27CQMiD4zNJ9AY9AIChDi9Oz6eRzObhdyvwuxVMRB3injFkpH7h+g4AwAtWOmQ0bWAumau6f2nL1Niu2czEM8jW2UuWIFY7JO6rgBt29OPL//1yuFwM3QE3PvfuS6ErLlkOeDDsNT33jAGfrsKrKXBW/I1nDOm3X7rJzK7ZNzovm3AfnYpX3b8Q90iLbZmZRFaOhSDWOiTuq5CLNnTggd//JfzhG7YBANZ1eDCTyGImkYVfV6RHL4inDem3C3HPGAVcstH8udakqoiWWzmhmjUKiKUNaRERxFqnprgzxu5hjE0yxsoWIWGM/QZjbJ/17wnG2IWNHyaxWLb0BRH2mbXehzrMCH4qloHPrcqUSUEiY2BkNgmP5sK5gyH5+BVnd0NTGI7VEHfpubcwchf7zuWpCQlBAPVF7vcCuKnK88cAXMs5vwDAXwK4uwHjIhrIYNgrfw44xL3TEv9YxsCpSArrOrzoDboh5k6Hu33Y2OWracvIbJlE6zz36biZY0+RO0GY1BR3zvljAGarPP8E51xUmtoNYH2DxkY0iHUdtrj7dAV+q0nHOkeLvoloGgNhDzTFhS6fLv/u7N5Azcg92wapkMISInEnCJNGe+7vA/CjBr8msUT6w275s19X4bMWPXX73fBoLsQzBiZjGfRZ2TW9Vgrl+i4fNnX5cHI2Cc4r2x25NvDcxb5pQpUgTBom7oyx62GK+x9X2eZ2xtgextieqampRu2aqIFbVaRg+9yKbK/X4dMQcGuIpQ1MWYueAKAv5IHqYugPutEXciOdK5StFy8QgpoxCkhlW9MEZCZOnjtBOGmIuDPGLgDwBQA3c85nKm3HOb+bc76Lc76rt7e3Ebsm6mQobEblft3puesIuBWcnk8hYxTkoqft/QGcMxiEqrhkND9VpVKkM7d8tkXWzEyCPHeCcLJkcWeMbQTwXQDv4ZwfWvqQiGYgMmacnnvYqyHgUaWnLiL3P7rpHHz7f7ym6LFqZYCdVkilVapjkRQu/cuf4FWrmUijEbaMUeAoFCh6J4ia5QcYY18HcB2AHsbYKIC/AKABAOf8LgCfANAN4LPWEnWDc76rWQMmzgyRMeN32557h0+DX1dlLXYh5JrigmZlS4rHqkfuHD5dQTKbrzipemgihplEFkem4tjaH2zIMTkRtgwA5AoFuF1Kla0JYvVTU9w557fWeP79AN7fsBERTWGow7RXfLoibZkOn4agR5WrVYUF46Q3UIe4GwUMhDw4Op2oOKkqxDeda45t4txv1ijArZK4E2sbWqG6RtgxFILqYhgIe+wJVa8ui44BdpTupMOnQVMYpuLVxD2PvpD5t5VsGeGJp3LNmXCdceyXJlUJgqpCrhles7kHz33iBoQ8ZnaMrrhwdq9fliJwqy6EPAs/Dowx9AbcVRtx5PIcvUEPGKtcPGzaityblU0zE89AV1zI5gs0qUoQoMh9TRHymCtSt/UH8cr/vgmbuv0IWIJurkwtX9a3N+iuEbkX4NVcCHm0isXDxArSZkTuuXwB0bQh7x6WmuuezuXxs0OUqkusbEjc1yhCyAOWRdNXxpIR9AbdNVMhddWF/pAbJ2fL91y1PffGi7uwggatdM+lRu7/ue80fuuep2WrQoJYiZC4r3GckXslaoq7UYCuKNg13IVnjs2WFVfpuTfBlhGWT39IiPvSPPdo2rSWqi3cIoh2h8R9jSMmVMtlygh6gx7MJDKykXYp2XwBmsrwms3dSGTzePHUPADgyFQcu/73T3BiJiEj92bYMiL90hb3pUXuYozU+INYyZC4r3Fsca8euXNevnYM59xMPVRcuPLsbgDAk0fMRcoHxqKYjmfx7Im5pqZCRlNmpN1jpW0uVZTT1t1FpklpmwQwMpvEy6ejrR7GqobEfY1Tly0TqLxKVVgguupCT8CNcwaCUtzFxeCFkYgU3GZ47sJG6Q6Y1SxzS5xQpci9+fzVf76Mj35nb6uHsaohcV/jCCtjU7e/4jZylWqZjBlhgWiK+VG68uxuPHN8FhkjL3PPnz5mV4xuhi0TS5veeI8Q9yV67uLugiL35jEZSyOZaU2RubUCifsaZ1t/EA9/6BpceXZXxW2EZTNVJtddpB3qqvlRumy4CxmjgFcn4pixLgYHx+16MqlsHtF0Djf808/wkuXNL5VoKgfGgA6fEPdGRe4kPs0ikszRnVGTIXEnsKUvWDHHHbC9bBG5p3N5Wd9dfEGFuIsyBxPR9AKPvifgRiqXx8mZJF6djBeJ/lKIpg0E3Crc1hiWKhpS3Kk2fNOYTWZpsVmTIXEnauLRXNAUJu2P6/7+p7jti88gls7ZkbtlywxYuebj0XRRSQAAWN/pRTqXl6+TMWpHxvkCl3cAlYilDYQ8mhzDUkUjY4l7ZpnE/ZP37cfXnjq5LPtqB/IFjvlUDgaViWgqJO5ETRhjVtmCHNK5PMajafzs0BRu/fxuOUEqIveegNmDdSKawUw8IxcWAWbbvlQuL/PH6/G0/+PZUVzzfx5FokrOeTSdQ9CjSt+/UbbMcon7/XvH8PPDK3tFbC5fqLvU8nwqB85pwrrZkLgTdRH0qIilDZmZsqnbh5dORWUGjYiaNcXMmpmYN22Zy4ZNLz/sNStQprJ5xDPma9QjnocmYkhk8zg9n6q4TSydQ8ijQRO2zFKzZbLLJ+5GvoDZZLZp1TKXi5s//Qt85tHDdW271vvdziWyy2L5kbgTdRG0Ivdoyoygh63sGmG9iMgdAPpDbozNpxBJ5TDc40dv0I3ugA6PpiCdyyNu2TL1pEWOWyUATs9XLgUQTRkIeVVoijlvkF3i7X7KEtrl+ALOJc0oth6Lqp0ZmU1WLD1Riqg/tFZtmav+9hH8449fafp+qCokURelkbuYOBV1XYQlAgADIQ/2js6Dc6Dbr+PC9WFwDnh1BelcATFhy9QhnqIaZTVxj2VyCHqCtue+RFG2PffmC64oy7DSI/e0ka/7Tqe0a5bLVXkyf7Vh5AtI5wqyGmszocidqAsp7tZq0CGrs9Nsmci9L+SRtWi6Azo+dcvF+JdbL4ZXU5DNFzCfFLZM/ZH7RDVxTxtN8dyXI3KfjjWvoNpykS9w5PK86jEcnoxLTz7iKAudK6zsi9piSVi5/QESd6JdkLaMZakMWj1ZRV0Xp7gPhOxJ1C6/2RAk4FbhtXr3CeGvFelxzjEhbJloGtF0Dt99bnTBNiJbZiVOqNqR+8oVd3GRTlc4Xy+MRPCGf/oZfuuLT2N8Pl3URH2tWTMxa74pUKZ3QqMhcSfqQkTusRJbRnruSrHnLuj22z97rPZ+Il++VrbMfConBXZ8Po1vPTOCD31rL0Yc3m4ym0e+wK3IvTGee3o5I3er5s5yZeY0A2EpVbpAiYv5zw9P4yPf3lvUZ3etTaqKTDGK3Im2IejREMsY8pZ6nYjcy06o2pG7qPcCAB5rG/FlT9ewZSYsv50x03M/YBWacta4EXMAIa8Gxhg0hS1JMAoFbpcfWA7PPb7yPXdxnipdoMTzlw93Ye9IBLOOZuarPR1y70gEX/zFMfl7gsSdaDdEC76xSAq64kK3tWp1tmzkboo7Y0CnzxZ3r15iy9QQNOG3b+sLYiKaxsHT5opW56ImsSAqaI1PU1xLmlB1CtTyRO7iXKxcW8auxVP+GMTzF27oQCxjyIs0sLJtmS88frRm+ufXnz6Jv37gZbmiW3xeyZYh2gYhnqciKQQ9KnyaAsaqe+4dXg2KIxNCeO7Cc60VGYtJ1As3hDGbyOLVSUvcHStfhU0kWghqimtJkbuzsNlyiLsshdyku4Q//e4+3L93rCmvLZCeewVxF89fuL4DAIrEfSXbMj8+MIEHXjxd9Nix6YQUcsAMZHJ5joS1dqKtbBnG2D2MsUnG2EsVnmeMsf/LGDvMGNvHGLuk8cMkWk3QEs9TcymEvBpcLga/rmIuYYqrMxWyw6dBV13o8utFryHEXXz2a/nMYjL1AksURLXHaactkyqO3HXVtSTP3SlQi/HBD45H616h6WRaLujhyJ/B31cjmTXwjWdG8FiVfrDzqdyS92t77uXPl3j8gvVhAOb7L0oZrWRxN3t6z5g4AAAgAElEQVT32pk/E9E0Xv+PP8WPD0zIx4SFKOxLscajLcQdwL0Abqry/JsAbLX+3Q7gc0sfFtFuiMj4VCQlLZqAW11QOAwwyxX0h9zSuhGICVVBLfEcj6bR6dPkgimBM3IXXy5x8dFbELmPRVJ40788jkcOTi56f84LVaM9/kMTcXBuTjqX46VT87jwf/0Y5378Qfztjw6e8X5qrQsQj/cG3bIchZhoX2p55kYQSWax5/hs7Q1LyBoFGVwApkVZ4MDRqYR8TFiQwr6UkXs72DKc88cAVDvymwF8mZvsBtDBGBts1ACJ9kBExslsHiGvKaTOD6hbLf4ovf2idXjTeQNFj4nIXVDLZ56IZtAf8mAgbAqBpjBs6PJKnxqwPcyQV5XbLEncs4uP3GcTWXCOmgXOSuGcYyaRkRfGeiZVdx+dqfv4Xhk37Q9nL9h0Lo/vPT8Kzrm0uQIeFbuPzixq7E5ECmStyN2turB9IAjALiPdDpH7l544gXd9/qmKbSQrkcsXEEvn5B2b+LyIO85CgcvPqrAvxXvh19tA3OtgHYARx++j1mPEKiLoEHIRxTtX2TltGQD40I3b8d7XnlX02AJxr8OWMcXdzMzZ0hfEQMgjfWrAkS3TIM89fQaRu4hMazUiufuxI0U17JPZPNK5gsw8qpXrfmImgVvu3o2H9o/XNS5RUjmZtcX9kZcn8Yff3GvWBbKykbb3B5c0vyAu0mkjX+Q3y+eNPHTVBcaYLe6h8uJ+ciaJ/zo4UfTYu7/wFH7wwqkzHl8tJmNpZPMF6YvXS9YooMCBhHV+s460XcAUdMMSfinuaQM+XSmai2oWjRD3cqMse6/FGLudMbaHMbZnamplV8FbawjbA7Cj5KAl7oqL1fVh9VSJ3D/18CH8+r89WfS8Ke5uBNwqwl4NOwZD6Pa7F0TuuuKSdw6a4kLWWIrnbn5B/bpSt00i/qaauOcLHH/9wEF8dfcJAMBv3vM07rz/AADULe4iNXSqTLvDcrxiiXvC0fFo1lo0dSqSwkQ0A5+uoMuvLyklUUTunJe3WTK5gkyDPccS936rIXvpe/WlJ4/jA197Xv6ezBr4+eFpPHVs8bZJvYj03kTGwN6RCD7wtefqiuLFsUZLSlhPxExxd6bsirmpeMZYFr8daIy4jwLY4Ph9PYCy0/Oc87s557s457t6e3sbsGtiuSgfuZtirSv1fYyckbtPV2Tk/tTRGfzLI6/ihZGIfF7c0ooWf/fctgsfeeM2dAf0Ys89ZZb7Fc1GNNW1JKESAh32anXbMkKUq9kq4nb8yFQcsXQOjx2awjf3mDe8Qtxr7U/4tvOpXNXtBFLcHZG7+NvT8ylMxsw7I1111byQ3bd3DB/42nNFjz340jhG55JFF+lyWT8ZIw+39d7v2tSFTp+G863JVaOk/EAym0cym5fjEReyxVpei0FE1YmMeSH54b7TMg23GuL9EiU5RGqvyPJyXoSdtsxy+O1AY8T9PgC/aWXNXAlgnnN+utYfESsLj6ZIEQ/KCVVrElOt72Pk0e3tugM60rk8cvkCPvqdfVZlRLsmeDxroMCBDq+ZcXPppi4Mhr3oDrjN211LwEVdGYGusCXluQtxD3m1uq2KWis0AVsADk/G8epkvOi5dZ31Re5CIOoR96lYBjOJLFysOHIXUer4fBqTUfPiqSuumse6++gMHto/Lm0Xzjk++PXn8JXdJ4rKDpQ7hnSuAI9mvvcbunx4/hM34rx1priX2jJiHGIuRUS/0/Hixi+NRJyTeMZYsN9qiLFLcbfGPhnLoFDgxZG7U9zbJXJnjH0dwJMAtjPGRhlj72OM3cEYu8Pa5AEARwEcBvB5AL/btNESLUWIqJhQdS4cqgddcUG4Nz0BNzJGARPRNE7OJjHc7QNgR36iuFjYqxW9Rk9AB+dmqVzAquXu2KZRnvuZRO6pKp6tEI25ZE5OXv7GFRuhuhjO7vVbr1N9f1Lck7XFXUTt5wyEihqdiAvD2HxaRu5urba4p7N55PJcnpOk9Xs8bRRF7plcAf/x7CiePGJP0GaMPNxqsSUnS0WU2DIiYhfjXI7IPSIjd7vXwGSZfsGliHNWassYBY7pREaOfTDssW2ZdBuJO+f8Vs75IOdc45yv55z/O+f8Ls75XdbznHP+e5zzzZzz8znne5o/bKIVSHEvsWVKM2UqwRiT1kxPwA2jwGXUNGhNmgqBdJYVcCL6uQrfPVoSuTdK3Dt8ixD3OiZUY4586B+9OA6vpuAvbz4PT/zJ6zAkPHfDPvY77z9QNBEK2LnS9UTuL1sLhS7Z1IFULi9z2aUtE0lhMpZBnxW51zpWkU4pLhTiYpXIGEV/mzHy+KefHMKXnzwuH3NG7gJxF1hqy2RLrI5Jyx6ZaWLkPueI3EUe+lSsti1TGrk7L5CT0QwmY2n4dQXrOrztGbkThEBMqooJ1cXaMoBdgqDHqjkjfOQey1sXAilESOxL0G0tjJqJZ5HO5XFgLIqzeuw8eE1Z/CImzrm0G8TFJezVkDXymIpl8Offf7GqJ13PhKoQQwB48dQ8tvUH4HIx9IU88FhRrYiAHz04iXt+cQx7js8VvcasFf3VI+7PnZzD+k4vNnWZ50ZcKMTfHp6KI5nNm+KuFkfuuXwBn7xvv0zpA4BkrniFpYhwE9l8seeeM9MDncdbLnJXK1TwFPMlIhoWReZiGaMplTPTubx83xIOW8bpl7/7C0/h/zxYvA6gUOAyE0YEIs6L3Ph8GlMx0/bq8OlFee4ryXMn1gilkXtgkROqgJ0xIxaxiJK3vVZELr7AIhoqtWXEwqiZRAaPHZpCKpfHG3fa+fS6urg89wdfOo2L7vwJvmo1qJaeu0dDNl/Azw9P4au7T+LViXjF1xBjrpa371zJCABb+4PyZ7cV1QpxEJbKbEmD8Uqee77A8dZP/xw/sVZGcs6x58QcLt3UCZ/1HonIW/ytuGPqD3ngVhXZOEPs/94njuNRx6KsVFaIumEdjx25l3ruiWy+6HjLRe7ClsmV2DLiIjOfWmiPzCSyePbEbNWWi4vFWVs+kTVkIxmnX37gdBRHporff+ekvVjI5AwAxqNp687Igy6/VuTrU+ROtB2lnruIQDS1/pxdYcuIapHidltkxaSyxV/ucp47YE6wPbh/HGGvhivP7pbPC1vm+HQC+8fmUY3vPT+KO776HOZTOTx/0oyS07kCdNUFr5XNI/ztSqs8AVvU64ncRTnk7Q5xFxc8cZE4NLGwhg5QOVsmksxi3+g89o2a2UajcylMxTLYtalTComwU+ZTObn0H4CM3AFbsOyFN/Z+bFvGiuCFuJdE7nNJs5xBrchdds2qYctMObz26VgGt33xGfzTjw+hUTjLDzttGSHuhQJHpEyPW2cAISN3axuzQXwa01bk3unTMZvMgnPeXp47QQhExG5H7vaS/3rxaApUF5OiLbIghGinZORufslKxT3k0aC6GMbnU3j4wATecG5/0YSuqAr51w+8jA9/a2/VsXz5yRPY3h/EhRs6MBYxo8F0Lg+PaubNc24LaqLE/3YiItfqE6qmAFy8oRMAsG3AIe4lK1TF4qPSScRKkXskJSaXzTE+e8K8UF2yqRM+XYi7Hblv6vLJv+0L2eIuxElccJ3Cl5KFr4r3lcgYRcInxhxNVY/c1QotEW1bxo7cRY2iA6ejiKWNBdlGS6FI3NOGbKYxaXnusbSZtVVqCTltLGe2jIuZd6ETVuTeG3Sj068jaxQQSeZgFDjZMkT7ITx3EcHLPPfFeO6agqBHldGqEIPeMp67iy1cpu1yMXT5ddz7xHFE08aCEgeicNhENL0g8nUyMpvE8ycjeNvF6zDc7cNYxPwyp7J5eHVFHpOIHJOZap67iNwr20GxtAG36sLOoRCAypF7PGNgdC5lnZsSWyZhN/Zwio0Qe2GZ7DkxC7+u4JyBEPzWHEciayBrFJDM5uUqUcBsiSgmxDN58zVF5O60hZJS3ItFPpkxiuwIcc5jaUPOY1TLlild9CQuMOLiPhXPYMegec6ethYyHZmKl10JeyY4M48Szsg9Wlw2QJzvP/vei/iHh14pGre4EGXzBbhVBQNhD146FUU8Y6Av5Eanz/zeiAbiFLkTbce2/gDWdXjhswQj6LaX/NeLR1cQ8KhSUIQYiCyYlMMbFtUnS/nIjdvxzks34MM3bMN124sXw4nCYdPxLOZTuYoi8MN95lKMX75gEEMdXpyeT5mNOow8vJoixUhMrJVmrjip13MPejS8+8pNuOvdl2AgbDc0keJqFPCqZckA5rnhnOPH+8eRzuURTRuyJoszehcCFZeRewQXb+yE4mKyREQiY8i/OWfAFEuP5kLQrdq2jBWNivdkrkjcjaJ9iMg9bkXuIjIX5yubL8g5hPKeewVbJm/bcvkCx0w8I1e1CnGPpY0iu2YpCOtJdTHEM3l5gZyOZ5AvcIe4m+Pac3wWL4xESiJ3y3PP5eHWXOgPeXDgdBQ+XcHVW3plT4ORueUV9+XZC7EquOXyjbjl8o3y98WmQgLA+k4vCgUuxXMmnoFfV6QIyQnVdG6BJSP4tcs24Ncu21D2OU1hyBoFTOUyyBpmp3mvoxrlt/eM4LM/PYJY2sDFGzuwocuHoQ4vcnlzRWwqmzcXbJV0jarmudeTLRNNGwh5VXT6ddx0XnFdPVVxQXUxpHN56bdv7PJhJpHBcyfncPtXnsWdN+8EAAx3+zEZy2A+lZNNUYRoxzLmxezViRiuvfpsAPZ7lMjm5XZn9fihq6YIMcaKLi4AFhS7ch5baSpkMptH2sgj7NWQzmUWVOz0aIopegsid2HLlJ9QjaZzmElkUODApm4ffLqCUxF7IvXIZAJ9QQ8WSzqXx62f340/e/O5uGy4Sx7jYIcHswmz9vpAyIPxaBqziawt7o5013QuXzyh6siWcasuvOX8Qaguhj9907nY2O2Tfzsya46fInei7Qk4aqjXyyd/ZSe+8Fu7ZIbIdDwLv6N5ttOWqSTu1dAUF1K5vBSJSKrY2nj62CxG55JIZQ28y7pQrbP6wZ6KpJDK5YtW4wpxr+q51zmh6qzPU4pHU5DOFXBwPAavpuDCDR2YiWdxbNqM9n76ilmLSaR9FkXuKTtyj2cMGAUuU0bFRTPpiNzDPg2DYY+8CxDHmpXiLjx3c/tcviBtiFiJuBvW5KmcQylTaz9jFOT7LVBcDC62MBXSuaRfWCO9QY+8sxP/l2av1MupSArPn4zICfRIMguP5kK3341xa39iUdlkLC0XH9kL1QpIG/bny6crJeKu4G0Xr8Pn3n0pNloL81oVuZO4E2dM8AwmVHXVBY+myNzumUQGAY9D3LNLF3cnpZOPc8kstvYFsf/Om/Cru8zoXywiOhVJIZ2zbBnHxcc5rnLUO6EaqjKR5tFcSBtm5L6tP4CegJkbLZqBi1Wtw0LcHV6xSLOLOQXcOndiQjWeMYrSS297zTBuucy8uIljlbZMiefuvGuxI3d7/7OJrGOC3FnUzbyTEKJXiqa4ymTL2KmwU475GJFdddXmbvh0BUem4nji8HSRjVUP4pjEhWcumUOnT0fArWLcSrHc3BsAYGbMiMhdvLeprIFMriAvSt0BXb4XovplKdJzn7HEnSZUiXbHo5nlBBbjuQuEoKRzBQTdqqw744zcQ1Ui3UqUfrkiyVJxz6HTX/y6QtzHIinpD8to1voSJ+qYUHXWxiklWuN43KqCdC6PsUgaG7p86Am4Ec8YOGxFqEJgRZmGSpG7vfjLWkWs23nu4i6mw6vhva89C++4dD0AQFcUOX7AFuj5VA5GvlB00Yo7vHbBTDwjj63YlrFXr5Z67oDIbCotP2AvYhJ3TX1Bt1wXsbnXj829ATxxeAa3ffEZ/Ot/le9hOpfI4h2fewKHSzJrxCS1uDhFkuaFye9W5J2KiNynohn5+UkbBXDOTVvGsG2ZnoAbsYyBQoEja9kypXT4dPQE3HjaaghCkTvR9jDGEPJqC0r51oPzSxDwqLLuTNqRCllaeqAeRBaGYKG4Z9HhK27/F/JoCLpVjEXSSOXMbBl3yTFVm1Atqq1iidPhyTj+c59dP6+0wFkpbs0sASBWNQpbZa+jUiZgR+4Rh7gL0Y6ljQU1eVSrHHIis/A5gXNClXOOmXhWCtB8KldkN8WzxbYMYAqxR1fgVl1FPn0snZPZL+Uj9+IFZ5xzOxUylZOlB3qDbvQGzfNxdm8Am3v9eGUihmy+UPF9OTIVx7Mn5mSJZYGM3K3xR2Tkbp8TYX1NxtLyeLKGOUFspkUWZApnt98Nzs3zkqkg7oqL4fZrzpJ3RhS5EyuCf3jnhXjva4cX/XfOL7tfV2XdmVTWbPgQbZgtU+y5m1/mha871OE1PfdssecuqNbIwZnnLYTwK08exx988/mK1StL8agK5pM5xDMGeoNumds9OpfC2Y7yCpvKRO7Cbknl8jJydp47v1tFImtgPiW6VhUfv1suYjInXY0Cx+Y+05qYS2aLBFTaMhkDzkQmt2NtgD0uO02yUuTurC1jFLjsrzqfyuHVyTiGwh54NGVB5C6oNNEt3of7944hkTFw92NHMJfIynr2InKfS2bR6dfkamvAjMaDHhUTjshdbAuYF/OM9b6Ki040ZV7Iyl3EAOA3rtgkP3eULUOsCN6wo/+M/q40cgfMujNmNkIB2Xyh4Z67WG3YWRK5A8BQhwevjMcwmzCj1tIJwFTVRUy2wMhFWGkDuTzHyFwK6zu9SOXyNSZUXXLCrSdQ3H/2hh39+MLPj8GtuuDTVQTdatEiIecxisVYYZ9T3BUkMnnoSg4Bt7rgHDkXMYk5hq19AewdiWA2kSsScTsVMoe+oEfWPfdoCjyagmjakLVqYumco8Veec/dWRVSRLZdPrNm/3Mn53CuleN+wfowNnR5pbA/e3IOs4lsxUlsYSXNJLL4tX97EvvHogh5NHnxE557JJlDh08v6ioW8ph9e49NJ1BwXK3k5KpRHLmL18sYeXT6F362APMC+6EbtuFLT55Y0JGsWVDkTrQEp5UjOjp5NFPc7YqQi489RMTd4dOgKawo8oqmcyhwlBX3dZ1enJxNosA5br1848LIvYbnLnL/ha0kbItj03G7z2s1W0ZVpDA7bRkA2NIXwJbegBx3yKsVCbrzGEW6YFHkrqtIZAxEUtmyF0xn+QExmbqt3xTR2URWRsfdft0uHJY20F+Sqy/e096AG4qLIZrO1Yjci20ZIe4iI2ZkNoVzBs0c9xt3DuDxP3odPJqCnUNh3Pvey9Ef8lScxBai72LA/jGzQuZYJOWwZczJ3kjKvJNzinvAo2JrXwCHJ+NFC7lKLRpzrLp8vUq2jOA9Vw3j4Q9dKxvLNBsSd6Il6OUid82cVKxUV6YeRJ2bnoAbYa9W5E2LCbPSCVUA2NBp2h1/efN5OHcwtECMqnnuqWxeCm/pMv2jUwlpAdSK3EW6YW/Azg4BzAYX/+2SdbjBukvq8GkLJlTFuTo1lzIXLzly+/1uFcls3pzULXNOnXnuduRuimokaYt7b9AtxT2WNtAftO8uPJoiXyfoURH0qIiljZqRu9OWkYIZtI9dRO7l8OlKxUqR4vE7rt2M37pqEwZCHpyKpKVYx9IGoikD+QKX2TICv1vB5r4AxqNpnIqk5Ge1tA4NYBeyi6ZMcdcr2DKtgGwZoiUoLmZFblxGTV7d9NyXJO5WxG029eBFKYPiy1k6oQoAt1y2Edv6g7j+nD4AdgYJAHT59eqLmIwC1ns1mUoJ2JH70emE/Lmq5+64k+kNmn1jhb2xoctXVBwtXBq5p3LY0hvAfCqHU5EUwl6tKDr06QpiaQMZgyFc5m7IOaGazpmR+xbLc59NZqVF1Rs0a6YY+QJSuTwGK0TuAbeKkEezBK9y5K5WsGV6HZaUWE1bDq91pyc4PBnD2z/zBP7zf14tLyrvv/psdPl1vDQWxVgkJc9bNJXDtOW/i94C4ly4VUUefyxtYFO3DydmkkUrdsUFW9xhxTNGxWyZVtE+IyHWHCKaK7VlKmV11IMt7mYdbacIii9nOVsm7NOksAPFdxYDIU9FcRcpcOJuQIiNiOyOTsWlP15rERNgTiZ2+XUwxtDt16EpDAOh4pWYTnFPWwu21lut+kbnUgvOm19XkcwaiCTLT1K7HamQ07EMGDMnmD2aC3OJhZG7ODanLWN67uY5C9QZueultkze3o/5mq6iWv3lzpnTltk7Mo9YxsDxmYR8H4S/PRj24PS8bcvEs4ZccNUd0OWEqrDOtvbZk7bi/It6+oB98e4K2OJu1tBpH0ltn5EQaw6nGAAiEivYnvuZ5Lk7xN20ZZyeqfm6XWXEvRTnl3Sow1NxhaqwEsTdgBAzMfF4dCoh0+6qzSGIc9Hp0+UFqjugY6jDC6Wkvk6PVXWQcy5FXvRhjWcWppD63SomYxkcnU5Iu6XoWB2LmKYTWXT5dCguhi6fjtlETgpoX9CDXJ4X1QMSQysbudf03CvYMgG7LHLpsTvx6kpRppKYs0hm83LM4n1c1+HF2Lxpy+iKmdVzwlpU1OW3J1SFPbOxyyc/S2IdRHGapyH/VvxeLVumFZC4Ey1DfBFEjrFXU5Beoi0jIu6egI4Or1Y02Sh6ZXaU8dwrvQ5gNrSoFLkLG0akuaUctoymMEzGMnLlY61FTECxJXH99j7ctHNgwbabun2IpQ3MJrLy+NZ32mV8F0TubgURq876NduKC60BxeUHZuNZ6fd3+vUiz11MHo7Pp63jUaUoOj33gLu+yF1VGHIGxw/3jeFvf3RwwYRqNUsGMD8v2XxBppuOzQtxN7s2uVWXLDw3GPYga5hZWKIswLGZhNyfFHcr0FAVF4Z7fPJvgWJxF3djAbe5RiNmLdgqzbJqJe0zEmLNIcRAFLcSqZDz0sZY/JSQ05YJl0w8ziWzUF1M2kD1jM3FTJsgaxTKdngSaZAdXityz+aRsVYwisnAfaPzNY9HiIJzMvHDN27Hn7753AXbihWUx2cS8viELQOUE3c7Kr14Y8eC13O5GFQXQ8bII5axV9J2+c0mE8lsHprC5N2JiJADbk2WZHarLrnwK+BWEfLW9tzNlogFPPjSOL69Z8T23INu7NrUiRt3Vk+zLa1HdMoq25zM5uViNIGIvgF7le/xaVPcnROqQcdiJuG7C3F3Zs6IGju64kLQoyKaziGbLyyqFEezaZ+REGsOER2LL5Tw3KdiGXT7ddnQYTEI62Oow4uwV0MsbcjIbi6ZQ4dPqysVTbVWzIa8mvzil4veRWTa4YjchSVz/rowAODHBybgYtUXr3jKRO6VGO42xf3YdFLejQyGPbLDUscCz9187ddu6a5YKkJM3sbTdo/PTqv3ZyprwKsp0pcWkXvQo8oLs7NeUP2eu2nLxNIGElm7VIFXV/Cd33kNXn9udXH36MXibtsyhlmXX6sk7uL8JRD2atBV14LIHQC2WBbWgNW83XkXGEubHa0UF0PAo2LWyjKiyJ0gADvSc6ZCZvOYiGbkpNpiOWcghO/ccRWu3tojRU543nOJhaUHqo5PVRDyaLL4Vrl0SNuW0eXvYsLx/HVh9ATcGOrw4B9+9cKqFyuZI17HcW/o8kFxMRybjsvIvcNrR5+lkbsYfzlLxj5WM4qOOXp89ofcGJ9PI5HNw6er0j4TC5cCDlvGjNxtWybk0RDLGPKclc+WMW0ZsdhJiHS9k5I+0eQka5ZNKPLcc5XFfVOPfecjLKiALiJ3W9yv396L89eF5eRqqeeuKy4wxhBwq7IXcDt57nXd9zLGbgLwLwAUAF/gnP9tyfMbAXwJQIe1zZ9wzh9o8FiJVYZngS1jluudjKVlrfIzYddwFwB7kjOSzKLLr2Muma1rMlWgqy6EvHZ0Wj5yt2wZR+TunGx7+s9eD8ZQ827Brdp2Ui00xYUNnV4cn07Ki0rYZ9bHcZbfFazv9MKtunDd9r5yLyePNZMzI3dhH23s8iFjFHByNgmfrsjz4IzcxeKtosjdIZBi8rViVch8AbG0mYYo7kLqLSHtdUTu86mcfH+S2bxVAM7eZ6dPMytv5go4y4rc07kCeqwVpuLYnJH7xRs7cf8Hf0leQItTIQ1pwYhSBeZxtk+8XHMkjDEFwGcAvAnADgC3MsZ2lGz25wC+xTm/GMAtAD7b6IESqw8RuQcdE6pGwYzARCPppSCW4IsvZ8SyZeoen+pC2OuI3K1Vqn/+/Rfxe197DoBty4jWfE5xD3hUuFysLhtoMZE7YBYQOzadkA2vg25VplqWivsNO/rx9J+9Aesc0WspuhW5xx2R+3qr1+qhiRi8uiJFX6yCDbpty8rMlrGzn0TGjqjsWE70dKvkrzhfItWwXt/a6bk7G3mYfV2LPXfGGIYse0XU5wHsbBdVcWHHYKjsoilxXM76QtF0Tl6EAm5NVtJcUeIO4HIAhznnRznnWQDfAHBzyTYcgDgrYQBjjRsisVpxqy4oLia/PELgpuPZM+qyU4oQObFKda5CXZmK49NcCHm0oj6kAPDiqSj+c99pHJ2KywlVj6ZIW0nYMotJ5RTnoJ7IHTArFx6fSeCFkQiGwl64LO8XWFgYjDFWVGumHG7VzBlPZvPSfhGrdiPJnBW5m69/cDyGLX0BeDSXvPB5HK0JA25V3iGNziXNip9lUhqFLSPO19wiI3fxeUll87IHrvi91JYBTGvGrbqKWhw6VwI/8PtX41ZHpzGBqFjqxMyGsiN3cYEqrSbaSuo5i+sAjDh+H7Uec/JJAO9mjI0CeADAB8u9EGPsdsbYHsbYnqmpqTMYLrGacKsuBNyqjGydkVYjIvehsBeMAV998gRy+QLmErm60iAFH75hO9772rPs239RWsDKw//q7pOy3K9HtcQ9V5ClBxZT/U+I+sYuX40tTc7q8SOZzePxV6fxris2Fu3vjFJIFZfMBhEXCWcGjldXi17/7vdcCsZYkecuLlBBjyqbcO8bna84yQ8PLW0AAA8ISURBVKgpLmQM+2Io9r9YW8asg29G7j0BHQlrQrXU598xFMKWvgA0xSWFv7uOiyljTF5IxF1FvsDthACHlbPSsmXK3VOWdiS4FcC9nPP1AN4M4CuMsQWvzTm/m3O+i3O+q7e38uQOsTYYDHuKBcQR9fQtwXMXDIQ9uPPm8/DIwUm843NPIJsvLMpzf9vF63D5WV12k2nRJNoSo28/OyIXRnk0l0zldNoy9XL11h48/KFrZb32WoiMD5+u4N1XbCra35muDxDL8Z0rhkUrPp+mIOBWcce1m3HPbZfhbKs6o9/puVvvn9+tYn2nF0Grpk2lSUZNcZVdQVzvpKTTlhmzasCs6/RZnnt+QZ+Bj75xO75zx2vMY7TOVU+gvs+DeC2nrSd6Bzgv4u2ULVPPp28UgLMb8XostF3eB+AmAOCcP8kY8wDoATDZiEESq5MP37gdH3TkjjvFfSkTqk7ec6UpfP9v9wls6vbhkk2di34NMWkoPPd42sCOwRAOnI7ix/vHAdiLeIrEfRGRO2NM5lXXg6i3/uuXbZCWi1g6X8uCKYdbXRi5A2ZmzmQsA5+ugDGGP3nTOUV/54zcuwPmytZuvxuMMZw7FMLTx2bLZsoApjg6G1cJW6Ze31p8XpJZ03Nf1+FFwK2UzZYx9+eCeCjk1TAZy8iSvfXuq8uvY9KaR9AdqZ+CdvLc6/n0PQNgK2PsLACnYE6Yvqtkm5MAXg/gXsbYuQA8AMh3IarijPYAO28ZgIwYG8F7rtwkRf5MEL5yImu2U0tk87h6aw8OjkfxvNUpyaMp1nJ402bQFNbUL/q6Di++eNtluPysLvnYkmwZ1SXzuJ0XpQ2dXjx7Yq7IMnPS5TcF3edWcdPOATz0B9fISeGdlrhXOg+lOffiLqhea8PZmnEsksJg2AOvpmI2kVqwiKkUEbl3Vai/XoqIyJ3zGboVuTtrBq2oVEjOucEY+wCAh2CmOd7DOd/PGLsTwB7O+X0APgzg84yxP4Rp2dzGOS/fTJIgKuCMtM40z70ZyMg9m5fWTE/AjeEeP45OmascPZpLliyOpXMIeupbLLUUnIXOAOC8dWHsGAwVlfutl3LNUwAzcgfsc1DKOy9dj51DIXlBcN597LAyTyq1YSwV99mEuYK43ORrOcRFN501O1BdtMFcfSvKD1RriiEmu+u2ZSzR9lntBM3yvmIR3sqN3GHlrD9Q8tgnHD8fAPDaxg6NWGuIL2NPQK+4krIViKyeZNauiBjwqDh3MGSLuzWhGk3nzFWey9RKzcnNF63DzReV5jrUhzPidIqVEHevXv54/G5VrisoZcdQyHrtyraMk/lUruJFpBxinUQql8ds3MyEyhh5KxWyUPGiAtiRez0TqoA9eeu17jYzRqEoW0aOqY089/YZCbHmEV+g3gakQTYSxhh8mtmqLu7w00VkqlsFqkQJWmeu+EqhXPMUwE6HXIzoCrb2BaEprO7IHVhc5KsqLuiKC9FUDrGMgW6/Dp+uSnunmi0T8mpwsYWlGiohRNur2wXS7Dx3Z7bMCrJlCGK5EJF7I9IgG43PbQp3zBG5S9vBkRI3l8yh21GfZaXg9LmdYiUqI55R+WXVJUswlKOcuNebBinwaC5ZDbLTryObLyBvzdJWs2Vu2NFfVDWy5n7U4sjdOf6iCdU2itxX1ieQWNWIL01/m0XugNWHNGvIyD3oVmWZXTHuCzd04BvPjODl01FcXsGqaFecoup3WDCDYS++9N8vx64zyDICgC/81mUVa7I7bZmQR5XNtReDV1dkNcguvy4zlYDqFsn12/twfZVyDKWI99jraEoixhoqmlAlcSeIBYjb6L42jdydXYgCHhX9IbdVs8Qc9xVW5krMUZ9lpeCsxV4azV5bpeBYLaplo4jIV3Ux9ATcprgvcq7Fq9mNxbv8uiwDAFSeyD0TxGuZE6rFC5qK8tzbKFumfS4zxJrHryv4yI3b8LaLz2xSsJl0+d2YS2RtcbdW1u4cCstc77N6/DLLZ8XZMmX842bjnJAU53CxDaY9miLr13T59SIrppots1hkiQzdEbmXsWUWe+fRTFbWJ5BY1TDG8IHXbW31MMrS49dxZDLusGXMW/G/+JUd0odnjOHKs7tx/94xWZ9lpSDFfRkvSpqcq9DkhO1ibQ3nRK+zXR5QfUJ1sRTbMpbnrpp3OJpill4w8rxqW8Dlpn0uMwTRxvQE3ZiOZ6SnK0rEbu0P4pKNth8trJmVZ8ssLNfbbDSXWARk1605E89d0OHVin5vZOTudYi7uABpRZPQWlv57QBF7gRRF91+HRmjgPFoGh7NVbHxxlWbuwFgUdUn24FyRbCaTTlbZrECKUQ37NWgKq6iyeDGeu6OVEhRRMwx1qBHRaHN1m2SuBNEHYh0vhMziaqWy+beAL76vivK9iptZ1riuTtsGXEntNgJVSHg3dbErdOmaZYtI/PclWJxF41b2gUSd4KoA1H3+/h0omZ0+0tbe5ZjSA3F3Qpxd9gyIuJetC1jiW5nGXFvRraMV7c999K1AVFHhct2oL1MIoJoU0TkfjqaXnGrT+vB3coJVfcSbBndrtYI2PVmgEZny9ipkGJBk+YYa1/QjXCbWXGr71NKEE1ApDhyvrzR7XIh+4G2JBVSO/MJVVGK1xJWn7s5E6qiGJtPV+UqVGfk/rG37Gg7W4Yid4KoA+dinJWWw14Pbm35I3fVYcsIUT7TbBlpyzgEvZHZK9dt78Nfvf08nDMQLBu59wbdsshau0DiThB1oCku2YVnOaPb5UIUvFrO/HxdLRO5L7LwlrdkQlVVXNCtln/11o2paz+6gt+4YpPVcs8ct7uNKpeWo71HRxBthBCQ1Ri5t2IRU3/IbLO4cyh05hOqJZE7YFoojbRkSpF57mr7LFgqB4k7QdSJmFRdjZ57f8gNFzM7Ly0XYa+Gn//x63Dhhg5pyyzWSilNhQRMX7yZ4m5ny7RPHZlyrL5PKUE0CSnuqzBy39Ttx3MfvwEdLcr4WOqEameRuCvIN3FBkZifKG020m6svk8pQTQJ0ZJtNUbuAFom7ADOOBXymq29+MD1W7DT6voEmOKeyzdP3MWEajsVCSvH6vyUEkQT6F7FtkyrETXRF7uqNOzT8JE3bi96rDvgRtYoNGxspZRbxNSO0KeUIOpkNXvuraY36Man33Uxrt565rXjBX/19vPQzDIvW/oC2NDlxVm9/ubtpAHQp5Qg6qR7ldsyreaXLxhqyOsMhps7Kbyhy4fH/+h1Td1HI6jrvoIxdhNj7BXG2GHG2J9U2ObXGGMHGGP7GWNfa+wwCaL1vHZLD3776rNwyRm2nCOI5aRmCMIYUwB8BsANAEYBPMMYu49zfsCxzVYAfwrgtZzzOcZY/c0JCWKFEHCr+NhbdrR6GARRF/VE7pcDOMw5P8o5zwL4BoCbS7b5bQCf4ZzPAQDnfLKxwyQIgiAWQz3ivg7AiOP3UesxJ9sAbGOM/YIxtpsxdlOjBkgQBEEsnnpmhspl6pfORasAtgK4DsB6AI8zxs7jnEeKXoix2wHcDgAbN25c9GAJgiCI+qgnch8FsMHx+3oAY2W2+QHnPMc5PwbgFZhiXwTn/G7O+S7O+a7e3qWnPBEEQRDlqUfcnwGwlTF2FmNMB3ALgPtKtvk+gOsBgDHWA9OmOdrIgRIEQRD1U1PcOecGgA8AeAjAywC+xTnfzxi7kzH2VmuzhwDMMMYOAHgUwEc55zPNGjRBEARRHcZb1LF7165dfM+ePS3ZN0EQxEqFMfYs53xXre3auzgCQRAEcUa0LHJnjE0BOHGGf94DYLqBw1np0PmwoXNhQ+eimNVyPjZxzmtmpLRM3JcCY2xPPbclawU6HzZ0LmzoXBSz1s4H2TIEQRCrEBJ3giCIVchKFfe7Wz2ANoPOhw2dCxs6F8WsqfOxIj13giAIojorNXInCIIgqrCs4l6r6QdjzM0Y+6b1/FOMsWHHc39qPf4KY+yNtV7TKpfwFGPsVes19Vr7WE7a5FzcxhibYoy9YP17f3OPujzLfC4+YD3GrVIZ4nHGGPu/1nP7GGOXNO+IK9Mm5+I6xti843PxieYdcXWW+Xz8P+vxlxhj9zDGNOvxtvhsLBrO+bL8A6AAOALgbAA6gL0AdpRs87sA7rJ+vgXAN62fd1jbuwGcZb2OUu01AXwLwC3Wz3cB+J1q+1jOf210Lm4D8OnlPv4Wn4uLAQwDOA6gx7GPNwP4EcwqqFcCeGoNn4vrAPywlZ+LFp2PN1vvPwPwdcf3pOWfjTP5t5yRez1NP24G8CXr5+8AeD1jjFmPf4NznuFm1cnD1uuVfU3rb15nvQas13xbjX0sJ+1yLtqBZTsXAMA5f55zfrzMOG4G8GVushtAB2NssKFHWpt2ORftwnKfjwes958DeBpmBVyxj1Z/NhbNcop7PU0/5DbcLFg2D6C7yt9WerwbQMR6jdJ9VdrHctIu5wIA3mHdan6HMeYs7bxcLOe5WOo4mk27nAsAuIoxtpcx9iPG2M7FHEQDacn5sOyY9wB4cBHjaDuWU9zrafpRaZtGPV7vOJpNu5yL+wEMc84vAPAw7AhoOVnOc7HUcTSbdjkXz8Fc4n4hgH+FWdK7FbTqfHwWwGOc88cXMY62YznFvd6mHxsAgDGmAggDmK3yt5Uen4Z566SWPF5tH8tJW5wLzvkM5zxjPf55AJcu6ajOjOU8F0sdR7Npi3PBOY9yzuPWzw8A0JwTrsvIsp8PxthfAOgF8KFFjqP9WC5zH2YrvqMwJzfERMbOkm1+D8WTI9+yft6J4smRozAnRiq+JoBvo3gS8Xer7WM5/7XRuRh07O/tAHav9nPheM3jKJ5EfAuKJ82eXsPnYgD2GpjLAZwUv6/m8wHg/QCeAOAt2UfLPxtndP6W+c16M4BDMGerP2Y9dieAt1o/e2AK0WGYExpnO/72Y9bfvQLgTdVe03r8bOs1Dluv6a61jzV4Lv4GwH7rA/4ogHPWwLn4nzAjMQNm9PUF6/H/374d2kAIBUEAHXo6RU3UcF1cT0gS2kFgvjiLggzvJWt+vppsxu2U5Df+70k+L85i+duLNcl8RxY35HGMt23M90m7cXVcqAIUcqEKUEi5AxRS7gCFlDtAIeUOUEi5AxRS7gCFlDtAoRP+eEtOHO64ygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(lrs[:200], losses[:200])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, epochs=5, learning_rate=0.01):\n",
    "    optimizer = get_optimizer(model, lr = learning_rate, wd =0)\n",
    "    for i in range(epochs):\n",
    "        model.train()  #Do this here because in val_metrics, model.eval() runs.\n",
    "        total = 0\n",
    "        sum_loss = 0\n",
    "        for x, y in train_dl:\n",
    "            x = x.cuda().float()\n",
    "            y = y.cuda()\n",
    "            out = model(x)\n",
    "            \n",
    "            loss = F.cross_entropy(out, y)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            batch = y.shape[0]\n",
    "            total += batch\n",
    "            sum_loss += batch*(loss.item())\n",
    "            \n",
    "        val_loss, val_acc, val_kappa = val_metrics(model, valid_dl)  #Her bir epoch'da validation set ile test ediyorsun.\n",
    "        print(\"train loss %.3f val loss %.3f val accuracy %.3f val kappa %.3f\" % (sum_loss/total, val_loss, val_acc, val_kappa))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2, 0]\n",
      "[0, 0, 3]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.935 val loss 1.042 val accuracy 0.623 val kappa 0.329\n",
      "[0, 2, 0]\n",
      "[0, 0, 2]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.872 val loss 0.913 val accuracy 0.681 val kappa 0.381\n",
      "[0, 2, 0]\n",
      "[0, 0, 2]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.865 val loss 1.040 val accuracy 0.605 val kappa 0.295\n",
      "[0, 2, 0]\n",
      "[0, 0, 2]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.876 val loss 0.963 val accuracy 0.622 val kappa 0.329\n",
      "[0, 2, 0]\n",
      "[0, 0, 2]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.907 val loss 0.911 val accuracy 0.709 val kappa 0.366\n"
     ]
    }
   ],
   "source": [
    "model = Net().cuda()\n",
    "train(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2, 0]\n",
      "[0, 0, 2]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9108756109584338, 0.7090805579276971, 0.36617075551337064)"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_metrics(model, valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "#modelin parametrelerini True ya da False yapiyorsun. Tüm parametreleri ama.\n",
    "#O yüzden bunu modelin bir kısmını seçip run ediyorsun.\n",
    "#Mesela Sequential modelin birkaç layerını seçip kullanabilirsin.\n",
    "#a layer is a model too, I think\n",
    "def set_trainable_attr(m, b=True):  #b=True yani it unfreezes by default.\n",
    "    for p in m.parameters(): p.requires_grad = b\n",
    "        \n",
    "def unfreeze(model, l):\n",
    "    top_model = model.top_model\n",
    "    set_trainable_attr(top_model[l])  #Mesela top_model içinde layerler olan bir list. İstediğin layerı unfreeze et.\n",
    "    \n",
    "#Zaten 8 layer var. 7 ve 6. indexdekiler en sonuncuları.\n",
    "unfreeze(model, 7)\n",
    "unfreeze(model, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2, 0]\n",
      "[0, 0, 0]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.750 val loss 0.727 val accuracy 0.772 val kappa 0.498\n",
      "[0, 2, 0]\n",
      "[0, 0, 0]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.587 val loss 0.692 val accuracy 0.765 val kappa 0.523\n",
      "[0, 2, 0]\n",
      "[0, 0, 1]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.433 val loss 0.773 val accuracy 0.752 val kappa 0.546\n",
      "[0, 2, 0]\n",
      "[0, 0, 1]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.243 val loss 0.984 val accuracy 0.733 val kappa 0.577\n",
      "[0, 2, 0]\n",
      "[0, 0, 1]\n",
      "50\n",
      "7026\n",
      "7026\n",
      "<class 'list'>\n",
      "<class 'list'>\n",
      "7026\n",
      "train loss 0.150 val loss 1.054 val accuracy 0.725 val kappa 0.557\n"
     ]
    }
   ],
   "source": [
    "# lower learning rate\n",
    "# 5 posta daha train ediyorum. Bu sefer layerlarin birazini unfreeze ederek.\n",
    "# val loss iyice azaliyor gibi.\n",
    "train(model, epochs=5, learning_rate=0.0001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Very Bottom Part of this doc is for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Net().cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = next(iter(train_dl))\n",
    "x = x.cuda().float()\n",
    "y = y.cuda().float()\n",
    "x.shape, y.shape, type(x), type(y)\n",
    "out = model(x)\n",
    "_, pred = torch.max(out, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 3, 1, 1,\n",
       "        1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "        1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 3, 1, 1, 1], device='cuda:0')"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-8.2738e-01,  4.3935e-01, -2.0654e-01, -4.9658e-01, -1.0522e+00],\n",
       "        [-7.5172e-01,  3.5475e-01, -3.7029e-01, -4.8058e-01, -9.9944e-01],\n",
       "        [-7.9705e-01,  3.6043e-01,  1.8179e-01, -4.9183e-01, -1.0892e+00],\n",
       "        [-1.0630e+00,  1.2893e+00, -6.7064e-01, -5.8770e-01, -8.9687e-01],\n",
       "        [-7.2993e-01,  1.2086e+00, -3.8347e-01, -5.4394e-01, -7.4223e-01],\n",
       "        [-1.1183e+00,  8.1864e-01, -5.8624e-01, -1.1010e+00, -1.5238e+00],\n",
       "        [-7.5229e-01, -1.3334e-03, -5.8160e-01, -3.6901e-01, -4.0382e-01],\n",
       "        [-5.9850e-01,  5.9732e-01, -3.2639e-01,  8.1230e-02, -1.0396e+00],\n",
       "        [-1.5461e+00,  4.9414e-01, -7.9799e-01, -2.6335e-01, -1.2811e+00],\n",
       "        [-1.1934e+00,  1.8424e-02, -1.2202e-03,  3.2946e-03, -1.0564e+00],\n",
       "        [-1.5891e+00,  2.3360e-01, -6.2440e-01, -9.7288e-02, -1.0854e+00],\n",
       "        [-5.5202e-01,  4.8498e-01, -5.0822e-01, -2.5746e-01, -9.5653e-01],\n",
       "        [-1.2583e+00,  4.1151e-01, -5.7392e-01,  3.3230e-01, -1.8313e+00],\n",
       "        [-8.4040e-01,  8.5225e-01, -1.4356e-01, -2.2183e-01, -1.0906e+00],\n",
       "        [-6.7821e-01,  8.7040e-02, -3.5938e-01, -1.3500e-01, -1.1800e+00],\n",
       "        [-1.0634e+00,  2.5160e-01, -5.6179e-01,  3.6358e-01, -1.9185e+00],\n",
       "        [-8.4403e-01,  3.5880e-01,  3.2787e-01, -4.8968e-01,  4.4618e-02],\n",
       "        [-8.8410e-01,  3.5345e-01, -2.8962e-01, -4.7662e-01, -1.0028e+00],\n",
       "        [-9.8591e-01,  1.3375e+00, -3.9628e-01,  3.3284e-02, -4.8958e-01],\n",
       "        [-1.2160e+00,  9.3370e-01, -5.8303e-02, -6.3288e-01, -9.3268e-01],\n",
       "        [-7.4207e-01,  9.0175e-01, -4.7104e-01, -6.9038e-01, -5.9087e-01],\n",
       "        [-3.8082e-01, -9.1117e-02, -2.7263e-01, -6.6979e-02, -1.4754e+00],\n",
       "        [-6.0674e-01,  6.1013e-01,  2.2982e-01, -5.9489e-01, -1.2924e+00],\n",
       "        [-9.5783e-01,  6.3934e-01, -3.1961e-01,  2.0024e-01, -1.2473e+00],\n",
       "        [-5.5297e-01,  1.1540e+00, -4.7352e-01, -3.9150e-01, -8.7231e-01],\n",
       "        [-1.0209e+00,  1.0677e+00, -8.0425e-01, -5.5663e-01, -7.9475e-01],\n",
       "        [-1.1707e+00,  5.5267e-01, -7.1643e-01, -3.9887e-01, -9.9507e-01],\n",
       "        [-9.3106e-01,  2.1695e+00, -3.9223e-01, -8.9240e-01, -5.9300e-01],\n",
       "        [-1.1194e+00,  8.8684e-01, -4.2599e-01, -1.3354e+00, -8.0845e-01],\n",
       "        [-1.2544e+00,  3.6654e-01, -2.0603e-01, -6.3600e-01, -7.7018e-01],\n",
       "        [-1.3344e+00,  3.8315e-01, -5.9196e-01, -3.8791e-01, -9.9269e-01],\n",
       "        [-1.1124e+00,  9.3911e-01, -7.6295e-01,  1.2384e-01, -1.6678e+00],\n",
       "        [-4.2140e-01,  1.1150e+00, -1.1682e-01,  2.0888e-01, -8.5319e-01],\n",
       "        [-8.9023e-01,  2.2000e-01, -6.6949e-01,  4.2543e-01, -1.1033e+00],\n",
       "        [-9.0739e-01,  3.4695e-01, -1.1031e+00,  2.0419e-01, -1.1963e+00],\n",
       "        [-9.8550e-01,  4.7432e-01, -5.2244e-01,  1.6976e-02, -8.4142e-01],\n",
       "        [-1.1602e+00,  5.3780e-01, -1.1989e-01,  1.7163e-02, -1.6733e+00],\n",
       "        [-9.0378e-01,  6.7437e-01, -3.5012e-01, -5.4585e-02, -7.0163e-01],\n",
       "        [-6.7531e-01,  1.5756e+00, -6.7022e-01, -4.1186e-01, -1.1764e+00],\n",
       "        [-1.4181e+00,  5.7813e-01, -4.8334e-01, -1.2755e-01, -2.9446e-01],\n",
       "        [-1.0398e+00,  1.0167e+00, -2.9493e-01, -4.6887e-01, -1.3058e+00],\n",
       "        [-9.6371e-01,  5.0497e-01, -1.1680e+00, -1.1721e-01, -1.4532e+00],\n",
       "        [-1.3085e+00,  1.4001e+00,  4.1654e-02, -1.0317e+00, -1.1993e+00],\n",
       "        [-1.2604e+00,  3.9723e-01, -2.5038e-01, -4.0408e-01, -1.2813e+00],\n",
       "        [-5.2197e-01,  4.5092e-01,  4.1342e-01, -3.1648e-01, -4.2069e-01],\n",
       "        [-1.1500e+00,  3.4631e-01, -5.4189e-01, -3.2815e-01, -1.1007e+00],\n",
       "        [-6.2021e-01,  6.4412e-01, -4.3039e-01, -1.1190e+00, -8.4726e-01],\n",
       "        [-6.5741e-01,  6.3153e-01, -6.7477e-02, -2.7359e-01, -8.7321e-01],\n",
       "        [-6.9815e-01,  7.5102e-01,  1.6397e-01,  5.4850e-01, -1.1840e+00],\n",
       "        [-7.8365e-01,  1.1746e+00, -8.9644e-03, -6.4958e-01, -1.8041e+00],\n",
       "        [-1.1415e+00,  1.2405e+00, -3.7043e-01, -9.3701e-01, -9.0754e-01],\n",
       "        [-1.1730e+00,  6.0087e-01,  1.6113e-02, -6.8910e-01, -1.8905e+00],\n",
       "        [-5.3200e-01,  5.5951e-01, -2.0247e-01, -1.6922e-01, -7.1141e-01],\n",
       "        [-7.9262e-01,  1.0460e+00, -1.3838e-02, -3.9546e-01, -1.7911e+00],\n",
       "        [-6.9411e-01,  1.0378e+00, -7.2988e-01, -3.6028e-01, -1.2056e+00],\n",
       "        [-7.4965e-01,  6.6636e-01,  1.0630e-01, -2.9428e-01, -9.9605e-01],\n",
       "        [-9.4345e-01,  9.2257e-01,  1.0993e-01, -2.7700e-01, -1.3721e+00],\n",
       "        [-1.1421e+00,  6.9604e-02, -8.1920e-01,  1.8953e-01, -1.9437e+00],\n",
       "        [-9.6402e-01,  4.8535e-01, -3.4182e-01, -1.6604e-01, -1.2623e+00],\n",
       "        [-3.8381e-01,  4.3712e-01,  3.0331e-01, -7.4898e-01, -6.1091e-01],\n",
       "        [-1.0902e+00,  7.9146e-01, -2.7056e-01,  8.1703e-01, -1.3677e+00],\n",
       "        [-1.2358e+00,  2.3999e-01, -9.0350e-01, -4.4172e-01, -1.3901e+00],\n",
       "        [-7.4451e-01,  8.5697e-01, -1.0138e-01, -2.0317e-01, -5.7757e-01],\n",
       "        [-1.0713e+00,  6.2292e-01, -6.6916e-02, -2.5528e-01, -8.7506e-01]],\n",
       "       device='cuda:0', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0., 0., 0., 0., 2., 0., 4., 0., 0., 0., 0., 2., 0., 0.,\n",
       "        0., 2., 0., 0., 0., 3., 2., 0., 2., 2., 0., 0., 0., 3., 2., 2., 0., 1.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 2., 1., 0., 2., 0., 2., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 2., 0., 2., 0., 0.], device='cuda:0')"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "qkappa = metrics.cohen_kappa_score(y.detach().cpu().numpy(),\n",
    "                                   pred.detach().cpu().numpy(),\n",
    "                                   weights=\"quadratic\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03961584633853543"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qkappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'view'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-205-bd0d4971e7f5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m#list(np.vstack(anan).reshape(-1))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0manan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'view'"
     ]
    }
   ],
   "source": [
    "anan = [[1, 2, 3], [3, 4, 5], [6, 7]]\n",
    "anan\n",
    "\n",
    "#list(np.vstack(anan).reshape(-1))\n",
    "anan.view(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(np.vstack(anan))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}