Resnet50_TL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm ,trange# for progress bar \n",
    "import os,sys,requests \n",
    "from concurrent.futures import ProcessPoolExecutor ,as_completed # Asynchronous Execution\n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "from pathlib import Path\n",
    "import re\n",
    "from pprint import pprint \n",
    "\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional  as F\n",
    "from torch.utils.data import Dataset , DataLoader\n",
    "\n",
    "import torchvision\n",
    "from torchvision import transforms ,models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Archive:  Birds.zip\n",
      "   creating: Birds/\n",
      "  inflating: Birds/Asian Paradise Flycatcher.csv  \n",
      "  inflating: Birds/Himalayan Monal.csv  \n",
      "  inflating: Birds/Indian Pitta.csv  \n",
      "  inflating: Birds/Mrs. Gould’s Sunbird.csv  \n",
      "  inflating: Birds/Oriental Dwarf Kingfisher.csv  \n",
      "  inflating: Birds/Red Headed Trogon.csv  \n",
      "  inflating: Birds/Sarus Crane.csv   \n",
      "  inflating: Birds/Satyr Tragopan.csv  \n",
      "  inflating: Birds/parrot.csv        \n",
      "  inflating: Birds/peacock.csv       \n",
      "total 620\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 61740 Nov  3  2018 'Asian Paradise Flycatcher.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 48812 Nov  3  2018 'Himalayan Monal.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 60080 Nov  3  2018 'Indian Pitta.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 54367 Nov  3  2018 'Mrs. Gould’s Sunbird.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 59841 Nov  3  2018 'Oriental Dwarf Kingfisher.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 55118 Nov  3  2018 'Red Headed Trogon.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 58170 Nov  3  2018 'Sarus Crane.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 51467 Nov  3  2018 'Satyr Tragopan.csv'\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 85959 Nov  3  2018  parrot.csv\n",
      "-rw-rw-r-- 1 ubuntu ubuntu 77909 Nov  3  2018  peacock.csv\n"
     ]
    }
   ],
   "source": [
    "!unzip Birds.zip\n",
    "!ls -l Birds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_url(url:str, dest:str, overwrite:bool=False, chunk_size=1024*1024, timeout=4):\n",
    "    \"Download `url` to `dest` unless it exists and not `overwrite`.\"\n",
    "    if os.path.exists(dest) and not overwrite: return\n",
    "\n",
    "    u = requests.get(url, stream=True, timeout=timeout)\n",
    "    try: file_size = int(u.headers[\"Content-Length\"])\n",
    "    except: pass\n",
    "        \n",
    "    with open(dest, 'wb') as f:\n",
    "        for chunk in u.iter_content(chunk_size=chunk_size):\n",
    "            f.write(chunk)\n",
    "\n",
    "def download_images(urls:str, dest:Path, max_pics:int=1000, max_workers:int=8, timeout=4,prefix=None,show_progress=True):\n",
    "    \"Download images listed in text file urls to path dest , at most max_pics\"\n",
    "    urls = open(urls).read().strip().split(\"\\n\")[:max_pics] # get all the urls in the list\n",
    "    dest = Path(dest) # get the path of the folder \n",
    "    dest.mkdir(exist_ok=True) # create all the parent and child directory if neccessary. \n",
    "\n",
    "    if max_workers: # use the power of multiprocessing\n",
    "        with ProcessPoolExecutor(max_workers=max_workers) as ex: # use a ProcessPoolExecutor for using pool of process to execute call asynchronously.\n",
    "             # submit schedules the callable and return a Future object on completion. We iterate through all the urls to create Future objects for each url \n",
    "            futures = [ex.submit(download_url, url, dest/f\"{i:08d}.jpg\", timeout=timeout) for i,url in enumerate(urls)]\n",
    "            if show_progress : \n",
    "                for f in tqdm(as_completed(futures), total=len(urls),desc=prefix): pass # show th progress bar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('Birds/Himalayan Monal.csv'),\n",
       " PosixPath('Birds/Red Headed Trogon.csv'),\n",
       " PosixPath('Birds/Mrs. Gould’s Sunbird.csv'),\n",
       " PosixPath('Birds/Oriental Dwarf Kingfisher.csv'),\n",
       " PosixPath('Birds/Indian Pitta.csv'),\n",
       " PosixPath('Birds/Sarus Crane.csv'),\n",
       " PosixPath('Birds/Asian Paradise Flycatcher.csv'),\n",
       " PosixPath('Birds/Satyr Tragopan.csv'),\n",
       " PosixPath('Birds/parrot.csv'),\n",
       " PosixPath('Birds/peacock.csv')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the path for all csv files\n",
    "birdspath = Path(\"Birds\")\n",
    "birdspath = [ Path(child) for child in birdspath.iterdir()]\n",
    "birdspath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HimalayanMonal',\n",
      " 'RedHeadedTrogon',\n",
      " 'MrsGould’sSunbird',\n",
      " 'OrientalDwarfKingfisher',\n",
      " 'IndianPitta',\n",
      " 'SarusCrane',\n",
      " 'AsianParadiseFlycatcher',\n",
      " 'SatyrTragopan',\n",
      " 'parrot',\n",
      " 'peacock']\n",
      "{'AsianParadiseFlycatcher': PosixPath('Birds/Asian Paradise Flycatcher.csv'),\n",
      " 'HimalayanMonal': PosixPath('Birds/Himalayan Monal.csv'),\n",
      " 'IndianPitta': PosixPath('Birds/Indian Pitta.csv'),\n",
      " 'MrsGould’sSunbird': PosixPath('Birds/Mrs. Gould’s Sunbird.csv'),\n",
      " 'OrientalDwarfKingfisher': PosixPath('Birds/Oriental Dwarf Kingfisher.csv'),\n",
      " 'RedHeadedTrogon': PosixPath('Birds/Red Headed Trogon.csv'),\n",
      " 'SarusCrane': PosixPath('Birds/Sarus Crane.csv'),\n",
      " 'SatyrTragopan': PosixPath('Birds/Satyr Tragopan.csv'),\n",
      " 'parrot': PosixPath('Birds/parrot.csv'),\n",
      " 'peacock': PosixPath('Birds/peacock.csv')}\n"
     ]
    }
   ],
   "source": [
    "all_birds_Name = [re.sub(r\".csv|\\.\\s|\\s*|\\’\" , r\"\", x.name) for x in birdspath] # Get all the birds name from the csv file\n",
    "pprint(all_birds_Name)\n",
    "folder_file = {x:y for x ,y in zip(all_birds_Name , birdspath)} # create a folder name the file name dictionary \n",
    "pprint(folder_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "HimalayanMonal: 100%|██████████| 512/512 [00:19<00:00, 25.78it/s]\n",
      "RedHeadedTrogon: 100%|██████████| 585/585 [00:22<00:00, 25.48it/s]\n",
      "MrsGould’sSunbird: 100%|██████████| 550/550 [00:17<00:00,  8.56it/s]\n",
      "OrientalDwarfKingfisher: 100%|██████████| 546/546 [00:16<00:00, 33.65it/s]\n",
      "IndianPitta: 100%|██████████| 648/648 [00:25<00:00,  2.15it/s]\n",
      "SarusCrane: 100%|██████████| 609/609 [00:20<00:00, 30.22it/s]\n",
      "AsianParadiseFlycatcher: 100%|██████████| 634/634 [00:22<00:00,  4.61it/s]\n",
      "SatyrTragopan: 100%|██████████| 544/544 [00:19<00:00, 27.70it/s]\n",
      "parrot: 100%|██████████| 847/847 [00:28<00:00, 29.92it/s]\n",
      "peacock: 100%|██████████| 810/810 [00:45<00:00,  3.62s/it]\n"
     ]
    }
   ],
   "source": [
    "for folder,file in folder_file.items():\n",
    "    path = Path(\"train\") # create a parent directory   \n",
    "    dest = path/folder # Specific folder for each bird\n",
    "    dest.mkdir(parents=True,exist_ok=True)\n",
    "    download_images(file, dest,prefix=folder,max_workers=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('HimalayanMonal', 500),\n",
       " ('SatyrTragopan', 530),\n",
       " ('OrientalDwarfKingfisher', 532),\n",
       " ('MrsGould’sSunbird', 543),\n",
       " ('RedHeadedTrogon', 577),\n",
       " ('SarusCrane', 601),\n",
       " ('AsianParadiseFlycatcher', 627),\n",
       " ('IndianPitta', 635),\n",
       " ('peacock', 796),\n",
       " ('parrot', 823)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "length = lambda iterrator : len(tuple(iterrator))\n",
    "birdsPerClassCount = { (path/folder).name : length((path/folder).iterdir()) for folder,file in folder_file.items() }\n",
    "sorted(birdsPerClassCount.items() , key=lambda x :x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PIL\n",
    "def verify_image(file:Path, delete:bool ,dest:Path=None, n_channels:int=3, **kwargs):\n",
    "    \"\"\"Check if the image in `file` exists, can be opend and has `n_channels`. If `delete`, removes it if it fails.\n",
    "        Result is stored in `dest` `img_format` and `kwargs` are passed to PIL.Image.save.\"\"\"\n",
    "    try:\n",
    "        img = PIL.Image.open(file)\n",
    "        assert isinstance(dest, Path), \"You should provide `dest` Path to save resized image\"\n",
    "        if not file.is_file(): return\n",
    "        if n_channels == 3: img = img.convert(\"RGB\")\n",
    "        img.save(file, **kwargs)\n",
    "        img = np.array(img)\n",
    "        img_channels = 1 if len(img.shape) == 2 else img.shape[2]\n",
    "        assert img_channels == n_channels, f\"Image {file} has {img_channels} instead of {n_channels}\"\n",
    "    except Exception as e:\n",
    "        #print(f'{e}')\n",
    "        if delete: file.unlink()\n",
    "\n",
    "def verify_images(path:Path, delete:bool=True, max_workers:int=4 , dest:Path=\".\",n_channels:int=3,**kwargs):\n",
    "    \"\"\"Check if the image in `path` exists, can be opened and has `n_channels`.\n",
    "    If `n_channels` is 3 – it'll try to convert image to RGB. If `delete`, removes it if it fails.\n",
    "    If `max_size` is specifided,\n",
    "    image is resized to the same ratio so that both sizes are less than `max_size`, using `interp`.\n",
    "    Result is stored in `dest`, `ext` forces an extension type, `img_format` and `kwargs` are\n",
    "    passed to PIL.Image.save. Use `max_workers` CPUs.\"\"\"\n",
    "    dest = path/Path(dest)\n",
    "    dest.mkdir(exist_ok=True)\n",
    "    with ProcessPoolExecutor(max_workers=max_workers) as ex:\n",
    "        files = [Path(file) for file in path.iterdir()]\n",
    "        futures = [ex.submit(verify_image, file, delete=delete,dest=dest, n_channels=n_channels, **kwargs) for file in files]\n",
    "    for f in tqdm(as_completed(futures), total=len(files) , desc=path.name): pass\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "HimalayanMonal: 100%|██████████| 500/500 [00:00<00:00, 232319.93it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "RedHeadedTrogon: 100%|██████████| 577/577 [00:00<00:00, 238528.82it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "MrsGould’sSunbird: 100%|██████████| 543/543 [00:00<00:00, 241927.67it/s]\n",
      "OrientalDwarfKingfisher: 100%|██████████| 532/532 [00:00<00:00, 232968.23it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "IndianPitta: 100%|██████████| 635/635 [00:00<00:00, 245767.56it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "SarusCrane: 100%|██████████| 601/601 [00:00<00:00, 239549.24it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "AsianParadiseFlycatcher: 100%|██████████| 627/627 [00:00<00:00, 243412.50it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "SatyrTragopan: 100%|██████████| 530/530 [00:00<00:00, 230791.23it/s]\n",
      "parrot: 100%|██████████| 823/823 [00:00<00:00, 247147.72it/s]\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "/home/ubuntu/instoried/lib/python3.6/site-packages/PIL/Image.py:993: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images\n",
      "  \"Palette images with Transparency expressed in bytes should be \"\n",
      "peacock: 100%|██████████| 796/796 [00:00<00:00, 250406.21it/s]\n"
     ]
    }
   ],
   "source": [
    "for folder,file in folder_file.items():\n",
    "    path = Path(\"train\")\n",
    "    verify_images(path/folder,delete=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('HimalayanMonal', 442),\n",
       " ('SatyrTragopan', 474),\n",
       " ('OrientalDwarfKingfisher', 475),\n",
       " ('MrsGould’sSunbird', 493),\n",
       " ('RedHeadedTrogon', 513),\n",
       " ('IndianPitta', 546),\n",
       " ('SarusCrane', 549),\n",
       " ('AsianParadiseFlycatcher', 560),\n",
       " ('parrot', 709),\n",
       " ('peacock', 729)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "birdsPerClassCountVerify = { (path/folder).name :len([*(path/folder).iterdir()]) for folder,file in folder_file.items() }\n",
    "sorted(birdsPerClassCountVerify.items() , key=lambda x :x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len of trainSet 4392 , len of testSet 1098\n"
     ]
    }
   ],
   "source": [
    "batch_size=128*torch.cuda.device_count()\n",
    "transform = transforms.Compose([transforms.Resize((224,224)) , \n",
    "                               transforms.ToTensor(),\n",
    "                               transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "                               ])\n",
    "myDataset = torchvision.datasets.ImageFolder(\"train\" , transform = transform)\n",
    "valid_no  = int(0.2 * len(myDataset))\n",
    "# so divide the data into trainset and testset\n",
    "trainSet,testSet = torch.utils.data.random_split(myDataset,(len(myDataset)-valid_no,valid_no))\n",
    "print(f\"len of trainSet {len(trainSet)} , len of testSet {len(testSet)}\")\n",
    "trainLoader  = DataLoader(trainSet , batch_size=batch_size ,shuffle=True) \n",
    "testLoader  = DataLoader(testSet , batch_size=batch_size ,shuffle=True)\n",
    "data_loader={\"train\":trainLoader , \"val\":testLoader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2048\n"
     ]
    }
   ],
   "source": [
    "model = models.resnet50(pretrained=True) # load the pretrained model\n",
    "num_features = model.fc.in_features # get the no of on_features in last Linear unit\n",
    "print(num_features)\n",
    "## freeze the entire convolution base\n",
    "for param in model.parameters():\n",
    "    param.requires_grad_(False)\n",
    "    \n",
    "def create_head(num_features , number_classes ,dropout_prob=0.5 ,activation_func =nn.ReLU):\n",
    "    features_lst = [num_features , num_features//2 , num_features//4]\n",
    "    layers = []\n",
    "    for in_f ,out_f in zip(features_lst[:-1] , features_lst[1:]):\n",
    "        layers.append(nn.Linear(in_f , out_f))\n",
    "        layers.append(activation_func())\n",
    "        layers.append(nn.BatchNorm1d(out_f))\n",
    "        if dropout_prob !=0 : layers.append(nn.Dropout(dropout_prob))\n",
    "    layers.append(nn.Linear(features_lst[-1] , number_classes))\n",
    "    return nn.Sequential(*layers)\n",
    "\n",
    "top_head = create_head(num_features , len(birdsPerClassCount)) # because ten classes\n",
    "model.fc = top_head # replace the fully connected layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "model_super_conv = copy.deepcopy(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "model = torch.nn.DataParallel(model).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.01)\n",
    "\n",
    "# specify loss function (categorical cross-entropy)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "model_super_conv = torch.nn.DataParallel(model_super_conv).to(device)\n",
    "optimizer_super_conv = optim.Adam(model_super_conv.parameters(), lr=0.01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import trange\n",
    "\n",
    "def train(model , data_loader , criterion , optimizer , num_epochs=5):\n",
    "\n",
    "  for epoch in trange(num_epochs,desc=\"Epochs\"):\n",
    "    result = []\n",
    "    for phase in ['train', 'val']:\n",
    "      if phase==\"train\":     # put the model in training mode\n",
    "        model.train()\n",
    "      else:     # put the model in validation mode\n",
    "        model.eval()\n",
    "       \n",
    "      # keep track of training and validation loss\n",
    "      running_loss = 0.0\n",
    "      running_corrects = 0.0  \n",
    "      \n",
    "      for data , target in data_loader[phase]:\n",
    "        #load the data and target to respective device\n",
    "        data , target = data.to(device)  , target.to(device)\n",
    "\n",
    "        with torch.set_grad_enabled(phase==\"train\"):\n",
    "          #feed the input\n",
    "          output = model(data)\n",
    "          #calculate the loss\n",
    "          loss = criterion(output,target)\n",
    "          preds = torch.argmax(output,1)\n",
    "\n",
    "          if phase==\"train\"  :\n",
    "            # backward pass: compute gradient of the loss with respect to model parameters \n",
    "            loss.backward()\n",
    "            # update the model parameters\n",
    "            optimizer.step()\n",
    "            # zero the grad to stop it from accumulating\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "\n",
    "        # statistics\n",
    "        running_loss += loss.item() * data.size(0)\n",
    "        running_corrects += torch.sum(preds == target.data).item()\n",
    "        \n",
    "        \n",
    "      epoch_loss = running_loss / len(data_loader[phase].dataset)\n",
    "      epoch_acc = running_corrects / len(data_loader[phase].dataset)\n",
    "\n",
    "      result.append('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "    print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epochs:  10%|█         | 1/10 [01:59<17:51, 119.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 1.4076 Acc: 0.5533', 'val Loss: 7.2398 Acc: 0.4827']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  20%|██        | 2/10 [03:38<15:06, 113.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.6342 Acc: 0.8071', 'val Loss: 1.7492 Acc: 0.7732']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  30%|███       | 3/10 [05:19<12:45, 109.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.5355 Acc: 0.8372', 'val Loss: 0.8349 Acc: 0.8506']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  40%|████      | 4/10 [06:59<10:39, 106.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.4618 Acc: 0.8618', 'val Loss: 0.7415 Acc: 0.8434']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  50%|█████     | 5/10 [08:39<08:43, 104.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.4255 Acc: 0.8723', 'val Loss: 0.5552 Acc: 0.8679']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  60%|██████    | 6/10 [10:19<06:53, 103.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.3882 Acc: 0.8864', 'val Loss: 0.5443 Acc: 0.8561']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  70%|███████   | 7/10 [11:59<05:07, 102.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.3585 Acc: 0.8925', 'val Loss: 0.5390 Acc: 0.8588']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  80%|████████  | 8/10 [13:39<03:23, 101.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.3416 Acc: 0.8939', 'val Loss: 0.4994 Acc: 0.8679']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  90%|█████████ | 9/10 [15:19<01:41, 101.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.3138 Acc: 0.9064', 'val Loss: 0.5047 Acc: 0.8652']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs: 100%|██████████| 10/10 [17:00<00:00, 100.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.3032 Acc: 0.9071', 'val Loss: 0.5204 Acc: 0.8634']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train(model,data_loader , criterion, optimizer,num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AsianParadiseFlycatcher :- 88.2883 %\n",
      "HimalayanMonal :- 77.1739 %\n",
      "IndianPitta :- 80.0 %\n",
      "MrsGould’sSunbird :- 79.7872 %\n",
      "OrientalDwarfKingfisher :- 89.011 %\n",
      "RedHeadedTrogon :- 77.0833 %\n",
      "SarusCrane :- 92.233 %\n",
      "SatyrTragopan :- 89.1892 %\n",
      "parrot :- 96.3768 %\n",
      "peacock :- 87.6543 %\n",
      "total acc is 86.33879781420765%\n"
     ]
    }
   ],
   "source": [
    "def perClassAccuracy(model ,classes):\n",
    "  class_correct = np.zeros(len(classes) ,dtype =np.int64 )\n",
    "  class_total = np.zeros_like(class_correct,dtype =np.int64 )\n",
    "  model.eval()\n",
    "  \n",
    "  for data ,target in data_loader[\"val\"]:\n",
    "    data,target =data.to(device) , target.to(device)\n",
    "    with torch.set_grad_enabled(False):\n",
    "      output =model(data)\n",
    "      preds = torch.argmax(output,1)\n",
    "      for prediction , label in zip(preds , target.data):\n",
    "        if prediction == label:\n",
    "          class_correct[prediction]+=1\n",
    "        class_total[label]+=1\n",
    "  per = np.round((100*class_correct/class_total) ,4)\n",
    "  out = \"\\n\".join([f\"{name} :- {acc} %\" for name ,acc in zip(classes , per)])\n",
    "  return out+\"\\ntotal acc is {0}%\".format(100* sum(class_correct)/sum(class_total))\n",
    "\n",
    "print(perClassAccuracy(model , myDataset.classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unfreezing these layer ['layer4', 'avgpool', 'fc']\n",
      "conv1 False\n",
      "bn1 False\n",
      "relu True\n",
      "maxpool True\n",
      "layer1 False\n",
      "layer2 False\n",
      "layer3 False\n",
      "layer4 True\n",
      "avgpool True\n",
      "fc True\n"
     ]
    }
   ],
   "source": [
    "def unfreeze(model,percent=0.25):\n",
    "  l = int(np.ceil(len(model.module._modules.keys())* percent))\n",
    "  l = list(model.module._modules.keys())[-l:]\n",
    "  print(f\"unfreezing these layer {l}\",)\n",
    "  for name in l:\n",
    "    for params in model.module._modules[name].parameters():\n",
    "      params.requires_grad_(True)\n",
    "\n",
    "def check_freeze(model):\n",
    "  for name ,layer in model.module._modules.items():\n",
    "    s = []\n",
    "    for l in layer.parameters():\n",
    "      s.append(l.requires_grad)\n",
    "    print(name ,all(s))\n",
    "# unfreeze half of the model\n",
    "unfreeze(model_super_conv ,0.30)\n",
    "# check which layer is freezed or not\n",
    "check_freeze(model_super_conv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.optim import lr_scheduler\n",
    "class LinearScheduler(lr_scheduler._LRScheduler):\n",
    "    \"\"\"Linearly increases the learning rate between two boundaries over a number of iterations.\"\"\"\n",
    "    def __init__(self, optimizer, end_lr, num_iter):\n",
    "        self.end_lr = end_lr\n",
    "        self.num_iter = num_iter\n",
    "        super(LinearScheduler,self).__init__(optimizer)\n",
    "\n",
    "    def get_lr(self):\n",
    "        # increement one by one\n",
    "        curr_iter = self.last_epoch + 1\n",
    "        # get the ratio\n",
    "        pct = curr_iter / self.num_iter\n",
    "        # calculate lr with this formulae start + pct * (end-start)\n",
    "        return [base_lr + pct * (self.end_lr - base_lr) for base_lr in self.base_lrs]\n",
    "\n",
    "\n",
    "class ExponentialScheduler(lr_scheduler._LRScheduler):\n",
    "    \"\"\"Exponentially increases the learning rate between two boundaries over a number of iterations.\"\"\"\n",
    "\n",
    "    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):\n",
    "        self.end_lr = end_lr\n",
    "        self.num_iter = num_iter\n",
    "        super(ExponentialScheduler,self).__init__(optimizer)\n",
    "\n",
    "    def get_lr(self):\n",
    "        curr_iter = self.last_epoch + 1\n",
    "        pct = curr_iter / self.num_iter\n",
    "        return [base_lr * (self.end_lr / base_lr) ** pct for base_lr in self.base_lrs]\n",
    "      \n",
    "class CosineScheduler(lr_scheduler._LRScheduler):\n",
    "    \"\"\"Cosine increases the learning rate between two boundaries over a number of iterations.\"\"\"\n",
    "\n",
    "    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):\n",
    "        self.end_lr = end_lr\n",
    "        self.num_iter = num_iter\n",
    "        super(CosineScheduler,self).__init__(optimizer)\n",
    "\n",
    "    def get_lr(self):\n",
    "        curr_iter = self.last_epoch + 1\n",
    "        pct = curr_iter / self.num_iter\n",
    "        cos_out = np.cos(np.pi * pct) + 1\n",
    "        return [self.end_lr + (base_lr - self.end_lr )/2 *cos_out for base_lr in self.base_lrs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LRFinder:\n",
    "  \n",
    "  def __init__(self, model  , optimizer , criterion ,start_lr=1e-7, device=None):\n",
    "    \n",
    "    self.model = model\n",
    "    # Move the model to the proper device\n",
    "    self.optimizer = optimizer\n",
    "    self.criterion = criterion\n",
    "    \n",
    "    ## save the model intial dict\n",
    "    self.save_file = Path(\"tmpfile\")\n",
    "    torch.save(self.model , self.save_file)    \n",
    "    if device is None:\n",
    "      self.device = next(model.parameters()).device\n",
    "    else:\n",
    "      self.device = device\n",
    "    self.model.to(self.device)\n",
    "    \n",
    "    self.history = {\"lr\":[] , \"losses\":[]}\n",
    "    for l in self.optimizer.param_groups:\n",
    "      l[\"initial_lr\"]=start_lr\n",
    "    \n",
    "    \n",
    "  def reset(self):\n",
    "    \"\"\" Resets the model to intial state \"\"\"\n",
    "    self.model = torch.load(self.save_file)\n",
    "    self.model.train()\n",
    "    self.save_file.unlink()\n",
    "    return self.model\n",
    "    \n",
    "  def calculateSmmothingValue(self ,beta):\n",
    "    n ,mov_avg=0,0\n",
    "    while True :\n",
    "      n+=1\n",
    "      value = yield\n",
    "      mov_avg = beta*mov_avg +(1-beta)*value\n",
    "      smooth = mov_avg / (1 - beta **n )\n",
    "      yield smooth\n",
    "    \n",
    "  def lrfind(self, trainLoader,end_lr=10,num_iter=50,step_mode=\"exp\", loss_smoothing_beta=0.99, diverge_th=4): \n",
    "        \"\"\"\n",
    "         Performs the lrfind test\n",
    "\n",
    "         Arguments:\n",
    "            trainLoader : The data loader\n",
    "            end_lr :  The maximum lr\n",
    "            num_iter : Max iteratiom\n",
    "            step_mode : The anneal function by default `exp` but can be either `linear` or `cos`\n",
    "            smooth_f : The loss smoothing factor, value should be between [0 , 1[\n",
    "            diverge_th: The max loss value after which training should be stooped\n",
    "        \"\"\"\n",
    "              # Reset test results\n",
    "        self.history = {\"lr\": [], \"losses\": []}\n",
    "        self.best_loss = None\n",
    "        self.smoothner = self.calculateSmmothingValue(loss_smoothing_beta)\n",
    "        \n",
    "        if step_mode.lower()==\"exp\":\n",
    "          lr_schedule = ExponentialScheduler(self.optimizer , end_lr  , num_iter,)\n",
    "        elif step_mode.lower()==\"cos\":\n",
    "          lr_schedule = CosineScheduler(self.optimizer , end_lr  , num_iter)\n",
    "        elif step.mode.lower()==\"linear\":\n",
    "          lr_schedule = LinearScheduler(self.optimizer , end_lr  , num_iter)\n",
    "        else:\n",
    "          raise ValueError(f\"expected mode is either {exp , cos ,linear} got {step_mode}\")\n",
    "        \n",
    "        if 0 < loss_smoothing_beta >=1:\n",
    "          raise ValueError(\"smooth_f is outside the range [0, 1[\")\n",
    "        \n",
    "        iterator = iter(trainLoader)\n",
    "        for each_iter in range(num_iter):\n",
    "          try:\n",
    "            data , target = next(iterator)\n",
    "          except StopIteration:\n",
    "            iterator = iter(trainLoader)\n",
    "            data , target = next(iterator)\n",
    "         \n",
    "          loss = self._train_batch(data , target)\n",
    "          \n",
    "          # Update the learning rate\n",
    "          lr_schedule.step()\n",
    "          self.history[\"lr\"].append(lr_schedule.get_lr()[0])\n",
    "          # Track the best loss and smooth it if smooth_f is specified\n",
    "          if each_iter == 0:\n",
    "              self.best_loss = loss\n",
    "          else:\n",
    "              next(self.smoothner)\n",
    "              self.best_loss = self.smoothner.send(loss)\n",
    "              if loss < self.best_loss:\n",
    "                  self.best_loss = loss\n",
    "\n",
    "          # Check if the loss has diverged; if it has, stop the test\n",
    "          self.history[\"losses\"].append(self.best_loss)\n",
    "          if loss > diverge_th * self.best_loss:\n",
    "              print(\"Stopping early, the loss has diverged\")\n",
    "              break\n",
    "\n",
    "        print(\"Learning rate search finished. See the graph with {finder_name}.plot()\")            \n",
    "  \n",
    "  def _train_batch(self,data,target):\n",
    "    # set to training mode\n",
    "    self.model.train()\n",
    "    #load data to device\n",
    "    data ,target = data.to(self.device) ,target.to(self.device)\n",
    "    \n",
    "    #forward pass\n",
    "    self.optimizer.zero_grad()\n",
    "    for l in self.optimizer.param_groups:\n",
    "        lr =l[\"lr\"] \n",
    "    output = self.model(data)\n",
    "    loss = self.criterion(output,target)\n",
    "    \n",
    "    #backward pass\n",
    "    loss.backward()\n",
    "    self.optimizer.step()\n",
    "    return loss.item()\n",
    "  \n",
    "  def plot(self):\n",
    "    losses = self.history[\"losses\"]\n",
    "    lr = self.history[\"lr\"]\n",
    "    plt.semilogx(lr,losses)\n",
    "    plt.xlabel(\"Learning rate\")\n",
    "    plt.ylabel(\"Losses \")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    }
   ],
   "source": [
    "lr_finder = LRFinder(model_super_conv, optimizer_super_conv, criterion, device=device)\n",
    "lr_finder.lrfind(data_loader[\"train\"], end_lr=1, step_mode=\"exp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3xUdb7/8ddnUklIJZUUQgk91CAoKqiggg3L2lZ3dVXEum753evuvb9td/15d6+uu+qq6+6qq2tZr4iiYkHBBhZK6BACoYWSnpBC+vf3xwwYMGVSZs7MnM/z8TgPZs6cOeedkOQz53u+5/sVYwxKKaXsy2F1AKWUUtbSQqCUUjanhUAppWxOC4FSStmcFgKllLI5LQRKKWVzwVYH6KmEhASTlZVldQyllPIr69atKzPGJHb0mt8VgqysLNauXWt1DKWU8isisq+z17RpSCmlbE4LgVJK2ZwWAqWUsjktBEopZXNaCJRSyua0ECillM3ZphA0tbSxJK8IHXZbKaVOZptC8Pr6In70r43c968NNDS3Wh1HKaV8ht/dUNZb10zLoLyuiYc+yGd3aS1/uTGXtNgBVsdSSinL2aYQiAh3nTOC0SlR3PfKBi597HOevGEqpw2N93qWltY2SmsbOVLdwJHqBg5XNwBw8cRUkqLC3dpHTUMzBSW1hAY5CA12fPNvsIPwkCAGhtnmv1Yp1Ufib23mubm5pq9DTOwqqWXh82vZX1HPry4dxw0zhvR4H1X1TYgIMQNCutyuvLaR9furWLuvgrx9VeyvqKekpoG2Dr7twQ7h/HHJfHf6EE4fNgiHQ056vbm1jU93lrIk7yDLtxXT2NLW6XETo8KYkBbD+LQYJqTHkJMWQ1K0e0VGKRV4RGSdMSa3w9fsWAgAqo81c98reazML+W60zL5xcVjGRAa1OG2xhj2ltezZm8Fa/dWsHZfJYWldQBEhweTHhdBetyAE/+GhTjYsL+KdfsqKSxzbhcSJIxPi2FE4kBSY8JJjgl3/hsdTmrMACrrm3jl6/28tq6IyvpmsgZFcP30TK6cks6BymO8kXeQtzYeoryuibiIEC6eMJizRyZijKGptY3G5jaaWttoammjvqmVgpIaNhdVs6u0luP/xcnRYUwdEscF41I4b0yynjUoZSNaCDrR2mZ4+IN8nvh4NwDhIQ5iB4QSMyCEmIgQYgaEYAxsOFBJWW0TADEDQsgdEsfUrDhCHA4OVNZTVHmMosp6DlQc45jrQnRcRAhTh8QxdUg8uVlx5KTFEB7ScaFpr6G5lfe2HOHFr/axZm8lImAMhAY7mDsmmQWT05g1MpHQYPeu89c1trDt8FE2FVWzuaiKVbvLKa1pJDTYwayRiczPcRaF6PCuz2yUUv5NC0E3Pi8oY2NRFVX1TVQfa6aqvpnqY86lubWNiemx5GbFMy0rjuGJA7/VZHOcMYbK+mbqGltIjxuASMfbuWtncQ1LNxwiI34A83JS++WPdWubYf3+SpZtPsy7m49w5GgDoUEOzsxO4JxRicwckcDQhMg+Z1dK+RYtBKpDbW2GvANVvLv5MO9tPUJR5TEABseEc8aIBM4ckcAZIwa5fQFbKeW7tBCobhlj2Fdez6rdZazaVcbq3eVU1TcDMGNYPI9cM4nUGO1uq5S/0kKgeqy1zbDt0FE+LSjliZW7GBAaxJ+vn8L0YYOsjqaU6oWuCoFt7ixWPRPkEHLSY7jrnBG8cddMosND+O7fvuLZVXt0mA6lAowWAtWt7OQo3rh7JrNHJfHrt7bx41c3cqxJh+lQKlB4rBCISIaIrBSRbSKyVUR+2ME2s0WkWkQ2uJZfeCqP6pvo8BCevnEqP5k7kjc2HOTKJ1dzoKLe6lhKqX7gyTOCFuAnxpixwAzgLhEZ28F2nxljJrmW33gwj+ojh0O457xsnvn+NA5U1nPJ45+zMr/E6lhKqT7yWCEwxhw2xqx3Pa4BtgNpnjqe8p5zRifx1t1nkhIdzs3PruHBd7fT3Nr5cBfHrdlbwY/+tYHdpbVeSKmUcpdXrhGISBYwGfiqg5dPF5GNIvKuiIzzRh7Vd1kJkbxx10y+Oz2Tv3xSyNV/+YKiyo6big5XH+Pel/P4zlNfsCTvIDc/u4by2kYvJ1ZKdcbjhUBEBgKLgfuMMUdPeXk9MMQYMxF4DHijk30sFJG1IrK2tLTUs4GV28JDgnjg8hwev34yu4prmf+nz3h/65ETrzc0t/LYRwWc+9AnvL/1CPeel81Lt02n+GgDC19Yp/NCKOUjPHofgYiEAG8D7xtj/uDG9nuBXGNMWWfb6H0EvmlfeR13v5TH5oPV3HRGFtOy4nnw3e0UVR5j3vgUfj5/DBnxEQAs23yYO19cz6UTB/OnayfpcBZKeYEl9xGI87f778D2zoqAiKS4tkNETnPlKfdUJuU5QwZF8todp/ODmUN5bvVe7nppPZGhwbx063SevGHqiSIAMD8nlX+7cBRLNx7ijx8WWJhaKQWenZhmJnAjsFlENrjW/RzIBDDGPAVcBdwhIi3AMeBao3cr+a2w4CB+cclYzh6ZQMnRRq6YkkZwUMefNe6YNZw9pXX86aMCshIiuHxyupfTKqWO81ghMMZ8DnR5zm+MeRx43FMZlDVmj0rqdhsR4YHLcyiqPMa/v7aZtNgIS2aLU0rpncXKQqHBDp66YSrp8QO4/YW17HVN4qOU8i4tBMpSMREhPPP9aQDc+eJ6Wjuaw1Mp5VFaCJTlshIi+fVl49l2+ChL8g5aHUcp29FCoHzCJRNSmZgew8Mf5Ov9BUp5mRYC5RNEhJ/PH8Ph6gb+/vkeq+MoZStaCJTPmD5sEHPGJPPkx7t1CAqlvEgLgfIp988bzbHmVh79SG80U8pbtBAonzIiaSDXTMvgxa/2U6ijlCrlFVoIlM+5b042ocEO/uf9fKujKGULWgiUz0mKCuf2s4fz7pYjrNtXYXUcpQKeFgLlk247eyhJUWE88M52dPgppTxLC4HySRGhwfx47kjW76/ivS1Hun+DUqrXtBAon/Wd3AxGJg/kd+/toKml+6kwlVK9o4VA+awgh/Cz+WPYW17PYyu0O6lSnqKFQPm0c0YlcdXUdP68chdf79ELx0p5ghYC5fN+dek4MuIjuO+VPKrrm62Oo1TA0UKgfN7AsGD+dO1kSmoa+fkbm7UXkVL9TAuB8guTMmL50dyRvLPpMK+tK7I6jlIBRQuB8huLZg1nxrB4frl0q85mplQ/0kKg/EaQQ3jkmkmEBDn44St5NLdql1Kl+oMWAuVXUmMG8N9X5LCxqJpHlu+0Oo5SAUELgfI783JSuXZaBk9+spsvdpdbHUcpv6eFQPmlX1wylqGDIvnJqxuoadAupUr1hRYC5ZciQoN56OqJHDnawP9btt3qOEr5NS0Eym9NyYzjtrOH8fLXB/hkZ6nVcZTyW1oIlF/70ZyRjEgayP2LN3FUm4iU6hUtBMqvhYcE8dB3JlJ8tIHfvr3N6jhK+SUtBMrvTcqIZdGs4by6toiVO0qsjqOU39FCoALCD+dkMzJ5IPe/vkkHplOqh7QQqIAQFhzEw9+ZRFltE7/RJiKlekQLgQoYOekx3Dl7OIvXF/HhtmKr4yjlN7QQqIByz7nZjE6J4mdLNmsvIqXcpIVABZTQYAcPXJ5DaU0jb244ZHUcpfyCxwqBiGSIyEoR2SYiW0Xkhx1sIyLyqIjsEpFNIjLFU3mUfUzJjGVUchSLdd4CpdziyTOCFuAnxpixwAzgLhEZe8o284Bs17IQeNKDeZRNiAhXTk1jw4EqdpfWWh1HKZ/nsUJgjDlsjFnvelwDbAfSTtnsMuB54/QlECsiqZ7KpOxjwaQ0HAKvr9ezAqW645VrBCKSBUwGvjrlpTTgQLvnRXy7WCAiC0VkrYisLS3VMWVU95KiwzkrO5El6w/S1qZzHCvVFY8XAhEZCCwG7jPGHO3NPowxTxtjco0xuYmJif0bUAWsK6akcai6gS8Ldc4Cpbri0UIgIiE4i8CLxpjXO9jkIJDR7nm6a51SfXbBuBSiwoJZvF5/pJTqiid7DQnwd2C7MeYPnWy2FPieq/fQDKDaGHPYU5mUvYSHBDE/J5V3txymrrHF6jhK+SxPnhHMBG4EzhWRDa5lvogsEpFFrm2WAYXALuCvwJ0ezKNs6Mqp6dQ3tfL+1iNWR1HKZwV7asfGmM8B6WYbA9zlqQxK5Q6JIyN+AIvXF3HFlHSr4yjlk/TOYhXQHA7hisnprN5dzqGqY1bHUconaSFQAe/KKekYA0vy9KKxUh3RQqACXuagCKZlxfH6+iKcrZFKqfa0EChbuHJKOrtL69hYVG11FKV8jhYCZQvzJ6QSFuzQISeU6oAWAmUL0eEhnD8uhaUbD9HY0mp1HKV8ihYCZRtXTEmjqr6ZlTt0vCql2tNCoGzjrBEJJEaFsSRPm4eUak8LgbKN4CAHF+WksjK/lFodckKpE7QQKFuZNz6FppY2Vu4osTqKUj5DC4GyldyseBKjwli2Wcc2VOo4LQTKVoIcwoXjUliZX0J9kzYPKf/xWUEpNQ3NHtm3FgJlO/NzUmlobtPeQ8pvHKlu4NZ/rOV37+3wyP61ECjbOW1oPAkDQ1m2RZuHlH94bEUBbcZw+9nDPbJ/LQTKdoIcwgXjUli5o4RjTXpzmfJt+8vr+deaA1w7LZOM+AiPHEMLgbKl+Tmp1De18slO7T2kfNsfP9xJcJBwz7kjPHYMLQTKlqYPjSc+MpRlm3XmMuW7dhbXsGTDQb5/ehZJ0eEeO44WAmVLwUEOLhiXzEfbi2lo1uYh5Zse/iCfyNBgFs3yzLWB47QQKNuaNz6VuqZWPt2pvYeU79l4oIr3txZz21nDiIsM9eixtBAo2zp9+CBiI0J4d4s2Dynf89AH+cRHhnLLWUM9fiwtBMq2QoIcnD82mQ+3FevQ1MqnfFlYzmcFZdwxazgDw4I9fjwtBMrW5uWkUtPYwucFZVZHUQoAYwwPvZ9PcnQYN54+xCvH1EKgbG3m8ASiw4N5R8ceUj7i4/xS1u6r5N7zsgkPCfLKMbstBCLyQxGJFqe/i8h6ETnfG+GU8rTQYAdzx6awfFsxTS1tVsdRNtfWZvif9/PJjI/g6twMrx3XnTOCHxhjjgLnA3HAjcB/ezSVUl500YQUahpaWLVbm4eUtf7++R62HT7Kj+ZmExLkvQYbd44krn/nAy8YY7a2W6eU35s5IoGosGCWbdLmIWWNltY2fvHmFh5Ytp05Y5K5dGKaV4/vTiFYJyIf4CwE74tIFKDn0CpghAUHMXdsMh9sK6a5VX+0lXdVH2vm5ufW8PwX+1h49jD+cuNUghze/aztTiG4BbgfmGaMqQdCgZs9mkopL5ufk0r1sWY+K9Cby5T37Cmr4/InVvFlYTm/v3ICP58/xutFANwrBAYYC9zreh4JeG7QC6UscPbIROIiQliSd8jqKMomVu8qY8GfV1FZ18Q/b5nO1dO8d3H4VO4UgieA04HrXM9rgD97LJFSFggNdnDxhMF8sPWIx2aBUgqcPYP+sXov33vma5KiwnjzrjOZPmyQpZncKQTTjTF3AQ0AxphKnM1DSgWUBZPTaGxp4/2txVZHUQEqb38llz+5ml8u3cpZ2Qm8fucZZA7yzBwDPeHOvcvNIhKEs4kIEUlELxarADQlM5YhgyJYklfEVVPTrY6jAkhJTQO/ezefxeuLSIoK45FrJrJgUhoivtEB051C8CiwBEgSkQeAq4D/7O5NIvIMcDFQYowZ38Hrs4E3gT2uVa8bY37jZm6l+p2IsGBSGo+uKOBIdQMpMXopTPVNU0sbz63ew6Mf7aKxpZVFs4Zz97kjvDJ+UE90m8YY86KIrAPOw3n/wAJjzHY39v0c8DjwfBfbfGaMudidoEp5w4LJafzpowLe3HCQ2z08BrwKTA3NrWw4UMWaPRUsyTtIYVkd541O4j8vHsvQhEir43Wo20IgIsOBPcaYP7s+xc8VkcPGmKqu3meM+VREsvolpVJeMjQhksmZsSzJ00Kg3FNd38yavRUnls0Hq2luNYjAuMHRPHvTNM4ZnWR1zC65c36yGMgVkRHAX4ClwEs4bzDrq9NFZCNwCPip667lbxGRhcBCgMzMzH44rFKdu3xyGr94cyvbDx9lTGq01XGUDyosreWj7SUs317Mun2VtLYZQoKECemx3HLmME4bGsfUzHhiIkKsjuoWdwpBmzGmRUSuAB43xjwmInn9cOz1wBBjTK2IzAfeALI72tAY8zTwNEBubq7ph2Mr1amLJwzmN29t4428g1oIFOAcGnrtvkqWbyvmw23FFJbVATA6JYo7Zg3nrOwEJmbEem200P7mbq+h64DvAZe41vW5zLkGsjv+eJmIPCEiCcYYHflLWSo+MpTZoxJ5c8Mh/u3C0Zbc6al8ywfbirn9hXWEBAkzhg3i+2dkcd6YJNLjrO/62R/cKQQ3A4uAB4wxe0RkKPBCXw8sIilAsTHGiMhpOO9pKO/rfpXqDwsmp/Hh9hK+KiznjBEJVsdRFtt26CgisOY/5hAbEXi3UbnTa2gbruElRCQOiDLG/K6794nIy8BsIEFEioBf4jqTMMY8hbMb6h0i0gIcA641xmizj/IJc8YkExUWzOt5B7UQKArL6kiLHRCQRQDc6zX0MXCpa9t1QImIrDLG/Lir9xljruvm9cdxdi9VyueEhwQxLyeFZZuP8F+XjWdAqH+2/ar+UVhay7DEgVbH8Bh3hpiIcbXnXwE8b4yZDszxbCylrLdgchq1jS0s365DTtiZMYY9ZXUM89F7APqDO4UgWERSgauBtz2cRymfMWPoIFJjwnkj76DVUZSFio82Ut/UyvBEexeC3wDvA7uNMWtEZBhQ4NlYSlnP4RAum5TGJztLKa9ttDqOskhhaS2AvZuGjDH/a4yZYIy5w/W80BhzpeejKWW9K6ak0dpmeGujzlNgV7td9wwMs/MZgYiki8gSESlxLYtFRIdmVLYwMjmKManRLNVCYFuFpbUMCAkiOSpwByF0p2noWZzDSgx2LW+51illCxdPSGX9/ioOVh2zOoqyQGFpHUMTInEE8I2F7hSCRGPMs8aYFtfyHJDo4VxK+YyLJ6QCsGzTYYuTKCsUltUGdLMQuFcIykXkBhEJci03oHcAKxsZMiiSnLQY3t6shcBuGltaKao8FtAXisG9QvADnF1HjwCHcd4RfJMHMynlcy6akMrGA1UcqKi3Ooryon3l9RhDQHcdBfd6De0zxlxqjEk0xiQZYxYA2mtI2cpFOc7moXf0rMBWTnQdTdAzgo50ObyEUoEmIz6CiRmxvKPXCWxld6mz62hWQmCMMtqZ3haCwL18rlQnLs5JZfPBava6+pWrwFdYWkdSVBhR4f4xwUxv9bYQ6CihynbmT9DmIbvZY4MeQ9BFIRCRGhE52sFSg/N+AqVsJS12AFMytXnITgrL6gK+xxB0UQiMMVHGmOgOlihjjDsT2igVcC6aMJhth4+euIioAldFXRNV9c0BPerocb1tGlLKlubnpADoWYENHC/2w+18RqCU+rbUmAFMy4rT6wQ2UFga+IPNHaeFQKkeuignlR1HathVUmN1FNULx5paeeGLvbS0tnW53e6yWkKChLTYAd4JZiEtBEr10LycVETgbW0e8kvvbz3C/31zKx92M/PcntI6hgyKJDgo8P9MBv5XqFQ/S44O57SseN7edBhjtCe1v8kvdp7JfbCt60JQGODTU7anhUCpXrh4Qiq7SmrZWay9h/xNgev/bMWOkk6bh1pa29hXbo+uo6CFQKleuXB8Kg6BdzbphDX+pqCkhqjwYKrqm1m7r7LDbYoqj9HcamxxoRi0ECjVK4lRYcwYNkibh/xMQ3Mr+yvquXZaBqFBDj7spHmosOx411EtBEqpLlw0IZXCsjp2HNHeQ/5iV0ktxsCkjDhOHz6I5duLOyzkx7uODg3wUUeP00KgVC+dPzYFEXh3yxGroyg37SpxftIfmTyQuWOT2VdeT0HJt6/zFJbVERsRQnxkqLcjWkILgVK9lBgVxrSseN7bot1I/cXO4hqCHcKQQZHMHZsMwPIOmocKS2tt02MItBAo1Sfzxqews7j2xCdN5dt2FtcyNCGS0GAHydHhTEyP6bAbaWGpfXoMgRYCpfrkwvHOsYf0rMA/7CqpYWRy1Innc8cms/FAFSVHG06sq2lopqSm0TY9hkALgVJ9khozgMmZsXqdwA80NLeyr6KeEUnffNKf42oe+nB7yYl1e8uc81IH+vSU7WkhUKqP5o1PYeuho+wv14ntfdnuUmePofZnBKOSo8iIH8Dybd8U8uNdR/WMQCnltnnjnTOXvbdVm4d82fE7irOTv/mkLyLMHZPCqt3l1DW2AM55ih0CQwYF9jzF7WkhUKqPMuIjGDc4WpuHfNzxHkNZg07+pD93bDJNLW18urMUcPYYSo+LICw4yIqYlvBYIRCRZ0SkRES2dPK6iMijIrJLRDaJyBRPZVHK0+aNTyFvfxWHq49ZHUV1oqCklixXj6H2pmXFETMghOWu0UidPYbs0ywEnj0jeA64sIvX5wHZrmUh8KQHsyjlURe6mofe17MCn1VQXMPI5G9fAA4OcnDu6CRW7CihubWNPWV1trpQDB4sBMaYT4GKLja5DHjeOH0JxIpIqqfyKOVJI5IGMjJ5IMu0EPik42MMZSdFdfj63LHJVNU3886mwxxrbtUzAi9KAw60e17kWvctIrJQRNaKyNrS0lKvhFOqpy4cn8qavRWU1jRaHUWdYndpLW3m5AvF7Z09MpHQIAd/+bQQwFZ3FYOfXCw2xjxtjMk1xuQmJiZaHUepDs0bn4Ix8ME2PSvwNcd7DLXvOtrewLBgzhgxiO2HjwLY6q5isLYQHAQy2j1Pd61Tyi+NTokia1AE72nzkM8pKOm4x1B7x8ceigwNIjk6zFvRfIKVhWAp8D1X76EZQLUxRjtiK78lIlw4PpUvdpdTVd9kdRzVzs7ijnsMtTdnjLMQDE2MRES8Fc0neLL76MvAF8AoESkSkVtEZJGILHJtsgwoBHYBfwXu9FQWpbxl3vgUWtpMhyNaKuvsKqklO6nr5p7k6HDmjk3mrGz7NT8He2rHxpjrunndAHd56vhKWWFCegxpsQN4b8sRvpOb0f0blMc1NLeyr7yOSyYO7nbbv34v1wuJfI9fXCxWyl84m4dS+KygjJqGZqvjKJw3iLUZOryHQDlpIVCqn80bn0JTaxsrdpR0v7HyuIIS51Sind1DoLQQKNXvpmTGMTgmnGc+30Nbm05sb7WdxTUEOYShNrs3oCe0ECjVzxwO4acXjGJjUTWL1xdZHcf2CopryRoU0WWPIbvT74xSHrBgUhpTMmP53Xv5HNVrBZYqKKnt9EYy5aSFQCkPcDiEX106jvK6Rh77qMDqOLZ1vMdQd11H7U4LgVIeMiE9lmtyM3h21V6d3N4ix3sMZesZQZe0ECjlQT+9YBQDQoP49Vtbcd46o7zpeI8hbRrqmhYCpTwoYWAYP5ozks8Kyk6aIF15R0FxLUEOISvBPtNO9oYWAqU87MbThzAyeSD/9fY2GppbrY5jKzuLa8gaZK9pJ3tDC4FSHhYS5OCXl4xjf0U9f/98j9VxbMU5xpA2C3VHC4FSXjBzRALzxqfw+IpdOq+xlzQ0t7K3vE6HlnCDFgKlvOTn88fQZgwPLtthdRRb0B5D7tNCoJSXZMRHcPus4SzdeIiNB6qsjhPwTowxpGcE3dJCoJQXLTx7GHERIfxh+U6rowS84z2GdIyh7mkhUMqLBoYFc/us4Xyys5R1+yqsjhPQCkpqGKI9htyihUApL/ve6UNIGBjKwx/oWYGntLUZ1u2rZPzgGKuj+AUtBEp5WURoMHfMHsHq3eV8sbvc6jgBafPBaspqmzhntP2mnewNLQRKWeC70zNJjg7jD8vzdegJD1ixowQRONuG8w/3hhYCpSwQHhLE3eeMYM3eSj4rKLM6TsBZmV/CpIxYBg0MszqKX9BCoJRFrp6WweCYcP6wfKeeFfSjkpoGNhVVc+6oJKuj+A0tBEpZJCw4iHvOy2bDgSpW5uuAdP3l4/xSAM4ZrYXAXVoIlLLQVVPTyYyP0LOCfrRyRwnJ0WGMGxxtdRS/oYVAKQuFBDm497xsthw8yvtbi62O4/eaWtr4rKCMc0YlISJWx/EbWgiUstiCSYMZlhDJI8t30tamZwV9sXZvBbWNLdos1ENaCJSyWHCQgx/OySa/uIa3Nx+2Oo5fW7GjhNAgB2eOSLA6il/RQqCUD7hkwmBGp0Tx8Af5NLW0WR3Hb63IL2H6sHgiw4KtjuJXtBAo5QMcDuH+eaPZV17PP7/cZ3Ucv7SvvI7C0jrO0W6jPaaFQCkfMWtkImdlJ/DoigKqjzVbHcfvrNjh7IJ7rl4f6DEtBEr5CBHhZ/PGUH2smSdW7rI6jt9ZmV/KsIRIsnTY6R7TQqCUDxk7OJorJqfz7Oq9HKiotzqO36hvauHLwnLtLdRLWgiU8jE/vWAkAjz0Qb7VUfzGql3lNLW0abNQL3m0EIjIhSKSLyK7ROT+Dl6/SURKRWSDa7nVk3mU8gepMQO49ayhvLnhEJuKdEpLd6zYUcLAsGCmZcVbHcUveawQiEgQ8GdgHjAWuE5Exnaw6b+MMZNcy988lUcpf7Jo1nAGRYbywDvbdeiJbhhj+Di/hDNHJBAarI0cveHJ79ppwC5jTKExpgl4BbjMg8dTKmBEhYdw35xsvtpTwUfbdUC6rmw/XMPh6gZtFuoDTxaCNOBAu+dFrnWnulJENonIayKS4cE8SvmVa0/LZFhiJA++u52W1pNvMjPGUFhay5sbDvLWxkOs3lVG/pEaymobabXZMBXHR26drbOR9ZrVt9+9BbxsjGkUkduBfwDnnrqRiCwEFgJkZmZ6N6FSFgkJcnD/haNZ+MI6nlu9l7Gp0azfX8n6/VXk7a+ksr7jew1EIC4ilGlZcTx+/RRCggK7uWTFjhJy0mJIigq3Oorf8mQhOAi0/4Sf7lp3gjGm/YStfwN+39GOjDFPA08D5Obm2uvjjrK1uWOTOS0rnt++s/3EuuGJkcwZk8yUIXFMyogl2CGU1UIxiacAAAxbSURBVDZRXtdIRV0TZbVNHKioZ0neQZ7/Yh+3nDnU67nb2gwfbi9m5ogEjwz3YIyh+GgjGw5Ukre/krvPze73Y9iJJwvBGiBbRIbiLADXAte330BEUo0xx0fZuhTYjlLqBBHh91dN4O1NhxiXFsPkjFhiI0K/tV128snPjTFU1DXxx+U7uWzSYBK8PGXjH5bv5PGVuzh92CCevXka4SFBfdpfbWMLX+4uZ/PB6hNLaU0jAKHBDubnpPRHbNsST/ZIEJH5wB+BIOAZY8wDIvIbYK0xZqmIPIizALQAFcAdxpgdXe0zNzfXrF271mOZlQoUu0trueCRT7liShq/v2qi14779qZD3P1SHlOHxLFuXyXzxqfw+PVTCHL0bn6AusYWLn9iFTuLaxGBEYkDyUmLISc9hpy0GMYOjiYi1OpWbt8nIuuMMbkdvuZvXdO0ECjlvv+3bDtPf1rIG3fNZFJGrMePt/VQNVc9+QVjB0fz0m3TefHL/fzm7W1cPz2TBxaM7/FkMcYY7n1lA+9sOsQfr53MeaOTdGTRXuqqEAT2VSSlbO6ec0eQGBXGL5du9fikN2W1jSx8fh2xESE8dcNUwoKD+MGZQ7lz9nBe+mo/jyzf2eN9/mP1Xt7aeIifnD+KSycO1iLgIVoIlApgUeEh3H/haDYeqGLx+iKPHaeppY07/7mestpGnr4xl8Sob65J/J8LRnFNbgaPrtjFc6v2uL3Pdfsq+O0725kzJok7Zg33RGzlooVAqQB3+eQ0pmTG8rv38jna4JnhrX/91la+3lvB76+aQE56zEmviQgPXD6e88cm86u3tvHmhoOd7OUbZbWN3PniegbHDuDhqyfh6OX1BeUeLQRKBTiHQ/jVpeMor2vk0Q8L+n3///xyHy9+tZ/bZw3jskkd3TPqnI7z0esmM31oPD95dSPvbTnS6dAZLa1t3PNSHlX1zTx5wxRiBoT0e2Z1Mi0EStnAhPRYrsnN4LnVe9lVUtNv+/2ysJxfLd3K7FGJ/NsFo7vcNjwkiL9+P5eRyVEs+uc65j7yKX/9tJCy2saTtnt4+U6+KCzntwvGM25wTCd7U/1Jew0pZRPltY3MfuhjJmXE8vwPTutxD55Tfb2ngpuf/ZrkmHCW3DnT7U/u9U0tvL3xMK+s2c/6/VUEO4Q5Y5K5ZloGjS2tLPrneq47LZMHr8jpUz51Mu0+qpQC4NlVe/j1W9v49aXj+P4ZWb3ez+rdZdzy3FoGx4bz0m0zSI7u3fAOBcU1vLr2AIvXH6SirgmAnLQY/nfR6X2+CU2dTAuBUgpwtr/f+vxaPs4v5bvTM/nlJeN6PHTzZwWl3Pb8WjLjI3jx1hkn9RDqraaWNj7aXsynBaXcfW42abED+rxPdTItBEqpE1rbDP/zfj5PfbKb3CFxPHHDFLcHbFuZX8LtL6xjWEIkL946nUFeHrpC9Z7eUKaUOiHIIdw/bzSPXTeZrYeOcsljn5O3v7Lb9324rZjbn19HdtJAXr5thhaBAKKFQCmbumTiYBbfcQYhQQ6u+cuXvLrmwLe2McZQ09DM0o2HWPTPdYxJjeKlW2cQF/ntge+U/9KmIaVsrrKuiXtezuPzXWWcMXwQxkBlfRMVdU1U1jfR3Or8GzElM5bnfnAa0eHar98fddU0pAN3KGVzcZGhPHfzNP6wfCfLtxUTGxFCZnwEkzJiiYsMJS4ihMSoMC4Yl6KjfAYoPSNQSikb0IvFSimlOqWFQCmlbE4LgVJK2ZwWAqWUsjktBEopZXNaCJRSyua0ECillM1pIVBKKZvzuxvKRKQaKAASgLJe7CIGqO7FNt2tO/X148872sZT2Tt7vbNsHT3399ztH/tKbneytn/s6dzuZOxoXaD9jLd/bIfcQ4wxiR2+YozxqwV42vXv2r68v6fbdLfu1Nfb5fzWNp7K3tnrnWXr6uvw19wdfQ1W53Ynqzdzu5Oxu6z6sxIYuY8v/tg09JYX3t/RNt2tO/X1t9zYpqe6e39nr3eWraPn/p67/WNfyX3qOqtzd7aN3X7G2z+2a27AD5uGjhORtaaTcTN8nb9m19ze5a+5wX+z2zW3P54RHPe01QH6wF+za27v8tfc4L/ZbZnbb88IlFJK9Q9/PiNQSinVD7QQKKWUzWkhUEopmwvIQiAiZ4nIUyLyNxFZbXUed4mIQ0QeEJHHROT7VufpCRGZLSKfub7vs63O0xMiEikia0XkYquzuEtExri+16+JyB1W5+kJEVkgIn8VkX+JyPlW53GXiAwTkb+LyGtWZ+mO62f6H67v83e7297nCoGIPCMiJSKy5ZT1F4pIvojsEpH7u9qHMeYzY8wi4G3gH57M2y5fn3MDlwHpQDNQ5Kmsp+qn7AaoBcLxUvZ+yg3w78Crnkn5bf30M77d9TN+NTDTk3nb66fsbxhjbgMWAdd4Mm+7fP2Ru9AYc4tnk3auh1/DFcBrru/zpd3uvC93o3liAc4GpgBb2q0LAnYDw4BQYCMwFsjB+ce+/ZLU7n2vAlH+khu4H7jd9d7X/Ol7Djhc70sGXvSj3HOBa4GbgIv9JbfrPZcC7wLX+9PPSrv3PQxM8cPcXvvd7MPX8DNgkmubl7rbdzA+xhjzqYhknbL6NGCXMaYQQEReAS4zxjwIdHg6LyKZQLUxpsaDcU/oj9wiUgQ0uZ62ei7tyfrre+5SCYR5Iuep+ul7PhuIxPnLc0xElhlj2nw9t2s/S4GlIvIO8JLnEp90zP74ngvw38C7xpj1nk3s1M8/45boydeA86w8HdiAGy0/PlcIOpEGHGj3vAiY3s17bgGe9Vgi9/Q09+vAYyJyFvCpJ4O5oUfZReQK4AIgFnjcs9G61KPcxpj/ABCRm4AyTxeBLvT0+z0b5+l/GLDMo8m619Of83uAOUCMiIwwxjzlyXBd6On3fBDwADBZRH7mKhhW6+xreBR4XEQuwo1hKPylEPSYMeaXVmfoKWNMPc4C5neMMa/jLGR+yRjznNUZesIY8zHwscUxesUY8yjOP1R+xRhTjvO6hs8zxtQBN7u7vc9dLO7EQSCj3fN01zpf56+5wX+za27v89fs/pq7vX75GvylEKwBskVkqIiE4ry4t9TiTO7w19zgv9k1t/f5a3Z/zd1e/3wNVlz97ubK+MvAYb7pQnmLa/18YCfOK+T/YXXOQMntz9k1t2YP9Nze+hp00DmllLI5f2kaUkop5SFaCJRSyua0ECillM1pIVBKKZvTQqCUUjanhUAppWxOC4EKGCJS6+Xj/U1Exnr5mPeJSIQ3j6kCn95HoAKGiNQaYwb24/6CjTEt/bU/N48pOH8vOxz8TkT2ArnGmDJv5lKBTc8IVEATkUQRWSwia1zLTNf600TkCxHJE5HVIjLKtf4mEVkqIiuAj8Q589rH4pwJbIeIvOj6Y41rfa7rca04Z5fbKCJfikiya/1w1/PNIvLbjs5aRCTLNbHI88AWIENEnhTnrGlbReTXru3uBQYDK0VkpWvd+a6vY72I/K+I9FshVDZi9W3TuujSXwtQ28G6l4AzXY8zge2ux9FAsOvxHGCx6/FNOG/fj3c9nw1U4xzMywF80W5/H+P8dA7OGdoucT3+PfCfrsdvA9e5Hi/qJGMW0AbMaLfu+PGDXMeZ4Hq+F0hwPU7AOVx5pOv5vwO/sPr/QRf/WwJ2GGqlXOYAY10f4gGiXZ+aY4B/iEg2zj/iIe3es9wYU9Hu+dfGmCIAEdmA8w/356ccpwnnH32AdThnPgM4HVjgevwS8FAnOfcZY75s9/xqEVmIc6j4VJwT52w65T0zXOtXub6+UJyFSqke0UKgAp0D5yfthvYrReRxYKUx5nLXrE8ft3u57pR9NLZ73ErHvzfNxhjTzTZdOXFMERkK/BSYZoypFJHncM4FfSrBWbSu6+GxlDqJXiNQge4DnDNiASAik1wPY/hm3PabPHj8L4ErXY+vdfM90TgLQ7XrWsO8dq/VAFHt9j1TREYAiEikiIzse2RlN1oIVCCJEJGidsuPgXuBXBHZJCLb+GaGqd8DD4pIHp49M74P+LGIbAJG4Lze0CVjzEYgD9iBszlpVbuXnwbeE5GVxphSnEXsZdf+vwBG9298ZQfafVQpD3L1+T9mjDEici3OC8eXWZ1Lqfb0GoFSnjUV5yTiAlQBP7A4j1LfomcESillc3qNQCmlbE4LgVJK2ZwWAqWUsjktBEopZXNaCJRSyua0ECillM39fxOvbMDeOGwpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_finder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_super_conv = lr_finder.reset()\n",
    "optimizer_super_conv = optim.Adam(model_super_conv.parameters(), lr=0.01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stepper():\n",
    "    \"Used to \\\"step\\\" from start,end (`vals`) over `n_iter` iterations on a schedule defined by `func`\"\n",
    "    \n",
    "    def __init__(self, val, n_iter:int, func):\n",
    "        self.start,self.end = val\n",
    "        self.n_iter = max(1,n_iter)\n",
    "        self.func = func\n",
    "        self.n = 0\n",
    "\n",
    "    def step(self):\n",
    "        \"Return next value along annealed schedule.\"\n",
    "        self.n += 1\n",
    "        return self.func(self.start, self.end, self.n/self.n_iter)\n",
    "\n",
    "    @property\n",
    "    def is_done(self):\n",
    "        \"Return `True` if schedule completed.\"\n",
    "        return self.n >= self.n_iter\n",
    "    \n",
    "# Annealing functions\n",
    "def annealing_no(start, end, pct):\n",
    "    \"No annealing, always return `start`.\"\n",
    "    return start\n",
    "  \n",
    "def annealing_linear(start, end, pct):\n",
    "    \"Linearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.\"\n",
    "    return start + pct * (end-start)\n",
    "  \n",
    "def annealing_exp(start, end, pct):\n",
    "    \"Exponentially anneal from `start` to `end` as pct goes from 0.0 to 1.0.\"\n",
    "    return start * (end/start) ** pct\n",
    "\n",
    "def annealing_cos(start, end, pct):\n",
    "    \"Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.\"\n",
    "    cos_out = np.cos(np.pi * pct) + 1\n",
    "    return end + (start-end)/2 * cos_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class OneCyclePolicy:\n",
    "  \n",
    "  def __init__(self,model , optimizer , criterion ,num_iteration,num_epochs,max_lr, momentum = (0.95,0.85) , div_factor=25 , pct_start=0.4, device=None ):\n",
    "    \n",
    "    self.model =model\n",
    "    self.optimizer = optimizer\n",
    "    self.criterion = criterion\n",
    "    self.num_epochs = num_epochs\n",
    "    if device is None:\n",
    "      self.device = next(model.parameters()).device\n",
    "    else:\n",
    "      self.device = device\n",
    "      \n",
    "    n = num_iteration * self.num_epochs\n",
    "    a1 = int(n*pct_start)\n",
    "    a2 = n-a1\n",
    "    self.phases = ((a1 , annealing_linear) , (a2 , annealing_cos))\n",
    "    min_lr = max_lr/div_factor\n",
    "    self.lr_scheds = self.steps((min_lr,max_lr) , (max_lr,min_lr/1e4))\n",
    "    self.mom_scheds =self.steps(momentum , momentum[::-1])\n",
    "    self.idx_s = 0\n",
    "    self.update_lr_mom(self.lr_scheds[0].start,self.mom_scheds[0].start)\n",
    "  \n",
    "  def steps(self, *steps):\n",
    "      \"Build anneal schedule for all of the parameters.\"\n",
    "      return [Stepper(step, n_iter, func=func)for (step,(n_iter,func)) in zip(steps, self.phases)]\n",
    "\n",
    "  def train(self, trainLoader , validLoader ):\n",
    "    self.model.to(self.device)\n",
    "    data_loader = {\"train\":trainLoader , \"val\":validLoader}\n",
    "    for epoch in tqdm(range(self.num_epochs),desc=\"Epochs\"):\n",
    "      result = []\n",
    "      for phase in ['train', 'val']:\n",
    "        if phase==\"train\":     # put the model in training mode\n",
    "          model.train()\n",
    "        else:     # put the model in validation mode\n",
    "          model.eval()\n",
    "\n",
    "        # keep track of training and validation loss\n",
    "        running_loss = 0.0\n",
    "        running_corrects = 0  \n",
    "\n",
    "        for data , target in data_loader[phase]:\n",
    "          #load the data and target to respective device\n",
    "          data , target = data.to(device)  , target.to(device)\n",
    "\n",
    "          with torch.set_grad_enabled(phase==\"train\"):\n",
    "            #feed the input\n",
    "            output = self.model(data)\n",
    "            #calculate the loss\n",
    "            loss = self.criterion(output,target)\n",
    "            preds = torch.argmax(output,1)\n",
    "\n",
    "            if phase==\"train\"  :\n",
    "              # backward pass: compute gradient of the loss with respect to model parameters \n",
    "              loss.backward()\n",
    "              # update the model parameters\n",
    "              self.optimizer.step()\n",
    "              # zero the grad to stop it from accumulating\n",
    "              self.optimizer.zero_grad()\n",
    "            \n",
    "              self.update_lr_mom(self.lr_scheds[self.idx_s].step() ,self.mom_scheds[self.idx_s].step() )\n",
    "\n",
    "              if self.lr_scheds[self.idx_s].is_done:\n",
    "                self.idx_s += 1\n",
    "          \n",
    "          # statistics\n",
    "          running_loss += loss.item() * data.size(0)\n",
    "          running_corrects += torch.sum(preds == target.data).item()\n",
    "\n",
    "\n",
    "        epoch_loss = running_loss / len(data_loader[phase].dataset)\n",
    "        epoch_acc = running_corrects/ len(data_loader[phase].dataset)\n",
    "\n",
    "        result.append('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "      print(result)\n",
    "\n",
    "  def update_lr_mom(self,lr=0.001,mom=0.99):\n",
    "    for l in self.optimizer.param_groups:\n",
    "      l[\"lr\"]=lr\n",
    "      if isinstance(self.optimizer , ( torch.optim.Adamax,torch.optim.Adam)):\n",
    "          l[\"betas\"] = ( mom, 0.999)\n",
    "      elif isinstance(self.optimizer, torch.optim.SGD):\n",
    "          l[\"momentum\"] =mom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epochs:  20%|██        | 1/5 [01:41<06:46, 101.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 2.0348 Acc: 0.3179', 'val Loss: 0.7256 Acc: 0.8097']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  40%|████      | 2/5 [03:22<05:04, 101.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.5220 Acc: 0.8541', 'val Loss: 0.4162 Acc: 0.8834']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  60%|██████    | 3/5 [05:04<03:22, 101.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.2613 Acc: 0.9271', 'val Loss: 0.3958 Acc: 0.8934']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs:  80%|████████  | 4/5 [06:45<01:41, 101.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.1407 Acc: 0.9620', 'val Loss: 0.3632 Acc: 0.9044']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epochs: 100%|██████████| 5/5 [08:27<00:00, 101.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train Loss: 0.0936 Acc: 0.9770', 'val Loss: 0.3751 Acc: 0.8980']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "fit_one_cycle = OneCyclePolicy(model_super_conv ,optimizer_super_conv , criterion,\n",
    "                               num_iteration=len(trainLoader)  , num_epochs =5 ,\n",
    "                               max_lr =0.001 ,device=device)\n",
    "fit_one_cycle.train(trainLoader,testLoader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AsianParadiseFlycatcher :- 90.991 %\n",
      "HimalayanMonal :- 84.7826 %\n",
      "IndianPitta :- 87.0 %\n",
      "MrsGould’sSunbird :- 89.3617 %\n",
      "OrientalDwarfKingfisher :- 93.4066 %\n",
      "RedHeadedTrogon :- 85.4167 %\n",
      "SarusCrane :- 95.1456 %\n",
      "SatyrTragopan :- 91.8919 %\n",
      "parrot :- 99.2754 %\n",
      "peacock :- 91.358 %\n",
      "total acc is 91.2568306010929%\n"
     ]
    }
   ],
   "source": [
    "print(perClassAccuracy(model_super_conv , myDataset.classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}