classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BhJjrDThK6Zl"
   },
   "source": [
    "# **0. Basics**\n",
    "Importing required modules and Defining some useful functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1659,
     "status": "ok",
     "timestamp": 1573317495244,
     "user": {
      "displayName": "친절배려양보",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDFkwnEU7dwPLhibVEPf81KRoxg6Fp6WyoycQ8S_w=s64",
      "userId": "13918714700772710322"
     },
     "user_tz": -540
    },
    "id": "I3KEZd2bd1Pm",
    "outputId": "ec9a9597-26e1-43aa-aa11-af810d9725fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The gpu to be used : GeForce GTX 1060 6GB\n"
     ]
    }
   ],
   "source": [
    "# For plotting\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import matplotlib.animation as animation\n",
    "from IPython.display import HTML\n",
    "\n",
    "# For utilities\n",
    "import os, shutil, time, sys\n",
    "\n",
    "# For conversion\n",
    "sys.path.insert(0, '../../')\n",
    "import opencv_transforms.transforms as TF\n",
    "import opencv_transforms.functional as FF\n",
    "import dataloader\n",
    "from PIL import Image\n",
    "import cv2\n",
    "import numbers\n",
    "\n",
    "# For everything\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.utils as vutils\n",
    "import random\n",
    "import tqdm\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# For our model\n",
    "import mymodels\n",
    "import torchvision.models\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# To ignore warning\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\", UserWarning)\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print(\"The gpu to be used : {}\".format(torch.cuda.get_device_name(0)))\n",
    "else:\n",
    "    print(\"No gpu detected\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3_cw23W9d1P5"
   },
   "source": [
    "# **1. Loading dataset**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "W0wM8z9abn-P"
   },
   "source": [
    "## 1.1 Dataloader\n",
    "\n",
    "The code of dataloader to load input videos is written in `dataloader.py`. Loaded data is 5D tensor with size of **[N, C, L, W, H]**.\n",
    "\n",
    "To pre-process input data, the module `opencv_transforms.transforms` and `opencv_transforms.functional` are imported. These are implemented with **openCV** so faster than `torchvision.transforms` which is based on **Pillow**.[2]\n",
    "\n",
    "[2] Jim Bohnslav,\"opencv_transforms\",https://github.com/jbohnslav/opencv_transforms,2020.1.13."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 92
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1035,
     "status": "ok",
     "timestamp": 1573317630904,
     "user": {
      "displayName": "친절배려양보",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDFkwnEU7dwPLhibVEPf81KRoxg6Fp6WyoycQ8S_w=s64",
      "userId": "13918714700772710322"
     },
     "user_tz": -540
    },
    "id": "mQe2xM2sAbVy",
    "outputId": "dc0237b9-6c86-430e-d239-214be395419b",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Training data... Done!\n",
      "Training data size : 2543\n",
      "Loading Validation data... Done!\n",
      "Validation data size : 679\n"
     ]
    }
   ],
   "source": [
    "# batch_size\n",
    "batch_size=4\n",
    "clip_length=16\n",
    "sampling_rate=4\n",
    "\n",
    "# Training\n",
    "print('Loading Training data...', end=' ')\n",
    "train_transforms = TF.Compose([\n",
    "    TF.Resize((121, 178))\n",
    "    ])\n",
    "train_imagefolder = dataloader.VideoFolder(\n",
    "    '../../dataset/aps_cut/train', \n",
    "    transform=train_transforms, \n",
    "    extensions=('avi'), \n",
    "    clip_length=clip_length,\n",
    "    sampling_rate=sampling_rate,\n",
    "    start_random=True,\n",
    "    )\n",
    "train_loader = torch.utils.data.DataLoader(train_imagefolder, batch_size=batch_size, shuffle=True)\n",
    "train_batch = next(iter(train_loader))\n",
    "print(\"Done!\")\n",
    "print(\"Training data size : {}\".format(len(train_imagefolder)))\n",
    "\n",
    "# Validation\n",
    "print('Loading Validation data...', end=' ')\n",
    "val_transforms = TF.Compose([\n",
    "    TF.Resize((121, 178)),\n",
    "    ])\n",
    "val_imagefolder = dataloader.VideoFolder(\n",
    "    '../../dataset/aps_cut/val', \n",
    "    transform=val_transforms, \n",
    "    extensions=('avi'), \n",
    "    clip_length=clip_length,\n",
    "    sampling_rate=sampling_rate,\n",
    "    start_random=False,\n",
    "    )\n",
    "val_loader = torch.utils.data.DataLoader(val_imagefolder, batch_size=batch_size, shuffle=False)\n",
    "val_batch = next(iter(val_loader))\n",
    "print(\"Done!\")\n",
    "print(\"Validation data size : {}\".format(len(val_imagefolder)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DcBgaktjD3Ul"
   },
   "source": [
    "## 1.2 Dataset Test\n",
    "\n",
    "Check the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_batch_iter = iter(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 216
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1797,
     "status": "ok",
     "timestamp": 1573317635511,
     "user": {
      "displayName": "친절배려양보",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDFkwnEU7dwPLhibVEPf81KRoxg6Fp6WyoycQ8S_w=s64",
      "userId": "13918714700772710322"
     },
     "user_tz": -540
    },
    "id": "90y9V_zqeb93",
    "outputId": "aad3aaf3-ffcc-48ed-ccca-da0c48adc607",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clip size: torch.Size([4, 3, 32, 112, 112])\n",
      "tensor([ 8, 23, 14, 10])\n",
      "tensor(255., device='cuda:0', dtype=torch.float64)\n",
      "Finished\n"
     ]
    }
   ],
   "source": [
    "temp_batch = next(temp_batch_iter)\n",
    "clip = temp_batch[0]\n",
    "label = temp_batch[1]\n",
    "print(\"Clip size: {0}\".format(clip.size()))\n",
    "print(label)\n",
    "print(clip.max())\n",
    "dataloader.play_video(clip[0])\n",
    "print('Finished')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cdYb-1jfE8Lh"
   },
   "source": [
    "# **2. Construct the Model**\n",
    "\n",
    "All models are implemented on `mymodels.py`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of parameters: 11924440\n",
      "Classifier7(\n",
      "  (dilation): DilationBlock(\n",
      "    (convlist): ModuleList(\n",
      "      (0): Sequential(\n",
      "        (0): Conv3DWS(3, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), bias=False)\n",
      "        (1): GroupNorm(16, 64, eps=1e-05, affine=True)\n",
      "        (2): ReLU(inplace=True)\n",
      "      )\n",
      "      (1): Sequential(\n",
      "        (0): Conv3DWS(3, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), dilation=(3, 1, 1), bias=False)\n",
      "        (1): GroupNorm(16, 64, eps=1e-05, affine=True)\n",
      "        (2): ReLU(inplace=True)\n",
      "      )\n",
      "      (2): Sequential(\n",
      "        (0): Conv3DWS(3, 64, kernel_size=(3, 1, 1), stride=(1, 1, 1), dilation=(5, 1, 1), bias=False)\n",
      "        (1): GroupNorm(16, 64, eps=1e-05, affine=True)\n",
      "        (2): ReLU(inplace=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (norm1): GroupNorm(16, 64, eps=1e-05, affine=True)\n",
      "  (pool1): MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv2): Conv3DWS(64, 128, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "  (norm2): GroupNorm(16, 128, eps=1e-05, affine=True)\n",
      "  (pool2): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv3): Conv3DWS(128, 256, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "  (norm3): GroupNorm(16, 256, eps=1e-05, affine=True)\n",
      "  (pool3): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv4): Conv3DWS(256, 512, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "  (norm4): GroupNorm(16, 512, eps=1e-05, affine=True)\n",
      "  (pool4): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv5): Conv3DWS(512, 512, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1), bias=False)\n",
      "  (norm5): GroupNorm(16, 512, eps=1e-05, affine=True)\n",
      "  (pool5): MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)\n",
      "  (fc): Linear(in_features=8192, out_features=24, bias=True)\n",
      "  (relu): ReLU()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = mymodels.Classifier7().to(device) \n",
    "num_params = sum(p.numel() for p in net.parameters() if p.requires_grad)\n",
    "print('Number of parameters: %d' % (num_params))\n",
    "print(net)\n",
    "\n",
    "torch.backends.cudnn.benchmark = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r0w7ENw_d1QZ"
   },
   "source": [
    "# **3. Train the Model**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8pNKJYmoGrbq"
   },
   "source": [
    "## 3.1 Set hyperparameters, optimizer, loss, etc.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NqQ3m5ALeJDR"
   },
   "outputs": [],
   "source": [
    "# Number of epoch\n",
    "num_epochs = 50\n",
    "# Learning rate\n",
    "lr = 0.001\n",
    "# Loss functions\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "# Setup optimizers\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=5e-4, nesterov=True)\n",
    "#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TJ6GnxGtlHyd"
   },
   "source": [
    "## 3.2 Train the model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_epoch=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lists to keep track of progress\n",
    "train_loss_list = []\n",
    "val_loss_list = []\n",
    "train_accr_list = []\n",
    "val_accr_list = []\n",
    "label_list = range(0, 24)\n",
    "zero_list = [0]*24\n",
    "train_accr_dict = dict(zip(label_list, zero_list))\n",
    "val_accr_dict = dict(zip(label_list, zero_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0dKi5zyQkubu",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Training Loop...\n",
      "#################################### Epoch: 1/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 2.1648547308115895 / Acc: 0.3314982304364923                    \n",
      "Execution time: 382.99021673202515\n",
      "[val]  Batch: 170/169 / Loss: 1.805237026558709 / Acc: 0.3917525773195876                     \n",
      "Execution time: 43.38793921470642\n",
      "Saving... Done!\n",
      "#################################### Epoch: 2/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 1.0558167158783864 / Acc: 0.6362563900904443                   \n",
      "Execution time: 426.8255512714386\n",
      "[val]  Batch: 170/169 / Loss: 1.0909516488329476 / Acc: 0.6067746686303387                   \n",
      "Execution time: 46.121655225753784\n",
      "Saving... Done!\n",
      "#################################### Epoch: 3/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.705988154560181 / Acc: 0.7463625639009044                    \n",
      "Execution time: 422.48017168045044\n",
      "[val]  Batch: 170/169 / Loss: 0.8203479256782686 / Acc: 0.6921944035346097                   \n",
      "Execution time: 46.185486793518066\n",
      "Saving... Done!\n",
      "#################################### Epoch: 4/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.5019663393825609 / Acc: 0.8183248132127409                    \n",
      "Execution time: 422.7843589782715\n",
      "[val]  Batch: 170/169 / Loss: 0.783605460478034 / Acc: 0.7290132547864506                    \n",
      "Execution time: 49.48569893836975\n",
      "Saving... Done!\n",
      "#################################### Epoch: 5/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.4042951350154907 / Acc: 0.8545025560361778                    \n",
      "Execution time: 434.2371428012848\n",
      "[val]  Batch: 170/169 / Loss: 0.7817132037698081 / Acc: 0.7437407952871871                   \n",
      "Execution time: 47.35635471343994\n",
      "Saving... Done!\n",
      "#################################### Epoch: 6/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.32530960350323396 / Acc: 0.8883208808493904                   \n",
      "Execution time: 437.5679304599762\n",
      "[val]  Batch: 170/169 / Loss: 0.8344114852056939 / Acc: 0.7407952871870398                   \n",
      "Execution time: 47.90270376205444\n",
      "Saving... Done!\n",
      "#################################### Epoch: 7/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.2733943259861354 / Acc: 0.9052300432559968                    \n",
      "Execution time: 448.86985754966736\n",
      "[val]  Batch: 170/169 / Loss: 0.6012964350485486 / Acc: 0.8070692194403535                   \n",
      "Execution time: 47.10413360595703\n",
      "Saving... Done!\n",
      "#################################### Epoch: 8/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.21054566286231793 / Acc: 0.9244986236728273                   \n",
      "Execution time: 450.78834652900696\n",
      "[val]  Batch: 170/169 / Loss: 0.4979930594673213 / Acc: 0.8379970544918999                    \n",
      "Execution time: 47.11205053329468\n",
      "Saving... Done!\n",
      "#################################### Epoch: 9/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.17879064787202853 / Acc: 0.9351160047188359                   \n",
      "Execution time: 452.91650557518005\n",
      "[val]  Batch: 170/169 / Loss: 0.6641145375761613 / Acc: 0.8085419734904271                   \n",
      "Execution time: 47.13955020904541\n",
      "Saving... Done!\n",
      "#################################### Epoch: 10/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.15921242884937065 / Acc: 0.9457333857648447                   \n",
      "Execution time: 451.3862655162811\n",
      "[val]  Batch: 170/169 / Loss: 0.46571431340806085 / Acc: 0.8512518409425626                   \n",
      "Execution time: 47.676549196243286\n",
      "Saving... Done!\n",
      "#################################### Epoch: 11/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.14625403311688084 / Acc: 0.9500589854502556                   \n",
      "Execution time: 452.92084765434265\n",
      "[val]  Batch: 170/169 / Loss: 0.4487973618402046 / Acc: 0.8645066273932254                    \n",
      "Execution time: 47.40813899040222\n",
      "Saving... Done!\n",
      "#################################### Epoch: 12/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.1251525796405646 / Acc: 0.9634290208415257                    \n",
      "Execution time: 452.1608338356018\n",
      "[val]  Batch: 170/169 / Loss: 0.4858343076284574 / Acc: 0.8438880706921944                    \n",
      "Execution time: 47.38929581642151\n",
      "Saving... Done!\n",
      "#################################### Epoch: 13/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.11394875508056851 / Acc: 0.9622493118364136                   \n",
      "Execution time: 450.8081612586975\n",
      "[val]  Batch: 170/169 / Loss: 0.41435411647071896 / Acc: 0.8645066273932254                   \n",
      "Execution time: 47.19203853607178\n",
      "Saving... Done!\n",
      "#################################### Epoch: 14/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.10994503596988757 / Acc: 0.9661816751867872                   \n",
      "Execution time: 449.96663308143616\n",
      "[val]  Batch: 170/169 / Loss: 0.5510736570442548 / Acc: 0.8217967599410898                   \n",
      "Execution time: 47.33140850067139\n",
      "Saving... Done!\n",
      "#################################### Epoch: 15/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.09844310059623425 / Acc: 0.9661816751867872                   \n",
      "Execution time: 453.7478783130646\n",
      "[val]  Batch: 170/169 / Loss: 0.5619302220829105 / Acc: 0.8424153166421208                   \n",
      "Execution time: 47.190760135650635\n",
      "Saving... Done!\n",
      "#################################### Epoch: 16/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.07918587145426706 / Acc: 0.9744396382225717                   \n",
      "Execution time: 449.8595106601715\n",
      "[val]  Batch: 170/169 / Loss: 0.3236112466441338 / Acc: 0.9013254786450663                    \n",
      "Execution time: 46.135621547698975\n",
      "Saving... Done!\n",
      "#################################### Epoch: 17/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.07383767050455463 / Acc: 0.974832874557609                     \n",
      "Execution time: 426.7507519721985\n",
      "[val]  Batch: 170/169 / Loss: 0.47017052396583275 / Acc: 0.8394698085419735                   \n",
      "Execution time: 46.03389096260071\n",
      "Saving... Done!\n",
      "#################################### Epoch: 18/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.07688656502055034 / Acc: 0.9787652379079826                   \n",
      "Execution time: 425.43330240249634\n",
      "[val]  Batch: 170/169 / Loss: 0.3379392546652343 / Acc: 0.8910162002945508                    \n",
      "Execution time: 45.941112756729126\n",
      "Saving... Done!\n",
      "#################################### Epoch: 19/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.05765266114691323 / Acc: 0.9807314195831695                    \n",
      "Execution time: 425.4831395149231\n",
      "[val]  Batch: 170/169 / Loss: 0.2930857421547745 / Acc: 0.9042709867452136                    \n",
      "Execution time: 45.95609927177429\n",
      "Saving... Done!\n",
      "#################################### Epoch: 20/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.07726808912487905 / Acc: 0.974832874557609                    \n",
      "Execution time: 423.86845993995667\n",
      "[val]  Batch: 170/169 / Loss: 0.2957869511816393 / Acc: 0.9072164948453608                    \n",
      "Execution time: 46.053837299346924\n",
      "Saving... Done!\n",
      "#################################### Epoch: 21/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.05695907984293684 / Acc: 0.9819111285882816                    \n",
      "Execution time: 426.2082018852234\n",
      "[val]  Batch: 170/169 / Loss: 0.2613310408346432 / Acc: 0.9042709867452136                    \n",
      "Execution time: 46.05284094810486\n",
      "Saving... Done!\n",
      "#################################### Epoch: 22/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.05963411661189246 / Acc: 0.9787652379079826                    \n",
      "Execution time: 421.9715323448181\n",
      "[val]  Batch: 170/169 / Loss: 0.30757732388723935 / Acc: 0.8910162002945508                   \n",
      "Execution time: 46.19346523284912\n",
      "Saving... Done!\n",
      "#################################### Epoch: 23/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.05862816241234717 / Acc: 0.9811246559182069                    \n",
      "Execution time: 426.8764181137085\n",
      "[val]  Batch: 170/169 / Loss: 0.286351106654913 / Acc: 0.898379970544919                      \n",
      "Execution time: 46.04087042808533\n",
      "Saving... Done!\n",
      "#################################### Epoch: 24/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.0633105566029916 / Acc: 0.9764058198977585                     \n",
      "Execution time: 432.05257296562195\n",
      "[val]  Batch: 170/169 / Loss: 0.2557856492336088 / Acc: 0.9042709867452136                    \n",
      "Execution time: 45.86733675003052\n",
      "Saving... Done!\n",
      "#################################### Epoch: 25/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.04717540169080534 / Acc: 0.9850570192685804                    \n",
      "Execution time: 426.17229890823364\n",
      "[val]  Batch: 170/169 / Loss: 0.29845123930075734 / Acc: 0.9042709867452136                   \n",
      "Execution time: 45.76860022544861\n",
      "Saving... Done!\n",
      "#################################### Epoch: 26/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03888973297285391 / Acc: 0.9862367282736925                    \n",
      "Execution time: 424.79498195648193\n",
      "[val]  Batch: 170/169 / Loss: 0.37229345859589386 / Acc: 0.8807069219440353                   \n",
      "Execution time: 45.78555631637573\n",
      "Saving... Done!\n",
      "#################################### Epoch: 27/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.04680195287883155 / Acc: 0.9842705465985057                    \n",
      "Execution time: 424.6763000488281\n",
      "[val]  Batch: 170/169 / Loss: 0.29562950081607553 / Acc: 0.8998527245949927                   \n",
      "Execution time: 45.79353380203247\n",
      "Saving... Done!\n",
      "#################################### Epoch: 28/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03293539612400014 / Acc: 0.9885961462839166                    \n",
      "Execution time: 423.3817594051361\n",
      "[val]  Batch: 170/169 / Loss: 0.3673051684348678 / Acc: 0.8910162002945508                    \n",
      "Execution time: 45.89227056503296\n",
      "Saving... Done!\n",
      "#################################### Epoch: 29/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.058957080976071434 / Acc: 0.9787652379079826                   \n",
      "Execution time: 443.7096092700958\n",
      "[val]  Batch: 170/169 / Loss: 0.3206544371873184 / Acc: 0.9101620029455081                    \n",
      "Execution time: 44.088463306427\n",
      "Saving... Done!\n",
      "#################################### Epoch: 30/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03510996763789649 / Acc: 0.9885961462839166                    \n",
      "Execution time: 365.71149158477783\n",
      "[val]  Batch: 170/169 / Loss: 0.2735867596866456 / Acc: 0.9116347569955817                    \n",
      "Execution time: 36.048733711242676\n",
      "Saving... Done!\n",
      "#################################### Epoch: 31/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.05139669629470879 / Acc: 0.9862367282736925                    \n",
      "Execution time: 400.364137172699\n",
      "[val]  Batch: 170/169 / Loss: 0.26104613045705083 / Acc: 0.9072164948453608                                      \n",
      "Execution time: 40.104926109313965\n",
      "Saving... Done!\n",
      "#################################### Epoch: 32/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.04567294780822706 / Acc: 0.9838773102634683                    \n",
      "Execution time: 399.10476517677307\n",
      "[val]  Batch: 170/169 / Loss: 0.264786841767526 / Acc: 0.918998527245949947                                      \n",
      "Execution time: 40.50552010536194\n",
      "Saving... Done!\n",
      "#################################### Epoch: 33/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.042714560919339103 / Acc: 0.9854502556036178                   \n",
      "Execution time: 406.47313141822815\n",
      "[val]  Batch: 170/169 / Loss: 0.3898968224321795 / Acc: 0.8762886597938144                                      \n",
      "Execution time: 39.64906549453735\n",
      "Saving... Done!\n",
      "#################################### Epoch: 34/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03895967090143476 / Acc: 0.9874164372788046                    \n",
      "Execution time: 408.4197814464569\n",
      "[val]  Batch: 170/169 / Loss: 0.2936713745123508 / Acc: 0.9131075110456554                    \n",
      "Execution time: 38.81927037239075\n",
      "Saving... Done!\n",
      "#################################### Epoch: 35/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03460420270253015 / Acc: 0.988989382618954                     \n",
      "Execution time: 397.3128571510315\n",
      "[val]  Batch: 170/169 / Loss: 0.30208966254485725 / Acc: 0.9116347569955817                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
      "Execution time: 33.97414231300354\n",
      "Saving... Done!\n",
      "#################################### Epoch: 36/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03499634801239985 / Acc: 0.9885961462839166                    \n",
      "Execution time: 350.30042243003845\n",
      "[val]  Batch: 170/169 / Loss: 0.34700005781316967 / Acc: 0.9160530191458026                   \n",
      "Execution time: 34.54697561264038\n",
      "Saving... Done!\n",
      "#################################### Epoch: 37/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.0383133615417906 / Acc: 0.9874164372788046                     \n",
      "Execution time: 347.81284284591675\n",
      "[val]  Batch: 170/169 / Loss: 0.24367325277848167 / Acc: 0.9189985272459499                   \n",
      "Execution time: 34.73818349838257\n",
      "Saving... Done!\n",
      "#################################### Epoch: 38/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.029992552099854186 / Acc: 0.9925285096342902                   \n",
      "Execution time: 349.7719979286194\n",
      "[val]  Batch: 170/169 / Loss: 0.30236355370028556 / Acc: 0.9027982326951399                   \n",
      "Execution time: 34.77009868621826\n",
      "Saving... Done!\n",
      "#################################### Epoch: 39/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.026901531350830767 / Acc: 0.9921352732992528                   \n",
      "Execution time: 350.67814350128174\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[val]  Batch: 170/169 / Loss: 0.25601682870131703 / Acc: 0.9189985272459499                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             \n",
      "Execution time: 35.0263454914093\n",
      "Saving... Done!\n",
      "#################################### Epoch: 40/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03009890825599445 / Acc: 0.9897758552890287                    \n",
      "Execution time: 350.8518810272217\n",
      "[val]  Batch: 170/169 / Loss: 0.30570927143799187 / Acc: 0.9204712812960235                   \n",
      "Execution time: 34.72227191925049\n",
      "Saving... Done!\n",
      "#################################### Epoch: 41/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.024811283697349695 / Acc: 0.9913488006291781                   \n",
      "Execution time: 350.2400929927826\n",
      "[val]  Batch: 170/169 / Loss: 0.3092394535250095 / Acc: 0.8924889543446244                    \n",
      "Execution time: 34.755940437316895\n",
      "Saving... Done!\n",
      "#################################### Epoch: 42/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.03264026054938064 / Acc: 0.9921352732992528                     \n",
      "Execution time: 350.3179590702057\n",
      "[val]  Batch: 170/169 / Loss: 0.2814799447122835 / Acc: 0.9042709867452136                    \n",
      "Execution time: 34.80427122116089\n",
      "Saving... Done!\n",
      "#################################### Epoch: 43/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.030490444628128333 / Acc: 0.9870232009437672                   \n",
      "Execution time: 350.5252261161804\n",
      "[val]  Batch: 170/169 / Loss: 0.2840493162531565 / Acc: 0.9042709867452136                    \n",
      "Execution time: 34.71317100524902\n",
      "Saving... Done!\n",
      "#################################### Epoch: 44/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.026997056290524894 / Acc: 0.9913488006291781                   \n",
      "Execution time: 350.077308177948\n",
      "[val]  Batch: 170/169 / Loss: 0.2718734708731234 / Acc: 0.9116347569955817                    \n",
      "Execution time: 34.76376795768738\n",
      "Saving... Done!\n",
      "#################################### Epoch: 45/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.02393207070024005 / Acc: 0.9913488006291781                    \n",
      "Execution time: 350.21967911720276\n",
      "[val]  Batch: 170/169 / Loss: 0.27238009674735203 / Acc: 0.9189985272459499                   \n",
      "Execution time: 34.80679965019226\n",
      "Saving... Done!\n",
      "#################################### Epoch: 46/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.026779212681646195 / Acc: 0.9921352732992528                   \n",
      "Execution time: 349.53807950019836\n",
      "[val]  Batch: 170/169 / Loss: 0.23331803882244936 / Acc: 0.9278350515463918                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      \n",
      "Execution time: 33.8504741191864\n",
      "Saving... Done!\n",
      "#################################### Epoch: 47/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.02827144737933688 / Acc: 0.9905623279591034                    \n",
      "Execution time: 341.5815181732178\n",
      "[val]  Batch: 170/169 / Loss: 0.2097388256632 / Acc: 0.9293078055964654                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n",
      "Execution time: 33.91729521751404\n",
      "Saving... Done!\n",
      "#################################### Epoch: 48/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.016729066655262456 / Acc: 0.9964608729846638                   \n",
      "Execution time: 342.3958809375763\n",
      "[val]  Batch: 170/169 / Loss: 0.24825935160113013 / Acc: 0.9234167893961708                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 \n",
      "Execution time: 34.088836669921875\n",
      "Saving... Done!\n",
      "#################################### Epoch: 49/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.01851599121769357 / Acc: 0.9941014549744396                    \n",
      "Execution time: 364.23268961906433\n",
      "[val]  Batch: 170/169 / Loss: 0.2158958516872333 / Acc: 0.93519882179676071                                      \n",
      "Execution time: 45.80535101890564\n",
      "Saving... Done!\n",
      "#################################### Epoch: 50/50 ####################################\n",
      "[train]  Batch: 636/635 / Loss: 0.014387073453092406 / Acc: 0.9960676366496264                   \n",
      "Execution time: 400.19371366500854\n",
      "[val]  Batch: 170/169 / Loss: 0.3544024420416583 / Acc: 0.893961708394698                     \n",
      "Execution time: 38.37640619277954\n",
      "Saving... Done!\n"
     ]
    }
   ],
   "source": [
    "# Training Loop\n",
    "print(\"Starting Training Loop...\")\n",
    "trainval_loaders = {'train': train_loader, 'val': val_loader}\n",
    "trainval_sizes = {x: len(trainval_loaders[x].dataset) for x in ['train', 'val']}\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    print(\"#################################### Epoch: {}/{} ####################################\".format(epoch + 1, num_epochs))\n",
    "    for phase in ['train', 'val']:\n",
    "        start_time = time.time()\n",
    "\n",
    "        iters = 0\n",
    "        running_loss = 0.0\n",
    "        running_corrects = 0.0\n",
    "        total_num = 0\n",
    "\n",
    "        if phase == 'train':\n",
    "            net.train()\n",
    "            #scheduler.step()\n",
    "        else:\n",
    "            net.eval()\n",
    "\n",
    "        for inputs, labels in trainval_loaders[phase]:\n",
    "            inputs = Variable(inputs, requires_grad=True).to(device=device, dtype=torch.float)\n",
    "            labels = Variable(labels).to(device=device, dtype=torch.int64)\n",
    "            b_size = inputs.size(0)\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            if phase == 'train':\n",
    "                outputs = net(inputs)\n",
    "            else:\n",
    "                with torch.no_grad():\n",
    "                    outputs = net(inputs)\n",
    "\n",
    "            probs = nn.Softmax(dim=1)(outputs)\n",
    "            preds = torch.max(probs, 1)[1]\n",
    "            loss = criterion(outputs, labels)\n",
    "\n",
    "            if phase == 'train':\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "            iters += 1\n",
    "                \n",
    "            total_num += b_size\n",
    "        \n",
    "            running_loss += loss.item() * inputs.size(0)\n",
    "            running_corrects += torch.sum(preds == labels.data)\n",
    "            \n",
    "            epoch_loss = running_loss / total_num\n",
    "            epoch_acc = running_corrects.double() / total_num\n",
    "            \n",
    "            print(\"\\r[{0}]  Batch: {1}/{2} / Loss: {3} / Acc: {4}\"\n",
    "                  .format(phase, iters, trainval_sizes[phase]//batch_size, epoch_loss, epoch_acc), end='                   ')\n",
    "\n",
    "        if phase == 'train':\n",
    "            train_loss_list.append(epoch_loss)\n",
    "            train_accr_list.append(epoch_acc)\n",
    "        else:\n",
    "            val_loss_list.append(epoch_loss)\n",
    "            val_accr_list.append(epoch_acc)\n",
    "\n",
    "        \n",
    "        stop_time = time.time()\n",
    "        print()\n",
    "        print(\"Execution time: \" + str(stop_time - start_time))\n",
    "    \n",
    "    save(net, epoch+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kUXNs5H0d1Qs"
   },
   "source": [
    "# **4. Save / Load the model**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SDbDiUFhITpN"
   },
   "source": [
    "## 4.1 Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 855,
     "status": "ok",
     "timestamp": 1573317647249,
     "user": {
      "displayName": "친절배려양보",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDFkwnEU7dwPLhibVEPf81KRoxg6Fp6WyoycQ8S_w=s64",
      "userId": "13918714700772710322"
     },
     "user_tz": -540
    },
    "id": "D3kjA054Icw4",
    "outputId": "f6917cf4-e28e-46f6-a05c-fe37ff1fcce5"
   },
   "outputs": [],
   "source": [
    "def save(net, epoch):\n",
    "    global loss_list, optimizer\n",
    "    print('Saving...', end=' ')\n",
    "    state = {\n",
    "        'epoch': current_epoch,\n",
    "        'net': net.state_dict(),\n",
    "        'train_loss_list' : train_loss_list,\n",
    "        'val_loss_list' : val_loss_list,\n",
    "        'train_accr_list' : train_accr_list,\n",
    "        'val_accr_list' : val_accr_list,\n",
    "        'optimizer': optimizer.state_dict(),\n",
    "        }\n",
    "    if not os.path.isdir('checkpoint'):\n",
    "        os.mkdir('checkpoint')\n",
    "    torch.save(state, './checkpoint/ckpt.pth')\n",
    "    print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(\"Current epoch : {}\".format(current_epoch))\n",
    "save(net, current_epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "USeQMBDZIzR2"
   },
   "source": [
    "## 4.2 Load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 55
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1179,
     "status": "ok",
     "timestamp": 1573317649116,
     "user": {
      "displayName": "친절배려양보",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDFkwnEU7dwPLhibVEPf81KRoxg6Fp6WyoycQ8S_w=s64",
      "userId": "13918714700772710322"
     },
     "user_tz": -540
    },
    "id": "qK2bR51IIzdt",
    "outputId": "cfb4dd47-b2c0-469a-99ea-551d218e11a9",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def load(netG):\n",
    "    global current_epoch, train_loss_list, val_loss_list, train_accr_list, val_accr_list, optimizer\n",
    "    print('Loading...', end=' ')\n",
    "    assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'\n",
    "    checkpoint = torch.load('./checkpoint/ckpt.pth')\n",
    "    net.load_state_dict(checkpoint['net'], strict=True)\n",
    "    current_epoch = checkpoint['epoch']\n",
    "    train_loss_list = checkpoint['train_loss_list'],\n",
    "    val_loss_list = checkpoint['val_loss_list'],\n",
    "    train_accr_list = checkpoint['train_accr_list'],\n",
    "    val_accr_list = checkpoint['val_accr_list'],\n",
    "    optimizer.load_state_dict(checkpoint['optimizer']),\n",
    "    print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading... Done!\n"
     ]
    }
   ],
   "source": [
    "load(net)\n",
    "train_loss_list = train_loss_list[0]\n",
    "train_accr_list = train_accr_list[0]\n",
    "val_loss_list = val_loss_list[0]\n",
    "val_accr_list = val_accr_list[0]"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "main.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:pytorch] *",
   "language": "python",
   "name": "conda-env-pytorch-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}