exercise_870.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Classification using a 7-Layer Neural Network\n",
    "\n",
    "Next, we measure the performance of a rather deep (7-layered) multi-layer neural network on the MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train - Epoch 1, Batch: 0, Loss: 2.303169\n",
      "Train - Epoch 1, Batch: 10, Loss: 1.753631\n",
      "Train - Epoch 1, Batch: 20, Loss: 1.393486\n",
      "Train - Epoch 1, Batch: 30, Loss: 1.100070\n",
      "Train - Epoch 1, Batch: 40, Loss: 0.753131\n",
      "Train - Epoch 1, Batch: 50, Loss: 0.817242\n",
      "Train - Epoch 1, Batch: 60, Loss: 0.573149\n",
      "Train - Epoch 1, Batch: 70, Loss: 0.693806\n",
      "Train - Epoch 1, Batch: 80, Loss: 0.454687\n",
      "Train - Epoch 1, Batch: 90, Loss: 0.360979\n",
      "Train - Epoch 1, Batch: 100, Loss: 0.294893\n",
      "Train - Epoch 1, Batch: 110, Loss: 0.309115\n",
      "Train - Epoch 1, Batch: 120, Loss: 0.248765\n",
      "Train - Epoch 1, Batch: 130, Loss: 0.206527\n",
      "Train - Epoch 1, Batch: 140, Loss: 0.273821\n",
      "Train - Epoch 1, Batch: 150, Loss: 0.200534\n",
      "Train - Epoch 1, Batch: 160, Loss: 0.158352\n",
      "Train - Epoch 1, Batch: 170, Loss: 0.211770\n",
      "Train - Epoch 1, Batch: 180, Loss: 0.214966\n",
      "Train - Epoch 1, Batch: 190, Loss: 0.186663\n",
      "Train - Epoch 1, Batch: 200, Loss: 0.257858\n",
      "Train - Epoch 1, Batch: 210, Loss: 0.230725\n",
      "Train - Epoch 1, Batch: 220, Loss: 0.153596\n",
      "Train - Epoch 1, Batch: 230, Loss: 0.175349\n",
      "Test Avg. Loss: 0.000165, Accuracy: 0.950600\n",
      "Train - Epoch 2, Batch: 0, Loss: 0.176835\n",
      "Train - Epoch 2, Batch: 10, Loss: 0.189274\n",
      "Train - Epoch 2, Batch: 20, Loss: 0.150204\n",
      "Train - Epoch 2, Batch: 30, Loss: 0.163925\n",
      "Train - Epoch 2, Batch: 40, Loss: 0.211774\n",
      "Train - Epoch 2, Batch: 50, Loss: 0.162938\n",
      "Train - Epoch 2, Batch: 60, Loss: 0.171201\n",
      "Train - Epoch 2, Batch: 70, Loss: 0.069721\n",
      "Train - Epoch 2, Batch: 80, Loss: 0.128826\n",
      "Train - Epoch 2, Batch: 90, Loss: 0.153154\n",
      "Train - Epoch 2, Batch: 100, Loss: 0.122970\n",
      "Train - Epoch 2, Batch: 110, Loss: 0.170024\n",
      "Train - Epoch 2, Batch: 120, Loss: 0.146375\n",
      "Train - Epoch 2, Batch: 130, Loss: 0.111477\n",
      "Train - Epoch 2, Batch: 140, Loss: 0.103808\n",
      "Train - Epoch 2, Batch: 150, Loss: 0.060358\n",
      "Train - Epoch 2, Batch: 160, Loss: 0.170306\n",
      "Train - Epoch 2, Batch: 170, Loss: 0.170058\n",
      "Train - Epoch 2, Batch: 180, Loss: 0.108431\n",
      "Train - Epoch 2, Batch: 190, Loss: 0.221638\n",
      "Train - Epoch 2, Batch: 200, Loss: 0.143246\n",
      "Train - Epoch 2, Batch: 210, Loss: 0.103257\n",
      "Train - Epoch 2, Batch: 220, Loss: 0.156117\n",
      "Train - Epoch 2, Batch: 230, Loss: 0.240083\n",
      "Test Avg. Loss: 0.000119, Accuracy: 0.965600\n",
      "Train - Epoch 3, Batch: 0, Loss: 0.103036\n",
      "Train - Epoch 3, Batch: 10, Loss: 0.123575\n",
      "Train - Epoch 3, Batch: 20, Loss: 0.086274\n",
      "Train - Epoch 3, Batch: 30, Loss: 0.082392\n",
      "Train - Epoch 3, Batch: 40, Loss: 0.050282\n",
      "Train - Epoch 3, Batch: 50, Loss: 0.135428\n",
      "Train - Epoch 3, Batch: 60, Loss: 0.105254\n",
      "Train - Epoch 3, Batch: 70, Loss: 0.032744\n",
      "Train - Epoch 3, Batch: 80, Loss: 0.108519\n",
      "Train - Epoch 3, Batch: 90, Loss: 0.161225\n",
      "Train - Epoch 3, Batch: 100, Loss: 0.069822\n",
      "Train - Epoch 3, Batch: 110, Loss: 0.095729\n",
      "Train - Epoch 3, Batch: 120, Loss: 0.073362\n",
      "Train - Epoch 3, Batch: 130, Loss: 0.156049\n",
      "Train - Epoch 3, Batch: 140, Loss: 0.069730\n",
      "Train - Epoch 3, Batch: 150, Loss: 0.081858\n",
      "Train - Epoch 3, Batch: 160, Loss: 0.126022\n",
      "Train - Epoch 3, Batch: 170, Loss: 0.122088\n",
      "Train - Epoch 3, Batch: 180, Loss: 0.097128\n",
      "Train - Epoch 3, Batch: 190, Loss: 0.109711\n",
      "Train - Epoch 3, Batch: 200, Loss: 0.072661\n",
      "Train - Epoch 3, Batch: 210, Loss: 0.083435\n",
      "Train - Epoch 3, Batch: 220, Loss: 0.075581\n",
      "Train - Epoch 3, Batch: 230, Loss: 0.027502\n",
      "Test Avg. Loss: 0.000096, Accuracy: 0.972500\n",
      "Train - Epoch 4, Batch: 0, Loss: 0.038074\n",
      "Train - Epoch 4, Batch: 10, Loss: 0.074823\n",
      "Train - Epoch 4, Batch: 20, Loss: 0.119210\n",
      "Train - Epoch 4, Batch: 30, Loss: 0.070250\n",
      "Train - Epoch 4, Batch: 40, Loss: 0.106562\n",
      "Train - Epoch 4, Batch: 50, Loss: 0.029538\n",
      "Train - Epoch 4, Batch: 60, Loss: 0.076894\n",
      "Train - Epoch 4, Batch: 70, Loss: 0.041506\n",
      "Train - Epoch 4, Batch: 80, Loss: 0.120401\n",
      "Train - Epoch 4, Batch: 90, Loss: 0.069571\n",
      "Train - Epoch 4, Batch: 100, Loss: 0.059365\n",
      "Train - Epoch 4, Batch: 110, Loss: 0.069996\n",
      "Train - Epoch 4, Batch: 120, Loss: 0.054353\n",
      "Train - Epoch 4, Batch: 130, Loss: 0.025855\n",
      "Train - Epoch 4, Batch: 140, Loss: 0.040052\n",
      "Train - Epoch 4, Batch: 150, Loss: 0.116047\n",
      "Train - Epoch 4, Batch: 160, Loss: 0.069517\n",
      "Train - Epoch 4, Batch: 170, Loss: 0.087004\n",
      "Train - Epoch 4, Batch: 180, Loss: 0.064960\n",
      "Train - Epoch 4, Batch: 190, Loss: 0.040938\n",
      "Train - Epoch 4, Batch: 200, Loss: 0.044485\n",
      "Train - Epoch 4, Batch: 210, Loss: 0.060270\n",
      "Train - Epoch 4, Batch: 220, Loss: 0.074513\n",
      "Train - Epoch 4, Batch: 230, Loss: 0.124009\n",
      "Test Avg. Loss: 0.000100, Accuracy: 0.972500\n"
     ]
    }
   ],
   "source": [
    "# Code adapted from https://github.com/activatedgeek/LeNet-5/\n",
    "\n",
    "from collections import OrderedDict\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision.datasets.mnist import MNIST\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "class MultilayerClassifier(nn.Module):\n",
    "    \"\"\"\n",
    "    Input - 1x32x32\n",
    "    Output - 10 (Output)\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        super(MultilayerClassifier, self).__init__()\n",
    "\n",
    "        self.fc = nn.Sequential(OrderedDict([\n",
    "            ('fc1', nn.Linear(32 * 32, 32 * 20)),\n",
    "            ('relu1', nn.ReLU()),\n",
    "            ('fc2', nn.Linear(32 * 20, 32 * 16)),\n",
    "            ('relu2', nn.ReLU()),\n",
    "            ('fc3', nn.Linear(32 * 16, 32 * 8)),\n",
    "            ('relu3', nn.ReLU()),\n",
    "            ('fc4', nn.Linear(32 * 8, 32 * 4)),\n",
    "            ('relu4', nn.ReLU()),\n",
    "            ('fc5', nn.Linear(32 * 4, 32 * 2)),\n",
    "            ('relu5', nn.ReLU()),\n",
    "            ('fc6', nn.Linear(32 * 2, 16 * 2)),\n",
    "            ('relu6', nn.ReLU()),\n",
    "            ('fc7', nn.Linear(16 * 2, 10 * 1)),\n",
    "        ]))\n",
    "\n",
    "    def forward(self, img):\n",
    "        output = img.view(-1, 32 * 32)\n",
    "        output = self.fc(output)\n",
    "        return output\n",
    "    \n",
    "\n",
    "data_train = MNIST('data/mnist',\n",
    "                   download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.Resize((32, 32)),\n",
    "                       transforms.ToTensor()]))\n",
    "data_test = MNIST('data/mnist',\n",
    "                  train=False,\n",
    "                  download=True,\n",
    "                  transform=transforms.Compose([\n",
    "                      transforms.Resize((32, 32)),\n",
    "                      transforms.ToTensor()]))\n",
    "data_train_loader = DataLoader(data_train, batch_size=256, shuffle=True, num_workers=8)\n",
    "data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8)\n",
    "\n",
    "net = MultilayerClassifier()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(net.parameters(), lr=2e-3)\n",
    "\n",
    "def train(epoch):\n",
    "    net.train()\n",
    "    loss_list, batch_list = [], []\n",
    "    for i,(images, labels) in enumerate(data_train_loader):\n",
    "        images, labels = Variable(images), Variable(labels)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        output = net(images)\n",
    "\n",
    "        loss = criterion(output, labels)\n",
    "\n",
    "        loss_list.append(loss.data.item())\n",
    "        batch_list.append(i+1)\n",
    "\n",
    "        if i % 10 == 0:\n",
    "            print('Train - Epoch %d, Batch: %d, Loss: %f' % (epoch, i, loss.data.item()))\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def test():\n",
    "    net.eval()\n",
    "    total_correct = 0\n",
    "    avg_loss = 0.0\n",
    "    for i, (images, labels) in enumerate(data_test_loader):\n",
    "        images, labels = Variable(images), Variable(labels)\n",
    "        output = net(images)\n",
    "        avg_loss += criterion(output, labels).sum()\n",
    "        pred = output.data.max(1)[1]\n",
    "        total_correct += pred.eq(labels.data.view_as(pred)).sum()\n",
    "\n",
    "    avg_loss /= len(data_test)\n",
    "    print('Test Avg. Loss: %f, Accuracy: %f' % (avg_loss.data.item(), float(total_correct) / len(data_test)))\n",
    "\n",
    "\n",
    "def train_and_test(epoch):\n",
    "    train(epoch)\n",
    "    test()\n",
    "\n",
    "\n",
    "def main():\n",
    "    for e in range(1, 5):  # Change 5 to 16 for better performance\n",
    "        train_and_test(e)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "The accuracy we get with seven layers is also really high at 97% but it's no improvement over 2 layers.  Can we do even better?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}