C&W_Vs_Distilled_model_ipynb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "C&W_Vs_Distilled_model.ipynb.ipynb",
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a160edc45f79431db7e5cacd5a024f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2651d00e02964a7f818a8f64d6b6bb2c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_20e92d0d853648c599fbbfc888aa28db",
              "IPY_MODEL_42585d1a8e314720a9e9f26645a05c8e",
              "IPY_MODEL_53614b0d4d1e4d2fa3faff4b00b28ac6"
            ]
          }
        },
        "2651d00e02964a7f818a8f64d6b6bb2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20e92d0d853648c599fbbfc888aa28db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d8b545357aa144b6b0988fca1031b1cd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9cab07aa0d8744adbfda73d549069a9f"
          }
        },
        "42585d1a8e314720a9e9f26645a05c8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e5dfba2a194e4d3c826dcb84693cb50a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 170498071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 170498071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_222e986e94ff49178d3422625a920bf3"
          }
        },
        "53614b0d4d1e4d2fa3faff4b00b28ac6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7be8d3eb833b47f683e54f19ff667c84",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170499072/? [00:03&lt;00:00, 37155659.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11c9daa4ef8a45f0b59bf8cbe94ddfb1"
          }
        },
        "d8b545357aa144b6b0988fca1031b1cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9cab07aa0d8744adbfda73d549069a9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5dfba2a194e4d3c826dcb84693cb50a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "222e986e94ff49178d3422625a920bf3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7be8d3eb833b47f683e54f19ff667c84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11c9daa4ef8a45f0b59bf8cbe94ddfb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Of5XR-tk-W3m",
        "outputId": "a20440bc-2401-4946-a487-849157106d08"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'assignment3-21-robustnet-1'...\n",
            "remote: Enumerating objects: 49, done.\u001b[K\n",
            "remote: Counting objects: 100% (49/49), done.\u001b[K\n",
            "remote: Compressing objects: 100% (40/40), done.\u001b[K\n",
            "remote: Total 49 (delta 17), reused 23 (delta 5), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (49/49), done.\n"
          ]
        }
      ],
      "source": [
        "#import project"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/assignment3-21-robustnet-1/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8TYPByy6-Z1c",
        "outputId": "4b71c96a-30a8-45e6-85c9-48f60241b62e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/assignment3-21-robustnet-1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JW_RYltB-gzq",
        "outputId": "4f91f983-71d2-45c1-c492-411cf3fafad5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 1)) (1.10.0+cu111)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 2)) (0.11.1+cu111)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 3)) (1.19.5)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 4)) (7.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 5)) (3.2.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->-r requirements.txt (line 1)) (3.10.0.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->-r requirements.txt (line 5)) (0.11.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->-r requirements.txt (line 5)) (3.0.6)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->-r requirements.txt (line 5)) (1.3.2)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->-r requirements.txt (line 5)) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->-r requirements.txt (line 5)) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#The attack is in the file carlini_wagner_l2.py\n",
        "from carlini_wagner_l2 import (\n",
        "   carlini_wagner_l2,\n",
        ")"
      ],
      "metadata": {
        "id": "VdTsQ3fh6a5b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!/usr/bin/env python3 \n",
        "import os\n",
        "import argparse\n",
        "import torch\n",
        "import torchvision\n",
        "import numpy as np\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch.utils.data\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "from attacks import *\n",
        "\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "\n",
        "valid_size = 1024 \n",
        "batch_size = 32 \n",
        "criterion = nn.NLLLoss()\n",
        "'''Basic neural network architecture (from pytorch doc).'''\n",
        "class Net(nn.Module):\n",
        "\n",
        "    model_file=\"models/default_model.pth\"\n",
        "    '''This file will be loaded to test your model. Use --model-file to load/store a different model.'''\n",
        "    \n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
        "        self.pool = nn.MaxPool2d(2, 2)\n",
        "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
        "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
        "        self.fc2 = nn.Linear(120, 84)\n",
        "        self.fc3 = nn.Linear(84, 10)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = self.pool(F.relu(self.conv1(x)))\n",
        "        x = self.pool(F.relu(self.conv2(x)))\n",
        "        x = torch.flatten(x, 1)\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        x = F.log_softmax(x, dim=1)\n",
        "        return x\n",
        "    \n",
        "\n",
        "    \n",
        "\n",
        "    def save(self, model_file):\n",
        "        '''Helper function, use it to save the model weights after training.'''\n",
        "        torch.save(self.state_dict(), model_file)\n",
        "\n",
        "    def load(self, model_file):\n",
        "        self.load_state_dict(torch.load(model_file, map_location=torch.device(device)))\n",
        "\n",
        "        \n",
        "    def load_for_testing(self, project_dir='./'):\n",
        "        '''This function will be called automatically before testing your\n",
        "           project, and will load the model weights from the file\n",
        "           specify in Net.model_file.\n",
        "           \n",
        "           You must not change the prototype of this function. You may\n",
        "           add extra code in its body if you feel it is necessary, but\n",
        "           beware that paths of files used in this function should be\n",
        "           refered relative to the root of your project directory.\n",
        "        '''        \n",
        "        self.load(os.path.join(project_dir, Net.model_file))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "####### Testing ############\n",
        "def test_natural(net, test_loader):\n",
        "    '''Basic testing function.'''\n",
        "\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    # since we're not training, we don't need to calculate the gradients for our outputs\n",
        "    with torch.no_grad():\n",
        "        for i,data in enumerate(test_loader, 0):\n",
        "            images, labels = data[0].to(device), data[1].to(device)\n",
        "            # calculate outputs by running images through the network\n",
        "            outputs = net(images)\n",
        "            # the class with the highest energy is what we choose as prediction\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "    return 100 * correct / total\n",
        "    \n",
        "######################## Attack Testing ########################\n",
        "\n",
        "\n",
        "def test_carlini_wagner(net,test_loader):\n",
        "  correct = 0\n",
        "  for i,data in enumerate(test_loader, 0):\n",
        "            images, labels = data[0].to(device), data[1].to(device)\n",
        "            adv_data = images.clone().detach().to(device)\n",
        "            # calculate outputs by running images through the network\n",
        "            adv_data.requires_grad = True\n",
        "            outputs = net(adv_data)\n",
        "            # the class with the highest energy is what we choose as prediction\n",
        "            _, nat_pred = torch.max(outputs.data, 1)\n",
        "            if nat_pred.item() != labels.item():\n",
        "              continue\n",
        "            #if test sample is correctly predicted\n",
        "            loss = criterion(outputs, labels)\n",
        "            net.zero_grad()\n",
        "            loss.backward()\n",
        "\n",
        "            data_grad = adv_data.grad.data\n",
        "            adv_data = carlini_wagner_l2(net,images,10,targeted=False,y=nat_pred)\n",
        "            output = net(adv_data)\n",
        "            _,adv_pred = torch.max(output.data, 1)\n",
        "            if adv_pred.item() == labels.item():\n",
        "              correct+=1\n",
        "            print(\"Test num \"+str(i))\n",
        "  final_acc = correct/float(len(test_loader))\n",
        "  print(\"\\tTest Accuracy = {} / {} = {}\".format( correct, len(test_loader), final_acc))\n",
        "  return 100*final_acc\n",
        "\n",
        "\n",
        "\n",
        "##########################################\n",
        "\n",
        "def get_train_loader(dataset, valid_size=1024, batch_size=32):\n",
        "    '''Split dataset into [train:valid] and return a DataLoader for the training part.'''\n",
        "\n",
        "    indices = list(range(len(dataset)))\n",
        "    train_sampler = torch.utils.data.SubsetRandomSampler(indices[valid_size:])\n",
        "    train = torch.utils.data.DataLoader(dataset, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "    return train\n",
        "\n",
        "def get_validation_loader(dataset, valid_size=1024, batch_size=32):\n",
        "    '''Split dataset into [train:valid] and return a DataLoader for the validation part.'''\n",
        "\n",
        "    indices = list(range(len(dataset)))\n",
        "    valid_sampler = torch.utils.data.SubsetRandomSampler(indices[:valid_size])\n",
        "    valid = torch.utils.data.DataLoader(dataset, sampler=valid_sampler, batch_size=batch_size)\n",
        "\n",
        "    return valid\n",
        "net = Net()\n",
        "net.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VjJVz4_J-tg2",
        "outputId": "119db95b-0f15-4cef-a389-837e418aab26"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Net(\n",
              "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
              "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
              "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cifar = torchvision.datasets.CIFAR10('./data/', download=True, transform=transforms.ToTensor())\n",
        "valid_loader = get_validation_loader(cifar, valid_size, 1)  # batch size one for attack test otherwise put 32\n",
        "net.load(\"models/distilled_model.pth\")\n",
        "acc = test_natural(net, valid_loader)\n",
        "print(\"Model natural accuracy (valid): {}\".format(acc))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98,
          "referenced_widgets": [
            "a160edc45f79431db7e5cacd5a024f1f",
            "2651d00e02964a7f818a8f64d6b6bb2c",
            "20e92d0d853648c599fbbfc888aa28db",
            "42585d1a8e314720a9e9f26645a05c8e",
            "53614b0d4d1e4d2fa3faff4b00b28ac6",
            "d8b545357aa144b6b0988fca1031b1cd",
            "9cab07aa0d8744adbfda73d549069a9f",
            "e5dfba2a194e4d3c826dcb84693cb50a",
            "222e986e94ff49178d3422625a920bf3",
            "7be8d3eb833b47f683e54f19ff667c84",
            "11c9daa4ef8a45f0b59bf8cbe94ddfb1"
          ]
        },
        "id": "V2ZCbAT0_F6J",
        "outputId": "e7c01a90-cd92-4571-ba2f-122078e3173c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a160edc45f79431db7e5cacd5a024f1f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data/\n",
            "Model natural accuracy (valid): 60.15625\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "acc = test_carlini_wagner(net, valid_loader)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7bu6sRPB_PCn",
        "outputId": "890acbc6-ff85-4f87-8715-82af532ce075"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test num 1 label:2 predicted 4\n",
            "Test num 2 label:9 predicted 1\n",
            "Test num 4 label:6 predicted 4\n",
            "Test num 5 label:6 predicted 3\n",
            "Test num 6 label:1 predicted 0\n",
            "Test num 8 label:3 predicted 4\n",
            "Test num 9 label:5 predicted 4\n",
            "Test num 10 label:2 predicted 6\n",
            "Test num 11 label:5 predicted 3\n",
            "Test num 12 label:9 predicted 1\n",
            "Test num 14 label:7 predicted 5\n",
            "Test num 16 label:2 predicted 5\n",
            "Test num 18 label:4 predicted 6\n",
            "Test num 19 label:1 predicted 8\n",
            "Test num 20 label:7 predicted 4\n",
            "Test num 21 label:5 predicted 3\n",
            "Test num 23 label:1 predicted 2\n",
            "Test num 24 label:1 predicted 9\n",
            "Test num 25 label:9 predicted 3\n",
            "Test num 26 label:5 predicted 2\n",
            "Test num 28 label:0 predicted 8\n",
            "Test num 29 label:8 predicted 9\n",
            "Test num 31 label:4 predicted 2\n",
            "Test num 33 label:8 predicted 1\n",
            "Test num 38 label:2 predicted 0\n",
            "Test num 40 label:6 predicted 2\n",
            "Test num 41 label:6 predicted 8\n",
            "Test num 42 label:1 predicted 9\n",
            "Test num 43 label:9 predicted 8\n",
            "Test num 44 label:2 predicted 4\n",
            "Test num 45 label:1 predicted 9\n",
            "Test num 46 label:5 predicted 3\n",
            "Test num 47 label:0 predicted 8\n",
            "Test num 48 label:0 predicted 2\n",
            "Test num 49 label:9 predicted 1\n",
            "Test num 54 label:2 predicted 4\n",
            "Test num 55 label:6 predicted 2\n",
            "Test num 56 label:8 predicted 9\n",
            "Test num 57 label:0 predicted 4\n",
            "Test num 58 label:8 predicted 6\n",
            "Test num 59 label:9 predicted 1\n",
            "Test num 60 label:4 predicted 5\n",
            "Test num 62 label:3 predicted 6\n",
            "Test num 63 label:5 predicted 7\n",
            "Test num 65 label:1 predicted 9\n",
            "Test num 66 label:1 predicted 9\n",
            "Test num 67 label:2 predicted 3\n",
            "Test num 68 label:6 predicted 3\n",
            "Test num 69 label:5 predicted 3\n",
            "Test num 70 label:5 predicted 4\n",
            "Test num 71 label:7 predicted 4\n",
            "Test num 72 label:7 predicted 3\n",
            "Test num 73 label:0 predicted 1\n",
            "Test num 74 label:7 predicted 4\n",
            "Test num 75 label:0 predicted 1\n",
            "Test num 76 label:2 predicted 4\n",
            "Test num 77 label:9 predicted 4\n",
            "Test num 78 label:6 predicted 3\n",
            "Test num 80 label:8 predicted 9\n",
            "Test num 81 label:4 predicted 8\n",
            "Test num 83 label:2 predicted 4\n",
            "Test num 84 label:9 predicted 1\n",
            "Test num 85 label:8 predicted 9\n",
            "Test num 87 label:8 predicted 0\n",
            "Test num 88 label:1 predicted 9\n",
            "Test num 89 label:6 predicted 3\n",
            "Test num 91 label:9 predicted 1\n",
            "Test num 92 label:9 predicted 0\n",
            "Test num 93 label:1 predicted 9\n",
            "Test num 94 label:8 predicted 0\n",
            "Test num 95 label:0 predicted 8\n",
            "Test num 96 label:6 predicted 4\n",
            "Test num 98 label:8 predicted 0\n",
            "Test num 99 label:4 predicted 7\n",
            "Test num 100 label:8 predicted 0\n",
            "Test num 102 label:7 predicted 5\n",
            "Test num 103 label:3 predicted 2\n",
            "Test num 104 label:6 predicted 3\n",
            "Test num 105 label:9 predicted 1\n",
            "Test num 106 label:2 predicted 6\n",
            "Test num 107 label:6 predicted 4\n",
            "Test num 108 label:8 predicted 1\n",
            "Test num 109 label:3 predicted 5\n",
            "Test num 112 label:7 predicted 4\n",
            "Test num 113 label:6 predicted 3\n",
            "Test num 115 label:9 predicted 3\n",
            "Test num 117 label:7 predicted 2\n",
            "Test num 119 label:2 predicted 9\n",
            "Test num 120 label:1 predicted 9\n",
            "Test num 121 label:4 predicted 3\n",
            "Test num 123 label:2 predicted 5\n",
            "Test num 124 label:7 predicted 4\n",
            "Test num 125 label:9 predicted 1\n",
            "Test num 127 label:8 predicted 0\n",
            "Test num 129 label:7 predicted 0\n",
            "Test num 130 label:0 predicted 2\n",
            "Test num 133 label:0 predicted 1\n",
            "Test num 134 label:8 predicted 9\n",
            "Test num 135 label:1 predicted 9\n",
            "Test num 136 label:6 predicted 2\n",
            "Test num 137 label:5 predicted 2\n",
            "Test num 138 label:3 predicted 8\n",
            "Test num 139 label:6 predicted 4\n",
            "Test num 140 label:1 predicted 8\n",
            "Test num 141 label:5 predicted 3\n",
            "Test num 142 label:6 predicted 2\n",
            "Test num 143 label:9 predicted 3\n",
            "Test num 144 label:4 predicted 8\n",
            "Test num 145 label:3 predicted 4\n",
            "Test num 146 label:1 predicted 9\n",
            "Test num 148 label:8 predicted 9\n",
            "Test num 149 label:3 predicted 4\n",
            "Test num 150 label:6 predicted 2\n",
            "Test num 151 label:4 predicted 3\n",
            "Test num 152 label:5 predicted 3\n",
            "Test num 155 label:0 predicted 8\n",
            "Test num 156 label:3 predicted 5\n",
            "Test num 157 label:9 predicted 1\n",
            "Test num 159 label:8 predicted 1\n",
            "Test num 160 label:0 predicted 1\n",
            "Test num 161 label:1 predicted 9\n",
            "Test num 162 label:2 predicted 4\n",
            "Test num 163 label:0 predicted 2\n",
            "Test num 164 label:1 predicted 0\n",
            "Test num 166 label:2 predicted 4\n",
            "Test num 167 label:1 predicted 9\n",
            "Test num 168 label:4 predicted 2\n",
            "Test num 170 label:2 predicted 4\n",
            "Test num 173 label:7 predicted 4\n",
            "Test num 176 label:8 predicted 0\n",
            "Test num 179 label:0 predicted 1\n",
            "Test num 181 label:8 predicted 0\n",
            "Test num 183 label:1 predicted 9\n",
            "Test num 184 label:8 predicted 5\n",
            "Test num 186 label:8 predicted 0\n",
            "Test num 187 label:7 predicted 4\n",
            "Test num 188 label:7 predicted 4\n",
            "Test num 192 label:5 predicted 3\n",
            "Test num 194 label:2 predicted 4\n",
            "Test num 195 label:8 predicted 9\n",
            "Test num 198 label:4 predicted 2\n",
            "Test num 199 label:0 predicted 4\n",
            "Test num 202 label:1 predicted 9\n",
            "Test num 203 label:4 predicted 2\n",
            "Test num 204 label:9 predicted 1\n",
            "Test num 206 label:5 predicted 3\n",
            "Test num 207 label:1 predicted 8\n",
            "Test num 210 label:2 predicted 4\n",
            "Test num 211 label:4 predicted 7\n",
            "Test num 212 label:1 predicted 0\n",
            "Test num 213 label:3 predicted 6\n",
            "Test num 214 label:1 predicted 8\n",
            "Test num 220 label:1 predicted 0\n",
            "Test num 221 label:3 predicted 5\n",
            "Test num 229 label:0 predicted 4\n",
            "Test num 230 label:0 predicted 4\n",
            "Test num 231 label:4 predicted 0\n",
            "Test num 232 label:0 predicted 8\n",
            "Test num 233 label:6 predicted 3\n",
            "Test num 234 label:9 predicted 6\n",
            "Test num 238 label:7 predicted 5\n",
            "Test num 242 label:4 predicted 2\n",
            "Test num 244 label:4 predicted 5\n",
            "Test num 245 label:8 predicted 9\n",
            "Test num 246 label:1 predicted 8\n",
            "Test num 247 label:4 predicted 2\n",
            "Test num 248 label:9 predicted 8\n",
            "Test num 249 label:9 predicted 1\n",
            "Test num 253 label:3 predicted 5\n",
            "Test num 256 label:6 predicted 4\n",
            "Test num 258 label:0 predicted 2\n",
            "Test num 259 label:6 predicted 2\n",
            "Test num 262 label:8 predicted 0\n",
            "Test num 266 label:9 predicted 1\n",
            "Test num 268 label:6 predicted 2\n",
            "Test num 270 label:1 predicted 9\n",
            "Test num 275 label:4 predicted 9\n",
            "Test num 277 label:9 predicted 8\n",
            "Test num 283 label:9 predicted 7\n",
            "Test num 284 label:8 predicted 0\n",
            "Test num 285 label:2 predicted 5\n",
            "Test num 287 label:4 predicted 2\n",
            "Test num 289 label:6 predicted 2\n",
            "Test num 290 label:8 predicted 0\n",
            "Test num 292 label:4 predicted 3\n",
            "Test num 294 label:1 predicted 9\n",
            "Test num 297 label:4 predicted 2\n",
            "Test num 299 label:5 predicted 2\n",
            "Test num 300 label:1 predicted 9\n",
            "Test num 301 label:6 predicted 3\n",
            "Test num 303 label:1 predicted 8\n",
            "Test num 304 label:1 predicted 9\n",
            "Test num 306 label:9 predicted 1\n",
            "Test num 308 label:4 predicted 2\n",
            "Test num 309 label:8 predicted 1\n",
            "Test num 310 label:8 predicted 0\n",
            "Test num 313 label:8 predicted 0\n",
            "Test num 314 label:6 predicted 3\n",
            "Test num 315 label:7 predicted 4\n",
            "Test num 317 label:8 predicted 1\n",
            "Test num 318 label:4 predicted 6\n",
            "Test num 319 label:8 predicted 0\n",
            "Test num 321 label:0 predicted 8\n",
            "Test num 323 label:1 predicted 9\n",
            "Test num 324 label:9 predicted 7\n",
            "Test num 325 label:7 predicted 4\n",
            "Test num 328 label:1 predicted 9\n",
            "Test num 332 label:7 predicted 4\n",
            "Test num 334 label:2 predicted 4\n",
            "Test num 338 label:9 predicted 1\n",
            "Test num 339 label:5 predicted 7\n",
            "Test num 340 label:0 predicted 2\n",
            "Test num 342 label:8 predicted 0\n",
            "Test num 344 label:3 predicted 5\n",
            "Test num 346 label:2 predicted 4\n",
            "Test num 347 label:9 predicted 1\n",
            "Test num 348 label:4 predicted 2\n",
            "Test num 349 label:9 predicted 1\n",
            "Test num 351 label:8 predicted 0\n",
            "Test num 352 label:1 predicted 9\n",
            "Test num 353 label:8 predicted 0\n",
            "Test num 356 label:3 predicted 5\n",
            "Test num 357 label:1 predicted 9\n",
            "Test num 358 label:7 predicted 3\n",
            "Test num 360 label:1 predicted 3\n",
            "Test num 362 label:9 predicted 6\n",
            "Test num 365 label:7 predicted 3\n",
            "Test num 366 label:5 predicted 3\n",
            "Test num 368 label:8 predicted 0\n",
            "Test num 371 label:5 predicted 3\n",
            "Test num 372 label:4 predicted 3\n",
            "Test num 373 label:5 predicted 3\n",
            "Test num 374 label:1 predicted 9\n",
            "Test num 375 label:1 predicted 9\n",
            "Test num 376 label:1 predicted 8\n",
            "Test num 377 label:6 predicted 4\n",
            "Test num 378 label:2 predicted 5\n",
            "Test num 379 label:1 predicted 9\n",
            "Test num 380 label:9 predicted 1\n",
            "Test num 381 label:9 predicted 1\n",
            "Test num 382 label:7 predicted 4\n",
            "Test num 383 label:8 predicted 1\n",
            "Test num 387 label:0 predicted 4\n",
            "Test num 389 label:8 predicted 5\n",
            "Test num 392 label:2 predicted 6\n",
            "Test num 396 label:3 predicted 5\n",
            "Test num 399 label:8 predicted 2\n",
            "Test num 400 label:1 predicted 9\n",
            "Test num 401 label:2 predicted 6\n",
            "Test num 402 label:3 predicted 5\n",
            "Test num 403 label:6 predicted 3\n",
            "Test num 404 label:2 predicted 4\n",
            "Test num 405 label:0 predicted 8\n",
            "Test num 406 label:7 predicted 4\n",
            "Test num 407 label:0 predicted 2\n",
            "Test num 408 label:2 predicted 6\n",
            "Test num 409 label:8 predicted 2\n",
            "Test num 412 label:0 predicted 3\n",
            "Test num 413 label:6 predicted 4\n",
            "Test num 414 label:8 predicted 0\n",
            "Test num 416 label:3 predicted 7\n",
            "Test num 417 label:1 predicted 9\n",
            "Test num 418 label:4 predicted 2\n",
            "Test num 419 label:2 predicted 4\n",
            "Test num 420 label:6 predicted 3\n",
            "Test num 422 label:6 predicted 1\n",
            "Test num 423 label:3 predicted 5\n",
            "Test num 424 label:9 predicted 1\n",
            "Test num 427 label:4 predicted 6\n",
            "Test num 428 label:0 predicted 6\n",
            "Test num 431 label:8 predicted 1\n",
            "Test num 435 label:1 predicted 9\n",
            "Test num 436 label:9 predicted 8\n",
            "Test num 438 label:5 predicted 3\n",
            "Test num 440 label:7 predicted 4\n",
            "Test num 441 label:4 predicted 7\n",
            "Test num 442 label:7 predicted 4\n",
            "Test num 443 label:0 predicted 2\n",
            "Test num 444 label:4 predicted 2\n",
            "Test num 445 label:8 predicted 0\n",
            "Test num 447 label:6 predicted 3\n",
            "Test num 448 label:2 predicted 0\n",
            "Test num 449 label:8 predicted 0\n",
            "Test num 450 label:6 predicted 4\n",
            "Test num 452 label:6 predicted 3\n",
            "Test num 453 label:8 predicted 0\n",
            "Test num 454 label:7 predicted 4\n",
            "Test num 455 label:1 predicted 9\n",
            "Test num 456 label:2 predicted 4\n",
            "Test num 457 label:1 predicted 9\n",
            "Test num 458 label:8 predicted 9\n",
            "Test num 459 label:2 predicted 4\n",
            "Test num 460 label:0 predicted 8\n",
            "Test num 464 label:2 predicted 3\n",
            "Test num 466 label:5 predicted 6\n",
            "Test num 467 label:1 predicted 7\n",
            "Test num 471 label:2 predicted 3\n",
            "Test num 472 label:7 predicted 4\n",
            "Test num 473 label:0 predicted 8\n",
            "Test num 475 label:1 predicted 9\n",
            "Test num 476 label:7 predicted 4\n",
            "Test num 478 label:1 predicted 9\n",
            "Test num 480 label:1 predicted 6\n",
            "Test num 482 label:4 predicted 6\n",
            "Test num 483 label:3 predicted 5\n",
            "Test num 484 label:7 predicted 4\n",
            "Test num 487 label:4 predicted 2\n",
            "Test num 488 label:8 predicted 0\n",
            "Test num 489 label:1 predicted 9\n",
            "Test num 493 label:8 predicted 9\n",
            "Test num 494 label:3 predicted 5\n",
            "Test num 495 label:0 predicted 4\n",
            "Test num 496 label:2 predicted 0\n",
            "Test num 497 label:6 predicted 2\n",
            "Test num 500 label:5 predicted 3\n",
            "Test num 501 label:9 predicted 1\n",
            "Test num 502 label:8 predicted 2\n",
            "Test num 505 label:8 predicted 4\n",
            "Test num 510 label:4 predicted 3\n",
            "Test num 511 label:8 predicted 2\n",
            "Test num 513 label:0 predicted 8\n",
            "Test num 520 label:6 predicted 2\n",
            "Test num 523 label:6 predicted 2\n",
            "Test num 528 label:7 predicted 4\n",
            "Test num 529 label:7 predicted 0\n",
            "Test num 530 label:4 predicted 6\n",
            "Test num 532 label:4 predicted 3\n",
            "Test num 534 label:5 predicted 3\n",
            "Test num 536 label:0 predicted 8\n",
            "Test num 537 label:3 predicted 4\n",
            "Test num 538 label:4 predicted 6\n",
            "Test num 541 label:5 predicted 3\n",
            "Test num 542 label:4 predicted 7\n",
            "Test num 543 label:1 predicted 6\n",
            "Test num 544 label:8 predicted 9\n",
            "Test num 545 label:0 predicted 1\n",
            "Test num 547 label:8 predicted 0\n",
            "Test num 548 label:6 predicted 4\n",
            "Test num 549 label:1 predicted 9\n",
            "Test num 552 label:1 predicted 8\n",
            "Test num 553 label:2 predicted 0\n",
            "Test num 555 label:6 predicted 0\n",
            "Test num 556 label:7 predicted 4\n",
            "Test num 558 label:6 predicted 3\n",
            "Test num 559 label:4 predicted 2\n",
            "Test num 562 label:1 predicted 8\n",
            "Test num 564 label:3 predicted 5\n",
            "Test num 565 label:9 predicted 0\n",
            "Test num 566 label:6 predicted 3\n",
            "Test num 567 label:0 predicted 8\n",
            "Test num 568 label:4 predicted 7\n",
            "Test num 570 label:2 predicted 4\n",
            "Test num 573 label:9 predicted 0\n",
            "Test num 574 label:8 predicted 9\n",
            "Test num 576 label:7 predicted 4\n",
            "Test num 578 label:6 predicted 3\n",
            "Test num 579 label:7 predicted 5\n",
            "Test num 581 label:0 predicted 8\n",
            "Test num 585 label:9 predicted 1\n",
            "Test num 586 label:7 predicted 4\n",
            "Test num 589 label:0 predicted 2\n",
            "Test num 591 label:7 predicted 4\n",
            "Test num 592 label:2 predicted 4\n",
            "Test num 593 label:7 predicted 4\n",
            "Test num 594 label:4 predicted 3\n",
            "Test num 595 label:2 predicted 7\n",
            "Test num 597 label:9 predicted 7\n",
            "Test num 599 label:6 predicted 2\n",
            "Test num 601 label:0 predicted 8\n",
            "Test num 602 label:7 predicted 4\n",
            "Test num 604 label:4 predicted 6\n",
            "Test num 605 label:6 predicted 1\n",
            "Test num 608 label:5 predicted 3\n",
            "Test num 610 label:3 predicted 6\n",
            "Test num 612 label:9 predicted 8\n",
            "Test num 613 label:8 predicted 0\n",
            "Test num 615 label:1 predicted 9\n",
            "Test num 618 label:1 predicted 9\n",
            "Test num 619 label:8 predicted 4\n",
            "Test num 620 label:9 predicted 5\n",
            "Test num 623 label:7 predicted 4\n",
            "Test num 624 label:8 predicted 0\n",
            "Test num 626 label:0 predicted 4\n",
            "Test num 627 label:7 predicted 4\n",
            "Test num 628 label:5 predicted 3\n",
            "Test num 629 label:0 predicted 6\n",
            "Test num 630 label:7 predicted 9\n",
            "Test num 633 label:3 predicted 8\n",
            "Test num 634 label:6 predicted 3\n",
            "Test num 635 label:7 predicted 5\n",
            "Test num 636 label:6 predicted 3\n",
            "Test num 639 label:2 predicted 0\n",
            "Test num 640 label:3 predicted 5\n",
            "Test num 642 label:2 predicted 3\n",
            "Test num 644 label:1 predicted 9\n",
            "Test num 645 label:1 predicted 8\n",
            "Test num 647 label:7 predicted 3\n",
            "Test num 648 label:4 predicted 5\n",
            "Test num 649 label:0 predicted 8\n",
            "Test num 650 label:9 predicted 1\n",
            "Test num 652 label:0 predicted 1\n",
            "Test num 653 label:7 predicted 4\n",
            "Test num 654 label:2 predicted 4\n",
            "Test num 656 label:9 predicted 1\n",
            "Test num 658 label:6 predicted 9\n",
            "Test num 659 label:6 predicted 4\n",
            "Test num 660 label:6 predicted 3\n",
            "Test num 662 label:9 predicted 1\n",
            "Test num 665 label:5 predicted 7\n",
            "Test num 666 label:4 predicted 3\n",
            "Test num 667 label:4 predicted 0\n",
            "Test num 669 label:6 predicted 4\n",
            "Test num 670 label:0 predicted 2\n",
            "Test num 671 label:0 predicted 4\n",
            "Test num 672 label:1 predicted 9\n",
            "Test num 677 label:8 predicted 0\n",
            "Test num 678 label:8 predicted 0\n",
            "Test num 683 label:8 predicted 0\n",
            "Test num 684 label:6 predicted 3\n",
            "Test num 685 label:5 predicted 3\n",
            "Test num 687 label:4 predicted 3\n",
            "Test num 689 label:5 predicted 3\n",
            "Test num 690 label:2 predicted 3\n",
            "Test num 691 label:5 predicted 3\n",
            "Test num 692 label:7 predicted 2\n",
            "Test num 693 label:1 predicted 9\n",
            "Test num 694 label:1 predicted 9\n",
            "Test num 699 label:9 predicted 5\n",
            "Test num 701 label:8 predicted 9\n",
            "Test num 703 label:0 predicted 8\n",
            "Test num 704 label:7 predicted 4\n",
            "Test num 705 label:6 predicted 3\n",
            "Test num 707 label:5 predicted 7\n",
            "Test num 708 label:1 predicted 0\n",
            "Test num 710 label:7 predicted 4\n",
            "Test num 711 label:0 predicted 2\n",
            "Test num 712 label:7 predicted 4\n",
            "Test num 713 label:9 predicted 1\n",
            "Test num 716 label:4 predicted 2\n",
            "Test num 717 label:4 predicted 3\n",
            "Test num 718 label:6 predicted 3\n",
            "Test num 720 label:6 predicted 2\n",
            "Test num 722 label:7 predicted 4\n",
            "Test num 723 label:6 predicted 4\n",
            "Test num 725 label:1 predicted 9\n",
            "Test num 726 label:6 predicted 2\n",
            "Test num 728 label:9 predicted 1\n",
            "Test num 729 label:7 predicted 4\n",
            "Test num 730 label:2 predicted 5\n",
            "Test num 732 label:9 predicted 8\n",
            "Test num 734 label:2 predicted 4\n",
            "Test num 735 label:4 predicted 3\n",
            "Test num 736 label:1 predicted 8\n",
            "Test num 738 label:5 predicted 3\n",
            "Test num 739 label:3 predicted 6\n",
            "Test num 741 label:3 predicted 5\n",
            "Test num 742 label:8 predicted 0\n",
            "Test num 743 label:9 predicted 4\n",
            "Test num 744 label:8 predicted 0\n",
            "Test num 745 label:9 predicted 7\n",
            "Test num 746 label:9 predicted 1\n",
            "Test num 747 label:7 predicted 9\n",
            "Test num 748 label:8 predicted 4\n",
            "Test num 751 label:6 predicted 3\n",
            "Test num 752 label:4 predicted 7\n",
            "Test num 753 label:8 predicted 0\n",
            "Test num 754 label:8 predicted 9\n",
            "Test num 755 label:2 predicted 5\n",
            "Test num 756 label:8 predicted 9\n",
            "Test num 757 label:6 predicted 3\n",
            "Test num 766 label:1 predicted 9\n",
            "Test num 767 label:4 predicted 2\n",
            "Test num 770 label:0 predicted 8\n",
            "Test num 771 label:9 predicted 8\n",
            "Test num 772 label:8 predicted 0\n",
            "Test num 775 label:5 predicted 3\n",
            "Test num 777 label:5 predicted 3\n",
            "Test num 778 label:1 predicted 8\n",
            "Test num 779 label:1 predicted 9\n",
            "Test num 780 label:1 predicted 8\n",
            "Test num 781 label:4 predicted 2\n",
            "Test num 782 label:4 predicted 7\n",
            "Test num 783 label:4 predicted 2\n",
            "Test num 784 label:4 predicted 5\n",
            "Test num 787 label:6 predicted 3\n",
            "Test num 789 label:6 predicted 9\n",
            "Test num 790 label:7 predicted 5\n",
            "Test num 792 label:3 predicted 7\n",
            "Test num 793 label:9 predicted 0\n",
            "Test num 795 label:9 predicted 1\n",
            "Test num 796 label:1 predicted 9\n",
            "Test num 797 label:9 predicted 1\n",
            "Test num 800 label:0 predicted 9\n",
            "Test num 802 label:9 predicted 1\n",
            "Test num 803 label:8 predicted 9\n",
            "Test num 805 label:6 predicted 4\n",
            "Test num 806 label:8 predicted 1\n",
            "Test num 808 label:2 predicted 4\n",
            "Test num 810 label:2 predicted 4\n",
            "Test num 811 label:6 predicted 4\n",
            "Test num 813 label:1 predicted 8\n",
            "Test num 819 label:9 predicted 1\n",
            "Test num 822 label:1 predicted 9\n",
            "Test num 828 label:8 predicted 5\n",
            "Test num 829 label:6 predicted 3\n",
            "Test num 830 label:9 predicted 3\n",
            "Test num 832 label:0 predicted 4\n",
            "Test num 833 label:9 predicted 8\n",
            "Test num 834 label:7 predicted 2\n",
            "Test num 835 label:2 predicted 8\n",
            "Test num 836 label:2 predicted 0\n",
            "Test num 838 label:3 predicted 6\n",
            "Test num 839 label:5 predicted 2\n",
            "Test num 841 label:5 predicted 7\n",
            "Test num 842 label:3 predicted 4\n",
            "Test num 845 label:6 predicted 2\n",
            "Test num 846 label:6 predicted 4\n",
            "Test num 848 label:1 predicted 8\n",
            "Test num 849 label:9 predicted 1\n",
            "Test num 852 label:7 predicted 4\n",
            "Test num 853 label:0 predicted 8\n",
            "Test num 854 label:0 predicted 2\n",
            "Test num 857 label:1 predicted 9\n",
            "Test num 858 label:2 predicted 4\n",
            "Test num 859 label:7 predicted 9\n",
            "Test num 864 label:4 predicted 6\n",
            "Test num 867 label:2 predicted 6\n",
            "Test num 868 label:5 predicted 3\n",
            "Test num 870 label:1 predicted 9\n",
            "Test num 871 label:9 predicted 3\n",
            "Test num 873 label:7 predicted 4\n",
            "Test num 876 label:8 predicted 0\n",
            "Test num 877 label:9 predicted 1\n",
            "Test num 878 label:9 predicted 3\n",
            "Test num 879 label:8 predicted 0\n",
            "Test num 881 label:7 predicted 4\n",
            "Test num 883 label:0 predicted 4\n",
            "Test num 884 label:5 predicted 3\n",
            "Test num 885 label:3 predicted 6\n",
            "Test num 886 label:9 predicted 1\n",
            "Test num 888 label:9 predicted 1\n",
            "Test num 890 label:1 predicted 9\n",
            "Test num 892 label:1 predicted 9\n",
            "Test num 893 label:1 predicted 9\n",
            "Test num 895 label:4 predicted 7\n",
            "Test num 898 label:6 predicted 4\n",
            "Test num 899 label:5 predicted 2\n",
            "Test num 902 label:1 predicted 9\n",
            "Test num 903 label:2 predicted 4\n",
            "Test num 905 label:6 predicted 2\n",
            "Test num 906 label:8 predicted 9\n",
            "Test num 908 label:9 predicted 8\n",
            "Test num 909 label:9 predicted 1\n",
            "Test num 910 label:0 predicted 1\n",
            "Test num 911 label:4 predicted 5\n",
            "Test num 912 label:7 predicted 4\n",
            "Test num 914 label:8 predicted 9\n",
            "Test num 920 label:0 predicted 8\n",
            "Test num 922 label:1 predicted 9\n",
            "Test num 923 label:5 predicted 7\n",
            "Test num 926 label:1 predicted 9\n",
            "Test num 930 label:4 predicted 2\n",
            "Test num 931 label:4 predicted 7\n",
            "Test num 932 label:0 predicted 8\n",
            "Test num 934 label:7 predicted 8\n",
            "Test num 937 label:8 predicted 3\n",
            "Test num 938 label:9 predicted 0\n",
            "Test num 939 label:7 predicted 4\n",
            "Test num 941 label:3 predicted 5\n",
            "Test num 943 label:6 predicted 3\n",
            "Test num 945 label:8 predicted 2\n",
            "Test num 947 label:0 predicted 4\n",
            "Test num 949 label:2 predicted 6\n",
            "Test num 950 label:2 predicted 4\n",
            "Test num 951 label:8 predicted 3\n",
            "Test num 953 label:2 predicted 8\n",
            "Test num 955 label:6 predicted 3\n",
            "Test num 956 label:0 predicted 2\n",
            "Test num 958 label:9 predicted 1\n",
            "Test num 959 label:8 predicted 0\n",
            "Test num 960 label:6 predicted 3\n",
            "Test num 963 label:4 predicted 2\n",
            "Test num 966 label:3 predicted 6\n",
            "Test num 967 label:8 predicted 0\n",
            "Test num 971 label:6 predicted 3\n",
            "Test num 973 label:8 predicted 0\n",
            "Test num 974 label:3 predicted 5\n",
            "Test num 976 label:8 predicted 1\n",
            "Test num 977 label:3 predicted 2\n",
            "Test num 978 label:8 predicted 0\n",
            "Test num 979 label:4 predicted 0\n",
            "Test num 982 label:8 predicted 1\n",
            "Test num 983 label:6 predicted 2\n",
            "Test num 984 label:8 predicted 0\n",
            "Test num 988 label:8 predicted 9\n",
            "Test num 989 label:7 predicted 8\n",
            "Test num 990 label:9 predicted 1\n",
            "Test num 991 label:3 predicted 5\n",
            "Test num 994 label:1 predicted 0\n",
            "Test num 995 label:0 predicted 8\n",
            "Test num 996 label:4 predicted 3\n",
            "Test num 997 label:1 predicted 9\n",
            "Test num 998 label:8 predicted 0\n",
            "Test num 999 label:7 predicted 0\n",
            "Test num 1000 label:1 predicted 9\n",
            "Test num 1002 label:1 predicted 9\n",
            "Test num 1004 label:0 predicted 1\n",
            "Test num 1007 label:3 predicted 6\n",
            "Test num 1012 label:2 predicted 4\n",
            "Test num 1013 label:2 predicted 4\n",
            "Test num 1014 label:0 predicted 2\n",
            "Test num 1015 label:6 predicted 3\n",
            "Test num 1016 label:6 predicted 3\n",
            "Test num 1018 label:0 predicted 8\n",
            "Test num 1019 label:7 predicted 5\n",
            "Test num 1022 label:4 predicted 6\n",
            "\tTest Accuracy = 0 / 1024 = 0.0\n"
          ]
        }
      ]
    }
  ]
}