Copy of german_signs3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SIpaxXnQsvTE"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import load\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2514,
     "status": "ok",
     "timestamp": 1586799530930,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "nJmca26iyZET",
    "outputId": "268370c3-dcdc-45fd-84b0-eaac774382c4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6910,
     "status": "ok",
     "timestamp": 1586799538547,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "wZ59xrUVtEgd",
    "outputId": "961312a4-f2a5-4853-b077-734ceac2cb80"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len of X =  3\n",
      "<class 'numpy.ndarray'>\n",
      "(39209, 32, 32, 3)\n",
      "(39209,)\n",
      "(12630, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "#unpack data from source\n",
    "\n",
    "data = np.load('./drive/My Drive/Colab Notebooks/data/csci4622_kaggle3/train_and_test.npz')  #what kind of file is this? zipped np.array file\n",
    "type(data)\n",
    "\n",
    "X = [data[key] for key in data]  #??? why does this work? \"data\" is not a dictionary data type but a NpzFile type\n",
    "\n",
    "print('len of X = ', len(X))  #there are three separate np arrays in the list: 1) train data 2) train labels 3) test data, no labels\n",
    "print(type(X[0]))\n",
    "\n",
    "xtrain0 = X[0]\n",
    "ytrain0 = X[1]  #ytrain0 contains values between 0-42; so 43 categories\n",
    "xtest0 = X[2]   #this is test data for final kaggle submission\n",
    "\n",
    "print(xtrain0.shape)\n",
    "print(ytrain0.shape)\n",
    "print(xtest0.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4486,
     "status": "ok",
     "timestamp": 1586799538548,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "gorc7EdSaDvE",
    "outputId": "ef3b836d-3d06-43ed-fccf-01339ec437e2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(ytrain0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4203,
     "status": "ok",
     "timestamp": 1586799539740,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "fN9GbsqEwgF8",
    "outputId": "5a6ea908-a0f6-4e44-ac34-122c69e123a5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26270, 32, 32, 3)\n",
      "(26270, 43)\n",
      "(12939, 32, 32, 3)\n",
      "(12939, 43)\n"
     ]
    }
   ],
   "source": [
    "#preprocess data - normalize input; divide all data by max pixel value, 255 ; we might also consider z normalization\n",
    "\n",
    "#hot encode the labels\n",
    "ymax = np.max(ytrain0)\n",
    "ytrain = tf.keras.utils.to_categorical(ytrain0, ymax+1)\n",
    "\n",
    "maxpixel_val = np.max(xtrain0) \n",
    "xtrain = xtrain0/maxpixel_val\n",
    "xtest = xtest0/maxpixel_val  #we should shuffle before making predictions\n",
    "\n",
    "x_tr, x_tst, y_tr, y_tst = train_test_split(xtrain, ytrain, test_size = 0.33,shuffle = True)  #results overfit if data is not shuffled!!!\n",
    "print(x_tr.shape)\n",
    "print(y_tr.shape)\n",
    "print(x_tst.shape)\n",
    "print(y_tst.shape)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1847,
     "status": "ok",
     "timestamp": 1586799539745,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "ZhpMj8ZBBwSb",
    "outputId": "ca75ebd8-d9dc-49e2-e823-64abbad8fb33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "print(ytrain0[0])  #before and after hot encoding\n",
    "print(ytrain[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nI4NTgHSMKIT"
   },
   "outputs": [],
   "source": [
    "#build model -  this is a function to build the model where activation type and batch\n",
    "#normalization can be specified as parameters so models can be changed without retyping\n",
    "\n",
    "def build_model(activation, batch_normalization = False):\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(8, (3,3), activation = activation, padding = 'same', name = 'conv1', \n",
    "                    input_shape = (32,32,3))) \n",
    "    if batch_normalization: \n",
    "        model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D((2,2), name = 'maxpool1')) \n",
    "    model.add(Conv2D(16, (3,3), activation = activation, padding = 'same', name = 'conv2' ))\n",
    "    if batch_normalization: \n",
    "        model.add(BatchNormalization())\n",
    "    model.add(MaxPooling2D((2,2), name = 'maxpool2'))\n",
    "\n",
    "#    model.add(Conv2D(32, (3,3), activation = 'relu', padding = 'same', name = 'conv3'))\n",
    "#    if batch_normalization: \n",
    "#        model.add(BatchNormalization())\n",
    "#    model.add(MaxPooling2D((2,2), name = 'maxpool3'))\n",
    "#    model.add(Conv2D(32, (3, 3), activation=activation, padding='same', name='conv_4'))\n",
    "#    if batch_normalization: \n",
    "#        model.add(BatchNormalization())\n",
    "#    model.add(MaxPooling2D((2, 2), name='maxpool_4'))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    #model.add(Dropout(0.5))\n",
    "    model.add(Dense(64, activation=activation, name='dense_1'))\n",
    "\n",
    "#    if batch_normalization: \n",
    "#        model.add(BatchNormalization())\n",
    "#    model.add(Dense(32, activation=activation, name='dense_2'))\n",
    "\n",
    "    model.add(Dense(43, activation='softmax', name='output'))  #there are 43 labels\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 386
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 633,
     "status": "ok",
     "timestamp": 1586799940002,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "dTMJVW0oMZzt",
    "outputId": "93174681-def4-434f-f6d0-b4e9fba5189a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1 (Conv2D)               (None, 32, 32, 8)         224       \n",
      "_________________________________________________________________\n",
      "maxpool1 (MaxPooling2D)      (None, 16, 16, 8)         0         \n",
      "_________________________________________________________________\n",
      "conv2 (Conv2D)               (None, 16, 16, 16)        1168      \n",
      "_________________________________________________________________\n",
      "maxpool2 (MaxPooling2D)      (None, 8, 8, 16)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                65600     \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 43)                2795      \n",
      "=================================================================\n",
      "Total params: 69,787\n",
      "Trainable params: 69,787\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = build_model(activation = 'sigmoid', batch_normalization = False)  # try 'relu' too\n",
    "opt = tf.keras.optimizers.SGD(learning_rate = .1, momentum = 0.0, nesterov=False)\n",
    "model.compile(loss = 'categorical_crossentropy', optimizer = opt, metrics = ['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 722
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 97191,
     "status": "ok",
     "timestamp": 1586799668503,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "vp-CKtK_Mbn4",
    "outputId": "afc455ac-3b68-4e3c-c71f-f1fb53998b6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 26270 samples, validate on 12939 samples\n",
      "Epoch 1/20\n",
      "26270/26270 [==============================] - 11s 412us/step - loss: 3.5086 - accuracy: 0.0542 - val_loss: 3.4866 - val_accuracy: 0.0560\n",
      "Epoch 2/20\n",
      "26270/26270 [==============================] - 5s 173us/step - loss: 3.4912 - accuracy: 0.0621 - val_loss: 3.4629 - val_accuracy: 0.0559\n",
      "Epoch 3/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 3.4104 - accuracy: 0.0887 - val_loss: 3.3586 - val_accuracy: 0.1005\n",
      "Epoch 4/20\n",
      "26270/26270 [==============================] - 5s 171us/step - loss: 3.3214 - accuracy: 0.1025 - val_loss: 3.2523 - val_accuracy: 0.1221\n",
      "Epoch 5/20\n",
      "26270/26270 [==============================] - 5s 174us/step - loss: 3.1281 - accuracy: 0.1643 - val_loss: 2.8921 - val_accuracy: 0.2360\n",
      "Epoch 6/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 2.4925 - accuracy: 0.3195 - val_loss: 2.1246 - val_accuracy: 0.3840\n",
      "Epoch 7/20\n",
      "26270/26270 [==============================] - 5s 171us/step - loss: 1.9534 - accuracy: 0.4172 - val_loss: 1.7780 - val_accuracy: 0.4475\n",
      "Epoch 8/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 1.6951 - accuracy: 0.4743 - val_loss: 1.6020 - val_accuracy: 0.4937\n",
      "Epoch 9/20\n",
      "26270/26270 [==============================] - 4s 169us/step - loss: 1.5212 - accuracy: 0.5222 - val_loss: 1.4371 - val_accuracy: 0.5558\n",
      "Epoch 10/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 1.3657 - accuracy: 0.5726 - val_loss: 1.2758 - val_accuracy: 0.6043\n",
      "Epoch 11/20\n",
      "26270/26270 [==============================] - 4s 170us/step - loss: 1.2155 - accuracy: 0.6263 - val_loss: 1.1361 - val_accuracy: 0.6535\n",
      "Epoch 12/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 1.0748 - accuracy: 0.6728 - val_loss: 1.0030 - val_accuracy: 0.7098\n",
      "Epoch 13/20\n",
      "26270/26270 [==============================] - 4s 171us/step - loss: 0.9418 - accuracy: 0.7189 - val_loss: 0.9005 - val_accuracy: 0.7202\n",
      "Epoch 14/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 0.8168 - accuracy: 0.7590 - val_loss: 0.7591 - val_accuracy: 0.7810\n",
      "Epoch 15/20\n",
      "26270/26270 [==============================] - 5s 172us/step - loss: 0.7050 - accuracy: 0.7951 - val_loss: 0.6556 - val_accuracy: 0.8175\n",
      "Epoch 16/20\n",
      "26270/26270 [==============================] - 4s 169us/step - loss: 0.6084 - accuracy: 0.8285 - val_loss: 0.5696 - val_accuracy: 0.8451\n",
      "Epoch 17/20\n",
      "26270/26270 [==============================] - 4s 171us/step - loss: 0.5258 - accuracy: 0.8586 - val_loss: 0.4902 - val_accuracy: 0.8719\n",
      "Epoch 18/20\n",
      "26270/26270 [==============================] - 4s 170us/step - loss: 0.4576 - accuracy: 0.8815 - val_loss: 0.4356 - val_accuracy: 0.8861\n",
      "Epoch 19/20\n",
      "26270/26270 [==============================] - 4s 169us/step - loss: 0.4011 - accuracy: 0.8992 - val_loss: 0.3882 - val_accuracy: 0.9003\n",
      "Epoch 20/20\n",
      "26270/26270 [==============================] - 4s 170us/step - loss: 0.3526 - accuracy: 0.9153 - val_loss: 0.3459 - val_accuracy: 0.9154\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7efe801d4fd0>"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x_tr, y_tr, epochs = 20, validation_data= (x_tst, y_tst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1804,
     "status": "ok",
     "timestamp": 1586799674783,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "oEVXvN8yf158",
    "outputId": "c0b67229-4cee-4626-a4f4-8e8aa3979b4e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12939/12939 [==============================] - 1s 91us/step\n",
      "Test accuracy: 0.9154\n"
     ]
    }
   ],
   "source": [
    "loss, accuracy = model.evaluate(x_tst, y_tst)\n",
    "print('Test accuracy: %.4f' % (accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sRf_XhQeldqz"
   },
   "source": [
    "##below is for error checking and visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1443,
     "status": "ok",
     "timestamp": 1586799680806,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "2woLArmNjpdJ",
    "outputId": "70702b11-393f-4b51-9dd4-1e6889ff73a9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12939/12939 [==============================] - 1s 62us/step\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_tst, verbose = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-ULOq39_kgeg"
   },
   "outputs": [],
   "source": [
    "y_TST = [np.argmax(y_tst[i]) for i in range(len(y_tst))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 700,
     "status": "ok",
     "timestamp": 1586799684999,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "ERU5jH6xpWxR",
    "outputId": "7d149e39-dc6a-44d7-d744-1c0a8c673ac3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[31, 13, 1, 9, 11]"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_TST[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 533,
     "status": "ok",
     "timestamp": 1586799687202,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "Ua3YOLX731Dz",
    "outputId": "6366cf09-c7e8-4435-f1e1-b2d5fbaa03dd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_tst[0:5]  #y_TST and y_tst both check "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yxSJPt-en2Br"
   },
   "outputs": [],
   "source": [
    "y_HAT = [np.argmax(yhat[i]) for i in range(len(yhat))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 529,
     "status": "ok",
     "timestamp": 1586799695127,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "1_NiK8FLtQEy",
    "outputId": "72809263-4deb-402d-f3f7-845e7a34e639"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[31, 13, 1, 9, 11]"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_HAT[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 332
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 794,
     "status": "ok",
     "timestamp": 1586799697128,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "Csy-LHTv0vz9",
    "outputId": "503ce7d7-d9ac-47f2-b19e-2a3e9ff1d56f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2625., 1669., 2325., 1354.,  961.,  965.,  522.,  737.,  746.,\n",
       "        1035.]),\n",
       " array([ 0. ,  4.2,  8.4, 12.6, 16.8, 21. , 25.2, 29.4, 33.6, 37.8, 42. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAPVklEQVR4nO3dfayfZX3H8fdnBXWZLi2ja/q0lbkuS00mmgZY9A+mEQpbVkwWAmHaGJLjH5Bg4tTiPzgNC5uPM3Mkx9FYEidrpo7GNGMdI3H+IfZUGVAY4QwltBR6XMWHkLDgvvvjXJ0/63k+p+d3ONf7lZz87vt7P133Fc6nN9d9/+6TqkKS1IdfGnYDJEnLx9CXpI4Y+pLUEUNfkjpi6EtSR84bdgNmcuGFF9a2bduG3QxJekU5evTo96tq/VTLVnTob9u2jbGxsWE3Q5JeUZI8Pd0yh3ckqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0JakjK/obuYu2adNwjvvss8M5riTNwit9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1JFV/cjm6Is3DOW4I0M5qiTNzit9SeqIoS9JHTH0Jakjhr4kdcTQl6SOzBr6SbYmeSDJY0mOJbml1T+S5ESSh9rP1QPb3JpkPMkTSa4cqO9qtfEke8/NKUmSpjOXRzZfBt5fVd9O8jrgaJLDbdmnq+oTgysn2QFcB7wB2AT8a5LfaYs/B7wDOA4cSXKwqh5bihORJM1u1tCvqpPAyTb94ySPA5tn2GQ3cE9VvQR8N8k4cElbNl5VTwEkuaeta+hL0jKZ15h+km3Am4AHW+nmJA8n2ZdkXattBp4Z2Ox4q01XP/sYI0nGkoxNTEzMp3mSpFnMOfSTvBb4MvC+qvoRcCfweuBiJv9P4JNL0aCqGq2qnVW1c/369UuxS0lSM6fXMCQ5n8nA/2JVfQWgqp4fWP554Gtt9gSwdWDzLa3GDHVJ0jKYy9M7Ae4CHq+qTw3UNw6s9k7g0TZ9ELguyauTXARsB74FHAG2J7koyauYvNl7cGlOQ5I0F3O50n8L8C7gkSQPtdqHgeuTXAwU8D3gvQBVdSzJASZv0L4M3FRVPwVIcjNwH7AG2FdVx5bwXCRJs5jL0zvfADLFokMzbHM7cPsU9UMzbSdJOrf8Rq4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR2Z02sY9AqyadNwjvvss8M5rqR5MfRXmdEXbxjKcUeGclRJ8+XwjiR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOjJr6CfZmuSBJI8lOZbklla/IMnhJE+2z3WtniSfTTKe5OEkbx7Y1562/pNJ9py705IkTWUuV/ovA++vqh3AZcBNSXYAe4H7q2o7cH+bB7gK2N5+RoA7YfIfCeA24FLgEuC2M/9QSJKWx6yhX1Unq+rbbfrHwOPAZmA3sL+tth+4pk3vBu6uSd8E1ibZCFwJHK6q01X1A+AwsGtJz0aSNKN5jekn2Qa8CXgQ2FBVJ9ui54ANbXoz8MzAZsdbbbr62ccYSTKWZGxiYmI+zZMkzWLOoZ/ktcCXgfdV1Y8Gl1VVAbUUDaqq0araWVU7169fvxS7lCQ1cwr9JOczGfhfrKqvtPLzbdiG9nmq1U8AWwc239Jq09UlSctkLk/vBLgLeLyqPjWw6CBw5gmcPcC9A/V3t6d4LgN+2IaB7gOuSLKu3cC9otUkScvkvDms8xbgXcAjSR5qtQ8DdwAHktwIPA1c25YdAq4GxoEXgfcAVNXpJB8DjrT1PlpVp5fkLCRJczJr6FfVN4BMs/jtU6xfwE3T7GsfsG8+DZQkLR2/kStJHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI7M+ofRNX+jo8NugSRNzSt9SeqIoS9JHXF451z44AeG3QJJmpJX+pLUEUNfkjpi6EtSRwx9SeqIoS9JHZk19JPsS3IqyaMDtY8kOZHkofZz9cCyW5OMJ3kiyZUD9V2tNp5k79KfiiRpNnO50v8CsGuK+qer6uL2cwggyQ7gOuANbZu/TbImyRrgc8BVwA7g+rauJGkZzfqcflV9Pcm2Oe5vN3BPVb0EfDfJOHBJWzZeVU8BJLmnrfvYvFssSVqwxYzp35zk4Tb8s67VNgPPDKxzvNWmq/+CJCNJxpKMTUxMLKJ5kqSzLTT07wReD1wMnAQ+uVQNqqrRqtpZVTvXr1+/VLuVJLHA1zBU1fNnppN8Hvhamz0BbB1YdUurMUNdkrRMFnSln2TjwOw7gTNP9hwErkvy6iQXAduBbwFHgO1JLkryKiZv9h5ceLMlSQsx65V+ki8BlwMXJjkO3AZcnuRioIDvAe8FqKpjSQ4weYP2ZeCmqvpp28/NwH3AGmBfVR1b8rORJM1oLk/vXD9F+a4Z1r8duH2K+iHg0LxaJ0laUn4jV5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXEP4yuJTG6djh/DH7khY8P5bjSK5VX+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOzhn6SfUlOJXl0oHZBksNJnmyf61o9ST6bZDzJw0nePLDNnrb+k0n2nJvTkSTN5Lw5rPMF4G+Auwdqe4H7q+qOJHvb/IeAq4Dt7edS4E7g0iQXALcBO4ECjiY5WFU/WKoTkaRzYXTtB4Zy3JEXPn5O9jvrlX5VfR04fVZ5N7C/Te8Hrhmo312TvgmsTbIRuBI4XFWnW9AfBnYtxQlIkuZuoWP6G6rqZJt+DtjQpjcDzwysd7zVpqtLkpbRom/kVlUxOWSzJJKMJBlLMjYxMbFUu5UksfDQf74N29A+T7X6CWDrwHpbWm26+i+oqtGq2llVO9evX7/A5kmSpjKXG7lTOQjsAe5on/cO1G9Ocg+TN3J/WFUnk9wH/MWZp3yAK4BbF95safhGR4dz3JGR4RxXq8OsoZ/kS8DlwIVJjjP5FM4dwIEkNwJPA9e21Q8BVwPjwIvAewCq6nSSjwFH2nofraqzbw5L8zasJysA+Ktz83SFdC7NGvpVdf00i94+xboF3DTNfvYB++bVOknSkvIbuZLUkYWO6Uv64JCGlkYcVtLCeaUvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xHfvSJq7TZuGc9xnnx3OcVchQ1/SijesP1izGhn6kuZs9MUbht0ELZKhL2nlG9ZrrFchb+RKUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZFGhn+R7SR5J8lCSsVa7IMnhJE+2z3WtniSfTTKe5OEkb16KE5Akzd1SvFr5D6rq+wPze4H7q+qOJHvb/IeAq4Dt7edS4M72KWkeRtf6mmEt3LkY3tkN7G/T+4FrBup316RvAmuTbDwHx5ckTWOxoV/AvyQ5mmSk1TZU1ck2/RywoU1vBp4Z2PZ4q/2cJCNJxpKMTUxMLLJ5kqRBix3eeWtVnUjy68DhJP85uLCqKknNZ4dVNQqMAuzcuXNe20qSZraoK/2qOtE+TwFfBS4Bnj8zbNM+T7XVTwBbBzbf0mqSpGWy4NBP8itJXndmGrgCeBQ4COxpq+0B7m3TB4F3t6d4LgN+ODAMJElaBosZ3tkAfDXJmf38fVX9c5IjwIEkNwJPA9e29Q8BVwPjwIvAexZxbEnSAiw49KvqKeCNU9T/G3j7FPUCblro8SRJi+c3ciWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOrLsoZ9kV5Inkown2bvcx5ekni1r6CdZA3wOuArYAVyfZMdytkGSerbcV/qXAONV9VRV/Q9wD7B7mdsgSd06b5mPtxl4ZmD+OHDp4ApJRoCRNvuTJE8s4ngXAt9fxParnf0zO/toZvbP7BbUR+/NJxZzzN+cbsFyh/6sqmoUGF2KfSUZq6qdS7Gv1cj+mZ19NDP7Z3YrrY+We3jnBLB1YH5Lq0mSlsFyh/4RYHuSi5K8CrgOOLjMbZCkbi3r8E5VvZzkZuA+YA2wr6qOncNDLskw0Spm/8zOPpqZ/TO7FdVHqapht0GStEz8Rq4kdcTQl6SOrMrQ91UPvyjJviSnkjw6ULsgyeEkT7bPdcNs4zAl2ZrkgSSPJTmW5JZWt4+aJK9J8q0k/9H66M9b/aIkD7bft39oD2l0K8maJN9J8rU2v6L6Z9WFvq96mNYXgF1n1fYC91fVduD+Nt+rl4H3V9UO4DLgpvbfjX30My8Bb6uqNwIXA7uSXAb8JfDpqvpt4AfAjUNs40pwC/D4wPyK6p9VF/r4qocpVdXXgdNnlXcD+9v0fuCaZW3UClJVJ6vq2236x0z+0m7GPvp/Neknbfb89lPA24B/bPWu+yjJFuAPgb9r82GF9c9qDP2pXvWweUhtWek2VNXJNv0csGGYjVkpkmwD3gQ8iH30c9rQxUPAKeAw8F/AC1X1clul99+3zwAfBP63zf8aK6x/VmPoawFq8tnd7p/fTfJa4MvA+6rqR4PL7COoqp9W1cVMfpv+EuB3h9ykFSPJHwGnqurosNsykxX37p0l4Kse5u75JBur6mSSjUxevXUryflMBv4Xq+orrWwfTaGqXkjyAPD7wNok57Wr2Z5/394C/HGSq4HXAL8K/DUrrH9W45W+r3qYu4PAnja9B7h3iG0Zqjb2ehfweFV9amCRfdQkWZ9kbZv+ZeAdTN77eAD4k7Zat31UVbdW1Zaq2sZk7vxbVd3ACuufVfmN3PYv7Wf42asebh9yk4YuyZeAy5l8zevzwG3APwEHgN8Angauraqzb/Z2IclbgX8HHuFn47EfZnJc3z4Ckvwekzci1zB5wXigqj6a5LeYfGDiAuA7wJ9W1UvDa+nwJbkc+LOq+qOV1j+rMvQlSVNbjcM7kqRpGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI/8HM9gO+ZXrxGcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.hist(y_HAT, color = 'r', alpha = 0.9)\n",
    "plt.hist(y_TST, color='b', alpha = 0.4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aeYCkHs_Mbui"
   },
   "source": [
    "missclassification can be seen in the non overlapping areas; magenta is common, blue and red areas are misclassification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 534,
     "status": "ok",
     "timestamp": 1586799743083,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "g835dvETMioo",
    "outputId": "ee1e7cf0-73b1-40e4-bccf-c819c7e996f0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12939"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "y_HAT[0:5]\n",
    "np.max(np.array(y_HAT))\n",
    "set(y_HAT)\n",
    "len(y_HAT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 559,
     "status": "ok",
     "timestamp": 1586799745530,
     "user": {
      "displayName": "Steven Su",
      "photoUrl": "",
      "userId": "12872522105664994411"
     },
     "user_tz": 360
    },
    "id": "0r1GR83v55xU",
    "outputId": "e33706e5-605b-4929-972d-c420e0979e89"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12939"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_TST[0:5]\n",
    "np.max(np.array(y_TST))\n",
    "set(y_TST)\n",
    "len(y_TST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NKO_E2Y5B4_Z"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyPmJA4hBfRBdrqRdmH05WdY",
   "mount_file_id": "19D6oAusolyAqbiz0L7_PIHRSMv0eAsTz",
   "name": "Copy of german_signs3.ipynb",
   "provenance": [
    {
     "file_id": "1Ys1JBCEmz69eskWG7x-PFnjZcyaSUlAx",
     "timestamp": 1586798975859
    },
    {
     "file_id": "1rgH0lPxAacqXN-WBthfm6MoRTyzuga_J",
     "timestamp": 1586746443733
    }
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}