classifier_keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "import numpy as np\n",
    "import scipy \n",
    "import tensorflow as tf\n",
    "\n",
    "import imageio\n",
    "import gzip\n",
    "from PIL import Image\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.cm as cmx\n",
    "import matplotlib.pyplot as plt\n",
    "#from tqdm import trange, tqdm\n",
    "\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from keras.optimizers import SGD, Adam, RMSprop, Optimizer\n",
    "from keras.callbacks import Callback\n",
    "from collections import OrderedDict\n",
    "\n",
    "from helpers import utils\n",
    "# from helpers import protocols\n",
    "# from helpers.keras_utils import LossHistory\n",
    "# from helpers.optimizers import KOOptimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "EXTRACT LABELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000,)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "# Extract labels from MNIST labels into vector\n",
    "def extract_labels(filename, num_images):\n",
    "  with gzip.open(filename) as bytestream:\n",
    "    bytestream.read(8)\n",
    "    buf = bytestream.read(1 * num_images)\n",
    "    labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)\n",
    "  return labels\n",
    "\n",
    "train_labels = extract_labels(\"MNIST-data/train-labels-idx1-ubyte.gz\", 60000)\n",
    "eval_labels = extract_labels(\"MNIST-data/t10k-labels-idx1-ubyte.gz\", 10000)\n",
    "print(np.shape(train_labels))\n",
    "print(np.shape(eval_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CONSTRUCT DATASETS\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# original train\n",
    "train_original = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_original = [\"MNIST-processed-training/original/original{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_original)):\n",
    "    img = np.array(Image.open(images_original[i]))\n",
    "    train_original[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_original))\n",
    "\n",
    "# original test\n",
    "eval_original = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_original = [\"MNIST-processed-test/original/test-original{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_original)):\n",
    "    img = np.array(Image.open(images2_original[i]))\n",
    "    eval_original[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_original))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# ROTATE 90 train\n",
    "train_rot90 = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_rot90 = [\"MNIST-processed-training/rot90/rot90{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_rot90)):\n",
    "    img = np.array(Image.open(images_rot90[i]))\n",
    "    train_rot90[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_rot90))\n",
    "\n",
    "# ROTATE 90 test\n",
    "eval_rot90 = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_rot90 = [\"MNIST-processed-test/rot90/test-rot90{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_rot90)):\n",
    "    img = np.array(Image.open(images2_rot90[i]))\n",
    "    eval_rot90[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_rot90))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# checkerboard train\n",
    "train_checkerboard = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_checkerboard = [\"MNIST-processed-training/checkerboard/fullcheck{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_checkerboard)):\n",
    "    img = np.array(Image.open(images_checkerboard[i]))\n",
    "    train_checkerboard[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_checkerboard))\n",
    "\n",
    "# checkerboard test\n",
    "eval_checkerboard = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_checkerboard = [\"MNIST-processed-test/checkerboard/test-checkerboard{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_checkerboard)):\n",
    "    img = np.array(Image.open(images2_checkerboard[i]))\n",
    "    eval_checkerboard[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_checkerboard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# INV train\n",
    "train_inv = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_inv = [\"MNIST-processed-training/Inv/inv{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_inv)):\n",
    "    img = np.array(Image.open(images_inv[i]))\n",
    "    train_inv[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_inv))\n",
    "\n",
    "# INV test\n",
    "eval_inv = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_inv = [\"MNIST-processed-test/inv/test-inv{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_inv)):\n",
    "    img = np.array(Image.open(images2_inv[i]))\n",
    "    eval_inv[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_inv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# invbot train\n",
    "train_invbot = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_invbot = [\"MNIST-processed-training/invbot/invbot{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_invbot)):\n",
    "    img = np.array(Image.open(images_invbot[i]))\n",
    "    train_invbot[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_invbot))\n",
    "\n",
    "# invbot test\n",
    "eval_invbot = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_invbot = [\"MNIST-processed-test/invbot/test-invbot{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_invbot)):\n",
    "    img = np.array(Image.open(images2_invbot[i]))\n",
    "    eval_invbot[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_invbot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# fliplr train\n",
    "train_fliplr = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_fliplr = [\"MNIST-processed-training/fliplr/fliplr{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_fliplr)):\n",
    "    img = np.array(Image.open(images_fliplr[i]))\n",
    "    train_fliplr[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_fliplr))\n",
    "\n",
    "# fliplr test\n",
    "eval_fliplr = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_fliplr = [\"MNIST-processed-test/fliplr/test-fliplr{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_fliplr)):\n",
    "    img = np.array(Image.open(images2_fliplr[i]))\n",
    "    eval_fliplr[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_fliplr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# cutud train\n",
    "train_cutud = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_cutud = [\"MNIST-processed-training/cutud/cutUD{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_cutud)):\n",
    "    img = np.array(Image.open(images_cutud[i]))\n",
    "    train_cutud[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_cutud))\n",
    "\n",
    "# cutud test\n",
    "eval_cutud = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_cutud = [\"MNIST-processed-test/cutud/test-cutud{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_cutud)):\n",
    "    img = np.array(Image.open(images2_cutud[i]))\n",
    "    eval_cutud[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_cutud))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# flipud train\n",
    "train_flipud = np.zeros((60000,28,28), dtype=np.float32)\n",
    "images_flipud = [\"MNIST-processed-training/flipud/flipud{0}.png\".format(k) for k in range(1,60000)]\n",
    "\n",
    "for i in range(len(images_flipud)):\n",
    "    img = np.array(Image.open(images_flipud[i]))\n",
    "    train_flipud[i, :, :] = img\n",
    " \n",
    "print(np.shape(train_flipud))\n",
    "\n",
    "# flipud test\n",
    "eval_flipud = np.zeros((10000,28,28), dtype=np.float32)\n",
    "images2_flipud = [\"MNIST-processed-test/flipud/test-flipud{0}.png\".format(k) for k in range(1,10001)]\n",
    "\n",
    "for i in range(len(images2_flipud)):\n",
    "    img = np.array(Image.open(images2_flipud[i]))\n",
    "    eval_flipud[i, :, :] = img\n",
    " \n",
    "print(np.shape(eval_flipud))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PARAMETERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Data params\n",
    "# input_dim = 784\n",
    "# output_dim = 10\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# # Network params\n",
    "# n_hidden_units = 2000\n",
    "# activation_fn = tf.nn.relu\n",
    "\n",
    "# Optimization params\n",
    "batch_size = 256\n",
    "num_classes = 10\n",
    "epochs = 5 # epochs per task\n",
    "# learning_rate=1e-3\n",
    "# xi = 0.1\n",
    "\n",
    "# Reset optimizer after each age\n",
    "# reset_optimizer = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train labels shape: (60000,)\n",
      "test labels shape: (10000,)\n",
      "x_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "train labels shape: (60000, 10)\n",
      "test labels shape: (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "# the data, train and test sets\n",
    "x_train = train_original\n",
    "x_test = eval_original\n",
    "y_train = train_labels\n",
    "y_test = eval_labels\n",
    "\n",
    "print('train labels shape:', y_train.shape)\n",
    "print('test labels shape:', y_test.shape)\n",
    "\n",
    "# ?\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "    \n",
    "# ?\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "print('train labels shape:', y_train.shape)\n",
    "print('test labels shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "x_train2 = train_rot90\n",
    "x_test2 = eval_rot90\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train2 = x_train2.reshape(x_train2.shape[0], 1, img_rows, img_cols)\n",
    "    x_test2 = x_test2.reshape(x_test2.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train2 = x_train2.reshape(x_train2.shape[0], img_rows, img_cols, 1)\n",
    "    x_test2 = x_test2.reshape(x_test2.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train2 = x_train2.astype('float32')\n",
    "x_test2 = x_test2.astype('float32')\n",
    "x_train2 /= 255\n",
    "x_test2 /= 255\n",
    "\n",
    "x_train3 = train_inv\n",
    "x_test3 = eval_inv\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train3 = x_train3.reshape(x_train3.shape[0], 1, img_rows, img_cols)\n",
    "    x_test3 = x_test3.reshape(x_test3.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train3 = x_train3.reshape(x_train3.shape[0], img_rows, img_cols, 1)\n",
    "    x_test3 = x_test3.reshape(x_test3.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train3 = x_train3.astype('float32')\n",
    "x_test3 = x_test3.astype('float32')\n",
    "x_train3 /= 255\n",
    "x_test3 /= 255\n",
    "\n",
    "x_train4 = train_flipud\n",
    "x_test4 = eval_flipud\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train4 = x_train4.reshape(x_train4.shape[0], 1, img_rows, img_cols)\n",
    "    x_test4 = x_test4.reshape(x_test4.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train4 = x_train4.reshape(x_train4.shape[0], img_rows, img_cols, 1)\n",
    "    x_test4 = x_test4.reshape(x_test4.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train4 = x_train4.astype('float32')\n",
    "x_test4 = x_test4.astype('float32')\n",
    "x_train4 /= 255\n",
    "x_test4 /= 255\n",
    "\n",
    "x_train5 = train_fliplr\n",
    "x_test5 = eval_fliplr\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train5 = x_train5.reshape(x_train5.shape[0], 1, img_rows, img_cols)\n",
    "    x_test5 = x_test5.reshape(x_test5.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train5 = x_train5.reshape(x_train5.shape[0], img_rows, img_cols, 1)\n",
    "    x_test5 = x_test5.reshape(x_test5.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train5 = x_train5.astype('float32')\n",
    "x_test5 = x_test5.astype('float32')\n",
    "x_train5 /= 255\n",
    "x_test5 /= 255\n",
    "\n",
    "x_train6 = train_cutud\n",
    "x_test6 = eval_cutud\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train6 = x_train6.reshape(x_train6.shape[0], 1, img_rows, img_cols)\n",
    "    x_test6 = x_test6.reshape(x_test6.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train6 = x_train6.reshape(x_train6.shape[0], img_rows, img_cols, 1)\n",
    "    x_test6 = x_test6.reshape(x_test6.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train6 = x_train6.astype('float32')\n",
    "x_test6 = x_test6.astype('float32')\n",
    "x_train6 /= 255\n",
    "x_test6 /= 255\n",
    "\n",
    "x_train7 = train_invbot\n",
    "x_test7 = eval_invbot\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train7 = x_train7.reshape(x_train7.shape[0], 1, img_rows, img_cols)\n",
    "    x_test7 = x_test7.reshape(x_test7.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train7 = x_train7.reshape(x_train7.shape[0], img_rows, img_cols, 1)\n",
    "    x_test7 = x_test7.reshape(x_test7.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train7 = x_train7.astype('float32')\n",
    "x_test7 = x_test7.astype('float32')\n",
    "x_train7 /= 255\n",
    "x_test7 /= 255\n",
    "\n",
    "x_train8 = train_checkerboard\n",
    "x_test8 = eval_checkerboard\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train8 = x_train8.reshape(x_train8.shape[0], 1, img_rows, img_cols)\n",
    "    x_test8 = x_test8.reshape(x_test8.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train8 = x_train8.reshape(x_train8.shape[0], img_rows, img_cols, 1)\n",
    "    x_test8 = x_test8.reshape(x_test8.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "x_train8 = x_train8.astype('float32')\n",
    "x_test8 = x_test8.astype('float32')\n",
    "x_train8 /= 255\n",
    "x_test8 /= 255\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CONSTRUCT MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(32, kernel_size=(3, 3),\n",
    "                 activation='relu',\n",
    "                 input_shape=input_shape))\n",
    "model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TRAIN MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 184s 3ms/step - loss: 0.3585 - acc: 0.8888 - val_loss: 0.0795 - val_acc: 0.9745\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 177s 3ms/step - loss: 0.1031 - acc: 0.9697 - val_loss: 0.0547 - val_acc: 0.9817\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 178s 3ms/step - loss: 0.0762 - acc: 0.9772 - val_loss: 0.0383 - val_acc: 0.9871\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 184s 3ms/step - loss: 0.0614 - acc: 0.9814 - val_loss: 0.0417 - val_acc: 0.9861\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 187s 3ms/step - loss: 0.0545 - acc: 0.9827 - val_loss: 0.0336 - val_acc: 0.9888\n",
      "Test loss: 0.033603799838290434\n",
      "Test accuracy: 0.9888\n"
     ]
    }
   ],
   "source": [
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])\n",
    "\n",
    "# Save trained model\n",
    "model.save('trained_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 183s 3ms/step - loss: 0.3223 - acc: 0.9047 - val_loss: 0.6549 - val_acc: 0.7928\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 197s 3ms/step - loss: 0.1119 - acc: 0.9674 - val_loss: 1.1198 - val_acc: 0.6905\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 177s 3ms/step - loss: 0.0842 - acc: 0.9754 - val_loss: 1.7272 - val_acc: 0.6029\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 171s 3ms/step - loss: 0.0699 - acc: 0.9793 - val_loss: 2.4287 - val_acc: 0.5139\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 164s 3ms/step - loss: 0.0602 - acc: 0.9824 - val_loss: 2.7129 - val_acc: 0.4359\n",
      "Original data set\n",
      "Test loss: 2.712919066810608\n",
      "Test accuracy: 0.4359\n",
      "Second data set\n",
      "Test loss: 0.036111295583665194\n",
      "Test accuracy: 0.9877\n"
     ]
    }
   ],
   "source": [
    "# model2 = Sequential()\n",
    "\n",
    "# model2.add(Conv2D(32, kernel_size=(3, 3),\n",
    "#                  activation='relu',\n",
    "#                  input_shape=input_shape))\n",
    "# model2.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "# model2.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model2.add(Dropout(0.25))\n",
    "# model2.add(Flatten())\n",
    "# model2.add(Dense(128, activation='relu'))\n",
    "# model2.add(Dropout(0.5))\n",
    "# model2.add(Dense(num_classes, activation='softmax'))\n",
    "# model2.compile(loss=keras.losses.categorical_crossentropy,\n",
    "#               optimizer=keras.optimizers.Adadelta(),\n",
    "#               metrics=['accuracy'])\n",
    "\n",
    "\n",
    "# Reload model\n",
    "model2 = load_model('trained_model.h5')\n",
    "\n",
    "# Continue training\n",
    "model2.fit(x_train2, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score1 = model2.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model2.evaluate(x_test2, y_test, verbose=0)\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('Second data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "# Save trained model\n",
    "model2.save('trained_model2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 157s 3ms/step - loss: 0.2013 - acc: 0.9419 - val_loss: 0.3987 - val_acc: 0.8580\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 155s 3ms/step - loss: 0.0849 - acc: 0.9741 - val_loss: 0.7368 - val_acc: 0.7565\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 154s 3ms/step - loss: 0.0657 - acc: 0.9805 - val_loss: 0.5341 - val_acc: 0.8086\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 156s 3ms/step - loss: 0.0539 - acc: 0.9839 - val_loss: 0.7304 - val_acc: 0.7647\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 155s 3ms/step - loss: 0.0468 - acc: 0.9859 - val_loss: 0.7390 - val_acc: 0.7626\n",
      "Original data set\n",
      "Test loss: 0.7389654237508774\n",
      "Test accuracy: 0.7626\n",
      "Second data set\n",
      "Test loss: 2.562846361351013\n",
      "Test accuracy: 0.4234\n",
      "Second data set\n",
      "Test loss: 0.03205857433127967\n",
      "Test accuracy: 0.9896\n"
     ]
    }
   ],
   "source": [
    "# model3 = Sequential()\n",
    "\n",
    "# model3.add(Conv2D(32, kernel_size=(3, 3),\n",
    "#                  activation='relu',\n",
    "#                  input_shape=input_shape))\n",
    "# model3.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "# model3.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model3.add(Dropout(0.25))\n",
    "# model3.add(Flatten())\n",
    "# model3.add(Dense(128, activation='relu'))\n",
    "# model3.add(Dropout(0.5))\n",
    "# model3.add(Dense(num_classes, activation='softmax'))\n",
    "# model3.compile(loss=keras.losses.categorical_crossentropy,\n",
    "#               optimizer=keras.optimizers.Adadelta(),\n",
    "#               metrics=['accuracy'])\n",
    "\n",
    "# Reload model\n",
    "model3 = load_model('trained_model2.h5')\n",
    "\n",
    "# Continue training\n",
    "model3.fit(x_train3, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score1 = model3.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model3.evaluate(x_test2, y_test, verbose=0)\n",
    "score3 = model3.evaluate(x_test3, y_test, verbose=0)\n",
    "\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('Second data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "print('Second data set')\n",
    "print('Test loss:', score3[0])\n",
    "print('Test accuracy:', score3[1])\n",
    "\n",
    "# Save trained model\n",
    "model3.save('trained_model3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 156s 3ms/step - loss: 0.3044 - acc: 0.9121 - val_loss: 3.2784 - val_acc: 0.2707\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 157s 3ms/step - loss: 0.1042 - acc: 0.9696 - val_loss: 3.8944 - val_acc: 0.2293\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 151s 3ms/step - loss: 0.0801 - acc: 0.9761 - val_loss: 4.1714 - val_acc: 0.2181\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 154s 3ms/step - loss: 0.0644 - acc: 0.9802 - val_loss: 5.4578 - val_acc: 0.1811\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 149s 2ms/step - loss: 0.0570 - acc: 0.9830 - val_loss: 5.4171 - val_acc: 0.1953\n",
      "Original data set\n",
      "Test loss: 5.417119498443603\n",
      "Test accuracy: 0.1953\n",
      "2 data set\n",
      "Test loss: 5.454559282684326\n",
      "Test accuracy: 0.2267\n",
      "3 data set\n",
      "Test loss: 3.50109836730957\n",
      "Test accuracy: 0.483\n",
      "4 data set\n",
      "Test loss: 0.028514384727807193\n",
      "Test accuracy: 0.9906\n"
     ]
    }
   ],
   "source": [
    "# model4 = Sequential()\n",
    "\n",
    "# model4.add(Conv2D(32, kernel_size=(3, 3),\n",
    "#                  activation='relu',\n",
    "#                  input_shape=input_shape))\n",
    "# model4.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "# model4.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model4.add(Dropout(0.25))\n",
    "# model4.add(Flatten())\n",
    "# model4.add(Dense(128, activation='relu'))\n",
    "# model4.add(Dropout(0.5))\n",
    "# model4.add(Dense(num_classes, activation='softmax'))\n",
    "# model4.compile(loss=keras.losses.categorical_crossentropy,\n",
    "#               optimizer=keras.optimizers.Adadelta(),\n",
    "#               metrics=['accuracy'])\n",
    "\n",
    "# Reload model\n",
    "model4 = load_model('trained_model3.h5')\n",
    "\n",
    "# Continue training\n",
    "model4.fit(x_train4, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score1 = model4.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model4.evaluate(x_test2, y_test, verbose=0)\n",
    "score3 = model4.evaluate(x_test3, y_test, verbose=0)\n",
    "score4 = model4.evaluate(x_test4, y_test, verbose=0)\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('2 data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "print('3 data set')\n",
    "print('Test loss:', score3[0])\n",
    "print('Test accuracy:', score3[1])\n",
    "\n",
    "print('4 data set')\n",
    "print('Test loss:', score4[0])\n",
    "print('Test accuracy:', score4[1])\n",
    "\n",
    "# Save trained model\n",
    "model4.save('trained_model4.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 183s 3ms/step - loss: 0.3263 - acc: 0.9058 - val_loss: 6.6734 - val_acc: 0.1322\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 180s 3ms/step - loss: 0.1090 - acc: 0.9681 - val_loss: 5.7889 - val_acc: 0.1702\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 179s 3ms/step - loss: 0.0822 - acc: 0.9765 - val_loss: 6.9074 - val_acc: 0.1506\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 178s 3ms/step - loss: 0.0698 - acc: 0.9797 - val_loss: 6.8747 - val_acc: 0.1660\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 172s 3ms/step - loss: 0.0607 - acc: 0.9824 - val_loss: 8.6029 - val_acc: 0.1096\n",
      "Original data set\n",
      "Test loss: 8.602926930236816\n",
      "Test accuracy: 0.1096\n",
      "2 data set\n",
      "Test loss: 4.607483555603027\n",
      "Test accuracy: 0.273\n",
      "3 data set\n",
      "Test loss: 4.280151822662353\n",
      "Test accuracy: 0.4227\n",
      "4 data set\n",
      "Test loss: 5.093793195343018\n",
      "Test accuracy: 0.4807\n",
      "5 data set\n",
      "Test loss: 0.03732895709029435\n",
      "Test accuracy: 0.9887\n"
     ]
    }
   ],
   "source": [
    "# model5 = Sequential()\n",
    "# model5.add(Conv2D(32, kernel_size=(3, 3),\n",
    "#                  activation='relu',\n",
    "#                  input_shape=input_shape))\n",
    "# model5.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "# model5.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model5.add(Dropout(0.25))\n",
    "# model5.add(Flatten())\n",
    "# model5.add(Dense(128, activation='relu'))\n",
    "# model5.add(Dropout(0.5))\n",
    "# model5.add(Dense(num_classes, activation='softmax'))\n",
    "# model5.compile(loss=keras.losses.categorical_crossentropy,\n",
    "#               optimizer=keras.optimizers.Adadelta(),\n",
    "#               metrics=['accuracy'])\n",
    "\n",
    "# Reload model\n",
    "model5 = load_model('trained_model4.h5')\n",
    "\n",
    "# Continue training\n",
    "model5.fit(x_train5, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score1 = model5.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model5.evaluate(x_test2, y_test, verbose=0)\n",
    "score3 = model5.evaluate(x_test3, y_test, verbose=0)\n",
    "score4 = model5.evaluate(x_test4, y_test, verbose=0)\n",
    "score5 = model5.evaluate(x_test5, y_test, verbose=0)\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('2 data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "print('3 data set')\n",
    "print('Test loss:', score3[0])\n",
    "print('Test accuracy:', score3[1])\n",
    "\n",
    "print('4 data set')\n",
    "print('Test loss:', score4[0])\n",
    "print('Test accuracy:', score4[1])\n",
    "\n",
    "\n",
    "print('5 data set')\n",
    "print('Test loss:', score5[0])\n",
    "print('Test accuracy:', score5[1])\n",
    "\n",
    "# Save trained model\n",
    "model5.save('trained_model5.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 168s 3ms/step - loss: 0.3295 - acc: 0.9045 - val_loss: 5.6291 - val_acc: 0.1603\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 180s 3ms/step - loss: 0.1117 - acc: 0.9677 - val_loss: 7.3592 - val_acc: 0.1332\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 154s 3ms/step - loss: 0.0835 - acc: 0.9753 - val_loss: 7.6739 - val_acc: 0.1436\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 156s 3ms/step - loss: 0.0686 - acc: 0.9794 - val_loss: 7.0773 - val_acc: 0.1450\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 155s 3ms/step - loss: 0.0617 - acc: 0.9822 - val_loss: 8.1081 - val_acc: 0.1136\n",
      "Original data set\n",
      "Test loss: 8.108143403625489\n",
      "Test accuracy: 0.1136\n",
      "2 data set\n",
      "Test loss: 4.581724743652344\n",
      "Test accuracy: 0.2402\n",
      "3 data set\n",
      "Test loss: 3.5188808303833006\n",
      "Test accuracy: 0.4677\n",
      "4 data set\n",
      "Test loss: 4.375547528076172\n",
      "Test accuracy: 0.4731\n",
      "5 data set\n",
      "Test loss: 0.040834705409735036\n",
      "Test accuracy: 0.9864\n"
     ]
    }
   ],
   "source": [
    "# model6 = Sequential()\n",
    "# model6.add(Conv2D(32, kernel_size=(3, 3),\n",
    "#                  activation='relu',\n",
    "#                  input_shape=input_shape))\n",
    "# model6.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "# model6.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model6.add(Dropout(0.25))\n",
    "# model6.add(Flatten())\n",
    "# model6.add(Dense(128, activation='relu'))\n",
    "# model6.add(Dropout(0.5))\n",
    "# model6.add(Dense(num_classes, activation='softmax'))\n",
    "# model6.compile(loss=keras.losses.categorical_crossentropy,\n",
    "#               optimizer=keras.optimizers.Adadelta(),\n",
    "#               metrics=['accuracy'])\n",
    "\n",
    "# Reload model\n",
    "model5 = load_model('trained_model4.h5')\n",
    "\n",
    "# Continue training\n",
    "model5.fit(x_train5, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score1 = model5.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model5.evaluate(x_test2, y_test, verbose=0)\n",
    "score3 = model5.evaluate(x_test3, y_test, verbose=0)\n",
    "score4 = model5.evaluate(x_test4, y_test, verbose=0)\n",
    "score5 = model5.evaluate(x_test5, y_test, verbose=0)\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('2 data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "print('3 data set')\n",
    "print('Test loss:', score3[0])\n",
    "print('Test accuracy:', score3[1])\n",
    "\n",
    "print('4 data set')\n",
    "print('Test loss:', score4[0])\n",
    "print('Test accuracy:', score4[1])\n",
    "\n",
    "\n",
    "print('5 data set')\n",
    "print('Test loss:', score5[0])\n",
    "print('Test accuracy:', score5[1])\n",
    "\n",
    "# Save trained model\n",
    "model5.save('trained_model5.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 182s 3ms/step - loss: 0.4091 - acc: 0.8797 - val_loss: 9.4218 - val_acc: 0.1602\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 159s 3ms/step - loss: 0.1331 - acc: 0.9605 - val_loss: 10.5342 - val_acc: 0.1369\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 151s 3ms/step - loss: 0.1022 - acc: 0.9704 - val_loss: 8.1295 - val_acc: 0.1778\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 156s 3ms/step - loss: 0.0838 - acc: 0.9752 - val_loss: 8.7704 - val_acc: 0.1769\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 155s 3ms/step - loss: 0.0724 - acc: 0.9785 - val_loss: 9.4676 - val_acc: 0.1624\n",
      "Original data set\n",
      "Test loss: 9.467570140075683\n",
      "Test accuracy: 0.1624\n",
      "2 data set\n",
      "Test loss: 5.590687184143066\n",
      "Test accuracy: 0.216\n",
      "3 data set\n",
      "Test loss: 3.4266992835998535\n",
      "Test accuracy: 0.5322\n",
      "4 data set\n",
      "Test loss: 2.7531645259857176\n",
      "Test accuracy: 0.5787\n",
      "5 data set\n",
      "Test loss: 0.2132988649956882\n",
      "Test accuracy: 0.9354\n",
      "6 data set\n",
      "Test loss: 0.03768906392035424\n",
      "Test accuracy: 0.9876\n"
     ]
    }
   ],
   "source": [
    "# Reload model\n",
    "model6 = load_model('trained_model5.h5')\n",
    "\n",
    "# Continue training\n",
    "model6.fit(x_train6, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score1 = model6.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model6.evaluate(x_test2, y_test, verbose=0)\n",
    "score3 = model6.evaluate(x_test3, y_test, verbose=0)\n",
    "score4 = model6.evaluate(x_test4, y_test, verbose=0)\n",
    "score5 = model6.evaluate(x_test5, y_test, verbose=0)\n",
    "score6 = model6.evaluate(x_test6, y_test, verbose=0)\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('2 data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "print('3 data set')\n",
    "print('Test loss:', score3[0])\n",
    "print('Test accuracy:', score3[1])\n",
    "\n",
    "print('4 data set')\n",
    "print('Test loss:', score4[0])\n",
    "print('Test accuracy:', score4[1])\n",
    "\n",
    "\n",
    "print('5 data set')\n",
    "print('Test loss:', score5[0])\n",
    "print('Test accuracy:', score5[1])\n",
    "\n",
    "print('6 data set')\n",
    "print('Test loss:', score6[0])\n",
    "print('Test accuracy:', score6[1])\n",
    "\n",
    "\n",
    "# Save trained model\n",
    "model6.save('trained_model6.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/5\n",
      "60000/60000 [==============================] - 157s 3ms/step - loss: 0.2613 - acc: 0.9309 - val_loss: 0.4489 - val_acc: 0.8767\n",
      "Epoch 2/5\n",
      "60000/60000 [==============================] - 154s 3ms/step - loss: 0.0899 - acc: 0.9730 - val_loss: 0.4068 - val_acc: 0.8979\n",
      "Epoch 3/5\n",
      "60000/60000 [==============================] - 155s 3ms/step - loss: 0.0708 - acc: 0.9792 - val_loss: 0.3240 - val_acc: 0.9127\n",
      "Epoch 4/5\n",
      "60000/60000 [==============================] - 149s 2ms/step - loss: 0.0602 - acc: 0.9820 - val_loss: 0.3660 - val_acc: 0.9073\n",
      "Epoch 5/5\n",
      "60000/60000 [==============================] - 150s 2ms/step - loss: 0.0533 - acc: 0.9843 - val_loss: 0.3952 - val_acc: 0.9059\n",
      "Original data set\n",
      "Test loss: 9.467570140075683\n",
      "Test accuracy: 0.1624\n",
      "2 data set\n",
      "Test loss: 5.590687184143066\n",
      "Test accuracy: 0.216\n",
      "3 data set\n",
      "Test loss: 3.4266992835998535\n",
      "Test accuracy: 0.5322\n",
      "4 data set\n",
      "Test loss: 2.7531645259857176\n",
      "Test accuracy: 0.5787\n",
      "5 data set\n",
      "Test loss: 0.2132988649956882\n",
      "Test accuracy: 0.9354\n",
      "6 data set\n",
      "Test loss: 0.03768906392035424\n",
      "Test accuracy: 0.9876\n",
      "7 data set\n",
      "Test loss: 7.3119373146057125\n",
      "Test accuracy: 0.1855\n"
     ]
    }
   ],
   "source": [
    "# Reload model\n",
    "model7 = load_model('trained_model6.h5')\n",
    "\n",
    "# Continue training\n",
    "model7.fit(x_train7, y_train, \n",
    "          batch_size=batch_size, \n",
    "          epochs=epochs, \n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score = model6.evaluate(x_test, y_test, verbose=0)\n",
    "score2 = model6.evaluate(x_test2, y_test, verbose=0)\n",
    "score3 = model6.evaluate(x_test3, y_test, verbose=0)\n",
    "score4 = model6.evaluate(x_test4, y_test, verbose=0)\n",
    "score5 = model6.evaluate(x_test5, y_test, verbose=0)\n",
    "score6 = model6.evaluate(x_test6, y_test, verbose=0)\n",
    "score7 = model6.evaluate(x_test7, y_test, verbose=0)\n",
    "\n",
    "print('Original data set')\n",
    "print('Test loss:', score1[0])\n",
    "print('Test accuracy:', score1[1])\n",
    "\n",
    "print('2 data set')\n",
    "print('Test loss:', score2[0])\n",
    "print('Test accuracy:', score2[1])\n",
    "\n",
    "print('3 data set')\n",
    "print('Test loss:', score3[0])\n",
    "print('Test accuracy:', score3[1])\n",
    "\n",
    "print('4 data set')\n",
    "print('Test loss:', score4[0])\n",
    "print('Test accuracy:', score4[1])\n",
    "\n",
    "\n",
    "print('5 data set')\n",
    "print('Test loss:', score5[0])\n",
    "print('Test accuracy:', score5[1])\n",
    "\n",
    "print('6 data set')\n",
    "print('Test loss:', score6[0])\n",
    "print('Test accuracy:', score6[1])\n",
    "\n",
    "print('7 data set')\n",
    "print('Test loss:', score7[0])\n",
    "print('Test accuracy:', score7[1])\n",
    "\n",
    "\n",
    "# Save trained model\n",
    "model7.save('trained_model7.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cnn_model_fn(features, labels, mode):\n",
    "\n",
    "    # Input Layer\n",
    "    input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n",
    "\n",
    "    # Convolutional layer 1\n",
    "    conv1 = tf.layers.conv2d(\n",
    "        inputs = input_layer,\n",
    "        filters = 32,\n",
    "        kernel_size=[5,5],\n",
    "        padding=\"same\",\n",
    "        activation=tf.nn.relu\n",
    "    )\n",
    "\n",
    "    # Pooling 1\n",
    "    pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)\n",
    "\n",
    "    # Convolutional layer 2 and pooling layer\n",
    "    conv2 = tf.layers.conv2d(\n",
    "        inputs=pool1,\n",
    "        filters=64,\n",
    "        kernel_size=[5,5],\n",
    "        padding=\"same\",\n",
    "        activation=tf.nn.relu\n",
    "    )\n",
    "\n",
    "    # Pooling 2 with flattening\n",
    "    pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2,2], strides=2)\n",
    "    pool2_flat=tf.reshape(pool2, [-1, 7 * 7 * 64])\n",
    "\n",
    "    # Dense layer with dropout \n",
    "    dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)\n",
    "    dropout=tf.layers.dropout(inputs=dense, rate=0.4, training=mode==tf.estimator.ModeKeys.TRAIN)\n",
    "    logits = tf.layers.dense(inputs=dropout, units=10)\n",
    "\n",
    "    # Generate predictions\n",
    "    predictions = {\n",
    "        \"classes\": tf.argmax(input=logits, axis=1),\n",
    "        \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n",
    "    }\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n",
    "\n",
    "    # Caluclate loss\n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
    "\n",
    "    # Configure training op\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n",
    "        train_op = optimizer.minimize(\n",
    "        loss=loss,\n",
    "        global_step=tf.train.get_global_step())\n",
    "        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
    "\n",
    "    # Add evaluation metrics\n",
    "    eval_metric_ops = {\n",
    "        \"accuracy\": tf.metrics.accuracy(labels=labels, predictions=predictions[\"classes\"])}\n",
    "\n",
    "    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Estimator\n",
    "mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn, model_dir=\"/tmp/mnist_convnet_model\")\n",
    "\n",
    "# Logging predictions\n",
    "tensors_to_log = {\"probabilities\": \"softmax_tensor\"}\n",
    "logging_hook = tf.train.LoggingTensorHook(\n",
    "    tensors=tensors_to_log, \n",
    "    every_n_iter=50)\n",
    "\n",
    "# Our application logic will be added here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(unused_argv):\n",
    "    # Training on original\n",
    "    train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={\"x\":train_original},\n",
    "        y=train_labels,\n",
    "        batch_size=1000,\n",
    "        num_epochs=None,\n",
    "        shuffle=True)\n",
    "    \n",
    "    mnist_classifier.train(\n",
    "        input_fn=train_input_fn,\n",
    "        steps=200,\n",
    "        hooks=[logging_hook])\n",
    "\n",
    "    # Evaluation on original\n",
    "    eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "        x={\"x\": eval_original},\n",
    "        y=eval_labels,\n",
    "        num_epochs=1,\n",
    "        shuffle=False)\n",
    "    \n",
    "    eval_results=mnist_classifier.evaluate(input_fn=eval_input_fn)\n",
    "    print(eval_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "  tf.app.run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}