05_convolutional_net.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, Convolution2D, MaxPooling2D, Dropout\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils\n",
    "from keras import initializations\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "nb_classes = 10\n",
    "nb_epoch = 100\n",
    "\n",
    "img_rows, img_cols = 28, 28         # input image dimensions\n",
    "pool_size = (2, 2)                  # size of pooling area for max pooling\n",
    "prob_drop_conv = 0.2                # drop probability for dropout @ conv layer\n",
    "prob_drop_hidden = 0.5              # drop probability for dropout @ fc layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_weights(shape, name=None):\n",
    "    return initializations.normal(shape, scale=0.01, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train original shape: (60000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# Load MNIST dataset\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "print('X_train original shape:', X_train.shape)\n",
    "\n",
    "if K.image_dim_ordering() == 'th':\n",
    "    # For Theano backend\n",
    "    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)\n",
    "    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    # For TensorFlow backend\n",
    "    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (60000, 28, 28, 1)\n",
      "60000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "X_train = X_train.astype('float32') / 255.\n",
    "X_test = X_test.astype('float32') / 255.\n",
    "Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "\n",
    "print('X_train shape:', X_train.shape)\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "convolution2d_1 (Convolution2D)  (None, 28, 28, 32)    320         convolution2d_input_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_1 (MaxPooling2D)    (None, 14, 14, 32)    0           convolution2d_1[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 14, 14, 32)    0           maxpooling2d_1[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_2 (Convolution2D)  (None, 14, 14, 64)    18496       dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_2 (MaxPooling2D)    (None, 7, 7, 64)      0           convolution2d_2[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 7, 7, 64)      0           maxpooling2d_2[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_3 (Convolution2D)  (None, 7, 7, 128)     73856       dropout_2[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "maxpooling2d_3 (MaxPooling2D)    (None, 4, 4, 128)     0           convolution2d_3[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)              (None, 2048)          0           maxpooling2d_3[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)              (None, 2048)          0           flatten_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 625)           1280625     dropout_3[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)              (None, 625)           0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 10)            6260        dropout_4[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 1379557\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Convolutional model\n",
    "model = Sequential()\n",
    "\n",
    "# conv1 layer\n",
    "model.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu', input_shape=input_shape, init=init_weights))\n",
    "model.add(MaxPooling2D(pool_size=pool_size, strides=(2,2), border_mode='same'))\n",
    "model.add(Dropout(prob_drop_conv))\n",
    "\n",
    "# conv2 layer\n",
    "model.add(Convolution2D(64, 3, 3, border_mode='same', activation='relu', init=init_weights))\n",
    "model.add(MaxPooling2D(pool_size=pool_size, strides=(2,2), border_mode='same'))\n",
    "model.add(Dropout(prob_drop_conv))\n",
    "\n",
    "# conv3 layer\n",
    "model.add(Convolution2D(128, 3, 3, border_mode='same', activation='relu', init=init_weights))\n",
    "model.add(MaxPooling2D(pool_size=pool_size, strides=(2,2), border_mode='same'))\n",
    "model.add(Flatten())\n",
    "model.add(Dropout(prob_drop_conv))\n",
    "\n",
    "# fc1 layer\n",
    "model.add(Dense(625, activation='relu', init=init_weights))\n",
    "model.add(Dropout(prob_drop_hidden))\n",
    "\n",
    "# fc2 layer\n",
    "model.add(Dense(10, activation='softmax', init=init_weights))\n",
    "\n",
    "opt = RMSprop(lr=0.001, rho=0.9)\n",
    "model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "60000/60000 [==============================] - 6s - loss: 0.4027 - acc: 0.8651     \n",
      "Epoch 2/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.1045 - acc: 0.9677     \n",
      "Epoch 3/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0732 - acc: 0.9773     \n",
      "Epoch 4/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0572 - acc: 0.9819     \n",
      "Epoch 5/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0493 - acc: 0.9848     \n",
      "Epoch 6/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0444 - acc: 0.9867     \n",
      "Epoch 7/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0407 - acc: 0.9878     \n",
      "Epoch 8/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0379 - acc: 0.9882     \n",
      "Epoch 9/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0354 - acc: 0.9897     \n",
      "Epoch 10/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0341 - acc: 0.9897     \n",
      "Epoch 11/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0310 - acc: 0.9910     \n",
      "Epoch 12/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0305 - acc: 0.9907     \n",
      "Epoch 13/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0309 - acc: 0.9908     \n",
      "Epoch 14/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0299 - acc: 0.9913     \n",
      "Epoch 15/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0294 - acc: 0.9913     \n",
      "Epoch 16/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0299 - acc: 0.9918     \n",
      "Epoch 17/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0298 - acc: 0.9913     \n",
      "Epoch 18/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0286 - acc: 0.9922     \n",
      "Epoch 19/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0268 - acc: 0.9920     \n",
      "Epoch 20/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0295 - acc: 0.9916     \n",
      "Epoch 21/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0296 - acc: 0.9917     \n",
      "Epoch 22/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0281 - acc: 0.9920     \n",
      "Epoch 23/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0294 - acc: 0.9920     \n",
      "Epoch 24/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0303 - acc: 0.9915     \n",
      "Epoch 25/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0319 - acc: 0.9912     \n",
      "Epoch 26/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0310 - acc: 0.9916     \n",
      "Epoch 27/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0300 - acc: 0.9913     \n",
      "Epoch 28/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0316 - acc: 0.9911     \n",
      "Epoch 29/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0325 - acc: 0.9913     \n",
      "Epoch 30/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0329 - acc: 0.9918     \n",
      "Epoch 31/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0326 - acc: 0.9910     \n",
      "Epoch 32/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0352 - acc: 0.9905     \n",
      "Epoch 33/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0348 - acc: 0.9904     \n",
      "Epoch 34/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0344 - acc: 0.9909     \n",
      "Epoch 35/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0380 - acc: 0.9906     \n",
      "Epoch 36/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0338 - acc: 0.9909     \n",
      "Epoch 37/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0343 - acc: 0.9911     \n",
      "Epoch 38/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0343 - acc: 0.9905     \n",
      "Epoch 39/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0355 - acc: 0.9905     \n",
      "Epoch 40/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0351 - acc: 0.9906     \n",
      "Epoch 41/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0382 - acc: 0.9903     \n",
      "Epoch 42/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0388 - acc: 0.9898     \n",
      "Epoch 43/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0380 - acc: 0.9896     \n",
      "Epoch 44/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0387 - acc: 0.9898     \n",
      "Epoch 45/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0445 - acc: 0.9892     \n",
      "Epoch 46/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0405 - acc: 0.9899     \n",
      "Epoch 47/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0409 - acc: 0.9897     \n",
      "Epoch 48/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0403 - acc: 0.9897     \n",
      "Epoch 49/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0390 - acc: 0.9903     \n",
      "Epoch 50/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0443 - acc: 0.9893     \n",
      "Epoch 51/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0414 - acc: 0.9897     \n",
      "Epoch 52/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0437 - acc: 0.9890     \n",
      "Epoch 53/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0467 - acc: 0.9893     \n",
      "Epoch 54/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0455 - acc: 0.9881     \n",
      "Epoch 55/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0441 - acc: 0.9892     \n",
      "Epoch 56/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0469 - acc: 0.9885     \n",
      "Epoch 57/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0474 - acc: 0.9886     \n",
      "Epoch 58/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0484 - acc: 0.9880     \n",
      "Epoch 59/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0505 - acc: 0.9878     \n",
      "Epoch 60/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0470 - acc: 0.9888     \n",
      "Epoch 61/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0478 - acc: 0.9882     \n",
      "Epoch 62/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0512 - acc: 0.9881     \n",
      "Epoch 63/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0562 - acc: 0.9876     \n",
      "Epoch 64/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0496 - acc: 0.9879     \n",
      "Epoch 65/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0504 - acc: 0.9879     \n",
      "Epoch 66/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0508 - acc: 0.9875     \n",
      "Epoch 67/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0548 - acc: 0.9872     \n",
      "Epoch 68/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0532 - acc: 0.9871     \n",
      "Epoch 69/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0508 - acc: 0.9879     \n",
      "Epoch 70/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0533 - acc: 0.9870     \n",
      "Epoch 71/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0524 - acc: 0.9873     \n",
      "Epoch 72/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0549 - acc: 0.9874     \n",
      "Epoch 73/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0558 - acc: 0.9861     \n",
      "Epoch 74/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0531 - acc: 0.9864     \n",
      "Epoch 75/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0539 - acc: 0.9864     \n",
      "Epoch 76/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0574 - acc: 0.9862     \n",
      "Epoch 77/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0574 - acc: 0.9862     \n",
      "Epoch 78/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0597 - acc: 0.9856     \n",
      "Epoch 79/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0582 - acc: 0.9856     \n",
      "Epoch 80/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0592 - acc: 0.9854     \n",
      "Epoch 81/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0571 - acc: 0.9865     \n",
      "Epoch 82/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0690 - acc: 0.9847     \n",
      "Epoch 83/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0598 - acc: 0.9856     \n",
      "Epoch 84/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0606 - acc: 0.9855     \n",
      "Epoch 85/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0621 - acc: 0.9849     \n",
      "Epoch 86/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0673 - acc: 0.9844     \n",
      "Epoch 87/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0599 - acc: 0.9850     \n",
      "Epoch 88/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0571 - acc: 0.9858     \n",
      "Epoch 89/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0651 - acc: 0.9841     \n",
      "Epoch 90/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0661 - acc: 0.9838     \n",
      "Epoch 91/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0673 - acc: 0.9833     \n",
      "Epoch 92/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0663 - acc: 0.9838     \n",
      "Epoch 93/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0736 - acc: 0.9827     \n",
      "Epoch 94/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0647 - acc: 0.9840     \n",
      "Epoch 95/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0660 - acc: 0.9837     \n",
      "Epoch 96/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0646 - acc: 0.9837     \n",
      "Epoch 97/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0635 - acc: 0.9844     \n",
      "Epoch 98/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0638 - acc: 0.9841     \n",
      "Epoch 99/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0685 - acc: 0.9832     \n",
      "Epoch 100/100\n",
      "60000/60000 [==============================] - 5s - loss: 0.0664 - acc: 0.9838     \n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "history = model.fit(X_train, Y_train, nb_epoch=nb_epoch, batch_size=batch_size, shuffle=True, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9984/10000 [============================>.] - ETA: 0sSummary: Loss over the test dataset: 0.03, Accuracy: 0.99\n"
     ]
    }
   ],
   "source": [
    "# Evaluate\n",
    "evaluation = model.evaluate(X_test, Y_test, batch_size=256, verbose=1)\n",
    "print('Summary: Loss over the test dataset: %.2f, Accuracy: %.2f' % (evaluation[0], evaluation[1]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}