tf_nn_epochs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "import random\n",
    "import functools\n",
    "from functools import reduce\n",
    "import math\n",
    "from math import ceil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image_size = 28\n",
    "input_num_units=28*28\n",
    "num_labels = 10\n",
    "output_num_units=num_labels\n",
    "learning_rate=0.1\n",
    "num_steps =501\n",
    "num_train_samples = 10000\n",
    "beta=0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# number of neurons in each layer\n",
    "input_num_units = 784\n",
    "hidden_num_units = 1024\n",
    "output_num_units = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 784) (200000, 10)\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n",
    "    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)#Concise 1-Hot Encoding in NUMPY\n",
    "    return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getSampleIndexes(population,sample_sz):\n",
    "    if(sample_sz>population.shape[0]):\n",
    "        return np.array(range(population.shape[0]))\n",
    "    rand_indexes=random.sample(range(population.shape[0]),sample_sz)\n",
    "    return population[rand_indexes,:],rand_indexes\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))/ predictions.shape[0])\n",
    "def isEqual(nparr1,nparr2):\n",
    "    totDims=len(nparr1.shape)\n",
    "    if np.sum([nparr1.shape[i]==nparr2.shape[i] for i in range(totDims)])!=totDims:\n",
    "        return False\n",
    "    eqItems=nparr1==nparr2\n",
    "    if np.sum(eqItems)==reduce(lambda x,y:x*y,nparr1.shape):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "def getEquiDistantIndexSplits(num_samples,batch_count):\n",
    "    #num_samples=population.shape[0]\n",
    "    n=int(ceil(num_samples/batch_count))\n",
    "    sub_sample_cnts=[batch_count for i in range(n)]\n",
    "    rem_indexes=num_samples%batch_count\n",
    "    sub_sample_cnts=[sub_sample_cnts[i]+1 if i <rem_indexes else sub_sample_cnts[i] for i in range(len(sub_sample_cnts))]\n",
    "    cum_subsmpl_cnts=np.cumsum(sub_sample_cnts)\n",
    "    indices=[np.arange(0,cum_subsmpl_cnts[i]) if i==0 else np.arange(cum_subsmpl_cnts[i-1],cum_subsmpl_cnts[i])\n",
    "            for i in range(cum_subsmpl_cnts.shape[0])]\n",
    "    return indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wt_h=None\n",
    "wt_o=None\n",
    "bias_h=None\n",
    "bias_o=None\n",
    "graph_random_weights=tf.Graph()\n",
    "with  graph_random_weights.as_default():\n",
    "    weights_hidden=tf.Variable(tf.random_normal([input_num_units, hidden_num_units]))\n",
    "    weights_output=tf.Variable(tf.random_normal([hidden_num_units, output_num_units]))\n",
    "    biases_hidden=tf.Variable(tf.random_normal([hidden_num_units]))\n",
    "    biases_output= tf.Variable(tf.random_normal([output_num_units]))\n",
    "with tf.Session(graph=graph_random_weights) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    wt_h,wt_o,bias_h,bias_o=session.run([weights_hidden,weights_output,biases_hidden,biases_output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "graph_nn = tf.Graph()\n",
    "def process_2layered_input(ip_tf,wt_hid,wt_op,bias_hid,bias_op):\n",
    "    layer_1_linear_op=tf.add(tf.matmul(ip_tf,wt_hid),bias_hid)\n",
    "    layer_1_non_lin_op=tf.nn.relu(layer_1_linear_op)\n",
    "    layer_2_linear_op=tf.add(tf.matmul(layer_1_non_lin_op,wt_op),bias_op)\n",
    "    #layer_2_non_lin_op=tf.nn.relu(layer_2_linear_op)\n",
    "    #return layer_2_non_lin_op\n",
    "    return layer_2_linear_op\n",
    "with graph_nn.as_default():\n",
    "    # define placeholders\n",
    "    x = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "    y = tf.placeholder(tf.float32, [None, output_num_units])\n",
    "    valid_dataset_tf = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "    test_dataset_tf = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "    weights_hidden=tf.Variable(wt_h)\n",
    "    weights_output=tf.Variable(wt_o)\n",
    "    biases_hidden=tf.Variable(bias_h)\n",
    "    biases_output= tf.Variable(bias_o)\n",
    "    output_layer=process_2layered_input(x,weights_hidden,weights_output,biases_hidden,biases_output)\n",
    "    total_loss=tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=output_layer)\n",
    "    loss_fun = tf.reduce_mean(total_loss)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss_fun)\n",
    "    train_prediction = output_layer\n",
    "    valid_prediction=process_2layered_input(valid_dataset_tf,weights_hidden,weights_output,biases_hidden,\n",
    "                                            biases_output)\n",
    "    test_prediction=process_2layered_input(test_dataset_tf,weights_hidden,weights_output,biases_hidden,\n",
    "                                            biases_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Neural Network  Terminology: EPOCHs\n",
    "    One Epoch :\n",
    "         A set of (Forward,Backward) passes, s.t. these passes combined cover ALL Training Examples\n",
    "    Batch Size :\n",
    "        Number of Training examples in one Forward/Backward Pass. \n",
    "        Directly affects the memory requirement of the Network. Higher the batch size, more memory required.\n",
    "    Number of Iterations :\n",
    "        Iteration :  1 Forward Pass followed by 1 Backward Pass\n",
    "        Network updates Weights Once & Only ONCE Every Iteration\n",
    "        Each Iteration uses [Batch Size] number of examples. \n",
    "    Example: Given 500 training examples and a Batch size of 100 ~>\n",
    "    `Network will require 5 Iterations to Complete 1 Epoch.\n",
    "\"\"\"\n",
    "\"\"\"\"\"\"\n",
    "TOTAL_TRAIN_EXAMPLES=train_dataset.shape[0]\n",
    "NUM_EPOCHS=10\n",
    "BATCH_SIZE=num_train_samples\n",
    "ITER_INDEXES=getEquiDistantIndexSplits(TOTAL_TRAIN_EXAMPLES,BATCH_SIZE)\n",
    "NUM_ITERS=len(ITER_INDEXES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 10000)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ITER_INDEXES),BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "(10000, 10)\n",
      "Computational Graph : Validation Passed!\n",
      "EPOCH # 0   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 381.152405\n",
      "Accuracy ~~~~~~~~~~~~> 29.51 31.93 34.61\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 55.995899\n",
      "Accuracy ~~~~~~~~~~~~> 74.59 73.97 82.57\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 1   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 54.879944\n",
      "Accuracy ~~~~~~~~~~~~> 75.07 73.33 82.07\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 48.548775\n",
      "Accuracy ~~~~~~~~~~~~> 75.4 74.33 82.68\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 2   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 48.463284\n",
      "Accuracy ~~~~~~~~~~~~> 74.99 74.03 82.43\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 42.102642\n",
      "Accuracy ~~~~~~~~~~~~> 77.33 75.98 84.38\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 3   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 41.590714\n",
      "Accuracy ~~~~~~~~~~~~> 76.95 76.39 84.6\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 39.023838\n",
      "Accuracy ~~~~~~~~~~~~> 77.34 76.3 84.74\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 4   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 38.850574\n",
      "Accuracy ~~~~~~~~~~~~> 77.51 76.81 84.93\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 35.505680\n",
      "Accuracy ~~~~~~~~~~~~> 78.0 76.79 85.42\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 5   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 35.560665\n",
      "Accuracy ~~~~~~~~~~~~> 77.97 77.34 85.48\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 32.910267\n",
      "Accuracy ~~~~~~~~~~~~> 78.44 77.35 85.93\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 6   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 32.954422\n",
      "Accuracy ~~~~~~~~~~~~> 78.56 77.64 86.12\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 31.153503\n",
      "Accuracy ~~~~~~~~~~~~> 78.95 77.46 86.04\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 7   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 31.229202\n",
      "Accuracy ~~~~~~~~~~~~> 78.81 78.1 86.2\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 30.405966\n",
      "Accuracy ~~~~~~~~~~~~> 79.18 77.65 85.95\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 8   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 30.225132\n",
      "Accuracy ~~~~~~~~~~~~> 78.88 78.44 86.54\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 28.107353\n",
      "Accuracy ~~~~~~~~~~~~> 79.88 78.18 86.43\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "EPOCH # 9   >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 0: 28.277599\n",
      "Accuracy ~~~~~~~~~~~~> 79.41 78.7 86.85\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 19: 27.229994\n",
      "Accuracy ~~~~~~~~~~~~> 79.83 78.1 86.41\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
     ]
    }
   ],
   "source": [
    "#With Epochs\n",
    "def getFeedDict(train_x,train_y):\n",
    "        return {x:train_x,y:train_y,valid_dataset_tf:valid_dataset,\n",
    "              test_dataset_tf:test_dataset}\n",
    "def validate_tf_comput_graph(session,tf_feed_dict):\n",
    "    _=session.run([optimizer],feed_dict=tf_feed_dict)\n",
    "    train_pred_1=session.run(train_prediction,feed_dict=tf_feed_dict)\n",
    "    print(train_pred_1.shape)\n",
    "    train_pred_2=session.run(train_prediction,feed_dict=tf_feed_dict)\n",
    "    if isEqual(train_pred_1,train_pred_2):\n",
    "        print(\"Computational Graph : Validation Passed!\")\n",
    "    else:\n",
    "        print(\"Computational Graph : Validation Failed!\")\n",
    "with tf.Session(graph=graph_nn) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    validate_tf_comput_graph(session,getFeedDict(train_dataset[ITER_INDEXES[0]],train_labels[ITER_INDEXES[0]]))\n",
    "    for epoch in range(NUM_EPOCHS):\n",
    "        print(\"EPOCH #\",epoch ,\"  >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")\n",
    "        for step,ITER_INDEX_SET in enumerate(ITER_INDEXES):\n",
    "            train_x=train_dataset[ITER_INDEX_SET]\n",
    "            train_y=train_labels[ITER_INDEX_SET]\n",
    "            _,loss,train_pred=session.run([optimizer,loss_fun, train_prediction],feed_dict=getFeedDict(train_x,train_y))\n",
    "            if step%19==0:\n",
    "                print('Loss at step %d: %f' % (step, loss))\n",
    "                valid_pred,test_pred=session.run([valid_prediction,test_prediction],feed_dict=getFeedDict(train_x,train_y))\n",
    "                train_acc=accuracy(train_pred, train_y)\n",
    "                valid_acc=accuracy(valid_pred, valid_labels)\n",
    "                test_acc=accuracy(test_pred,test_labels)\n",
    "                print(\"Accuracy ~~~~~~~~~~~~>\",train_acc,valid_acc,test_acc )\n",
    "                print(\">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}