Tensorflow - CIFAR10 recognition.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "#numpy.set_printoptions(threshold=numpy.nan)\n",
    "def unpickle(file):\n",
    "    import pickle\n",
    "    with open(file, 'rb') as fo:\n",
    "        dict = pickle.load(fo, encoding='bytes')\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cifar10_name = {\n",
    "    0: \"airplane\",\n",
    "    1: \"automobile\",\n",
    "    2: \"bird\",\n",
    "    3: \"cat\",\n",
    "    4: \"deer\",\n",
    "    5: \"dog\",\n",
    "    6: \"frog\",\n",
    "    7: \"horse\",\n",
    "    8: \"ship\",\n",
    "    9: \"truck\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    return tf.Variable(tf.truncated_normal(shape, stddev=0.1))\n",
    "def bias_variable(shape):\n",
    "    return tf.Variable(tf.constant(0.1, shape=shape))\n",
    "\n",
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "def next_batch(num, data, labels):\n",
    "    '''\n",
    "    Return a total of `num` random samples and labels. \n",
    "    '''\n",
    "    idx = np.arange(0 , len(data))\n",
    "    np.random.shuffle(idx)\n",
    "    idx = idx[:num]\n",
    "    data_shuffle = [data[i] for i in idx]\n",
    "    labels_shuffle = [labels[i] for i in idx]\n",
    "\n",
    "    return np.asarray(data_shuffle), np.asarray(labels_shuffle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data1 = unpickle(\"cifar-10-batches-py/data_batch_1\")\n",
    "data2 = unpickle(\"cifar-10-batches-py/data_batch_2\")\n",
    "data3 = unpickle(\"cifar-10-batches-py/data_batch_3\")\n",
    "data4 = unpickle(\"cifar-10-batches-py/data_batch_4\")\n",
    "data5 = unpickle(\"cifar-10-batches-py/data_batch_5\")\n",
    "data = {**data1, **data2, **data3, **data4, **data5}\n",
    "\n",
    "test = unpickle(\"cifar-10-batches-py/test_batch\")\n",
    "lables = unpickle(\"cifar-10-batches-py/batches.meta\")[b'label_names']\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 3072])\n",
    "\n",
    "x_image = tf.reshape(x, [-1, 32, 32, 3])\n",
    "W_conv1 = weight_variable([5, 5, 3, 32])\n",
    "b_conv1 = bias_variable([32])\n",
    "\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1)\n",
    "\n",
    "W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "b_conv2 = bias_variable([64])\n",
    "\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)\n",
    "\n",
    "W_fc1 = weight_variable([8 * 8 * 64, 1024])\n",
    "b_fc1 = bias_variable([1024])\n",
    "\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 8*8*64])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "\n",
    "W_fc2 = weight_variable([1024, 64])\n",
    "b_fc2 = bias_variable([64])\n",
    "\n",
    "h_fc2 = tf.matmul(h_fc1, W_fc2) + b_fc2\n",
    "\n",
    "keep_prob = tf.placeholder(tf.float32)\n",
    "h_fc2_drop = tf.nn.dropout(h_fc2, keep_prob)\n",
    "\n",
    "W_fc3 = weight_variable([64, 10])\n",
    "b_fc3 = bias_variable([10])\n",
    "\n",
    "y = tf.matmul(h_fc2_drop, W_fc3) + b_fc3\n",
    "\n",
    "result = tf.nn.softmax(y)\n",
    "\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learningrate = tf.placeholder(tf.float32, [])\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "train_step = tf.train.AdamOptimizer(learning_rate=learningrate).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data = []\n",
    "train_labels = []\n",
    "for i in range(len(data[b'labels'])):\n",
    "    img_dat = data[b'data'][i]\n",
    "    label_dat = np.zeros(10)\n",
    "    label_dat[data[b'labels'][i]] = 1\n",
    "    \n",
    "    train_data.append(img_dat)\n",
    "    train_labels.append(label_dat)\n",
    "    \n",
    "    \n",
    "test_data = []\n",
    "test_labels = []\n",
    "for i in range(len(test[b'labels'])):\n",
    "    img_dat = data[b'data'][i]\n",
    "    label_dat = np.zeros(10)\n",
    "    label_dat[test[b'labels'][i]] = 1\n",
    "    \n",
    "    test_data.append(img_dat)\n",
    "    test_labels.append(label_dat)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(init)\n",
    "\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "ename": "NotFoundError",
     "evalue": "Key beta2_power_8 not found in checkpoint\n\t [[Node: save_9/RestoreV2_287 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_9/Const_0, save_9/RestoreV2_287/tensor_names, save_9/RestoreV2_287/shape_and_slices)]]\n\nCaused by op 'save_9/RestoreV2_287', defined at:\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tornado\\ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tornado\\stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tornado\\stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-88-7466c5a4cacd>\", line 6, in <module>\n    saver = tf.train.Saver()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 1040, in __init__\n    self.build()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 1070, in build\n    restore_sequentially=self._restore_sequentially)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 675, in build\n    restore_sequentially, reshape)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 402, in _AddRestoreOps\n    tensors = self.restore_op(filename_tensor, saveable, preferred_shard)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 242, in restore_op\n    [spec.tensor.dtype])[0])\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\ops\\gen_io_ops.py\", line 668, in restore_v2\n    dtypes=dtypes, name=name)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 763, in apply_op\n    op_def=op_def)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2327, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1226, in __init__\n    self._traceback = _extract_stack()\n\nNotFoundError (see above for traceback): Key beta2_power_8 not found in checkpoint\n\t [[Node: save_9/RestoreV2_287 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_9/Const_0, save_9/RestoreV2_287/tensor_names, save_9/RestoreV2_287/shape_and_slices)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1021\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1022\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1023\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1003\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1004\u001b[1;33m                                  status, run_metadata)\n\u001b[0m\u001b[0;32m   1005\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type, value, traceback)\u001b[0m\n\u001b[0;32m     65\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m                 \u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     67\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[1;34m()\u001b[0m\n\u001b[0;32m    465\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 466\u001b[1;33m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[0;32m    467\u001b[0m   \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNotFoundError\u001b[0m: Key beta2_power_8 not found in checkpoint\n\t [[Node: save_9/RestoreV2_287 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_9/Const_0, save_9/RestoreV2_287/tensor_names, save_9/RestoreV2_287/shape_and_slices)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-89-b1e5bb60c3b0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#Load the model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0msaver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msess\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"./tf-models/cifar10.ckpt\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, sess, save_path)\u001b[0m\n\u001b[0;32m   1426\u001b[0m       \u001b[1;32mreturn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1427\u001b[0m     sess.run(self.saver_def.restore_op_name,\n\u001b[1;32m-> 1428\u001b[1;33m              {self.saver_def.filename_tensor_name: save_path})\n\u001b[0m\u001b[0;32m   1429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1430\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    765\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 767\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    768\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    963\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m--> 965\u001b[1;33m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[0;32m    966\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1013\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[1;32m-> 1015\u001b[1;33m                            target_list, options, run_metadata)\n\u001b[0m\u001b[0;32m   1016\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[1;32mc:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1033\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1034\u001b[0m           \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1035\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1036\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1037\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNotFoundError\u001b[0m: Key beta2_power_8 not found in checkpoint\n\t [[Node: save_9/RestoreV2_287 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_9/Const_0, save_9/RestoreV2_287/tensor_names, save_9/RestoreV2_287/shape_and_slices)]]\n\nCaused by op 'save_9/RestoreV2_287', defined at:\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tornado\\ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tornado\\stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tornado\\stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-88-7466c5a4cacd>\", line 6, in <module>\n    saver = tf.train.Saver()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 1040, in __init__\n    self.build()\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 1070, in build\n    restore_sequentially=self._restore_sequentially)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 675, in build\n    restore_sequentially, reshape)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 402, in _AddRestoreOps\n    tensors = self.restore_op(filename_tensor, saveable, preferred_shard)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\training\\saver.py\", line 242, in restore_op\n    [spec.tensor.dtype])[0])\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\ops\\gen_io_ops.py\", line 668, in restore_v2\n    dtypes=dtypes, name=name)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 763, in apply_op\n    op_def=op_def)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 2327, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"c:\\users\\bibows\\appdata\\local\\programs\\python\\python35\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1226, in __init__\n    self._traceback = _extract_stack()\n\nNotFoundError (see above for traceback): Key beta2_power_8 not found in checkpoint\n\t [[Node: save_9/RestoreV2_287 = RestoreV2[dtypes=[DT_FLOAT], _device=\"/job:localhost/replica:0/task:0/cpu:0\"](_recv_save_9/Const_0, save_9/RestoreV2_287/tensor_names, save_9/RestoreV2_287/shape_and_slices)]]\n"
     ]
    }
   ],
   "source": [
    "#Load the model\n",
    "saver.restore(sess, \"./tf-models/cifar10.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test error 2068.83, train error 2031.65\n",
      "test error 1867.44, train error 1847.32"
     ]
    }
   ],
   "source": [
    "try:    \n",
    "    while True:\n",
    "        test_batch =next_batch(25, test_data, test_labels)\n",
    "        train_batch = next_batch(25, train_data, train_labels)\n",
    "        te = cross_entropy.eval(feed_dict={x: test_batch[0], y_: test_batch[1], keep_prob: 1.0})\n",
    "        tee = cross_entropy.eval(feed_dict={x: train_batch[0], y_: train_batch[1], keep_prob: 1.0})\n",
    "        print(\"\\rtest error %g, train error %g\"%(te, tee), end=\"\")\n",
    "        for i in range(11):\n",
    "            batch = next_batch(200, train_data, train_labels)\n",
    "            sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.3, learningrate: 3e-6})\n",
    "except KeyboardInterrupt:\n",
    "    print(\"\")\n",
    "test_batch =next_batch(200, test_data, test_labels)\n",
    "train_batch = next_batch(200, train_data, train_labels)\n",
    "te = cross_entropy.eval(feed_dict={x: test_batch[0], y_: test_batch[1], keep_prob: 1.0})\n",
    "tee = cross_entropy.eval(feed_dict={x: train_batch[0], y_: train_batch[1], keep_prob: 1.0})\n",
    "print(\"test error %g, train error %g\"%(te, tee), end=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the model\n",
    "saver.save(sess, \"./tf-models/cifar10.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = random.randint(0,len(test_data))\n",
    "num = 5\n",
    "\n",
    "res = sess.run(result, feed_dict={x: test_data[image:image+num], keep_prob: 1})\n",
    "\n",
    "for j,e in enumerate(res):\n",
    "    data = []\n",
    "    for i in range(32):\n",
    "        tmp = []\n",
    "        for k in range(32):\n",
    "            img_dat_r = test[b'data'][image+j][32*i+k-1]\n",
    "            img_dat_g = test[b'data'][image+j][1024+32*i+k-1]\n",
    "            img_dat_b = test[b'data'][image+j][2048+32*i+k-1]\n",
    "            tmp.append([img_dat_r,img_dat_g,img_dat_b])\n",
    "        data.append(tmp)\n",
    "    dat = np.array(data, dtype=np.float32)\n",
    "    \n",
    "    fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "    fig_size[0] = 2\n",
    "    fig_size[1] = 2\n",
    "    plt.rcParams[\"figure.figsize\"] = fig_size\n",
    "    \n",
    "    plt.imshow(dat)\n",
    "    plt.show()\n",
    "    \n",
    "    print(\"suggested label: {:s}\\nconfidence: {:.0f}%\\ncorrect label: {:s}\".format(cifar10_name[e.argmax()], e.max()*100, cifar10_name[test_labels[image+j].argmax()]))\n",
    "    \n",
    "    #for i in range(10):\n",
    "    #    print(lables[i], \" \\t\", e[i], \" \\t\", test_labels[image+j][i])\n",
    "    #print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}