Convolutional Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#a toy example of convolutional neural network\n",
    "#code written on python3.5 using only numpy for forward and backward passes\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams[\"figure.figsize\"] = [12, 9]\n",
    "\n",
    "image = np.array([\n",
    "    [1, 2, 3, 4, 5, 6],\n",
    "    [7, 8, 9, 10,11,12],\n",
    "    [13,14,15,16,17,18],\n",
    "    [13,14,15,16,17,18],\n",
    "    [7, 8, 9, 10,11,12],\n",
    "    [1, 2, 3, 4, 5, 6]])\n",
    "\n",
    "y = 10\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "np.random.seed(1234)\n",
    "kernel = 2*np.random.random((5, 5)) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv_forw(image, kernel):\n",
    "    output = np.zeros([image.shape[0]-kernel.shape[0]+1, image.shape[1]-kernel.shape[1]+1])\n",
    "    for i in range(output.shape[0]):\n",
    "        for j in range(output.shape[1]):\n",
    "            part_of_the_image = image[i:kernel.shape[0]+i, j:kernel.shape[1]+j]\n",
    "            #print('kernel part:', '\\n', part_of_the_image)\n",
    "            output[i,j] = (np.multiply(part_of_the_image, kernel)).sum()\n",
    "\n",
    "    return output\n",
    "\n",
    "def conv_back(image, kernel):\n",
    "    #rotate kernel by 180 degrees for backpropagation \n",
    "    kernel = np.flip(kernel, axis=1)\n",
    "    kernel = np.flip(kernel, axis=0)\n",
    "    image_zeros = np.zeros([image.shape[0]+kernel.shape[0]*2-2, image.shape[1]+kernel.shape[1]*2-2])\n",
    "    #zero padding of input image around the border for valid backpropagation of conv layer\n",
    "    for i in range(image.shape[0]):\n",
    "        for j in range(image.shape[1]):\n",
    "            image_zeros[i+kernel.shape[0]-1, j+kernel.shape[1]-1] = image[i,j]\n",
    "    image = image_zeros\n",
    "    output = conv_forw(image, kernel)\n",
    "    return output\n",
    "\n",
    "def maxpooling(image):\n",
    "    heigh = 2\n",
    "    width = 2\n",
    "\n",
    "    #zero padding\n",
    "    if image.shape[0]%heigh != 0:\n",
    "        image_padded_heigh = (image.shape[0]//heigh)*heigh + heigh\n",
    "    else:\n",
    "        image_padded_heigh = (image.shape[0]//heigh)*heigh\n",
    "\n",
    "    if image.shape[1]%width != 0:\n",
    "        image_padded_width = (image.shape[1]//width)*width + width\n",
    "    else:\n",
    "        image_padded_width = (image.shape[1]//width)*width\n",
    "\n",
    "    image_padded = np.zeros([image_padded_heigh, image_padded_width])\n",
    "    for i in range(image.shape[0]):\n",
    "        for j in range(image.shape[1]):\n",
    "            image_padded[i, j] = image[i,j]\n",
    "\n",
    "    #maxpooling\n",
    "    image_back = np.zeros([image_padded_heigh, image_padded_width])\n",
    "    output = np.zeros([int(image_padded_heigh/heigh), int(image_padded_width/width)])\n",
    "    for i in range(output.shape[0]):\n",
    "        for j in range(output.shape[1]):\n",
    "            part_of_the_image = image_padded[i*heigh:i*heigh+heigh, j*width:j*width+width]\n",
    "            output[i,j] = (np.max(part_of_the_image))\n",
    "            #index of maximum value for backprop\n",
    "            max_index = np.unravel_index(part_of_the_image.argmax(), part_of_the_image.shape)\n",
    "            image_back[max_index[0]+i*heigh, max_index[1]+j*width] = 1\n",
    "    return output, image_back\n",
    "\n",
    "#need to change to leaky relu\n",
    "def relu_forw(image):\n",
    "    output = np.copy(image)\n",
    "    output[output<0]=0\n",
    "    return output\n",
    "\n",
    "def relu_back(image):\n",
    "    output = np.copy(image)\n",
    "    output[output<=0]=0\n",
    "    output[output>0]=1\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAIMCAYAAADcq5vIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3WuMpNd5H/j/U5fuaQ41vEgjhhrSJmPTF0pZr+yBIsdB\n1mtlIdoxTG0QCDTsmMkKFnYtJ3LghSGuPxiLBQHvrpF1jDUFEJJjeqMVzchOxBi21zJjwwgCSR5d\nEImkaFGiJZIixdGVlCgO53L2Q709UzNT1V3Vtxp2/X5Aod96662u0/2K4p9PP+ecaq0FAACYTW/R\nAwAAgJcSARoAAOYgQAMAwBwEaAAAmIMADQAAcxCgAQBgDgI0AADMQYAGAIA5CNAAADAHARoAAOYw\nWPQANvOKV7yi3XDDDYseBgAA+9hHPvKRL7XWDs9y7SUfoG+44YYcO3Zs0cMAAGAfq6rPzXqtFg4A\nAJiDAA0AAHMQoAEAYA4CNAAAzEGABgCAOQjQAAAwBwEaAADmIEADAMAcBGgAAJiDAA0AAHMQoAEA\nYA4CNAAAzEGABgCAOQjQAAAwBwEaAADmIEADAMAcBGgAAJiDAA0AAHMQoAEAYA4C9AS//O8+kR/5\ntT9f9DAAALgECdATnD7T8o0TpxY9DAAALkEC9ASDfuX0mbboYQAAcAkSoCcY9Ho5JUADADCBAD1B\nv6cCDQDAZAL0BINe5dSZM4seBgAAlyABegIVaAAAphGgJxhVoAVoAAAuJkBP0O/10lpUoQEAuIgA\nPcGgX0miDxoAgIsI0BMMeqMArQINAMCFBOgJ+r31CrQADQDA+QToCc5WoE8L0AAAnE+AnqDfH/1a\nVKABALiQAD2BHmgAAKYRoCc41wNtFQ4AAM4nQE+wXoE+pQcaAIALCNATWIUDAIBpBOgJht0kQj3Q\nAABcSICeQA80AADTCNATWIUDAIBpBOgJ9EADADCNAD3BoKcHGgCAyQToCfqWsQMAYAoBeoJB3yRC\nAAAmE6An0AMNAMA0AvQEw/UeaC0cAABcQICeQAUaAIBpBOgJ1nugrcIBAMCFBOgJ7EQIAMA0AvQE\ndiIEAGAaAXoC60ADADCNAD3B+k6EJhECAHAhAXqCc5MI9UADAHA+AXqCgWXsAACYQoCeoG8SIQAA\nUwjQE+iBBgBgmk0DdFX9VlU9U1WfHDv3f1bVp6rqv1TVv6uqK8deu6OqHq2qR6rqjWPnf6CqPtG9\n9htVVTv/4+wMFWgAAKaZpQL920luueDcB5K8prX2XyX5qyR3JElV3ZzktiSv7t5zV1X1u/e8M8nP\nJrmpe1z4PS8ZA8vYAQAwxaYBurX2F0m+csG5P2mtneqefjDJdd3xrUnuba2daK09luTRJK+rqmuT\nHGqtfbC11pL8TpI37dQPsdN6vUqVnQgBALjYTvRA/w9J/qg7PpLk8bHXnujOHemOLzx/yRr0Sg80\nAAAX2VaArqpfTnIqyXt2Zjhnv+9bq+pYVR07fvz4Tn7rmQ16PT3QAABcZMsBuqr+SZIfT/JTXVtG\nkjyZ5Pqxy67rzj2Zc20e4+cnaq3d3Vo72lo7evjw4a0OcVsGvdIDDQDARbYUoKvqliS/lOQnWmvP\nj710f5Lbqmq1qm7MaLLgh1trTyV5tqpe362+8TNJ3r/Nse+qfr/sRAgAwEUGm11QVe9N8sNJXlFV\nTyT5lYxW3VhN8oFuNboPttb+x9bag1V1X5KHMmrteFtr7XT3rX4uoxU91jLqmf6jXML0QAMAMMmm\nAbq19pMTTr97g+vvTHLnhPPHkrxmrtEtUL9XeqABALiInQinGPR6KtAAAFxEgJ6i36ucOq0HGgCA\n8wnQU+iBBgBgEgF6ikFfDzQAABcToKfo64EGAGACAXqKgVU4AACYQICeoq8HGgCACQToKUYVaKtw\nAABwPgF6itEydirQAACcT4CeYtDXwgEAwMUE6CnsRAgAwCQC9BR6oAEAmESAnkIPNAAAkwjQU9iJ\nEACASQToKfq9ngANAMBFBOgpBjZSAQBgAgF6ilEPtEmEAACcT4CeQgUaAIBJBOgpTCIEAGASAXoK\nG6kAADCJAD1Fv6cCDQDAxQToKUY90CYRAgBwPgF6ChVoAAAmEaCnsAoHAACTCNBT9Hu9tBZVaAAA\nziNATzHoV5LogwYA4DwC9BSD3ihAq0ADADBOgJ6i31uvQAvQAACcI0BPcbYCfVqABgDgHAF6in5/\n9KtRgQYAYJwAPYUeaAAAJhGgpzjXA20VDgAAzhGgp1ivQJ/SAw0AwBgBegqrcAAAMIkAPcWwm0So\nBxoAgHEC9BR6oAEAmESAnsIqHAAATCJAT6EHGgCASQToKQY9PdAAAFxMgJ5ivQJ98rQeaAAAzhGg\npxj09UADAHAxAXqKgR5oAAAmEKCnONsDbSdCAADGCNBTWIUDAIBJBOgp9EADADCJAD2FnQgBAJhE\ngJ7CToQAAEwiQE9xtgJtEiEAAGME6CnWV+EwiRAAgHEC9BTrkwj1QAMAME6AnmJwditvFWgAAM4R\noKcY9LsWjtMq0AAAnCNAT2ErbwAAJhGgpxh2FeiTKtAAAIwRoKfo9ypVlrEDAOB8AvQGhv1eTlqF\nAwCAMQL0Boa9UoEGAOA8AvQGBv2eVTgAADiPAL2BYb9y0iocAACM2TRAV9VvVdUzVfXJsXNXV9UH\nqurT3derxl67o6oerapHquqNY+d/oKo+0b32G1VVO//j7KxBTwUaAIDzzVKB/u0kt1xw7h1JHmit\n3ZTkge55qurmJLcleXX3nruqqt+9551JfjbJTd3jwu95yRn09UADAHC+TQN0a+0vknzlgtO3Jrmn\nO74nyZvGzt/bWjvRWnssyaNJXldV1yY51Fr7YGutJfmdsfdcskarcAjQAACcs9Ue6Gtaa091x08n\nuaY7PpLk8bHrnujOHemOLzw/UVW9taqOVdWx48ePb3GI2zfolRYOAADOs+1JhF1FeUfLtK21u1tr\nR1trRw8fPryT33oug34vJ7VwAAAwZqsB+otdW0a6r890559Mcv3Yddd1557sji88f0kb9iunbKQC\nAMCYrQbo+5Pc3h3fnuT9Y+dvq6rVqroxo8mCH+7aPZ6tqtd3q2/8zNh7LlkDG6kAAHCBwWYXVNV7\nk/xwkldU1RNJfiXJrya5r6rekuRzSd6cJK21B6vqviQPJTmV5G2ttdPdt/q5jFb0WEvyR93jkjZq\n4VCBBgDgnE0DdGvtJ6e89IYp19+Z5M4J548lec1co1uwYb/ywkkBGgCAc+xEuAEbqQAAcCEBegPD\nflmFAwCA8wjQGxj0elbhAADgPAL0BmzlDQDAhQToDYy28laBBgDgHAF6A9aBBgDgQgL0BmzlDQDA\nhQToDdjKGwCACwnQGxitA60CDQDAOQL0BkbrQKtAAwBwjgC9gUG/cuqMCjQAAOcI0BsY9Ho5faal\nNSEaAIARAXoDw34liZU4AAA4S4DewKA/+vVYiQMAgHUC9AYGPRVoAADOJ0BvYLhegbYSBwAAHQF6\nA4OuB9pKHAAArBOgNzDsjX491oIGAGCdAL2BsxVoPdAAAHQE6A1YhQMAgAsJ0BsYWoUDAIALCNAb\nOFuBFqABAOgI0BtY74E+qYUDAICOAL2B9VU4VKABAFgnQG/g3CocKtAAAIwI0BsYnm3hUIEGAGBE\ngN7AoGcrbwAAzidAb+DsJEI90AAAdAToDQxtpAIAwAUE6A0Mzm6kIkADADAiQG9gvQKthQMAgHUC\n9AbOLWMnQAMAMCJAb+DsKhx6oAEA6AjQGxhahQMAgAsI0BsY9K0DDQDA+QToDaxXoE/ZiRAAgI4A\nvYFhb30VDhVoAABGBOgN9HqVXlmFAwCAcwToTQz6vZy0CgcAAB0BehPDXqlAAwBwlgC9iUG/ZxUO\nAADOEqA3MexXTlqFAwCAjgC9iUFPBRoAgHME6E0M+nqgAQA4R4DexLDf08IBAMBZAvQmBr3SwgEA\nwFkC9CYG/V5OauEAAKAjQG9i2K+cspEKAAAdAXoTAxupAAAwRoDexKDfy4t6oAEA6AjQm1gdWAca\nAIBzBOhNDE0iBABgjAC9iWG/clIFGgCAjgC9iWG/lxdPCdAAAIwI0JtYMYkQAIAxAvQmVgY9LRwA\nAJwlQG/CJEIAAMYJ0JsY9ns5qQcaAIDOtgJ0Vf2Lqnqwqj5ZVe+tqgNVdXVVfaCqPt19vWrs+juq\n6tGqeqSq3rj94e++4aByQgsHAACdLQfoqjqS5J8nOdpae02SfpLbkrwjyQOttZuSPNA9T1Xd3L3+\n6iS3JLmrqvrbG/7uW+mPeqBb08YBAMD2WzgGSdaqapDksiRfSHJrknu61+9J8qbu+NYk97bWTrTW\nHkvyaJLXbfPzd91Kv5fWktNnBGgAALYRoFtrTyb5tSSfT/JUkq+31v4kyTWttae6y55Ock13fCTJ\n42Pf4onu3CVtOBj9ikwkBAAg2V4Lx1UZVZVvTPKqJAer6qfHr2mjvoe5k2dVvbWqjlXVsePHj291\niDti2B/9iqwFDQBAsr0Wjr+f5LHW2vHW2skkv5/k7yT5YlVdmyTd12e6659Mcv3Y+6/rzl2ktXZ3\na+1oa+3o4cOHtzHE7VvpV5LYjRAAgCTbC9CfT/L6qrqsqirJG5I8nOT+JLd319ye5P3d8f1Jbquq\n1aq6MclNST68jc/fE+sVaJupAACQjCYBbklr7UNV9b4kH01yKsnHktyd5PIk91XVW5J8Lsmbu+sf\nrKr7kjzUXf+21trpbY5/160MBGgAAM7ZcoBOktbaryT5lQtOn8ioGj3p+juT3Lmdz9xrKtAAAIyz\nE+Emzk4iPGUVDgAABOhNrQy6SYQq0AAARIDelBYOAADGCdCbWFkP0JaxAwAgAvSm1nci1MIBAEAi\nQG9q5ewkQgEaAAABelPneqCtwgEAgAC9KRupAAAwToDexLBvGTsAAM4RoDexYhk7AADGCNCbGJpE\nCADAGAF6E0M90AAAjBGgN7FiFQ4AAMYI0Js4O4lQCwcAABGgN1VVGfZLCwcAAEkE6JkM+z0VaAAA\nkgjQMxn2eyrQAAAkEaBnsjLo5UWTCAEAiAA9kxUVaAAAOgL0DEwiBABgnQA9A5MIAQBYJ0DPwCRC\nAADWCdAzMIkQAIB1AvQMVvq9nNTCAQBABOiZDAcmEQIAMCJAz2DY7+VFARoAgAjQM7EKBwAA6wTo\nGawMrMIBAMCIAD2D0U6EVuEAAECAnomdCAEAWCdAz0APNAAA6wToGViFAwCAdQL0DFZNIgQAoCNA\nz2BoEiEAAB0BegbDfi+nz7ScPiNEAwAsOwF6BsNBJYk2DgAABOhZrPRHvyYTCQEAEKBnsDIY/ZpO\nWsoOAGDpCdAzGHYVaBMJAQAQoGewHqBtpgIAgAA9g2F/NIlQDzQAAAL0DFbXe6AFaACApSdAz+Bc\nD7QADQCw7AToGQjQAACsE6BnsB6gT5hECACw9AToGayc3YnQMnYAAMtOgJ7BSr+fxEYqAAAI0DMZ\nDixjBwDAiAA9gxUbqQAA0BGgZ3BgOGrhOHHq9IJHAgDAognQM1jfSMUqHAAACNAzWF2vQJ8UoAEA\nlp0APYNzFWgtHAAAy06AnsGgV+mVFg4AAATomVRVVgd9ARoAAAF6VqvDXk6c1MIBALDsBOgZrQ56\necEkQgCApSdAz2jUwqECDQCw7LYVoKvqyqp6X1V9qqoerqofrKqrq+oDVfXp7utVY9ffUVWPVtUj\nVfXG7Q9/76wOenqgAQDYdgX6XyX549ba9yT5viQPJ3lHkgdaazcleaB7nqq6OcltSV6d5JYkd1VV\nf5ufv2cODE0iBABgGwG6qq5I8veSvDtJWmsvtta+luTWJPd0l92T5E3d8a1J7m2tnWitPZbk0SSv\n2+rn77VRBVoLBwDAsttOBfrGJMeT/Ouq+lhVvauqDia5prX2VHfN00mu6Y6PJHl87P1PdOdeEkar\ncKhAAwAsu+0E6EGS70/yztbaa5N8M127xrrWWkvS5v3GVfXWqjpWVceOHz++jSHuHOtAAwCQbC9A\nP5Hkidbah7rn78soUH+xqq5Nku7rM93rTya5fuz913XnLtJau7u1drS1dvTw4cPbGOLO0cIBAECy\njQDdWns6yeNV9d3dqTckeSjJ/Ulu787dnuT93fH9SW6rqtWqujHJTUk+vNXP32tW4QAAIBm1YWzH\nP0vynqpaSfLZJP80o1B+X1W9Jcnnkrw5SVprD1bVfRmF7FNJ3tZae8mUdFcHfT3QAABsL0C31j6e\n5OiEl94w5fo7k9y5nc9clNWhFg4AAOxEODMtHAAAJAL0zFYH/bxw8nRGC4sAALCsBOgZHRj2cqYl\np84I0AAAy0yAntHqYLTruDYOAIDlJkDPaHU4+lWdOGkiIQDAMhOgZ7Q66AK0CjQAwFIToGekhQMA\ngESAntm5CrQWDgCAZSZAz+hcD7QKNADAMhOgZ6SFAwCARICemRYOAAASAXpmZyvQWjgAAJaaAD2j\nA10P9Asq0AAAS02AnpEKNAAAiQA9s7OrcJhECACw1AToGZlECABAIkDPzDJ2AAAkAvTMVgY2UgEA\nQICeWb9XGfZLCwcAwJIToOewOuhr4QAAWHIC9BxWBz0VaACAJSdAz2F10NMDDQCw5AToORwYauEA\nAFh2AvQcVga9vHBSCwcAwDIToOewqgINALD0BOg5mEQIAIAAPYdRgFaBBgBYZgL0HFYHfatwAAAs\nOQF6DqtDLRwAAMtOgJ6DFg4AAAToOdjKGwAAAXoOo50ItXAAACwzAXoOdiIEAECAnsN6D3RrbdFD\nAQBgQQToOawOR7+uF0+rQgMALCsBeg6rg36S5AVrQQMALC0Beg6rg9Gvy1rQAADLS4Cew9kArQIN\nALC0BOg5rA5HLRxW4gAAWF4C9By0cAAAIEDP4VyAVoEGAFhWAvQc1lfh0AMNALC8BOg5HBhq4QAA\nWHYC9BzOVqC1cAAALC0Beg6rQz3QAADLToCew7l1oLVwAAAsKwF6DgeG61t5C9AAAMtKgJ7DWheg\nvyVAAwAsLQF6DusV6G+9qAcaAGBZCdBz6PcqK4OeCjQAwBIToOe0NuzrgQYAWGIC9JwuW+nnWy8K\n0AAAy0qAntPasJ/nVaABAJaWAD2nA0MVaACAZSZAz2ltRQ80AMAyE6DntDbsW4UDAGCJCdBzWjOJ\nEABgqQnQc1KBBgBYbgL0nNZMIgQAWGrbDtBV1a+qj1XVH3TPr66qD1TVp7uvV41de0dVPVpVj1TV\nG7f72YuwtqICDQCwzHaiAv32JA+PPX9HkgdaazcleaB7nqq6OcltSV6d5JYkd1VVfwc+f08d0MIB\nALDUthWgq+q6JP8gybvGTt+a5J7u+J4kbxo7f29r7URr7bEkjyZ53XY+fxEuW+nnxVNncvpMW/RQ\nAABYgO1WoH89yS8lOTN27prW2lPd8dNJrumOjyR5fOy6J7pzLylrw1HRXBUaAGA5bTlAV9WPJ3mm\ntfaRade01lqSuUu1VfXWqjpWVceOHz++1SHuigMrXYA2kRAAYCltpwL9Q0l+oqr+Osm9SX6kqv5N\nki9W1bVJ0n19prv+ySTXj73/uu7cRVprd7fWjrbWjh4+fHgbQ9x56xVouxECACynLQfo1todrbXr\nWms3ZDQ58D+21n46yf1Jbu8uuz3J+7vj+5PcVlWrVXVjkpuSfHjLI18QLRwAAMttsAvf81eT3FdV\nb0nyuSRvTpLW2oNVdV+Sh5KcSvK21tpLLoVe1rVwPK+FAwBgKe1IgG6t/XmSP++Ov5zkDVOuuzPJ\nnTvxmYtyYKgHGgBgmdmJcE5rK3qgAQCWmQA9Jz3QAADLTYCe02WWsQMAWGoC9JzWe6CfV4EGAFhK\nAvSczvZAq0ADACwlAXpOBwajX5keaACA5SRAz2nQ72Wl3xOgAQCWlAC9BWsr/Tx/4tSihwEAwAII\n0Ftw+eog39QDDQCwlAToLbhspZ9vqkADACwlAXoLDqpAAwAsLQF6Cw6uqkADACwrAXoLDq4MBGgA\ngCUlQG/BqIVDgAYAWEYC9BYcXO3n+RN6oAEAlpEAvQUHVwf5hhYOAIClJEBvwcGVQU6cOpNTp88s\neigAAOwxAXoLDq4OksRSdgAAS0iA3oKDK/0ksRIHAMASEqC3YL0C/byVOAAAlo4AvQUHV0cV6G9Y\niQMAYOkI0FtwcKWrQGvhAABYOgL0Fqy3cFjKDgBg+QjQW3CuB1oLBwDAshGgt2B9FQ4VaACA5SNA\nb8HZdaAFaACApSNAb8HasJ8qG6kAACwjAXoLer3KZcO+CjQAwBISoLfo4OrARioAAEtIgN6ig6sD\nG6kAACwhAXqLDq72baQCALCEBOgtumxlYBk7AIAlJEBv0eWrg3xTDzQAwNIRoLfospV+ntcDDQCw\ndAToLbp8dZDntHAAACwdAXqLDq0N89wLJxc9DAAA9pgAvUVXrA3zwskzOXFKGwcAwDIRoLfo0IFB\nkuS5F7RxAAAsEwF6iw6tDZMkX/+WNg4AgGUiQG/RoQOjAP2sAA0AsFQE6C06tDZq4XhWCwcAwFIR\noLfoijUVaACAZSRAb9F6C4ceaACA5SJAb9H6JMJnrQUNALBUBOgtWh30stLv5dlv6YEGAFgmAvQW\nVVUOrQ1UoAEAlowAvQ2H1oYmEQIALBkBehsOHRiaRAgAsGQE6G04tDa0DjQAwJIRoLfh0IFBnlOB\nBgBYKgL0NlyxNjSJEABgyQjQ23BobdQD3Vpb9FAAANgjAvQ2HDowzMnTLS+cPLPooQAAsEcE6G04\ntDZIYjdCAIBlIkBvw6ED3XbeJhICACwNAXobrljrArQKNADA0hCgt+FQF6BtpgIAsDwE6G04dKDr\ngf6WzVQAAJbFlgN0VV1fVX9WVQ9V1YNV9fbu/NVV9YGq+nT39aqx99xRVY9W1SNV9cad+AEW6ZAW\nDgCApbOdCvSpJL/YWrs5yeuTvK2qbk7yjiQPtNZuSvJA9zzda7cleXWSW5LcVVX97Qx+0UwiBABY\nPlsO0K21p1prH+2On0vycJIjSW5Nck932T1J3tQd35rk3tbaidbaY0keTfK6rX7+pWBl0MvasK8H\nGgBgiexID3RV3ZDktUk+lOSa1tpT3UtPJ7mmOz6S5PGxtz3RnXtJO7Q20AMNALBEth2gq+ryJL+X\n5Bdaa8+Ov9ZGe1zPvc91Vb21qo5V1bHjx49vd4i76tCBoR5oAIAlsq0AXVXDjMLze1prv9+d/mJV\nXdu9fm2SZ7rzTya5fuzt13XnLtJau7u1drS1dvTw4cPbGeKuO7QmQAMALJPtrMJRSd6d5OHW2r8c\ne+n+JLd3x7cnef/Y+duqarWqbkxyU5IPb/XzLxWHDmjhAABYJoNtvPeHkvzjJJ+oqo935/6XJL+a\n5L6qekuSzyV5c5K01h6sqvuSPJTRCh5va62d3sbnXxKuWBvmM8e/uehhAACwR7YcoFtr/ylJTXn5\nDVPec2eSO7f6mZeiK9aG+drzLy56GAAA7BE7EW7T1QdX8+wLp3Ly9JlFDwUAgD0gQG/T1ZevJEm+\n+k1VaACAZSBAb9PLD44C9JcFaACApSBAb9PVB1WgAQCWiQC9TSrQAADLRYDepqu6AP0VARoAYCkI\n0Nt01WUrqVKBBgBYFgL0NvV7lSvXhvnKN08seigAAOwBAXoHXH1wRQsHAMCSEKB3wMsPrubL3xCg\nAQCWgQC9A1SgAQCWhwC9A15+uQANALAsBOgd8PKDK/nK8y/m9Jm26KEAALDLBOgdcPXBlbSWfPV5\nVWgAgP1OgN4Br3jZapLkS9+wlB0AwH4nQO+Aaw4dSJJ88VkBGgBgvxOgd8A1LxsF6GeefWHBIwEA\nYLcJ0DvglYdGLRzPPKcCDQCw3wnQO+DAsJ9DBwb5ogo0AMC+J0DvkGsOHcgzeqABAPY9AXqHvPLQ\nar74nAo0AMB+J0DvkGtepgINALAMBOgd8spDB/LMcy+kNbsRAgDsZwL0Dvkbh1Zz8nTLl75hN0IA\ngP1MgN4hr7pyLUny1Ne/teCRAACwmwToHbIeoL/wNRMJAQD2MwF6h1x7xWg3wqdVoAEA9jUBeodc\nfXAlK/1envq6CjQAwH4mQO+Qqsq1Vx7Ik19TgQYA2M8E6B10/VWX5fGvCtAAAPuZAL2DrrtqLU9+\n9flFDwMAgF0kQO+gI1eu5UvfeDEvnDy96KEAALBLBOgddOSq0VJ2T2jjAADYtwToHfTtLz+YJPnc\nl7+54JEAALBbBOgd9B2HRwH6s8cFaACA/UqA3kFXXraSqw+u5LNfEqABAPYrAXqHXX/VWp6wEgcA\nwL4lQO+w66++LJ/7sgANALBfCdA77LuueVk+/5Xn840TpxY9FAAAdoEAvcNuvvZQkuSRp59d8EgA\nANgNAvQO+95XjQL0Q18QoAEA9iMBeoe96ooDuWJtmIeeem7RQwEAYBcI0DusqnLztYfy0FMq0AAA\n+5EAvQu+99pDeeTpZ3P6TFv0UAAA2GEC9C64+VWH8sLJM3nMhioAAPuOAL0L1lfieFgbBwDAviNA\n74LvfOXlGfZLHzQAwD4kQO+ClUEv3/nKl6lAAwDsQwL0Lvnea1+WP3/keM6YSAgAsK8I0LvkOw5f\nniT5zPFvLHgkAADsJAF6l7zx1X8jSfKHn3h6wSMBAGAnCdC75DsOH0yvkr/8668seigAAOwgAXqX\nVFX+m+86nP/06Jf0QQMA7CMC9C76b7/nlUmSP35QGwcAwH4hQO+i//61R5Ik93/8CwseCQAAO0WA\n3kUvOzDM3zpyRf74wadz8vSZRQ8HAIAdIEDvsjd1Vehf/9O/WvBIAADYCQL0LvuZH/z2JMlv/tln\n8sLJ0wseDQAA2yVA77Jhv5df/O++K0ny8//vRxc8GgAAtmvPA3RV3VJVj1TVo1X1jr3+/EX4+R/5\nzvQq+dOHn8m//9iTix4OAADbsKcBuqr6SX4zyY8muTnJT1bVzXs5hkWoqvyHf/Z3kyS/8Lsfz/s/\nLkQDALxUDfb4816X5NHW2meTpKruTXJrkof2eBx77tWvuiJ3/dT35+fe89G8/d6P53f/8vH8z2/8\n7qwOdNEPH3WMAAAGGklEQVQAAKy78rKVHLlybdHD2NBeB+gjSR4fe/5Ekr+9x2NYmB/7W9fm9/6n\nH8xPvetD+c+f+XL+4V3/edFDAgC4pLz56HX5P/7R9y16GBva6wA9k6p6a5K3Jsm3fdu3LXg0O+sH\nvv3qfOp/+9F86LNfzte+dXLRwwEAuKRc6tXnZO8D9JNJrh97fl137jyttbuT3J0kR48ebXsztL31\nt//myxc9BAAAtmCvG3D/MslNVXVjVa0kuS3J/Xs8BgAA2LI9rUC31k5V1c8n+f+S9JP8Vmvtwb0c\nAwAAbMee90C31v4wyR/u9ecCAMBOsIYaAADMQYAGAIA5CNAAADAHARoAAOYgQAMAwBwEaAAAmIMA\nDQAAcxCgAQBgDgI0AADMQYAGAIA5CNAAADAHARoAAOYgQAMAwBwEaAAAmIMADQAAcxCgAQBgDgI0\nAADMoVprix7DhqrqeJLPLeCjX5HkSwv4XPaW+7wc3Of9zz1eDu7zcljUff721trhWS685AP0olTV\nsdba0UWPg93lPi8H93n/c4+Xg/u8HF4K91kLBwAAzEGABgCAOQjQ09296AGwJ9zn5eA+73/u8XJw\nn5fDJX+f9UADAMAcVKABAGAOAvQEVXVLVT1SVY9W1TsWPR5mV1XXV9WfVdVDVfVgVb29O391VX2g\nqj7dfb1q7D13dPf6kap649j5H6iqT3Sv/UZV1SJ+Jiarqn5Vfayq/qB77h7vM1V1ZVW9r6o+VVUP\nV9UPus/7T1X9i+7/rz9ZVe+tqgPu80tfVf1WVT1TVZ8cO7dj97WqVqvqd7vzH6qqG/by5xOgL1BV\n/SS/meRHk9yc5Cer6ubFjoo5nEryi621m5O8Psnbuvv3jiQPtNZuSvJA9zzda7cleXWSW5Lc1f1v\nIEnemeRnk9zUPW7Zyx+ETb09ycNjz93j/edfJfnj1tr3JPm+jO63+7yPVNWRJP88ydHW2muS9DO6\nj+7zS99v5+J7sJP39S1Jvtpa+84k/1eS/33XfpIJBOiLvS7Jo621z7bWXkxyb5JbFzwmZtRae6q1\n9tHu+LmM/oV7JKN7eE932T1J3tQd35rk3tbaidbaY0keTfK6qro2yaHW2gfbaKLA74y9hwWrquuS\n/IMk7xo77R7vI1V1RZK/l+TdSdJae7G19rW4z/vRIMlaVQ2SXJbkC3GfX/Jaa3+R5CsXnN7J+zr+\nvd6X5A17+VcHAfpiR5I8Pvb8ie4cLzHdn3Nem+RDSa5prT3VvfR0kmu642n3+0h3fOF5Lg2/nuSX\nkpwZO+ce7y83Jjme5F93rTrvqqqDcZ/3ldbak0l+LcnnkzyV5OuttT+J+7xf7eR9Pfue1tqpJF9P\n8vLdGfbFBGj2paq6PMnvJfmF1tqz4691/xVr+ZmXqKr68STPtNY+Mu0a93hfGCT5/iTvbK29Nsk3\n0/25d537/NLX9cDemtF/ML0qycGq+unxa9zn/emlfl8F6Is9meT6sefXded4iaiqYUbh+T2ttd/v\nTn+x+1NQuq/PdOen3e8nu+MLz7N4P5TkJ6rqrzNqsfqRqvo3cY/3myeSPNFa+1D3/H0ZBWr3eX/5\n+0kea60db62dTPL7Sf5O3Of9aifv69n3dO0/VyT58q6N/AIC9MX+MslNVXVjVa1k1NR+/4LHxIy6\n/qd3J3m4tfYvx166P8nt3fHtSd4/dv62bjbvjRlNUPhw9yemZ6vq9d33/Jmx97BArbU7WmvXtdZu\nyOifz//YWvvpuMf7Smvt6SSPV9V3d6fekOShuM/7zeeTvL6qLuvuzxsymrviPu9PO3lfx7/XP8ro\n3wV7V9FurXlc8EjyY0n+KslnkvzyosfjMde9+7sZ/UnovyT5ePf4sYz6oh5I8ukkf5rk6rH3/HJ3\nrx9J8qNj548m+WT32v+dbuMhj0vnkeSHk/xBd+we77NHkv86ybHun+d/n+Qq93n/PZL8r0k+1d2j\n/yfJqvv80n8keW9Gfe0nM/qL0lt28r4mOZDk32Y04fDDSf7mXv58diIEAIA5aOEAAIA5CNAAADAH\nARoAAOYgQAMAwBwEaAAAmIMADQAAcxCgAQBgDgI0AADM4f8H1Bte4Lril+MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b15de10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error total: -0.004250398520801113\n"
     ]
    }
   ],
   "source": [
    "#forward and backward passes\n",
    "error = []\n",
    "for i in range(10000):\n",
    "    conv_forw_result = conv_forw(image, kernel)\n",
    "    #print('convolution feedforward result', '\\n', conv_forw_result)\n",
    "\n",
    "    relu_forw_result = relu_forw(conv_forw_result)\n",
    "    #print('relu feedforward result', '\\n', relu_forw_result)\n",
    "\n",
    "    maxpool_forw_result, maxpool_back_result = maxpooling(relu_forw_result)\n",
    "    #print('maxpooling feedforward result', '\\n', maxpool_forw_result)\n",
    "    #print('maxpooling backprop result', '\\n', maxpool_back_result)\n",
    "\n",
    "    error_total = (maxpool_forw_result-y)**2\n",
    "    error.append(int(error_total))\n",
    "\n",
    "    backprop = learning_rate*maxpool_back_result*(maxpool_forw_result-y)\n",
    "\n",
    "    backprop = learning_rate*relu_forw_result*backprop\n",
    "\n",
    "    relu_back_result = relu_back(backprop)\n",
    "\n",
    "    backprop = learning_rate*relu_back_result*conv_forw_result\n",
    "\n",
    "    kernel_back = conv_forw(image, backprop)\n",
    "\n",
    "    kernel = kernel - learning_rate*kernel_back\n",
    "\n",
    "plt.plot(error)\n",
    "plt.show()\n",
    "print('error total:', float(maxpool_forw_result-y))\n",
    "\n",
    "#for later, ad second conv layer\n",
    "#conv_back_result = conv_back(_, kernel)\n",
    "#print('convolution backward result', '\\n', conv_back_result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}