PG_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from gym import wrappers\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_obs = 80 * 80        # dimensionality of observations\n",
    "h = 200                # number of hidden layer neurons\n",
    "n_actions = 3          # number of available actions\n",
    "learning_rate = 1e-3\n",
    "gamma = 0.99           # discount factor for reward\n",
    "decay = 0.99           # decay rate for RMSProp gradients\n",
    "save_path = './models/pong.ckpt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### check dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not os.path.exists('./models'):\n",
    "    os.makedirs('./models')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### gamespace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-05-16 02:55:57,729] Making new env: Pong-v0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"Pong-v0\")\n",
    "# env = wrappers.Monitor(env, '/tmp/pong-pg-tensorflow', video_callable=lambda count: count % 300 == 0, force=True)\n",
    "observation = env.reset()\n",
    "prev_x = None\n",
    "xs, rs, ys = [],[],[]\n",
    "running_reward = None\n",
    "reward_sum = 0\n",
    "episode_number = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### initialize model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_model = {}\n",
    "\n",
    "with tf.variable_scope('layer_one', reuse=False):\n",
    "    xavier_l1 = tf.truncated_normal_initializer(mean=0, stddev=1./np.sqrt(n_obs), dtype=tf.float32)\n",
    "    tf_model['W1'] = tf.get_variable(\"W1\", [n_obs, h], initializer=xavier_l1)\n",
    "    \n",
    "with tf.variable_scope('layer_two', reuse=False):\n",
    "    xavier_l2 = tf.truncated_normal_initializer(mean=0, stddev=1./np.sqrt(h), dtype=tf.float32)\n",
    "    tf_model['W2'] = tf.get_variable(\"W2\", [h, n_actions], initializer=xavier_l2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tf_discount_rewards(tf_r): # tf_r shape:(game_steps, 1)\n",
    "    discount_f = lambda a, v: a*gamma + v;\n",
    "    tf_r_reverse = tf.scan(discount_f, tf.reverse(tf_r, [True, False]))\n",
    "    tf_discounted_r = tf.reverse(tf_r_reverse, [True, False])\n",
    "    return tf_discounted_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tf_policy_forward(x): # x shape:(1, n_obs)\n",
    "    h = tf.matmul(x, tf_model['W1'])\n",
    "    h = tf.nn.relu(h)\n",
    "    logp = tf.matmul(h, tf_model['W2'])\n",
    "    p = tf.nn.softmax(logp)\n",
    "    return p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### downsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepro(I):\n",
    "    \"\"\" prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector \"\"\"\n",
    "    I = I[35:195]    # crop\n",
    "    I = I[::2,::2,0] # downsample by factor of 2\n",
    "    I[I == 144] = 0  # erase background (background type 1)\n",
    "    I[I == 109] = 0  # erase background (background type 2)\n",
    "    I[I != 0] = 1    # everything else (paddles, ball) just set to 1\n",
    "    return I.astype(np.float).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "observation, _, _, _ = env.step(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(210, 160, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAD8CAYAAADpCEEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADhdJREFUeJzt3X+s1fV9x/Hnq1gwka7lhyMEcUBCm4DZblfifrQaN2dF\nuxTdHw4yDd3MribOlLTLApqsZglJ1xX9Z9EGIxmbDmSjVpK6bkiammZt9WIoAooCQuAGob0203XN\nLPDeH9/PbY/Xe7nnnvc5nu85ez2Sm/v9fr7f7znvby4vvz/8nvdRRGBmrftAtwsw63UOkVmSQ2SW\n5BCZJTlEZkkOkVlSx0IkaaWkw5KOSFrfqfcx6zZ14v8TSZoGvArcAJwCXgDWRMShtr+ZWZd16kh0\nNXAkIo5FxDvAdmBVh97LrKsu6dDrLgBONsyfAn5ropUlXfRwuPBXprWpLLPmnXzr/I8j4vLJ1utU\niCYlaRAYBJh16Qf40nUf7lYp47rhd39nytvs/s/vdaCS3jf0hc9MeZsVD36zA5VMzbpv/eREM+t1\n6nRuGFjYMH9FGfuFiNgcESsiYsXM6epQGWad16kQvQAslbRY0nRgNbCrQ+9l1lUdOZ2LiHOS/gL4\nd2AasCUiDnbivcy6rWPXRBHxDPBMp17//Tbe9U4r1002/vVOK9dNdeEnFsySHCKzJIfILMkhMkty\niMySHCKzJIfILMkhMktyiMySHCKzJIfILMkhMkvq2ofyeo0fNm2fXn7YdDw+EpklOURmSQ6RWZKv\niSbgpiPtU4emI53U8pFI0kJJ35Z0SNJBSZ8v4w9IGpa0r/zc3L5yzeoncyQ6B3wxIl6U9CFgr6Td\nZdlDEfHVfHlm9ddyiCLiNHC6TL8t6WWqpo1TNnvxVdz++J5WSzHriHVz5za1XltuLEhaBHwc+EEZ\nulfSfklbJM1qx3uY1VU6RJJmAjuBdRHxFvAIsAQYoDpSbZpgu0FJQ5KGRkZGsmWYdU0qRJI+SBWg\nJyLi6wARcSYizkfEBeBRqub279HYAXXOnDmZMsy6KnN3TsBjwMsR8WDD+PyG1W4FDrRenln9Ze7O\nfRK4A3hJ0r4ydh+wRtIAEMBx4K5UhWY1l7k7911gvE70fdP11KwZfuzHLMkhMktyiMySavEA6puv\nH+Dx25d2uwyzlvhIZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJD\nZJaUeopb0nHgbeA8cC4iVkiaDTwJLKL6ePhtEfGTXJlm9dWOI9HvRcRARKwo8+uBPRGxFNhT5s36\nVidO51YBW8v0VuCWDryHWW1kQxTAs5L2ShosY/NKi2GAN4B5yfcwq7XsJ1s/FRHDkn4V2C3plcaF\nERGSYrwNS+gGAWZd6vsb1rtS/3ojYrj8Pgs8RdXt9MxoA8fy++wE2/6iA+rM6eN13jLrDZkOqJeV\nr1RB0mXAp6m6ne4C1pbV1gJPZ4s0q7PM6dw84KmqmzCXAP8cEd+S9AKwQ9KdwAngtnyZZvWV6YB6\nDPiNccZHgOszRZn1El/RmyU5RGZJDpFZkkNkluQQmSU5RGZJDpFZkkNkluQQmSU5RGZJDpFZkkNk\nluQQmSU5RGZJDpFZkkNkluQQmSW1/MlWSR+j6nQ6agnw18BHgD8HflTG74uIZ1qu0KzmMh8PPwwM\nAEiaBgxTdfz5U+ChiPhqWyo0q7l2nc5dDxyNiBNtej2zntGuEK0GtjXM3ytpv6Qtkma16T3Maikd\nIknTgc8C/1KGHqG6PhoATgObJthuUNKQpKH/fmfcJqlmPaEdR6KbgBcj4gxARJyJiPMRcQF4lKor\n6nu4A6r1i3aEaA0Np3KjLYSLW6m6opr1reyXfF0G3ADc1TD8FUkDVN8YcXzMMrO+kwpRRPwUmDNm\n7I5URWY9xk8smCU5RGZJDpFZkkNkluQQmSU5RGZJDpFZkkNkluQQmSU5RGZJDpFZkkNklpR6ANWs\nLoa+8Jl3za948Jvv23v7SGSW5BCZJTlEZkkOkVnSpCEqba/OSjrQMDZb0m5Jr5XfsxqWbZB0RNJh\nSTd2qnCzumjmSPQPwMoxY+uBPRGxFNhT5pG0jKoH3fKyzcOlO6pZ35o0RBHxHPDmmOFVwNYyvRW4\npWF8e0T8b0S8DhxhgpZZZv2i1WuieRFxuky/Acwr0wuAkw3rnSpj7+HmjdYv0jcWIiKo2mNNdTs3\nb7S+0GqIzow2aSy/z5bxYWBhw3pXlDGzvtVqiHYBa8v0WuDphvHVkmZIWgwsBZ7PlWhWb5M+Oydp\nG3AdMFfSKeBLwJeBHZLuBE4AtwFExEFJO4BDwDngnog436HazWph0hBFxJoJFl0/wfobgY2Zosx6\niZ9YMEtyiMyS/Hki6wvv5+eHxvKRyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJL\ncojMkhwisySHyCzJITJLarUD6t9JekXSfklPSfpIGV8k6WeS9pWfr3WyeLM6aLUD6m7gqoj4deBV\nYEPDsqMRMVB+7m5PmWb11VIH1Ij4j4g4V2a/T9Uay+z/pXZcE/0Z8G8N84vLqdx3JF0z0UbugGr9\nIvXxcEn3U7XGeqIMnQaujIgRSZ8AviFpeUS8NXbbiNgMbAa48sOXOEXWs1o+Ekn6HPCHwJ+UVsKU\nRvYjZXovcBT4aBvqNKutlkIkaSXwV8BnI+J/GsYvH/0qFUlLqDqgHmtHoWZ11WoH1A3ADGC3JIDv\nlztx1wJ/I+nnwAXg7ogY+7UsZn2l1Q6oj02w7k5gZ7Yos17iJxbMkhwisySHyCzJITJLcojMkhwi\nsySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMklrtgPqApOGGTqc3Nyzb\nIOmIpMOSbuxU4WZ10WoHVICHGjqdPgMgaRmwGlhetnl4tHGJWb9qqQPqRawCtpfWWa8DR4CrE/WZ\n1V7mmuje0tB+i6RZZWwBcLJhnVNl7D3cAdX6RashegRYAgxQdT3dNNUXiIjNEbEiIlbMnK4WyzDr\nvpZCFBFnIuJ8RFwAHuWXp2zDwMKGVa8oY2Z9q9UOqPMbZm8FRu/c7QJWS5ohaTFVB9TncyWa1Vur\nHVCvkzQABHAcuAsgIg5K2gEcomp0f09EnO9M6Wb10NYOqGX9jcDGTFFmvcRPLJglOURmSQ6RWZJD\nZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJDZJbUavPGJxsaNx6X\ntK+ML5L0s4ZlX+tk8WZ1MOknW6maN/498I+jAxHxx6PTkjYB/9Ww/tGIGGhXgWZ118zHw5+TtGi8\nZZIE3Ab8fnvLMusd2Wuia4AzEfFaw9jicir3HUnXJF/frPaaOZ27mDXAtob508CVETEi6RPANyQt\nj4i3xm4oaRAYBJh1qe9vWO9q+V+vpEuAPwKeHB0rPbhHyvRe4Cjw0fG2dwdU6xeZQ8AfAK9ExKnR\nAUmXj34LhKQlVM0bj+VKNKu3Zm5xbwO+B3xM0ilJd5ZFq3n3qRzAtcD+csv7X4G7I6LZb5Qw60mt\nNm8kIj43zthOYGe+LLPe4St6sySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJL\ncojMkhwisySHyCwp+/Hwtpi9+Cpuf3xPt8swe5d1c+c2tZ6PRGZJDpFZUjMfD18o6duSDkk6KOnz\nZXy2pN2SXiu/ZzVss0HSEUmHJd3YyR0w67ZmjkTngC9GxDLgt4F7JC0D1gN7ImIpsKfMU5atBpYD\nK4GHR5uXmPWjSUMUEacj4sUy/TbwMrAAWAVsLattBW4p06uA7aV91uvAEeDqdhduVhdTuiYq7YQ/\nDvwAmBcRp8uiN4B5ZXoBcLJhs1NlzKwvNR0iSTOpOvmsG9vRNCICiKm8saRBSUOShkZGRqayqVmt\nNBUiSR+kCtATEfH1MnxG0vyyfD5wtowPAwsbNr+ijL1LYwfUOXPmtFq/Wdc1c3dOwGPAyxHxYMOi\nXcDaMr0WeLphfLWkGZIWU3VBfb59JZvVSzNPLHwSuAN4afTLvID7gC8DO0pH1BNUX7FCRByUtAM4\nRHVn756ION/2ys1qopkOqN8FJuo4f/0E22wENibqMusZfmLBLMkhMktyiMySHCKzJIfILEnVwwZd\nLkL6EfBT4MfdrqWN5tI/+9NP+wLN78+vRcTlk61UixABSBqKiBXdrqNd+ml/+mlfoP3749M5sySH\nyCypTiHa3O0C2qyf9qef9gXavD+1uSYy61V1OhKZ9aSuh0jSytLQ5Iik9d2upxWSjkt6SdI+SUNl\nbMJGLnUjaYuks5IONIz1bCOaCfbnAUnD5W+0T9LNDcty+xMRXfsBpgFHgSXAdOCHwLJu1tTifhwH\n5o4Z+wqwvkyvB/6223VepP5rgd8EDkxWP7Cs/J1mAIvL329at/ehif15APjLcdZN70+3j0RXA0ci\n4lhEvANsp2p00g8mauRSOxHxHPDmmOGebUQzwf5MJL0/3Q5RvzQ1CeBZSXslDZaxiRq59Ip+bERz\nr6T95XRv9PQ0vT/dDlG/+FREDAA3UfXlu7ZxYVTnDT17G7TX6y8eobpsGABOA5va9cLdDlFTTU3q\nLiKGy++zwFNUpwMTNXLpFalGNHUTEWci4nxEXAAe5ZenbOn96XaIXgCWSlosaTpV59RdXa5pSiRd\nJulDo9PAp4EDTNzIpVf0VSOa0f8gFLdS/Y2gHftTgzspNwOvUt0Vub/b9bRQ/xKquzs/BA6O7gMw\nh6q98mvAs8Dsbtd6kX3YRnWK83Oqa4I7L1Y/cH/5ex0Gbup2/U3uzz8BLwH7S3Dmt2t//MSCWVK3\nT+fMep5DZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSf8HB1ZBgDOwCkYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd15b06be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(observation.shape)\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(160, 160, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD8CAYAAAB+fLH0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADQNJREFUeJzt3X/sXfVdx/Hni3aFAeOXRVJpt5YFMUh0kAbnnMSE/QCG\ndMZkKRHDHAkxwQkOQ2D8sf2zhDlF/cctOFDiEIIMssZtCMPNxUQY0BVoKT8KDGgtlDGVuRlY2ds/\n7um8n663P+7P78XnI/mm53zu+fZ87rm3r55zevt9paqQpF0OmvUEJC0shoKkhqEgqWEoSGoYCpIa\nhoKkxsRCIclZSR5PsiXJlZPaj6TxyiQ+p5BkEfAE8F5gK3A/cH5VPTr2nUkaq0mdKZwObKmqp6vq\nNeAWYM2E9iVpjBZP6Pc9Hni+b30r8CuDNj58yUF1zJu9vSFN0vOvvP7dqjp2X9tNKhT2KcnFwMUA\nRx9yEJe/64hZTUX6f+GyO//j2f3ZblJ/PW8DVvStL+/GfqKqrquq1VW1+vAlmdA0JB2oSYXC/cCJ\nSVYlWQKsBdZNaF+Sxmgilw9VtTPJHwD/BCwCbqiqTZPYl6Txmtg9har6CvCVSf3+kibDW/6SGoaC\npIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqS\nGoaCpIahIKkxdCgkWZHk60keTbIpyaXd+DFJ7k7yZPfr0eObrqRJG+VMYSdweVWdDLwTuCTJycCV\nwD1VdSJwT7cuaU4MHQpVtb2q1nfL3wc206uLWwPc2G12I/DBUScpaXrGck8hyUrgVOA+4Liq2t49\n9AJw3IDvuTjJA0ke+O/Xxt98LWk4I4dCksOBLwKXVdUr/Y9Vr+d+j3/irY2TFqaRQiHJm+gFwk1V\ndXs3/GKSZd3jy4Ado01R0jSN8q8PAa4HNlfVtX0PrQMu7JYvBL40/PQkTdsotXG/Bvwu8EiSDd3Y\nx4FrgFuTXAQ8C3xotClKmqahQ6Gq/hUYdDPgzGF/X0mz5ScaJTUMBUkNQ0FSw1CQ1DAUJDUMBUkN\nQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUmNcfyI90VJvp3kH7t1\na+OkOTaOM4VL6bVD7WJtnDTHRu19WA58APh837C1cdIcG/VM4S+AK4Af941ZGyfNsVHKYM4FdlTV\ng4O2sTZOmj+jlsGcl+Qc4BDgiCRfoKuNq6rt1sZJ82eUKvqrqmp5Va0E1gL/XFUXYG2cNNcm8TmF\na4D3JnkSeE+3LmlOjHL58BNV9Q3gG93yy1gbJ80tP9EoqWEoSGoYCpIahoKkhqEgqWEoSGoYCpIa\nhoKkhqEgqWEoSGoYCpIahoKkhqEgqWEoSGoYCpIahoKkhqEgqTGWn7wkaTgPfOwDPzW2+tovz2Am\n/2fUMpijktyW5LEkm5P8qrVx0nwb9fLhL4E7q+oXgF+mVx9nbZw0x0YpgzkSOAO4HqCqXquq/8Ta\nOGmujXKmsAp4CfibrnX680kOYz9r4yQtTKOEwmLgNOCzVXUq8AN2u1TYW22cXZLSwjRKKGwFtlbV\nfd36bfRC4sWuLo691cbZJSktTKPUxr0APJ/kpG7oTOBRrI2T5tqon1P4KHBTkiXA08Dv0QuaW5Nc\nBDwLfGjEfUiaopFCoao2AKv38JC1cdKc8mPOkhqGgqSGoSCpYShIahgKkhqGgqSGoSCpYShIahgK\nkhqGgqSGoSCp4Q9ulWZo1j+kdU88U5DUMBQkNQwFSQ1DQVLDUJDUMBQkNUatjfujJJuSbExyc5JD\nrI2T5tsoDVHHA38IrK6qU4BFwFqsjZPm2qiXD4uBNydZDBwK/DvWxklzbZTeh23AnwLPAduB/6qq\nu7A2Tppro1w+HE3vrGAV8HPAYUku6N/G2jhp/oxy+fAe4JmqeqmqfgTcDrwLa+OkuTZKKDwHvDPJ\noUlCrwBmM9bGSXNt6P8lWVX3JbkNWA/sBL4NXAccjrVx0twatTbuE8Andht+FWvjpLnlJxolNQwF\nSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQk\nNQwFSY19hkKSG5LsSLKxb2xgNVySq5JsSfJ4kvdPauKSJmN/zhT+Fjhrt7E9VsMlOZleddwvdt/z\nV0kWjW22kiZun6FQVd8Evrfb8KBquDXALVX1alU9A2wBTh/TXCVNwbD3FAZVwx0PPN+33dZuTNKc\nGPlG496q4fbG2jhpYRo2FAZVw20DVvRtt7wb+ynWxkkL07ChMKgabh2wNsnBSVYBJwLfGm2KkqZp\nnw1RSW4GfgNYmmQrvUaoa9hDNVxVbUpyK/AovSq5S6rq9QnNXdIE7DMUqur8AQ/tsRquqj4FfGqU\nSUmaHT/RKKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhq\nGAqSGoaCpIahIKlhKEhqDFsb95kkjyV5OMkdSY7qe8zaOGmODVsbdzdwSlX9EvAEcBVYGye9EQxV\nG1dVd1XVzm71Xnr9DmBtnDT3xnFP4SPAV7tla+OkOTdSKCS5ml6/w01DfK+1cdICtM/eh0GSfBg4\nFziz65OEA6yNA64DeOuRi00FaYEY6kwhyVnAFcB5VfXDvoesjZPm3LC1cVcBBwN3JwG4t6p+39o4\naf4NWxt3/V62tzZOmmN+olFSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUM\nBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDWG6pLse+zyJJVkad+YXZLSHBu2S5IkK4D3Ac/1\njdklKc25obokO39Or/uhv8jFLklpzg1bBrMG2FZVD+320H53SVobJy1MB1wbl+RQ4OP0Lh2GZm2c\ntDAN0yX5dmAV8FDXDrUcWJ/kdA6gS1LSwnTAlw9V9UhV/WxVrayqlfQuEU6rqhewS1Kae/vzT5I3\nA/8GnJRka5KLBm1bVZuAXV2Sd2KXpDR3hu2S7H985W7rdklKc8xPNEpqGAqSGoaCpIahIKlhKEhq\nGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKkx\nTO/D2B2z6hQu+MI9s56G9IZ22dKl+94ISNXsy5mSvAT8APjurOcCLMV59HMerXmex9uq6th9bbQg\nQgEgyQNVtdp5OA/nMdt5eE9BUsNQkNRYSKFw3awn0HEeLefResPPY8HcU5C0MCykMwVJC8DMQyHJ\nWUkeT7IlyZVT3O+KJF9P8miSTUku7cY/mWRbkg3d1zlTmMt3kjzS7e+BbuyYJHcnebL79egJz+Gk\nvue8IckrSS6bxvFIckOSHUk29o0NfP5JrureL48nef+E5/GZJI8leTjJHUmO6sZXJvmfvuPyuQnP\nY+DrMPbjUVUz+wIWAU8BJwBLgIeAk6e072XAad3yW4AngJOBTwJ/POXj8B1g6W5jfwJc2S1fCXx6\nyq/LC8DbpnE8gDOA04CN+3r+3Wv0EHAwsKp7/yya4DzeByzulj/dN4+V/dtN4Xjs8XWYxPGY9ZnC\n6cCWqnq6ql4DbgHWTGPHVbW9qtZ3y98HNgPHT2Pf+2kNcGO3fCPwwSnu+0zgqap6dho7q6pvAt/b\nbXjQ818D3FJVr1bVM8AWeu+jicyjqu6qqp3d6r3A8nHs60DnsRdjPx6zDoXjgef71rcygz+YSVYC\npwL3dUMf7U4Xb5j0aXungK8leTDJxd3YcVW1vVt+AThuCvPYZS1wc9/6tI8HDH7+s3zPfAT4at/6\nqu5U/l+S/PoU9r+n12Hsx2PWoTBzSQ4HvghcVlWvAJ+ldznzDmA78GdTmMa7q+odwNnAJUnO6H+w\neueJU/lnoiRLgPOAf+iGZnE8GtN8/oMkuRrYCdzUDW0H3tq9bh8D/j7JEROcwtReh1mHwjZgRd/6\n8m5sKpK8iV4g3FRVtwNU1YtV9XpV/Rj4a8Z0aro3VbWt+3UHcEe3zxeTLOvmuQzYMel5dM4G1lfV\ni92cpn48OoOe/9TfM0k+DJwL/E4XUHSn6y93yw/Su5b/+UnNYS+vw9iPx6xD4X7gxCSrur+h1gLr\nprHjJAGuBzZX1bV948v6NvstYOPu3zvmeRyW5C27lund2NpI7zhc2G12IfClSc6jz/n0XTpM+3j0\nGfT81wFrkxycZBVwIvCtSU0iyVnAFcB5VfXDvvFjkyzqlk/o5vH0BOcx6HUY//GYxN3TA7zTeg69\nO/9PAVdPcb/vpndK+jCwofs6B/g74JFufB2wbMLzOIHe3eOHgE27jgHwM8A9wJPA14BjpnBMDgNe\nBo7sG5v48aAXQtuBH9G7Jr5ob88fuLp7vzwOnD3heWyhd82+6z3yuW7b3+5erw3AeuA3JzyPga/D\nuI+Hn2iU1Jj15YOkBcZQkNQwFCQ1DAVJDUNBUsNQkNQwFCQ1DAVJjf8FGbLqm+CckCAAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd15a39588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# crop\n",
    "observation = observation[35:195]\n",
    "print(observation.shape)\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80, 80)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACyhJREFUeJzt3X/IXYV9x/H3ZzHS0W7VWBcyky7+IZZQMOmCUyxjU7Om\nndj9JQodUgr+0w2FjqL9b38M/Ku0f4xCUTuhrp2zSkWKklnLNijWWN1aE9M4pyROTaordg62mX73\nxz2lT9P8OM+vm5zn+37Bw3PPuffhnEN4P+fe+9ycb6oKSf382pneAUlnhvFLTRm/1JTxS00Zv9SU\n8UtNGb/U1LLiT7I7yYEkLyS5faV2StLqy1I/5JNkHfAjYBdwGHgKuKmq9q3c7klaLecs42cvB16o\nqhcBknwd+Dhw0vjft2Fdbd2yfhmblHQqLx36P3785rGMeexy4r8IOLRg+TDwe6f6ga1b1vO9x7Ys\nY5OSTuXyjxw6/YMGq/6GX5JbkuxNsvfoG8dWe3OSRlpO/K8AC0/jm4d1v6SqvlxVO6tq54UXrFvG\n5iStpOXE/xRwSZKLk5wL3Ag8vDK7JWm1Lfk1f1W9k+TPgMeAdcA9VfXciu2ZpFW1nDf8qKpvAd9a\noX2RNEd+wk9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxf\nasr4paaMX2rK+KWmTht/knuSHEnywwXrNiTZk+Tg8P381d1NSSttzJn/b4Ddx627HXi8qi4BHh+W\nJU3IaeOvqn8E3jxu9ceBe4fb9wJ/ssL7JWmVLfU1/8aqenW4/RqwcYX2R9KcLPsNv5qN+T3pqF/H\ndUlnp6XG/3qSTQDD9yMne6DjuqSz01Ljfxi4ebh9M/DNldkdSfMy5k99XwO+C1ya5HCSTwF3AruS\nHASuHZYlTchpx3VV1U0nueuaFd4XSXPkJ/ykpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaM\nX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qasw1/LYkeSLJviTPJbl1WO/ILmnCxpz53wE+\nU1XbgCuATyfZhiO7pEkbM67r1ar6/nD7p8B+4CIc2SVN2qJe8yfZCuwAnsSRXdKkjY4/yXuAbwC3\nVdVbC+871cgux3VJZ6dR8SdZzyz8+6rqwWH1qJFdjuuSzk5j3u0PcDewv6o+v+AuR3ZJE3baiT3A\nVcCfAj9I8uyw7nPMRnTdP4zvehm4YXV2UdJqGDOu65+BnORuR3ZJE+Un/KSmjF9qyvilpoxfasr4\npaaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2pqzDX8JC3TR357\n+6+se+w/nj3BI+dnzNV735Xke0n+ZZjV95fDemf1SRM25mn//wBXV9VlwHZgd5IrcFafNGljZvVV\nVf3XsLh++Cqc1SdN2tiJPeuGa/YfAfZU1ehZfY7rks5Oo+KvqmNVtR3YDFye5IPH3X/SWX2O65LO\nTov6U19V/QR4AtjNyFl9ks5OY97tvzDJecPtXwd2Ac/jrD5p0sb8nX8TcG+Sdcx+WdxfVY8k+S7O\n6pMma8ysvn8Fdpxg/Rs4q0+aLD/eKzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8Yv\nNWX8UlNevVeagzN9pd4T8cwvNWX8UlPGLzVl/FJTxi81NTr+4dr9zyR5ZFh2XJc0YYs5898K7F+w\n7LguacLGTuzZDPwxcNeC1Y7rkiZs7Jn/C8BngZ8tWOe4LmnCxgztuA44UlVPn+wxjuuSpmfMx3uv\nAq5P8jHgXcBvJvkqw7iuqnrVcV3S9IwZ0X1HVW2uqq3AjcC3q+oTOK5LmrTl/J3/TmBXkoPAtcOy\npIlY1P/qq6rvAN8ZbjuuS5owP+EnNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81\nZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU6Mu45XkJeCnwDHgnaramWQD8HfAVuAl4Iaq+s/V2U1J\nK20xZ/4/rKrtVbVzWHZclzRhy3na77guacLGxl/APyR5Osktw7pR47oknZ3GXrr7w1X1SpLfAvYk\neX7hnVVVSU44rmv4ZXELwPsvWtSVwiWtolFn/qp6Zfh+BHgIuJxhXBfAqcZ1OatPOjuNGdT57iS/\n8fPbwB8BP8RxXdKkjXkevhF4KMnPH/+3VfVokqeA+5N8CngZuGH1dlPSSjtt/FX1InDZCdY7rkua\nMD/hJzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxS\nU8YvNWX8UlOj4k9yXpIHkjyfZH+SK5NsSLInycHh+/mrvbOSVs7YM/8XgUer6gPMrue3H8d1SZM2\n5tLd7wV+H7gboKr+t6p+guO6pEkbc+a/GDgKfCXJM0nuGq7f77guacLGxH8O8CHgS1W1A3ib457i\nV1Uxm+f3K5LckmRvkr1H3zi23P2VtELGxH8YOFxVTw7LDzD7ZeC4LmnCTht/Vb0GHEpy6bDqGmAf\njuuSJm3s2Nw/B+5Lci7wIvBJZr84HNclTdSo+KvqWWDnCe5yXJc0UX7CT2rK+KWmjF9qyvilpoxf\nasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4pabGDO24NMmzC77e\nSnKb47qkaRtz9d4DVbW9qrYDvwv8N/AQjuuSJm2xT/uvAf6tql7GcV3SpC02/huBrw23HdclTdjo\n+Idr9l8P/P3x9zmuS5qexZz5Pwp8v6peH5Yd1yVN2GLiv4lfPOUHx3VJkzYq/mEk9y7gwQWr7wR2\nJTkIXDssS5qIseO63gYuOG7dGziuS5osP+EnNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl\n/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzWV2bCdOW0sOQq8Dfx4bhud\nr/exNo/N45qO36mqC8c8cK7xAyTZW1U757rROVmrx+ZxrU0+7ZeaMn6pqTMR/5fPwDbnZa0em8e1\nBs39Nb+ks4NP+6Wm5hp/kt1JDiR5Icnt89z2SkqyJckTSfYleS7JrcP6DUn2JDk4fD//TO/rUiRZ\nl+SZJI8My2vluM5L8kCS55PsT3LlWjm2pZhb/EnWAX8NfBTYBtyUZNu8tr/C3gE+U1XbgCuATw/H\ncjvweFVdAjw+LE/RrcD+Bctr5bi+CDxaVR8ALmN2jGvl2BavqubyBVwJPLZg+Q7gjnltf5WP7ZvA\nLuAAsGlYtwk4cKb3bQnHsplZBFcDjwzr1sJxvRf4d4b3uRasn/yxLfVrnk/7LwIOLVg+PKybtCRb\ngR3Ak8DGqnp1uOs1YOMZ2q3l+ALwWeBnC9atheO6GDgKfGV4SXNXknezNo5tSXzDbxmSvAf4BnBb\nVb218L6anUom9aeUJNcBR6rq6ZM9ZorHNTgH+BDwparawexj5r/0FH/Cx7Yk84z/FWDLguXNw7pJ\nSrKeWfj3VdWDw+rXk2wa7t8EHDlT+7dEVwHXJ3kJ+DpwdZKvMv3jgtkzzcNV9eSw/ACzXwZr4diW\nZJ7xPwVckuTiJOcCNwIPz3H7KyZJgLuB/VX1+QV3PQzcPNy+mdl7AZNRVXdU1eaq2srs3+fbVfUJ\nJn5cAFX1GnAoyaXDqmuAfayBY1uqef+vvo8xe025Drinqv5qbhtfQUk+DPwT8AN+8dr4c8xe998P\nvB94Gbihqt48Izu5TEn+APiLqrouyQWsgeNKsh24CzgXeBH4JLMT4OSPbSn8hJ/UlG/4SU0Zv9SU\n8UtNGb/UlPFLTRm/1JTxS00Zv9TU/wOSSiekASs+cwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd13761cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# downsample by factor of 2\n",
    "observation = observation[::2,::2,0]\n",
    "print(observation.shape)\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 92, 144], dtype=uint8), array([  16, 6384]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 可觀察到144占了多數，應該是背景，為不必要資訊\n",
    "np.unique(observation, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACydJREFUeJzt3V+oZfV5h/Hn21EzjWmjpukwdWz1QgxDwZl0MIqhtOq0\nJhXTK1FIkRCYm7QopATNXS8KXoXkogSCmgqxSa1RIhIUawxtIVjHaJs4ox1rFceqY5MGU0tNTN5e\n7BU8mcyfNWfvs8+s8z4fOJy9fvsMay3kOfvP2a43VYWkfn5pvQ9A0vowfqkp45eaMn6pKeOXmjJ+\nqSnjl5qaK/4kVyZ5JsmzSW5a1EFJWntZ7Yd8kmwC/g3YDRwEHgOuq6p9izs8SWvllDn+7UXAs1X1\nHECSrwAfAY4a/2l5R23m9Dl2KelY/o83+FG9mTE/O0/8ZwMvrtg+CHzgWP9gM6fzgVw+xy4lHcuj\n9fDon50n/lGS7AH2AGzmnWu9O0kjzfOG30vAOSu2tw1rP6eqvlBVu6pq16m8Y47dSVqkeeJ/DDg/\nyXlJTgOuBe5bzGFJWmurftpfVW8l+VPgQWATcHtVPbWwI5O0puZ6zV9VXwe+vqBjkbREfsJPasr4\npaaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvil\npo4bf5LbkxxK8t0Va2cleSjJgeH7mWt7mJIWbcwj/18DVx62dhPwcFWdDzw8bEuakOPGX1X/AHz/\nsOWPAHcMt+8A/njBxyVpja32Nf+Wqnp5uP0KsGVBxyNpSeZ+w69mY36POuo3yZ4ke5Ps/TFvzrs7\nSQuy2vhfTbIVYPh+6Gg/6Lgu6eS02vjvA64fbl8PfG0xhyNpWcb8qe/LwLeAC5IcTPJx4BZgd5ID\nwBXDtqQJOe64rqq67ih3Xb7gY5G0RH7CT2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvil\npoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4pabGXMPvnCSPJNmX5KkkNwzrjuySJmzMI/9bwCer\najtwMfCJJNtxZJc0aWPGdb1cVd8ebv8Q2A+cjSO7pEk7odf8Sc4FdgKP4sguadJGx5/kXcBXgRur\n6vWV9x1rZJfjuqST06j4k5zKLPw7q+qeYXnUyC7HdUknpzHv9ge4DdhfVZ9ZcZcju6QJO+7EHuBS\n4E+A7yR5clj7NLMRXXcN47teAK5Zm0OUtBbGjOv6JyBHuduRXdJE+Qk/qSnjl5oyfqkp45eaMn6p\nKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmhpzDT9Jc3rwP5/8\nhbU//I0d63Akbxtz9d7NSf45yb8Ms/r+Ylh3Vp80YWOe9r8JXFZVFwI7gCuTXIyz+qRJGzOrr6rq\nf4bNU4evwll90qSNndizabhm/yHgoaoaPavPcV3SyWlU/FX1k6raAWwDLkry24fdf9RZfY7rkk5O\nJ/Snvqr6AfAIcCUjZ/VJOjmNebf/vUnOGG7/MrAbeBpn9UmTNubv/FuBO5JsYvbL4q6quj/Jt3BW\nnzRZY2b1/Suw8wjr38NZfdJk+fFeqSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6p\nKeOXmvLqvdISrPeVeo/ER36pKeOXmjJ+qSnjl5oyfqmp0fEP1+5/Isn9w7bjuqQJO5FH/huA/Su2\nHdclTdjYiT3bgD8Cbl2x7LguacLGPvJ/FvgU8NMVa47rkiZszNCOq4BDVfX40X7GcV3S9Iz5eO+l\nwNVJPgxsBn41yZcYxnVV1cuO65KmZ8yI7puraltVnQtcC3yjqj6K47qkSZvn7/y3ALuTHACuGLYl\nTcQJ/V99VfVN4JvDbcd1SRPmJ/ykpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWm\njF9qyvilpoxfasr4paaMX2rK+KWmjF9qatRlvJI8D/wQ+AnwVlXtSnIW8LfAucDzwDVV9d9rc5iS\nFu1EHvl/v6p2VNWuYdtxXdKEzfO033Fd0oSNjb+Av0/yeJI9w9qocV2STk5jL939wap6KcmvAw8l\neXrlnVVVSY44rmv4ZbEHYDPvnOtgJS3OqEf+qnpp+H4IuBe4iGFcF8CxxnU5q086OY0Z1Hl6kl/5\n2W3gD4Dv4rguadLGPO3fAtyb5Gc//zdV9UCSx4C7knwceAG4Zu0OU9KiHTf+qnoOuPAI647rkibM\nT/hJTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU\n8UtNGb/U1Kj4k5yR5O4kTyfZn+SSJGcleSjJgeH7mWt9sJIWZ+wj/+eAB6rqfcyu57cfx3VJkzbm\n0t3vBn4XuA2gqn5UVT/AcV3SpI155D8PeA34YpInktw6XL/fcV3ShI2J/xTg/cDnq2on8AaHPcWv\nqmI2z+8XJNmTZG+SvT/mzXmPV9KCjIn/IHCwqh4dtu9m9svAcV3ShB03/qp6BXgxyQXD0uXAPhzX\nJU3a2Cm9fwbcmeQ04DngY8x+cTiuS5qoUfFX1ZPAriPc5bguaaL8hJ/UlPFLTRm/1JTxS00Zv9SU\n8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNjRnacUGSJ1d8vZ7k\nRsd1SdM25uq9z1TVjqraAfwO8L/AvTiuS5q0E33afznw71X1Ao7rkibtROO/FvjycNtxXdKEjY5/\nuGb/1cDfHX6f47qk6TmRR/4PAd+uqleHbcd1SRN2IvFfx9tP+cFxXdKkjYp/GMm9G7hnxfItwO4k\nB4Arhm1JEzF2XNcbwHsOW/sejuuSJstP+ElNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/\n1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTWU2bGdJO0teA94A/mtpO12u\nX2NjnpvnNR2/VVXvHfODS40fIMneqtq11J0uyUY9N89rY/Jpv9SU8UtNrUf8X1iHfS7LRj03z2sD\nWvprfkknB5/2S00tNf4kVyZ5JsmzSW5a5r4XKck5SR5Jsi/JU0luGNbPSvJQkgPD9zPX+1hXI8mm\nJE8kuX/Y3ijndUaSu5M8nWR/kks2yrmtxtLiT7IJ+CvgQ8B24Lok25e1/wV7C/hkVW0HLgY+MZzL\nTcDDVXU+8PCwPUU3APtXbG+U8/oc8EBVvQ+4kNk5bpRzO3FVtZQv4BLgwRXbNwM3L2v/a3xuXwN2\nA88AW4e1rcAz631sqziXbcwiuAy4f1jbCOf1buA/GN7nWrE++XNb7dcyn/afDby4YvvgsDZpSc4F\ndgKPAluq6uXhrleALet0WPP4LPAp4Kcr1jbCeZ0HvAZ8cXhJc2uS09kY57YqvuE3hyTvAr4K3FhV\nr6+8r2YPJZP6U0qSq4BDVfX40X5miuc1OAV4P/D5qtrJ7GPmP/cUf8LntirLjP8l4JwV29uGtUlK\nciqz8O+sqnuG5VeTbB3u3wocWq/jW6VLgauTPA98BbgsyZeY/nnB7Jnmwap6dNi+m9kvg41wbquy\nzPgfA85Pcl6S04BrgfuWuP+FSRLgNmB/VX1mxV33AdcPt69n9l7AZFTVzVW1rarOZfbf5xtV9VEm\nfl4AVfUK8GKSC4aly4F9bIBzW61l/199H2b2mnITcHtV/eXSdr5AST4I/CPwHd5+bfxpZq/77wJ+\nE3gBuKaqvr8uBzmnJL8H/HlVXZXkPWyA80qyA7gVOA14DvgYswfAyZ/bavgJP6kp3/CTmjJ+qSnj\nl5oyfqkp45eaMn6pKeOXmjJ+qan/B9gIGjUx3xfcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd137696a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# erase background (background type 1)\n",
    "observation[observation == 144] = 0\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACydJREFUeJzt3V+oZfV5h/Hn21EzjWmjpukwdWz1QgxDwZl0MIqhtOq0\nJhXTK1FIkRCYm7QopATNXS8KXoXkogSCmgqxSa1RIhIUawxtIVjHaJs4ox1rFceqY5MGU0tNTN5e\n7BU8mcyfNWfvs8+s8z4fOJy9fvsMay3kOfvP2a43VYWkfn5pvQ9A0vowfqkp45eaMn6pKeOXmjJ+\nqSnjl5qaK/4kVyZ5JsmzSW5a1EFJWntZ7Yd8kmwC/g3YDRwEHgOuq6p9izs8SWvllDn+7UXAs1X1\nHECSrwAfAY4a/2l5R23m9Dl2KelY/o83+FG9mTE/O0/8ZwMvrtg+CHzgWP9gM6fzgVw+xy4lHcuj\n9fDon50n/lGS7AH2AGzmnWu9O0kjzfOG30vAOSu2tw1rP6eqvlBVu6pq16m8Y47dSVqkeeJ/DDg/\nyXlJTgOuBe5bzGFJWmurftpfVW8l+VPgQWATcHtVPbWwI5O0puZ6zV9VXwe+vqBjkbREfsJPasr4\npaaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvil\npo4bf5LbkxxK8t0Va2cleSjJgeH7mWt7mJIWbcwj/18DVx62dhPwcFWdDzw8bEuakOPGX1X/AHz/\nsOWPAHcMt+8A/njBxyVpja32Nf+Wqnp5uP0KsGVBxyNpSeZ+w69mY36POuo3yZ4ke5Ps/TFvzrs7\nSQuy2vhfTbIVYPh+6Gg/6Lgu6eS02vjvA64fbl8PfG0xhyNpWcb8qe/LwLeAC5IcTPJx4BZgd5ID\nwBXDtqQJOe64rqq67ih3Xb7gY5G0RH7CT2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvil\npoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4pabGXMPvnCSPJNmX5KkkNwzrjuySJmzMI/9bwCer\najtwMfCJJNtxZJc0aWPGdb1cVd8ebv8Q2A+cjSO7pEk7odf8Sc4FdgKP4sguadJGx5/kXcBXgRur\n6vWV9x1rZJfjuqST06j4k5zKLPw7q+qeYXnUyC7HdUknpzHv9ge4DdhfVZ9ZcZcju6QJO+7EHuBS\n4E+A7yR5clj7NLMRXXcN47teAK5Zm0OUtBbGjOv6JyBHuduRXdJE+Qk/qSnjl5oyfqkp45eaMn6p\nKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmhpzDT9Jc3rwP5/8\nhbU//I0d63Akbxtz9d7NSf45yb8Ms/r+Ylh3Vp80YWOe9r8JXFZVFwI7gCuTXIyz+qRJGzOrr6rq\nf4bNU4evwll90qSNndizabhm/yHgoaoaPavPcV3SyWlU/FX1k6raAWwDLkry24fdf9RZfY7rkk5O\nJ/Snvqr6AfAIcCUjZ/VJOjmNebf/vUnOGG7/MrAbeBpn9UmTNubv/FuBO5JsYvbL4q6quj/Jt3BW\nnzRZY2b1/Suw8wjr38NZfdJk+fFeqSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6p\nKeOXmvLqvdISrPeVeo/ER36pKeOXmjJ+qSnjl5oyfqmp0fEP1+5/Isn9w7bjuqQJO5FH/huA/Su2\nHdclTdjYiT3bgD8Cbl2x7LguacLGPvJ/FvgU8NMVa47rkiZszNCOq4BDVfX40X7GcV3S9Iz5eO+l\nwNVJPgxsBn41yZcYxnVV1cuO65KmZ8yI7puraltVnQtcC3yjqj6K47qkSZvn7/y3ALuTHACuGLYl\nTcQJ/V99VfVN4JvDbcd1SRPmJ/ykpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWm\njF9qyvilpoxfasr4paaMX2rK+KWmjF9qatRlvJI8D/wQ+AnwVlXtSnIW8LfAucDzwDVV9d9rc5iS\nFu1EHvl/v6p2VNWuYdtxXdKEzfO033Fd0oSNjb+Av0/yeJI9w9qocV2STk5jL939wap6KcmvAw8l\neXrlnVVVSY44rmv4ZbEHYDPvnOtgJS3OqEf+qnpp+H4IuBe4iGFcF8CxxnU5q086OY0Z1Hl6kl/5\n2W3gD4Dv4rguadLGPO3fAtyb5Gc//zdV9UCSx4C7knwceAG4Zu0OU9KiHTf+qnoOuPAI647rkibM\nT/hJTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU\n8UtNGb/U1Kj4k5yR5O4kTyfZn+SSJGcleSjJgeH7mWt9sJIWZ+wj/+eAB6rqfcyu57cfx3VJkzbm\n0t3vBn4XuA2gqn5UVT/AcV3SpI155D8PeA34YpInktw6XL/fcV3ShI2J/xTg/cDnq2on8AaHPcWv\nqmI2z+8XJNmTZG+SvT/mzXmPV9KCjIn/IHCwqh4dtu9m9svAcV3ShB03/qp6BXgxyQXD0uXAPhzX\nJU3a2Cm9fwbcmeQ04DngY8x+cTiuS5qoUfFX1ZPAriPc5bguaaL8hJ/UlPFLTRm/1JTxS00Zv9SU\n8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNjRnacUGSJ1d8vZ7k\nRsd1SdM25uq9z1TVjqraAfwO8L/AvTiuS5q0E33afznw71X1Ao7rkibtROO/FvjycNtxXdKEjY5/\nuGb/1cDfHX6f47qk6TmRR/4PAd+uqleHbcd1SRN2IvFfx9tP+cFxXdKkjYp/GMm9G7hnxfItwO4k\nB4Arhm1JEzF2XNcbwHsOW/sejuuSJstP+ElNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/\n1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTWU2bGdJO0teA94A/mtpO12u\nX2NjnpvnNR2/VVXvHfODS40fIMneqtq11J0uyUY9N89rY/Jpv9SU8UtNrUf8X1iHfS7LRj03z2sD\nWvprfkknB5/2S00tNf4kVyZ5JsmzSW5a5r4XKck5SR5Jsi/JU0luGNbPSvJQkgPD9zPX+1hXI8mm\nJE8kuX/Y3ijndUaSu5M8nWR/kks2yrmtxtLiT7IJ+CvgQ8B24Lok25e1/wV7C/hkVW0HLgY+MZzL\nTcDDVXU+8PCwPUU3APtXbG+U8/oc8EBVvQ+4kNk5bpRzO3FVtZQv4BLgwRXbNwM3L2v/a3xuXwN2\nA88AW4e1rcAz631sqziXbcwiuAy4f1jbCOf1buA/GN7nWrE++XNb7dcyn/afDby4YvvgsDZpSc4F\ndgKPAluq6uXhrleALet0WPP4LPAp4Kcr1jbCeZ0HvAZ8cXhJc2uS09kY57YqvuE3hyTvAr4K3FhV\nr6+8r2YPJZP6U0qSq4BDVfX40X5miuc1OAV4P/D5qtrJ7GPmP/cUf8LntirLjP8l4JwV29uGtUlK\nciqz8O+sqnuG5VeTbB3u3wocWq/jW6VLgauTPA98BbgsyZeY/nnB7Jnmwap6dNi+m9kvg41wbquy\nzPgfA85Pcl6S04BrgfuWuP+FSRLgNmB/VX1mxV33AdcPt69n9l7AZFTVzVW1rarOZfbf5xtV9VEm\nfl4AVfUK8GKSC4aly4F9bIBzW61l/199H2b2mnITcHtV/eXSdr5AST4I/CPwHd5+bfxpZq/77wJ+\nE3gBuKaqvr8uBzmnJL8H/HlVXZXkPWyA80qyA7gVOA14DvgYswfAyZ/bavgJP6kp3/CTmjJ+qSnj\nl5oyfqkp45eaMn6pKeOXmjJ+qan/B9gIGjUx3xfcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd13631a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# everything else (paddles, ball) just set to 1\n",
    "observation[observation != 0] = 1\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0, 1], dtype=uint8), array([6384,   16]))\n",
      "(80, 80)\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(observation, return_counts=True))\n",
    "print(observation.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### downsampling (another case)\n",
    "* 以下只是舉出另外一種erase background，若想重現這段code，請reset env，並不走任何一個action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "observation = env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(210, 160, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAD8CAYAAADpCEEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADhZJREFUeJzt3X+s1fV9x/Hna4BAFFP8UUaQDkiwCSzb7Upck1bj5lrR\nNKXuDwdZDG6mVxNn2qTLBppsZglJ50r9Y4s2EMnY4lA2auUPN4ekrWkyKmAoAooCQuQGLy1dpu3E\n9sJ7f3w/tz1e74Vzz/sczvecvR7Jzf2ez/f7Pef9zc2L7w++3/dRRGBmrfu1bhdg1uscIrMkh8gs\nySEyS3KIzJIcIrOkjoVI0nJJhyUdkbSmU59j1m3qxP8TSZoCvA58FjgJ7AZWRcShtn+YWZd1ak90\nA3AkIo5FxM+Bp4AVHfoss66a2qH3nQe81fD6JPC7Ey0s6YK7w4/OmdGmssyad3r47I8j4tqLLdep\nEF2UpEFgEGDWldO4+77ru1XKuL5089JJr7Pxuwc7UEnvO/v+85NeZ8b0WztQyeT8/SMHTzSzXKcO\n54aA+Q2vrytjvxQRGyJiWUQsmzlzSofKMOu8ToVoN7BY0kJJlwErge0d+iyzrurI4VxEjEj6M+B5\nYAqwKSJ8rGN9qWPnRBHxHPBcp97/UhvvfKeV8yYb/3ynlfOmuvAdC2ZJDpFZkkNkluQQmSU5RGZJ\nDpFZkkNkluQQmSU5RGZJDpFZkkNkluQQmSV17aG8XuObTdunl282HY/3RGZJDpFZkkNkluRzogm4\n6Uj71KHpSCe1vCeSNF/SdyQdknRQ0pfL+MOShiTtKz+3t69cs/rJ7IlGgK9GxMuSZgF7Je0o8x6N\niK/nyzOrv5ZDFBGngFNl+l1Jr1I1bZy0n46MsGv4TKulmHVVWy4sSFoAfAL4QRl6QNJ+SZskzW7H\nZ5jVVTpEkq4AtgFfiYh3gMeBRcAA1Z5q/QTrDUraI2nPyNnz2TLMuiYVIknTqAL0ZER8CyAihiPi\nXEScBzZSNbf/kMYOqFNn+Eq79a7M1TkBTwCvRsQ3GsbnNix2B3Cg9fLM6i9zde7TwF3AK5L2lbEH\ngVWSBoAAjgP3pio0q7nM1bnvAxpnVt90PTVrhk9GzJIcIrMkh8gsqRY3oF4xdSqfmnN1t8sw+4Dd\nvN3Uct4TmSU5RGZJDpFZkkNkluQQmSU5RGZJDpFZkkNkluQQmSU5RGZJDpFZUq1CtGv4jLv+WM+p\nVYjMelHqLm5Jx4F3gXPASEQsk3QV8DSwgOrx8Dsj4r9zZZrVVzv2RL8XEQMRsay8XgPsjIjFwM7y\n2qxvdeJ5ohXAzWV6M/Bd4C+bWdHPFFkvyu6JAnhB0l5Jg2VsTmkxDPA2MCf5GWa1lt0TfSYihiR9\nFNgh6bXGmRERkmK8FUvoBgFmXTktWYZZ96T2RBExVH6fBp6h6nY6PNrAsfw+PcG6v+yAOnPmlEwZ\nZl2V6YB6eflKFSRdDnyOqtvpdmB1WWw18Gy2SLM6yxzOzQGeqboJMxX4l4j4D0m7ga2S7gFOAHfm\nyzSrr0wH1GPAb48zfga4JVOUWS/xHQtmSQ6RWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6R\nWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSS0/2Srp41SdTkctAv4K+AjwJeBHZfzBiHiu5QrN\nai7zePhhYABA0hRgiKrjz58Aj0bE19tSoVnNtetw7hbgaEScaNP7mfWMdoVoJbCl4fUDkvZL2iRp\ndps+w6yW0iGSdBnwBeBfy9DjVOdHA8ApYP0E6w1K2iNpz3vvncuWYdY17dgT3Qa8HBHDABExHBHn\nIuI8sJGqK+qHuAOq9Yt2hGgVDYdyoy2EizuouqKa9a3sl3xdDnwWuLdh+BFJA1TfGHF8zDyzvpMK\nUUT8DLh6zNhdqYrMeozvWDBLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySH\nyCzJITJLcojMkhwisySHyCwp9VCeWV2cff/5D7yeMf3WS/bZF90TlbZXpyUdaBi7StIOSW+U37Mb\n5q2VdETSYUmXbkvMuqSZw7l/BJaPGVsD7IyIxcDO8hpJS6h60C0t6zxWuqOa9a2LhigiXgR+MmZ4\nBbC5TG8Gvtgw/lREvB8RbwJHmKBlllm/aPXCwpyIOFWm3wbmlOl5wFsNy50sYx/i5o3WL9JX5yIi\nqNpjTXY9N2+0vtBqiIZHmzSW36fL+BAwv2G568qYWd9qNUTbgdVlejXwbMP4SknTJS0EFgMv5Uo0\nq7eL/j+RpC3AzcA1kk4Cfw18Ddgq6R7gBHAnQEQclLQVOASMAPdHhE94rK9dNEQRsWqCWbdMsPw6\nYF2mKLNe4tt+zJIcIrMkh8gsySEyS3KIzJIcIrMkP09kfeFSPj80lvdEZkkOkVmSQ2SW5BCZJTlE\nZkkOkVmSQ2SW5BCZJTlEZkkOkVlSqx1Q/07Sa5L2S3pG0kfK+AJJ70naV36+2cnizeqg1Q6oO4Df\njIjfAl4H1jbMOxoRA+XnvvaUaVZfLXVAjYj/jIiR8nIXVWsss/+X2nFO9KfAvze8XlgO5b4n6caJ\nVnIHVOsXqUchJD1E1RrryTJ0CvhYRJyR9Eng25KWRsQ7Y9eNiA3ABoA5vz5z0h1Uzeqi5T2RpLuB\nzwN/XFoJUxrZnynTe4GjwPVtqNOstloKkaTlwF8AX4iI/20Yv3b0q1QkLaLqgHqsHYWa1VWrHVDX\nAtOBHZIAdpUrcTcBfyPpF8B54L6IGPu1LGZ9pdUOqE9MsOw2YFu2KLNe4jsWzJIcIrMkh8gsySEy\nS3KIzJIcIrMkh8gsySEyS3KIzJIcIrMkh8gsySEyS3KIzJIcIrMkh8gsySEyS3KIzJJa7YD6sKSh\nhk6ntzfMWyvpiKTDkrr3bbRml0irHVABHm3odPocgKQlwEpgaVnnsdHGJWb9qqUOqBewAniqtM56\nEzgC3JCoz6z2MudED5SG9pskzS5j84C3GpY5WcY+xB1QrV+0GqLHgUXAAFXX0/WTfYOI2BARyyJi\n2cyZPuKz3tVSiCJiOCLORcR5YCO/OmQbAuY3LHpdGTPrW612QJ3b8PIOYPTK3XZgpaTpkhZSdUB9\nKVeiWb212gH1ZkkDQADHgXsBIuKgpK3AIapG9/dHhE94rK+1tQNqWX4dsC5TlFkv8R0LZkkOkVmS\nQ2SW5BCZJTlEZkkOkVmSQ2SW5BCZJTlEZkkOkVmSQ2SW5BCZJTlEZkkOkVmSQ2SW5BCZJbXavPHp\nhsaNxyXtK+MLJL3XMO+bnSzerA4u+mQrVfPGfwD+aXQgIv5odFrSeuB/GpY/GhED7SrQrO6aeTz8\nRUkLxpsnScCdwO+3tyyz3pE9J7oRGI6INxrGFpZDue9JujH5/ma118zh3IWsArY0vD4FfCwizkj6\nJPBtSUsj4p2xK0oaBAYBZl05LVmGWfe0vCeSNBX4Q+Dp0bHSg/tMmd4LHAWuH299d0C1fpE5nPsD\n4LWIODk6IOna0W+BkLSIqnnjsVyJZvXWzCXuLcB/AR+XdFLSPWXWSj54KAdwE7C/XPL+N+C+iGj2\nGyXMelKrzRuJiLvHGdsGbMuXZdY7fMeCWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJD\nZJbkEJklOURmSQ6RWZJDZJaUfTy8LX46MsKu4TPdLsOsJd4TmSU5RGZJzTwePl/SdyQdknRQ0pfL\n+FWSdkh6o/ye3bDOWklHJB2WdGsnN8Cs25rZE40AX42IJcCngPslLQHWADsjYjGws7ymzFsJLAWW\nA4+NNi8x60cXDVFEnIqIl8v0u8CrwDxgBbC5LLYZ+GKZXgE8VdpnvQkcAW5od+FmdTGpc6LSTvgT\nwA+AORFxqsx6G5hTpucBbzWsdrKMmfWlpkMk6QqqTj5fGdvRNCICiMl8sKRBSXsk7Rk5e34yq5rV\nSlMhkjSNKkBPRsS3yvCwpLll/lzgdBkfAuY3rH5dGfuAxg6oU2f4IqH1rmauzgl4Ang1Ir7RMGs7\nsLpMrwaebRhfKWm6pIVUXVBfal/JZvXSzB0LnwbuAl4Z/TIv4EHga8DW0hH1BNVXrBARByVtBQ5R\nXdm7PyLOtb1ys5popgPq9wFNMPuWCdZZB6xL1GXWM3wyYpbkEJklOURmSQ6RWZJDZJak6maDLhch\n/Qj4GfDjbtfSRtfQP9vTT9sCzW/Pb0TEtRdbqBYhApC0JyKWdbuOdumn7emnbYH2b48P58ySHCKz\npDqFaEO3C2izftqeftoWaPP21OacyKxX1WlPZNaTuh4iSctLQ5MjktZ0u55WSDou6RVJ+yTtKWMT\nNnKpG0mbJJ2WdKBhrGcb0UywPQ9LGip/o32Sbm+Yl9ueiOjaDzAFOAosAi4Dfggs6WZNLW7HceCa\nMWOPAGvK9Brgb7td5wXqvwn4HeDAxeoHlpS/03RgYfn7Ten2NjSxPQ8Dfz7Osunt6fae6AbgSEQc\ni4ifA09RNTrpBxM1cqmdiHgR+MmY4Z5tRDPB9kwkvT3dDlG/NDUJ4AVJeyUNlrGJGrn0in5sRPOA\npP3lcG/08DS9Pd0OUb/4TEQMALdR9eW7qXFmVMcNPXsZtNfrLx6nOm0YAE4B69v1xt0OUVNNTeou\nIobK79PAM1SHAxM1cukVqUY0dRMRwxFxLiLOAxv51SFbenu6HaLdwGJJCyVdRtU5dXuXa5oUSZdL\nmjU6DXwOOMDEjVx6RV81ohn9B6G4g+pvBO3YnhpcSbkdeJ3qqshD3a6nhfoXUV3d+SFwcHQbgKup\n2iu/AbwAXNXtWi+wDVuoDnF+QXVOcM+F6gceKn+vw8Bt3a6/ye35Z+AVYH8Jztx2bY/vWDBL6vbh\nnFnPc4jMkhwisySHyCzJITJLcojMkhwisySHyCzp/wAaskqp1XfSOgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd13604828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(observation.shape)\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(160, 160, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD8CAYAAAB+fLH0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADS1JREFUeJzt3X3MnfVdx/H3x5Zn6ICBtdK6lgUxuOhGCKJOYsIeAJFu\nMZklm2GOpDFBBJ0hMP7Y/lnCNp0uJrrgwBFFCDLImmUPMNxcTCyjdAVaykOBAa2lZV2U6Ris7Osf\n5+o8v653H87jfcb7lTQ953euu9f3XOfuh+s6PdyfVBWStMfPTHsASfOLoSCpYShIahgKkhqGgqSG\noSCpMbZQSHJ+kseSbElyzbj2I2m0Mo7PKSRZADwOvB3YCtwPXFJVj4x8Z5JGalxnCmcDW6rqqap6\nBbgNWDmmfUkaoYVj+nNPAZ7ru78V+LW5Nj7q6IW1aNFhYxpFEsDOHT/4TlWdfKDtxhUKB5RkNbAa\n4LhFh/H7l75xWqNIrwl/8/FNzxzMduO6fNgGLOu7v7Rb+7GquqGqzqqqs446asGYxpB0qMYVCvcD\npyVZkeRwYBWwZkz7kjRCY7l8qKrdSf4Y+AqwALipqjaNY1+SRmts7ylU1ReBL47rz5c0Hn6iUVLD\nUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSY15Fwprd+xi7Y5d0x5Des2ad6EgaboMBUkNQ0FSw1CQ\n1JjaT16ayzmLXz/tEaTXNM8UJDUMBUkNQ0FSw1CQ1DAUJDUGDoUky5J8LckjSTYlubJbPzHJPUme\n6H4/YXTjShq3Yc4UdgMfrKozgHOAy5OcAVwD3FtVpwH3dvclzYiBQ6GqtlfV+u7294DN9OriVgI3\nd5vdDLxr2CElTc5I3lNIshx4C3AfsLiqtncPPQ8snuNrVidZl2TdSy+9OooxJI3A0KGQ5Fjgc8BV\nVfVi/2PV67nfZ9e9tXHS/DRUKCQ5jF4g3FJVd3bLO5Is6R5fAuwcbkRJkzTMvz4EuBHYXFWf7Hto\nDXBpd/tS4PODjydp0ob5H6J+E/gD4OEkG7q1DwHXA7cnuQx4BnjPcCNKmqSBQ6Gq/h3IHA+fN+if\nK2m6/ESjpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlh\nKEhqGAqSGoaCpIahIKkxih/xviDJt5J8obtvbZw0w0ZxpnAlvXaoPayNk2bYsL0PS4HfAT7Tt2xt\nnDTDhj1T+GvgauBHfWvWxkkzbJgymIuAnVX1wFzbWBsnzZ5hy2AuTnIhcCSwKMk/0dXGVdV2a+Ok\n2TNMFf21VbW0qpYDq4B/rar3YW2cNNPG8TmF64G3J3kCeFt3X9KMGOby4ceq6uvA17vbu7A2TppZ\nfqJRUsNQkNQwFCQ1DAVJDUNBUsNQkNQwFCQ1DAVJDUNBUsNQkNQwFCQ1DAVJDUNBUsNQkNQwFCQ1\nDAVJDUNBUsNQkNQYtgzm+CR3JHk0yeYkv25tnDTbhj1T+BTw5ar6JeBX6dXHWRsnzbBhymBeB5wL\n3AhQVa9U1X9hbZw004Y5U1gBvAD8Q9c6/Zkkx3CQtXGS5qdhfsT7QuBM4Iqqui/Jp9jrUqGqKsk+\na+OSrAZWAxy36LAhxpBm1w9e/spPrB15xDunMMn/G+ZMYSuwtaru6+7fQS8kdnR1ceyvNs4uSWl+\nGqY27nnguSSnd0vnAY9gbZw004ZtiLoCuCXJ4cBTwB/SC5rbk1wGPAO8Z8h9SJqgoUKhqjYAZ+3j\nIWvjpBnlJxolNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwF\nSQ1DQVLDUJDUMBQkNYb9cWyShjDtn9y8L8PWxv1pkk1JNia5NcmR1sZJs22YhqhTgD8BzqqqNwEL\ngFVYGyfNtGHfU1gIHJVkIXA08J9YGyfNtGF6H7YBfwE8C2wH/ruq7sbaOGmmDXP5cAK9s4IVwM8D\nxyR5X/82VVXAnLVxSdYlWffSS68OOoakERvm8uFtwNNV9UJV/RC4E/gNrI2TZtowofAscE6So5OE\nXgHMZqyNk2bawJ9T6Jqm7wDWA7uBbwE3AMdibZw0s4atjfsw8OG9ll/G2jhpZvkxZ0kNQ0FSw1CQ\n1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FS\n44ChkOSmJDuTbOxbm7MaLsm1SbYkeSzJ/CvKk7RfB3Om8Fng/L3W9lkNl+QMetVxv9x9zd8m8ee3\nSzPkgKFQVd8AvrvX8lzVcCuB26rq5ap6GtgCnD2iWSVNwKDvKcxVDXcK8Fzfdlu7NUkzYug3GvdX\nDbc/1sZJ89OgoTBXNdw2YFnfdku7tZ9gbZw0Pw0aCnNVw60BViU5IskK4DTgm8ONKGmSDtgQleRW\n4LeBk5JspdcIdT37qIarqk1JbgceoVcld3lVeW0gzZADhkJVXTLHQ/ushquqjwIfHWYoSdPjJxol\nNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDU\nMBQkNQwFSY1Ba+M+keTRJA8luSvJ8X2PWRsnzbBBa+PuAd5UVb8CPA5cC9bGST8NBqqNq6q7q2p3\nd3ctvX4HsDZOmnmjeE/hA8CXutvWxkkzbqhQSHIdvX6HWwb4WmvjpHlo4FBI8n7gIuC9XZ8kWBsn\nzbyBQiHJ+cDVwMVV9f2+h6yNk2bcoLVx1wJHAPckAVhbVX9kbZw0+watjbtxP9tbGyfNMD/RKKlh\nKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIahIKlhKEhqGAqSGoaCpIah\nIKlhKEhqGAqSGgN1SfY99sEkleSkvjW7JKUZNmiXJEmWAe8Anu1bs0tSmnEDdUl2/ope90P1rdkl\nKc24QctgVgLbqurBvR466C5Ja+Ok+emAvQ97S3I08CF6lw4Dq6obgBsAFv/cUXWAzSVNyCGHAvBG\nYAXwYNcOtRRYn+RsDqFLUtL8dMiXD1X1cFX9bFUtr6rl9C4Rzqyq57FLUpp5B/NPkrcC/wGcnmRr\nksvm2raqNgF7uiS/jF2S0swZtEuy//Hle923S1KaYX6iUVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQk\nNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSQ1DQVLDUJDUMBQkNQwFSY1Beh9G7n92\n72btjl3THkMSkKrplzMleQH4X+A7054FOAnn6OccrVme4w1VdfKBNpoXoQCQZF1VneUczuEc053D\n9xQkNQwFSY35FAo3THuAjnO0nKP1Uz/HvHlPQdL8MJ/OFCTNA1MPhSTnJ3ksyZYk10xwv8uSfC3J\nI0k2JbmyW/9Ikm1JNnS/LpzALN9O8nC3v3Xd2olJ7knyRPf7CWOe4fS+57whyYtJrprE8UhyU5Kd\nSTb2rc35/JNc232/PJbknWOe4xNJHk3yUJK7khzfrS9P8lLfcfn0mOeY83UY+fGoqqn9AhYATwKn\nAocDDwJnTGjfS4Azu9vHAY8DZwAfAf58wsfh28BJe619HLimu30N8LEJvy7PA2+YxPEAzgXOBDYe\n6Pl3r9GDwBHAiu77Z8EY53gHsLC7/bG+OZb3bzeB47HP12Ecx2PaZwpnA1uq6qmqegW4DVg5iR1X\n1faqWt/d/h6wGThlEvs+SCuBm7vbNwPvmuC+zwOerKpnJrGzqvoG8N29lud6/iuB26rq5ap6GthC\n7/toLHNU1d1Vtbu7uxZYOop9Heoc+zHy4zHtUDgFeK7v/lam8BczyXLgLcB93dIV3eniTeM+be8U\n8NUkDyRZ3a0trqrt3e3ngcUTmGOPVcCtffcnfTxg7uc/ze+ZDwBf6ru/ojuV/7ckvzWB/e/rdRj5\n8Zh2KExdkmOBzwFXVdWLwN/Ru5x5M7Ad+MsJjPHWqnozcAFweZJz+x+s3nniRP6ZKMnhwMXAv3RL\n0zgejUk+/7kkuQ7YDdzSLW0HfqF73f4M+Ocki8Y4wsReh2mHwjZgWd/9pd3aRCQ5jF4g3FJVdwJU\n1Y6qerWqfgT8PSM6Nd2fqtrW/b4TuKvb544kS7o5lwA7xz1H5wJgfVXt6Gaa+PHozPX8J/49k+T9\nwEXAe7uAojtd39XdfoDetfwvjmuG/bwOIz8e0w6F+4HTkqzo/gu1ClgziR0nCXAjsLmqPtm3vqRv\ns3cDG/f+2hHPcUyS4/bcpvfG1kZ6x+HSbrNLgc+Pc44+l9B36TDp49Fnrue/BliV5IgkK4DTgG+O\na4gk5wNXAxdX1ff71k9OsqC7fWo3x1NjnGOu12H0x2Mc754e4jutF9J75/9J4LoJ7vet9E5JHwI2\ndL8uBP4ReLhbXwMsGfMcp9J79/hBYNOeYwC8HrgXeAL4KnDiBI7JMcAu4HV9a2M/HvRCaDvwQ3rX\nxJft7/kD13XfL48BF4x5ji30rtn3fI98utv297rXawOwHvjdMc8x5+sw6uPhJxolNaZ9+SBpnjEU\nJDUMBUkNQ0FSw1CQ1DAUJDUMBUkNQ0FS4/8AMyHyw2TrhTgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd136910f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# crop\n",
    "observation = observation[35:195]\n",
    "print(observation.shape)\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        ..., \n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43]],\n",
       "\n",
       "       [[109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        ..., \n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43]],\n",
       "\n",
       "       [[109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        ..., \n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43]],\n",
       "\n",
       "       ..., \n",
       "       [[109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        ..., \n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43]],\n",
       "\n",
       "       [[109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        ..., \n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43],\n",
       "        [109, 118,  43]],\n",
       "\n",
       "       [[ 53,  95,  24],\n",
       "        [ 53,  95,  24],\n",
       "        [ 53,  95,  24],\n",
       "        ..., \n",
       "        [ 53,  95,  24],\n",
       "        [ 53,  95,  24],\n",
       "        [ 53,  95,  24]]], dtype=uint8)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(80, 80)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAC1tJREFUeJzt3V/oXoV9x/H3Z1FxtGs16iQ16eKFWELBpAtOsYxNzUw7\nsbsSha5SCt50Q6GjaO92MfCqtBejIGrnqGvnrFKRomTWsg2KVatba6KLc0ri1Phnxc7BNtPvLp5T\n/DVNzPn9e345v+/7BeH3nPM84ZyDvH/neR5PzjdVhaR+fm2td0DS2jB+qSnjl5oyfqkp45eaMn6p\nKeOXmlpW/El2J3k2yXNJblqpnZK0+rLUi3ySbAD+FdgFHAQeA66tqr0rt3uSVstJy/i7FwLPVdXz\nAEm+BXwKOGb8Z27cUFu3nLyMTUp6Ly8c+D9ef/Nwxrx2OfGfAxxYsHwQ+J33+gtbt5zMDx/asoxN\nSnovF15x4PgvGqz6F35Jrk/yeJLHX3vj8GpvTtJIy4n/JWDhaXzzsO6XVNWtVbWzqnaedcaGZWxO\n0kpaTvyPAeclOTfJKcA1wP0rs1uSVtuSP/NX1TtJ/gR4CNgA3FFVT6/YnklaVcv5wo+q+i7w3RXa\nF0lz5BV+UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlNrFv8VH9rOFR/avlabl9rzzC81ZfxSU8YvNWX8\nUlPL+oc9y/HQfzy1VpuWhGd+qS3jl5oyfqkp45eaMn6pqePGn+SOJIeS/GTBuo1J9iTZP/w8fXV3\nU9JKG3Pm/ytg9xHrbgIerqrzgIeHZUkTctz4q+ofgDePWP0p4M7h8Z3AH63wfklaZUv9zH92Vb08\nPH4FOHuF9kfSnCz7C7+ajfk95qhfx3VJJ6alxv9qkk0Aw89Dx3qh47qkE9NS478fuG54fB3wnZXZ\nHUnzMuZ/9X0T+AFwfpKDST4H3ALsSrIfuHxYljQhx/1XfVV17TGeumyF90XSHHmFn9SU8UtNGb/U\nlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS02NuYff\nliSPJNmb5OkkNwzrHdklTdiYM/87wBeqahtwEfD5JNtwZJc0aWPGdb1cVT8aHv8M2AecgyO7pElb\n1Gf+JFuBHcCjOLJLmrTR8Sd5P/Bt4Maqemvhc+81sstxXdKJaVT8SU5mFv5dVXXvsHrUyC7HdUkn\npjHf9ge4HdhXVV9e8JQju6QJO+7EHuAS4I+BHyd5alj3JWYjuu4exne9CFy9OrsoaTWMGdf1T0CO\n8bQju6SJ8go/qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oy\nfqkp45eaMn6pKeOXmjJ+qakxd+89NckPk/zzMKvvz4f1zuqTJmzMmf9/gEur6gJgO7A7yUU4q0+a\ntDGz+qqq/mtYPHn4UzirT5q0sRN7Ngz37D8E7Kmq0bP6HNclnZjGDO2gqg4D25OcBtyX5KNHPF9J\njjqrr6puBW4F2HnBqUd9jbTe/fVbZ/7Kus984PU12JN3Lerb/qr6KfAIsJuRs/oknZjGfNt/1nDG\nJ8mvA7uAZ3BWnzRpY972bwLuTLKB2S+Lu6vqgSQ/wFl90mSNmdX3L8COo6x/A2f1SZPlFX5SU8Yv\nNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTo+7e\nK2l51vpOvUcz+sw/3Lv/ySQPDMuO65ImbDFv+28A9i1YdlyXNGFjJ/ZsBv4QuG3Basd1SRM29sz/\nFeCLwM8XrHNclzRhY4Z2XAkcqqonjvWaqipmwzuP9tytVbWzqnaedcaGpe+ppBU15tv+S4CrknwS\nOBX4QJJvMIzrqqqXHdclTc+YEd03V9XmqtoKXAN8r6o+jeO6pElbzkU+twC7kuwHLh+WJU3Eoi7y\nqarvA98fHjuuS5owL++VmjJ+qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOX\nmjJ+qSnjl5oyfqkp45eaMn6pqVG38UryAvAz4DDwTlXtTLIR+FtgK/ACcHVV/efq7KaklbaYM//v\nV9X2qto5LDuuS5qw5bztd1yXNGFj4y/g75M8keT6Yd2ocV2STkxjb9398ap6KclvAnuSPLPwyaqq\nJEcd1zX8srge4MPnLOpO4ZJW0agzf1W9NPw8BNwHXMgwrgvgvcZ1OatPOjGNGdT5viS/8YvHwB8A\nP8FxXdKkjXkffjZwX5JfvP5vqurBJI8Bdyf5HPAicPXq7aaklXbc+KvqeeCCo6x3XJc0YV7hJzVl\n/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8\nUlOj4k9yWpJ7kjyTZF+Si5NsTLInyf7h5+mrvbOSVs7YM/9XgQer6iPM7ue3D8d1SZM25tbdHwR+\nF7gdoKr+t6p+iuO6pEkbc+Y/F3gN+HqSJ5PcNty/33Fd0oSNif8k4GPA16pqB/A2R7zFr6piNs/v\nVyS5PsnjSR5/7Y3Dy91fSStkTPwHgYNV9eiwfA+zXwaO65Im7LjxV9UrwIEk5w+rLgP24rguadLG\njs39U+CuJKcAzwOfZfaLw3Fd0kSNir+qngJ2HuUpx3VJE+UVflJTxi81ZfxSU8YvNWX8UlPGLzVl\n/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzU1ZmjH+UmeWvDnrSQ3Oq5L\nmrYxd+99tqq2V9V24LeB/wbuw3Fd0qQt9m3/ZcC/VdWLOK5LmrTFxn8N8M3hseO6pAkbHf9wz/6r\ngL878jnHdUnTs5gz/yeAH1XVq8Oy47qkCVtM/Nfy7lt+cFyXNGmj4h9Gcu8C7l2w+hZgV5L9wOXD\nsqSJGDuu623gjCPWvYHjuqTJ8go/qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqkp45eaMn6p\nKeOXmjJ+qSnjl5oyfqkp45eaMn6pKeOXmjJ+qSnjl5oyfqmpzIbtzGljyWvA28Drc9vofJ3J+jw2\nj2s6fquqzhrzwrnGD5Dk8araOdeNzsl6PTaPa33ybb/UlPFLTa1F/LeuwTbnZb0em8e1Ds39M7+k\nE4Nv+6Wm5hp/kt1Jnk3yXJKb5rntlZRkS5JHkuxN8nSSG4b1G5PsSbJ/+Hn6Wu/rUiTZkOTJJA8M\ny+vluE5Lck+SZ5LsS3Lxejm2pZhb/Ek2AH8JfALYBlybZNu8tr/C3gG+UFXbgIuAzw/HchPwcFWd\nBzw8LE/RDcC+Bcvr5bi+CjxYVR8BLmB2jOvl2BavqubyB7gYeGjB8s3AzfPa/iof23eAXcCzwKZh\n3Sbg2bXetyUcy2ZmEVwKPDCsWw/H9UHg3xm+51qwfvLHttQ/83zbfw5wYMHywWHdpCXZCuwAHgXO\nrqqXh6deAc5eo91ajq8AXwR+vmDdejiuc4HXgK8PH2luS/I+1sexLYlf+C1DkvcD3wZurKq3Fj5X\ns1PJpP5XSpIrgUNV9cSxXjPF4xqcBHwM+FpV7WB2mfkvvcWf8LEtyTzjfwnYsmB587BukpKczCz8\nu6rq3mH1q0k2Dc9vAg6t1f4t0SXAVUleAL4FXJrkG0z/uGD2TvNgVT06LN/D7JfBeji2JZln/I8B\n5yU5N8kpwDXA/XPc/opJEuB2YF9VfXnBU/cD1w2Pr2P2XcBkVNXNVbW5qrYy++/zvar6NBM/LoCq\negU4kOT8YdVlwF7WwbEt1bz/Vd8nmX2m3ADcUVV/MbeNr6AkHwf+Efgx7342/hKzz/13Ax8GXgSu\nrqo312QnlynJ7wF/VlVXJjmDdXBcSbYDtwGnAM8Dn2V2Apz8sS2FV/hJTfmFn9SU8UtNGb/UlPFL\nTRm/1JTxS00Zv9SU8UtN/T9EzCMNyMN0tgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd1362b828>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# downsample by factor of 2\n",
    "observation = observation[::2,::2,0]\n",
    "print(observation.shape)\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 53, 101, 109], dtype=uint8), array([   2,   16, 6382]))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(observation, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAC1pJREFUeJzt3W+oJYV5x/Hvr+vqNqaJbprK1rXVF2KQgrvpEhVDaV23\nNamYvhKFBAmBfWOLQkrQvOuLgq9C8kICoqZCbFJrlIgEZWsMbSFs1Wib6GrWWsW16tqkwdRSE5On\nL84Ebzb7Z/bec8/duc/3A8s9M+csM4N878w5np0nVYWkfn5trXdA0towfqkp45eaMn6pKeOXmjJ+\nqSnjl5paUfxJLk/ybJLnktw4r52StPqy3C/5JNkAfB/YBRwAHgWuqaqn57d7klbLSSv4ux8Cnquq\n5wGSfBX4GHDE+E/OKbWJU1ewSUlH83+8yU/qrYx57UriPxN4acnyAeDCo/2FTZzKhdm5gk1KOpq9\n9fDo164k/lGS7AZ2A2ziXau9OUkjreQDv5eBs5Ysbx3W/ZKqurWqdlTVjo2csoLNSZqnlcT/KHBu\nknOSnAxcDdw/n92StNqWfdlfVW8n+XPgIWADcEdVPTW3PZO0qlb0nr+qvgF8Y077ImmB/Iaf1JTx\nS00Zv9SU8UtNGb/UlPFLTRm/1NSaxb//lgvZf8tR/x2QpFXkmV9qyvilpoxfasr4paZW/WYeR3Lu\ndXvXatOS8MwvtWX8UlPGLzVl/FJTxi81dcz4k9yR5GCS7y1ZtznJniT7h5+nr+5uSpq3MWf+vwEu\nP2TdjcDDVXUu8PCwLGlCjhl/Vf0j8MNDVn8MuHN4fCfwZ3PeL0mrbLnv+c+oqleGx68CZ8xpfyQt\nyIo/8KvZmN8jjvpNsjvJY0ke+ylvrXRzkuZkufG/lmQLwPDz4JFe6Lgu6cS03PjvB64dHl8LfH0+\nuyNpUcb8r76vAN8GzktyIMmngJuBXUn2A5cNy5Im5Jj/qq+qrjnCUzvnvC+SFshv+ElNGb/UlPFL\nTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9TUmHv4\nnZXkkSRPJ3kqyfXDekd2SRM25sz/NvDpqjofuAi4Lsn5OLJLmrQx47peqarvDI9/DOwDzsSRXdKk\nHdd7/iRnA9uBvTiyS5q00fEneTfwNeCGqnpj6XNHG9nluC7pxDQq/iQbmYV/V1XdO6weNbLLcV3S\niWnMp/0Bbgf2VdXnljzlyC5pwo45sQe4BPgE8N0kTw7rPstsRNfdw/iuF4GrVmcXJa2GMeO6/hnI\nEZ52ZJc0UX7DT2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWm\njF9qyvilpoxfasr4paaMX2pqzN17NyX5lyT/Oszq+6thvbP6pAkbc+Z/C7i0qi4AtgGXJ7kIZ/VJ\nkzZmVl9V1f8MixuHP4Wz+qRJGzuxZ8Nwz/6DwJ6qGj2rz3Fd0olpzNAOqupnwLYkpwH3Jfm9Q56v\nJIed1VdVtwK3Arwnmw/7Gmm9e+g/n/yVdX/y29vWYE/ecVyf9lfVj4BHgMsZOatP0olpzKf97x/O\n+CT5dWAX8AzO6pMmbcxl/xbgziQbmP2yuLuqHkjybZzVJ03WmFl9/wZsP8z6H+CsPmmy/Iaf1JTx\nS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNGb/U1Ki7\n90pambW+U+/hjD7zD/fufyLJA8Oy47qkCTuey/7rgX1Llh3XJU3Y2Ik9W4E/BW5bstpxXdKEjT3z\nfx74DPDzJesc1yVN2JihHVcAB6vq8SO9pqqK2fDOwz13a1XtqKodGzll+Xsqaa7GfNp/CXBlko8C\nm4D3JPkyw7iuqnrFcV3S9IwZ0X1TVW2tqrOBq4FvVtXHcVyXNGkr+ZLPzcCuJPuBy4ZlSRNxXF/y\nqapvAd8aHjuuS5owv94rNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8Yv\nNWX8UlPGLzVl/FJTxi81ZfxSU6Nu45XkBeDHwM+At6tqR5LNwN8BZwMvAFdV1X+vzm5KmrfjOfP/\nUVVtq6odw7LjuqQJW8llv+O6pAkbG38B/5Dk8SS7h3WjxnVJOjGNvXX3h6vq5SS/BexJ8szSJ6uq\nkhx2XNfwy2I3wCbetaKdlTQ/o878VfXy8PMgcB/wIYZxXQBHG9flrD7pxDRmUOepSX7jF4+BPwa+\nh+O6pEkbc9l/BnBfkl+8/m+r6sEkjwJ3J/kU8CJw1ertpqR5O2b8VfU8cMFh1juuS5owv+EnNWX8\nUlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxSU8YvNWX8UlPGLzVl/FJTxi81ZfxS\nU6PiT3JaknuSPJNkX5KLk2xOsifJ/uHn6au9s5LmZ+yZ/wvAg1X1AWb389uH47qkSRtz6+73An8A\n3A5QVT+pqh/huC5p0sac+c8BXge+lOSJJLcN9+93XJc0YWPiPwn4IPDFqtoOvMkhl/hVVczm+f2K\nJLuTPJbksZ/y1kr3V9KcjIn/AHCgqvYOy/cw+2XguC5pwo4Zf1W9CryU5Lxh1U7gaRzXJU3a2Cm9\nfwHcleRk4Hngk8x+cTiuS5qoUfFX1ZPAjsM85bguaaL8hp/UlPFLTRm/1JTxS00Zv9SU8UtNGb/U\nlPFLTRm/1JTxS00Zv9SU8UtNGb/UlPFLTRm/1JTxS00Zv9SU8UtNjRnacV6SJ5f8eSPJDY7rkqZt\nzN17n62qbVW1Dfh94H+B+3BclzRpx3vZvxP496p6Ecd1SZN2vPFfDXxleOy4LmnCRsc/3LP/SuDv\nD33OcV3S9BzPmf8jwHeq6rVh2XFd0oQdT/zX8M4lPziuS5q0UfEPI7l3AfcuWX0zsCvJfuCyYVnS\nRIwd1/Um8L5D1v0Ax3VJk+U3/KSmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpoxfasr4paaM\nX2rK+KWmjF9qyvilpoxfasr4paaMX2rK+KWmjF9qyvilpjIbtrOgjSWvA28C/7WwjS7Wb7I+j83j\nmo7frar3j3nhQuMHSPJYVe1Y6EYXZL0em8e1PnnZLzVl/FJTaxH/rWuwzUVZr8fmca1DC3/PL+nE\n4GW/1NRC409yeZJnkzyX5MZFbnuekpyV5JEkTyd5Ksn1w/rNSfYk2T/8PH2t93U5kmxI8kSSB4bl\n9XJcpyW5J8kzSfYluXi9HNtyLCz+JBuAW4CPAOcD1yQ5f1Hbn7O3gU9X1fnARcB1w7HcCDxcVecC\nDw/LU3Q9sG/J8no5ri8AD1bVB4ALmB3jejm241dVC/kDXAw8tGT5JuCmRW1/lY/t68Au4Flgy7Bu\nC/DsWu/bMo5lK7MILgUeGNath+N6L/AfDJ9zLVk/+WNb7p9FXvafCby0ZPnAsG7SkpwNbAf2AmdU\n1SvDU68CZ6zRbq3E54HPAD9fsm49HNc5wOvAl4a3NLclOZX1cWzL4gd+K5Dk3cDXgBuq6o2lz9Xs\nVDKp/5WS5ArgYFU9fqTXTPG4BicBHwS+WFXbmX3N/Jcu8Sd8bMuyyPhfBs5asrx1WDdJSTYyC/+u\nqrp3WP1aki3D81uAg2u1f8t0CXBlkheArwKXJvky0z8umF1pHqiqvcPyPcx+GayHY1uWRcb/KHBu\nknOSnAxcDdy/wO3PTZIAtwP7qupzS566H7h2eHwts88CJqOqbqqqrVV1NrP/Pt+sqo8z8eMCqKpX\ngZeSnDes2gk8zTo4tuVa9L/q+yiz95QbgDuq6q8XtvE5SvJh4J+A7/LOe+PPMnvffzfwO8CLwFVV\n9cM12ckVSvKHwF9W1RVJ3sc6OK4k24DbgJOB54FPMjsBTv7YlsNv+ElN+YGf1JTxS00Zv9SU8UtN\nGb/UlPFLTRm/1JTxS039P0gwF6DhwGMUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdd02fd8208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# erase background (background type 2)\n",
    "observation[observation == 109] = 0\n",
    "plt.imshow(observation)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf placeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_x = tf.placeholder(dtype=tf.float32, shape=[None, n_obs], name=\"tf_x\")\n",
    "tf_y = tf.placeholder(dtype=tf.float32, shape=[None, n_actions], name=\"tf_y\")\n",
    "tf_epr = tf.placeholder(dtype=tf.float32, shape=[None, 1], name=\"tf_epr\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf reward processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_discounted_epr = tf_discount_rewards(tf_epr)\n",
    "tf_mean, tf_variance = tf.nn.moments(tf_discounted_epr, [0], shift=None, name=\"reward_moments\")\n",
    "tf_discounted_epr -= tf_mean\n",
    "tf_discounted_epr /= tf.sqrt(tf_variance + 1e-6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf optimizer op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf_aprob = tf_policy_forward(tf_x)\n",
    "loss = tf.nn.l2_loss(tf_y - tf_aprob)\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=decay)\n",
    "tf_grads = optimizer.compute_gradients(loss, var_list=tf.trainable_variables(), grad_loss=tf_discounted_epr)\n",
    "train_op = optimizer.apply_gradients(tf_grads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf graph initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### try load saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no save model to load. starting new session\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.global_variables ())\n",
    "# load_was_success = True\n",
    "try:\n",
    "    save_dir = '/'.join(save_path.split('/')[:-1])\n",
    "    ckpt = tf.train.get_checkpoint_state(save_dir)\n",
    "    load_path = ckpt.model_checkpoint_path\n",
    "    saver.restore(sess, load_path)\n",
    "except:\n",
    "    print(\"no save model to load. starting new session\")\n",
    "else:\n",
    "    print(\"loaded model: {}\".format(load_path))\n",
    "    saver = tf.train.Saver(tf.global_variables ())\n",
    "    episode_number = int(load_path.split('-')[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ep 0: reward: -19.0, mean reward: -19.000000\n",
      "\tep 1: reward: -21.0\n",
      "\tep 2: reward: -18.0\n",
      "\tep 3: reward: -21.0\n",
      "\tep 4: reward: -21.0\n",
      "\tep 5: reward: -19.0\n",
      "\tep 6: reward: -21.0\n",
      "\tep 7: reward: -20.0\n",
      "\tep 8: reward: -21.0\n",
      "\tep 9: reward: -20.0\n",
      "ep 10: reward: -20.0, mean reward: -19.114931\n",
      "\tep 11: reward: -21.0\n",
      "\tep 12: reward: -21.0\n",
      "\tep 13: reward: -21.0\n",
      "\tep 14: reward: -20.0\n",
      "\tep 15: reward: -19.0\n",
      "\tep 16: reward: -21.0\n",
      "\tep 17: reward: -19.0\n",
      "\tep 18: reward: -20.0\n",
      "\tep 19: reward: -20.0\n",
      "ep 20: reward: -21.0, mean reward: -19.227636\n",
      "\tep 21: reward: -21.0\n",
      "\tep 22: reward: -21.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-9693db150159>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m     \u001b[0;31m# step the envirnoment and get new measurements\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0;31m# action 1 = 不動, action 2 = 向上, action 3 = 向下\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m     \u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m     \u001b[0mreward_sum\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/gym/core.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m         \"\"\"\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/gym/wrappers/time_limit.py\u001b[0m in \u001b[0;36m_step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_episode_started_at\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Cannot call env.step() before calling reset()\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_elapsed_steps\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/gym/core.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m         \"\"\"\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/gym/envs/atari/atari_env.py\u001b[0m in \u001b[0;36m_step\u001b[0;34m(self, a)\u001b[0m\n\u001b[1;32m     79\u001b[0m             \u001b[0mnum_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnp_random\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframeskip\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframeskip\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0mreward\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0male\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m         \u001b[0mob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_obs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/atari_py/ale_python_interface.py\u001b[0m in \u001b[0;36mact\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0male_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgame_over\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "observation = env.reset()\n",
    "\n",
    "while True:\n",
    "    # preprocess the observation, set input to network to be difference image\n",
    "    cur_x = prepro(observation)\n",
    "    x = cur_x - prev_x if prev_x is not None else np.zeros(n_obs)\n",
    "    prev_x = cur_x\n",
    "    \n",
    "    # stochastically sample a policy from the network\n",
    "    feed = {tf_x: np.reshape(x, (1,-1))} # reshape x (80, 80) to (1, 6400)\n",
    "    aprob = sess.run(tf_aprob, feed) \n",
    "    aprob = aprob[0,:] # turn array([[...]]) to array([...]), aprob shape is (1, 3)\n",
    "    action = np.random.choice(n_actions, p=aprob)\n",
    "    label = np.zeros_like(aprob)\n",
    "    label[action] = 1\n",
    "    \n",
    "    # step the envirnoment and get new measurements\n",
    "    # action 1 = 不動, action 2 = 向上, action 3 = 向下\n",
    "    observation, reward, done, info = env.step(action+1)\n",
    "    reward_sum += reward\n",
    "    \n",
    "    # record game history\n",
    "    xs.append(x)\n",
    "    ys.append(label)\n",
    "    rs.append(reward)\n",
    "    \n",
    "    if done:\n",
    "        # update running reward\n",
    "        running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01\n",
    "        \n",
    "        \n",
    "        # parameter update\n",
    "        feed = {tf_x: np.vstack(xs), tf_epr: np.vstack(rs), tf_y: np.vstack(ys)}\n",
    "        _ = sess.run(train_op, feed)\n",
    "        \n",
    "        \n",
    "        # print progress console\n",
    "        if episode_number % 10 == 0:\n",
    "            print('ep {}: reward: {}, mean reward: {:3f}'.format(episode_number, reward_sum, running_reward))\n",
    "        else:\n",
    "            print('\\tep {}: reward: {}'.format(episode_number, reward_sum))\n",
    "            \n",
    "        episode_number += 1\n",
    "        \n",
    "        \n",
    "        # reset\n",
    "        xs, rs, ys = [], [], []\n",
    "        observation = env.reset()\n",
    "        reward_sum = 0\n",
    "        \n",
    "        \n",
    "        # save model\n",
    "        if episode_number % 50 == 0:\n",
    "            saver.save(sess, save_path, global_step=episode_number)\n",
    "            print(\"SAVED MODEL #{}\".format(episode_number))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### put it all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-05-11 10:31:43,851] Making new env: Pong-v0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no save model to load. starting new session\n",
      "ep 0: reward: -20.0, mean reward: -20.000000\n",
      "\tep 1: reward: -21.0\n",
      "\tep 2: reward: -19.0\n",
      "\tep 3: reward: -21.0\n",
      "\tep 4: reward: -18.0\n",
      "\tep 5: reward: -21.0\n",
      "\tep 6: reward: -21.0\n",
      "\tep 7: reward: -19.0\n",
      "\tep 8: reward: -21.0\n",
      "\tep 9: reward: -21.0\n",
      "ep 10: reward: -19.0, mean reward: -20.009513\n",
      "\tep 11: reward: -21.0\n",
      "\tep 12: reward: -19.0\n",
      "\tep 13: reward: -21.0\n",
      "\tep 14: reward: -21.0\n",
      "\tep 15: reward: -21.0\n",
      "\tep 16: reward: -21.0\n",
      "\tep 17: reward: -21.0\n",
      "\tep 18: reward: -21.0\n",
      "\tep 19: reward: -19.0\n",
      "ep 20: reward: -20.0, mean reward: -20.055966\n",
      "\tep 21: reward: -21.0\n",
      "\tep 22: reward: -21.0\n",
      "\tep 23: reward: -21.0\n",
      "\tep 24: reward: -19.0\n",
      "\tep 25: reward: -21.0\n",
      "\tep 26: reward: -19.0\n",
      "\tep 27: reward: -20.0\n",
      "\tep 28: reward: -21.0\n",
      "\tep 29: reward: -20.0\n",
      "ep 30: reward: -21.0, mean reward: -20.088588\n",
      "\tep 31: reward: -20.0\n",
      "\tep 32: reward: -20.0\n",
      "\tep 33: reward: -21.0\n",
      "\tep 34: reward: -21.0\n",
      "\tep 35: reward: -20.0\n",
      "\tep 36: reward: -21.0\n",
      "\tep 37: reward: -19.0\n",
      "\tep 38: reward: -21.0\n",
      "\tep 39: reward: -21.0\n",
      "ep 40: reward: -21.0, mean reward: -20.128457\n",
      "\tep 41: reward: -21.0\n",
      "\tep 42: reward: -21.0\n",
      "\tep 43: reward: -19.0\n",
      "\tep 44: reward: -20.0\n",
      "\tep 45: reward: -19.0\n",
      "\tep 46: reward: -21.0\n",
      "\tep 47: reward: -21.0\n",
      "\tep 48: reward: -19.0\n",
      "\tep 49: reward: -18.0\n",
      "SAVED MODEL #50\n",
      "ep 50: reward: -20.0, mean reward: -20.105414\n",
      "\tep 51: reward: -20.0\n",
      "\tep 52: reward: -19.0\n",
      "\tep 53: reward: -20.0\n",
      "\tep 54: reward: -21.0\n",
      "\tep 55: reward: -21.0\n",
      "\tep 56: reward: -19.0\n",
      "\tep 57: reward: -21.0\n",
      "\tep 58: reward: -20.0\n",
      "\tep 59: reward: -21.0\n",
      "ep 60: reward: -21.0, mean reward: -20.125029\n",
      "\tep 61: reward: -21.0\n",
      "\tep 62: reward: -19.0\n",
      "\tep 63: reward: -20.0\n",
      "\tep 64: reward: -21.0\n",
      "\tep 65: reward: -21.0\n",
      "\tep 66: reward: -20.0\n",
      "\tep 67: reward: -21.0\n",
      "\tep 68: reward: -19.0\n",
      "\tep 69: reward: -21.0\n",
      "ep 70: reward: -20.0, mean reward: -20.141708\n",
      "\tep 71: reward: -20.0\n",
      "\tep 72: reward: -21.0\n",
      "\tep 73: reward: -20.0\n",
      "\tep 74: reward: -20.0\n",
      "\tep 75: reward: -20.0\n",
      "\tep 76: reward: -21.0\n",
      "\tep 77: reward: -21.0\n",
      "\tep 78: reward: -19.0\n",
      "\tep 79: reward: -21.0\n",
      "ep 80: reward: -21.0, mean reward: -20.166794\n",
      "\tep 81: reward: -21.0\n",
      "\tep 82: reward: -20.0\n",
      "\tep 83: reward: -21.0\n",
      "\tep 84: reward: -21.0\n",
      "\tep 85: reward: -20.0\n",
      "\tep 86: reward: -20.0\n",
      "\tep 87: reward: -21.0\n",
      "\tep 88: reward: -20.0\n",
      "\tep 89: reward: -21.0\n",
      "ep 90: reward: -19.0, mean reward: -20.188319\n",
      "\tep 91: reward: -21.0\n",
      "\tep 92: reward: -21.0\n",
      "\tep 93: reward: -21.0\n",
      "\tep 94: reward: -21.0\n",
      "\tep 95: reward: -21.0\n",
      "\tep 96: reward: -20.0\n",
      "\tep 97: reward: -21.0\n",
      "\tep 98: reward: -20.0\n",
      "\tep 99: reward: -20.0\n",
      "SAVED MODEL #100\n",
      "ep 100: reward: -18.0, mean reward: -20.206623\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-8a586a84350a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    139\u001b[0m         \u001b[0;31m# parameter update\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    140\u001b[0m         \u001b[0mfeed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mtf_x\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf_epr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf_y\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 141\u001b[0;31m         \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    765\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    768\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1016\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1020\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1023\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/nailo/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1002\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1003\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from gym import wrappers\n",
    "\n",
    "n_obs = 80 * 80        # dimensionality of observations\n",
    "h = 200                # number of hidden layer neurons\n",
    "n_actions = 3          # number of available actions\n",
    "learning_rate = 1e-3\n",
    "gamma = 0.99           # discount factor for reward\n",
    "decay = 0.99           # decay rate for RMSProp gradients\n",
    "save_path = './models/pong.ckpt'\n",
    "\n",
    "\n",
    "if not os.path.exists('./models'):\n",
    "    os.makedirs('./models')\n",
    "\n",
    "\n",
    "env = gym.make(\"Pong-v0\")\n",
    "# env = wrappers.Monitor(env, '/tmp/pong-pg-tensorflow', video_callable=lambda count: count % 300 == 0, force=True)\n",
    "observation = env.reset()\n",
    "prev_x = None\n",
    "xs, rs, ys = [],[],[]\n",
    "running_reward = None\n",
    "reward_sum = 0\n",
    "episode_number = 0\n",
    "\n",
    "\n",
    "\n",
    "tf_model = {}\n",
    "with tf.variable_scope('layer_one', reuse=False):\n",
    "    xavier_l1 = tf.truncated_normal_initializer(mean=0, stddev=1./np.sqrt(n_obs), dtype=tf.float32)\n",
    "    tf_model['W1'] = tf.get_variable(\"W1\", [n_obs, h], initializer=xavier_l1)\n",
    "    \n",
    "with tf.variable_scope('layer_two', reuse=False):\n",
    "    xavier_l2 = tf.truncated_normal_initializer(mean=0, stddev=1./np.sqrt(h), dtype=tf.float32)\n",
    "    tf_model['W2'] = tf.get_variable(\"W2\", [h, n_actions], initializer=xavier_l2)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def tf_discount_rewards(tf_r): # tf_r shape:(game_steps, 1)\n",
    "    discount_f = lambda a, v: a*gamma + v;\n",
    "    tf_r_reverse = tf.scan(discount_f, tf.reverse(tf_r, [True, False]))\n",
    "    tf_discounted_r = tf.reverse(tf_r_reverse, [True, False])\n",
    "    return tf_discounted_r\n",
    "\n",
    "\n",
    "def tf_policy_forward(x): # x shape:(1, n_obs)\n",
    "    h = tf.matmul(x, tf_model['W1'])\n",
    "    h = tf.nn.relu(h)\n",
    "    logp = tf.matmul(h, tf_model['W2'])\n",
    "    p = tf.nn.softmax(logp)\n",
    "    return p\n",
    "\n",
    "\n",
    "def prepro(I):\n",
    "    \"\"\" prepro 210x160x3 uint8 frame into 6400 (80x80) 1D float vector \"\"\"\n",
    "    I = I[35:195]    # crop\n",
    "    I = I[::2,::2,0] # downsample by factor of 2\n",
    "    I[I == 144] = 0  # erase background (background type 1)\n",
    "    I[I == 109] = 0  # erase background (background type 2)\n",
    "    I[I != 0] = 1    # everything else (paddles, ball) just set to 1\n",
    "    return I.astype(np.float).ravel()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "tf_x = tf.placeholder(dtype=tf.float32, shape=[None, n_obs], name=\"tf_x\")\n",
    "tf_y = tf.placeholder(dtype=tf.float32, shape=[None, n_actions], name=\"tf_y\")\n",
    "tf_epr = tf.placeholder(dtype=tf.float32, shape=[None, 1], name=\"tf_epr\")\n",
    "\n",
    "\n",
    "tf_discounted_epr = tf_discount_rewards(tf_epr)\n",
    "tf_mean, tf_variance = tf.nn.moments(tf_discounted_epr, [0], shift=None, name=\"reward_moments\")\n",
    "tf_discounted_epr -= tf_mean\n",
    "tf_discounted_epr /= tf.sqrt(tf_variance + 1e-6)\n",
    "\n",
    "\n",
    "tf_aprob = tf_policy_forward(tf_x)\n",
    "loss = tf.nn.l2_loss(tf_y - tf_aprob)\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=decay)\n",
    "tf_grads = optimizer.compute_gradients(loss, var_list=tf.trainable_variables(), grad_loss=tf_discounted_epr)\n",
    "train_op = optimizer.apply_gradients(tf_grads)\n",
    "\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "\n",
    "\n",
    "saver = tf.train.Saver(tf.global_variables ())\n",
    "# load_was_success = True\n",
    "try:\n",
    "    save_dir = '/'.join(save_path.split('/')[:-1])\n",
    "    ckpt = tf.train.get_checkpoint_state(save_dir)\n",
    "    load_path = ckpt.model_checkpoint_path\n",
    "    saver.restore(sess, load_path)\n",
    "except:\n",
    "    print(\"no save model to load. starting new session\")\n",
    "else:\n",
    "    print(\"loaded model: {}\".format(load_path))\n",
    "    saver = tf.train.Saver(tf.global_variables ())\n",
    "    episode_number = int(load_path.split('-')[-1])\n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "while True:\n",
    "    # preprocess the observation, set input to network to be difference image\n",
    "    cur_x = prepro(observation)\n",
    "    x = cur_x - prev_x if prev_x is not None else np.zeros(n_obs)\n",
    "    prev_x = cur_x\n",
    "    \n",
    "    # stochastically sample a policy from the network\n",
    "    feed = {tf_x: np.reshape(x, (1,-1))} # reshape x (80, 80) to (1, 6400)\n",
    "    aprob = sess.run(tf_aprob, feed) \n",
    "    aprob = aprob[0,:] # turn array([[...]]) to array([...]), aprob shape is (1, 3)\n",
    "    action = np.random.choice(n_actions, p=aprob)\n",
    "    label = np.zeros_like(aprob)\n",
    "    label[action] = 1\n",
    "    \n",
    "    # step the envirnoment and get new measurements\n",
    "    # action 1 = 不動, action 2 = 向上, action 3 = 向下\n",
    "    observation, reward, done, info = env.step(action+1)\n",
    "    reward_sum += reward\n",
    "    \n",
    "    # record game history\n",
    "    xs.append(x)\n",
    "    ys.append(label)\n",
    "    rs.append(reward)\n",
    "    \n",
    "    if done:\n",
    "        # update running reward\n",
    "        running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01\n",
    "        \n",
    "        \n",
    "        # parameter update\n",
    "        feed = {tf_x: np.vstack(xs), tf_epr: np.vstack(rs), tf_y: np.vstack(ys)}\n",
    "        _ = sess.run(train_op, feed)\n",
    "        \n",
    "        \n",
    "        # print progress console\n",
    "        if episode_number % 10 == 0:\n",
    "            print('ep {}: reward: {}, mean reward: {:3f}'.format(episode_number, reward_sum, running_reward))\n",
    "        else:\n",
    "            print('\\tep {}: reward: {}'.format(episode_number, reward_sum))\n",
    "            \n",
    "        episode_number += 1\n",
    "        \n",
    "        \n",
    "        # reset\n",
    "        xs, rs, ys = [], [], []\n",
    "        observation = env.reset()\n",
    "        reward_sum = 0\n",
    "        \n",
    "        \n",
    "        # save model\n",
    "        if episode_number % 50 == 0:\n",
    "            saver.save(sess, save_path, global_step=episode_number)\n",
    "            print(\"SAVED MODEL #{}\".format(episode_number))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}