DQN with FrozenLake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from util import to_onehot, moving_average, Agent\n",
    "import gym\n",
    "from gym.envs.registration import register, spec\n",
    "\n",
    "plt.style.use('fivethirtyeight')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MY_ENV_NAME='FrozenLakeNonskid4x4-v1'\n",
    "\n",
    "register(\n",
    "    id=MY_ENV_NAME,\n",
    "    entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
    "    kwargs={'map_name': '4x4', 'is_slippery': False},\n",
    "    timestep_limit=500,\n",
    "    reward_threshold=0.8196\n",
    ")\n",
    "env = gym.make(MY_ENV_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SAMPLE_BATCH_SIZE = 32\n",
    "OBS_SPACE = env.observation_space.n  # size of state space\n",
    "ACTION_SPACE = env.action_space.n  # 0 = left; 1 = down; 2 = right;  3 = up\n",
    "N_EPISODES = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scenario 1: single agent training\n",
    "the agent is responsible for learning value and policy simultaneously. \n",
    "\n",
    "the original atari approach used a single DQN agent. it was discovered later that an actor/critic approach works better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from agent import Agent\n",
    "agent = Agent(OBS_SPACE, ACTION_SPACE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "history = []\n",
    "num_steps = []\n",
    "\n",
    "for episode in range(N_EPISODES):\n",
    "    \n",
    "    state = env.reset()  # state is a int number corresponding to the agent's position in a board\n",
    "    done = False\n",
    "    steps = 0\n",
    "    \n",
    "    while not done:\n",
    "        \n",
    "        # propose an action\n",
    "        curr_state_encoded = to_onehot(OBS_SPACE, state).reshape(1, OBS_SPACE)\n",
    "        action = agent.act(curr_state_encoded)\n",
    "        \n",
    "        # what are the consequences of taking that action?\n",
    "        next_state, reward, done, transmit_prob = env.step(action)\n",
    "        next_state_encoded = to_onehot(OBS_SPACE, next_state).reshape(1, OBS_SPACE)\n",
    "        \n",
    "        # store memories for experience replay (prevents catastrophic forgetting)\n",
    "        agent.remember(curr_state_encoded, action, reward, next_state_encoded, done)\n",
    "        \n",
    "        # move to next state\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "    agent.replay(SAMPLE_BATCH_SIZE)\n",
    "    \n",
    "    # if in final state, then mark 1 for success\n",
    "    if state == (OBS_SPACE - 1):\n",
    "        history.append(1)\n",
    "    else:\n",
    "        history.append(0)\n",
    "    num_steps.append(steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(moving_average(history, 500))\n",
    "plt.ylabel(\"probability of successive outcome\")\n",
    "plt.xlabel(\"episodes\")\n",
    "plt.title(\"RL\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(moving_average(num_steps, 500))\n",
    "plt.ylabel(\"number of steps required for success\")\n",
    "plt.xlabel(\"episodes\")\n",
    "plt.title(\"RL\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# approach 2: actor/critic model\n",
    "\n",
    "a critic network (aka value network) predicts a single value for a specific location.\n",
    "\n",
    "the value network will place higher values near the final winning position (and low values in the \"hole\" positions). it will place the higest value on the winning final position.\n",
    "\n",
    "the value network expresses the the most efficient path to the policy network. it is no different from a critic telling an actor how to act (and similarly, it is up to the actor to follow the critic's suggestions.)\n",
    "\n",
    "the actor (aka policy network) predicts the best action from the current state. the best action is quantified as (1) reward + (2) discounted next value - (3) predicted value. the actor will no longer select the < action/next-state > with the highest value; instead it will select the < action/next-state > that results in the greatest change in value.\n",
    "\n",
    "* http://www.rage.net/~greg/2016-07-05-ActorCritic-with-OpenAI-Gym.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from critic import Critic\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "critic = Critic(OBS_SPACE, ACTION_SPACE)\n",
    "gamma = .9  # weighting\n",
    "\n",
    "OBS_SQR= int(math.sqrt(OBS_SPACE))\n",
    "STATEGRID = np.zeros((OBS_SQR,OBS_SQR))\n",
    "STATEGRID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for episode in range(N_EPISODES):\n",
    "    \n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    steps = 0\n",
    "    \n",
    "    while not done:\n",
    "        \n",
    "        # ask critic to predict value for current state        \n",
    "        curr_state_encoded = to_onehot(OBS_SPACE, state).reshape(1, OBS_SPACE)\n",
    "        orig_val = critic.predict(curr_state_encoded)\n",
    "        \n",
    "        # take an action\n",
    "        action = agent.act(curr_state_encoded)\n",
    "        \n",
    "        # what are the consequences of taking that action?\n",
    "        next_state, reward, done, transmit_prob = env.step(action)\n",
    "        next_state_encoded = to_onehot(OBS_SPACE, next_state).reshape(1, OBS_SPACE)\n",
    "        \n",
    "        # get critic's prediction on next state\n",
    "        new_val = critic.predict(next_state_encoded)\n",
    "        \n",
    "        # determine target value\n",
    "        if not done:\n",
    "            target = (gamma * new_val)\n",
    "        else:\n",
    "            target = (gamma * reward)  # max value is discounted reward at final location\n",
    "        best_val = max((orig_val*gamma), target)\n",
    "        \n",
    "        # each state is encoded with its \"best\" target value\n",
    "        # the target value is equal to the discounted next value\n",
    "        # we dont care which action led to this value!\n",
    "        critic.remember((curr_state_encoded, best_val))\n",
    "        \n",
    "        # if terminal, append another replay for final location (otherwise final loc will never be recorded)\n",
    "        if done:\n",
    "            critic.remember((next_state_encoded, reward))\n",
    "        \n",
    "        # move on to next state and continue\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "    \n",
    "    critic.replay(SAMPLE_BATCH_SIZE)\n",
    "    \n",
    "    if episode % 500 == 0:\n",
    "        print (episode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "critic.plot_value(STATEGRID)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## actor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from actor import Actor\n",
    "actor = Actor(OBS_SPACE, ACTION_SPACE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for episode in range(500):\n",
    "    \n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    steps = 0\n",
    "    \n",
    "    while not done:\n",
    "        # get value for current state from critic\n",
    "        curr_state_encoded = to_onehot(OBS_SPACE, state).reshape(1, OBS_SPACE)\n",
    "        orig_val = critic.predict(curr_state_encoded)\n",
    "        \n",
    "        # propose an action\n",
    "        action = actor.act(curr_state_encoded)\n",
    "        \n",
    "        # what are the consequences of taking that action?\n",
    "        next_state, reward, done, transmit_prob = env.step(action)\n",
    "        next_state_encoded = to_onehot(OBS_SPACE, next_state).reshape(1, OBS_SPACE)\n",
    "        \n",
    "        # get value for next state from critic\n",
    "        new_val = critic.predict(next_state_encoded)\n",
    "        \n",
    "        # The actor is updated\n",
    "        # by using the difference of the value the critic\n",
    "        # placed on the old state vs. the value the critic\n",
    "        # places on the new state.. encouraging the actor\n",
    "        # to move into more valuable states.\n",
    "        \n",
    "        # actor learns to predict a set of actor deltas\n",
    "        actor_delta = new_val - orig_val                \n",
    "        actor.remember([curr_state_encoded, action, actor_delta])\n",
    "    \n",
    "        # move on to next state and continue\n",
    "        state = next_state\n",
    "        steps += 1\n",
    "    \n",
    "        # TODO: inside while loop or not?\n",
    "        actor.replay()\n",
    "    \n",
    "    if episode % 250 == 0:\n",
    "        print (episode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "obs_sqr = 4\n",
    "np_w_cri_r = np.zeros((obs_sqr, obs_sqr))\n",
    "working_state = STATEGRID.copy()\n",
    "for x in range(0, obs_sqr):\n",
    "    for y in range(0, obs_sqr):\n",
    "        my_state = working_state.copy()\n",
    "        \n",
    "        my_state[x,y] = 1  # Place the player at a given X/Y location.\n",
    "\n",
    "        # And now have the critic model predict the state value\n",
    "        # with the player in that location.\n",
    "        value = actor.predict(my_state.reshape(1, OBS_SPACE))\n",
    "        np_w_cri_r[x,y] = np.argmax(value)\n",
    "np_w_cri_r.shape\n",
    "plt.pcolor(np_w_cri_r)\n",
    "plt.title(\"ACTION Network\")\n",
    "plt.colorbar()\n",
    "plt.xlabel(\"X\")\n",
    "plt.ylabel(\"Y\")\n",
    "plt.gca().invert_yaxis()\n",
    "plt.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 0 = left; 1 = down; 2 = right;  3 = up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}