Reinforcement_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "Reinforcement learning.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "3-LcjjWYQiVS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "S_t-state\n",
        "a_t=action\n",
        "r_t+1=reward at next state\n",
        "alpha-learning rate\n",
        "gamma-discount\n",
        "epsilon-exploration\n",
        "Q-value-->maximum reward that can be taken in a state\n",
        "Q(s_t,a_t)=(r_t+1)+gamma(r_t+2)+gamma^2(r_t+3)+gamma^3(r_t+4)+........\n",
        "Q(s_t+1,a_t+1)=(r_t+2)+gamma(r_t+3)+gamma^2(r_t+4)+gamma^3(r_t+5)+........\n",
        "Q(s_t,a_t)=r_t+1 + gammaQ(s_t+1,a_t+1)\n",
        "Q(s_t,a_t)=r_t+1 + gamma*Q_max(s_t+1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IbxHkXfpQiVc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " ___________\n",
        "|__|__|__|__|      16 columns determines state,4 rows determines(action ie. top,down,left,right)\n",
        "|__|__|__|__|\n",
        "|__|__|__|__|=====>\n",
        "|__|__|__|__|\n",
        "\n",
        "S-start\n",
        "H-Hole\n",
        "F-forzen\n",
        "G-gole"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pnh4H9HYQiVj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from IPython.display import clear_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmUgNK3EQiVp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gym\n",
        "import random\n",
        "import numpy as np\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tTs1zJb9QiVy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from gym.envs.registration import register"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OasdK8FYQiV6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#https://github.com/openai/gym/blob/master/gym/envs/__init__.py\n",
        "\n",
        "try:\n",
        "    register(\n",
        "        id='FrozenLakeNoSlip-v0',\n",
        "        entry_point='gym.envs.toy_text:FrozenLakeEnv',\n",
        "        kwargs={'map_name' : '4x4','is_slippery':False},\n",
        "        max_episode_steps=100,\n",
        "        reward_threshold=0.78, # optimum = .8196\n",
        "    )\n",
        "except:\n",
        "    pass\n",
        "\n",
        "env_name='FrozenLakeNoSlip-v0'\n",
        "env=gym.make(env_name)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aeJZ1fQoQiWB",
        "colab_type": "code",
        "outputId": "4e271a18-9188-4492-8341-736e73c7b305",
        "colab": {}
      },
      "source": [
        "env.observation_space"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Discrete(16)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E1B0YOfHQiWL",
        "colab_type": "code",
        "outputId": "57edf7e4-6f9e-4d9b-a1cf-04e919ab7a74",
        "colab": {}
      },
      "source": [
        "env.action_space"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Discrete(4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-x07vuHtQiWU",
        "colab_type": "code",
        "outputId": "4d7239a7-0f18-4a9a-af84-e94bc1ec1a93",
        "colab": {}
      },
      "source": [
        "type(env.action_space)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "gym.spaces.discrete.Discrete"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GtI6phQtQiWa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class  Agent():\n",
        "    def __init__(self,env):\n",
        "        self.env=env\n",
        "        self.actionSize=env.action_space.n   #4\n",
        "        self.stateSize=env.observation_space.n  #16\n",
        "        \n",
        "    def getAction(self,state):\n",
        "        action=self.env.action_space.sample()\n",
        "        #or  action=random.choice(range(self.actionSize))\n",
        "        \n",
        "        #action=np.random.uniform(self.env.action_space.low,\n",
        "        #                       self.env.action_space.high,         for continuous\n",
        "        #                      self.env.action_space.shape)\n",
        "        \n",
        "        return action\n",
        "agent=Agent(env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Xl6S4yuQiWf",
        "colab_type": "code",
        "outputId": "330bc69a-91ea-4722-fed2-f472d0df94cb",
        "colab": {}
      },
      "source": [
        "for ep in range(20):\n",
        "    state=env.reset()\n",
        "    done=False\n",
        "    while not done:\n",
        "        action=agent.getAction(state)\n",
        "        state,reward,done,info=env.step(action)\n",
        "        env.render()\n",
        "        time.sleep(0.05)\n",
        "        clear_output(wait=True)\n",
        "        #if reward==1:\n",
        "         #   break\n",
        "env.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  (Right)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HFF\u001b[41mG\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCWl28asQiWp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V2TWRUHMQiWu",
        "colab_type": "text"
      },
      "source": [
        "# Q-Learning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QtiBIr_GQiWz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class  Agent():\n",
        "    def __init__(self,env,discountRate=0.97,learningRate=0.01):\n",
        "        self.env=env\n",
        "        self.actionSize=env.action_space.n   #4\n",
        "        self.stateSize=env.observation_space.n  #16\n",
        "        \n",
        "        self.explorationRate=1.0\n",
        "        self.discountRate=discountRate\n",
        "        self.learningRate=learningRate\n",
        "        \n",
        "        self.qTable=1e-4*np.random.random([self.stateSize, self.actionSize])\n",
        "        \n",
        "    def getAction(self,state):\n",
        "        qState=self.qTable[state]\n",
        "        actionGreedy=np.argmax(qState)\n",
        "        actionRandom=self.env.action_space.sample()\n",
        "        if random.random()<self.explorationRate:\n",
        "            action=actionRandom \n",
        "        else:\n",
        "            action=actionGreedy\n",
        "        return action\n",
        "    \n",
        "    def train(self,state,action,nextState,reward,done):\n",
        "        if done:\n",
        "            qNext=np.zeros([self.actionSize])\n",
        "        else:\n",
        "            qNext=self.qTable[nextState]\n",
        "        qTarget=reward+self.discountRate*np.max(qNext)\n",
        "        qUpdate=qTarget-self.qTable[state,action]\n",
        "        self.qTable[state,action]+=self.learningRate*qUpdate\n",
        "        \n",
        "        if done:\n",
        "            self.explorationRate*=0.99\n",
        "            \n",
        "agent=Agent(env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yTUOCyy8QiW5",
        "colab_type": "code",
        "outputId": "875d9bfa-0490-44db-e8dd-705984d5c8f9",
        "colab": {}
      },
      "source": [
        "totalReward=0\n",
        "for ep in range(100):\n",
        "    state=env.reset()\n",
        "    done=False\n",
        "    while not done:\n",
        "        action=agent.getAction(state)\n",
        "        nextState,reward,done,info=env.step(action)\n",
        "        agent.train(state, action, nextState, reward, done)\n",
        "        \n",
        "        print(f's:{state},a:{action}')\n",
        "        state=nextState\n",
        "        \n",
        "        totalReward+=reward\n",
        "        print(f'Ep:{ep},Goal:{totalReward},Explore:{agent.explorationRate}')\n",
        "        \n",
        "        env.render()\n",
        "        print(agent.qTable)\n",
        "        time.sleep(0.05)\n",
        "        clear_output(wait=True)\n",
        "env.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "s:14,a:2\n",
            "Ep:99,Goal:43.0,Explore:0.13397967485796175\n",
            "  (Right)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HFF\u001b[41mG\u001b[0m\n",
            "[[7.56395168e-05 9.45823816e-05 7.41979865e-05 7.61343662e-05]\n",
            " [8.08994251e-05 2.49032650e-05 7.44601852e-05 6.57981636e-05]\n",
            " [3.85665267e-05 5.43058199e-05 3.63199902e-05 3.59834771e-05]\n",
            " [1.95018261e-05 6.46909715e-05 2.40466680e-05 6.90188723e-05]\n",
            " [6.52149870e-05 2.83977460e-04 1.10431550e-05 7.63533275e-05]\n",
            " [3.05358414e-05 4.44658280e-05 1.91291334e-05 5.50356009e-06]\n",
            " [7.31174695e-05 6.50897838e-04 5.12037007e-05 6.98077298e-05]\n",
            " [9.41768709e-05 4.91222028e-05 3.46179454e-05 2.33618581e-05]\n",
            " [7.76734358e-05 2.78592869e-05 1.87930899e-03 3.81886872e-05]\n",
            " [6.62668179e-05 1.30740151e-05 1.26477369e-02 7.00684977e-05]\n",
            " [1.93623472e-04 7.96018428e-02 5.95309917e-05 8.03073106e-05]\n",
            " [3.92084674e-05 7.29395092e-05 1.28698768e-05 7.63095707e-05]\n",
            " [7.77027535e-05 7.13308314e-06 9.39821301e-05 3.68425613e-05]\n",
            " [4.09219683e-05 4.94298200e-05 3.65103299e-05 2.06698445e-04]\n",
            " [5.18579145e-05 4.67627893e-03 3.70219646e-01 1.91081567e-04]\n",
            " [2.21069340e-05 7.30378650e-05 7.86143262e-05 8.25894224e-05]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8EYM1NrhQiW9",
        "colab_type": "text"
      },
      "source": [
        "#  Q-Learning with Neural Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uHkb-pLLQiW_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ID2A1BBqQiXG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class  Agent():\n",
        "    def __init__(self,env,discountRate=0.97,learningRate=0.01):\n",
        "        self.env=env\n",
        "        self.actionSize=env.action_space.n   #4\n",
        "        self.stateSize=env.observation_space.n  #16\n",
        "        \n",
        "        self.explorationRate=1.0\n",
        "        self.discountRate=discountRate\n",
        "        self.learningRate=learningRate\n",
        "        \n",
        "        tf.reset_default_graph()\n",
        "        self.stateIn=tf.placeholder(tf.int32,shape=[1])\n",
        "        self.actionIn=tf.placeholder(tf.int32,shape=[1])\n",
        "        self.targetIn=tf.placeholder(tf.float32,shape=[1])\n",
        "        \n",
        "        self.state=tf.one_hot(self.stateIn,depth=self.stateSize)\n",
        "        self.action=tf.one_hot(self.actionIn,depth=self.actionSize)\n",
        "        \n",
        "        self.qState=tf.layers.dense(self.state,units=self.actionSize, name='QTable')\n",
        "        self.qAction=tf.reduce_sum(tf.multiply(self.qState,self.action),axis=1)\n",
        "        \n",
        "        self.loss=tf.reduce_sum(tf.square(self.targetIn-self.qAction))\n",
        "        self.optimizer=tf.train.AdamOptimizer(self.learningRate).minimize(self.loss)\n",
        "        \n",
        "        self.sess=tf.Session()\n",
        "        self.sess.run(tf.global_variables_initializer())\n",
        "        \n",
        "    def getAction(self,state):\n",
        "        qState=self.sess.run(self.qState, feed_dict={self.stateIn:[state]})\n",
        "        actionGreedy=np.argmax(qState)\n",
        "        actionRandom=self.env.action_space.sample()\n",
        "        if random.random()<self.explorationRate:\n",
        "            action=actionRandom \n",
        "        else:\n",
        "            action=actionGreedy\n",
        "        return action\n",
        "    \n",
        "    def train(self,state,action,nextState,reward,done):\n",
        "        if done:\n",
        "            qNext=np.zeros([self.actionSize])\n",
        "        else:\n",
        "            qNext=self.sess.run(self.qState, feed_dict={self.stateIn:[nextState]})\n",
        "        qTarget=reward+self.discountRate*np.max(qNext)\n",
        "        \n",
        "        feed={\n",
        "            self.stateIn:[state],\n",
        "            self.actionIn:[action],\n",
        "            self.targetIn:[qTarget]\n",
        "        }\n",
        "        self.sess.run(self.optimizer, feed_dict=feed)\n",
        "        \n",
        "        if done:\n",
        "            self.explorationRate*=0.99\n",
        "        \n",
        "    def __del__(self):\n",
        "        self.sess.close()\n",
        "            \n",
        "agent=Agent(env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pfmbDSM_QiXO",
        "colab_type": "code",
        "outputId": "ff510d53-f2c0-4a71-cd0d-712377be728b",
        "colab": {}
      },
      "source": [
        "totalReward=0\n",
        "for ep in range(100):\n",
        "    state=env.reset()\n",
        "    done=False\n",
        "    while not done:\n",
        "        action=agent.getAction(state)\n",
        "        nextState,reward,done,info=env.step(action)\n",
        "        agent.train(state, action, nextState, reward, done)\n",
        "        \n",
        "        print(f's:{state},a:{action}')\n",
        "        state=nextState\n",
        "        \n",
        "        totalReward+=reward\n",
        "        print(f'Ep:{ep},Goal:{totalReward},Explore:{agent.explorationRate}')\n",
        "        \n",
        "        env.render()\n",
        "        with tf.variable_scope('QTable', reuse=True):\n",
        "            weights=agent.sess.run(tf.get_variable('kernel'))\n",
        "            print(weights)\n",
        "        \n",
        "        time.sleep(0.05)\n",
        "        clear_output(wait=True)\n",
        "        \n",
        "env.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "s:14,a:2\n",
            "Ep:99,Goal:17.0,Explore:0.36603234127322926\n",
            "  (Right)\n",
            "SFFF\n",
            "FHFH\n",
            "FFFH\n",
            "HFF\u001b[41mG\u001b[0m\n",
            "[[ 0.2803828   0.35699728  0.18399435  0.35858876]\n",
            " [ 0.28144774 -0.5374817   0.15907648  0.27208474]\n",
            " [ 0.33027396 -0.10502584  0.16348262  0.36058888]\n",
            " [ 0.11249182 -0.2907312   0.21272534  0.1573511 ]\n",
            " [ 0.31486636  0.38480148 -0.647843    0.37505475]\n",
            " [-0.39050534 -0.31017357 -0.52743286 -0.28745803]\n",
            " [-0.39061326 -0.3525641  -0.2906056   0.3319691 ]\n",
            " [ 0.4671831   0.06217569  0.36608124 -0.10325885]\n",
            " [ 0.3358321  -0.48684606  0.26382816  0.41029578]\n",
            " [ 0.32928905  0.45160905  0.2800546  -0.41874123]\n",
            " [ 0.37605     0.48248896  0.18519376  0.35058293]\n",
            " [ 0.44320905  0.4936825   0.45347178 -0.16480187]\n",
            " [-0.14257866  0.0737763   0.27911562  0.41187555]\n",
            " [-0.5435657   0.4504788   0.33276573  0.46915168]\n",
            " [ 0.39861926  0.4466499   0.36276028  0.44972998]\n",
            " [ 0.21421516  0.16343725  0.08921695 -0.34565684]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GOAND5GwQiXT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}