Framework_Research.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Framework Research",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jonbaer/googlecolab/blob/master/Framework_Research.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMx8VriV50SN",
        "colab_type": "text"
      },
      "source": [
        "# A Comparison of Reinforcement Learning Frameworks\n",
        "\n",
        "This notebook is a companion to the blog post of the same name by Winder Research. Please view the blog post to gain context of what this is.\n",
        "\n",
        "This is presented in a very raw format. It was primarily used to test out each of the frameworks. But if it is useful to you, then by all means feel free share it."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LxMHnPbhag16",
        "colab_type": "text"
      },
      "source": [
        "## Warning\n",
        "\n",
        "This notebook is quite unstable. Each framework needs slightly different requirements and they all interact differently with colab.\n",
        "\n",
        "Generally, if you see an error, a runtime restart usually fixes the problem. The cause is usually because the wrong version of a library is loaded."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "79QxZGNrMBf1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import glob\n",
        "import io\n",
        "import base64\n",
        "from IPython.display import HTML\n",
        "from IPython import display as ipythondisplay\n",
        "\n",
        "def show_vid(path):\n",
        "  video = io.open(path, 'r+b').read()\n",
        "  encoded = base64.b64encode(video)\n",
        "  ipythondisplay.display(HTML(data='''<video alt=\"test\" autoplay \n",
        "              loop controls style=\"height: 400px;\">\n",
        "              <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "           </video>'''.format(encoded.decode('ascii'))))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W8-PdEqsw1_Z",
        "colab_type": "text"
      },
      "source": [
        "# OpenAI Gym"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WXfPkxDbL4Oa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install gym pyvirtualdisplay > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mG1E-p-8L40f",
        "colab_type": "code",
        "outputId": "f3e4387b-d785-41a2-c6f7-e1d50d7e0d1e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "from pyvirtualdisplay import Display\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Display cmd_param=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1400x900x24', ':1017'] cmd=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1400x900x24', ':1017'] oserror=None return_code=None stdout=\"None\" stderr=\"None\" timeout_happened=False>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8uTzw9C1L5IX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gym\n",
        "from gym.wrappers.monitoring.video_recorder import VideoRecorder # Because we want to record a video\n",
        "\n",
        "env = gym.make(\"CartPole-v1\") # Create the cartpole environment\n",
        "env.reset()\n",
        "rec = VideoRecorder(env)      # Create the video recorder\n",
        "rec.capture_frame()           # Capture the starting position\n",
        "while True:\n",
        "    action = env.action_space.sample()                   # Use a random action\n",
        "    observation, reward, done, info = env.step(action)   # to take a single step in the environment\n",
        "    rec.capture_frame()                                  # and record\n",
        "    if done:\n",
        "           break                                         # If the pole has fallen, quit.\n",
        "rec.close()  # Close the recording\n",
        "env.close()  # Close the environment"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a4vKeZrXwsYI",
        "colab_type": "code",
        "outputId": "3991da93-4fce-4c56-8504-71ee35ce16a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        "show_vid(rec.path)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAACkxtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAACNGWIhAAv//72rvzLK0cLlS4dWXuzUfLoSXL9iDB9aAAAAwAAAwAAJuKiZ0WFMeJsgAAALmAIWElDyDzETFWKgSvGXweETYA52T1wF6UkO2lA/DdQcbjfswJ8/+e1YGF3/cCoE++Fu0WLo4lxPNPqQR8hwsLZHAlnZ5uEN0mCBP+tAZFj6oZ5b7WsMvPtQSnbXnKUhPnxi3yDyZaQUlIKpFCodsZw0f/m7RDSa6r8UW75drlxXkuNw7BBQVz6oH3jNGS8ms72QjF+USwnmIo+qOb6lid/bQk1NI+tERBACQjz7+N4Je5quypjzyylyN3gLn5jiMjGo7xyeDMEoAagClpgK+mVGHlKun1objB+fBXTH+0glHsglMBQBHK54vACpnFQYtiVOduWUjY2A6oi4v/P/7z8LW8XIM/k2urAmMBK2AADXvSUnzScje0Kn+8NjkRrUPPRsZdbV+ELunr3onGUNYtKq8IrcUDpcqOxPZHM9U9G4Bi/zvG542ZPFxVBqbk1us0QACUhprFT6X+9rgXJso/mcDx5g6xHsfNJfNu987qqAHGl0FTJrmu50PmUsVdIYhKQFD1714W49y881R2lRTHwOOGUuxGfr8ZF0YJKX/y0CORySoLqhgxjFcZFNHTGQgRpLjlbVT7J7jYAACre+6tzWC2rTnBLkLKZyoX60XPKl5oyjSnPRPTvBLRSdgx/6pCrczp+uc0CLKWFsHW52WUBd4FpvmjCAAADAAADAACRgQAAALtBmiRsQv/+jLAAABqdndCL0iguiH0ey0yPQ0kfaSfco4XgktnqbYPGnkVUAJrVejJ4wB0vLhHYSYpgtCASR3B8Dgz+HYd+aWvxzbMiIxzXChbdYVj52+Bp7E3IgKZ/ei7gLOi7Njl/HJEeGqshqx7GoyzW8FEQLxuRJrPYJuAz132/3qftjKVO90VL+vDNkSinHnyEI9C/d7Qku4LcZojjHNyNWuRiR8oaZnwAAVnS1rlIq1/KqI9QTZIcAAAARUGeQniEfwAAFqrAHMbTz4QHHNJPz0ofy/Y4rWxQKdas4QDxw8FAkZEW3Gv3fY5i6+lmbKkpcOlN0AAAAwAZ/q5cvoQDpwAAADQBnmF0R/8AACPDF3cVeV9XB0fc4Dtixo8QWuPDzkmSIpsKAbpkOWIAAAMAAKZg7BIGoBAwAAAAQwGeY2pH/wAAIthcwBWCGBDSWiRuB7uhWhQF9B0/I1Z3Y8yayndSrn9jLGRR45VoQVZEe2Hdj8aq4AAAB3Xtk7MIB00AAACnQZpoSahBaJlMCFf//jhAAAEN+agQmqQA3UvZ2P2zgC9UX9dNjvNpu0QD8ta2H6+9rrCfW/eC6VZ0n6TYA3Fj2e/VRziOnVTTy+QE7lBN7gvsKemXRd20Ih9DAxzT0AghIKPaBB00VJekrxB0NUmbOmzecjKNkkmWXtg35LprFVcm9JylsZuUI5q8pfEcZXFfLGH7oOne7mDd+2bC1Ds+ukVR0+0oUb8AAABMQZ6GRREsI/8AABa8Q8yCEyAc2/RbfWP4VBwd/W7GVhA/2q8AaU/4pHFc9gJdsAINhH5mdOZQpU9eIjiJUhTpQ8viE0Q8PgEr+8ZouQAAADYBnqV0R/8AAA1/R4shFDLQaLX9XEc5B6hWn2LcC0Aa1s8E5zfAATJn4+SzUSsW0iZn560Qm4EAAABDAZ6nakf/AAAjkieKDNu7kyflfsM3ItFEjGEYhsTaGzGTqjfN5uB6xpP1NZ7SiDqhNpPKWDDma1Uwh13TDWFQ7LQm4AAAAOFBmqxJqEFsmUwI//yEAAAP3XJafaEAOgKQ8HBntMEtqie6Z6DrKmM5KeWEFn7QN92ooN4CS/vdZb3BA3fzraeahMW2p285enKNE6ei1yby4aaUEWSx7lvT7JBOJdgc/1t81Rw5LOcqZIk37RGvExy5xm4CsWIwUGVcrcGYMrp27s23vl9PFrDOlQWJ91VBKdUhQAV7pZ3AAw/RDPIGBbLil0DCNBZAFDvJ3AccjAxTUqWtJMKk9cO16i817GoJSTYFS3k90+VZEkqRMe46VQA4KlbBLcrgtV+YZGtSRPW2vDAAAAC7QZ7KRRUsI/8AABa7XqpJQAi/b7L0uuNnzbfQXC8mEsd4NVanVY4lMFs2d4rl05KcF3SQ9xGXstaggyCHrtoWVpLI+NduvoaNq8hWWUzRx303AlySZmynYfUFqEHvots0W049I+JE6DjOBWEcb16mF0ZXb7l5zhGyuHWBQVRSgFJzRI3TTL/8o3uyenbjDZhIk1/Y6NMbAphuGlExAmpVYbqsToPK3cTejoaGW6ohHkbkI34l4X9Vu5WW9QAAAFIBnul0R/8AACTC3GranNB8qTKFVDHJVGkurr0Pneh4oGVYO5k/lHQJpfuBCfeFpIfoOw0G7LWf2xsaVZ1mvvSFTXEqQIOmUyokodwH9evlbutgAAAAWQGe62pH/wAAJLGcmXoRgjYcoxdeAAT15miTHL7rkH3uIRwh/RYxCY8rd93dWiFfN6t+jj8Hja8K6c2ldHf/XgKgA+RmdItfTXkM5NhTlPEBDbPLFMuVAHpAAAADr21vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAAEEAAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAALZdHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAAEEAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAJYAAABkAAAAAAAJGVkdHMAAAAcZWxzdAAAAAAAAAABAAABBAAAAgAAAQAAAAACUW1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAMgAAAA0AVcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAAAfxtaW5mAAAAFHZtaGQAAAABAAAAAAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAAG8c3RibAAAAJhzdHNkAAAAAAAAAAEAAACIYXZjMQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAJYAZAASAAAAEgAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABj//wAAADJhdmNDAWQAH//hABlnZAAfrNlAmDPl4QAAAwABAAADAGQPGDGWAQAGaOvjyyLAAAAAGHN0dHMAAAAAAAAAAQAAAA0AAAEAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAAB4Y3R0cwAAAAAAAAANAAAAAQAAAgAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAAcc3RzYwAAAAAAAAABAAAAAQAAAA0AAAABAAAASHN0c3oAAAAAAAAAAAAAAA0AAATqAAAAvwAAAEkAAAA4AAAARwAAAKsAAABQAAAAOgAAAEcAAADlAAAAvwAAAFYAAABdAAAAFHN0Y28AAAAAAAAAAQAAADAAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGlyYXBwbAAAAAAAAAAAAAAAAC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjU3LjgzLjEwMA==\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zmEGVt9Hwyxs",
        "colab_type": "code",
        "outputId": "6febfefc-ec53-4bdc-90dd-d4e747d147a7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        }
      },
      "source": [
        "!pip install --upgrade --no-cache-dir dopamine-rl\n",
        "!pip install gin-config"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting dopamine-rl\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2e/1c/99be0c9325dad3ce11ec6088408ebdb5bc7194791fd40df8b6be28a23973/dopamine_rl-2.0.5.tar.gz (54kB)\n",
            "\r\u001b[K     |██████                          | 10kB 13.9MB/s eta 0:00:01\r\u001b[K     |████████████                    | 20kB 3.0MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 30kB 4.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 40kB 2.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 51kB 3.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 61kB 4.3MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: gin-config>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (0.1.4)\n",
            "Requirement already satisfied, skipping upgrade: absl-py>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (0.7.1)\n",
            "Requirement already satisfied, skipping upgrade: opencv-python>=3.4.1.15 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (3.4.5.20)\n",
            "Requirement already satisfied, skipping upgrade: gym>=0.10.5 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (0.10.11)\n",
            "Requirement already satisfied, skipping upgrade: enum34>=1.1.6 in /usr/local/lib/python3.6/dist-packages (from gin-config>=0.1.1->dopamine-rl) (1.1.6)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from gin-config>=0.1.1->dopamine-rl) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from opencv-python>=3.4.1.15->dopamine-rl) (1.16.4)\n",
            "Requirement already satisfied, skipping upgrade: requests>=2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.10.5->dopamine-rl) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from gym>=0.10.5->dopamine-rl) (1.3.0)\n",
            "Requirement already satisfied, skipping upgrade: pyglet>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.10.5->dopamine-rl) (1.3.2)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (2019.3.9)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym>=0.10.5->dopamine-rl) (0.16.0)\n",
            "Building wheels for collected packages: dopamine-rl\n",
            "  Building wheel for dopamine-rl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-67cj6qhy/wheels/75/01/fa/43ebcfe0f37d8c27bb1b7b6fa213caa0340e783fe754a2af8f\n",
            "Successfully built dopamine-rl\n",
            "Installing collected packages: dopamine-rl\n",
            "  Found existing installation: dopamine-rl 1.0.5\n",
            "    Uninstalling dopamine-rl-1.0.5:\n",
            "      Successfully uninstalled dopamine-rl-1.0.5\n",
            "Successfully installed dopamine-rl-2.0.5\n",
            "Requirement already satisfied: gin-config in /usr/local/lib/python3.6/dist-packages (0.1.4)\n",
            "Requirement already satisfied: enum34>=1.1.6 in /usr/local/lib/python3.6/dist-packages (from gin-config) (1.1.6)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from gin-config) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tKqeB5QuvHTL",
        "colab_type": "code",
        "outputId": "7b117547-a2f6-495c-c18a-1803833d0052",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 360
        }
      },
      "source": [
        "# @title Necessary imports and globals.\n",
        "\n",
        "import numpy as np\n",
        "import os\n",
        "from dopamine.discrete_domains import run_experiment\n",
        "from dopamine.colab import utils as colab_utils\n",
        "from absl import flags\n",
        "import gin.tf\n",
        "import tensorflow as tf\n",
        "\n",
        "BASE_PATH = '/tmp/colab_dopamine_run'  # @param\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0625 16:10:35.975975 140516433803136 lazy_loader.py:50] \n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "W0625 16:10:35.984353 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/utils.py:34: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n",
            "W0625 16:10:35.986600 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/utils.py:34: The name tf.gfile.Exists is deprecated. Please use tf.io.gfile.exists instead.\n",
            "\n",
            "W0625 16:10:35.991888 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/utils.py:43: The name tf.train.SessionRunHook is deprecated. Please use tf.estimator.SessionRunHook instead.\n",
            "\n",
            "W0625 16:10:35.995024 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/agents/dqn/dqn_agent.py:98: The name tf.train.RMSPropOptimizer is deprecated. Please use tf.compat.v1.train.RMSPropOptimizer instead.\n",
            "\n",
            "W0625 16:10:36.000683 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/agents/rainbow/rainbow_agent.py:77: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N7sYGfq_vPEk",
        "colab_type": "code",
        "outputId": "9013941a-4654-473e-fc14-16da0ccc6889",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "# @title Load the configuration for DQN.\n",
        "\n",
        "DQN_PATH = os.path.join(BASE_PATH, 'dqn')\n",
        "# Modified from dopamine/agents/dqn/config/dqn_cartpole.gin\n",
        "dqn_config = \"\"\"\n",
        "# Hyperparameters for a simple DQN-style Cartpole agent. The hyperparameters\n",
        "# chosen achieve reasonable performance.\n",
        "import dopamine.discrete_domains.gym_lib\n",
        "import dopamine.discrete_domains.run_experiment\n",
        "import dopamine.agents.dqn.dqn_agent\n",
        "import dopamine.replay_memory.circular_replay_buffer\n",
        "import gin.tf.external_configurables\n",
        "\n",
        "DQNAgent.observation_shape = %gym_lib.CARTPOLE_OBSERVATION_SHAPE\n",
        "DQNAgent.observation_dtype = %gym_lib.CARTPOLE_OBSERVATION_DTYPE\n",
        "DQNAgent.stack_size = %gym_lib.CARTPOLE_STACK_SIZE\n",
        "DQNAgent.network = @gym_lib.cartpole_dqn_network\n",
        "DQNAgent.gamma = 0.99\n",
        "DQNAgent.update_horizon = 1\n",
        "DQNAgent.min_replay_history = 500\n",
        "DQNAgent.update_period = 4\n",
        "DQNAgent.target_update_period = 100\n",
        "DQNAgent.epsilon_fn = @dqn_agent.identity_epsilon\n",
        "DQNAgent.tf_device = '/gpu:0'  # use '/cpu:*' for non-GPU version\n",
        "DQNAgent.optimizer = @tf.train.AdamOptimizer()\n",
        "\n",
        "tf.train.AdamOptimizer.learning_rate = 0.001\n",
        "tf.train.AdamOptimizer.epsilon = 0.0003125\n",
        "\n",
        "create_gym_environment.environment_name = 'CartPole'\n",
        "create_gym_environment.version = 'v0'\n",
        "create_agent.agent_name = 'dqn'\n",
        "TrainRunner.create_environment_fn = @gym_lib.create_gym_environment\n",
        "Runner.num_iterations = 100\n",
        "Runner.training_steps = 100\n",
        "Runner.evaluation_steps = 100\n",
        "Runner.max_steps_per_episode = 200  # Default max episode length.\n",
        "\n",
        "WrappedReplayBuffer.replay_capacity = 50000\n",
        "WrappedReplayBuffer.batch_size = 128\n",
        "\"\"\"\n",
        "gin.parse_config(dqn_config, skip_unknown=False)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0625 16:10:36.035696 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:32: The name tf.train.exponential_decay is deprecated. Please use tf.compat.v1.train.exponential_decay instead.\n",
            "\n",
            "W0625 16:10:36.037590 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:33: The name tf.train.inverse_time_decay is deprecated. Please use tf.compat.v1.train.inverse_time_decay instead.\n",
            "\n",
            "W0625 16:10:36.039106 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:48: The name tf.losses.absolute_difference is deprecated. Please use tf.compat.v1.losses.absolute_difference instead.\n",
            "\n",
            "W0625 16:10:36.041050 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:50: The name tf.losses.hinge_loss is deprecated. Please use tf.compat.v1.losses.hinge_loss instead.\n",
            "\n",
            "W0625 16:10:36.043989 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:51: The name tf.losses.huber_loss is deprecated. Please use tf.compat.v1.losses.huber_loss instead.\n",
            "\n",
            "W0625 16:10:36.045200 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:52: The name tf.losses.log_loss is deprecated. Please use tf.compat.v1.losses.log_loss instead.\n",
            "\n",
            "W0625 16:10:36.046976 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/gin/tf/external_configurables.py:54: The name tf.losses.mean_pairwise_squared_error is deprecated. Please use tf.compat.v1.losses.mean_pairwise_squared_error instead.\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KTPXgnXsvYo9",
        "colab_type": "code",
        "outputId": "80452ff6-4e8f-4fc1-fa0a-d293cda6a4a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 785
        }
      },
      "source": [
        "# @title Train DQN on Cartpole\n",
        "tf.reset_default_graph()\n",
        "dqn_runner = run_experiment.create_runner(DQN_PATH, schedule='continuous_train')\n",
        "print('Will train DQN agent, please be patient, may be a while...')\n",
        "dqn_runner.run_experiment()\n",
        "print('Done training!')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0625 16:10:36.073584 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/run_experiment.py:506: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
            "\n",
            "W0625 16:10:36.077062 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/logger.py:48: The name tf.gfile.MakeDirs is deprecated. Please use tf.io.gfile.makedirs instead.\n",
            "\n",
            "W0625 16:10:36.079235 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/run_experiment.py:183: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
            "\n",
            "W0625 16:10:36.084795 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/run_experiment.py:186: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "W0625 16:10:36.086268 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/run_experiment.py:191: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "W0625 16:10:36.122981 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/agents/dqn/dqn_agent.py:193: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "W0625 16:10:36.131702 140516433803136 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/dopamine/replay_memory/circular_replay_buffer.py:790: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "tf.py_func is deprecated in TF V2. Instead, there are two\n",
            "    options available in V2.\n",
            "    - tf.py_function takes a python function which manipulates tf eager\n",
            "    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to\n",
            "    an ndarray (just call tensor.numpy()) but having access to eager tensors\n",
            "    means `tf.py_function`s can use accelerators such as GPUs as well as\n",
            "    being differentiable using a gradient tape.\n",
            "    - tf.numpy_function maintains the semantics of the deprecated tf.py_func\n",
            "    (it is not differentiable, and manipulates numpy arrays). It drops the\n",
            "    stateful argument making all functions stateful.\n",
            "    \n",
            "W0625 16:10:36.167903 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/agents/dqn/dqn_agent.py:247: The name tf.make_template is deprecated. Please use tf.compat.v1.make_template instead.\n",
            "\n",
            "W0625 16:10:36.193719 140516433803136 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/layers/python/layers/layers.py:1634: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.flatten instead.\n",
            "W0625 16:10:37.203975 140516433803136 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "W0625 16:10:37.399099 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/agents/dqn/dqn_agent.py:325: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
            "\n",
            "W0625 16:10:37.711280 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/checkpointer.py:78: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Will train DQN agent, please be patient, may be a while...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0625 16:10:37.905010 140516433803136 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/dopamine/discrete_domains/logger.py:89: The name tf.gfile.GFile is deprecated. Please use tf.io.gfile.GFile instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0625 16:10:39.675085 140516433803136 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:960: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use standard file APIs to delete files with this prefix.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Done training!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XJScim0svpia",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6x_dR7UC8UNU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gym\n",
        "from gym.wrappers.monitoring.video_recorder import VideoRecorder # Because we want to record a video\n",
        "\n",
        "rec = VideoRecorder(dqn_runner._environment.environment)\n",
        "action = dqn_runner._initialize_episode()\n",
        "rec.capture_frame()\n",
        "while True:\n",
        "    observation, reward, is_terminal = dqn_runner._run_one_step(action)\n",
        "    rec.capture_frame()\n",
        "    if is_terminal:\n",
        "      break                                         # If the pole has fallen, quit.\n",
        "    else:\n",
        "      action = dqn_runner._agent.step(reward, observation)\n",
        "dqn_runner._end_episode(reward)\n",
        "rec.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDA9aL6e9UoE",
        "colab_type": "code",
        "outputId": "966b8cfe-41ce-4901-8bdc-1d78e21d36de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        "show_vid(rec.path)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAALhdtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAB+WWIhAAz//727L4FNf2f0JcRLMXaSnA+KqSAgHc0wAAAAwAAAwAAFgn0I7DkqgN3QAAAHGAFBCwCPCVC2EhH2OkN/we4nsgCsgSuJlL4UfXW7KGLTCCZdGj+2achfhRkCP+JmHoCAffTMo+a3XqqRD4TMoZnFUjvKYzsz9UqbuiwCXI7iK+hlU2Q8wLSJ3c7lr3MiHY3jK7O26vljH7Mq52LU9jC/8gB1numWn8TRKjpXC49p73KL3hPIrL5XMLFC1q3Cfna6vreAEN6V1EGKi/1XPM3HHWkzXHnG3mWMzaYB3rh2puxb29JnDN5XIBie+BCC1gbn02d3LYBuuDJiVKY6OG5lLEl1JyibenhJjg5l1iXoAK5kIIJigWn26/AFFT3MAWPaB9Ib3y62X+MwcbDFZuIq7U/p1FBi7FMzG5OKa2di1MUuWjJA2Q5R2TCi6miLqvYObaTCTAsQDyQYTm1lKcVqq4v8gunNg+6gAvwVeMz7nNOhQPbqxJ9Tmtv084fQJQNIWDnrtrt7h9Js52zlcA98NR2/ARpehBAL3TFuS6gThs0tP+Gw0PLBcG1WZqEi9LNmNH/ug4AYl8IvIauIlzM3wa2Ipsxt6AV7mB2NiU0jRWEgBNUXNZRMZKxUzW5/0lq2J7ZFq64S5PJwAAAAwAAAwAAHFEAAADDQZokbEM//p4QAABFSbhcgCJDRkqMWwIKuKAYhDWV3fk5HOIf2T4vAFmWrgM2DBEp7V8SMn+ympv472i99EeKYy2cYGa4CN2VMg/gr+WPO8SNNfkkAIZwn268TjdhsGXzQCwjRVewIPptwinrci/gzl1sbL5Q8vzPiWNE8GdEjRLZRKGcqxKLavwOM0buzCG8EYjgESwq/HTRRdcntwEoNPYlA9zbT2/0i7Fd5/3YAADeumfmp+mpT9UrTEJh7fguAHFwAAAATEGeQniEfwAAFmwQsGPAFbUN/PDyN9Wf8jXCeVoZFea9xzF5i0QfZqY/R5Qi5LYXYd5Fhz+ZkeSGqoAWzGHtS+AgAAADABQ2LllgPyEAAAA9AZ5hdEf/AAAjwxd63D63u79/CvVcIe/+TLzDNjxfRICunRtyehcd3sMsM0rXwfQAAAMAAAMADiseaoB8wAAAAFQBnmNqR/8AACO++KtPfYFj2jkdfEw0kly0FyAgnC6koApEGzsclgAQ+5p7CbhPH1sWEtoymTwSeU0RmWCrAfGb/tNYakJ4lM1QwAAARYatUgywHzEAAADBQZpoSahBaJlMCGf//p4QAABFe4yHovP7ABF9Mx0lLP8hUdXF+Lp8vm+SAeoGbV6O1vOBET7XIMhHxhjWDdMPLmhz6he5tID7mBR0RxQR1eJaSWGtV+MWAdNPmBOIZ32E9tVXlPvIEdFj6U4LlbZILuUlfatBAy7pVn85i8jBLKXXIxxTyyUOoj9Fiqh0WOnTp9KnvPsrnuu//z/1r6E7WI8nukdOM10AX+26BMrPVexqNuNAzGERhZRw9IK44ZPb+QAAAEdBnoZFESwj/wAAFqS9y1Fvuvml00mWy69crKKbCEgvAlb5LyVABp7WWLXE9K+qOVE6Wu8PDhbCGBXREH8MhWsEkCzZKlkwIQAAAEcBnqV0R/8AACPXFQAcK3RfOV0/rnlHRovvW53ouGEnUMch8/lLO12f4uIadbNtrFzZqeNly5dbw345qAZdXfa7zZBKg5U7MQAAAFcBnqdqR/8AACNjm5RGCnsORRIfj7BImGz1zALoklo3pMl89Dqu+JSxm5g7AJRX0x1U0reAAQdTo1b8e0iaVFW8TW+vJOGL1/1gPuroxCr1IH7JWsAdY0YAAABjQZqsSahBbJlMCF///oywAABGJDR9pBPctHRx2Pf4udjSBu8C6ZGV6m6o13/+LBl/FKPHeQyfVkmyDu51I2cuBUNtWuvrCUNSAhATCqtxxTERE8OVLy/kLOPNTiwqkyTSrZ/QAAAAY0GeykUVLCP/AAAWu4rejYMm925uTyjRvaNAhgHv5DCNZf/pVqPa85LGPWTvV+eHzGC/fdyro6mDQKqzAAHvxu90uNHslv/QlDEwjyv88wx2SgCWXDUDQcy6Yv2KXYrvYHBl4QAAADIBnul0R/8AACOYXdsKZMInDRWhW0USZLEiTce8nRTIgTdD4eHdZr7s6HTssBExdleG4AAAADkBnutqR/8AAA2HXXYAXBYQk53zBaeRUZp8Gf0dsgC4ckgQq7TT/B4ZUr1JV3CYn4mzqN6TDeDh5NoAAAC4QZrwSahBbJlMCF///oywAABIAs/wuAM4wBW/UMe5OwTaSPzC9z2Gk9WdA+l5S4wcWvI2qf6kLpbFP164iA2hqnegbSnhPPgO7iMVGQdaHXJ5Pti8xcQyDhKFl7C6EiP+SxTkwaGWSvfP6z+pPDZxdmHT4p9jm49qNTwWMr+VorarXSEUmSORtOnPrdTjywT0kYGsxfkYWfJ7hvbcVW9GzmNfQgvBe1Z/mmK0+3ZSxyZ/QuKx2UEOqQAAAFJBnw5FFSwj/wAAF0uKSZoRv0FFYmV354NvCjk6hAbCApWryXlIjfBOzncZej1+pRLW8AlCGewRsAITEpduemWdXGZzNPWkyanh5KEzfqBeQ8+BAAAAPgGfLXRH/wAAJNUhAFAn9d5sf4hoBe2EkzrhET0Knfw8xCb+K0hX+cMYcZGvbI5ka+bCrDvAUz4LaJsTrW3BAAAAQgGfL2pH/wAAI7I8mwTLSfmaPlKj7Fyc43wmv77bftvlxxzVToAoTS30hwAE1XhPfedoqkGuFHy4H20hlBjSDQV3ZgAAAGVBmzNJqEFsmUwIX//+jLAAAEgUW0UtujADo/UXuz069rASNps1ZX+idos1kFCaTKzSUXqA30hRk4i1vSz55bFVyGzxwkqOl9OYn983SsrcZL4Lx//607Dafwln8qGkjVK5qcVVHQAAAEdBn1FFFSwj/wAAF1BWhLQw9pi9KvkBC5LAM03zaWZqemx/yWbLceLF0CNAcanc0ARpa8nN+ZQwieYg8KaoUtl0fFPk9PPJsQAAADwBn3JqR/8AAA3Ptot+1yx/eODtmlYfSRgloQABbF8ahmAKgcB9JFCDpEEXNdc4bWnLMK5Q9jvNxcKXAIAAAABsQZt3SahBbJlMCF///oywAABIEHsxsawQBvvVL0H82saI4oufQNBSlw8xb+QjhvPBCWUetFhl+O3Eq7/DG3NaY4l/VFcOqxyxlf3jSzY95dl79Pz9SR3QleUVG/1zwAk7luIxmanZ6J3uWHuwAAAAOUGflUUVLCP/AAAXQqwewIQAIvYEzW+D4ba6uti+4sMJ5f2XQeJ6zGk0kAPTDASUF9ddiTB65IyakQAAAC4Bn7R0R/8AACSsCNPVQBInr9o7VP0J/rBNvugteCwyPSm6zBeLCeVy2Ep36EWAAAAALQGftmpH/wAAJK5e0LQ+cgWQ7o2c9OAGkXX3BABcMITHb3qpsYgtsuRGB3lTaQAAAMVBm7tJqEFsmUwIX//+jLAAAEgw4IS0EbuAK1P4v024///Ba6zybb8M4GBHjef1Vn2wdPfLSNB6D0LM4/dJi0e+niQZcMFE3/9K7izevLR2V65hoLP4mf9IWkWb4gvNkAePej6v9gGQ5/ngs74B62UeIlnSLUqdfYKKOESpe48GptREEbUpJDCv5znG7peZW2dyE7t3KnP5b5DhetBnD5oYILjT0ufuC6C0Bb/5FcaJUjbgF1+vaptuKROv1tmqHBR7kf3PnwAAADxBn9lFFSwj/wAAF0ID6NMPOxmayIlW4v2dnSzcEM47oeVWMq9qCIleXz4nwo9HBwmBe0oNOzdjXd2u9MAAAAAtAZ/4dEf/AAAkqepOOUfHqD+2+6BByA+zhcec18o7Jio9KfpCQOOKgKWYWiLBAAAAQAGf+mpH/wAAJL1lPq00qz0+qoH/BvkStE5aERISSQQG0zstzDXd4WraG96ABCml3BgyEx4mq3kg1oK1OsNmybAAAAB7QZv8SahBbJlMCGf//p4QAABHRQXgm3lJwQfG4ipZACatVJSV1mPM4kc/ntESG/prGyZCxp7Z3p00Ia2ns+nH+KGWyzbOlBU1utu0yurmD/SoWpO+UcVW7FboU8wLBQMudxnnlmkVkQxW7PUNHrNgcNBNxmcFnfjOe3vbAAAAekGaAEnhClJlMCGf/p4QAABJQ+n6bty69gAjA1qBKl/R9BKkniS2BZB8W9Mge/D8b5585JklUsS8Z21VE/9j7cCaqsCFS00WQFLJ5pqRlvN81CKylOkxY8D+UyQOwExLoUlpmXUukokiehcjPZyjGgUlP7fTCzFFnvFxAAAAUUGePkU0TCP/AAAX6XkeG3EndULkmgbRyfHbsbT5eGRMKUgiM9nsXGRB71Bx8fs8QwQWwEgY4rw+AEydNb+01aXZiWLlqh5dsKuGOulEeu/pgAAAADUBnl10R/8AACQ4M3fKjdeBRC2C02ZyEDoxaOdYdIF5ksDzet+9zZwnQTyaYMmtCmzCGi3NSAAAAC0Bnl9qR/8AACW/AmSMo84d3BLlicK4yh3INOsx12sfKFoeLtZpUIj+7OrH/MEAAABzQZpESahBaJlMCGf//p4QAABJQ+ib4AIyAffB2yWInjTA9ZpkC8OWdjzsLP4av3lOSqHBiCBCqsG/OlgH09mQytJCedVkNY5wK56kDDSrCstQ1HfBxrQP5EJR2ZFa2u0l3W9EvfZ5o2wP7kLSO4AfbS4TjgAAAFBBnmJFESwj/wAAF+ibLpWSf5FHpyokW1lpUfM7fNBELBLlYFllCtZyr4qv/KDYeZAeHnxm9JiPZH3F+a+AD79xNaxkbMZJ7zTkplvf/tQN6QAAADkBnoF0R/8AACXAmd3nokiPRfjRhTm1cnJitPJfs4IFzr7Fn6x8QbxN8Zmw/D+5aONSiRd4RP/+toAAAAA7AZ6Dakf/AAAlrl7MxcKtu7DCurzYgQ6h6hn8jE1YIW2gbAwyXOLmoebnv49cJ1jj1rq7c/jAUbs1tSEAAACQQZqISahBbJlMCGf//p4QAABJe5CmUzfqrAA41CIQanetbRNI4BOZYSr7lLUz1V7B4rv4AEO4F6d9Gl6Voaz4AAm2TajUuV6JquvjnKmLzg/OR0Opx+/xMgEi6KaYgYPY8w78rZzhhiGTUGnQC8uw4cv/5r6hEiCVwAmVjj9UzG49Lkw5299mEkJMdLhHqSJ5AAAAVEGepkUVLCP/AAAX13pSmiKcEQBdm8M917F1FMXz+D5P8YIY5tvmFyRI85xqfrMaPGn73eSScY6W9NBjR2I8uAEy2Ni+AbPq2HhYNzjrTZzEXaWFgQAAAD8BnsV0R/8AACXAmd3phF6y8NwtkIVTdrZ4YZ2yZysTCPDgycCCI2Os0QTffYF7a3ZuVPygCkFLJkprLDe6tw8AAABGAZ7Hakf/AAAlrVvDiSoOiLKXsymcVrjaAThH8lO99T0tOAcrSdc1RHvf90N9bOrFkcRKv8Iq4swQAmTHjl82FQvz8n+u4AAAAIpBmsxJqEFsmUwIX//+jLAAAEwBiXXVgEIWd9D4rvBbuy0Vivq4r76FNBVGxnC4J9NhSDafRCEYHmztecxiwCpAXjh6xVFQPUR7mRWt4ec/DTOi0klejtjJw7xuixyvgBXvOZxLXY+lH+bFesW8Rd/w1CUzSjC/jDmqHrQCauekdgW2PDH9oiczeFAAAABOQZ7qRRUsI/8AABiJhs0JL49kJ2THge63WHimcQiHtxJlTyrahmF0ayrYjhfwXOC5d9m+WJJnvp2MKi6Vlen53SsAAmWz3AftIfvsaQdPAAAAPgGfCXRH/wAAJam5hP3AOALgS5ZZSXMDyXWWOtNy74+WIxYv1d5D3alQLpcs5/RtSLGlYjc93VBjKCthxVbgAAAAOAGfC2pH/wAAJr7mr7XX6ZHEQjRoC3opFGAUv1OqV6YGvQ0HceXmJzZaSv74nQQUF72BNdTJufgQAAAAY0GbD0moQWyZTAhn//6eEAAAS3MtSvNGegE6ttAVAgMAPCAKjHifPjkaSBHncO2nXGY5hPJkoXQRiL+7aUSAjV3jA+LYz4zaxSK9EsF5HmaC6VVr/Pfw2gHJBI3VjnAgw4n/QQAAAEFBny1FFSwj/wAAGIibTfUUr88G7/1j2U/ZpZWgEem7Tda8ACpASqf0Sn+7eQ1Sp1peZxyg3Pxinu/MKgcs+829nwAAAFcBn05qR/8AACa9ZT2dC64eNuM27OC1R/fAg0RYYe4skoDEJYa/X+cKpMOCaaGCqBCy3xyOPUUAH83W4eQDtxBGD7KdOslBq4hTmZVnaeNoxaMc4TAiB58AAABqQZtTSahBbJlMCGf//p4QAABLekZMd3zqYIiBO0c5ZoD7lvUOIDAkqCRA3QAk2HBy9+RtytssNwAtBF1hCFtExang4jhhLph1QxPcbFiXj90387ljNtr/aIXbHWjXPBP+llW+yGBwngmGgAAAAFhBn3FFFSwj/wAAGIh0ZxVYYuD19VSwx2wosbBLWHPaBzskOF3gQoMPNWYPiCoJJnsnWm9O205oAFpAftCfj/JO2S0hplHw1fT52xdd/sd5OUofQucmY13AAAAAOQGfkHRH/wAAJr/vbp1ArHBSUrIVysfhzf4cdOfm91yu4w00XiDK5E/3Iq2wAPv+k/ntbwuay0tumQAAAEMBn5JqR/8AACa9JvGokYYOwwJaFPa68xcvnZYsW+RvEYISUMdnJ4nD/wqDv/gLwCRwVexJrMuBR7nyX5ct3MkofVTAAAAAeEGbl0moQWyZTAhf//6MsAAATiTKBJlnazWeGhdFdnNyz3lFRfVwsqNTA78CR7qv/N6ACTQgty29s+yz6zmmiJqI9gX2LB8iW4HW0xCY5jzQQtqe8g4FXxaVB+YDqlPVZrflH5FtEBCdybH1FWlUOAmZlw6+1IMNlwAAAGtBn7VFFSwj/wAAGR91HMBJIAR5g2C4nGZVKKVV8wlPwynIKRaxUgdpgQdzOdM72WhMwzHFqXq0yFB6riad1rytn1Both55QWRWqHACJl8PNc64zU1/SgXYS22jIlEapK1BbumvVMMv1NiEjQAAAE0Bn9R0R/8AACe+vBK8D5A6pzCJI1uhGkYdykLbjgsDgkkrMAAtgwkXksXiJSWHglo9qyEkXXBO54ea/RxzhVAiK9wgKMSb4RNxtFVJwAAAAFABn9ZqR/8AACfXMxqQIY+VlfdUsyHsRQ3e6YYKvvbHNaadT9Vy8VFCv1GEwxlWxS4i0JEHamCYk0RCBQAHkD/0f2b4fmwTfD821d8xO7avgQAAAG9Bm9tJqEFsmUwIX//+jLAAAFAqco+MTdbidj+gvQ/+rwqFOrV3oILKRcjjkcu2JSE4r9yzCUrSDeiDvHIAJADYyg6QrTNH/AlUN0IVfA6FdRX/L2llp/tO/X9OVly/Nd9roZFgIHPfi+DPN34taoEAAABlQZ/5RRUsI/8AABnJfHyyYLf9ql/vJ5PzIZa5fxg8twFl7yU3B6wRDBEkWUbpyKc8ADbTGZ559uUemODLI9VPdrycwvYaiHYmL47DxzywfYv1W+GmrYuxq2AKiUmfKd7ZjqblWVAAAAA5AZ4YdEf/AAAnvqsffy9U2PtY32yd6gqKmlJ5t5tuHyOQbuP9DIgMm2Ea5yLUUukCuMotS7lXK8XBAAAASwGeGmpH/wAAKOJm3MA1P9o4LruFyL+X2gEGgPGWLy5nHx4GAbXuMDHum+vJ4SOVzPmqSdHUp+AA1+xXHBYIPvF7YiJJEqDx+SF8fAAAAIBBmh5JqEFsmUwIX//+jLAAAFBCxVF5IARirIMTzTVQ+CYYOAaCj25W0Q0CCn4JaKh7U7C+IDmuAfUWTch83ftLGzzNRB+iEMJmGit8aIWyLFmH2DiaAGK+HBum2mphVu5rogOs0/8U45fLbVW1/tQcNW9hA7UaCZBObCRvKSiA+QAAAEtBnjxFFSwj/wAAGcumuK49eG9PlLfBa8TfH5Zbw/M0f3mW6E/baUMkEHMXsmpNaSJnONusJ8vkHP0K0VUvyvIdtdHBBXORZ2Z+kbEAAABIAZ5dakf/AAAoznf/c9nj2WOgT75KVH6M1BMA0cV2Q/HWNsGM889M5zoeiAMAaEsKjMBNLcD9JO6+WoAP7vvT9w2jzUyjk13AAAAAh0GaQkmoQWyZTAhX//44QAABPecGyU6h7OZtXdzruB7qOKc3BRNFUqKWl3Al+gISajx/yGNpkADWmrL2rjaLo62hOyZ+Btz7pFF4GX8vLBwUPgGf3W0Xk0Lqf9bwFF81hkNFxdvXdZWjD1XIcq/2+0QBc0UWR6G099Hu+XOGxGGwjF/+3lt1wAAAAEVBnmBFFSwj/wAAGl+SDWWottsWdMdgiZCIF/45deqaBH9bRrZTOwmb0izd6NmYBeoQEKfUsTtIRcs21sl2ptAw7U1KbaEAAABPAZ6fdEf/AAAqHjKKrCgLQ5QnefxzSHAxZ+1dcHze+BTVY/hx4S6Mg0SwCThEXN3xFl8AAD9ugroJ0QcXvceNzDmny21whsmHXdSOB5e2gAAAAD4BnoFqR/8AACoCnJ/0hj81Ybt8n8F12ckKpi85Ta3sYpkclXbwm2bSd0oqAXkC1nqmFOLukcKy48x3abRbQQAAAFdBmoNJqEFsmUwIX//+jLAAAFKCxDxIgCFtuEF3e4GC6spreebjFsDJe4F0A0cgcIGRCHhJldyq7ipwxrbeN7X54Bqb5SeRCnqLgft73DIe6O0mvRxlzjQAAACJQZqmSeEKUmUwIX/+jLAAAFS9BTo8SieTZrFeCZnbOCNlu8Kz3vL5TZ7+hmk0MhmHwOu0UyeIAWdS80GEm+ZFQqyprA4Pubzq/vO45hvDpB8CgTsx3PfV5qGG8hFhZe1PyfycPcb5w9wixXDD1WoDtICExhjY5a1SrAkiTJ8OH7ESeaOYRvfg6jEAAABHQZ7ERTRMI/8AABsJfId6YLY+6dK/r1yOMKRsv9FHvvTyWxmf7Uy7qFCDkDyK0b2h1tkn82QHvrs4lG2+tJf3Czgp5FyuFg8AAABHAZ7lakf/AAArIpyMQsYZwXkzMUbmOWM3Ute5F/rCgx6Zbo/UIpw+KqYJ6ijRBnNQBHg8gAQil1j7NJqK7KIDcEosuHQzFTEAAABkQZroSahBaJlMFPC//oywAABUqkqcEAQsv7rlngBjlHKTKrBRj5fikAN15tRv2jshUqmjmpAy6z/+MLitan1/c5B9yzqfVQQ7Znivyqc1VSimFhYjRjtMStZtL9cM8Rv2FYZv4QAAAFMBnwdqR/8AACs2wOUUVh4b1brQHiqZtrjTodw1TFEdUJU1TqEfO2Hbx4Giyrmuq6GDXN2aScsAATK8i4CWVWmEggL8aJRrBljUexHBOuGIrw65OAAAAJtBmwtJ4QpSZTAhX/44QAABUTetg2wA6R2Uw4KTUxingkPyV+7CqIEUDIpd0BflwE18KWF8jZz/usS377hzc3qpQROctfm0XdIkgbr5kmkFMh2Upn8SuBIQtk3Rv318T/I8C4dWCP47oXpP05Wvvf6Auob4nDdzeHcJRZkMqw71FgQbaFKIp8vwZ3Ni1Pl3fe2N2MdkoNdRTVRqUAAAAE9BnylFNEwj/wAAG6l8kCWEAQyN6jTwbiF7LTr1ePEE6iEs/+3RR6vSOCDeYSbL36aULYLj+vRg0RXUVFrs5SABvJmVrD4Tdui0Bkm3wHTBAAAAOwGfSmpH/wAALEMOrJ7XxzDviBkhaacKawoNRHE9xA0cNLqu+7Oa+8GCf7GEyh+f3OH4Bq4NpFAegBiwAAAAa0GbTEmoQWiZTAhf//6MsAAAVunxX7xQNf/IksQdnp4fUhMs76JHOBPAV18UlI6EoaKZ/OwHn8HJnrL8KHQP4DjPHHnQnhbw7dg6wbve01awtRLW119YSidAlOGwus97/THgiUHx+xfoLEzQAAAAd0Gbb0nhClJlMCF//oywAABZKniOSiKsZsytzUgLst3iEj7y1+c2aRs5aiWsPgGOzZJXd+KKRSxQj04HUWAC11qMeo8lYZkK8DrRbvqtdLAama1fJugXoCswNP0ofmJwkaUhZuMTTt/gP63LuKS27GHLr1gs0bVBAAAAR0GfjUU0TCP/AAAcWGusvZu8M4MYwLlVCtfM6XxKqnfWVlkHw4vhxgb5jooHbTlej0xMK11WbCOMoBginxb5p0cZzmHqLyyhAAAARwGfrmpH/wAALWMOWv6XqLhwOmN//ifThwimTjQs2QDrlGXQumR62WNTtV6vuYqlbLqVQOoeWbAAmagTvC38DHwxKQfE+b5hAAAAZUGbsUmoQWiZTBTwv/6MsAAAWSpirkAC2dTdAm0tw4z5H1R7m/eByxww36EyBo49zAFoUa1Bx/PblLM7Ou5BL2ww/+pJyEesCefVTB2sibQSLnbdTXveqSkgPk+wtSPyDrXBLOa2AAAAQwGf0GpH/wAALWMOjVjvvga/UNktK0rZyc/QzxnUt6Tsw6S9e3/ru3UaJI8IavpiZlAhNcX+sU9B36U2p3VeYmRUQoIAAAB6QZvUSeEKUmUwIX/+jLAAAFuT+pVWiUe0csA+e6plJrCVSfu+rl5i+ecv6Xv75zi9BjnSpY1pyJL9zUgpLZ/mc7Hau/GEWfWQix7WD+pua5CL8/B/tf5qqh0KUtTL3MvsqQd0iSEWGnC4e4zxZpLRLOta3ra2lnxUL8cAAABEQZ/yRTRMI/8AABz8vW8eAK18v0aZBSgl+JRp9IuhUwE/QLduGdiE9IWXBRA5x2eucl/xGOMO3gzk1Ecw8Vg5tzGgHTAAAABEAZ4Takf/AAAugw4f9I3b+7Tw9dCW5AAzj6wVUkqiwt+++c3xz8i1mHLcCGC20qLa+hbw/GHxGsYEeyTdUdxrnEEJQcAAAACGQZoYSahBaJlMCFf//jhAAAFqjqNIE+5cfKD5i1eMf19h2YbpxHfbG/RcaoI8VUThnQBW/Oh3OisoiaqjB6PuFedEH02dD6qGct2w+vyF9pCJQm2mia/tEASxHpEKTE8m4V7V5RLRZwuj/0kQEX7I2nv94BvxsUMNOfcIhBa30AD9oOoXMEEAAABSQZ42RREsI/8AAB2svxkjUQ6nZrGpJQtiVg6o3B0TpZPk96gsgAVUQ66UGNoQ20Df4dX+cWnPVm4CjxuSDx7+PsQBdEF/DQS4HCNd+kMhvXgDZgAAAEYBnlV0R/8AAC/XaYrngBuCdGq2JP+8kh8xrLs2QjGsNFJapXegznAgyaLsEY11mn3Kqy51CS/Fz0pp94vydsYBHpAMbDGDAAAATgGeV2pH/wAAL5ikI9GAIjwP57vHvjOjODpXkOxAbY7jct1CshcvPvRNK2ERXQZCc8orTClidD2g0G+T2+GbbF28ge1T+2mdnHuYtFz6ywAAAJ1BmllJqEFsmUwIX//+jLAAAGAdmQBCrajHrBCIVbuXoIVlQ8ZUOrM+cuLKxUY8QB83gv54FU5leSh+RZabRAHQyH4yqpL+jSHPKgw1qrheYHKV0HAVKWwBYuN7wIALcsUTGgKXD9/mekPE2KauXK2jzBpS2DLvsgwwz0VtEPIpsyDPuPJ1GLwZeM0Ah/9/M/+WAuQickLgMyl89XhgAAAAhkGafEnhClJlMCF//oywAABgKQ1eM5c7UU+sHsgBL3z+U1hm+0i7VIK1BgC7DzHrhh/ULmoK0WTbTVrR2y1sQB4E4H0yNe4a17nX6sJ96TZBTxUYSiTZyiMGutH//H2c/JKKDAy9j9ObDAC7PeqW8YxSkmvptGhoZAAO+v4g5MY82bwQcQMtAAAAYEGemkU0TCP/AAAeXMKZI1BxFn262C4RNZUUfI6UHkGkCupnbka44vu4yG83xXaGK5O7WxqJmyGs1d+D9gwbbKFZH4g22kACal7aCVCeb67hIc++nuPmxOdjx0YwYtoH+AAAAEUBnrtqR/8AADD3zAaIzEofTPH1eD7wJPzBFlDK1T/o+9Lxy8cxPQMmZDfDUbqAANoAU/5erxDk+xPa+6oA+BfEvaWrC/kAAACEQZq+SahBaJlMFPCv/jhAAAF9Pcr2707uXLG/3NjKVKKxDWr+hgqSbBXiHhdMPHTrgVWKOIAhZR8EyE0/5LWnKl9m9mmBeaycMJlnJd1mJw3+uqBST66XvDwrLU8+8mPHFPMffpptFm0Q5fxdy1G+6qfnwNb4iEVSrtaWqVhQdscNRe9BAAAAQQGe3WpH/wAAMjfL7b2ZAoZF4WrUghAX7Ij4FUbBxgebwW8HGoqXJu/mQQUAG1CEOgzJ4B/gUAbILySBOLF29STgAAAAaEGawUnhClJlMCFf/jhAAAGIHTyspHbyoXlY3DxMjnwjN3Wui5GD9guYHpJZnaq7+7OcOKlUAKZ7NcFyKskF+iW88SVBp4tzTvDNHjGksDI0iVmIqYHxU26/1UUqkOjSrUGaEICjs1XAAAAAU0Ge/0U0TCP/AAAfyGutP6MAgY/anb6SEsSrWHInNHCFXLKjLWwsoCWBtAtBmmuHJY4pg+FNXYzxvLncQ2AEsPKezwLYe6WzlE8LzHP/vxVTe2NBAAAAVQGfAGpH/wAAM3fXRUwBymj9G/Oo9SxhehKOHrqz9j56YURHbVXH6cOZsM1Ghgde01IEXoPhwcBRs2cJk/fPOyQTISHBhU/nZOviLgq1au4pZnz6xicAAABuQZsCSahBaJlMCFf//jhAAAGHO/ENgBGRQZuZP3eTfwv3JbD+6pxaHtfzcH+MDotXez4JpNLDK3rEm1N02STfGxwdftZjh7+jqWZNMqKre5EevUIVKnaW6VbVp7FB3wu0R4BAWCRTMXAagmp5MqEAAACDQZskSeEKUmUwURLCf/3xAAADA8pI8l29SpXUOtkUiABDsu07MkqL+pFjjh9Fyi17zAVcQvnGbZQJM1f2nMbpoBJthn48GSq0kHeInE3JbxOHp9/aVvYlsIg5O9igG3YLIodsU2JWtuxx8U6vtLthLWds8GlJrPKP7DfA2YXRU/GEwjgAAABLAZ9Dakf/AAA0t8vtvYSrJjxNevDTfnCceoqxivoXRMaefuoR/o+B6S0EJVWNS95zHOgsDUY+JeyJe22xa1diqRS7NtL6gtdKJ9vJAAAAlUGbRUnhDomUwIT//fEAAAMDyWwz7cAVmuEyC1Ax/p8HBusdrF6UU3K95swUPD7r8GQD5HsT4pOjQl03aJ8DocMiIIZEr3Wh65tOdOMEU2lS8szBd9f4A/ZLNIA8tNuk2v68KFvWdAYQL2qkku6pN6t8mEzSoK5kpnQvfZKUn0g1sLvxAMCDp8tIq0bOsNeskdKC3q7TAAAAk0GbZ0nhDyZTBRU8I//94QAABkZnVaLIAIR3QQwlk0kGCbB1tiBCHuJ+TSdmkoWgnwv/WDbIGsu2wPvqe9A2am0G6q5muaWmoDrENnWdQgOxcfasoz8aSyDAkYi9spnBA77YA4B/tYECAtYyDlf+wp+3x+c3FvwtHDMXdHC8sLG2xbecDruuoQZl/b4/te/I8xvd8QAAAEgBn4ZqR/8AADX33M29mQ2IgIYdX9eB/yV4xR4HNBxsnX1I3i9fOoYJc0sUlZvjSmqs0TIu0b0OGgtv0t2OiYl1m/twqlM4Fh0AAAB2QZuJSeEPJlMFPCP//eEAAAZt8jV6SAWs2LhODZnTm04dxzaaFhhkt6Au9NS7BIBxr8llnsbY1a2npz9QlYsT1RDIQmTHeiqOtPhGWGI5kE6NqPIUiw7FuYx1hy5tOYx+HRTRZeHFHbDfcoMk2Aj0WYkaoSZSgAAAAEQBn6hqR/8AADdSUTaR4tJEzbjDASaPZwemvPqER/iU54RyUaMBHjIycCCu4HnGDoposoC1/lyqCL4P4K8yKmuvsDYq/wAAAINBm6tJ4Q8mUwU8I//94QAABpU9IgKjvN7eXJ83hLvpzbjGhe9FungWqE+/8s86Bc1mnxSep+O+gYAEw0ZovE1ePNwr6/FXiTWAHEC3OVR4iebjE16cPa1VcQfpSmAvWuvxts7WNCN6cMpxyelmQ9QKDjGg5CpHcsXNhYRxkbawf80rIQAAADgBn8pqR/8AADiyUQOID7950pR7rVtZY0yBJGRAniuQbCGKd08cOIsK0fgdo8EQQ9eKne9cBVNU4gAAAGZBm81J4Q8mUwU8I//94QAABpRU7u7/KTEb3eGiQAs5fTlHnLQUMNIxeg1Ft1V6Te9nuzjaFCzeFLPZrZSGfyg96puU0yEsQwbL7JzkhL/FobFxhOgz275kPVln3LOrDQPoUiRSCtAAAAAvAZ/sakf/AAA4kP/H3BoFGu/IpYhygm8UTQrWXCpuqNj5zCuKcIrkAxUK6f472qkAAABLQZvwSeEPJlMCEf/94QAABp/Rv8wcQjRqH+OCvqct3f27dvgNV0UPqFhNUFlX0cqM2nRwCv6cT/8k5FNXIyDHjZ75LOSVNJVPoPThAAAANkGeDkURPCP/AAAjuoMpCNZLQcpunXqJ0TU5beU4Fyst241wKkJ+AAVhP3IqLK3yjBp0V4aqqQAAADUBni9qR/8AADi8ndc0XOU6hA0O07Fafl0roDW1tEKYln+3riPv5FNqWAAQYdwmkiV8HCqL4AAAADNBmjJJqEFomUwU8f/8hAAAAwFj9HE5exUC2EO6gV+gYjrwGT1cx5ku5W28GPJ9xgPtrMAAAAAiAZ5Rakf/AAADAxBSEwr3Xq8jt6IoiEOCHppmtvo2CTWQHwAACCdtb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAI/AABAAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAHUXRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAI/AAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAACWAAAAZAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAACPwAAAIAAAEAAAAABsltZGlhAAAAIG1kaGQAAAAAAAAAAAAAAAAAADIAAABzAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAAZ0bWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAGNHN0YmwAAACYc3RzZAAAAAAAAAABAAAAiGF2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAACWAGQAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY//8AAAAyYXZjQwFkAB//4QAZZ2QAH6zZQJgz5eEAAAMAAQAAAwBkDxgxlgEABmjr48siwAAAABhzdHRzAAAAAAAAAAEAAABzAAABAAAAABRzdHNzAAAAAAAAAAEAAAABAAADWGN0dHMAAAAAAAAAaQAAAAEAAAIAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAEAAAAAAIAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAIAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAEAAAAAAIAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABAAAAAACAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAAAgAAAAABAAAEAAAAAAIAAAEAAAAAAQAAAwAAAAABAAABAAAAAAEAAAQAAAAAAgAAAQAAAAABAAACAAAAAAEAAAQAAAAAAgAAAQAAAAABAAADAAAAAAEAAAEAAAAAAQAABAAAAAACAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAAAgAAAAABAAAEAAAAAAIAAAEAAAAAAQAAAwAAAAABAAABAAAAAAEAAAQAAAAAAgAAAQAAAAABAAACAAAAAAEAAAMAAAAAAQAAAQAAAAABAAACAAAAAAEAAAMAAAAAAQAAAQAAAAABAAADAAAAAAEAAAEAAAAAAQAAAwAAAAABAAABAAAAAAEAAAMAAAAAAQAAAQAAAAABAAAEAAAAAAIAAAEAAAAAAQAAAwAAAAABAAABAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAcwAAAAEAAAHgc3RzegAAAAAAAAAAAAAAcwAABK8AAADHAAAAUAAAAEEAAABYAAAAxQAAAEsAAABLAAAAWwAAAGcAAABnAAAANgAAAD0AAAC8AAAAVgAAAEIAAABGAAAAaQAAAEsAAABAAAAAcAAAAD0AAAAyAAAAMQAAAMkAAABAAAAAMQAAAEQAAAB/AAAAfgAAAFUAAAA5AAAAMQAAAHcAAABUAAAAPQAAAD8AAACUAAAAWAAAAEMAAABKAAAAjgAAAFIAAABCAAAAPAAAAGcAAABFAAAAWwAAAG4AAABcAAAAPQAAAEcAAAB8AAAAbwAAAFEAAABUAAAAcwAAAGkAAAA9AAAATwAAAIQAAABPAAAATAAAAIsAAABJAAAAUwAAAEIAAABbAAAAjQAAAEsAAABLAAAAaAAAAFcAAACfAAAAUwAAAD8AAABvAAAAewAAAEsAAABLAAAAaQAAAEcAAAB+AAAASAAAAEgAAACKAAAAVgAAAEoAAABSAAAAoQAAAIoAAABkAAAASQAAAIgAAABFAAAAbAAAAFcAAABZAAAAcgAAAIcAAABPAAAAmQAAAJcAAABMAAAAegAAAEgAAACHAAAAPAAAAGoAAAAzAAAATwAAADoAAAA5AAAANwAAACYAAAAUc3RjbwAAAAAAAAABAAAAMAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTcuODMuMTAw\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O2F6QlhSC9aO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MBUK5zjU2Vju",
        "colab_type": "text"
      },
      "source": [
        "# Ray\n",
        "\n",
        "Note that if you running this code on Google colab then you must restart the runtime after uninstalling pyarrow below or else ray will error out."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mY0_6Ni32XfJ",
        "colab_type": "code",
        "outputId": "353e7dbd-e02a-4621-fa3d-7e7e8adb141e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!pip uninstall -y pyarrow\n",
        "!pip install tensorflow ray[rllib] > /dev/null 2>&1\n",
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install pyvirtualdisplay > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[33mWARNING: Skipping pyarrow as it is not installed.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9nB-T7HOHHx9",
        "colab_type": "code",
        "outputId": "3e98c1da-b0f0-4e59-b052-d31ce777698b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "from pyvirtualdisplay import Display\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0625 19:30:17.028367 140180297578368 abstractdisplay.py:151] xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Display cmd_param=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1400x900x24', ':1001'] cmd=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1400x900x24', ':1001'] oserror=None return_code=None stdout=\"None\" stderr=\"None\" timeout_happened=False>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F4PIYdNlHngX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import ray\n",
        "from ray import tune\n",
        "\n",
        "ray.init()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPn2avMTBkVT",
        "colab_type": "code",
        "outputId": "3d32ad04-934d-4c66-aea9-31e2cb54bc03",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "tune.run(\n",
        "    \"DQN\",\n",
        "    stop={\"episode_reward_mean\": 100},\n",
        "    config={\n",
        "        \"env\": \"CartPole-v0\",\n",
        "        \"num_gpus\": 0,\n",
        "        \"num_workers\": 1,\n",
        "        \"lr\": tune.grid_search([0.01, 0.001, 0.0001]),\n",
        "        \"monitor\": False,\n",
        "    },\n",
        ")\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-06-25 19:44:53,022\tINFO tune.py:61 -- Tip: to resume incomplete experiments, pass resume='prompt' or resume=True to run()\n",
            "2019-06-25 19:44:53,024\tINFO tune.py:232 -- Starting a new experiment.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 1.9/13.7 GB\n",
            "\n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 1.9/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m WARNING: Logging before flag parsing goes to stderr.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:02.178079 140633306634112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:03,067\tINFO policy_evaluator.py:311 -- Creating policy evaluation worker 0 on CPU (please ignore any CUDA init errors)\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:03.074652: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:03.074966: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x6397640 executing computations on platform Host. Devices:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:03.075003: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:03.084947 140633306634112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/models/fcnet.py:37: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Use keras.layers.dense instead.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:03.780741 140633306634112 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:03.921737 140633306634112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:663: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:03.938131 140633306634112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:03.957903 140633306634112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:337: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Use `tf.random.categorical` instead.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:07.931466: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:08,751\tINFO policy_evaluator.py:731 -- Built policy map: {'default_policy': <ray.rllib.agents.dqn.dqn_policy.DQNTFPolicy object at 0x7fe651b4dbe0>}\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:08,751\tINFO policy_evaluator.py:732 -- Built preprocessor map: {'default_policy': <ray.rllib.models.preprocessors.NoPreprocessor object at 0x7fe651b4db00>}\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:08,751\tINFO policy_evaluator.py:343 -- Built filter map: {'default_policy': <ray.rllib.utils.filter.NoFilter object at 0x7fe651b369e8>}\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m W0625 19:45:08.852919 140633306634112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/policy/tf_policy.py:509: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m Prefer Variable.assign which has equivalent behavior in 2.X.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m WARNING: Logging before flag parsing goes to stderr.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:15.471442 140282351548288 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:16,250\tINFO policy_evaluator.py:311 -- Creating policy evaluation worker 1 on CPU (please ignore any CUDA init errors)\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:16.284778: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:16.285065: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x6539b80 executing computations on platform Host. Devices:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:16.285107: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:16.293445 140282351548288 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/models/fcnet.py:37: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Use keras.layers.dense instead.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:16.937118 140282351548288 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:17.066888 140282351548288 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:663: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:17.078192 140282351548288 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:17.099149 140282351548288 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:337: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Use `tf.random.categorical` instead.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:21.391268: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m W0625 19:45:22.213191 140282351548288 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/policy/tf_policy.py:509: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m Prefer Variable.assign which has equivalent behavior in 2.X.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,773\tINFO policy_evaluator.py:437 -- Generating sample batch of size 4\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,774\tINFO sampler.py:308 -- Raw obs from env: { 0: { 'agent0': np.ndarray((4,), dtype=float64, min=-0.017, max=0.047, mean=0.019)}}\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,774\tINFO sampler.py:309 -- Info return from env: {0: {'agent0': None}}\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,779\tINFO sampler.py:407 -- Preprocessed obs: np.ndarray((4,), dtype=float64, min=-0.017, max=0.047, mean=0.019)\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,780\tINFO sampler.py:411 -- Filtered obs: np.ndarray((4,), dtype=float64, min=-0.017, max=0.047, mean=0.019)\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,782\tINFO sampler.py:525 -- Inputs to compute_actions():\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m { 'default_policy': [ { 'data': { 'agent_id': 'agent0',\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                                   'env_id': 0,\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                                   'info': None,\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                                   'obs': np.ndarray((4,), dtype=float64, min=-0.017, max=0.047, mean=0.019),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                                   'prev_action': np.ndarray((), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                                   'prev_reward': 0.0,\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                                   'rnn_state': []},\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'type': 'PolicyEvalData'}]}\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:22,783\tINFO tf_run_builder.py:92 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:23,078\tINFO sampler.py:552 -- Outputs of compute_actions():\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m { 'default_policy': ( np.ndarray((1,), dtype=int64, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                       [],\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                       { 'q_values': np.ndarray((1, 2), dtype=float32, min=-0.027, max=0.034, mean=0.003)})}\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:23,102\tINFO sample_batch_builder.py:161 -- Trajectory fragment after postprocess_trajectory():\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m { 'agent0': { 'data': { 'actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'agent_index': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'dones': np.ndarray((4,), dtype=bool, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'eps_id': np.ndarray((4,), dtype=int64, min=166857152.0, max=166857152.0, mean=166857152.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'infos': np.ndarray((4,), dtype=object, head={}),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'new_obs': np.ndarray((4, 4), dtype=float32, min=-0.571, max=0.411, mean=-0.009),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'obs': np.ndarray((4, 4), dtype=float32, min=-0.571, max=0.411, mean=-0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'prev_actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'prev_rewards': np.ndarray((4,), dtype=float32, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'q_values': np.ndarray((4, 2), dtype=float32, min=-0.407, max=0.669, mean=0.065),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'rewards': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         't': np.ndarray((4,), dtype=int64, min=0.0, max=3.0, mean=1.5),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'unroll_id': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m                         'weights': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m               'type': 'SampleBatch'}}\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m 2019-06-25 19:45:23,109\tINFO policy_evaluator.py:474 -- Completed sample batch:\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m { 'data': { 'actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'agent_index': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'dones': np.ndarray((4,), dtype=bool, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'eps_id': np.ndarray((4,), dtype=int64, min=166857152.0, max=166857152.0, mean=166857152.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'infos': np.ndarray((4,), dtype=object, head={}),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'new_obs': np.ndarray((4, 4), dtype=float32, min=-0.571, max=0.411, mean=-0.009),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'obs': np.ndarray((4, 4), dtype=float32, min=-0.571, max=0.411, mean=-0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'prev_actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'prev_rewards': np.ndarray((4,), dtype=float32, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'q_values': np.ndarray((4, 2), dtype=float32, min=-0.407, max=0.669, mean=0.065),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'rewards': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             't': np.ndarray((4,), dtype=int64, min=0.0, max=3.0, mean=1.5),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'unroll_id': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m             'weights': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m   'type': 'SampleBatch'}\n",
            "\u001b[2m\u001b[36m(pid=1357)\u001b[0m \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0625 19:45:35.022074 140180297578368 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/ray/tune/logger.py:114: The name tf.Summary is deprecated. Please use tf.compat.v1.Summary instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-45-34\n",
            "  done: false\n",
            "  episode_len_mean: 23.476190476190474\n",
            "  episode_reward_max: 65.0\n",
            "  episode_reward_mean: 23.476190476190474\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 42\n",
            "  episodes_total: 42\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: .nan\n",
            "    learner: {}\n",
            "    max_exploration: 1.0\n",
            "    min_exploration: 1.0\n",
            "    num_steps_sampled: 1000\n",
            "    num_steps_trained: 0\n",
            "    num_target_updates: 1\n",
            "    opt_peak_throughput: 0.0\n",
            "    opt_samples: .nan\n",
            "    replay_time_ms: .nan\n",
            "    sample_time_ms: 49.79\n",
            "    update_time_ms: 41.697\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21874249636471926\n",
            "    mean_inference_ms: 2.3641448159079683\n",
            "    mean_processing_ms: 0.4266613608711844\n",
            "  time_since_restore: 25.655725240707397\n",
            "  time_this_iter_s: 25.655725240707397\n",
            "  time_total_s: 25.655725240707397\n",
            "  timestamp: 1561491934\n",
            "  timesteps_since_restore: 1000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 1000\n",
            "  training_iteration: 1\n",
            "  \u001b[2m\u001b[36m(pid=1351)\u001b[0m /usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3118: RuntimeWarning: Mean of empty slice.\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m   out=out, **kwargs)\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m /usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m   ret = ret.dtype.type(ret / rcount)\n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 2.7/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 25 s, 1 iter, 1000 ts, 23.5 rew\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:35,181\tINFO policy_evaluator.py:564 -- Training on concatenated sample batches:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m { 'count': 32,\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m   'policy_batches': { 'default_policy': { 'data': { 'actions': np.ndarray((32,), dtype=int64, min=0.0, max=1.0, mean=0.469),\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                                     'batch_indexes': np.ndarray((32,), dtype=int64, min=4.0, max=996.0, mean=587.625),\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                                     'dones': np.ndarray((32,), dtype=bool, min=0.0, max=1.0, mean=0.062),\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                                     'new_obs': np.ndarray((32, 4), dtype=float32, min=-2.521, max=1.52, mean=-0.0),\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                                     'obs': np.ndarray((32, 4), dtype=float32, min=-2.188, max=1.324, mean=-0.001),\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                                     'rewards': np.ndarray((32,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                                     'weights': np.ndarray((32,), dtype=float64, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                           'type': 'SampleBatch'}},\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m   'type': 'MultiAgentBatch'}\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:35,182\tINFO tf_run_builder.py:92 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m 2019-06-25 19:45:36,837\tINFO policy_evaluator.py:586 -- Training output:\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m { 'default_policy': { 'learner_stats': { 'cur_lr': 0.009999999776482582,\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                          'max_q': 1.1752741,\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                          'mean_q': 0.024537522,\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                          'mean_td_error': -1.3309821,\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                          'min_q': -0.6301401,\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                                          'model': {}},\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m                       'td_error': np.ndarray((32,), dtype=float32, min=-2.45, max=-0.995, mean=-1.331)}}\n",
            "\u001b[2m\u001b[36m(pid=1351)\u001b[0m \n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-45-55\n",
            "  done: false\n",
            "  episode_len_mean: 22.647727272727273\n",
            "  episode_reward_max: 65.0\n",
            "  episode_reward_mean: 22.647727272727273\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 46\n",
            "  episodes_total: 88\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.136\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 10.136509895324707\n",
            "        mean_q: 4.959753036499023\n",
            "        mean_td_error: 0.09046021103858948\n",
            "        min_q: -1.421177864074707\n",
            "        model: {}\n",
            "    max_exploration: 0.902\n",
            "    min_exploration: 0.902\n",
            "    num_steps_sampled: 2000\n",
            "    num_steps_trained: 8000\n",
            "    num_target_updates: 3\n",
            "    opt_peak_throughput: 2636.761\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.294\n",
            "    sample_time_ms: 17.708\n",
            "    update_time_ms: 16.322\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21640006948366358\n",
            "    mean_inference_ms: 2.346191855923145\n",
            "    mean_processing_ms: 0.4644291276221862\n",
            "  time_since_restore: 46.31636142730713\n",
            "  time_this_iter_s: 20.66063618659973\n",
            "  time_total_s: 46.31636142730713\n",
            "  timestamp: 1561491955\n",
            "  timesteps_since_restore: 2000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 2000\n",
            "  training_iteration: 2\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 3.1/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 46 s, 2 iter, 2000 ts, 22.6 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-46-13\n",
            "  done: false\n",
            "  episode_len_mean: 24.05\n",
            "  episode_reward_max: 65.0\n",
            "  episode_reward_mean: 24.05\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 37\n",
            "  episodes_total: 125\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 13.433\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 11.252812385559082\n",
            "        mean_q: 7.739860534667969\n",
            "        mean_td_error: 0.24563732743263245\n",
            "        min_q: 1.4648414850234985\n",
            "        model: {}\n",
            "    max_exploration: 0.804\n",
            "    min_exploration: 0.804\n",
            "    num_steps_sampled: 3000\n",
            "    num_steps_trained: 16000\n",
            "    num_target_updates: 5\n",
            "    opt_peak_throughput: 2382.226\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.592\n",
            "    sample_time_ms: 18.614\n",
            "    update_time_ms: 14.796\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21164938384185533\n",
            "    mean_inference_ms: 2.3014191868624265\n",
            "    mean_processing_ms: 0.48551082355061537\n",
            "  time_since_restore: 64.08357524871826\n",
            "  time_this_iter_s: 17.767213821411133\n",
            "  time_total_s: 64.08357524871826\n",
            "  timestamp: 1561491973\n",
            "  timesteps_since_restore: 3000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 3000\n",
            "  training_iteration: 3\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 3.5/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 64 s, 3 iter, 3000 ts, 24.1 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-46-35\n",
            "  done: false\n",
            "  episode_len_mean: 26.99\n",
            "  episode_reward_max: 84.0\n",
            "  episode_reward_mean: 26.99\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 30\n",
            "  episodes_total: 155\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 16.355\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 14.368959426879883\n",
            "        mean_q: 10.971094131469727\n",
            "        mean_td_error: -0.12635385990142822\n",
            "        min_q: 1.8363313674926758\n",
            "        model: {}\n",
            "    max_exploration: 0.706\n",
            "    min_exploration: 0.706\n",
            "    num_steps_sampled: 4000\n",
            "    num_steps_trained: 24000\n",
            "    num_target_updates: 7\n",
            "    opt_peak_throughput: 1956.621\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.316\n",
            "    sample_time_ms: 27.831\n",
            "    update_time_ms: 26.833\n",
            "  iterations_since_restore: 4\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21525097977003477\n",
            "    mean_inference_ms: 2.3046314065218723\n",
            "    mean_processing_ms: 0.5042053778484477\n",
            "  time_since_restore: 86.47922849655151\n",
            "  time_this_iter_s: 22.395653247833252\n",
            "  time_total_s: 86.47922849655151\n",
            "  timestamp: 1561491995\n",
            "  timesteps_since_restore: 4000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 4000\n",
            "  training_iteration: 4\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 3.9/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 86 s, 4 iter, 4000 ts, 27 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-46-54\n",
            "  done: false\n",
            "  episode_len_mean: 30.71\n",
            "  episode_reward_max: 135.0\n",
            "  episode_reward_mean: 30.71\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 29\n",
            "  episodes_total: 184\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.108\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 16.436054229736328\n",
            "        mean_q: 11.233628273010254\n",
            "        mean_td_error: -0.0525309219956398\n",
            "        min_q: 0.7026607990264893\n",
            "        model: {}\n",
            "    max_exploration: 0.608\n",
            "    min_exploration: 0.608\n",
            "    num_steps_sampled: 5000\n",
            "    num_steps_trained: 32000\n",
            "    num_target_updates: 9\n",
            "    opt_peak_throughput: 2268.207\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.101\n",
            "    sample_time_ms: 20.21\n",
            "    update_time_ms: 15.275\n",
            "  iterations_since_restore: 5\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21852470235082538\n",
            "    mean_inference_ms: 2.302752371916792\n",
            "    mean_processing_ms: 0.5072248129186153\n",
            "  time_since_restore: 104.60253190994263\n",
            "  time_this_iter_s: 18.123303413391113\n",
            "  time_total_s: 104.60253190994263\n",
            "  timestamp: 1561492014\n",
            "  timesteps_since_restore: 5000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 5000\n",
            "  training_iteration: 5\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 4.3/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 104 s, 5 iter, 5000 ts, 30.7 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-47-12\n",
            "  done: false\n",
            "  episode_len_mean: 25.37\n",
            "  episode_reward_max: 135.0\n",
            "  episode_reward_mean: 25.37\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 55\n",
            "  episodes_total: 239\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 22.466\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 17.45759391784668\n",
            "        mean_q: 14.142037391662598\n",
            "        mean_td_error: 0.0840936154127121\n",
            "        min_q: 2.043250322341919\n",
            "        model: {}\n",
            "    max_exploration: 0.51\n",
            "    min_exploration: 0.51\n",
            "    num_steps_sampled: 6000\n",
            "    num_steps_trained: 40000\n",
            "    num_target_updates: 11\n",
            "    opt_peak_throughput: 1424.358\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 9.171\n",
            "    sample_time_ms: 38.182\n",
            "    update_time_ms: 30.429\n",
            "  iterations_since_restore: 6\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22242843339891485\n",
            "    mean_inference_ms: 2.3158546509693823\n",
            "    mean_processing_ms: 0.5077375959607151\n",
            "  time_since_restore: 122.99372792243958\n",
            "  time_this_iter_s: 18.39119601249695\n",
            "  time_total_s: 122.99372792243958\n",
            "  timestamp: 1561492032\n",
            "  timesteps_since_restore: 6000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 6000\n",
            "  training_iteration: 6\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 4.7/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 122 s, 6 iter, 6000 ts, 25.4 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-47-34\n",
            "  done: false\n",
            "  episode_len_mean: 27.5\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 27.5\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 21\n",
            "  episodes_total: 260\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.017\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 19.506223678588867\n",
            "        mean_q: 14.458924293518066\n",
            "        mean_td_error: 0.18865829706192017\n",
            "        min_q: 2.290074110031128\n",
            "        model: {}\n",
            "    max_exploration: 0.41200000000000003\n",
            "    min_exploration: 0.41200000000000003\n",
            "    num_steps_sampled: 7000\n",
            "    num_steps_trained: 48000\n",
            "    num_target_updates: 13\n",
            "    opt_peak_throughput: 2662.812\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 6.062\n",
            "    sample_time_ms: 17.245\n",
            "    update_time_ms: 16.187\n",
            "  iterations_since_restore: 7\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.221944735613367\n",
            "    mean_inference_ms: 2.3152819749157114\n",
            "    mean_processing_ms: 0.5073270390141147\n",
            "  time_since_restore: 144.80893993377686\n",
            "  time_this_iter_s: 21.81521201133728\n",
            "  time_total_s: 144.80893993377686\n",
            "  timestamp: 1561492054\n",
            "  timesteps_since_restore: 7000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 7000\n",
            "  training_iteration: 7\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 5.1/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 144 s, 7 iter, 7000 ts, 27.5 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-47-52\n",
            "  done: false\n",
            "  episode_len_mean: 31.8\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 31.8\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 18\n",
            "  episodes_total: 278\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 15.571\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 20.148395538330078\n",
            "        mean_q: 15.422500610351562\n",
            "        mean_td_error: 0.7547379732131958\n",
            "        min_q: -0.6351772546768188\n",
            "        model: {}\n",
            "    max_exploration: 0.31400000000000006\n",
            "    min_exploration: 0.31400000000000006\n",
            "    num_steps_sampled: 8000\n",
            "    num_steps_trained: 56000\n",
            "    num_target_updates: 15\n",
            "    opt_peak_throughput: 2055.134\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.594\n",
            "    sample_time_ms: 19.541\n",
            "    update_time_ms: 14.815\n",
            "  iterations_since_restore: 8\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22114980577876384\n",
            "    mean_inference_ms: 2.3151321462854857\n",
            "    mean_processing_ms: 0.5077901426039685\n",
            "  time_since_restore: 162.89738845825195\n",
            "  time_this_iter_s: 18.088448524475098\n",
            "  time_total_s: 162.89738845825195\n",
            "  timestamp: 1561492072\n",
            "  timesteps_since_restore: 8000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 8000\n",
            "  training_iteration: 8\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 5.5/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 162 s, 8 iter, 8000 ts, 31.8 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-48-13\n",
            "  done: false\n",
            "  episode_len_mean: 36.63\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 36.63\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 21\n",
            "  episodes_total: 299\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 32.087\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 22.289743423461914\n",
            "        mean_q: 18.164962768554688\n",
            "        mean_td_error: 0.954929769039154\n",
            "        min_q: 4.5791826248168945\n",
            "        model: {}\n",
            "    max_exploration: 0.21599999999999997\n",
            "    min_exploration: 0.21599999999999997\n",
            "    num_steps_sampled: 9000\n",
            "    num_steps_trained: 64000\n",
            "    num_target_updates: 17\n",
            "    opt_peak_throughput: 997.279\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 13.207\n",
            "    sample_time_ms: 53.158\n",
            "    update_time_ms: 47.116\n",
            "  iterations_since_restore: 9\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2220726314636928\n",
            "    mean_inference_ms: 2.32128869537571\n",
            "    mean_processing_ms: 0.5112224875230866\n",
            "  time_since_restore: 184.02075815200806\n",
            "  time_this_iter_s: 21.123369693756104\n",
            "  time_total_s: 184.02075815200806\n",
            "  timestamp: 1561492093\n",
            "  timesteps_since_restore: 9000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 9000\n",
            "  training_iteration: 9\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 5.9/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 184 s, 9 iter, 9000 ts, 36.6 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-48-32\n",
            "  done: false\n",
            "  episode_len_mean: 45.5\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 45.5\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 10\n",
            "  episodes_total: 309\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.672\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 23.260173797607422\n",
            "        mean_q: 17.905406951904297\n",
            "        mean_td_error: 0.7549974918365479\n",
            "        min_q: 0.2857074737548828\n",
            "        model: {}\n",
            "    max_exploration: 0.118\n",
            "    min_exploration: 0.118\n",
            "    num_steps_sampled: 10000\n",
            "    num_steps_trained: 72000\n",
            "    num_target_updates: 19\n",
            "    opt_peak_throughput: 2180.969\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.132\n",
            "    sample_time_ms: 19.88\n",
            "    update_time_ms: 13.338\n",
            "  iterations_since_restore: 10\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22262001160767092\n",
            "    mean_inference_ms: 2.324437007920996\n",
            "    mean_processing_ms: 0.5127655956712358\n",
            "  time_since_restore: 202.99509835243225\n",
            "  time_this_iter_s: 18.974340200424194\n",
            "  time_total_s: 202.99509835243225\n",
            "  timestamp: 1561492112\n",
            "  timesteps_since_restore: 10000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 10000\n",
            "  training_iteration: 10\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 202 s, 10 iter, 10000 ts, 45.5 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-48-49\n",
            "  done: false\n",
            "  episode_len_mean: 53.9\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 53.9\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 8\n",
            "  episodes_total: 317\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 15.668\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 25.832168579101562\n",
            "        mean_q: 20.994808197021484\n",
            "        mean_td_error: 0.5020767450332642\n",
            "        min_q: 1.0851041078567505\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 11000\n",
            "    num_steps_trained: 80000\n",
            "    num_target_updates: 21\n",
            "    opt_peak_throughput: 2042.387\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.054\n",
            "    sample_time_ms: 17.712\n",
            "    update_time_ms: 12.553\n",
            "  iterations_since_restore: 11\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22279482075901882\n",
            "    mean_inference_ms: 2.3249119033147085\n",
            "    mean_processing_ms: 0.5132906540012638\n",
            "  time_since_restore: 220.1499080657959\n",
            "  time_this_iter_s: 17.154809713363647\n",
            "  time_total_s: 220.1499080657959\n",
            "  timestamp: 1561492129\n",
            "  timesteps_since_restore: 11000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 11000\n",
            "  training_iteration: 11\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 220 s, 11 iter, 11000 ts, 53.9 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-49-11\n",
            "  done: false\n",
            "  episode_len_mean: 58.51\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 58.51\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 32\n",
            "  episodes_total: 349\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 25.898\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 26.53645133972168\n",
            "        mean_q: 22.66250228881836\n",
            "        mean_td_error: 2.0660226345062256\n",
            "        min_q: 1.3101022243499756\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 12000\n",
            "    num_steps_trained: 88000\n",
            "    num_target_updates: 23\n",
            "    opt_peak_throughput: 1235.601\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 12.826\n",
            "    sample_time_ms: 43.856\n",
            "    update_time_ms: 42.578\n",
            "  iterations_since_restore: 12\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22490543893462983\n",
            "    mean_inference_ms: 2.330897105599346\n",
            "    mean_processing_ms: 0.5158389932975641\n",
            "  time_since_restore: 241.78713989257812\n",
            "  time_this_iter_s: 21.637231826782227\n",
            "  time_total_s: 241.78713989257812\n",
            "  timestamp: 1561492151\n",
            "  timesteps_since_restore: 12000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 12000\n",
            "  training_iteration: 12\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 241 s, 12 iter, 12000 ts, 58.5 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-49-28\n",
            "  done: false\n",
            "  episode_len_mean: 57.14\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 57.14\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 19\n",
            "  episodes_total: 368\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.037\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 28.307544708251953\n",
            "        mean_q: 20.607501983642578\n",
            "        mean_td_error: 0.07437895238399506\n",
            "        min_q: 2.303727149963379\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 13000\n",
            "    num_steps_trained: 96000\n",
            "    num_target_updates: 25\n",
            "    opt_peak_throughput: 2658.372\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.748\n",
            "    sample_time_ms: 15.622\n",
            "    update_time_ms: 14.813\n",
            "  iterations_since_restore: 13\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.225231693524459\n",
            "    mean_inference_ms: 2.326772936020629\n",
            "    mean_processing_ms: 0.5144085065475473\n",
            "  time_since_restore: 258.94025325775146\n",
            "  time_this_iter_s: 17.15311336517334\n",
            "  time_total_s: 258.94025325775146\n",
            "  timestamp: 1561492168\n",
            "  timesteps_since_restore: 13000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 13000\n",
            "  training_iteration: 13\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 258 s, 13 iter, 13000 ts, 57.1 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-49-46\n",
            "  done: false\n",
            "  episode_len_mean: 60.86\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 60.86\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 8\n",
            "  episodes_total: 376\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 15.345\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 31.66539764404297\n",
            "        mean_q: 26.759105682373047\n",
            "        mean_td_error: 0.40918561816215515\n",
            "        min_q: 10.561456680297852\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 14000\n",
            "    num_steps_trained: 104000\n",
            "    num_target_updates: 27\n",
            "    opt_peak_throughput: 2085.331\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.811\n",
            "    sample_time_ms: 18.4\n",
            "    update_time_ms: 16.597\n",
            "  iterations_since_restore: 14\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22552789161641543\n",
            "    mean_inference_ms: 2.3253294724161964\n",
            "    mean_processing_ms: 0.5135805455099609\n",
            "  time_since_restore: 276.41886711120605\n",
            "  time_this_iter_s: 17.47861385345459\n",
            "  time_total_s: 276.41886711120605\n",
            "  timestamp: 1561492186\n",
            "  timesteps_since_restore: 14000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 14000\n",
            "  training_iteration: 14\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 276 s, 14 iter, 14000 ts, 60.9 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-50-08\n",
            "  done: false\n",
            "  episode_len_mean: 68.12\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 68.12\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 382\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 17.523\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 31.756122589111328\n",
            "        mean_q: 25.30614471435547\n",
            "        mean_td_error: 0.6534069776535034\n",
            "        min_q: 6.462457656860352\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 15000\n",
            "    num_steps_trained: 112000\n",
            "    num_target_updates: 29\n",
            "    opt_peak_throughput: 1826.121\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 8.344\n",
            "    sample_time_ms: 25.248\n",
            "    update_time_ms: 30.824\n",
            "  iterations_since_restore: 15\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2257991150969248\n",
            "    mean_inference_ms: 2.3257322139908267\n",
            "    mean_processing_ms: 0.5129298687544933\n",
            "  time_since_restore: 298.3582694530487\n",
            "  time_this_iter_s: 21.93940234184265\n",
            "  time_total_s: 298.3582694530487\n",
            "  timestamp: 1561492208\n",
            "  timesteps_since_restore: 15000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 15000\n",
            "  training_iteration: 15\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 298 s, 15 iter, 15000 ts, 68.1 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-50-25\n",
            "  done: false\n",
            "  episode_len_mean: 72.61\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 72.61\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 9\n",
            "  episodes_total: 391\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.426\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 33.20109176635742\n",
            "        mean_q: 25.591720581054688\n",
            "        mean_td_error: 0.7972341775894165\n",
            "        min_q: 1.529518723487854\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 16000\n",
            "    num_steps_trained: 120000\n",
            "    num_target_updates: 31\n",
            "    opt_peak_throughput: 2218.288\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.374\n",
            "    sample_time_ms: 17.78\n",
            "    update_time_ms: 13.279\n",
            "  iterations_since_restore: 16\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2261459303615622\n",
            "    mean_inference_ms: 2.324564090271762\n",
            "    mean_processing_ms: 0.5114383997433136\n",
            "  time_since_restore: 315.54818987846375\n",
            "  time_this_iter_s: 17.18992042541504\n",
            "  time_total_s: 315.54818987846375\n",
            "  timestamp: 1561492225\n",
            "  timesteps_since_restore: 16000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 16000\n",
            "  training_iteration: 16\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 315 s, 16 iter, 16000 ts, 72.6 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-50-42\n",
            "  done: false\n",
            "  episode_len_mean: 82.36\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 82.36\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 397\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.298\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 34.14920425415039\n",
            "        mean_q: 28.687511444091797\n",
            "        mean_td_error: 0.026291191577911377\n",
            "        min_q: -0.5627329349517822\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 17000\n",
            "    num_steps_trained: 128000\n",
            "    num_target_updates: 33\n",
            "    opt_peak_throughput: 2238.074\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.485\n",
            "    sample_time_ms: 17.524\n",
            "    update_time_ms: 12.174\n",
            "  iterations_since_restore: 17\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22618269654393114\n",
            "    mean_inference_ms: 2.322901293155722\n",
            "    mean_processing_ms: 0.5101773393033363\n",
            "  time_since_restore: 332.5962128639221\n",
            "  time_this_iter_s: 17.048022985458374\n",
            "  time_total_s: 332.5962128639221\n",
            "  timestamp: 1561492242\n",
            "  timesteps_since_restore: 17000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 17000\n",
            "  training_iteration: 17\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 332 s, 17 iter, 17000 ts, 82.4 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-51-03\n",
            "  done: false\n",
            "  episode_len_mean: 84.94\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 84.94\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 403\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.222\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 36.4993896484375\n",
            "        mean_q: 28.19709587097168\n",
            "        mean_td_error: 3.3464465141296387\n",
            "        min_q: 0.985515832901001\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 18000\n",
            "    num_steps_trained: 136000\n",
            "    num_target_updates: 35\n",
            "    opt_peak_throughput: 2249.978\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.307\n",
            "    sample_time_ms: 18.195\n",
            "    update_time_ms: 13.303\n",
            "  iterations_since_restore: 18\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2263902883451366\n",
            "    mean_inference_ms: 2.3225187748280174\n",
            "    mean_processing_ms: 0.5093981811437501\n",
            "  time_since_restore: 354.06765699386597\n",
            "  time_this_iter_s: 21.471444129943848\n",
            "  time_total_s: 354.06765699386597\n",
            "  timestamp: 1561492263\n",
            "  timesteps_since_restore: 18000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 18000\n",
            "  training_iteration: 18\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 354 s, 18 iter, 18000 ts, 84.9 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-51-20\n",
            "  done: false\n",
            "  episode_len_mean: 90.5\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 90.5\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 408\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.773\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 37.15810012817383\n",
            "        mean_q: 32.240360260009766\n",
            "        mean_td_error: 2.6342294216156006\n",
            "        min_q: 0.16170671582221985\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 19000\n",
            "    num_steps_trained: 144000\n",
            "    num_target_updates: 37\n",
            "    opt_peak_throughput: 2505.193\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.914\n",
            "    sample_time_ms: 13.441\n",
            "    update_time_ms: 16.093\n",
            "  iterations_since_restore: 19\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22645700331520716\n",
            "    mean_inference_ms: 2.321473836737147\n",
            "    mean_processing_ms: 0.5084911138623238\n",
            "  time_since_restore: 371.06088042259216\n",
            "  time_this_iter_s: 16.993223428726196\n",
            "  time_total_s: 371.06088042259216\n",
            "  timestamp: 1561492280\n",
            "  timesteps_since_restore: 19000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 19000\n",
            "  training_iteration: 19\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 371 s, 19 iter, 19000 ts, 90.5 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-51-38\n",
            "  done: false\n",
            "  episode_len_mean: 92.57\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 92.57\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 414\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 20.351\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 38.68680953979492\n",
            "        mean_q: 28.02227783203125\n",
            "        mean_td_error: 3.5009024143218994\n",
            "        min_q: -1.2860262393951416\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 20000\n",
            "    num_steps_trained: 152000\n",
            "    num_target_updates: 39\n",
            "    opt_peak_throughput: 1572.411\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 12.323\n",
            "    sample_time_ms: 29.116\n",
            "    update_time_ms: 26.884\n",
            "  iterations_since_restore: 20\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.226605495561724\n",
            "    mean_inference_ms: 2.320857705095431\n",
            "    mean_processing_ms: 0.5076464307121643\n",
            "  time_since_restore: 388.4334969520569\n",
            "  time_this_iter_s: 17.37261652946472\n",
            "  time_total_s: 388.4334969520569\n",
            "  timestamp: 1561492298\n",
            "  timesteps_since_restore: 20000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 20000\n",
            "  training_iteration: 20\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 388 s, 20 iter, 20000 ts, 92.6 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-51-59\n",
            "  done: false\n",
            "  episode_len_mean: 97.68\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 97.68\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 419\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 11.831\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 38.56929397583008\n",
            "        mean_q: 33.38999557495117\n",
            "        mean_td_error: 0.1679849624633789\n",
            "        min_q: 6.21500301361084\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 21000\n",
            "    num_steps_trained: 160000\n",
            "    num_target_updates: 41\n",
            "    opt_peak_throughput: 2704.863\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 6.163\n",
            "    sample_time_ms: 14.725\n",
            "    update_time_ms: 16.722\n",
            "  iterations_since_restore: 21\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22676038549579097\n",
            "    mean_inference_ms: 2.320789419519303\n",
            "    mean_processing_ms: 0.5071002973014123\n",
            "  time_since_restore: 409.15626525878906\n",
            "  time_this_iter_s: 20.722768306732178\n",
            "  time_total_s: 409.15626525878906\n",
            "  timestamp: 1561492319\n",
            "  timesteps_since_restore: 21000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 21000\n",
            "  training_iteration: 21\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'RUNNING': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1351], 409 s, 21 iter, 21000 ts, 97.7 rew\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-06-25 19:52:16,217\tINFO ray_trial_executor.py:187 -- Destroying actor for trial DQN_CartPole-v0_0_lr=0.01. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result for DQN_CartPole-v0_0_lr=0.01:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-52-16\n",
            "  done: true\n",
            "  episode_len_mean: 105.34\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 105.34\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 424\n",
            "  experiment_id: 6dd8564ee1634871b517d2893d4bb054\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.193\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.009999999776482582\n",
            "        max_q: 40.740264892578125\n",
            "        mean_q: 36.757164001464844\n",
            "        mean_td_error: 3.181473731994629\n",
            "        min_q: 11.437118530273438\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 22000\n",
            "    num_steps_trained: 168000\n",
            "    num_target_updates: 43\n",
            "    opt_peak_throughput: 2254.627\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.22\n",
            "    sample_time_ms: 16.0\n",
            "    update_time_ms: 13.907\n",
            "  iterations_since_restore: 22\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1351\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22664485708847493\n",
            "    mean_inference_ms: 2.3191346288962404\n",
            "    mean_processing_ms: 0.506270176683215\n",
            "  time_since_restore: 426.25695037841797\n",
            "  time_this_iter_s: 17.100685119628906\n",
            "  time_total_s: 426.25695037841797\n",
            "  timestamp: 1561492336\n",
            "  timesteps_since_restore: 22000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 22000\n",
            "  training_iteration: 22\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'PENDING': 2})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tPENDING\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m WARNING: Logging before flag parsing goes to stderr.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:28.410444 139827059848960 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m WARNING: Logging before flag parsing goes to stderr.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:28.810771 139959780681472 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:29,573\tINFO policy_evaluator.py:311 -- Creating policy evaluation worker 0 on CPU (please ignore any CUDA init errors)\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:29.585766: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:29.586056: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x6f24140 executing computations on platform Host. Devices:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:29.586097: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:29.598034 139827137337216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/models/fcnet.py:37: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Use keras.layers.dense instead.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:30.410632 139827137337216 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:30.538113 139827137337216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:663: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:30.553140 139827137337216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:30.572837 139827137337216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:337: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Use `tf.random.categorical` instead.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:32.203620: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:32,396\tINFO policy_evaluator.py:731 -- Built policy map: {'default_policy': <ray.rllib.agents.dqn.dqn_policy.DQNTFPolicy object at 0x7f2b8535eba8>}\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:32,396\tINFO policy_evaluator.py:732 -- Built preprocessor map: {'default_policy': <ray.rllib.models.preprocessors.NoPreprocessor object at 0x7f2b8535eac8>}\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:32,396\tINFO policy_evaluator.py:343 -- Built filter map: {'default_policy': <ray.rllib.utils.filter.NoFilter object at 0x7f2b8534a9b0>}\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m W0625 19:52:32.427026 139827137337216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/policy/tf_policy.py:509: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Prefer Variable.assign which has equivalent behavior in 2.X.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:32,447\tINFO policy_evaluator.py:311 -- Creating policy evaluation worker 1 on CPU (please ignore any CUDA init errors)\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:32.524994: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:32.525288: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x58ce140 executing computations on platform Host. Devices:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:32.525331: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:32.540523 139959858169728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/models/fcnet.py:37: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Use keras.layers.dense instead.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:33.961279 139959858169728 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:34.413255 139959858169728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:663: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:34.468911 139959858169728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:34.553624 139959858169728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:337: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Use `tf.random.categorical` instead.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:39.951203: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m W0625 19:52:40.166751 139959858169728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/policy/tf_policy.py:509: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m Prefer Variable.assign which has equivalent behavior in 2.X.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,305\tINFO policy_evaluator.py:437 -- Generating sample batch of size 4\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,306\tINFO sampler.py:308 -- Raw obs from env: { 0: { 'agent0': np.ndarray((4,), dtype=float64, min=-0.032, max=-0.002, mean=-0.018)}}\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,306\tINFO sampler.py:309 -- Info return from env: {0: {'agent0': None}}\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,306\tINFO sampler.py:407 -- Preprocessed obs: np.ndarray((4,), dtype=float64, min=-0.032, max=-0.002, mean=-0.018)\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,307\tINFO sampler.py:411 -- Filtered obs: np.ndarray((4,), dtype=float64, min=-0.032, max=-0.002, mean=-0.018)\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,308\tINFO sampler.py:525 -- Inputs to compute_actions():\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m { 'default_policy': [ { 'data': { 'agent_id': 'agent0',\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                                   'env_id': 0,\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                                   'info': None,\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                                   'obs': np.ndarray((4,), dtype=float64, min=-0.032, max=-0.002, mean=-0.018),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                                   'prev_action': np.ndarray((), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                                   'prev_reward': 0.0,\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                                   'rnn_state': []},\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'type': 'PolicyEvalData'}]}\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,308\tINFO tf_run_builder.py:92 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,370\tINFO sampler.py:552 -- Outputs of compute_actions():\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m { 'default_policy': ( np.ndarray((1,), dtype=int64, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                       [],\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                       { 'q_values': np.ndarray((1, 2), dtype=float32, min=0.028, max=0.039, mean=0.033)})}\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,378\tINFO sample_batch_builder.py:161 -- Trajectory fragment after postprocess_trajectory():\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m { 'agent0': { 'data': { 'actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'agent_index': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'dones': np.ndarray((4,), dtype=bool, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'eps_id': np.ndarray((4,), dtype=int64, min=1929675161.0, max=1929675161.0, mean=1929675161.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'infos': np.ndarray((4,), dtype=object, head={}),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'new_obs': np.ndarray((4, 4), dtype=float32, min=-0.61, max=0.389, mean=-0.049),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'obs': np.ndarray((4, 4), dtype=float32, min=-0.61, max=0.389, mean=-0.046),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'prev_actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'prev_rewards': np.ndarray((4,), dtype=float32, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'q_values': np.ndarray((4, 2), dtype=float32, min=-0.161, max=0.213, mean=0.014),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'rewards': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         't': np.ndarray((4,), dtype=int64, min=0.0, max=3.0, mean=1.5),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'unroll_id': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m                         'weights': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m               'type': 'SampleBatch'}}\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m 2019-06-25 19:52:40,380\tINFO policy_evaluator.py:474 -- Completed sample batch:\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m { 'data': { 'actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'agent_index': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'dones': np.ndarray((4,), dtype=bool, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'eps_id': np.ndarray((4,), dtype=int64, min=1929675161.0, max=1929675161.0, mean=1929675161.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'infos': np.ndarray((4,), dtype=object, head={}),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'new_obs': np.ndarray((4, 4), dtype=float32, min=-0.61, max=0.389, mean=-0.049),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'obs': np.ndarray((4, 4), dtype=float32, min=-0.61, max=0.389, mean=-0.046),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'prev_actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'prev_rewards': np.ndarray((4,), dtype=float32, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'q_values': np.ndarray((4, 2), dtype=float32, min=-0.161, max=0.213, mean=0.014),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'rewards': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             't': np.ndarray((4,), dtype=int64, min=0.0, max=3.0, mean=1.5),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'unroll_id': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m             'weights': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m   'type': 'SampleBatch'}\n",
            "\u001b[2m\u001b[36m(pid=1480)\u001b[0m \n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-52-53\n",
            "  done: false\n",
            "  episode_len_mean: 20.224489795918366\n",
            "  episode_reward_max: 45.0\n",
            "  episode_reward_mean: 20.224489795918366\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 49\n",
            "  episodes_total: 49\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: .nan\n",
            "    learner: {}\n",
            "    max_exploration: 1.0\n",
            "    min_exploration: 1.0\n",
            "    num_steps_sampled: 1000\n",
            "    num_steps_trained: 0\n",
            "    num_target_updates: 1\n",
            "    opt_peak_throughput: 0.0\n",
            "    opt_samples: .nan\n",
            "    replay_time_ms: .nan\n",
            "    sample_time_ms: 51.847\n",
            "    update_time_ms: 41.733\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22524815577488916\n",
            "    mean_inference_ms: 2.4427986526108167\n",
            "    mean_processing_ms: 0.4763884263319688\n",
            "  time_since_restore: 20.76395058631897\n",
            "  time_this_iter_s: 20.76395058631897\n",
            "  time_total_s: 20.76395058631897\n",
            "  timestamp: 1561492373\n",
            "  timesteps_since_restore: 1000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 1000\n",
            "  training_iteration: 1\n",
            "  \u001b[2m\u001b[36m(pid=1477)\u001b[0m /usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3118: RuntimeWarning: Mean of empty slice.\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m   out=out, **kwargs)\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m /usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m   ret = ret.dtype.type(ret / rcount)\n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 20 s, 1 iter, 1000 ts, 20.2 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:53,369\tINFO policy_evaluator.py:564 -- Training on concatenated sample batches:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m { 'count': 32,\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m   'policy_batches': { 'default_policy': { 'data': { 'actions': np.ndarray((32,), dtype=int64, min=0.0, max=1.0, mean=0.438),\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                                     'batch_indexes': np.ndarray((32,), dtype=int64, min=97.0, max=994.0, mean=523.5),\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                                     'dones': np.ndarray((32,), dtype=bool, min=0.0, max=1.0, mean=0.125),\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                                     'new_obs': np.ndarray((32, 4), dtype=float32, min=-1.741, max=1.11, mean=-0.03),\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                                     'obs': np.ndarray((32, 4), dtype=float32, min=-1.653, max=1.338, mean=-0.03),\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                                     'rewards': np.ndarray((32,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                                     'weights': np.ndarray((32,), dtype=float64, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                           'type': 'SampleBatch'}},\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m   'type': 'MultiAgentBatch'}\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:53,370\tINFO tf_run_builder.py:92 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2019-06-25 19:52:53,763\tINFO policy_evaluator.py:586 -- Training output:\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m { 'default_policy': { 'learner_stats': { 'cur_lr': 0.0010000000474974513,\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                          'max_q': 0.65825725,\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                          'mean_q': 0.10509057,\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                          'mean_td_error': -1.1052336,\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                          'min_q': -0.18940288,\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                                          'model': {}},\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m                       'td_error': np.ndarray((32,), dtype=float32, min=-1.69, max=-0.395, mean=-1.105)}}\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m \n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-53-11\n",
            "  done: false\n",
            "  episode_len_mean: 19.59\n",
            "  episode_reward_max: 60.0\n",
            "  episode_reward_mean: 19.59\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 53\n",
            "  episodes_total: 102\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.832\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 3.5823497772216797\n",
            "        mean_q: 2.6962220668792725\n",
            "        mean_td_error: -0.0963807925581932\n",
            "        min_q: 2.1035587787628174\n",
            "        model: {}\n",
            "    max_exploration: 0.902\n",
            "    min_exploration: 0.902\n",
            "    num_steps_sampled: 2000\n",
            "    num_steps_trained: 8000\n",
            "    num_target_updates: 3\n",
            "    opt_peak_throughput: 2157.529\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 4.85\n",
            "    sample_time_ms: 17.734\n",
            "    update_time_ms: 14.129\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22465074773156993\n",
            "    mean_inference_ms: 2.3858665365656724\n",
            "    mean_processing_ms: 0.4719299003037024\n",
            "  time_since_restore: 38.682934522628784\n",
            "  time_this_iter_s: 17.918983936309814\n",
            "  time_total_s: 38.682934522628784\n",
            "  timestamp: 1561492391\n",
            "  timesteps_since_restore: 2000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 2000\n",
            "  training_iteration: 2\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 38 s, 2 iter, 2000 ts, 19.6 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-53-29\n",
            "  done: false\n",
            "  episode_len_mean: 21.96\n",
            "  episode_reward_max: 79.0\n",
            "  episode_reward_mean: 21.96\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 36\n",
            "  episodes_total: 138\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 31.108\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 5.433751583099365\n",
            "        mean_q: 4.20675802230835\n",
            "        mean_td_error: 0.17087721824645996\n",
            "        min_q: 1.6126424074172974\n",
            "        model: {}\n",
            "    max_exploration: 0.804\n",
            "    min_exploration: 0.804\n",
            "    num_steps_sampled: 3000\n",
            "    num_steps_trained: 16000\n",
            "    num_target_updates: 5\n",
            "    opt_peak_throughput: 1028.684\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 13.57\n",
            "    sample_time_ms: 43.793\n",
            "    update_time_ms: 48.191\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2237253584430922\n",
            "    mean_inference_ms: 2.3395911162254293\n",
            "    mean_processing_ms: 0.47090866199726267\n",
            "  time_since_restore: 57.151427268981934\n",
            "  time_this_iter_s: 18.46849274635315\n",
            "  time_total_s: 57.151427268981934\n",
            "  timestamp: 1561492409\n",
            "  timesteps_since_restore: 3000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 3000\n",
            "  training_iteration: 3\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 57 s, 3 iter, 3000 ts, 22 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-53-50\n",
            "  done: false\n",
            "  episode_len_mean: 27.41\n",
            "  episode_reward_max: 124.0\n",
            "  episode_reward_mean: 27.41\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 19\n",
            "  episodes_total: 157\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.728\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 6.9576263427734375\n",
            "        mean_q: 5.0213623046875\n",
            "        mean_td_error: 0.19702567160129547\n",
            "        min_q: 1.1526267528533936\n",
            "        model: {}\n",
            "    max_exploration: 0.706\n",
            "    min_exploration: 0.706\n",
            "    num_steps_sampled: 4000\n",
            "    num_steps_trained: 24000\n",
            "    num_target_updates: 7\n",
            "    opt_peak_throughput: 2514.057\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.688\n",
            "    sample_time_ms: 19.225\n",
            "    update_time_ms: 13.737\n",
            "  iterations_since_restore: 4\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22491233752931603\n",
            "    mean_inference_ms: 2.332001655203279\n",
            "    mean_processing_ms: 0.47239714450113085\n",
            "  time_since_restore: 77.48808932304382\n",
            "  time_this_iter_s: 20.33666205406189\n",
            "  time_total_s: 77.48808932304382\n",
            "  timestamp: 1561492430\n",
            "  timesteps_since_restore: 4000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 4000\n",
            "  training_iteration: 4\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 77 s, 4 iter, 4000 ts, 27.4 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-54-07\n",
            "  done: false\n",
            "  episode_len_mean: 34.95\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 34.95\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 14\n",
            "  episodes_total: 171\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.595\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 9.489399909973145\n",
            "        mean_q: 7.497003555297852\n",
            "        mean_td_error: 0.1075868308544159\n",
            "        min_q: 0.36846432089805603\n",
            "        model: {}\n",
            "    max_exploration: 0.608\n",
            "    min_exploration: 0.608\n",
            "    num_steps_sampled: 5000\n",
            "    num_steps_trained: 32000\n",
            "    num_target_updates: 9\n",
            "    opt_peak_throughput: 2192.552\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 4.837\n",
            "    sample_time_ms: 17.964\n",
            "    update_time_ms: 13.311\n",
            "  iterations_since_restore: 5\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22434878945572068\n",
            "    mean_inference_ms: 2.329001895681448\n",
            "    mean_processing_ms: 0.47252838795332136\n",
            "  time_since_restore: 94.78808426856995\n",
            "  time_this_iter_s: 17.299994945526123\n",
            "  time_total_s: 94.78808426856995\n",
            "  timestamp: 1561492447\n",
            "  timesteps_since_restore: 5000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 5000\n",
            "  training_iteration: 5\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 94 s, 5 iter, 5000 ts, 35 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-54-26\n",
            "  done: false\n",
            "  episode_len_mean: 43.46\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 43.46\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 11\n",
            "  episodes_total: 182\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 28.683\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 11.2942533493042\n",
            "        mean_q: 9.172256469726562\n",
            "        mean_td_error: 0.1438504308462143\n",
            "        min_q: 0.9475095272064209\n",
            "        model: {}\n",
            "    max_exploration: 0.51\n",
            "    min_exploration: 0.51\n",
            "    num_steps_sampled: 6000\n",
            "    num_steps_trained: 40000\n",
            "    num_target_updates: 11\n",
            "    opt_peak_throughput: 1115.624\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 12.732\n",
            "    sample_time_ms: 53.233\n",
            "    update_time_ms: 42.809\n",
            "  iterations_since_restore: 6\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22391784963905245\n",
            "    mean_inference_ms: 2.3263084866035\n",
            "    mean_processing_ms: 0.4731359951196545\n",
            "  time_since_restore: 113.83148384094238\n",
            "  time_this_iter_s: 19.043399572372437\n",
            "  time_total_s: 113.83148384094238\n",
            "  timestamp: 1561492466\n",
            "  timesteps_since_restore: 6000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 6000\n",
            "  training_iteration: 6\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 113 s, 6 iter, 6000 ts, 43.5 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-54-46\n",
            "  done: false\n",
            "  episode_len_mean: 51.49\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 51.49\n",
            "  episode_reward_min: 11.0\n",
            "  episodes_this_iter: 7\n",
            "  episodes_total: 189\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 11.742\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 13.049267768859863\n",
            "        mean_q: 11.475282669067383\n",
            "        mean_td_error: 0.17313659191131592\n",
            "        min_q: 6.212714672088623\n",
            "        model: {}\n",
            "    max_exploration: 0.41200000000000003\n",
            "    min_exploration: 0.41200000000000003\n",
            "    num_steps_sampled: 7000\n",
            "    num_steps_trained: 48000\n",
            "    num_target_updates: 13\n",
            "    opt_peak_throughput: 2725.156\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.516\n",
            "    sample_time_ms: 13.445\n",
            "    update_time_ms: 19.604\n",
            "  iterations_since_restore: 7\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22400143101662814\n",
            "    mean_inference_ms: 2.327450421728966\n",
            "    mean_processing_ms: 0.47406544108751875\n",
            "  time_since_restore: 134.00549340248108\n",
            "  time_this_iter_s: 20.174009561538696\n",
            "  time_total_s: 134.00549340248108\n",
            "  timestamp: 1561492486\n",
            "  timesteps_since_restore: 7000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 7000\n",
            "  training_iteration: 7\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 134 s, 7 iter, 7000 ts, 51.5 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-55-04\n",
            "  done: false\n",
            "  episode_len_mean: 60.5\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 60.5\n",
            "  episode_reward_min: 12.0\n",
            "  episodes_this_iter: 9\n",
            "  episodes_total: 198\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.74\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 15.730767250061035\n",
            "        mean_q: 11.998941421508789\n",
            "        mean_td_error: 0.8489376306533813\n",
            "        min_q: 3.0820980072021484\n",
            "        model: {}\n",
            "    max_exploration: 0.31400000000000006\n",
            "    min_exploration: 0.31400000000000006\n",
            "    num_steps_sampled: 8000\n",
            "    num_steps_trained: 56000\n",
            "    num_target_updates: 15\n",
            "    opt_peak_throughput: 2170.986\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.313\n",
            "    sample_time_ms: 18.261\n",
            "    update_time_ms: 15.266\n",
            "  iterations_since_restore: 8\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2235127187746346\n",
            "    mean_inference_ms: 2.326163154163003\n",
            "    mean_processing_ms: 0.4749435469115014\n",
            "  time_since_restore: 151.43413734436035\n",
            "  time_this_iter_s: 17.428643941879272\n",
            "  time_total_s: 151.43413734436035\n",
            "  timestamp: 1561492504\n",
            "  timesteps_since_restore: 8000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 8000\n",
            "  training_iteration: 8\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 151 s, 8 iter, 8000 ts, 60.5 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-55-25\n",
            "  done: false\n",
            "  episode_len_mean: 68.02\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 68.02\n",
            "  episode_reward_min: 12.0\n",
            "  episodes_this_iter: 9\n",
            "  episodes_total: 207\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 27.863\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 17.070598602294922\n",
            "        mean_q: 14.220478057861328\n",
            "        mean_td_error: -0.07086397707462311\n",
            "        min_q: -0.2378854751586914\n",
            "        model: {}\n",
            "    max_exploration: 0.21599999999999997\n",
            "    min_exploration: 0.21599999999999997\n",
            "    num_steps_sampled: 9000\n",
            "    num_steps_trained: 64000\n",
            "    num_target_updates: 17\n",
            "    opt_peak_throughput: 1148.486\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 15.434\n",
            "    sample_time_ms: 54.064\n",
            "    update_time_ms: 51.473\n",
            "  iterations_since_restore: 9\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22352062153007116\n",
            "    mean_inference_ms: 2.3285916194335106\n",
            "    mean_processing_ms: 0.4766942218445135\n",
            "  time_since_restore: 172.37276458740234\n",
            "  time_this_iter_s: 20.938627243041992\n",
            "  time_total_s: 172.37276458740234\n",
            "  timestamp: 1561492525\n",
            "  timesteps_since_restore: 9000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 9000\n",
            "  training_iteration: 9\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 172 s, 9 iter, 9000 ts, 68 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-55-42\n",
            "  done: false\n",
            "  episode_len_mean: 76.79\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 76.79\n",
            "  episode_reward_min: 12.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 213\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 13.751\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 19.384471893310547\n",
            "        mean_q: 15.10909366607666\n",
            "        mean_td_error: 1.1014864444732666\n",
            "        min_q: 0.19603198766708374\n",
            "        model: {}\n",
            "    max_exploration: 0.118\n",
            "    min_exploration: 0.118\n",
            "    num_steps_sampled: 10000\n",
            "    num_steps_trained: 72000\n",
            "    num_target_updates: 19\n",
            "    opt_peak_throughput: 2327.082\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.687\n",
            "    sample_time_ms: 19.026\n",
            "    update_time_ms: 13.832\n",
            "  iterations_since_restore: 10\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.22342514054802132\n",
            "    mean_inference_ms: 2.329320456446208\n",
            "    mean_processing_ms: 0.4775101109325773\n",
            "  time_since_restore: 190.10449481010437\n",
            "  time_this_iter_s: 17.731730222702026\n",
            "  time_total_s: 190.10449481010437\n",
            "  timestamp: 1561492542\n",
            "  timesteps_since_restore: 10000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 10000\n",
            "  training_iteration: 10\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 190 s, 10 iter, 10000 ts, 76.8 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-55-59\n",
            "  done: false\n",
            "  episode_len_mean: 85.35\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 85.35\n",
            "  episode_reward_min: 12.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 218\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 13.099\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 20.676137924194336\n",
            "        mean_q: 15.825045585632324\n",
            "        mean_td_error: 1.187561273574829\n",
            "        min_q: 8.837031364440918\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 11000\n",
            "    num_steps_trained: 80000\n",
            "    num_target_updates: 21\n",
            "    opt_peak_throughput: 2443.014\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.465\n",
            "    sample_time_ms: 17.459\n",
            "    update_time_ms: 14.526\n",
            "  iterations_since_restore: 11\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2231879752942026\n",
            "    mean_inference_ms: 2.3283466067222163\n",
            "    mean_processing_ms: 0.47805628393717925\n",
            "  time_since_restore: 206.96319389343262\n",
            "  time_this_iter_s: 16.858699083328247\n",
            "  time_total_s: 206.96319389343262\n",
            "  timestamp: 1561492559\n",
            "  timesteps_since_restore: 11000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 11000\n",
            "  training_iteration: 11\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 206 s, 11 iter, 11000 ts, 85.3 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-56-20\n",
            "  done: false\n",
            "  episode_len_mean: 94.42\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 94.42\n",
            "  episode_reward_min: 12.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 224\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 29.91\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 22.205656051635742\n",
            "        mean_q: 16.45809555053711\n",
            "        mean_td_error: 0.25076040625572205\n",
            "        min_q: -0.5990676879882812\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 12000\n",
            "    num_steps_trained: 88000\n",
            "    num_target_updates: 23\n",
            "    opt_peak_throughput: 1069.888\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 14.216\n",
            "    sample_time_ms: 46.072\n",
            "    update_time_ms: 46.708\n",
            "  iterations_since_restore: 12\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2229798006905996\n",
            "    mean_inference_ms: 2.3284880726103894\n",
            "    mean_processing_ms: 0.47902753657325464\n",
            "  time_since_restore: 227.79639101028442\n",
            "  time_this_iter_s: 20.833197116851807\n",
            "  time_total_s: 227.79639101028442\n",
            "  timestamp: 1561492580\n",
            "  timesteps_since_restore: 12000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 12000\n",
            "  training_iteration: 12\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 1, 'RUNNING': 1, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1477], 227 s, 12 iter, 12000 ts, 94.4 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-06-25 19:56:37,699\tINFO ray_trial_executor.py:187 -- Destroying actor for trial DQN_CartPole-v0_1_lr=0.001. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result for DQN_CartPole-v0_1_lr=0.001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-56-37\n",
            "  done: true\n",
            "  episode_len_mean: 101.62\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 101.62\n",
            "  episode_reward_min: 12.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 229\n",
            "  experiment_id: 0e448c35d47f480abb7df1669ee7a6e3\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.525\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 0.0010000000474974513\n",
            "        max_q: 23.366138458251953\n",
            "        mean_q: 18.686616897583008\n",
            "        mean_td_error: 0.29401248693466187\n",
            "        min_q: 1.5325549840927124\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 13000\n",
            "    num_steps_trained: 96000\n",
            "    num_target_updates: 25\n",
            "    opt_peak_throughput: 2554.883\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.296\n",
            "    sample_time_ms: 17.463\n",
            "    update_time_ms: 13.157\n",
            "  iterations_since_restore: 13\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1477\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2227291313273063\n",
            "    mean_inference_ms: 2.3272408465797056\n",
            "    mean_processing_ms: 0.4795320740654556\n",
            "  time_since_restore: 244.81317853927612\n",
            "  time_this_iter_s: 17.0167875289917\n",
            "  time_total_s: 244.81317853927612\n",
            "  timestamp: 1561492597\n",
            "  timesteps_since_restore: 13000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 13000\n",
            "  training_iteration: 13\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'PENDING': 1})\n",
            "PENDING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tPENDING\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m WARNING: Logging before flag parsing goes to stderr.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:47.863310 140407442122496 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:50,274\tINFO policy_evaluator.py:311 -- Creating policy evaluation worker 0 on CPU (please ignore any CUDA init errors)\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:50.301204: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:50.301521: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x547c140 executing computations on platform Host. Devices:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:50.301563: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:50.313474 140407519610752 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/models/fcnet.py:37: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Use keras.layers.dense instead.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m WARNING: Logging before flag parsing goes to stderr.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:51.260465 140221811672832 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m non-resource variables are not supported in the long term\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:51.313372 140407519610752 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:51.480411 140407519610752 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:663: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:51.492747 140407519610752 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:51.520086 140407519610752 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:337: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Use `tf.random.categorical` instead.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:53.622441: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:53,828\tINFO policy_evaluator.py:731 -- Built policy map: {'default_policy': <ray.rllib.agents.dqn.dqn_policy.DQNTFPolicy object at 0x7fb2a6ae0b38>}\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:53,829\tINFO policy_evaluator.py:732 -- Built preprocessor map: {'default_policy': <ray.rllib.models.preprocessors.NoPreprocessor object at 0x7fb2a6ae0a58>}\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:56:53,829\tINFO policy_evaluator.py:343 -- Built filter map: {'default_policy': <ray.rllib.utils.filter.NoFilter object at 0x7fb2a6aca940>}\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m W0625 19:56:53.866215 140407519610752 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/policy/tf_policy.py:509: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m Prefer Variable.assign which has equivalent behavior in 2.X.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:53,875\tINFO policy_evaluator.py:311 -- Creating policy evaluation worker 1 on CPU (please ignore any CUDA init errors)\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:53.943009: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:53.943312: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x6f10140 executing computations on platform Host. Devices:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:53.943350: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:53.956141 140221889161088 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/models/fcnet.py:37: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Use keras.layers.dense instead.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:54.875648 140221889161088 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:54.999589 140221889161088 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:663: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:55.014354 140221889161088 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/util/decorator_utils.py:145: GraphKeys.VARIABLES (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Use `tf.GraphKeys.GLOBAL_VARIABLES` instead.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:55.033025 140221889161088 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/agents/dqn/dqn_policy.py:337: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Use `tf.random.categorical` instead.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:58.501487: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m W0625 19:56:59.239954 140221889161088 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/ray/rllib/policy/tf_policy.py:509: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Instructions for updating:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m Prefer Variable.assign which has equivalent behavior in 2.X.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,783\tINFO policy_evaluator.py:437 -- Generating sample batch of size 4\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,783\tINFO sampler.py:308 -- Raw obs from env: { 0: { 'agent0': np.ndarray((4,), dtype=float64, min=-0.012, max=0.029, mean=0.01)}}\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,784\tINFO sampler.py:309 -- Info return from env: {0: {'agent0': None}}\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,784\tINFO sampler.py:407 -- Preprocessed obs: np.ndarray((4,), dtype=float64, min=-0.012, max=0.029, mean=0.01)\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,784\tINFO sampler.py:411 -- Filtered obs: np.ndarray((4,), dtype=float64, min=-0.012, max=0.029, mean=0.01)\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,790\tINFO sampler.py:525 -- Inputs to compute_actions():\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m { 'default_policy': [ { 'data': { 'agent_id': 'agent0',\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                                   'env_id': 0,\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                                   'info': None,\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                                   'obs': np.ndarray((4,), dtype=float64, min=-0.012, max=0.029, mean=0.01),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                                   'prev_action': np.ndarray((), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                                   'prev_reward': 0.0,\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                                   'rnn_state': []},\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'type': 'PolicyEvalData'}]}\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:56:59,790\tINFO tf_run_builder.py:92 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:57:00,030\tINFO sampler.py:552 -- Outputs of compute_actions():\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m { 'default_policy': ( np.ndarray((1,), dtype=int64, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                       [],\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                       { 'q_values': np.ndarray((1, 2), dtype=float32, min=-0.005, max=0.041, mean=0.018)})}\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:57:00,059\tINFO sample_batch_builder.py:161 -- Trajectory fragment after postprocess_trajectory():\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m { 'agent0': { 'data': { 'actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'agent_index': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'dones': np.ndarray((4,), dtype=bool, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'eps_id': np.ndarray((4,), dtype=int64, min=1320783969.0, max=1320783969.0, mean=1320783969.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'infos': np.ndarray((4,), dtype=object, head={}),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'new_obs': np.ndarray((4, 4), dtype=float32, min=-0.28, max=0.312, mean=0.012),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'obs': np.ndarray((4, 4), dtype=float32, min=-0.28, max=0.312, mean=0.011),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'prev_actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.25),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'prev_rewards': np.ndarray((4,), dtype=float32, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'q_values': np.ndarray((4, 2), dtype=float32, min=-0.298, max=0.119, mean=-0.007),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'rewards': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         't': np.ndarray((4,), dtype=int64, min=0.0, max=3.0, mean=1.5),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'unroll_id': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m                         'weights': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m               'type': 'SampleBatch'}}\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m 2019-06-25 19:57:00,065\tINFO policy_evaluator.py:474 -- Completed sample batch:\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m { 'data': { 'actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.5),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'agent_index': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'dones': np.ndarray((4,), dtype=bool, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'eps_id': np.ndarray((4,), dtype=int64, min=1320783969.0, max=1320783969.0, mean=1320783969.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'infos': np.ndarray((4,), dtype=object, head={}),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'new_obs': np.ndarray((4, 4), dtype=float32, min=-0.28, max=0.312, mean=0.012),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'obs': np.ndarray((4, 4), dtype=float32, min=-0.28, max=0.312, mean=0.011),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'prev_actions': np.ndarray((4,), dtype=int64, min=0.0, max=1.0, mean=0.25),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'prev_rewards': np.ndarray((4,), dtype=float32, min=0.0, max=1.0, mean=0.75),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'q_values': np.ndarray((4, 2), dtype=float32, min=-0.298, max=0.119, mean=-0.007),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'rewards': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             't': np.ndarray((4,), dtype=int64, min=0.0, max=3.0, mean=1.5),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'unroll_id': np.ndarray((4,), dtype=int64, min=0.0, max=0.0, mean=0.0),\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m             'weights': np.ndarray((4,), dtype=float32, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m   'type': 'SampleBatch'}\n",
            "\u001b[2m\u001b[36m(pid=1532)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m /usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3118: RuntimeWarning: Mean of empty slice.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m   out=out, **kwargs)\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m /usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m   ret = ret.dtype.type(ret / rcount)\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-57-10\n",
            "  done: false\n",
            "  episode_len_mean: 23.61904761904762\n",
            "  episode_reward_max: 65.0\n",
            "  episode_reward_mean: 23.61904761904762\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 42\n",
            "  episodes_total: 42\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: .nan\n",
            "    learner: {}\n",
            "    max_exploration: 1.0\n",
            "    min_exploration: 1.0\n",
            "    num_steps_sampled: 1000\n",
            "    num_steps_trained: 0\n",
            "    num_target_updates: 1\n",
            "    opt_peak_throughput: 0.0\n",
            "    opt_samples: .nan\n",
            "    replay_time_ms: .nan\n",
            "    sample_time_ms: 52.548\n",
            "    update_time_ms: 41.494\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.1673948514711607\n",
            "    mean_inference_ms: 2.0979017644495404\n",
            "    mean_processing_ms: 0.4023381403752497\n",
            "  time_since_restore: 16.779335021972656\n",
            "  time_this_iter_s: 16.779335021972656\n",
            "  time_total_s: 16.779335021972656\n",
            "  timestamp: 1561492630\n",
            "  timesteps_since_restore: 1000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 1000\n",
            "  training_iteration: 1\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 16 s, 1 iter, 1000 ts, 23.6 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:57:10,971\tINFO policy_evaluator.py:564 -- Training on concatenated sample batches:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m { 'count': 32,\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m   'policy_batches': { 'default_policy': { 'data': { 'actions': np.ndarray((32,), dtype=int64, min=0.0, max=1.0, mean=0.562),\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                                     'batch_indexes': np.ndarray((32,), dtype=int64, min=15.0, max=905.0, mean=443.062),\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                                     'dones': np.ndarray((32,), dtype=bool, min=0.0, max=1.0, mean=0.031),\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                                     'new_obs': np.ndarray((32, 4), dtype=float32, min=-1.776, max=1.81, mean=0.014),\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                                     'obs': np.ndarray((32, 4), dtype=float32, min=-1.46, max=1.466, mean=0.016),\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                                     'rewards': np.ndarray((32,), dtype=float32, min=1.0, max=1.0, mean=1.0),\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                                     'weights': np.ndarray((32,), dtype=float64, min=1.0, max=1.0, mean=1.0)},\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                           'type': 'SampleBatch'}},\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m   'type': 'MultiAgentBatch'}\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:57:10,971\tINFO tf_run_builder.py:92 -- Executing TF run without tracing. To dump TF timeline traces to disk, set the TF_TIMELINE_DIR environment variable.\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m 2019-06-25 19:57:12,392\tINFO policy_evaluator.py:586 -- Training output:\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m \n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m { 'default_policy': { 'learner_stats': { 'cur_lr': 9.999999747378752e-05,\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                          'max_q': 0.3494134,\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                          'mean_q': -0.008002334,\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                          'mean_td_error': -1.1162376,\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                          'min_q': -0.95385194,\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                                          'model': {}},\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m                       'td_error': np.ndarray((32,), dtype=float32, min=-1.954, max=-0.885, mean=-1.116)}}\n",
            "\u001b[2m\u001b[36m(pid=1528)\u001b[0m \n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-57-31\n",
            "  done: false\n",
            "  episode_len_mean: 21.813186813186814\n",
            "  episode_reward_max: 65.0\n",
            "  episode_reward_mean: 21.813186813186814\n",
            "  episode_reward_min: 8.0\n",
            "  episodes_this_iter: 49\n",
            "  episodes_total: 91\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.809\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 2.80509352684021\n",
            "        mean_q: 1.5948587656021118\n",
            "        mean_td_error: -0.37133386731147766\n",
            "        min_q: 0.304401695728302\n",
            "        model: {}\n",
            "    max_exploration: 0.902\n",
            "    min_exploration: 0.902\n",
            "    num_steps_sampled: 2000\n",
            "    num_steps_trained: 8000\n",
            "    num_target_updates: 3\n",
            "    opt_peak_throughput: 2160.843\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.025\n",
            "    sample_time_ms: 16.403\n",
            "    update_time_ms: 16.563\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.18919026476909298\n",
            "    mean_inference_ms: 2.1498738770631625\n",
            "    mean_processing_ms: 0.43883316363879565\n",
            "  time_since_restore: 37.77490758895874\n",
            "  time_this_iter_s: 20.995572566986084\n",
            "  time_total_s: 37.77490758895874\n",
            "  timestamp: 1561492651\n",
            "  timesteps_since_restore: 2000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 2000\n",
            "  training_iteration: 2\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 37 s, 2 iter, 2000 ts, 21.8 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-57-48\n",
            "  done: false\n",
            "  episode_len_mean: 19.63\n",
            "  episode_reward_max: 70.0\n",
            "  episode_reward_mean: 19.63\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 53\n",
            "  episodes_total: 144\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.023\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 5.438018321990967\n",
            "        mean_q: 3.7796945571899414\n",
            "        mean_td_error: 0.2288440763950348\n",
            "        min_q: 2.7328860759735107\n",
            "        model: {}\n",
            "    max_exploration: 0.804\n",
            "    min_exploration: 0.804\n",
            "    num_steps_sampled: 3000\n",
            "    num_steps_trained: 16000\n",
            "    num_target_updates: 5\n",
            "    opt_peak_throughput: 2661.503\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.862\n",
            "    sample_time_ms: 14.545\n",
            "    update_time_ms: 16.654\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.19905376923225127\n",
            "    mean_inference_ms: 2.1437728089946844\n",
            "    mean_processing_ms: 0.4572294260616122\n",
            "  time_since_restore: 54.637760162353516\n",
            "  time_this_iter_s: 16.862852573394775\n",
            "  time_total_s: 54.637760162353516\n",
            "  timestamp: 1561492668\n",
            "  timesteps_since_restore: 3000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 3000\n",
            "  training_iteration: 3\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 54 s, 3 iter, 3000 ts, 19.6 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-58-09\n",
            "  done: false\n",
            "  episode_len_mean: 20.67\n",
            "  episode_reward_max: 70.0\n",
            "  episode_reward_mean: 20.67\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 41\n",
            "  episodes_total: 185\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 30.081\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 6.959627151489258\n",
            "        mean_q: 5.477382183074951\n",
            "        mean_td_error: 0.24937304854393005\n",
            "        min_q: 3.6120212078094482\n",
            "        model: {}\n",
            "    max_exploration: 0.706\n",
            "    min_exploration: 0.706\n",
            "    num_steps_sampled: 4000\n",
            "    num_steps_trained: 24000\n",
            "    num_target_updates: 7\n",
            "    opt_peak_throughput: 1063.81\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 16.83\n",
            "    sample_time_ms: 46.889\n",
            "    update_time_ms: 48.571\n",
            "  iterations_since_restore: 4\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.19799667573005156\n",
            "    mean_inference_ms: 2.1444988748419425\n",
            "    mean_processing_ms: 0.46063875629303086\n",
            "  time_since_restore: 75.24170470237732\n",
            "  time_this_iter_s: 20.603944540023804\n",
            "  time_total_s: 75.24170470237732\n",
            "  timestamp: 1561492689\n",
            "  timesteps_since_restore: 4000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 4000\n",
            "  training_iteration: 4\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 75 s, 4 iter, 4000 ts, 20.7 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-58-27\n",
            "  done: false\n",
            "  episode_len_mean: 27.83\n",
            "  episode_reward_max: 129.0\n",
            "  episode_reward_mean: 27.83\n",
            "  episode_reward_min: 9.0\n",
            "  episodes_this_iter: 17\n",
            "  episodes_total: 202\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 15.693\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 9.254096984863281\n",
            "        mean_q: 7.305184364318848\n",
            "        mean_td_error: 0.3624095916748047\n",
            "        min_q: 4.243045806884766\n",
            "        model: {}\n",
            "    max_exploration: 0.608\n",
            "    min_exploration: 0.608\n",
            "    num_steps_sampled: 5000\n",
            "    num_steps_trained: 32000\n",
            "    num_target_updates: 9\n",
            "    opt_peak_throughput: 2039.16\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 4.502\n",
            "    sample_time_ms: 18.402\n",
            "    update_time_ms: 13.957\n",
            "  iterations_since_restore: 5\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.19973458361098306\n",
            "    mean_inference_ms: 2.1531601802578906\n",
            "    mean_processing_ms: 0.4645465230986466\n",
            "  time_since_restore: 93.35607981681824\n",
            "  time_this_iter_s: 18.114375114440918\n",
            "  time_total_s: 93.35607981681824\n",
            "  timestamp: 1561492707\n",
            "  timesteps_since_restore: 5000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 5000\n",
            "  training_iteration: 5\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 93 s, 5 iter, 5000 ts, 27.8 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-58-44\n",
            "  done: false\n",
            "  episode_len_mean: 34.9\n",
            "  episode_reward_max: 129.0\n",
            "  episode_reward_mean: 34.9\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 14\n",
            "  episodes_total: 216\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 16.49\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 11.73971939086914\n",
            "        mean_q: 9.087392807006836\n",
            "        mean_td_error: 0.49879103899002075\n",
            "        min_q: 4.547214508056641\n",
            "        model: {}\n",
            "    max_exploration: 0.51\n",
            "    min_exploration: 0.51\n",
            "    num_steps_sampled: 6000\n",
            "    num_steps_trained: 40000\n",
            "    num_target_updates: 11\n",
            "    opt_peak_throughput: 1940.586\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 4.95\n",
            "    sample_time_ms: 18.62\n",
            "    update_time_ms: 13.889\n",
            "  iterations_since_restore: 6\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.20178541298844144\n",
            "    mean_inference_ms: 2.1625280715305113\n",
            "    mean_processing_ms: 0.46823731328822016\n",
            "  time_since_restore: 110.60806632041931\n",
            "  time_this_iter_s: 17.251986503601074\n",
            "  time_total_s: 110.60806632041931\n",
            "  timestamp: 1561492724\n",
            "  timesteps_since_restore: 6000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 6000\n",
            "  training_iteration: 6\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 110 s, 6 iter, 6000 ts, 34.9 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-59-05\n",
            "  done: false\n",
            "  episode_len_mean: 42.98\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 42.98\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 8\n",
            "  episodes_total: 224\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 29.297\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 12.842874526977539\n",
            "        mean_q: 11.095352172851562\n",
            "        mean_td_error: 1.435236930847168\n",
            "        min_q: 6.04903507232666\n",
            "        model: {}\n",
            "    max_exploration: 0.41200000000000003\n",
            "    min_exploration: 0.41200000000000003\n",
            "    num_steps_sampled: 7000\n",
            "    num_steps_trained: 48000\n",
            "    num_target_updates: 13\n",
            "    opt_peak_throughput: 1092.271\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 14.019\n",
            "    sample_time_ms: 47.03\n",
            "    update_time_ms: 51.769\n",
            "  iterations_since_restore: 7\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2039082325073965\n",
            "    mean_inference_ms: 2.1714319314757713\n",
            "    mean_processing_ms: 0.4710043440186222\n",
            "  time_since_restore: 131.62126111984253\n",
            "  time_this_iter_s: 21.013194799423218\n",
            "  time_total_s: 131.62126111984253\n",
            "  timestamp: 1561492745\n",
            "  timesteps_since_restore: 7000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 7000\n",
            "  training_iteration: 7\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 131 s, 7 iter, 7000 ts, 43 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-59-23\n",
            "  done: false\n",
            "  episode_len_mean: 51.16\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 51.16\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 7\n",
            "  episodes_total: 231\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.979\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 14.996505737304688\n",
            "        mean_q: 12.30805778503418\n",
            "        mean_td_error: 0.9493804574012756\n",
            "        min_q: 2.624488353729248\n",
            "        model: {}\n",
            "    max_exploration: 0.31400000000000006\n",
            "    min_exploration: 0.31400000000000006\n",
            "    num_steps_sampled: 8000\n",
            "    num_steps_trained: 56000\n",
            "    num_target_updates: 15\n",
            "    opt_peak_throughput: 2465.515\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.47\n",
            "    sample_time_ms: 17.739\n",
            "    update_time_ms: 14.576\n",
            "  iterations_since_restore: 8\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.2056474091037645\n",
            "    mean_inference_ms: 2.1785789384329513\n",
            "    mean_processing_ms: 0.47295435502116917\n",
            "  time_since_restore: 149.3323473930359\n",
            "  time_this_iter_s: 17.71108627319336\n",
            "  time_total_s: 149.3323473930359\n",
            "  timestamp: 1561492763\n",
            "  timesteps_since_restore: 8000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 8000\n",
            "  training_iteration: 8\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 149 s, 8 iter, 8000 ts, 51.2 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_19-59-40\n",
            "  done: false\n",
            "  episode_len_mean: 61.24\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 61.24\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 7\n",
            "  episodes_total: 238\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 14.849\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 17.52777671813965\n",
            "        mean_q: 14.724264144897461\n",
            "        mean_td_error: 2.0154964923858643\n",
            "        min_q: 6.840526103973389\n",
            "        model: {}\n",
            "    max_exploration: 0.21599999999999997\n",
            "    min_exploration: 0.21599999999999997\n",
            "    num_steps_sampled: 9000\n",
            "    num_steps_trained: 64000\n",
            "    num_target_updates: 17\n",
            "    opt_peak_throughput: 2155.073\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.608\n",
            "    sample_time_ms: 19.331\n",
            "    update_time_ms: 13.409\n",
            "  iterations_since_restore: 9\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.20718070949933348\n",
            "    mean_inference_ms: 2.184206945665524\n",
            "    mean_processing_ms: 0.4745656719742625\n",
            "  time_since_restore: 166.6489224433899\n",
            "  time_this_iter_s: 17.316575050354004\n",
            "  time_total_s: 166.6489224433899\n",
            "  timestamp: 1561492780\n",
            "  timesteps_since_restore: 9000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 9000\n",
            "  training_iteration: 9\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 166 s, 9 iter, 9000 ts, 61.2 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_20-00-02\n",
            "  done: false\n",
            "  episode_len_mean: 68.22\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 68.22\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 7\n",
            "  episodes_total: 245\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 26.995\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 19.054399490356445\n",
            "        mean_q: 15.726737022399902\n",
            "        mean_td_error: 0.0757984071969986\n",
            "        min_q: 3.500086784362793\n",
            "        model: {}\n",
            "    max_exploration: 0.118\n",
            "    min_exploration: 0.118\n",
            "    num_steps_sampled: 10000\n",
            "    num_steps_trained: 72000\n",
            "    num_target_updates: 19\n",
            "    opt_peak_throughput: 1185.394\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 14.94\n",
            "    sample_time_ms: 45.825\n",
            "    update_time_ms: 41.457\n",
            "  iterations_since_restore: 10\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.20895519487772155\n",
            "    mean_inference_ms: 2.192039539225627\n",
            "    mean_processing_ms: 0.47690937123060706\n",
            "  time_since_restore: 188.01024866104126\n",
            "  time_this_iter_s: 21.361326217651367\n",
            "  time_total_s: 188.01024866104126\n",
            "  timestamp: 1561492802\n",
            "  timesteps_since_restore: 10000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 10000\n",
            "  training_iteration: 10\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 188 s, 10 iter, 10000 ts, 68.2 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_20-00-19\n",
            "  done: false\n",
            "  episode_len_mean: 77.54\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 77.54\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 6\n",
            "  episodes_total: 251\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.57\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 20.904531478881836\n",
            "        mean_q: 17.812469482421875\n",
            "        mean_td_error: 0.7284952402114868\n",
            "        min_q: 5.501935958862305\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 11000\n",
            "    num_steps_trained: 80000\n",
            "    num_target_updates: 21\n",
            "    opt_peak_throughput: 2545.657\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 6.065\n",
            "    sample_time_ms: 15.628\n",
            "    update_time_ms: 16.939\n",
            "  iterations_since_restore: 11\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.20960694377635342\n",
            "    mean_inference_ms: 2.1927057837513435\n",
            "    mean_processing_ms: 0.47675229511756373\n",
            "  time_since_restore: 204.95941853523254\n",
            "  time_this_iter_s: 16.949169874191284\n",
            "  time_total_s: 204.95941853523254\n",
            "  timestamp: 1561492819\n",
            "  timesteps_since_restore: 11000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 11000\n",
            "  training_iteration: 11\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 204 s, 11 iter, 11000 ts, 77.5 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_20-00-36\n",
            "  done: false\n",
            "  episode_len_mean: 86.61\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 86.61\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 7\n",
            "  episodes_total: 258\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.725\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 23.473066329956055\n",
            "        mean_q: 20.113628387451172\n",
            "        mean_td_error: 0.5041059255599976\n",
            "        min_q: 8.765495300292969\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 12000\n",
            "    num_steps_trained: 88000\n",
            "    num_target_updates: 23\n",
            "    opt_peak_throughput: 2514.731\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.116\n",
            "    sample_time_ms: 11.853\n",
            "    update_time_ms: 17.04\n",
            "  iterations_since_restore: 12\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21009559516077225\n",
            "    mean_inference_ms: 2.191926043938401\n",
            "    mean_processing_ms: 0.4762168237036545\n",
            "  time_since_restore: 221.80113077163696\n",
            "  time_this_iter_s: 16.84171223640442\n",
            "  time_total_s: 221.80113077163696\n",
            "  timestamp: 1561492836\n",
            "  timesteps_since_restore: 12000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 12000\n",
            "  training_iteration: 12\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 221 s, 12 iter, 12000 ts, 86.6 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n",
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_20-00-57\n",
            "  done: false\n",
            "  episode_len_mean: 94.3\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 94.3\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 263\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 23.675\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 24.643156051635742\n",
            "        mean_q: 20.828256607055664\n",
            "        mean_td_error: 2.705711841583252\n",
            "        min_q: 7.012941837310791\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 13000\n",
            "    num_steps_trained: 96000\n",
            "    num_target_updates: 25\n",
            "    opt_peak_throughput: 1351.628\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 10.215\n",
            "    sample_time_ms: 39.287\n",
            "    update_time_ms: 37.457\n",
            "  iterations_since_restore: 13\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.210729030693145\n",
            "    mean_inference_ms: 2.1933999872337484\n",
            "    mean_processing_ms: 0.4763524480256622\n",
            "  time_since_restore: 243.2731375694275\n",
            "  time_this_iter_s: 21.472006797790527\n",
            "  time_total_s: 243.2731375694275\n",
            "  timestamp: 1561492857\n",
            "  timesteps_since_restore: 13000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 13000\n",
            "  training_iteration: 13\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 2/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 2, 'RUNNING': 1})\n",
            "RUNNING trials:\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tRUNNING, [2 CPUs, 0 GPUs], [pid=1528], 243 s, 13 iter, 13000 ts, 94.3 rew\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-06-25 20:01:14,624\tINFO ray_trial_executor.py:187 -- Destroying actor for trial DQN_CartPole-v0_2_lr=0.0001. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result for DQN_CartPole-v0_2_lr=0.0001:\n",
            "  custom_metrics: {}\n",
            "  date: 2019-06-25_20-01-14\n",
            "  done: true\n",
            "  episode_len_mean: 102.42\n",
            "  episode_reward_max: 200.0\n",
            "  episode_reward_mean: 102.42\n",
            "  episode_reward_min: 10.0\n",
            "  episodes_this_iter: 5\n",
            "  episodes_total: 268\n",
            "  experiment_id: 75a0bdcf9f8c491d9c0f027ebb3da5f4\n",
            "  hostname: 16244a9ca08f\n",
            "  info:\n",
            "    grad_time_ms: 12.32\n",
            "    learner:\n",
            "      default_policy:\n",
            "        cur_lr: 9.999999747378752e-05\n",
            "        max_q: 26.84748077392578\n",
            "        mean_q: 20.581289291381836\n",
            "        mean_td_error: 5.6687822341918945\n",
            "        min_q: 6.800024509429932\n",
            "        model: {}\n",
            "    max_exploration: 0.020000000000000018\n",
            "    min_exploration: 0.020000000000000018\n",
            "    num_steps_sampled: 14000\n",
            "    num_steps_trained: 104000\n",
            "    num_target_updates: 27\n",
            "    opt_peak_throughput: 2597.464\n",
            "    opt_samples: 32.0\n",
            "    replay_time_ms: 5.815\n",
            "    sample_time_ms: 14.992\n",
            "    update_time_ms: 16.876\n",
            "  iterations_since_restore: 14\n",
            "  node_ip: 172.28.0.2\n",
            "  num_healthy_workers: 1\n",
            "  num_metric_batches_dropped: 0\n",
            "  off_policy_estimator: {}\n",
            "  pid: 1528\n",
            "  policy_reward_mean: {}\n",
            "  sampler_perf:\n",
            "    mean_env_wait_ms: 0.21117533896720814\n",
            "    mean_inference_ms: 2.1940731963836173\n",
            "    mean_processing_ms: 0.4763186581842119\n",
            "  time_since_restore: 260.2600600719452\n",
            "  time_this_iter_s: 16.9869225025177\n",
            "  time_total_s: 260.2600600719452\n",
            "  timestamp: 1561492874\n",
            "  timesteps_since_restore: 14000\n",
            "  timesteps_this_iter: 1000\n",
            "  timesteps_total: 14000\n",
            "  training_iteration: 14\n",
            "  \n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 3})\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1528], 260 s, 14 iter, 14000 ts, 102 rew\n",
            "\n",
            "== Status ==\n",
            "Using FIFO scheduling algorithm.\n",
            "Resources requested: 0/2 CPUs, 0/0 GPUs\n",
            "Memory usage on this node: 6.2/13.7 GB\n",
            "Result logdir: /root/ray_results/DQN\n",
            "Number of trials: 3 ({'TERMINATED': 3})\n",
            "TERMINATED trials:\n",
            " - DQN_CartPole-v0_0_lr=0.01:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1351], 426 s, 22 iter, 22000 ts, 105 rew\n",
            " - DQN_CartPole-v0_1_lr=0.001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1477], 244 s, 13 iter, 13000 ts, 102 rew\n",
            " - DQN_CartPole-v0_2_lr=0.0001:\tTERMINATED, [2 CPUs, 0 GPUs], [pid=1528], 260 s, 14 iter, 14000 ts, 102 rew\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[DQN_CartPole-v0_0_lr=0.01,\n",
              " DQN_CartPole-v0_1_lr=0.001,\n",
              " DQN_CartPole-v0_2_lr=0.0001]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NTUss8bQOLr2",
        "colab_type": "text"
      },
      "source": [
        "# Keras-RL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2-ESg06OZ2R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install gym pyvirtualdisplay > /dev/null 2>&1\n",
        "!pip install keras-rl > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Q6kdtRtBnL0",
        "colab_type": "code",
        "outputId": "0c936072-d855-4a16-80ac-ebfcabd36632",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from pyvirtualdisplay import Display\n",
        "from gym.wrappers.monitor import Monitor\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()\n",
        "\n",
        "import numpy as np\n",
        "import gym\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Activation, Flatten\n",
        "from keras.optimizers import Adam\n",
        "\n",
        "from rl.agents.dqn import DQNAgent\n",
        "from rl.policy import BoltzmannQPolicy\n",
        "from rl.memory import SequentialMemory\n",
        "\n",
        "\n",
        "ENV_NAME = 'CartPole-v0'\n",
        "\n",
        "\n",
        "# Get the environment and extract the number of actions.\n",
        "env = gym.make(ENV_NAME)\n",
        "np.random.seed(123)\n",
        "env.seed(123)\n",
        "nb_actions = env.action_space.n\n",
        "\n",
        "# Next, we build a very simple model.\n",
        "model = Sequential()\n",
        "model.add(Flatten(input_shape=(1,) + env.observation_space.shape))\n",
        "model.add(Dense(16))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dense(16))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dense(16))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dense(nb_actions))\n",
        "model.add(Activation('linear'))\n",
        "print(model.summary())\n",
        "\n",
        "# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and\n",
        "# even the metrics!\n",
        "memory = SequentialMemory(limit=5000, window_length=1)\n",
        "policy = BoltzmannQPolicy()\n",
        "dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=10,\n",
        "               target_model_update=1e-2, policy=policy)\n",
        "dqn.compile(Adam(lr=1e-3), metrics=['mae'])\n",
        "\n",
        "# Okay, now it's time to learn something! We visualize the training here for show, but this\n",
        "# slows down training quite a lot. You can always safely abort the training prematurely using\n",
        "# Ctrl + C.\n",
        "dqn.fit(env, nb_steps=2500, visualize=True, verbose=2)\n",
        "\n",
        "# After training is done, we save the final weights.\n",
        "dqn.save_weights('dqn_{}_weights.h5f'.format(ENV_NAME), overwrite=True)\n",
        "\n",
        "# Finally, evaluate our algorithm for 5 episodes.\n",
        "dqn.test(Monitor(env, '.'), nb_episodes=5, visualize=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0625 20:17:55.664429 140008805877632 abstractdisplay.py:151] xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "flatten_2 (Flatten)          (None, 4)                 0         \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 16)                80        \n",
            "_________________________________________________________________\n",
            "activation_5 (Activation)    (None, 16)                0         \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 16)                272       \n",
            "_________________________________________________________________\n",
            "activation_6 (Activation)    (None, 16)                0         \n",
            "_________________________________________________________________\n",
            "dense_7 (Dense)              (None, 16)                272       \n",
            "_________________________________________________________________\n",
            "activation_7 (Activation)    (None, 16)                0         \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 2)                 34        \n",
            "_________________________________________________________________\n",
            "activation_8 (Activation)    (None, 2)                 0         \n",
            "=================================================================\n",
            "Total params: 658\n",
            "Trainable params: 658\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "Training for 2500 steps ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/rl/memory.py:39: UserWarning: Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!\n",
            "  warnings.warn('Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!')\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "   31/2500: episode: 1, duration: 1.962s, episode steps: 31, steps per second: 16, episode reward: 31.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.419 [0.000, 1.000], mean observation: 0.013 [-1.185, 1.776], loss: 0.462357, mean_absolute_error: 0.519609, mean_q: 0.093506\n",
            "   44/2500: episode: 2, duration: 0.126s, episode steps: 13, steps per second: 103, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.692 [0.000, 1.000], mean observation: -0.069 [-1.867, 1.224], loss: 0.353314, mean_absolute_error: 0.539802, mean_q: 0.283920\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/rl/memory.py:39: UserWarning: Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!\n",
            "  warnings.warn('Not enough entries to sample without replacement. Consider increasing your warm-up phase to avoid oversampling!')\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "   64/2500: episode: 3, duration: 0.191s, episode steps: 20, steps per second: 105, episode reward: 20.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.550 [0.000, 1.000], mean observation: -0.070 [-1.375, 0.833], loss: 0.230543, mean_absolute_error: 0.559043, mean_q: 0.496762\n",
            "   83/2500: episode: 4, duration: 0.170s, episode steps: 19, steps per second: 112, episode reward: 19.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.421 [0.000, 1.000], mean observation: 0.074 [-0.817, 1.505], loss: 0.116567, mean_absolute_error: 0.603374, mean_q: 0.820932\n",
            "   98/2500: episode: 5, duration: 0.131s, episode steps: 15, steps per second: 114, episode reward: 15.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.600 [0.000, 1.000], mean observation: -0.090 [-1.368, 0.825], loss: 0.051994, mean_absolute_error: 0.665641, mean_q: 1.135469\n",
            "  114/2500: episode: 6, duration: 0.153s, episode steps: 16, steps per second: 105, episode reward: 16.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.750 [0.000, 1.000], mean observation: -0.069 [-2.711, 1.766], loss: 0.034317, mean_absolute_error: 0.721667, mean_q: 1.310650\n",
            "  128/2500: episode: 7, duration: 0.134s, episode steps: 14, steps per second: 105, episode reward: 14.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.429 [0.000, 1.000], mean observation: 0.111 [-0.934, 1.483], loss: 0.023438, mean_absolute_error: 0.752646, mean_q: 1.434234\n",
            "  151/2500: episode: 8, duration: 0.230s, episode steps: 23, steps per second: 100, episode reward: 23.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.304 [0.000, 1.000], mean observation: 0.050 [-1.760, 2.679], loss: 0.018919, mean_absolute_error: 0.814541, mean_q: 1.603421\n",
            "  162/2500: episode: 9, duration: 0.100s, episode steps: 11, steps per second: 110, episode reward: 11.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.273 [0.000, 1.000], mean observation: 0.133 [-1.133, 1.962], loss: 0.027421, mean_absolute_error: 0.886094, mean_q: 1.760541\n",
            "  175/2500: episode: 10, duration: 0.147s, episode steps: 13, steps per second: 88, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.769 [0.000, 1.000], mean observation: -0.118 [-2.439, 1.549], loss: 0.043164, mean_absolute_error: 0.915887, mean_q: 1.770880\n",
            "  208/2500: episode: 11, duration: 0.277s, episode steps: 33, steps per second: 119, episode reward: 33.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.424 [0.000, 1.000], mean observation: 0.004 [-1.381, 1.780], loss: 0.043504, mean_absolute_error: 1.006226, mean_q: 2.006425\n",
            "  223/2500: episode: 12, duration: 0.127s, episode steps: 15, steps per second: 118, episode reward: 15.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.667 [0.000, 1.000], mean observation: -0.096 [-1.845, 1.032], loss: 0.064242, mean_absolute_error: 1.090164, mean_q: 2.153071\n",
            "  235/2500: episode: 13, duration: 0.101s, episode steps: 12, steps per second: 118, episode reward: 12.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.250 [0.000, 1.000], mean observation: 0.124 [-1.329, 2.154], loss: 0.039186, mean_absolute_error: 1.147887, mean_q: 2.317453\n",
            "  249/2500: episode: 14, duration: 0.118s, episode steps: 14, steps per second: 119, episode reward: 14.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.357 [0.000, 1.000], mean observation: 0.103 [-0.965, 1.601], loss: 0.058394, mean_absolute_error: 1.212103, mean_q: 2.445915\n",
            "  264/2500: episode: 15, duration: 0.135s, episode steps: 15, steps per second: 111, episode reward: 15.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.333 [0.000, 1.000], mean observation: 0.086 [-1.187, 2.013], loss: 0.065669, mean_absolute_error: 1.279796, mean_q: 2.562931\n",
            "  289/2500: episode: 16, duration: 0.201s, episode steps: 25, steps per second: 124, episode reward: 25.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.360 [0.000, 1.000], mean observation: 0.054 [-1.365, 2.263], loss: 0.077237, mean_absolute_error: 1.372554, mean_q: 2.760953\n",
            "  339/2500: episode: 17, duration: 0.383s, episode steps: 50, steps per second: 131, episode reward: 50.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.400 [0.000, 1.000], mean observation: -0.120 [-1.895, 2.080], loss: 0.097134, mean_absolute_error: 1.536714, mean_q: 3.044846\n",
            "  366/2500: episode: 18, duration: 0.200s, episode steps: 27, steps per second: 135, episode reward: 27.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.556 [0.000, 1.000], mean observation: -0.072 [-1.607, 0.789], loss: 0.111714, mean_absolute_error: 1.716993, mean_q: 3.394187\n",
            "  384/2500: episode: 19, duration: 0.143s, episode steps: 18, steps per second: 126, episode reward: 18.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.556 [0.000, 1.000], mean observation: -0.056 [-1.265, 0.809], loss: 0.099610, mean_absolute_error: 1.834031, mean_q: 3.583561\n",
            "  406/2500: episode: 20, duration: 0.164s, episode steps: 22, steps per second: 134, episode reward: 22.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.636 [0.000, 1.000], mean observation: -0.036 [-2.278, 1.562], loss: 0.115071, mean_absolute_error: 1.904394, mean_q: 3.730945\n",
            "  418/2500: episode: 21, duration: 0.092s, episode steps: 12, steps per second: 131, episode reward: 12.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.333 [0.000, 1.000], mean observation: 0.135 [-0.961, 1.649], loss: 0.203659, mean_absolute_error: 1.977715, mean_q: 3.810464\n",
            "  431/2500: episode: 22, duration: 0.097s, episode steps: 13, steps per second: 134, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.308 [0.000, 1.000], mean observation: 0.109 [-1.383, 2.208], loss: 0.173238, mean_absolute_error: 2.032999, mean_q: 3.896467\n",
            "  444/2500: episode: 23, duration: 0.097s, episode steps: 13, steps per second: 134, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.846 [0.000, 1.000], mean observation: -0.090 [-2.724, 1.795], loss: 0.160158, mean_absolute_error: 2.097915, mean_q: 4.069421\n",
            "  460/2500: episode: 24, duration: 0.421s, episode steps: 16, steps per second: 38, episode reward: 16.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.438 [0.000, 1.000], mean observation: 0.100 [-0.586, 1.215], loss: 0.197669, mean_absolute_error: 2.138733, mean_q: 4.136207\n",
            "  484/2500: episode: 25, duration: 0.617s, episode steps: 24, steps per second: 39, episode reward: 24.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.458 [0.000, 1.000], mean observation: 0.040 [-0.805, 1.402], loss: 0.195306, mean_absolute_error: 2.222576, mean_q: 4.289960\n",
            "  513/2500: episode: 26, duration: 0.851s, episode steps: 29, steps per second: 34, episode reward: 29.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.414 [0.000, 1.000], mean observation: 0.114 [-0.994, 2.303], loss: 0.190291, mean_absolute_error: 2.347659, mean_q: 4.481844\n",
            "  527/2500: episode: 27, duration: 0.470s, episode steps: 14, steps per second: 30, episode reward: 14.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.714 [0.000, 1.000], mean observation: -0.128 [-2.159, 1.175], loss: 0.208396, mean_absolute_error: 2.415468, mean_q: 4.676350\n",
            "  540/2500: episode: 28, duration: 0.433s, episode steps: 13, steps per second: 30, episode reward: 13.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.308 [0.000, 1.000], mean observation: 0.088 [-1.228, 2.024], loss: 0.288400, mean_absolute_error: 2.464157, mean_q: 4.698053\n",
            "  556/2500: episode: 29, duration: 0.494s, episode steps: 16, steps per second: 32, episode reward: 16.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.250 [0.000, 1.000], mean observation: 0.096 [-1.518, 2.460], loss: 0.203208, mean_absolute_error: 2.525871, mean_q: 4.854614\n",
            "  575/2500: episode: 30, duration: 0.617s, episode steps: 19, steps per second: 31, episode reward: 19.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: 0.101 [-0.546, 1.083], loss: 0.179367, mean_absolute_error: 2.625908, mean_q: 5.067637\n",
            "  599/2500: episode: 31, duration: 0.670s, episode steps: 24, steps per second: 36, episode reward: 24.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.583 [0.000, 1.000], mean observation: -0.088 [-1.868, 0.961], loss: 0.192668, mean_absolute_error: 2.683361, mean_q: 5.191103\n",
            "  621/2500: episode: 32, duration: 0.640s, episode steps: 22, steps per second: 34, episode reward: 22.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.500 [0.000, 1.000], mean observation: -0.083 [-1.089, 0.734], loss: 0.359855, mean_absolute_error: 2.788295, mean_q: 5.244081\n",
            "  663/2500: episode: 33, duration: 0.819s, episode steps: 42, steps per second: 51, episode reward: 42.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.405 [0.000, 1.000], mean observation: 0.057 [-1.550, 2.659], loss: 0.244082, mean_absolute_error: 2.884288, mean_q: 5.529559\n",
            "  679/2500: episode: 34, duration: 0.129s, episode steps: 16, steps per second: 124, episode reward: 16.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.375 [0.000, 1.000], mean observation: 0.100 [-0.936, 1.741], loss: 0.219697, mean_absolute_error: 2.978552, mean_q: 5.722777\n",
            "  695/2500: episode: 35, duration: 0.136s, episode steps: 16, steps per second: 118, episode reward: 16.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.438 [0.000, 1.000], mean observation: 0.090 [-0.812, 1.418], loss: 0.317440, mean_absolute_error: 3.057877, mean_q: 5.787911\n",
            "  707/2500: episode: 36, duration: 0.101s, episode steps: 12, steps per second: 119, episode reward: 12.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.250 [0.000, 1.000], mean observation: 0.110 [-1.202, 2.006], loss: 0.335140, mean_absolute_error: 3.134155, mean_q: 5.957223\n",
            "  716/2500: episode: 37, duration: 0.078s, episode steps: 9, steps per second: 116, episode reward: 9.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.889 [0.000, 1.000], mean observation: -0.135 [-2.291, 1.390], loss: 0.455019, mean_absolute_error: 3.081002, mean_q: 5.752834\n",
            "  753/2500: episode: 38, duration: 0.295s, episode steps: 37, steps per second: 125, episode reward: 37.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.405 [0.000, 1.000], mean observation: 0.031 [-1.347, 2.129], loss: 0.383251, mean_absolute_error: 3.179728, mean_q: 6.017925\n",
            "  778/2500: episode: 39, duration: 0.199s, episode steps: 25, steps per second: 126, episode reward: 25.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.520 [0.000, 1.000], mean observation: -0.083 [-1.330, 0.602], loss: 0.269417, mean_absolute_error: 3.292142, mean_q: 6.328763\n",
            "  792/2500: episode: 40, duration: 0.119s, episode steps: 14, steps per second: 118, episode reward: 14.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.429 [0.000, 1.000], mean observation: 0.084 [-0.798, 1.307], loss: 0.413019, mean_absolute_error: 3.396261, mean_q: 6.476095\n",
            "  821/2500: episode: 41, duration: 0.237s, episode steps: 29, steps per second: 122, episode reward: 29.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.690 [0.000, 1.000], mean observation: 0.007 [-3.105, 2.142], loss: 0.334732, mean_absolute_error: 3.461922, mean_q: 6.679079\n",
            "  831/2500: episode: 42, duration: 0.085s, episode steps: 10, steps per second: 117, episode reward: 10.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.300 [0.000, 1.000], mean observation: 0.107 [-1.223, 1.875], loss: 0.290274, mean_absolute_error: 3.509072, mean_q: 6.804790\n",
            "  866/2500: episode: 43, duration: 0.276s, episode steps: 35, steps per second: 127, episode reward: 35.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.457 [0.000, 1.000], mean observation: 0.124 [-0.583, 1.741], loss: 0.375185, mean_absolute_error: 3.604916, mean_q: 6.951423\n",
            "  906/2500: episode: 44, duration: 0.343s, episode steps: 40, steps per second: 117, episode reward: 40.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.525 [0.000, 1.000], mean observation: -0.047 [-1.382, 0.829], loss: 0.356091, mean_absolute_error: 3.721753, mean_q: 7.188008\n",
            "  926/2500: episode: 45, duration: 0.164s, episode steps: 20, steps per second: 122, episode reward: 20.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.350 [0.000, 1.000], mean observation: 0.094 [-1.334, 2.282], loss: 0.326631, mean_absolute_error: 3.848352, mean_q: 7.465901\n",
            "  952/2500: episode: 46, duration: 0.213s, episode steps: 26, steps per second: 122, episode reward: 26.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.462 [0.000, 1.000], mean observation: 0.116 [-0.561, 1.289], loss: 0.265114, mean_absolute_error: 3.958275, mean_q: 7.730439\n",
            "  970/2500: episode: 47, duration: 0.150s, episode steps: 18, steps per second: 120, episode reward: 18.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.389 [0.000, 1.000], mean observation: 0.105 [-0.757, 1.554], loss: 0.228334, mean_absolute_error: 4.093364, mean_q: 8.068156\n",
            "  996/2500: episode: 48, duration: 0.228s, episode steps: 26, steps per second: 114, episode reward: 26.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.577 [0.000, 1.000], mean observation: -0.055 [-1.415, 0.870], loss: 0.291499, mean_absolute_error: 4.119025, mean_q: 8.100605\n",
            " 1015/2500: episode: 49, duration: 0.159s, episode steps: 19, steps per second: 119, episode reward: 19.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.474 [0.000, 1.000], mean observation: 0.084 [-0.601, 1.082], loss: 0.282077, mean_absolute_error: 4.292354, mean_q: 8.460251\n",
            " 1075/2500: episode: 50, duration: 0.501s, episode steps: 60, steps per second: 120, episode reward: 60.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.533 [0.000, 1.000], mean observation: -0.098 [-2.195, 0.985], loss: 0.347626, mean_absolute_error: 4.315539, mean_q: 8.494741\n",
            " 1123/2500: episode: 51, duration: 0.404s, episode steps: 48, steps per second: 119, episode reward: 48.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.500 [0.000, 1.000], mean observation: 0.052 [-0.819, 0.870], loss: 0.315527, mean_absolute_error: 4.607578, mean_q: 9.098517\n",
            " 1155/2500: episode: 52, duration: 0.292s, episode steps: 32, steps per second: 110, episode reward: 32.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.469 [0.000, 1.000], mean observation: 0.040 [-0.810, 1.344], loss: 0.401000, mean_absolute_error: 4.732345, mean_q: 9.364408\n",
            " 1189/2500: episode: 53, duration: 0.280s, episode steps: 34, steps per second: 121, episode reward: 34.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.500 [0.000, 1.000], mean observation: -0.084 [-1.041, 0.551], loss: 0.317945, mean_absolute_error: 4.915459, mean_q: 9.731077\n",
            " 1210/2500: episode: 54, duration: 0.176s, episode steps: 21, steps per second: 120, episode reward: 21.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.524 [0.000, 1.000], mean observation: -0.061 [-1.422, 0.969], loss: 0.449451, mean_absolute_error: 4.990871, mean_q: 9.800324\n",
            " 1287/2500: episode: 55, duration: 0.628s, episode steps: 77, steps per second: 123, episode reward: 77.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.558 [0.000, 1.000], mean observation: 0.131 [-1.666, 1.912], loss: 0.340181, mean_absolute_error: 5.189955, mean_q: 10.310872\n",
            " 1380/2500: episode: 56, duration: 0.800s, episode steps: 93, steps per second: 116, episode reward: 93.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.484 [0.000, 1.000], mean observation: -0.096 [-1.134, 0.875], loss: 0.319214, mean_absolute_error: 5.507593, mean_q: 11.044703\n",
            " 1493/2500: episode: 57, duration: 2.740s, episode steps: 113, steps per second: 41, episode reward: 113.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.504 [0.000, 1.000], mean observation: 0.302 [-0.701, 1.621], loss: 0.455874, mean_absolute_error: 5.997399, mean_q: 12.009386\n",
            " 1596/2500: episode: 58, duration: 2.955s, episode steps: 103, steps per second: 35, episode reward: 103.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.515 [0.000, 1.000], mean observation: 0.170 [-1.216, 1.357], loss: 0.486948, mean_absolute_error: 6.373150, mean_q: 12.794644\n",
            " 1766/2500: episode: 59, duration: 1.771s, episode steps: 170, steps per second: 96, episode reward: 170.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.482 [0.000, 1.000], mean observation: -0.118 [-1.176, 0.927], loss: 0.463826, mean_absolute_error: 7.070574, mean_q: 14.322657\n",
            " 1935/2500: episode: 60, duration: 1.355s, episode steps: 169, steps per second: 125, episode reward: 169.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.479 [0.000, 1.000], mean observation: -0.137 [-1.521, 0.953], loss: 0.498070, mean_absolute_error: 7.884391, mean_q: 16.005653\n",
            " 2069/2500: episode: 61, duration: 1.095s, episode steps: 134, steps per second: 122, episode reward: 134.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.485 [0.000, 1.000], mean observation: -0.159 [-1.323, 0.971], loss: 0.671410, mean_absolute_error: 8.749821, mean_q: 17.768614\n",
            " 2156/2500: episode: 62, duration: 0.694s, episode steps: 87, steps per second: 125, episode reward: 87.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.471 [0.000, 1.000], mean observation: -0.213 [-1.494, 0.987], loss: 1.436515, mean_absolute_error: 9.399087, mean_q: 18.981615\n",
            " 2274/2500: episode: 63, duration: 0.928s, episode steps: 118, steps per second: 127, episode reward: 118.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.475 [0.000, 1.000], mean observation: -0.203 [-1.265, 1.061], loss: 1.190817, mean_absolute_error: 9.848555, mean_q: 19.917814\n",
            " 2411/2500: episode: 64, duration: 1.394s, episode steps: 137, steps per second: 98, episode reward: 137.000, mean reward: 1.000 [1.000, 1.000], mean action: 0.467 [0.000, 1.000], mean observation: -0.216 [-1.799, 0.827], loss: 1.430575, mean_absolute_error: 10.416906, mean_q: 21.049749\n",
            "done, took 32.964 seconds\n",
            "Testing for 5 episodes ...\n",
            "Episode 1: reward: 187.000, steps: 187\n",
            "Episode 2: reward: 144.000, steps: 144\n",
            "Episode 3: reward: 200.000, steps: 200\n",
            "Episode 4: reward: 158.000, steps: 158\n",
            "Episode 5: reward: 200.000, steps: 200\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f561331ceb8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M6WCwR8HB916",
        "colab_type": "code",
        "outputId": "0c36b9e3-0112-4485-e4d4-3e59a67397d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        "show_vid(next(glob.iglob(\"*.mp4\")))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAMDhtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAACI2WIhAAv//72rvzLK0cLlS4dWXuzUfLoSXL9iDB9aAAAAwAAAwAAJuKiZ0WFMeJsgAAALmAIWElDyDzETFWKgSvGXyzo4bANZEHl6Gl+2JXIs/k+BGAf48HZBXSxrGH+bMxz1opn1ji9dugTL6khGOrqJBGfCZtQ3hb8UO+4VZZCrwyhf+aQ7iglZ6Iw0n1RaY+KcwkK00tT+2CFZ9gIaVLEPgf9M0BRhOQbF9Noz1p51cZU6Uth7vbfRA5sM5Fc1NhzEJ63NpHVcivlgUvuzXEPK0FOl/xOhFUQeQH4nD1kyiUo9MgGtfW88AAlTyWAFQ5rOASMSTJFtl1+fkk2k1FjtJw8nMdiUAtylEQNzrrTI5T/mQXbMFQ4G75mNgkaof7DPhBe/LDIaJ3cv7y9KGttJapt8EVVGFAzAFtlWoNZMK4364KXNnwvfocwUMQjL7xbqfkLtN6E2aDYQ48XEs+pypqfVdeMN8LGrmfiQOE1YJHQPXJEm6BJUQG09Y+uY9KUmqhp5Wy4RlACd65ZAIf9VV5Dj+w7yJjSB3nOB/Vu04801AnWhnjmU4I0V6jjmnMM5xjudC2ISYjIbmgOTFyja2Vo9iMRO/8yzI8f8pdgYiXlMxtCtz8BtABRCU2xQpn7VtdyEZB+JUuBPvGyLYSrhIsjTey59Hn71hW1CkCkXH8JIaJ+/ibfLfH4EQXWnxFuHxxkpZB0kAAAAwAAAwAAh4EAAAChQZokbEL//oywAAAajLiogKivF5yjX9DVYBjl2Kbcp0OlvYlV4s9GT3Ps81cF8tq30LvlfBMMAtroeP0tf5E2nub0872cDPaLNba/xgj+rIR44Ogv9H+sDaHjs4iCfwY5HHu0xOsgrvpI3S5WxJJL5QZqpSZlR1gnu0jPI5ldRjcJD3jio6cZ/RQAbL24z8Ws/BeV+73W1PK/me1M7AvQJHgAAABJQZ5CeIR/AAADAzhS6r1e70Kn7YXvc2Sgn9WQZM7bAroW22AFq3dOFW88Uz02ygL1002qzhRrj3irAIAAAAMAEBiVLTBrnhzXIwAAADEBnmF0R/8AAAUgST/IOYKHbkguf4BBgJ6k9nVc4ZzK7gAAAwAAAwBWIbn+OOmXDYScAAAALgGeY2pH/wAABR5a/n974QvjOTD46WQ7d6skO0RyWRKlGs270AAGVV4NyrnhAS8AAABvQZpmSahBaJlMFPDP/p4QAABFSW1gA6R9u93EBeRkhh8dmgKDqZ88y6cJ/BogTkZ70V5qQ2jDMI34bGx09mjUvrIl1t6S/EVkxHdpbtX/clznYytrB6yXaYNzp9owSxBoC4sXshXQa/opdfvG80h/AAAAKAGehWpH/wAAI78dNb9mLsPOUUvMIP6OqZT9MYj2NtIFykk3wKEIDAkAAACOQZqKSeEKUmUwIZ/+nhAAAAn73NAwC7YAmSH9YVHEVF4/CaqsDrONHXX4LTYEfujWpUL0QDUeXl1t8bKeJvCDDidhRHxnGvjAGeqCdm0tFVTvnUdY3kNrSd6KdgMBTYgXVEBruFt8zSE8B7+ubn4YybR8nN3F+f+NNkv9a5urh9IAHrspBQWsHvKg0Aw+oQAAAEZBnqhFNEwj/wAAAwNNup4XNzY5kngzKlyW+/fkk5NukSuu4pcNh2ZFLBd5VrL672K6TWXSR/g876VFnq4t4ppGyyFNcMGAAAAAMQGex3RH/wAABUBTSNFxQFikdx5M8ALH+37jPf1Cm9U9NgwDeY95aZJZ3Cn3edAeA7oAAAAgAZ7Jakf/AAAFQy5mSBEKcR6ZNxP2Ue9lAi6FC2rgBlUAAABqQZrOSahBaJlMCGf//p4QAABFRQXvV37X30YCESChIrXGuigAXyTyD/MkxYW3I4YtL/WEZfCAv53x58uAuZYR2T6QFRvj11q1ODhhOYkIaTRbw7tXHBRCgUfDcdk8LQ4K5ye2kuDVQW5l8AAAACVBnuxFESwj/wAAFrxObR3zxq6bm1myRNkuWyvpYDkQODz8HeXgAAAAKgGfC3RH/wAABUBcz85zuGgAsIm99fPRtzBcum/1jDCgoL++qJyCOgCtgQAAABkBnw1qR/8AACOyJ7mhBltgvUQfdWG9iFBBAAAAQkGbEkmoQWyZTAhn//6eEAAARWVBUfTNIEga6cx6Visb4VjuACU5nK8PlGe2AN6rOds7mLPPw2aJ+eEN2E8EJdxY2QAAACRBnzBFFSwj/wAAAwNNux0KA66W9HnmMGZGQBxO2qZNuKg+hwQAAAAsAZ9PdEf/AAAFQFK26AC6ADXPZk8M5ZE+b8p6P7qNLfJkQ9bgv/a3fh99FRYAAAAaAZ9Rakf/AAAjx8efeKy/RQZnzbkr9gf6UWEAAABhQZtWSahBbJlMCGf//p4QAABHRQUfipiAN50yUOgCs7h1NHeNM/CpEf/WiWhs5fvDwIn9wCJ24xhjFDJ5CProUorpx9r/WEcapsDkbTkMerhM9cScSxTdktcl7kJBiTb78AAAACRBn3RFFSwj/wAAF0UrN2tRavsETYttO91QOJdbn7/dMVya+xYAAAA1AZ+TdEf/AAAkq/hX5TcAFcep+nl27qxdOYotWAlg8WlZvwwzZt5FaK4dS5VITqpKpZhlSPcAAAAhAZ+Vakf/AAAFQTAJOHcEOr91AC26rgF+W/iaCVXnUwYEAAAAVUGbmkmoQWyZTAhn//6eEAAAR1SpPuQBGuXeIeXS/zizfavyassLZIABOO76FUYJ/RLSJQQ9mpMdya75E2ySnO+DZONDGDLsrbPD9HoV7ykmS4UanE0AAAAzQZ+4RRUsI/8AABdFE4vcLHHCAG66kmn9/TqEII4+M7oBVTnnMX+Xuh48HvNTNRbritLxAAAAKQGf13RH/wAAJMMCMT5nVpumuyH9Hm+UkAcql7yLh0akT9i7W1bLpru4AAAAKAGf2WpH/wAAI78fhmLileUKoARVRk5vYWjfl0lDVlfCBPOuf4XyWUEAAABRQZveSahBbJlMCGf//p4QAABHUT6vtTEAbzoc2q2qmUYIXD9iuldmOU6UbibylVY3YiNVH1MZdAjqozAxrRz7j+ekfaAVPJqe4Jg7A9WGzXG0AAAAK0Gf/EUVLCP/AAAWytCABxioUsa31ZUVBVdtd16e+yRkEsa8njh97fEKIsEAAAAdAZ4bdEf/AAAkwxd1jKAXUJOjhwaGmzfAbvJd1IEAAAAXAZ4dakf/AAAkscwJb6lyitD6Gktq2BAAAAA9QZoCSahBbJlMCGf//p4QAABHVKk+7qaAF+M4Xn8AGNei8hwZ07OlNApqd7IP+B8xiukjKx4d6xVlkmfhGAAAADJBniBFFSwj/wAAF0UrOdvhACZOmjHe85DFLfiMXEFecKk+sjkAIMLQZTV1HAfi5muqYQAAACkBnl90R/8AACTDF2ipxtVtjdHsADjdCn5HI6lNlEzogCVcf5cYrkGndwAAAC4BnkFqR/8AAA3Ptow3PkOkAC4n+DQ/ndu2Xvyhc3YqDGwdJ+yKu6AScwnKc5WxAAAARUGaRkmoQWyZTAhn//6eEAAAR0UFHxpPMRfwWr+YE2T929eF1rGpObUA/X/uOFl1jKMPCVbdCIeq0qEXHN/wdaOQgi3DIAAAAEJBnmRFFSwj/wAAF0U3lt7adfcYAPfhKQGhxRbtx9CjUi7TlnwqQpngndyJfIXc6aj3Fjq1ejWM6TQA6r98n3uxTYEAAAAtAZ6DdEf/AAAj2FzAEScxegZdCOnpAGoVE0WGoTKWOZJ1lG49o7ZzlRwwEJdTAAAANAGehWpH/wAAI78dNRkNW8LgAshMEx9hnpu5av+X4y8Qvs8ywUwx3e+rr/U/3sz7C+n5q/0AAABEQZqKSahBbJlMCGf//p4QAABHUQltO9qVvDXyj9cFJYA+WHn7LEZpS2IPejjzCTTIKD/OXFk6pd4iVNhfUkTJKJDSnkEAAAA2QZ6oRRUsI/8AABdDR/txOgeC45QYqngAmRyvELF/PGeojeOogprx+SRAYV77dHanp0fjZQ2AAAAAJQGex3RH/wAAJKv4W7VQALb92/T829Yn4nQaDSwaanCTgC4GnNwAAAAxAZ7Jakf/AAAkvwQnEtDJpCsUqBAAueq39NyZAF0/yS/ysZdmE18PHW2PdbCeTMD1IQAAAFNBms5JqEFsmUwIZ//+nhAAAEdwelSSNAChDTuYWW7xFOfDe+en+T2jPm3s/v8nNYIHvjdDQr51vZ9+IzpTouLo5TOBFS+UNesTjkygvCp+y2pfHAAAAGBBnuxFFSwj/wAAF0xDzR5vAAThpW1X7pWCQWsm9qq16mQNmfBGPoorp3R+Yhc3rz84aZ1zptroBQvd300zDOX2ktMW7enI/zmsA1b1IJZBhFqa8I99yfqulqZaB21uu9wAAAAbAZ8LdEf/AAANzgDTYQyIZq13ReydBlA/5B0xAAAAMQGfDWpH/wAAJLHMCW/bTHDQZUbIAWk/U9Lx3/UPSXBGbmyUm0+Kp+4fXOoicf71+1MAAABkQZsSSahBbJlMCGf//p4QAABHZI4agAA4W7sFi+Y6DXJEtaqSlL2wX9PW8pnlBKkl1RXMSYmIQb/MjrPWrbJQMyAM0Zrg6yGV8uuankAQm2K9hxj/shjKw7smes3l0o8YAXnHIQAAAEFBnzBFFSwj/wAAF0uKd404D1x9fcx2HXLev4+kA1m5a8+ekPJW7MZFOzt/ThO74CBTKmB/OeNAHHD4W5GwjvNakAAAACIBn090R/8AACS3/vQzclp5xzgknbd1pb2+CKRn9wbDWKbMAAAANgGfUWpH/wAAJLHMEEYoOgCENIkRcPqqEvMHOrpjCBem3F9N60XzNRf6buvcSiTCGTtt8N97gQAAAFBBm1ZJqEFsmUwIZ//+nhAAAEdEWnPN7ogWl0VPKXNJMC6+hwTnGJtjBXklWp8CfX2wkmqctLbBEx+01pWBb4DymdMbSVPyHsVPAmBU2wWGeAAAAD5Bn3RFFSwj/wAAF0MVjl5p54qpEauSzwCLLruYGCCNz3gQB461AAE7dSl6/LC+WDTnoC7A0g8feQPvLN6tSAAAAEQBn5N0R/8AACSfEwYJlCI6AEqe9DO96wl2O6Qu570nwgDS9zmVXk13ZhRQhky0MIh8axD03ISxj18X4Z6qA3wxkCltSQAAAC8Bn5VqR/8AACSuYAI1MkfdQGsw6AWUFjlRAnHiL8zo7gLq8qhMVlqMQkOu+stB8wAAAH1Bm5pJqEFsmUwIZ//+nhAAAEdFCdyAIkN3dc6c1UuDDv315ghlbs808zAPf2yeWkv/Dx8rql04h9gLDQzvJL47sLE2nIAF+FqOVB96k0h6ZATcdgJMznyD91QRErFJ92BgOH16VBmfPfr2cBz6oOvfb5wWwuArGNWQfTzy/wAAAEdBn7hFFSwj/wAAF0NIi971dPpT4i633qiALL04CoBaPiO8vSx0oEcMMxyhzZzLwypntuFc2QJmOyK/C2e/PMa9Wz8VYZLNSQAAADQBn9d0R/8AACSsOQYT4MT/TBw2AC4YQ77Xy7HPbJRV827lArYUoXmjTHWN19vNn3pTMgygAAAAKgGf2WpH/wAAJMcr/Dj9D3zzBv7zTb5pckAJv8b318AnRYlMH2FAwIePcQAAAFpBm95JqEFsmUwIZ//+nhAAAEdUl7X6wk3zXXJVQTE/UO76GU0DFzZWNOSgAcWlwtHhOg8MuvL3nuohZsy5X1z/LniIbGunuH5OyOKx70lyilFeRz8t9W2oyu0AAAA6QZ/8RRUsI/8AABdLitHmNjLwk5gqdzu1qVdbY6/bdRTmbAbU3rQ5VKVK56qKa8IqJ3ymG/5dklgtSQAAAEQBnht0R/8AACSsNuGJYAWLLf1rB9//n887nAziUGD1T+dzSIri9UVRN85HyhCsiihwC24HLd3CcnZiuNgxkWM6aRFLwQAAACYBnh1qR/8AACSuYAULTYYcAC0/t76mtc0WvROMmAEHMz0AoAOjPgAAAF1BmgJJqEFsmUwIZ//+nhAAAElEWfYm7M5gIpC6fFvab4ACVHxaqQweVxs0J5bORFAcvZWkKOzUOD+IeFt0UfPRB+qjk+Zvu8kTG4H+5rTU7IqIMdeu0NsZVQOR9mAAAAA8QZ4gRRUsI/8AABf294A5fqXY2TcpsXF3/WGRwlU5qlq9ExJgLljUATU9NQ9NoBrjIgKIV26UcW79ZX1JAAAAQgGeX3RH/wAAJcM+AqXcRUAFqI3CKNGsQEZMkYENOJVBXWbpIryT2yddGXpFYOhrt3AO5y0AvxNwQNiBG2puCyuNgAAAADcBnkFqR/8AACWuXU/KMDsaK8tzDskHAAL9Imbb7/c68IzLVzULbRQ0ujE42TJn+42CajpXkECBAAAAWkGaRkmoQWyZTAhn//6eEAAASURaIpusAHQN4XntR5rmeAm9ZwSCmFokuX4ErtWeNyy1Zcu95q8GQJlrdoJQ8ffq5+wNbaYKIHBLVJii9M2HO42wyoditKFv7AAAAEJBnmRFFSwj/wAAF+CHtwojvNbIQAF1Fxj/y/7EllcGsKZvsgfOq/M51dTUGMJsV4ydKsMUa7N2+zGCTWVJ1L8sNn0AAAA5AZ6DdEf/AAAlwuzhd0yinZnkALRfTOd71hQpYYJsUPAcwxzwlZjSmub6uqGCbzTIfZBVRHRivcQJAAAAPwGehWpH/wAADiT1OU3SuWYry0hIAHG6FPyOMiaibKJnwmwQj2jlJnjrr86j/A1AOcor3hCk4ggcAuolAbf98wAAAGBBmopJqEFsmUwIZ//+nhAAAElUmwmMf4Rmccaqe/1a7xX5XAA/dGP7GZ+W1gTY3pKzg+yiwwffvVefIdXDGjeXM8i0acCrCfIa+/RQgqcJtgSPLzWsFy+YhaIfLfdP7JEAAABAQZ6oRRUsI/8AABfoypRkjWMlQLGsVzSdzXN2BVQHTlcuGoMAbAd54hrXpDoNda9w6ZS6YAyp9KxNJnwOtxE5AgAAAC8Bnsd0R/8AACWsCNPXvo4MyQKl0jXJSFZwTdKscSRWFFsAAh9B40urPAaqW6dz4AAAADcBnslqR/8AACWuXtC0hM8zSqsuodYdThfcN9sypyi65AAbDCBjDIQ19usmoJUz072Wl5l4UyBBAAAAb0GazkmoQWyZTAhn//6eEAAASVDd1wKbIubHHl1S0J6StfgdeNiD9wH3EwWib45Yau8rtpxZSS1SEgAANqh3k+PTNR+1fnFt9CCgEn3ZAtL2WgYhVYOxi7lIVAN34KImOm7aDsoUroYfD0kVWDfXrgAAAEtBnuxFFSwj/wAAF9c/+4jZVV+wkARkV1s1FdwKqzumtiCFhJ83pnQ3JyHEYWuWuCWRRx9ayFSsWOdifXRjoASzCMe3oqkyarty7Z8AAAApAZ8LdEf/AAAlqepnauRjSY9hBXpNX1Q89n4P40YAINWeyhU++/mzz4EAAAA+AZ8Nakf/AAAlPcwOjUytuDOEL+RWR4jQzUAC1EbhFL62fcxAwDjN2dnfuoubCfGBNMPELJwfe/HbG4O1WfEAAAB1QZsSSahBbJlMCGf//p4QAABJUN3rcreTmS0C6IrUSmtRVClxps87uU0u+JneENsK4tEv641v7nSNpw0H9DfNXz496S2nL9Rfm+1oldn3fvlvc2ISVDyr67UDkn/M9CwecRqzBvIA9cd1OAGD6wD94/rQNlXhAAAAPUGfMEUVLCP/AAAX66AZxeR/5MojS6AAEg9VKFeqGjPwiSVxVrC/2+ASXoo6sWqVWSbMYbhcuFYnRI56Zz4AAABRAZ9PdEf/AAAlt8v6GSP72m4YABdRBb9PD/rZ4r9lrV7HGShYLr4YN03FZzvXGsvrArfcaJIBM0NEOHNznqVmwnEVcJMqbiY7MANMNINBAkCAAAAANQGfUWpH/wAAJa73VhbIv0f3/AAXDALmIF9D3pAS+R+0KiXGBwy+t+eiZOn32KgZ9a1M5hAhAAAAaUGbVkmoQWyZTAhf//6MsAAAShCX0aMYZmkAGPlSL1UVeezd52jsFEmiUzIKrJ5ujU6dpJHlz2U1QzHLJQjvNKykeZCtgxPtD5FW7JMFkip4LHpKB130cxd3BhmkPfmLgphKlXItRIiJwAAAAENBn3RFFSwj/wAAF9/sj6paLNkkEVJqVw9wnnEIZmV4r5ZKqoZBLdqBVfX6xNloACItbVcbf/RCBCX0LFBgmkGabjFgAAAAPQGfk3RH/wAAJcCaoZ56vkOABdADD9ogiIvlfQ0NZnYtyaGS6hNpRS+PP4OC7czsm8qqndLCHUvL8pc5SicAAAA1AZ+Vakf/AAAlrl/nMwW66j5+WQAXPV827ziWP8sxgJ89K8NWFgK8ZHdDPtxWH7laTylznnwAAABdQZuaSahBbJlMCF///oywAABMFFBGwmumyXY0BIaeqWt//SijHpcyOn1HdEL7wOQQdgUXoJThscVcrqQ2aSme///iteXkE4aKQFVSHiMGGRIZiWhs6GqPGMWqqNFzAAAAUUGfuEUVLCP/AAAYf+weoxtHICp4l9K1s5dWn0LRtS0poRrGdzAELgt1IrVG6zoMGbeOQ+WY2XJPphNACZyXdgQMDoDB1mDSLsugj/Hdxn5+fQAAADgBn9d0R/8AACbCXeA+3w1xYfJJFPXACWKeH0lTFng9AXc30OD5P+s5UUGN3ABN9q/f3fEeY98QIAAAAEYBn9lqR/8AACauXjOBeUSdcUM8khru2c8ATsr1TLuF4PZmBSd+CcUan0RS59BY3/vo0bX4LkTSsPS/1KHyJGbNlvy5vIM/AAAAfkGb3EmoQWyZTBRMM//+nhAAAEtEWfYm7QGNLw+LQgBBXRHCC9hrETW/Q/uBPOL4VbHcFLdwNicH/OJwJOakgLl28syRpdq4aqOxX9JhIWKk6fj36PcnXvLIIEiBYJKs7dgnu68zx/5spXan7OgNQ+7BZkhpfGF4JNvPyrm/SAAAAC4Bn/tqR/8AACauXjOBeUOzsIdYJ8VisJ2TJDE7TipKULukFKgA+NK+Deu18WfBAAAAZkGb4EnhClJlMCF//oywAABMJHX4c0D5SczEFUAok4oQDG2/zSPUSTAxAzUvU9k6h5m51Dlf+fTCWlceA6Lcni+ZbXCYhsSGOhvX3Bcpk5mbo1XFkOLsYFZvWpSzRZ/bAW6fwkkJsQAAAEhBnh5FNEwj/wAAGIibYfSs8Y0UAwi9funKNwFmxXbCYqKLDp2C+pT1F/VCTEALe8VVu0zyoJCkyHW7XgkrylK7ze3Wb65/5UAAAAA7AZ49dEf/AAAmwJnd6YEhfrgdFxvYHKrNw0dhs++wlZrAp2KISUEF9i5cABK9HoA9qhbTI26hUQr5EWAAAAA9AZ4/akf/AAAmvYMuz04sXk2UKcGKz7tx98kh1YgPLpa3WJbP3MTwfhg8SAFvAIXujXiDYha5W/VqWzMO4QAAAHNBmiRJqEFomUwIX//+jLAAAEwQoeYMALBRa7EwpBscW4+Dz8JonDHryys5fL24C9pP7U1yh3ukT8YbtxeZRXSEKTBKd4Nbdj/nBMMAJmu4S8KjEyP++UgH5Dv6E3vEk9pZztBAkFaG6RK4djL9mvQ9r2uMAAAAN0GeQkURLCP/AAAYfx7Wp9xvdIa9Dyb/lrwomXvMmloQE7Ejgews8ZkDgN4QMK5Yexts7Lh5HcEAAAAvAZ5hdEf/AAAmwJkyfLIN36lVcR+9klvV5+BOy2m4TpGEikOdQ4MnCisljOCm9wIAAAA7AZ5jakf/AAAmrmAFFhlvS7eDGtNm7uoKdYgBbYvqF+mDde/v6W8GEnQmZM4ymq6R5z/JlwowS4Eu8qEAAABuQZpmSahBbJlMFEwz//6eEAAAS0Rac3TQBdHb3AA29cLExI3XmnIYAJCq4JSWc83I5coPBme7bVszwVzxtpxxs6qwWQruMOTteDcugt/8BzdPK5Amkmq07jTtkRaWPvCLfugzUeXXAEc9Z1kZ8DEAAABCAZ6Fakf/AAAmrl+0XaAQNF5p6ouc9pEYIqJBBm0IzOA2iG7vo6bp049g4SpFPhDgA9rQ/dCINilXEh3Ls3UnHTgRAAAAcEGaiknhClJlMCGf/p4QAABNQ+hMM+sue/LO9QApJsSp96CzM5PHm2GLZqQyFRKsdF+xCCFMj6cyUCSScwHM62qa5oWzwFojcZOxBGmRiL6szFRffQA/mLmd6wtj41Yp9+/B7TUjPiWTiKUmMGj7dZEAAABJQZ6oRTRMI/8AABkf6+XZSQAj3XY3LZglmJDQGZH+n9uQBFzT9S/0qMzMyofPc/C9BnVSlI9sYiL6B1E2cKjyZZsASjNkTx/yHgAAAEABnsd0R/8AACfALWwgB0TMJnW3J8n/XRVuQW9rtBJZOLNK+oZ6DDs+VqNvzAq7NN4c/S5tvpGy3AeiAJlNQaNSAAAAQQGeyWpH/wAAJ8VT0zzd6djHsKM92+MePORDjryVD0jLDXbKfIcbABMmkBd75300l0EiSQpxok5S+zGOENngEHNBAAAAckGazkmoQWiZTAhf//6MsAAAThCm0UG/1wFmZy2tDJgG4tIKoJs5P0TZBCeJHR+ARkaA7wDnLSMeUpNAeHB0xYkUsrRPElGrNUu7nsf++n1uL1pTLLrEMkhaBjyYn89hvGykbcN4eJwc+qg9biFcYJ+QgAAAAFNBnuxFESwj/wAAGR9DvQteaeYvag9y/k9uhTB2CeQxUmKSOWGAV/MPqhOot8iQ2vrHIOECROK6yAAnUSpSHFw0zSLU9OpcopJwVWcs+JgeHQDQgAAAAEkBnwt0R/8AACfanRx8N8kf1gAjOM4UuJ1sSp8eL2hqhFavHFXaAcz/keQy8gbsh34W2wvg7LKoUsnJxmuc90+gQ+gvTHNaKah5AAAARwGfDWpH/wAAJ0CL2ebbqTAsKj2DvNp2eKAzeSjcIpg94wZ/mIpAW8eaQDtf58zqrnOdgtzH8tJc6fUAH7Q+GFYN71aocqkhAAAAhkGbEkmoQWyZTAhf//6MsAAAUCpMt4thFEe0lKsgHAA6R64S+gzy/arWyBeX6ZPsdqhDdGVVojFIG9sxUPd7juPmeg8RRmm1jceblKK0ynpQI+4K6WfV8DFOHXpiofxtf612NtgMqhjnWdQi9HnqGyoVA/2sonWq/VZFANk4PTWjoS3uE0bpAAAAWEGfMEUVLCP/AAAZyYLDSjdHV78R2q6DM72uszqw3mfs6obsaqGxJnuxk3qzoDJZhdOit5NYvH1LiR8i4ALisswxjiSH9qPAO7eXJLDPNBoRIxyt4FwxV9MAAABAAZ9PdEf/AAAn2p3hEdqcBtnyLpxLpCwMilClOF31vqdYECf9lObXnzewZeivSql3Z1x9ymW8bPUjYccPV/AqYAAAADoBn1FqR/8AACj5V4i/HYtQHPXKPIO8VHmilZIgEaMpF29qM8ag6Nyo38RTiOlqUpHWSpwqXv6zslttAAAAX0GbVUmoQWyZTAhf//6MsAAAUCoaRJqgtA+wA0O0M1FpJxyHM4G1TFiiYT7r46k39Siz37l3QRxJxiSosJm8cUtWuIZY2RdShBMLPoJQHDFRbYAJQwI/YQtWh9TK868gAAAAQkGfc0UVLCP/AAAZyJtP8qjDpfzPqpzSV8IdIBjG4w3aDMvSXt0inyEIXIux72uy4/cNsLkACyYO7REkQGGB0rSN1gAAAD4Bn5RqR/8AACjkUv/UDqQkczUay6A58UCaVE/MUGRJ9uQdYqDzX4LEpdl+M9G3iTT1VRjAHc3oxQS7VjzobQAAAIZBm5dJqEFsmUwUTDP//p4QAABPa1nyUALZ661Ydq97baLtFJijQ/lE/IK0xugVgsDpt1Zc9HjNduJxowdkEJi9a+qiW2dTDyBDsTDT7oCucq3m+XwwILWdzVo0F/kAe1P1hTtqRskRZEA3coK2ZCGbhslT05WJgNsr1/PWJj29sDM7QMrLrAAAAFsBn7ZqR/8AACjkGTJw+n6L7dPScN+kC0paDNPXqfusOccKfmpMjYWsfKvkUtXmA33/fEPgAmmGh6aezhwScKgYWaj8adnDj1c+1DJWgl7JwgX5tibDJo2m40fBAAAAqEGbu0nhClJlMCGf/p4QAABRu9WlbDgCtvGRarq+rCRd5wmtdl2cCfS3b89nU1AtjTZ2i7/7yu+8VUFSKLoZdSd0Llt3/Mp94ysXFu0O47ldX6CcDZUr1m5iUWWX71kYDVdUHjraoBD/sWWy8nQebOmUdmvVkXqfBMAmCCwhuKHNOmJ6fFKrQ8iojW9aN4k78OLOJz8dAgt8m9fNT+37Pbus6ltEAT9y2QAAAF1Bn9lFNEwj/wAAGml8fQMkAI9jEjX/XtoPjx7Sw41JHyv5vEnMDtfF5nIYEWEiqTLvkYFJ5QYhb/M0ns5oRoaL73MQZrLeyMM1+3hicKxBDYAweuC+iq/ts0Z4D3AAAABMAZ/4dEf/AAAqHjPdmcE2xlH/PS9rbNNPBGBeE7IUMmY9JIpHmw74KlWBe5r9L/8jPpGQqchwfw8AJpc0/pcF5so1L2hNt2MakkIHuQAAAE0Bn/pqR/8AACoWjmtca9mvAZN9i0jK6qxAMrBcrTUSfeKzBYOFeP0UQcevTid54Lvz/0XT82SWnCi5It9DwTbtwoAPrzSRv2YuVKEZQAAAAJdBm/9JqEFomUwIX//+jLAAAFJqbKuQAOPTf5s5Ho4BkmGqNbLxlGfddPrHPqoXCBBLhkzvjP/c3iVPmwkWgE8QJUO9Rec41CqV/09/CMaYxDFbOKNCt4H4c/xEOm3POyq3+1Vvb7AffH0V9/aGspLpj+P2tAHH1Lahwt5r0iWTX1lEdbvfEzaFhUo+a9v4gbblM2QwCgzBAAAAVUGeHUURLCP/AAAaX3zjCACwnwzeWreppKAxp5Trfu05tlISTWy2Bw0/1T+Sqc1WA+R0ZnnW4BoCjDc82TlYBDZYZ27KX0Ij15gzbmFW6jnJJK9XzgUAAAA/AZ48dEf/AAAqGYEMJKioIbhNQ6ZXZTjXwZRJgFQGBWQQJ6vyIWUq+xY2Puk/ETmJcYoNC1j1GDXOxIuuj8jKAAAAVAGePmpH/wAAKiMlZ7NWum9WIP9pnnUgvkHlQEeG+MaOZXnBFnBD9XKrM93WXSMAAH3xBa9UZgfOy2Fde5cnCRhIcFb718QcrVE7KMLrGZLFkV84EAAAAJtBmiNJqEFsmUwIX//+jLAAAFSqbKuQAOMWxcf0SkPwmKXdJw1AT9LOJ5f1sB+s/yL8yM+oJYvhytt50frzmC2zpaGvAe2KntUZh7DOYHxXBFiHPSk74U5r/FRvbBH46kppA3MLoNR4sRc+KUgO6CGtdhdKkYtrv6tKsdhGMy03VIuOO5TWqV7nP9S4d2paiEVsnjoNrUHuS0u4UQAAAFFBnkFFFSwj/wAAGv+Fr0LYapzpc6iFcYMqfBzL4VAmbyAJlF2hzQ2se9I0I7SJLjqghyFcX1ADqTxGzupWIigBLY5R5+9if4sS6EnjT/nNycAAAABMAZ5gdEf/AAArPhwwH/s+RdeX0kvbNsRFwuT3C+JPKiMjLeH4C0cRi2yx7u687z5Crp6BIKXCsooc4U3iGtHCYAHmo5JW2L7CDVX/gQAAAFkBnmJqR/8AACsinJv/o45GiC86g7aq7N2OqffrHbF8RBP0TdE6KfflNYBewN5B0KViclwANopxy5MMLZ2LgM0Z4hLqv2S/rqvKEG4GcSsdkyK4Vq4Ci6EPgAAAAHxBmmdJqEFsmUwIX//+jLAAAFb/VgAFBgB/pbHbu5a2sbV3MpXbdyMowg77bUq04ESap2gDeP2RNHeBFDeXn4gLukFlxjyd9MxWZh1zyiOgBvA1g++zQs3RtbyKmqYj+zJG5xpLGnFBjQfNdwc5DM8NDeEZ0s4uOWbt7xixAAAAZEGehUUVLCP/AAAbqXyF69AAOM8ZZMmhINO4Phd3js7e/sxlKG/2kwTpirB9uCzrDQ6hhKCaxUSKIr8hxwyMFhXJpD7EtK6Bh6RFfnyF63Kt8DqGKKlRbbU0tzF/tHUFjc5Lu6EAAABXAZ6kdEf/AAArHrv/gOYzsnwDB+XE+ODkdCYFCVJfis1jjCWLS4tZFf1XuVtN+QAmrchmwPk2rAzcT2bLsZs/fD0Kf/boV8T7Ne1B2sw8cMoRLje5LjXdAAAAVAGepmpH/wAALEKbqSiIWuDbxlR6WrPOSHRhMVkji+KdDmcJu09J6sn7JVdegATpCf/EUtnvQy0GcsBTxave9f2vpUISXszlixuWI/XRiYDtrMdQ8QAAAH1BmqpJqEFsmUwIV//+OEAAAU+NzAOAARjiHAPVReydDAj1CG4FlwvSzpSZyltRBXJxVnO9pcIcpdyT3Odq7KoTZ42pMOVCH+njibbbyftY0goVEo7M3Yn1GO/5UX2mp+jYYKlp+KSnqX/swseP3mI9SC70Ur1asJnpKcnR3AAAAEtBnshFFSwj/wAAG6hjMVigMxz2oPVvf4/O1NXp4O27rJ3cs3Vd2RiXdEP5kUAt/yOWNXiQyR25HXYACaacL/vuFKbpxANGs2j6BvwAAABBAZ7pakf/AAAsYyExOvXAkf3pkU4pfvnIxhaFcK4vyVJlqDpgZeSisVHmZnE/QEcM33oE7IKlqGaUkfnpOW8WlYEAAAB6QZrrSahBbJlMCFf//jhAAAFYiQlcIpEk0OJPI2Ug0ANzFW4D3fu/aKeEY03L8l4JOLxSeCwdDy0N0VenZcODm7i7MOs1xw8SsgNKSduN//SE4zPhZX1SFwZYjfF4bPuGsLLjHhtlOL3eR6e26/4rKn5U8CGM9zRxoaAAAAB7QZsOSeEKUmUwIT/98QAAAwNKUqYKAAjI44KU5L09dkmx/x3ebYZkXTRk6R+kZxCHa3ms+8MIRHKcO6xccIvzL/XTmHfXJst9Y3HMcFjQYAHXABMKXYLCmX1SMpfM1JZQYD7FFPlS9GJHzc+RmkXrYR7M9FKdktDBSh0gAAAAS0GfLEU0TCP/AAAcWyLHAJnvYdAEiv0k4B/xA/A2Eow+ebDSHu6Pc5ZfRewg2/AlK4DODdKRCNlPy+jdsIyj2AnnsFxVp5cYT9uZcQAAAEsBn01qR/8AAC1inSyp38aNPdxYGf2oWFytQTlL+c7Eg5XKEtT1cGOnVFD3LPQ4Gkv9aexdABaJmOI9K1BwkIgn7IoMqexJm7q4zVEAAABvQZtQSahBaJlMFPH//IQAABRscXnbmAK07c6R1yzxs4mzCyXRS88bYRapRHZx0NEjGI0poWdfQdeGHcjkmtJHn9ZWIspL7pHenb1QnKTW+zGo0Tni634ycr9jllAJw5C/kMrPqWYlvXmaeJGeBHKBAAAATQGfb2pH/wAALplXPX7l9L79F3lqAAWxbRo+79dyGZI6T50Z1GxU43bQp8QoNa1M8ytEB5yhUBBXVx3XjeVWT+hgSTAApzR4HvA3gMeAAAAJx21vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAAtUAAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAjxdHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAAtUAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAJYAAABkAAAAAAAJGVkdHMAAAAcZWxzdAAAAAAAAAABAAALVAAAAgAAAQAAAAAIaW1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAMgAAAJEAVcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAACBRtaW5mAAAAFHZtaGQAAAABAAAAAAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAAfUc3RibAAAAJhzdHNkAAAAAAAAAAEAAACIYXZjMQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAJYAZAASAAAAEgAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABj//wAAADJhdmNDAWQAH//hABlnZAAfrNlAmDPl4QAAAwABAAADAGQPGDGWAQAGaOvjyyLAAAAAGHN0dHMAAAAAAAAAAQAAAJEAAAEAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAASAY3R0cwAAAAAAAACOAAAAAQAAAgAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAMAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAADAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAMAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAEAAAAAAIAAAEAAAAAAQAAAwAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABAAAAAACAAABAAAAAAEAAAIAAAAAAQAABAAAAAACAAABAAAAAAEAAAMAAAAAAQAAAQAAAAAcc3RzYwAAAAAAAAABAAAAAQAAAJEAAAABAAACWHN0c3oAAAAAAAAAAAAAAJEAAATZAAAApQAAAE0AAAA1AAAAMgAAAHMAAAAsAAAAkgAAAEoAAAA1AAAAJAAAAG4AAAApAAAALgAAAB0AAABGAAAAKAAAADAAAAAeAAAAZQAAACgAAAA5AAAAJQAAAFkAAAA3AAAALQAAACwAAABVAAAALwAAACEAAAAbAAAAQQAAADYAAAAtAAAAMgAAAEkAAABGAAAAMQAAADgAAABIAAAAOgAAACkAAAA1AAAAVwAAAGQAAAAfAAAANQAAAGgAAABFAAAAJgAAADoAAABUAAAAQgAAAEgAAAAzAAAAgQAAAEsAAAA4AAAALgAAAF4AAAA+AAAASAAAACoAAABhAAAAQAAAAEYAAAA7AAAAXgAAAEYAAAA9AAAAQwAAAGQAAABEAAAAMwAAADsAAABzAAAATwAAAC0AAABCAAAAeQAAAEEAAABVAAAAOQAAAG0AAABHAAAAQQAAADkAAABhAAAAVQAAADwAAABKAAAAggAAADIAAABqAAAATAAAAD8AAABBAAAAdwAAADsAAAAzAAAAPwAAAHIAAABGAAAAdAAAAE0AAABEAAAARQAAAHYAAABXAAAATQAAAEsAAACKAAAAXAAAAEQAAAA+AAAAYwAAAEYAAABCAAAAigAAAF8AAACsAAAAYQAAAFAAAABRAAAAmwAAAFkAAABDAAAAWAAAAJ8AAABVAAAAUAAAAF0AAACAAAAAaAAAAFsAAABYAAAAgQAAAE8AAABFAAAAfgAAAH8AAABPAAAATwAAAHMAAABRAAAAFHN0Y28AAAAAAAAAAQAAADAAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGlyYXBwbAAAAAAAAAAAAAAAAC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjU3LjgzLjEwMA==\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NfRnvEQkB_se",
        "colab_type": "code",
        "outputId": "2f498c25-e1f4-4a43-94f3-0a62d3ef536b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAANyhtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAB/WWIhAAz//727L4FNf2f0JcRLMXaSnA+KqSAgHc0wAAAAwAAAwAAFgn0I7DkqgN3QAAAHGAFBCwCPCVC2EhH2OlcDYD/lmnaNFVW+jEb838jHr8lgAAWqDz0IV4Sapc2Y/3FVIp4ylb1rL6vL9ake1e2YBtY68uKQBmbhvlm/hNRbWf/v5OgL1RDh7uwBBeLPJJJSmExXihAPRwh0sdDGPRV/ByVMQezWskj0KmGYIOE2SJNu6uxxReHK9gNAsap1Xmgc7n9G9HEoAMbBZm4WLD9c8le4wKXVTWylT++j2hYd+wC/7B9+yYJ4P2hwXcE43gZJtgVwmd3FlUB4b7iecJqj4JgnEhf/L36lEXu61BMysdIDmM3EOO07hXFuUYClaBuMVGNoFZgdTygwl8M7w54YMrABnGR4/S+FprlLAbaOzsFUFnJB78yMOCSz7iFmXuoen3slGY3zBuYUbFtVF3faj1oJWpnaa9ZwqMR3822WW5LMGg/zqmKgbBJlnzG42I0UC55Im0m1tBlDbgecdpvdsTZ5uzNGxALFW6jYyzJBkb0ffA0K+sZ7Y4ASzWTki1SNnJmd0Fb3eHTvEgewEgFGW2mHWF/iKXhU/Xx54FIT7hEnUkhJRGyVuKLM2SY1lnUfoILUkQAAeAAALmkdcAAAAMAAAMAAAMAAB3RAAABAkGaJGxDP/6eEAAARUtYeACgwGlnPXX92ZHl01jGDd5B6C+LGl5HzHDgROqN/mMydFzL4s43tPRR6+EEK0qnAzFJH4Pv8in0xUNq8PziAhgVFoFpfqDbqb6z0+T6wgrujE5cvklINXvMZaBSqerGAGPv73fCj5RW/8aipevZvQ9SGq5PP48lYn/lV7e6poDuS/k0YslkKMUNyRQmLsS+V00ixwfv7dIM3nJrymlT7sT3310HZllsuQCOzok6gf8cXVQZfPsGtlSSLz7opHW09rkXFkGsSXrsO9GSEQ2I+VQxSHBwDUACtT7YCgff9cXKaldtU0IFynTO3znsNsn5RTVHcAAAAFFBnkJ4hH8AABa1UowYp9OWc9qKEfI5MH+idIbtpjFtxsXyGmYgPKVzC4lrQ/rbmbZ4pqIy/B5mi/4OtEojW4fzRR00LYACYu6tuwo2qZYAGzEAAAAvAZ5hdEf/AAAjwzguoTPEmggHY4VYTMguoAttKbxeJAAAAwAAvN7ACk5ValUABswAAAAvAZ5jakf/AAANhJ6bVJnbl0Rrfzpzd8rdx5XY1uAAAAMAAAMC7AACCMFtUqgAN6EAAACBQZpoSahBaJlMCGf//p4QAABFSYDYAUGA0svxOpwECRVqWLr3B17/rEZJr1VEpyb2qyURHQoGyfEU4EjkXv39REGuVE5TnAozeUEigwvF9js+at1irVL6Q/qbyHBCh55fQ0USsxsGV392009G5AAAAwAADKgAAA4b3cMltUywAN6BAAAAREGehkURLCP/AAAIaXeJFMNrQeMfgZ9gAtSKZ74YefupVNRooKeq0vXNIvgvfl0cAAADACL64NzmfQAnH8Xo+rU8IAE3AAAANgGepXRH/wAAI7jBO3G/DUnaLPc+itXO2crFYAAAAwAAJjt47ktxQG1+e9QVBxJeu5SqAAD5gQAAADIBnqdqR/8AAAMB8e7d49ntEEGRx5FwTpY6oxgLMtcvqP0M8huAAAADADBiZTVGqoB+QAAAAFtBmqxJqEFsmUwIZ//+nhAAAAMDueorHV2mQRGDgmfAhvoCfRb429lT3nTvnK/MGwvVsjufe5mvzMUR2RK+mjkW8zyXU5ngGPrTJ9D0/UolmkMcRSSeiELoZYIuAAAAKkGeykUVLCP/AAAIcU/q4V/n5cmHx4R/vheHY6iPgAAAAwAFnuqnuywIOQAAADQBnul0R/8AAAMB8IkHbFr2qIJcv8fg5xwAmWiD6Bhoaj+RIOk4MAAAAwAAAwGnjLcIywKmAAAAGAGe62pH/wAAAwHxgA8djHA6ISjUQ0OLugAAADhBmvBJqEFsmUwIZ//+nhAAAEVIlyAOUbwFaXVp//2UWHYX0j3X0k/wSvMHXhHQUmjCaXtX7abePQAAACdBnw5FFSwj/wAAFrVZk2nBLkW0g6vtRp9deFN9YlC9E1ScfnL+58EAAAAeAZ8tdEf/AAAjrD0izp/oRBGhfstofqJ+4HcoAmKhAAAAHQGfL2pH/wAAI8fHSNBwgqFr6SgPyOEeQtepH2zAAAAAKUGbNEmoQWyZTAhn//6eEAAAAwCHe/jWEO4mysR0aYRwQJ46WCJeZTvzAAAALkGfUkUVLCP/AAAIcU/q5lAQBEPbKsZnwA8ssQAaqRd/kQUWS8U1Pdmr69yl8+EAAAAZAZ9xdEf/AAADAfCKUh28f1lTd05dYT3ugAAAACkBn3NqR/8AAAMB8X/k/YsvjEhUAJYQ3V6X5XLW4KVWe5Y43ytxbBIJUAAAAENBm3hJqEFsmUwIZ//+nhAAAEVFDTWACw+u93doRHA4yafqoNfWjixD8sS9WjPGrI275339lJSwv8iqXEqYkw5rAGBBAAAAGUGflkUVLCP/AAAWvE6hcwdYQUf2/ZJwCTgAAAARAZ+1dEf/AAAkw0Cqmf4A0IEAAAAWAZ+3akf/AAAkx7+M1EZ4OtD+m8BgQQAAAERBm7xJqEFsmUwIZ//+nhAAABpXOuAXcyjsnv9N6ce+GUjJ2CY74epHg1VxX38Lz3RmAkVrxJ6cBJP6ztQ6W9Ez/PTZvwAAACJBn9pFFSwj/wAAF06kGlOIN87MiBbzmFM3OoPgFSlj7MbBAAAAIwGf+XRH/wAAJKwY7Z0l8vgVAqACRRC7gH3QvhduZjC2xswYAAAAGQGf+2pH/wAAJL8azrlLpEGGwhkOFrqMUkEAAABsQZvgSahBbJlMCGf//p4QAABFbV7rCuAHHtztMgK7VHorEdJh4D7JA2RSEzStoNfWjiyBcRCJ//WBvm739C4Cnh6y4MmfZlfU2FFpNkiYl5SvB9didA+KOdIYhTZkB8+8G0h6hp4+7IkEXVO/AAAAIUGeHkUVLCP/AAAXUJs/YkTl9CYxoDNMPota5RumRCw7gAAAABwBnj10R/8AACSsOQZOLEnK3zCvQDQvB3b/yIwIAAAAGAGeP2pH/wAAJL8EJxh6qCq3QvWntoETcQAAAExBmiRJqEFsmUwIZ//+nhAAAEVUvr3rCT4UqZsiUiAC0zFw0MS+mbS2bWJOhtz4giCUsrv/v21VuKctvrAxKDDQshiwIovIkgD3Jyi4AAAAKUGeQkUVLCP/AAAXUJjnPOfZACaNnZ/HVbQ/nM4YRH0Xk2QPc/bESAEnAAAAGgGeYXRH/wAAJKt0hRh/j1n6jhvqe13+OAh4AAAAKAGeY2pH/wAAJL8ay0UTx2QAWLLfMQL7ZQ0I0yLKnxos1G9SUP6cvIEAAAA+QZpoSahBbJlMCGf//p4QAAADA7nqKwKAu9ad8f9scax9qutoNRwaK5UjLIoAV46j4nMKY1UAvsNbPYNO8tEAAAAiQZ6GRRUsI/8AAAixT+jLg9OZ/fDLbY5ksEreeTDUu7HJTQAAABcBnqV0R/8AAA3OFJF28vxUU4vi0mxFwQAAABoBnqdqR/8AAA3UnhsVmd7D6ZIOVz3ZnozsWAAAAHFBmqxJqEFsmUwIZ//+nhAAAEVjZ2spyAIfGaYDl1iDVGUSxw9LxYEoGlJi+2nb68IvG/nVMlLIWunRtz55vqt43NrNFYT9+rKrLcI1Ae+4DvDil6//eR9LM5AaLo9C/rCLj9phiX1HatsJjPbZ1LgEYgAAACRBnspFFSwj/wAAFweUHnr2mG5fozZUure4AUR2z/VikVrCnJsAAAAVAZ7pdEf/AAAkrDkGTNe2A5+5r5LYAAAAGQGe62pH/wAAJL8EJxh6rKpguVJCqmIZ97gAAABUQZrwSahBbJlMCGf//p4QAABHum9Ak9rJFAEE7NGXi5VCrpK+r5g1Jfurcbi0B8R3LdnMguKamkaTjbN+foFEr8XdD1RyQzb9LZ3rCo0OIMXJmnBBAAAAJEGfDkUVLCP/AAAXUJopLjilTjkIid9c8WLftYk9FbJrRhHFTQAAACoBny10R/8AACSsCptp7/8ISHJVx88ALFnwJ35C9rvBPfNwsTjAJkQEaXkAAAAYAZ8vakf/AAAkvxwG62ExaLuyuYISUD44AAAAXkGbNEmoQWyZTAhn//6eEAAAR0UFRY2tzol3cEzo2wTKACIPW/eHfedgQr7900HvIX6wj4PKT2BR22keMVXvguzxGSOorpKY5tWGrM80+1sAZOdb6mlfaC6Pk7UOO+AAAAAnQZ9SRRUsI/8AABdQnB1rQzJvBUA77RNIVwdFOskrAjpr9X51cMvBAAAAGAGfcXRH/wAAJKw8Ws+WTZlkln0DllmNqQAAACUBn3NqR/8AACSxzAlvSe9Hm9IAAv0iZtj+bnXhGZauaeVIIDFgAAAAOkGbeEmoQWyZTAhn//6eEAAARVE+9VnvPnllF5w+6tyoePHu9mlWmnICJC0oB+dmW7EnlOYAWQWYhA8AAAAvQZ+WRRUsI/8AABa1WwKWSVvsTavZZm15AAdX1YLZR0WKz8WWUD9uZmhgCNXD/CwAAAAgAZ+1dEf/AAAjrD0izHMhg5gAj3aP3OjLyySjyVxNojcAAAAbAZ+3akf/AAADAAqGY9BY85MR3ayF32+U482hAAAAaUGbvEmoQWyZTAhn//6eEAAAR1S/BoA4p4DGHz7WzW64S1dLhBxPyxL+ljrCP+BLpBinHyWy7sCZ6W/rxiFta6WwrUdaeQ9Q2TYysd22bfPLyeSXLeJo9xjLp60jwRhp3UHOiB4YCQ5TzAAAADNBn9pFFSwj/wAAF0UPB4VF6+Z9OYAODSlHU60drzlAyqAP21nuJP9ErndeM50zm4Wv3dEAAAAmAZ/5dEf/AAAkq/hX5TcADgs29bO45A757YDc+G8zcExCxoCAxYAAAAAtAZ/7akf/AAAjvxxYr4VCo6gA4LNvWzuONzQG2A3PhrvWxqq/YurhdtCb0tSBAAAARkGb4EmoQWyZTAhn//6eEAAAR1E+r68YA6398eFnHTJKABo2YVNBvPr9872idlh2V37TEwawe304h2Y4ksiUusQPDimByoEAAAAdQZ4eRRUsI/8AABdMQ823FcysmUlEy+GVvjS3+YAAAAAoAZ49dEf/AAADAAqG3yZEABEGFv6Eywn1NV0lF2I2+Ph60Rk4hCKS9AAAACIBnj9qR/8AACSxzAlv3dkgAcF6n308ir8sqC4wJf2XiAtpAAAAR0GaJEmoQWyZTAhn//6eEAAARz6WgApELImHHG4A/MMWQFYQn9dEXNL/0x8HIdwpKkPBMom+3AVrtTgx+fZBY3xAzpxfZKOAAAAAIUGeQkUVLCP/AAAXTEPPjEHoqwo1+MOOAVQjnG3oZKSPgQAAABsBnmF0R/8AAAMAvuANISasfnFG++oCDyn51rwAAAAQAZ5jakf/AAAkvwQrwAAx4QAAAFpBmmhJqEFsmUwIZ//+nhAAAEdRPp4zwBKYfIbphAu5z+RHgux9ABGJaSlO3RdI3xlPmftaKZF2Tnjc5wO3FIXAbmyexG0s8ihk3PFAldDgTT663Y1Oz09VI/EAAAAkQZ6GRRUsI/8AABdFKzdrWl64ooPJsivbUy87VQCFdWi077uhAAAAGwGepXRH/wAAJKv4W3gSnVfl37dJ55VNo7NMwQAAABwBnqdqR/8AAAMAGwsFobOPA07iMEdvYqVfYr3AAAAASUGarEmoQWyZTAhn//6eEAAAR0UE9xxhv9oL4tAH55+fJsUK8YN4LR5LGN9TfcMCcJTlDU6CCrLKIhYljlwxGasCOqPZmlwLZyoAAAAoQZ7KRRUsI/8AABdFLc1QBWpRNpA8iPGfmkgHR0QYzCb7qACPNAuSsQAAACMBnul0R/8AACSr+Ft4GFXIAFqI2etlkR0CuH/mGTf/6vpceQAAACQBnutqR/8AACOyMT8TCa1TAAkdWz1ssHQqFK3ik3AdJ+kESJwAAABEQZrwSahBbJlMCGf//p4QAABHVKk+4/yBb6Z25y3vrKfEdDY9ZAAUqj4Po0p2dTMw2Ax5qZskNebm6pjAqH4sE6ZiGoEAAAAmQZ8ORRUsI/8AABdQVnfG/NZ/ODpRolMo6keGB5NQoWcN1O/xo+EAAAAuAZ8tdEf/AAAkq/hX4BJeC8M/mdu2ACH4cCh3PtTARsnV52jv1IlOOea4+gr5eQAAACUBny9qR/8AACS/HFVfFtBJgASKIXcA+kLXnVrbws0HKj0788PAAAAAX0GbNEmoQWyZTAhn//6eEAAAR2jyQAATQ24W00naYb//j8skFydRfIj20BGevpV2Z3l3b4vLHZX0ILHUfW4Hf9oNZEqVdVSGIUpjEgEBqV81UFn7SzidFCHsNiruwL3ZAAAAO0GfUkUVLCP/AAAXUFZ3xvzXgnAB+2jTtPqwUkwh69vcVpzF67Gzogz7tjUlj2QCBjhP+uYazVhJiNWBAAAAKQGfcXRH/wAAJKv4V+AU/4ACR1ZmwyN09emps4JWdTvXInb/S0btYfzAAAAAOAGfc2pH/wAAJL8dLBkN8mTUANufqel5ADuHVTKbiJnUhMRxq+jTIXk8DvIRCrnSDy17w28cryI+AAAAQEGbeEmoQWyZTAhn//6eEAAAR2SOFhZbdJVmlyvnUqzwmOgopU0hz743mUjamO3cvo5rcuAFFiM9bvxrgutp3YEAAAA7QZ+WRRUsI/8AABdDR7cM8bqQAmpWlTCig/nLdqQQ31CcZO02dW8bLsVMzzgmGe2gRl7s7C6l6H8l3dAAAAAgAZ+1dEf/AAAkq/hbeBfVZ1iu4KMP7mliu/aAdVZE2YEAAAAgAZ+3akf/AAAkvwQnEtDJp7VykqOrvSgAC/zz1PfVW5UAAABnQZu8SahBbJlMCGf//p4QAABHRQXggs6Ib+3EAoEnBvQ/xxBTbqsWpkywXMRwIC3KYNsdH37kBToWs+VfbQfnfiU94rsNwy6lpmTD4EORgfpuH+HpZ7YaC4hLbl2J630B7FSFCB9WwAAAADxBn9pFFSwj/wAAF0xDzR4YYIkGHIFNvHEYAVMP+PFFTF7iq59W5aQOR8vEuLb6Q+5y0DI3J/FIUGbx0R8AAAA8AZ/5dEf/AAANzgDSFI5AAXE/rl+5QxmnphDd2iC7Vk/HmXHR07qD4zr+NFSpDK3/8K9j6rvaTiM0wab0AAAAJQGf+2pH/wAAJD3MDh2VtkRipKoxNclGjv9US2hHDl9l0M4lgakAAAB1QZvgSahBbJlMCGf//p4QAABHRFyBwA6BvC86ltlgfbDZ3L7eJAV6e50yLOErgj3zRkHk4HH7CAXFoBcYcJOkfivjFoPT6EN4XkchK2Oqf9l2EsUy8B4RDU2ZV6hfcm1uJQfasPF0uEcJMFVKm24epX+QCYSBAAAAUUGeHkUVLCP/AAAXQ0iPeOggCJe8UfOem/hNrD73MkB2SEMLYHpxRWNMCM4nf3zRZ4erzh6Ug7aPufkN06UxGAwOQ9CpnO29aHEs/MEouI8VlAAAADMBnj10R/8AACSsO1MHABcTCXMPKJkKPqL2XLkzAQoUV+TDDijfgJS4BBJT8woYnY7g4IAAAAAsAZ4/akf/AAAkqN5EwhdjT9PvoU8WdRdaFaiNSEltSchGZgAud39Aqcj3jjUAAABKQZokSahBbJlMCF///oywAABIM+mJMAgDecrR4U37cQvzOWQb/VyIu+R2IY61wbdosONbj7BgeY3/fCVGCVrcKrjlLu0ZeudJyYAAAAA9QZ5CRRUsI/8AABdOaeEtT14Z41Tjp+sW20V4H7TOfGx+gBEHUkxv3Yb31A4tobnp933mheQ7duqLZuZLuQAAADYBnmF0R/8AAA3OE5rBwAXE/wZyS0RQeqS/HPMjlaTfXOdozpeyXKALf9s282FkZglIMXvv73AAAAApAZ5jakf/AAAkxyv+EOMLS1U7wBazvJpRVr+/o2k7eaXReRyc4VJ9OvEAAABYQZpoSahBbJlMCF///oywAABIAfJ8xiG9kwqHaxSNP9CIOhz3jNYAOGNciJlXz6jfjLzon3+FElUutppnHhb6EUaUGjI+NdjMXXr+Uk9n0uBYXqBG9s7s7QAAAD1BnoZFFSwj/wAAF0NIi+JMJUjWZPTTmarRhC3EU31Wjc+okyMhsGx8d+Rg4JuDpEIDV5wGuD0FJei5BvhZAAAAPAGepXRH/wAADcS3684skXQgAlj81aUz0CstXGvuxussBk7vw+X1i8fRQeSZdC3C9YUx1QT7R5prvEtPdQAAAC8BnqdqR/8AACSyLXjbOITfyRqr5gBbd7RechcZZy+6YD+OfwJGcwjtzFIKeCqiwAAAAGBBmqpJqEFsmUwUTDP//p4QAABHRQXvng2c3ZckOg8aH4AXXS5affTBbXckOUYI7QaIanPDE2x+5I3zOZNi4BCD9oiCtGODAf66owtIJd6SGMm4PvSEg7Dzb977m09Pt+AAAAA0AZ7Jakf/AAAkvxrNrkd/YWkwAts83yNXSd5fcEcIKT+WzyKhYLZBhxL8iAhOMcVYWIZQWQAAAEdBms5J4QpSZTAhn/6eEAAASURZ9irEANKvM3VKqX34FwO1Gl1Wxx/4DSA2H6lXPibImCSI+S20dFvBuXUjiY1A0SseScNQ4AAAAFBBnuxFNEwj/wAAF+CHtw25HK3e/OAATTZljKhIS9LLEvH2vOKFRmfNevK0uCT/bP0ziIsrrR2Q3F+lU7yUWJGkWWrDrqMNqPasWnLp6kZrPgAAAD0Bnwt0R/8AACXDF2ippPQguWNtsFoAJHVuEUaNYgI14jSi+Khnmj+UoljIfhxEGAxhlTUYGXg4lKdLUq05AAAAOAGfDWpH/wAAJbIlxAIACMixf0zAdRTxtRrqzijPRSFE0CsvAdaMUcL+bCdgixKx5Ai4opwx1As5AAAAV0GbEkmoQWiZTAhn//6eEAAASURbbgBsdltcJ4g6n/clzHXhbi+Ydr0GbsH7lsDx0V5eadykO1k/Uon41s+mzPouFNrVAwpseWs0RGIU3m4RfGVIcDpDeQAAAFNBnzBFESwj/wAAF+CHtw25HC9mePsv7LgDY0STDnTHzIrjig0FyLEwi2JUNwbNRremp9oV6IKAjY81J7dMztxO+cl25v14WUsRHPeF3puG5gbtOAAAADUBn090R/8AACWsNt8GXHc0TEy7V7+QADgLq9kpZTgi6vfr63sIErcezMOqhgoGUxFlXMbPgAAAAEsBn1FqR/8AACWxy/yIzgAlQsXxvgSsfi2fy30Iha/ln6St0XaWeVpVN4auPjiEK4OH+QeOmHpiGw7RG7w41KtyEv32vxhbWHYfdYEAAABiQZtWSahBbJlMCGf//p4QAABJaJPOg/tRH4ALTMXDQsGMn0zaYX1nWFTxTVbYdIxXs1qwRxbVOBKuMsWlsZi8CHgJ4kJMyF6bqvBgIF3dX8q30Tx+J/GJClDMXARxzu/3kMEAAABLQZ90RRUsI/8AABfom04dhIGLgAPfE/xC0o9n6hE+QzWp8eX8Vs5N6L2EOQNPI23zP9oB4hWu+DTkc/QZQsLyQr3kKWP4zkKujs+AAAAAMAGfk3RH/wAAJcCZ3elekTlCkAC0Z3D3fcg/wZQuGPknEuVqA1QzacQuEX12u+B2gQAAADABn5VqR/8AACW+9T4gxys5YNfmotw7g/6B9+xwAF0AGrQaMUb4pZByJj4vTP7B7rAAAABSQZuaSahBbJlMCF///oywAABKJAKEwmgDllQoosQpjt3MP0IMDbKsDyWliK6IGLokfKysI1eWvXARlYj9+fVTEgS+fy2t4IMxJH5xDptpeCtYYQAAAEdBn7hFFSwj/wAAF+jKlGRW9Vh/q1YACdT8jA7npKZ9mB2iFlCp0/ZYq2yI1irbSQywM/vmZqdw1aAqSF9wnJPq8TdYIJJrPwAAADwBn9d0R/8AACXDF2KzwAhKxzV/p4AW2eb5GhnCeDy3BD1sFW363N3iiedHdq2aNtnL6dE3Wf9ZggCqG04AAABDAZ/Zakf/AAAlvYNjHfxdcfu/56T7yA/t9aABbgBrncOE7mKAuYyzDw0+8ocCZ3z1AZcH8yu4cJmlrguskWEefb9pwQAAAFdBm91JqEFsmUwIZ//+nhAAAElDYkkACMiJXA++WH9DHSH5AogKXmNLaUf4JoAXOIq1exIq/aPO8wPP7Lr/WPhj3mrKXrU8yEeWZ53p2VzoBYYoDkr3p9UAAAA1QZ/7RRUsI/8AABfthOIP2Jn1UBMmhw9PWOFNAFcF9t1YCxmZDGs9K2LCcEie0cwosuiZs+EAAABFAZ4cakf/AAAlvYcggA4sImJkRWF0lu/487aXdnJUW5Ht0dsTik382++bX1eXXQfQfBIufMX5Obe4ipDWnSjlQLy3VKtPAAAAeEGaAUmoQWyZTAhn//6eEAAASVDmUG2ACMj6n1UIx2CTgxmW3x/t2xMxEiAvqXA3etp9QdVM/cwD9HLjHoqN/CIisKFpwdIN+NTqbAuDVzqaRFMUge7FXfL+PYo0vdSMwvElTeasSlQmlk9QbGtbfZu3WecEOa06vgAAAEtBnj9FFSwj/wAAF+1wvGbfvw3OAEsPJFGupPN//BTRNesoNTRPTsdOaQTa4cnR+Xu2/Z6CWhT/mED8asUg46UkEhHyGawPzU3DusAAAAA4AZ5edEf/AAAlqer3m+8OwMYTQovHCWQqQdiODNJ71u3le8AAmRnzWQpJkdceUEOy5FCiWQyBacEAAAA3AZ5Aakf/AAAlrl//q6AC56vm3ecSx/lqq9b73MGlwJ/rzFNgqcMApd5H0PbzVktW+wJAnVq6wAAAAH5BmkVJqEFsmUwIZ//+nhAAAEtDkAEAT4VbJvLr09bSlCATXf1WwBRYyM6Tfb+tNh/p+Hwvk8jfBerYrKO7ULHlKWgMiFoJabOqKjK+XDy+usc0MfSFhk213sIth4A04G7BYjW2fU9Vya3fqn3vSKB3ildPHyaB5Cz4xY20+nUAAAA+QZ5jRRUsI/8AABiJhsXUvsLcAATVI9nRSTsbUrpJbAhgp/0YZ3j7WmvRdxUtMxkjTcu/9TYKZCq5SUosWZ8AAAA6AZ6CdEf/AAAlb6Ha5iqi6DZACVQP/e4FDvyYGbx/RS+Z/7HxgV20HDDZ62u9ilntskUfvOKHRoy+sQAAAD8BnoRqR/8AACa9g665wRsaEISAE7K9Uy7heDoIPvg62EQYAEeVFgZjRUTU4JBUtcCO+3NNA0GtRbXd9VIz+fEAAACOQZqJSahBbJlMCGf//p4QAABLUOsJ1AF4To1WtL5S76Y9KyEZItb1k3+FFZDwIO5pfto2fvu18b6/MUUtx0oINiEMuA4ndQbKDZTQ8jMV7lphVaBRN4j29BsyBrpNlUNTL0agV709/1HLGZlSZDmvKhutgwFh2wbApE/ng3YR+1vSIZ4nFSHOV7JtCtQgQQAAADJBnqdFFSwj/wAAGH/sRKXCgqIcpGb2Jip32Ojz/xWQVkqeDKYa7i9gXO71Av2cCAxz4QAAAEUBnsZ0R/8AACap6lO7XbV+VVB2PkALcaKXpyYgBKatYj5apCClKLAgIr8a9xsellBvli5edl8/mFxVhkO2Qm3+b6KO0e4AAAA9AZ7Iakf/AAAmrl4zgugAnXpzkLetpmIQNksxd3HdFq1aqZHsdquF0ADYn03KTT8ImzYVjA7/qXvHeRP2VAAAAGhBms1JqEFsmUwIZ//+nhAAAEtj/wv5Kf+oBvAWLRvhbnngm8qL/TaXzAgJWH1OetXLToaqR99bm04o4/kIlAo3MWRkQAXUfcLOh99rCmKB/YWq13m1NcamRiM0Xy+ASp94fcclQ1I1gQAAAEtBnutFFSwj/wAAGH/sXFFkWSNTn8+cvETQ55+XYlDlUD/nylw+EEgr0Vk5XLlkHXuorxV4ZjL5xyyKgQ/pI4n3+0sFfBK/oR2zdwgAAAAwAZ8KdEf/AAAmwJkfzs1KUvgWkuPZnvjFg2oFoNRjPNmQFZUzsRMYABOlHqtL/sqAAAAALAGfDGpH/wAAJr2DLs7kSlkWT9PPducBcAwmVnykxmPVwH7ht/54JSyePRs/AAAAh0GbEUmoQWyZTAhn//6eEAAAS0PNsG7Q9HAZR5GAGzPK81Xfdce0riZiepmaCufCnVXkeEPisSeaC6VwdaHkkLJFiCllu3eggpHFHjPCZM9ve9NsVrDFmzNVI0C43X2yQPwVFbv4wiKi6Tggc1v/7++kZ5Tp2GxMwYjRLSD6dmV52layzkRF1QAAAElBny9FFSwj/wAAGIibgrVbYnEZYHGPsl4dHvRd8EaQQAoUTNTOhZr5HBUwF6qCJiW2RQAEzkyrszuwnj8LGszc+3X7TVv7qa2VAAAAMgGfTnRH/wAAJsCalcmDUe1PjR97WkhdpKjePLNPrHOGTaZP66W77tWxNW/Y8o8BQLZUAAAAPQGfUGpH/wAAJr2EBAWGnykoeaKIg/MUKrk18/wZvmRsfkovwhkSMVAAnWG2oUN6bTk/fw3vci3Mrzgmz4AAAACEQZtVSahBbJlMCGf//p4QAABNUO/FwKlCowEdq/ekAQ7yz/1vyxIAcuK2Bcn+vnWJXqMHU0Gf57ZWRewG1tvCeajs6JNwinS76Eb8oygS5tuv3+ZqzxPQ+4dn2uDJcjr2Qmim3V573AK6r6rzIi/8xpCJcbX2TOpz82s+lhUHa/Th/Zp7AAAAVEGfc0UVLCP/AAAZNveAat+lmhtYJGICnaFRSHnCb28gaNGVEZQdr60E06wv6FXwDWHkbyUi5t+qrXfNeTMD+I+9aiLUs5Wm6wwK7RYnJtWsko6dwgAAADoBn5J0R/8AACfeSYQJijQMB9VmmOk4PhKEQ+EwBz7Hl8x4FPzqJ/bT5gBB7zZ6Kba/oi3diPNTX7hAAAAAPgGflGpH/wAAJ8VT0zpJ+dbnsRsOEG/nxMYe2Sjz1DQndW65dd78krLEqAD5CrVbrJq55defwV5kgGp7h72VAAAAXEGbmUmoQWyZTAhf//6MsAAAThCmy0MleeIgJ7TRu+YRCltRp1li97i+LkXE6gemd88Uond9r3M3692j5ZzqQfRLUplxgEICwDjYAH8Ffgx3/3dWO8C43ZZJqKSIAAAATEGft0UVLCP/AAAZKGMkb2aoAiUPo1po0IAl+madVkBHG/rhPp3ccaCZ7Cfnx0pYUr6dggew5pg9HT+HGaDJKsM4qhpk4a/oFT2BR+8AAAA6AZ/WdEf/AAAn21cmKIYKj0YbOaMkNRnbyzWw+U6Rhnk9S9ruD2ohYACasX5rkY5PSZEEslg7VVtrwQAAAEMBn9hqR/8AACfXqRxKsVUHdna8Iom9xp30Htir3iRtogHCZGHM+K9UXFYlpOJ0gAHvXSoBI9W5vgFrjDN6PJxf2zdwAAAAhEGb3UmoQWyZTAhf//6MsAAATkCHl/SlbdaPHk3XFg2cMQAW1WRoqZ/HR1eG1NelI3tMh0j9MHHjyz8JnpGc5Y9FrUgpLZ/mEGK/Al775RTthUQQZLfJqC4OPm5l9ql9UxtcF7RFGGa5eKbf5f7SQBuFSvUtt9/mwukaQu6FV68k6uQygQAAAFBBn/tFFSwj/wAAGQ/L2BmCCXZV67E+309VdlyVif/wS34oF/XmjWujQP1fF9TrlqIopwpMNIpJIHfYoSDOwOYXoV0XACNHod3Bbw6jrWh2qAAAADcBnhp0R/8AACdJ6dPCBjkVFfw9HkbQXT2RXZ+ufZ7u4lCYay8ZQZunHILEg5fNHAwVbudL82LBAAAARQGeHGpH/wAAJ8QaYXG12RIBjpxFrxX1/N+HdrradDgQoMlD1wFEqQGuE/yeDZ/Hl8AFvEtbL7bm+9MhxUtcy+nxl/02XQAAAHFBmgFJqEFsmUwIX//+jLAAAFAqMjiVrrLRdmy6RYzo24/s07ACEACM2Rx8uMbDuvE/EWj0z2r2y+/Jckr4yatHUYFE5VZGdUJN4HCK2+J700TFahgyHp2s+84vomhTgClNpMO6e0YMK5SCFeHYafSFQAAAAExBnj9FFSwj/wAAGb9Dtf1kKADkoFNGuYxJUMax9tgLjpY0psbiNMx/pJYKPtdPdfBck8buAvepsbXoHcSpUZ6MPOgDo8lv1OFQ39qgAAAAQwGeXnRH/wAAKP4z3ZmmZonwR0nvb7FIIB9l+jKbxSUGHmR6bHbdUutvhpOxX/AAJylnSJBc/97uOe/nGyvUYJOn2qEAAABTAZ5Aakf/AAAo5PIYH5LQ+8GNXlUC+VqOJEYcAKB7m4giA8FewAD7e99bFEH97Z5SGdLusKlbf4AOioDA66kt+E/3Nv/zhgjiGn498SzF7UjvaoAAAACWQZpCSahBbJlMCGf//p4QAABPbC+cABFrzQANZrZmXLJreMIapCthhCznvWNYYXND5Wjz7pKwwJnPr6Bu0bGLl3paXLH8VnWWImYUDbVULtvn17rIb39mBp+Y/B46+NSpc+5WRZjMjrbwZ1axftM2Ati4BHVfNiwOU+n+jLnZ9dYzE9a22fVehTu08YGC83fP6Kz9uSJXAAAAfUGaZknhClJlMCF//oywAABQKmL1nACMDGYUS3hWgA3pxE+owe40D58U2qRYlSw7cdBdu/HkBKrAGG+zyWJFDETeVNUDz2slUy1Gjw/xusU5t7u0yohiAp9zUYeR1SUieohz+nX6j8M/++H/+euF/65ZUjFSXwRoHigmS6jkAAAARkGehEU0TCP/AAAZyHRnFQVIjdzDLgww1eYEaZMMeMG2t+KjqS2oa0rWOUubzuq4IxtRcdkYliw5d0GVqA8xKhrGVwRty7cAAABOAZ6jdEf/AAAqHliPWP6grBjcR1QaEY5REEn/jdi5ASkTS7HaVm06nQgAlfQr8H3QvcX6VNT5r5SQfiJhxSF0/s9M/kz16YdWEl9TDqcXAAAAVAGepWpH/wAAKiP8OJBYuB0m7yxB3gika9qfj+rD4rUlX+RH7ay2e677fGOiroAJjhPwV7fTkGjpQMjk2kLXNE1A/TZxb2RBbxyIJnb+gD/sVjOwIQAAAHNBmqpJqEFomUwIX//+jLAAAFJqa5FzADoFqGK+K8zuH9wAvCXXYcwOYGxG8DV4+HQOZ5MORxCPlA1BdnP/i4O2nPRaZBkaxmbmvP/NBt6BFAfXrkG1oUiEWXhuwWuvJE3/MsTCJInA3k58yrRNE5tLaFgNAAAAP0GeyEURLCP/AAAaX3/V+AOVa1ovrqIFIW2y2noEyaCoPA9TLCoxjXR2cwbyLJGoA1F7Ld0B+5vi5U21CU71gQAAAFkBnud0R/8AACoam0pLH9aV9zBNy61i/CAfRR/BuNm7+ASCT62qalKsxdRoeMlAi2AAhTtH+yJroxS/A1LevZ4IYfo6btdBk4qVTJhIHNAcl1CXAuiQw2UOywAAAEoBnulqR/8AACoCSAye1wnwbbkhiet6drPjEtfijVisI2JbKcnjmh3A4SGI3RcEoBACE4aHA1iX1k7am9FPhuiSXvC2Xz2RCIrbAwAAAG1BmuxJqEFsmUwUTC///oywAABSahquQAOMDqCE5t0JnjkT6jB+RyGPbJjXy8Ka2UN/xsAIoV3iH6jzgv//bkp0UGTb3OQwPnabQudTdmcCmJK93hD+cfnUqJkmBY74cdHncOAU5u5olnDAO2KAAAAASAGfC2pH/wAAKgKc8wcPjN0R0k3/wgyFmSgIoh9Yrq9FRTfWsPc4KyCAVJsABL8wpLu4sz0CAwQ7qbXXrZm8h7UeLHjfxd048AAAAIVBmxBJ4QpSZTAhf/6MsAAAVKpuK5AA4d1XMC/2AAmY1Beteh6gTirrBcvqfew9v2Aasj6xloVQ/hs6UvRgn5MfqK+BeqcCtW9klNeVrSoVz+tzs/doDGWzQ9fCp6Qd21nsmamnI8vfCIsR24VO2Q5Yngu3GTOcv7dxFxHylJa7d3jdBoaBAAAARkGfLkU0TCP/AAAa/4WvQthxBBAGtE3KYfso7Nf1evtrFaSVeVgl5CnzIuXgu8R/JyplweGekBdMX9kNeGm9uPFeRGuQwIEAAABOAZ9NdEf/AAArPinvInzRepAYZDOkNg27QPgmBToSw7673vFBbvQ91NiRN+/bRzULSuABI92mylwDG3fdIkFmGx3HYCSxgvNKTtdIpEDhAAAASgGfT2pH/wAAK0Mla7+kMV4AKj+gaxE9tGwREc5APhWwnDKMyTuzYNxEqr6eaADaP6jtil58lYLseiEC25w/fx90zgcG/KzvNa7gAAAAnUGbVEmoQWiZTAhf//6MsAAAVvGZACEepQl/nzAKqAjf83i+oJ/NaT7Y0bybBaX/NkTPve5J/ZhgnZJCdXqlo0JXho1PoV8AeYJDpWp+oCaogEGQKpNW4IjO1BEZNxMyOMw/d4upGyJJMuoQxDEulOX1qoJkRRIm4ekDN/NGxtxa39lvjiJvE7eccjZkLwf8UzBuF/IF/yiCwA4WtEAAAABXQZ9yRREsI/8AABupgtSrwABxcQvRmjXhwThLrjO6HCB1tf20fjGe8onPzTfGmtZbdzrIxv++OBVQjiPSWG2c93juwML+hFPI9xVP6tT8Uw++RQ60GzdxAAAAWQGfkXRH/wAAKxP78o2L7duJ7KqEALbNTzS72lu1M35y28TUjrw7Lu1TTE5sPqdQicZNGDBjqfaW+74SiRFFTty2rySNLxoLwqneWLzYauMH507vdhR9OwyQAAAAVwGfk2pH/wAALFlQXdgUXvRim5AkzPuhUU3q21OBz+nlP9AbpNPp6QhRp1i6NFs/xPb0ACUu8lsl9nyBIaWAp8kDOkMc0CBU0uAjfo5IxSS7sqz5jHT1gAAAAJlBm5hJqEFsmUwIV//+OEAAAVC3GNEevJo1Anc0JcUm8tEM/KBMICRyi0OeQAki1Uq8ZY0lT/Zruilk46AxHqLbUCGRyL+8S0ofWUngjJnUiMkuo1SD2QYl0Zt78Vsg02iv4ckZGHPunzmy1rTXDGQfGUPztWorjBAs0yH1TcEEh01N+1pnH/H92mALDGseRm7fY1e/qkovFZEAAABZQZ+2RRUsI/8AABtTB2xpaO3lv7u5u/ekqi5sBDu2FuMo0Eo67XSB3snYGJFfLAAN0iGioCuuLWYyLtAC7FDELH6uSwpV6Kvwnt1h42EeMBE15U/jTdKUtFgAAABIAZ/VdEf/AAAsWbXrNJvxutfhNFzegavk10f1dhSvW1V374xHmktR2LZJk0W1dg59mEWa3xc6RdHoGoYBOY4JWMBsGgryjO1BAAAAQwGf12pH/wAALDjNas6IfqfNtIz5SToBvGY0PB7zt4sLoULcvSgqIo8OipJqI7UlmMnsSwNuvSAAXUXocmbfBAvgQYMAAAB2QZvbSahBbJlMCP/8hAAAFGS80RgCtvGtB8vzMsvdTaJlgbT98e7hA+pM2Hz7Zp0G9U71+5n+x7k7SyTrWXeV1SI+pEP2PTleUYVoN9EZueHwDE3XcT2tCUMglo4OQeP5P6TWcIMo/4vMr7D5oTHc1fbRR7+0+AAAAFlBn/lFFSwj/wAAHFbexIaWY6fg5wqz96nxS5zB4rJSm6MobWnnpYxYIZhv1zHvOXlZS3zL00gAgBk8dIhH28PTXw8CKn3I6jEXX7meMFVVh1MPR5Xln5nm2wAAAEkBnhpqR/8AAC12jrU9HPmZAPJln0TRIw5lZnOBl+gzPX0OgJk83wYdH13iRGPQxetUKzIqYjMIm9sErko15cQAlaj6e4Kog9CAAAAL021vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAA6wAAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAr9dHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAA6wAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAJYAAABkAAAAAAAJGVkdHMAAAAcZWxzdAAAAAAAAAABAAAOsAAAAgAAAQAAAAAKdW1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAMgAAALwAVcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAACiBtaW5mAAAAFHZtaGQAAAABAAAAAAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAAngc3RibAAAAJhzdHNkAAAAAAAAAAEAAACIYXZjMQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAJYAZAASAAAAEgAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABj//wAAADJhdmNDAWQAH//hABlnZAAfrNlAmDPl4QAAAwABAAADAGQPGDGWAQAGaOvjyyLAAAAAGHN0dHMAAAAAAAAAAQAAALwAAAEAAAAAFHN0c3MAAAAAAAAAAQAAAAEAAAXgY3R0cwAAAAAAAAC6AAAAAQAAAgAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAAAwAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABAAAAAACAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAIAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAMAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAEAAAAAAIAAAEAAAAAHHN0c2MAAAAAAAAAAQAAAAEAAAC8AAAAAQAAAwRzdHN6AAAAAAAAAAAAAAC8AAAEswAAAQYAAABVAAAAMwAAADMAAACFAAAASAAAADoAAAA2AAAAXwAAAC4AAAA4AAAAHAAAADwAAAArAAAAIgAAACEAAAAtAAAAMgAAAB0AAAAtAAAARwAAAB0AAAAVAAAAGgAAAEgAAAAmAAAAJwAAAB0AAABwAAAAJQAAACAAAAAcAAAAUAAAAC0AAAAeAAAALAAAAEIAAAAmAAAAGwAAAB4AAAB1AAAAKAAAABkAAAAdAAAAWAAAACgAAAAuAAAAHAAAAGIAAAArAAAAHAAAACkAAAA+AAAAMwAAACQAAAAfAAAAbQAAADcAAAAqAAAAMQAAAEoAAAAhAAAALAAAACYAAABLAAAAJQAAAB8AAAAUAAAAXgAAACgAAAAfAAAAIAAAAE0AAAAsAAAAJwAAACgAAABIAAAAKgAAADIAAAApAAAAYwAAAD8AAAAtAAAAPAAAAEQAAAA/AAAAJAAAACQAAABrAAAAQAAAAEAAAAApAAAAeQAAAFUAAAA3AAAAMAAAAE4AAABBAAAAOgAAAC0AAABcAAAAQQAAAEAAAAAzAAAAZAAAADgAAABLAAAAVAAAAEEAAAA8AAAAWwAAAFcAAAA5AAAATwAAAGYAAABPAAAANAAAADQAAABWAAAASwAAAEAAAABHAAAAWwAAADkAAABJAAAAfAAAAE8AAAA8AAAAOwAAAIIAAABCAAAAPgAAAEMAAACSAAAANgAAAEkAAABBAAAAbAAAAE8AAAA0AAAAMAAAAIsAAABNAAAANgAAAEEAAACIAAAAWAAAAD4AAABCAAAAYAAAAFAAAAA+AAAARwAAAIgAAABUAAAAOwAAAEkAAAB1AAAAUAAAAEcAAABXAAAAmgAAAIEAAABKAAAAUgAAAFgAAAB3AAAAQwAAAF0AAABOAAAAcQAAAEwAAACJAAAASgAAAFIAAABOAAAAoQAAAFsAAABdAAAAWwAAAJ0AAABdAAAATAAAAEcAAAB6AAAAXQAAAE0AAAAUc3RjbwAAAAAAAAABAAAAMAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTcuODMuMTAw\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0EM3ifftw_4R",
        "colab_type": "text"
      },
      "source": [
        "# Tensorforce"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZscPrS2ExBl8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install tensorforce tensorforce[tf] > /dev/null 2>&1\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ELCSz5FOxGhZ",
        "colab_type": "code",
        "outputId": "a020b2e5-4e40-41c8-e660-fa7af8e514cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        }
      },
      "source": [
        "from pyvirtualdisplay import Display\n",
        "from gym.wrappers.monitor import Monitor\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()\n",
        "\n",
        "import urllib.request, json \n",
        "import time\n",
        "\n",
        "from tensorforce import TensorForceError\n",
        "from tensorforce.agents import Agent\n",
        "from tensorforce.execution import Runner\n",
        "from tensorforce.contrib.openai_gym import OpenAIGym\n",
        "\n",
        "environment = OpenAIGym(\n",
        "    gym_id=\"CartPole-v0\",\n",
        "    monitor=\".\",\n",
        "    monitor_safe=False,\n",
        "    monitor_video=10,\n",
        "    visualize=True\n",
        ")\n",
        "\n",
        "with urllib.request.urlopen(\"https://raw.githubusercontent.com/tensorforce/tensorforce/master/examples/configs/dqn.json\") as url:\n",
        "  agent = json.loads(url.read().decode())\n",
        "  print(agent)\n",
        "with urllib.request.urlopen(\"https://raw.githubusercontent.com/tensorforce/tensorforce/master/examples/configs/mlp2_network.json\") as url:\n",
        "  network = json.loads(url.read().decode())\n",
        "  print(network)\n",
        "  \n",
        "agent = Agent.from_spec(\n",
        "  spec=agent,\n",
        "  kwargs=dict(\n",
        "    states=environment.states,\n",
        "    actions=environment.actions,\n",
        "    network=network\n",
        "  )\n",
        ")\n",
        "\n",
        "runner = Runner(\n",
        "    agent=agent,\n",
        "    environment=environment,\n",
        "    repeat_actions=1\n",
        ")\n",
        "\n",
        "runner.run(\n",
        "    num_timesteps=200,\n",
        "    num_episodes=200,\n",
        "    max_episode_timesteps=200,\n",
        "    deterministic=True,\n",
        "    testing=False,\n",
        "    sleep=None\n",
        ")\n",
        "runner.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0626 12:55:04.124785 139879278864256 abstractdisplay.py:151] xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "{'type': 'dqn_agent', 'update_mode': {'unit': 'timesteps', 'batch_size': 64, 'frequency': 4}, 'memory': {'type': 'replay', 'capacity': 10000, 'include_next_states': True}, 'optimizer': {'type': 'clipped_step', 'clipping_value': 0.1, 'optimizer': {'type': 'adam', 'learning_rate': 0.001}}, 'discount': 0.99, 'entropy_regularization': None, 'double_q_model': True, 'target_sync_frequency': 1000, 'target_update_weight': 1.0, 'actions_exploration': {'type': 'epsilon_anneal', 'initial_epsilon': 0.5, 'final_epsilon': 0.0, 'timesteps': 10000}, 'saver': {'directory': None, 'seconds': 600}, 'summarizer': {'directory': None, 'labels': ['graph', 'total-loss']}, 'execution': {'type': 'single', 'session_config': None, 'distributed_spec': None}}\n",
            "[{'type': 'dense', 'size': 32, 'activation': 'relu'}, {'type': 'dense', 'size': 32, 'activation': 'relu'}]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0626 12:55:06.718917 139879278864256 monitored_session.py:240] Graph was finalized.\n",
            "I0626 12:55:06.886334 139879278864256 session_manager.py:500] Running local_init_op.\n",
            "I0626 12:55:06.908234 139879278864256 session_manager.py:502] Done running local_init_op.\n",
            "I0626 12:55:08.367528 139879278864256 <ipython-input-10-bfd5102b369a>:54] Learning finished. Total episodes: 22\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dazPgmxLzdIu",
        "colab_type": "code",
        "outputId": "08e1a740-6503-4f8d-f59e-61b76203807e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        "show_vid(next(glob.iglob(\"*.mp4\")))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAACk5tZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAB72WIhAAv//72rvzLK0cLlS4dWXuzUfLoSXL9iDB9aAAAAwAAAwAAJuKiZ0WFMeJsgAAALmAIWElDyDzETFWKgSvGXwOeIADnZPXApDeigNz18HKF7za6ip9m+CSqjGGNpBSThX1H0GyjoBGEzDhsiDhGCf3z0DiGGjVh9hwvczxGc8EZ4mREcgpSKwj8FmuhP/Cti20qUWVDjd7AnBRysJV19UlUlolr7EIifpQYFLVrNacd50ryo/e1JBZQ01sMW57MF7xE5ac0LjKTYxhFsdDvEp52M50ZMDuu/hAoAcWTlzn1qggUaUd12cfJqoVonf2r7UIAdhN6vuPIXwqoLRfBeS1ZO5EoU9gVGhegGM+QkALzQDvIr/IUW+9+f6b+bh1prmgzsi1sGsRKea/2Nc63OpsZKMRd+0F82vreAJRpDtHb9TvKil11LCYbiz9gCtx4XZlPKEcVGlNQm87ScpYE2aWMseG3b2maGnEiF8Z06dCA8B41VZ7S8fjtAYPXnzPEeIqf1WpmLvtlw5mJOEi7ia1cYjwJhy8y5yp321SucetK/DFqv7zaO/yU3h0JssLP+BwCdcyEfEfUvIzCl78qIApwnvR9HYtfjW0tCQM5QZHJi1KaAVE4KV5ot8JMCLQHUfvXyAAAAwAAAwADNwAAALFBmiRsQr/+OEAAAQ0bUoCwA6R1XmHONnw35CBAPbPFAv899RO0T6zuCkvDJCPr2UpXjBYZDXsxqUrJR/NMe4s6QhWMKFRi0DUSZVvXgeoVfKUCylN/7/irhLyR6pfEN4+9ZaBYYF+Drf9RSu/NPE1lCxo7eF3l4PlsR0NLH99jtBh0wFQMOTEEbJC5umbFzOmZWsFtzA2wrGurELwrPvDcmOWaerRQqgzIc3apq7oADPgAAAB1QZ5CeIR/AAAWs16Psw67wYHuRqTSrJgAXPca73mcjdUtgbCysMQ3DT7PUplwQ+KKhfkdFb3fpTRGfGMbp7yON/NRhwqw6PoRjMr4TGNvhNOri9HGqj/X6tOnbbcmkLtS+jXhu5R5Xns9ADCrv8h4tqmWABsxAAAAUgGeYXRH/wAAI8NCVi117CYswSmjQBIE7PqI+PFiyJ4KRUpszr3Ve2YgEYGZaAJH9WOzC2dILCjexVBgHOfLq01vsduejdsYAACDztC21KoADjgAAABbAZ5jakf/AAAjvwQnEslps2jcXezpGTPABI4W+7RJ2TKMDXvTKfvMbxznaoAbOVYWqlrEgmXw/dOlZZEc6NCT7oUm5Eem2kJ5LhK25df7FgAAGizlb3dqmWABWwAAAO1BmmdJqEFomUwIV//+OEAAAQz2Qhfhn27MlAERzIx/ZAHmmB0gNhTVBt+SGWoz9Pzs0Jotcq4LWt27X2oTiyIZ1kPmjjnfdeIy+zXZF+Qt3dpa/qg4DfvtNDyV9EmxHk1xXhvnw6wwadiI+SJkBO4CObJi/1RXoOXgNIDF9BQ0L55fg+2RsCL4uWj3UTKLxNS65O+ijD9mFi5wKsoH58SejFkJnJ+sILYhLM9f9rIpFfyYAHH5F+opFqfg/41A8f7v3r4Yigw3nvm8vook7UEnpNPfvb5CSRyDPgg2CetkYQ/MWqZYQPCl0LQdPAkAAAB6QZ6FRREsI/8AABa8RSlRIAOHt6TF1B/jZpIVm7LQxSPsYpD7/S2Qy3786ystqxbgaUlB3EcUpQMseoUpIyztX/BLw7w9iPTwubp/X6Dbibjg0d1Q3TC2wRcktmBUQAAAB07X7mVvNokTM9H/oEPGLmpA0a+9alUABqUAAABhAZ6makf/AAAjrl42Y1SAEQ2Y7HByVqPlXDWUnMePLDqPPQZc8tdlTZWvd4jH9gT8tyZKcP5aYRdQl2HTMtqUYBtcQlPrRSWKkEBpzYdX+P8U0QTTU+ABdIRgFkmpcIDagQAAAMBBmqlJqEFsmUwUTCP//eEAAAQUvir+AARkaRZQ3FzgjJwD90GtaWI2R4cCGiVUZr6M2wmU3ss4rLmjDBDPufODkJX81OPyDdBPhdTBsftcUI96PzUXTIjMdwmw1OW9qV8hJPCBLxfs7I33wtWhUM7R6Q2nd/VmwWA5a57PQatEQ8BOlhLIqDYFHlGTT5fcZQACVmGkUykRJCueeajjbdG13c03q/B4onyA0h6oBOHuCvI7bcBe4SXTRmnoQCZAM+AAAABwAZ7Iakf/AAAjraGFUV1IAch9H4JfiN7adt1gbC7krSbY7r4Joc4TBazP0DjhYAt7oHQlxjxPAAA+khJqkOocb7cIoXA6IuNLyyjSoHk1f8STKv7EZ3lcL5vTaQNRq5WOzgfQxIAVAAA3oOYnExYCygAAAK5BmspJ4QpSZTAj//yEAAAP0/M4IQ9lmJP4SOhWAnsAEp3EJ0pSI1SAlA//SOSeFUhi1Wb4IoGuALs7Q0vjjaJoWK0T2tUJRYua3+5N0G7wXDQ2QjJMD4L5HsUU1E/7q6iIuY2xYMIQifxn4c3aprSfC+EZFf6UNKuEA/lWaGeKgNVm+KzlTKZgyJ5M9AHUPMMg4gnyNxZx0Gk4xGLliCtmwCzQNDEcbDfp2EFAm4EAAAOPbW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAAANwAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAArl0cmFrAAAAXHRraGQAAAADAAAAAAAAAAAAAAABAAAAAAAAANwAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAlgAAAGQAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAADcAAACAAABAAAAAAIxbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAyAAAACwBVxAAAAAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAB3G1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAAZxzdGJsAAAAmHN0c2QAAAAAAAAAAQAAAIhhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAlgBkABIAAAASAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAAMmF2Y0MBZAAf/+EAGWdkAB+s2UCYM+XhAAADAAEAAAMAZA8YMZYBAAZo6+PLIsAAAAAYc3R0cwAAAAAAAAABAAAACwAAAQAAAAAUc3RzcwAAAAAAAAABAAAAAQAAAGBjdHRzAAAAAAAAAAoAAAABAAACAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABAAAAAACAAABAAAAAAEAAAMAAAAAAQAAAQAAAAABAAACAAAAABxzdHNjAAAAAAAAAAEAAAABAAAACwAAAAEAAABAc3RzegAAAAAAAAAAAAAACwAABKUAAAC1AAAAeQAAAFYAAABfAAAA8QAAAH4AAABlAAAAxAAAAHQAAACyAAAAFHN0Y28AAAAAAAAAAQAAADAAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGlyYXBwbAAAAAAAAAAAAAAAAC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjU3LjgzLjEwMA==\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5uZR1TcRY8k",
        "colab_type": "text"
      },
      "source": [
        "# Coach\n",
        "\n",
        "Note: Once you have installed the below, you will need to restart the runtime. Coach needs a newer version of gym."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kiltiMTRdle",
        "colab_type": "code",
        "outputId": "8678e9ce-0181-4414-c3fe-9eae9305b5c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 326
        }
      },
      "source": [
        "!pip install rl_coach > /dev/null 2>&1\n",
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install pyvirtualdisplay > /dev/null 2>&1\n",
        "!pip install gym==0.12.5"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting gym==0.12.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0c/c4/307107c687f75267d645415d57db8c0a6e29e20ac30d8f4a10e8030b6737/gym-0.12.5.tar.gz (1.5MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5MB 2.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym==0.12.5) (1.3.0)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym==0.12.5) (1.16.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from gym==0.12.5) (1.12.0)\n",
            "Requirement already satisfied: pyglet>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym==0.12.5) (1.3.2)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym==0.12.5) (0.16.0)\n",
            "Building wheels for collected packages: gym\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /root/.cache/pip/wheels/cf/a5/c9/87967963aa32540d543e51bcf0d0fc19c5d68b8f49598d3b98\n",
            "Successfully built gym\n",
            "Installing collected packages: gym\n",
            "  Found existing installation: gym 0.10.11\n",
            "    Uninstalling gym-0.10.11:\n",
            "      Successfully uninstalled gym-0.10.11\n",
            "Successfully installed gym-0.12.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g8TeV1kcRazX",
        "colab_type": "code",
        "outputId": "0921a525-b6b3-468d-f1e3-f7fcbacb7923",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from pyvirtualdisplay import Display\n",
        "from gym.wrappers.monitor import Monitor\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()\n",
        "\n",
        "import tensorflow as tf\n",
        "tf.reset_default_graph()\n",
        "\n",
        "from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters\n",
        "from rl_coach.environments.gym_environment import GymVectorEnvironment\n",
        "from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager\n",
        "from rl_coach.graph_managers.graph_manager import ScheduleParameters\n",
        "from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps\n",
        "from rl_coach.base_parameters import VisualizationParameters\n",
        "global experiment_path; experiment_path = '.'\n",
        "\n",
        "\n",
        "# schedule\n",
        "schedule_params = ScheduleParameters()\n",
        "schedule_params.improve_steps = TrainingSteps(200)\n",
        "schedule_params.steps_between_evaluation_periods = EnvironmentSteps(200)\n",
        "schedule_params.evaluation_steps = EnvironmentEpisodes(200)\n",
        "schedule_params.heatup_steps = EnvironmentSteps(0)\n",
        "\n",
        "graph_manager = BasicRLGraphManager(\n",
        "    agent_params=ClippedPPOAgentParameters(),\n",
        "    env_params=GymVectorEnvironment(level='CartPole-v0'),\n",
        "    schedule_params=schedule_params,\n",
        "    vis_params=VisualizationParameters(dump_mp4=True)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0626 20:39:25.517042 140162082756480 abstractdisplay.py:151] xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0RpyoDfRv4Q",
        "colab_type": "code",
        "outputId": "488c3a65-c158-4406-c12e-21e59fd0d708",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "graph_manager.improve()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[30;46mCreating graph - name: BasicRLGraphManager\u001b[0m\n",
            "\u001b[30;46mCreating agent - name: agent\u001b[0m\n",
            "\u001b[30;46mStarting to improve simple_rl_graph task index 0\u001b[0m\n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m1 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m23 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m2 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m37 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m3 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m56 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m4 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m76 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m5 \u001b[94mTotal reward: \u001b[0m25.0 \u001b[94mSteps: \u001b[0m101 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m6 \u001b[94mTotal reward: \u001b[0m36.0 \u001b[94mSteps: \u001b[0m137 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m7 \u001b[94mTotal reward: \u001b[0m24.0 \u001b[94mSteps: \u001b[0m161 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTraining\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m175 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[30;46magent: Starting evaluation phase\u001b[0m\n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m28.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m37.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m35.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m22.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m41.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m10.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m25.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m31.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m31.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m28.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m10.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m27.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m28.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m36.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m35.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m22.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m32.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m26.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m36.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m30.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m24.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m30.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m34.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m36.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m31.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m10.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m29.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m34.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m25.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m31.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m35.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m22.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m26.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m42.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m27.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m29.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m33.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m10.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m12.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m35.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m10.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m24.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m31.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m21.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m20.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m13.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m18.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m15.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m19.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m33.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m23.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m16.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m17.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m11.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[95mTesting\u001b[0m - \u001b[94mName: \u001b[0mmain_level/agent \u001b[94mWorker: \u001b[0m0 \u001b[94mEpisode: \u001b[0m8 \u001b[94mTotal reward: \u001b[0m14.0 \u001b[94mSteps: \u001b[0m200 \u001b[94mTraining iteration: \u001b[0m0 \n",
            "\u001b[30;46magent: Finished evaluation phase. Success rate = 0.0, Avg Total Reward = 18.74\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5YULZSB0fLM7",
        "colab_type": "code",
        "outputId": "085639f0-52f6-48ac-9acd-0c36d53d44fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        "show_vid(next(glob.iglob(\"experiments/test/**/videos/*.mp4\")))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAACZ9tZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTEwIHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAACBGWIhAAQ//73gb8yy18iuslx+ed9LKzPPOQ8cl2JrrjQAAADAAADAAFHh7Xrjd2DW0nQAAAawAOkH8F5GDHQI+PkdC4lJ/+GrkgRs+h9RSwnlnfhp5r2J+z9kc9UO+n9ab01rY711un6qFSof4pfYQbky64S2c15whj/OvVrOPX8M0PhbcqFF0ZYU1Iev0KxHo179F+bjy1Lc0ukP/VmmZzdfNN+/qTGbs6ELZdJe2JvkRun2GPwT5slbsKFBuw2FNmCYmOoJ2RqsI/OR3qMoNhcCo/mKyxZIXnZTNS5hgwAzpL6cHUDK7AJxxdA2Cv0WlQ2VhzAXswiZSZf/BtYf+6T/OWqF0Q6SqOHhSiMJbhbTaWOaD/t7QZIwt/bwZrWn125RUeu0pWOtndmiuxYT4LfRKdM37j/YoiIUQX17WGrZrP++in4Jl3x6umw7k4xUeznNIn5hx9WROd6MZPqVcLAz2WzvOyshwER9nV/Nil0eYNu+KctbUBvqDmLrzuBLe0qqCYS2F2yVSP4qD7wf/CCbiDzd5gs5JF582mxkyQaI2lD7b+AlolY8IQNIK8iPDRAb5JcadJ3a7FqH0OTAEMZg9w/tzOguqzeuypXl8CVuNebx4VgCQFSDC2sijew+skw5Lk1AeRcRQVOSfX+PmKOwBnYitZfG2oeoAAAAwAAAwAHlQAAARtBmiRsQQ/+qlUAAAMCUZvTVtieAC6rz1FOzo+qKrzeVyVzhS+LMEA8Q61rvueOev7xW7dGbK3FcC2gtMKsjGET+10gaqrhijtR9tvy/zLu31O0aQpURw/Mq+xeZe+IfrwZ3ILGjR+JC/WD/6Llo6HAAAkP8fWSIJ7yPbKVhmMbdOXzAmVM5ClPwQg1pXJ0odDel7fQymb/b1eodPhBChnVAmpfnkv5A5iM7GEk2zTTpMaYkdcNAvpr29srilo2CbRXpYSt7jwSNOi0Vfwj/vkzzirQiNhF8kR5kAbpq96Jixne7/VVbg/5BrH11WdA+I9AAl9g01m0Dq3ry4JDXDT7+5I9vRa3uKgXZk6NOGC8DzZ2Wiy90gEw93hDAAAAOkGeQniHfwAABHQ0SlmvQAcUj60P/APgnq74RnBldN39ZnmXVONIjY2YN6CX0JLA0khlVlM25zyuFRkAAAA2AZ5hdEN/AAAGb6vg2Ryja3MsMb0yLhMACdveDHKR5uZwUvr7KFsp5e27d5uNXz2LDvwpQhOSAAAASwGeY2pDfwAABnABmdY50WwAf0Faode/eWPm+vFSW/0JwGYnMz1X8C7YRZltuR7MVrQEGvrncZYmiqsJLbiYvKYXHCrIxb9rf8qHwQAAARBBmmdJqEFomUwId//+qZYAAAkCgOiy1VqHoAOLSYtMN1mDfAAw8loAZ5DxHY+wMPpWtOybyJF9GKlC7bZUXmib8/5oDFybXWcG2fRbsCbY3vFGVPiyxbY7iOfF1WAX1ZrkZhex/p+we1hda16hUKoWuxB+3NLtlptbkmFQNxEM5JinB7oxQQrcXZjtXHYNFNtFOp5u3x9h9HHGd+koPgAXmouZ1CTUvEoyaB9/+F1Uy6sIYhHh25aryWZm8YoxhE5RMfxFP8KE9A7ifvPtY3rKBGvbY2whfEgtlK2kOLKCSX5lgStUmzOmpZd4+ZcCE7Oe9vXDRDhLP7EUAFFA+EEU5QBOE55okj15Kj4yY673wQAAAGFBnoVFESw3/wAABm5wQqUrnEvzRy9p7HmATm4AU9NGjzXW2mMGAdqgxv9WlY+w88z1UrHF5sP7pBQx13c9uQ74kM1fudgpr9Pojp9gf0QN7UBhhPZNE74HJIj3ctABB7GTAAAAZwGepmpDfwAABnCjtKQA3O3uAsI+g7FB2nwaoGWLuN0FN0R3AR4qmm3YxdgzO2O8ZYAeF1uoWwUg8xgmheNE4UP8E8a1dUJdsHycFj/x4wsDk9qhUYY3PRO0eVarC2o9LosgXLnFjs0AAAEPQZqoSahBbJlMCG///qeEAAAR75GkD88QIMgDooDOVw8snhl14+/vPHnkQQ1UPO3Mvs+1zCCSNZPt/p8faOYAMXdeN5BLMD7ZJVzV25jAQsQsmJQQwBxBQrxu/JnfkQ90346gXPVaRuPHYD71shbxQK9/ABijxJ8e2CQRkFCe7S1ARJSwEAUUtLYfMMoR+QAIe8tX/hAa6oB2FeZEn8wWK9cPekPWMHL1tRWUgYAq8/ObdbCDjACM24/3AxNQugdIBTuy75rkFBImeII5FnlN2uxhfSqd8qwlu0EAfAaJEHB/vqCX2Skw2Z+gwR54IlVQOLIk0I8Ckyatodvq6xAR2cOcJAhrsnwXwwCWbpjZ4AAAA3dtb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAADhAABAAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAACoXRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAADhAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAACWAAAAZAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAA4QAAAgAAAEAAAAAAhltZGlhAAAAIG1kaGQAAAAAAAAAAAAAAAAAACgAAAAkAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAAHEbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAABhHN0YmwAAACYc3RzZAAAAAAAAAABAAAAiGF2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAACWAGQAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY//8AAAAyYXZjQwFkABb/4QAZZ2QAFqzZQJgz5eEAAAMAAQAAAwAUDxYtlgEABmjr48siwAAAABhzdHRzAAAAAAAAAAEAAAAJAAAEAAAAABRzdHNzAAAAAAAAAAEAAAABAAAAUGN0dHMAAAAAAAAACAAAAAEAAAgAAAAAAQAAFAAAAAABAAAIAAAAAAEAAAAAAAAAAQAABAAAAAABAAAQAAAAAAIAAAQAAAAAAQAACAAAAAAcc3RzYwAAAAAAAAABAAAAAQAAAAkAAAABAAAAOHN0c3oAAAAAAAAAAAAAAAkAAAS6AAABHwAAAD4AAAA6AAAATwAAARQAAABlAAAAawAAARMAAAAUc3RjbwAAAAAAAAABAAAAMAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTcuODMuMTAw\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uO2ZwOs9gVGl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WwQjsKpzG14N",
        "colab_type": "text"
      },
      "source": [
        "# MAgent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kiv6ElECIqHU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install pyvirtualdisplay > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kArCZjNmG21T",
        "colab_type": "code",
        "outputId": "91f65762-b9c7-47e1-d4d8-c8ba99e222b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!git clone https://github.com/geek-ai/MAgent.git\n",
        "!sudo apt-get install cmake libboost-system-dev libjsoncpp-dev libwebsocketpp-dev\n",
        "%cd MAgent\n",
        "!bash build.sh"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'MAgent' already exists and is not an empty directory.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "cmake is already the newest version (3.10.2-1ubuntu2).\n",
            "libboost-system-dev is already the newest version (1.65.1.0ubuntu1).\n",
            "libjsoncpp-dev is already the newest version (1.7.4-3).\n",
            "libwebsocketpp-dev is already the newest version (0.7.0-11).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-410\n",
            "Use 'sudo apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 16 not upgraded.\n",
            "/content/MAgent\n",
            "-- The C compiler identification is GNU 7.4.0\n",
            "-- The CXX compiler identification is GNU 7.4.0\n",
            "-- Check for working C compiler: /usr/bin/cc\n",
            "-- Check for working C compiler: /usr/bin/cc -- works\n",
            "-- Detecting C compiler ABI info\n",
            "-- Detecting C compiler ABI info - done\n",
            "-- Detecting C compile features\n",
            "-- Detecting C compile features - done\n",
            "-- Check for working CXX compiler: /usr/bin/c++\n",
            "-- Check for working CXX compiler: /usr/bin/c++ -- works\n",
            "-- Detecting CXX compiler ABI info\n",
            "-- Detecting CXX compiler ABI info - done\n",
            "-- Detecting CXX compile features\n",
            "-- Detecting CXX compile features - done\n",
            "-- Configuring done\n",
            "-- Generating done\n",
            "-- Build files have been written to: /content/MAgent/build\n",
            "\u001b[35m\u001b[1mScanning dependencies of target render\u001b[0m\n",
            "\u001b[35m\u001b[1mScanning dependencies of target testlib\u001b[0m\n",
            "[  3%] \u001b[32mBuilding CXX object CMakeFiles/render.dir/src/render/backend/data.cc.o\u001b[0m\n",
            "[  6%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/discrete_snake/DiscreteSnake.cc.o\u001b[0m\n",
            "[  9%] \u001b[32mBuilding CXX object CMakeFiles/render.dir/src/render/backend/render.cc.o\u001b[0m\n",
            "[ 12%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/discrete_snake/Map.cc.o\u001b[0m\n",
            "[ 15%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/discrete_snake/RenderGenerator.cc.o\u001b[0m\n",
            "[ 18%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/gridworld/AgentType.cc.o\u001b[0m\n",
            "[ 21%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/gridworld/GridWorld.cc.o\u001b[0m\n",
            "[ 24%] \u001b[32mBuilding CXX object CMakeFiles/render.dir/src/render/backend/text.cc.o\u001b[0m\n",
            "[ 27%] \u001b[32mBuilding CXX object CMakeFiles/render.dir/src/render/backend/utility/config.cc.o\u001b[0m\n",
            "[ 30%] \u001b[32mBuilding CXX object CMakeFiles/render.dir/src/render/backend/utility/logger.cc.o\u001b[0m\n",
            "[ 33%] \u001b[32mBuilding CXX object CMakeFiles/render.dir/src/render/backend/websocket.cc.o\u001b[0m\n",
            "[ 36%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/gridworld/Map.cc.o\u001b[0m\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:\u001b[m\u001b[K In member function ‘\u001b[01m\u001b[Kvoid magent::gridworld::Map::render()\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:621:5:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n",
            "     \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K (int x = 0; x < w; x++)\n",
            "     \u001b[01;35m\u001b[K^~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:622:29:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’\n",
            "         printf(\"=\");        \u001b[01;36m\u001b[Kputs\u001b[m\u001b[K(\"\");\n",
            "                             \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:624:5:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n",
            "     \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K (int x = 0; x < w; x++)\n",
            "     \u001b[01;35m\u001b[K^~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:625:29:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’\n",
            "         printf(\"%2d \", x);  \u001b[01;36m\u001b[Kputs\u001b[m\u001b[K(\"\");\n",
            "                             \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:672:5:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n",
            "     \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K (int x = 0; x < w; x++)\n",
            "     \u001b[01;35m\u001b[K^~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:673:29:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’\n",
            "         printf(\"=\");        \u001b[01;36m\u001b[Kputs\u001b[m\u001b[K(\"\\n\");\n",
            "                             \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "[ 39%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/gridworld/RenderGenerator.cc.o\u001b[0m\n",
            "[ 42%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/gridworld/RewardEngine.cc.o\u001b[0m\n",
            "[ 45%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/gridworld/test.cc.o\u001b[0m\n",
            "[ 48%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/runtime_api.cc.o\u001b[0m\n",
            "[ 51%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/temp_c_booster.cc.o\u001b[0m\n",
            "[ 54%] \u001b[32mBuilding CXX object CMakeFiles/testlib.dir/src/utility/utility.cc.o\u001b[0m\n",
            "[ 57%] \u001b[32m\u001b[1mLinking CXX executable testlib\u001b[0m\n",
            "[ 57%] Built target testlib\n",
            "\u001b[35m\u001b[1mScanning dependencies of target magent\u001b[0m\n",
            "[ 60%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/discrete_snake/DiscreteSnake.cc.o\u001b[0m\n",
            "[ 63%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/discrete_snake/Map.cc.o\u001b[0m\n",
            "[ 66%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/discrete_snake/RenderGenerator.cc.o\u001b[0m\n",
            "[ 69%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/gridworld/AgentType.cc.o\u001b[0m\n",
            "[ 72%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/gridworld/GridWorld.cc.o\u001b[0m\n",
            "[ 75%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/gridworld/Map.cc.o\u001b[0m\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:\u001b[m\u001b[K In member function ‘\u001b[01m\u001b[Kvoid magent::gridworld::Map::render()\u001b[m\u001b[K’:\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:621:5:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n",
            "     \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K (int x = 0; x < w; x++)\n",
            "     \u001b[01;35m\u001b[K^~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:622:29:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’\n",
            "         printf(\"=\");        \u001b[01;36m\u001b[Kputs\u001b[m\u001b[K(\"\");\n",
            "                             \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:624:5:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n",
            "     \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K (int x = 0; x < w; x++)\n",
            "     \u001b[01;35m\u001b[K^~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:625:29:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’\n",
            "         printf(\"%2d \", x);  \u001b[01;36m\u001b[Kputs\u001b[m\u001b[K(\"\");\n",
            "                             \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:672:5:\u001b[m\u001b[K \u001b[01;35m\u001b[Kwarning: \u001b[m\u001b[Kthis ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’ clause does not guard... [\u001b[01;35m\u001b[K-Wmisleading-indentation\u001b[m\u001b[K]\n",
            "     \u001b[01;35m\u001b[Kfor\u001b[m\u001b[K (int x = 0; x < w; x++)\n",
            "     \u001b[01;35m\u001b[K^~~\u001b[m\u001b[K\n",
            "\u001b[01m\u001b[K/content/MAgent/src/gridworld/Map.cc:673:29:\u001b[m\u001b[K \u001b[01;36m\u001b[Knote: \u001b[m\u001b[K...this statement, but the latter is misleadingly indented as if it were guarded by the ‘\u001b[01m\u001b[Kfor\u001b[m\u001b[K’\n",
            "         printf(\"=\");        \u001b[01;36m\u001b[Kputs\u001b[m\u001b[K(\"\\n\");\n",
            "                             \u001b[01;36m\u001b[K^~~~\u001b[m\u001b[K\n",
            "[ 78%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/gridworld/RenderGenerator.cc.o\u001b[0m\n",
            "[ 81%] \u001b[32m\u001b[1mLinking CXX executable render/render\u001b[0m\n",
            "[ 81%] Built target render\n",
            "[ 84%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/gridworld/RewardEngine.cc.o\u001b[0m\n",
            "[ 87%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/gridworld/test.cc.o\u001b[0m\n",
            "[ 90%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/runtime_api.cc.o\u001b[0m\n",
            "[ 93%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/temp_c_booster.cc.o\u001b[0m\n",
            "[ 96%] \u001b[32mBuilding CXX object CMakeFiles/magent.dir/src/utility/utility.cc.o\u001b[0m\n",
            "[100%] \u001b[32m\u001b[1mLinking CXX shared library libmagent.so\u001b[0m\n",
            "[100%] Built target magent\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tD4HIEPyHHBk",
        "colab_type": "code",
        "outputId": "92c78e20-68e5-4644-f37c-6c634d9e0a0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!PYTHONPATH=$(pwd)/python:$PYTHONPATH python examples/api_demo.py"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0627 09:24:39.824534 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:81: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "W0627 09:24:39.843531 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:91: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
            "\n",
            "W0627 09:24:39.844067 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:93: The name tf.get_variable_scope is deprecated. Please use tf.compat.v1.get_variable_scope instead.\n",
            "\n",
            "W0627 09:24:39.844260 139869075769216 deprecation.py:323] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:165: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.keras.layers.Conv2D` instead.\n",
            "W0627 09:24:39.848146 139869075769216 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "W0627 09:24:40.109013 139869075769216 deprecation.py:323] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:170: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.dense instead.\n",
            "W0627 09:24:40.565698 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:110: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
            "\n",
            "W0627 09:24:40.723029 139869075769216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/clip_ops.py:286: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "W0627 09:24:40.868260 139869075769216 deprecation.py:323] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:118: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "W0627 09:24:40.869464 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:119: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "W0627 09:24:40.880762 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/dqn.py:129: The name tf.get_collection is deprecated. Please use tf.compat.v1.get_collection instead.\n",
            "\n",
            "2019-06-27 09:24:40.894404: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz\n",
            "2019-06-27 09:24:40.894620: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x17292c0 executing computations on platform Host. Devices:\n",
            "2019-06-27 09:24:40.894652: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "2019-06-27 09:24:40.965628: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "tcmalloc: large alloc 2097152000 bytes == 0x5622000 @  0x7f35ce5031e7 0x7f35cb09aca1 0x7f35cb0ff778 0x7f35cb0ffd47 0x7f35cb19a038 0x4f8925 0x4fa6c0 0x4f6128 0x4f426e 0x5a1481 0x512a60 0x53ee21 0x57ec0c 0x4f88ba 0x4fa6c0 0x4f6128 0x4f42e7 0x5a1481 0x512a60 0x53ee21 0x57ec0c 0x4f88ba 0x4f98c7 0x4f6128 0x4f9023 0x6415b2 0x64166a 0x643730 0x62b26e 0x4b4cb0 0x7f35ce100b97\n",
            "tcmalloc: large alloc 1698693120 bytes == 0x87d90000 @  0x7f35ce5031e7 0x7f35cb09aca1 0x7f35cb0ff778 0x7f35cb0ffd47 0x7f35cb19a038 0x4f8925 0x4fa6c0 0x4f6128 0x4f426e 0x5a1481 0x512a60 0x53ee21 0x57ec0c 0x4f88ba 0x4fa6c0 0x4f6128 0x4f42e7 0x5a1481 0x512a60 0x53ee21 0x57ec0c 0x4f88ba 0x4f98c7 0x4f6128 0x4f9023 0x6415b2 0x64166a 0x643730 0x62b26e 0x4b4cb0 0x7f35ce100b97\n",
            "W0627 09:24:41.767916 139869075769216 deprecation_wrapper.py:119] From /content/MAgent/python/magent/builtin/tf_model/base.py:54: The name tf.train.Saver is deprecated. Please use tf.compat.v1.train.Saver instead.\n",
            "\n",
            "W0627 09:24:41.812864 139869075769216 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use standard file APIs to check for files with this prefix.\n",
            "nums: 200 vs 200\n",
            "step: 0\t predators' reward: 19\t preys' reward: -25\n",
            "step: 10\t predators' reward: 8\t preys' reward: -11\n",
            "step: 20\t predators' reward: 13\t preys' reward: -18\n",
            "step: 30\t predators' reward: 33\t preys' reward: -44\n",
            "step: 40\t predators' reward: 32\t preys' reward: -42\n",
            "step: 50\t predators' reward: 34\t preys' reward: -45\n",
            "step: 60\t predators' reward: 46\t preys' reward: -60\n",
            "step: 70\t predators' reward: 41\t preys' reward: -55\n",
            "step: 80\t predators' reward: 48\t preys' reward: -63\n",
            "step: 90\t predators' reward: 49\t preys' reward: -65\n",
            "step: 100\t predators' reward: 52\t preys' reward: -69\n",
            "step: 110\t predators' reward: 55\t preys' reward: -71\n",
            "step: 120\t predators' reward: 58\t preys' reward: -76\n",
            "step: 130\t predators' reward: 59\t preys' reward: -78\n",
            "step: 140\t predators' reward: 54\t preys' reward: -71\n",
            "step: 150\t predators' reward: 57\t preys' reward: -76\n",
            "step: 160\t predators' reward: 53\t preys' reward: -70\n",
            "step: 170\t predators' reward: 56\t preys' reward: -74\n",
            "step: 180\t predators' reward: 57\t preys' reward: -76\n",
            "step: 190\t predators' reward: 58\t preys' reward: -77\n",
            "step: 200\t predators' reward: 54\t preys' reward: -71\n",
            "step: 210\t predators' reward: 55\t preys' reward: -74\n",
            "step: 220\t predators' reward: 56\t preys' reward: -75\n",
            "step: 230\t predators' reward: 59\t preys' reward: -78\n",
            "step: 240\t predators' reward: 56\t preys' reward: -74\n",
            "step: 250\t predators' reward: 57\t preys' reward: -76\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "er-LL4JPSyNn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fwFE3DHvZkGp",
        "colab_type": "text"
      },
      "source": [
        "# SLM-Lab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRZ7xb2UZujC",
        "colab_type": "code",
        "outputId": "6a3bb544-fa61-4c79-add4-b5d549e547ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "%cd /content\n",
        "!ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "build  MAgent  sample_data  SLM-Lab\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MpwnK_chZrOR",
        "colab_type": "code",
        "outputId": "952c6fc2-fe31-4e76-a697-ba0811782b3b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!git clone https://github.com/kengz/SLM-Lab.git\n",
        "%cd /content/SLM-Lab"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'SLM-Lab' already exists and is not an empty directory.\n",
            "/content/SLM-Lab\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q7gUiqjUaA8B",
        "colab_type": "code",
        "outputId": "6ea0fc1c-e312-4ce3-ae28-7269fd210582",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        }
      },
      "source": [
        "!pip install pydash==4.2.1 regex==2019.05.25 ujson==1.35 colorlog==4.0.2 ray==0.7.0 torch==1.1.0"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pydash==4.2.1 in /usr/local/lib/python3.6/dist-packages (4.2.1)\n",
            "Requirement already satisfied: regex==2019.05.25 in /usr/local/lib/python3.6/dist-packages (2019.5.25)\n",
            "Requirement already satisfied: ujson==1.35 in /usr/local/lib/python3.6/dist-packages (1.35)\n",
            "Requirement already satisfied: colorlog==4.0.2 in /usr/local/lib/python3.6/dist-packages (4.0.2)\n",
            "Requirement already satisfied: ray==0.7.0 in /usr/local/lib/python3.6/dist-packages (0.7.0)\n",
            "Requirement already satisfied: torch==1.1.0 in /usr/local/lib/python3.6/dist-packages (1.1.0)\n",
            "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (3.7.4)\n",
            "Requirement already satisfied: numpy>=1.14 in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (1.16.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (3.0.12)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (0.4.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (3.13)\n",
            "Requirement already satisfied: redis in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (3.2.1)\n",
            "Requirement already satisfied: funcsigs in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (1.0.2)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (1.11)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (7.0)\n",
            "Requirement already satisfied: six>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (1.12.0)\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.6/dist-packages (from ray==0.7.0) (3.6.4)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest->ray==0.7.0) (1.3.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest->ray==0.7.0) (1.8.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest->ray==0.7.0) (0.7.1)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest->ray==0.7.0) (7.0.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest->ray==0.7.0) (41.0.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest->ray==0.7.0) (19.1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6V9xyvSCZlaf",
        "colab_type": "code",
        "outputId": "7acf2fbf-114a-4cbb-bea3-a5ff19cafd09",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "!python run_lab.py slm_lab/spec/demo.json dqn_cartpole dev"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"run_lab.py\", line 77, in <module>\n",
            "    mp.set_start_method('spawn')  # for distributed pytorch to work\n",
            "  File \"/usr/lib/python3.6/multiprocessing/context.py\", line 242, in set_start_method\n",
            "    raise RuntimeError('context has already been set')\n",
            "RuntimeError: context has already been set\n",
            "\u001b[0m"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lvXXF5REZ-TJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cF-iG1xoK-Qf",
        "colab_type": "text"
      },
      "source": [
        "# DeeR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6F5KwN1WK_NR",
        "colab_type": "code",
        "outputId": "79c6a956-b118-469c-c635-0907acba05fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "!pip install git+git://github.com/VINF/deer.git@master\n",
        "!git clone https://github.com/VinF/deer.git"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+git://github.com/VINF/deer.git@master\n",
            "  Cloning git://github.com/VINF/deer.git (to revision master) to /tmp/pip-req-build-q7rqqsoq\n",
            "  Running command git clone -q git://github.com/VINF/deer.git /tmp/pip-req-build-q7rqqsoq\n",
            "Requirement already satisfied (use --upgrade to upgrade): deer==0.4 from git+git://github.com/VINF/deer.git@master in /usr/local/lib/python3.6/dist-packages\n",
            "Building wheels for collected packages: deer\n",
            "  Building wheel for deer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-ed1ixq74/wheels/22/ce/b3/62316fcd701b75f1c255c84026db5efc518cd9595488dc7e2a\n",
            "Successfully built deer\n",
            "Cloning into 'deer'...\n",
            "remote: Enumerating objects: 1, done.\u001b[K\n",
            "remote: Counting objects: 100% (1/1), done.\u001b[K\n",
            "remote: Total 2977 (delta 0), reused 0 (delta 0), pack-reused 2976\u001b[K\n",
            "Receiving objects: 100% (2977/2977), 12.83 MiB | 18.82 MiB/s, done.\n",
            "Resolving deltas: 100% (2043/2043), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j0h1JZyQLARC",
        "colab_type": "code",
        "outputId": "89a543ec-55b2-486e-c7de-b3f179d02c45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "%cd /content/deer/examples/toy_env\n",
        "import numpy as np\n",
        "from deer.agent import NeuralAgent\n",
        "from deer.learning_algos.q_net_keras import MyQNetwork\n",
        "from Toy_env import MyEnv as Toy_env\n",
        "import deer.experiment.base_controllers as bc\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/deer/examples/toy_env\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q17quzgsLNOE",
        "colab_type": "code",
        "outputId": "1bba3025-3796-47e8-ec35-5a212af7bcc4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "rng = np.random.RandomState(123456)\n",
        "\n",
        "# --- Instantiate environment ---\n",
        "env = Toy_env(rng)\n",
        "\n",
        "# --- Instantiate qnetwork ---\n",
        "qnetwork = MyQNetwork(\n",
        "    environment=env,\n",
        "    random_state=rng)\n",
        "\n",
        "# --- Instantiate agent ---\n",
        "agent = NeuralAgent(\n",
        "    env,\n",
        "    qnetwork,\n",
        "    random_state=rng)\n",
        "\n",
        "# --- Bind controllers to the agent ---\n",
        "# Before every training epoch, we want to print a summary of the agent's epsilon, discount and \n",
        "# learning rate as well as the training epoch number.\n",
        "agent.attach(bc.VerboseController())\n",
        "\n",
        "# During training epochs, we want to train the agent after every action it takes.\n",
        "# Plus, we also want to display after each training episode (!= than after every training) the average bellman\n",
        "# residual and the average of the V values obtained during the last episode.\n",
        "agent.attach(bc.TrainerController())\n",
        "\n",
        "# All previous controllers control the agent during the epochs it goes through. However, we want to interleave a \n",
        "# \"test epoch\" between each training epoch. We do not want these test epoch to interfere with the training of the \n",
        "# agent. Therefore, we will disable these controllers for the whole duration of the test epochs interleaved this \n",
        "# way, using the controllersToDisable argument of the InterleavedTestEpochController. The value of this argument \n",
        "# is a list of the indexes of all controllers to disable, their index reflecting in which order they were added.\n",
        "agent.attach(bc.InterleavedTestEpochController(\n",
        "    epoch_length=500, \n",
        "    controllers_to_disable=[0, 1]))\n",
        "    \n",
        "# --- Run the experiment ---\n",
        "agent.run(n_epochs=100, epoch_length=1000)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0628 08:24:58.046045 140240255469440 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "W0628 08:24:58.102065 140240255469440 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "W0628 08:24:58.130126 140240255469440 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "W0628 08:24:58.245688 140240255469440 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "W0628 08:24:58.367403 140240255469440 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "W0628 08:24:58.369060 140240255469440 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Average (on the epoch) training loss: 0.14866669476032257\n",
            "Episode average V value: 0.06669619510631186\n",
            "epoch 1:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.1299528181552887\n",
            "Episode average V value: 0.17194478684011846\n",
            "epoch 2:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.1677785962820053\n",
            "Episode average V value: 0.17891222828626632\n",
            "epoch 3:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24310679733753204\n",
            "Episode average V value: 0.16639169204235077\n",
            "epoch 4:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2669770419597626\n",
            "Episode average V value: 0.19760323057323695\n",
            "epoch 5:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.25460633635520935\n",
            "Episode average V value: 0.19152317226708712\n",
            "epoch 6:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24626144766807556\n",
            "Episode average V value: 0.20997467203438283\n",
            "epoch 7:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2426571100950241\n",
            "Episode average V value: 0.22858887243270873\n",
            "epoch 8:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2492290884256363\n",
            "Episode average V value: 0.2499886567145586\n",
            "epoch 9:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24580413103103638\n",
            "Episode average V value: 0.24533822299540042\n",
            "epoch 10:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.2475874423980713\n",
            "Episode average V value: 0.25221630134382045\n",
            "epoch 11:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24534495174884796\n",
            "Episode average V value: 0.26229080337286\n",
            "epoch 12:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24298596382141113\n",
            "Episode average V value: 0.2742143989623607\n",
            "epoch 13:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2448553740978241\n",
            "Episode average V value: 0.28240874762047746\n",
            "epoch 14:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24037162959575653\n",
            "Episode average V value: 0.2842316195070744\n",
            "epoch 15:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23441393673419952\n",
            "Episode average V value: 0.2967251349389553\n",
            "epoch 16:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23110082745552063\n",
            "Episode average V value: 0.2956966770887375\n",
            "epoch 17:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23833075165748596\n",
            "Episode average V value: 0.2895745251178741\n",
            "epoch 18:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23833151161670685\n",
            "Episode average V value: 0.32200525000691416\n",
            "epoch 19:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24186325073242188\n",
            "Episode average V value: 0.3323148148059845\n",
            "epoch 20:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.23957884311676025\n",
            "Episode average V value: 0.32721942535042764\n",
            "epoch 21:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23604904115200043\n",
            "Episode average V value: 0.3382917032241821\n",
            "epoch 22:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23805876076221466\n",
            "Episode average V value: 0.3396150012910366\n",
            "epoch 23:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2404842972755432\n",
            "Episode average V value: 0.3428701400756836\n",
            "epoch 24:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24001102149486542\n",
            "Episode average V value: 0.34811299562454223\n",
            "epoch 25:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2445892095565796\n",
            "Episode average V value: 0.35235521736741066\n",
            "epoch 26:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2390148639678955\n",
            "Episode average V value: 0.3461126935184002\n",
            "epoch 27:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24403664469718933\n",
            "Episode average V value: 0.326702396184206\n",
            "epoch 28:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23458734154701233\n",
            "Episode average V value: 0.32797212672233583\n",
            "epoch 29:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23550090193748474\n",
            "Episode average V value: 0.34903469508886337\n",
            "epoch 30:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.24813178181648254\n",
            "Episode average V value: 0.3412800631924478\n",
            "epoch 31:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23992019891738892\n",
            "Episode average V value: 0.3611905106306076\n",
            "epoch 32:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23419632017612457\n",
            "Episode average V value: 0.35170292419195176\n",
            "epoch 33:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23996758460998535\n",
            "Episode average V value: 0.34424702629446985\n",
            "epoch 34:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24222393333911896\n",
            "Episode average V value: 0.3342800041735172\n",
            "epoch 35:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24448421597480774\n",
            "Episode average V value: 0.3492643491923809\n",
            "epoch 36:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.246259406208992\n",
            "Episode average V value: 0.361055488973856\n",
            "epoch 37:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24161389470100403\n",
            "Episode average V value: 0.3519581006169319\n",
            "epoch 38:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24014119803905487\n",
            "Episode average V value: 0.3629315204024315\n",
            "epoch 39:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2478182464838028\n",
            "Episode average V value: 0.3543600558577654\n",
            "epoch 40:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.24092596769332886\n",
            "Episode average V value: 0.3477752946317196\n",
            "epoch 41:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23740343749523163\n",
            "Episode average V value: 0.3545894278585911\n",
            "epoch 42:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23870299756526947\n",
            "Episode average V value: 0.36899810734391214\n",
            "epoch 43:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24006721377372742\n",
            "Episode average V value: 0.3578974869847298\n",
            "epoch 44:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23428574204444885\n",
            "Episode average V value: 0.3425828560590744\n",
            "epoch 45:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24165146052837372\n",
            "Episode average V value: 0.3577576987147331\n",
            "epoch 46:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24048280715942383\n",
            "Episode average V value: 0.3626994613111019\n",
            "epoch 47:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24021755158901215\n",
            "Episode average V value: 0.36791615417599677\n",
            "epoch 48:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23480340838432312\n",
            "Episode average V value: 0.3694310404062271\n",
            "epoch 49:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23788335919380188\n",
            "Episode average V value: 0.3876477759480476\n",
            "epoch 50:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.24242755770683289\n",
            "Episode average V value: 0.3774426821470261\n",
            "epoch 51:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23547230660915375\n",
            "Episode average V value: 0.39513669803738594\n",
            "epoch 52:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24521930515766144\n",
            "Episode average V value: 0.39393924778699874\n",
            "epoch 53:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23927246034145355\n",
            "Episode average V value: 0.37250123772025107\n",
            "epoch 54:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2386237233877182\n",
            "Episode average V value: 0.3632540809320616\n",
            "epoch 55:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23191416263580322\n",
            "Episode average V value: 0.36676007267832755\n",
            "epoch 56:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23549246788024902\n",
            "Episode average V value: 0.36764910486340524\n",
            "epoch 57:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24701936542987823\n",
            "Episode average V value: 0.3534671650826931\n",
            "epoch 58:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24182376265525818\n",
            "Episode average V value: 0.36596403670310973\n",
            "epoch 59:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24058209359645844\n",
            "Episode average V value: 0.36497137424824116\n",
            "epoch 60:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.23614482581615448\n",
            "Episode average V value: 0.36535736778378486\n",
            "epoch 61:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2349364310503006\n",
            "Episode average V value: 0.36054942283034325\n",
            "epoch 62:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24697758257389069\n",
            "Episode average V value: 0.36750818526744844\n",
            "epoch 63:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24067190289497375\n",
            "Episode average V value: 0.3660389856696129\n",
            "epoch 64:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24071337282657623\n",
            "Episode average V value: 0.3670588257610798\n",
            "epoch 65:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23453699052333832\n",
            "Episode average V value: 0.35906021493673324\n",
            "epoch 66:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24032771587371826\n",
            "Episode average V value: 0.38413021314144136\n",
            "epoch 67:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23888398706912994\n",
            "Episode average V value: 0.37869823477170367\n",
            "epoch 68:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23857510089874268\n",
            "Episode average V value: 0.37726695850491526\n",
            "epoch 69:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24073335528373718\n",
            "Episode average V value: 0.3803598977923393\n",
            "epoch 70:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.2388656884431839\n",
            "Episode average V value: 0.36965302614359047\n",
            "epoch 71:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24649550020694733\n",
            "Episode average V value: 0.3710278814435005\n",
            "epoch 72:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23795965313911438\n",
            "Episode average V value: 0.35902621963620185\n",
            "epoch 73:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23648931086063385\n",
            "Episode average V value: 0.3528077877163887\n",
            "epoch 74:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2418311983346939\n",
            "Episode average V value: 0.3630599780082703\n",
            "epoch 75:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2365661859512329\n",
            "Episode average V value: 0.34754120716452597\n",
            "epoch 76:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2365422397851944\n",
            "Episode average V value: 0.32974090617895124\n",
            "epoch 77:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2352670580148697\n",
            "Episode average V value: 0.347865422219038\n",
            "epoch 78:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23633356392383575\n",
            "Episode average V value: 0.3736473198707875\n",
            "epoch 79:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2397204488515854\n",
            "Episode average V value: 0.3722566829919815\n",
            "epoch 80:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.23461899161338806\n",
            "Episode average V value: 0.37629220718145373\n",
            "epoch 81:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24454990029335022\n",
            "Episode average V value: 0.3685927130281925\n",
            "epoch 82:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23686754703521729\n",
            "Episode average V value: 0.3605931355655193\n",
            "epoch 83:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.22746500372886658\n",
            "Episode average V value: 0.3612159802913666\n",
            "epoch 84:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23943407833576202\n",
            "Episode average V value: 0.37843764750091163\n",
            "epoch 85:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2371247261762619\n",
            "Episode average V value: 0.3776245658099651\n",
            "epoch 86:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23652857542037964\n",
            "Episode average V value: 0.36898776891827584\n",
            "epoch 87:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23806288838386536\n",
            "Episode average V value: 0.3647692542374134\n",
            "epoch 88:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23927144706249237\n",
            "Episode average V value: 0.368241210848093\n",
            "epoch 89:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2376956194639206\n",
            "Episode average V value: 0.372055983543396\n",
            "epoch 90:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n",
            "Average (on the epoch) training loss: 0.23382866382598877\n",
            "Episode average V value: 0.3658715925954126\n",
            "epoch 91:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23265646398067474\n",
            "Episode average V value: 0.36536942955851553\n",
            "epoch 92:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23395469784736633\n",
            "Episode average V value: 0.35864550712704657\n",
            "epoch 93:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23540213704109192\n",
            "Episode average V value: 0.36361665150523187\n",
            "epoch 94:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23510639369487762\n",
            "Episode average V value: 0.3606573850810528\n",
            "epoch 95:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23598256707191467\n",
            "Episode average V value: 0.3659165444970131\n",
            "epoch 96:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.2426307499408722\n",
            "Episode average V value: 0.36468412360548974\n",
            "epoch 97:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24167782068252563\n",
            "Episode average V value: 0.3571422270536423\n",
            "epoch 98:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.24225278198719025\n",
            "Episode average V value: 0.36729977947473524\n",
            "epoch 99:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Average (on the epoch) training loss: 0.23632098734378815\n",
            "Episode average V value: 0.3567864058969973\n",
            "epoch 100:\n",
            "Learning rate: 0.005\n",
            "Discount factor: 0.9\n",
            "Epsilon: 0.1\n",
            "Testing score per episode (id: 0) is 0.0 (average over 1 episode(s))\n",
            "Summary Perf\n",
            "A plot of the policy obtained has been saved under the name plot.png\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEECAYAAADd88i7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmUHcV97791ZzSLJDRC1kWIRRpW\nsYjF9iWxMBYYHZI8ICzm4ENyDMmxw5iE53jBKxA/YewANgEnNsk51weHgwnkYRbDcfwI2IIIEggZ\nZBYBMhZoAWGhQctomxlpNPX+qHt1+/a93dNrVXX393NOn+mu21NVvVT/6vf71a9KSClBCCGEtKNk\nugKEEELshUKCEEKIJxQShBBCPKGQIIQQ4gmFBCGEEE8oJAghhHhihZAQQpSFEFIIwfG4hBBiEcKG\nOAkhxAQAAQBSSmG4OoQQQmp0mq6AEOIBKAGxHcAM9++zZ8+W/f39kfMfGhpCuVyO/P9ZhtfOay8a\nvPZo1/7CCy94dtCNCwkAlwB4G20EBAD09/djcHAwcuaVSiXW/2cZXjuvvWjw2qNduxDeBhyjPgkh\nxAgASCnnmawHIYSQ9pjWJHoAwOmwFkJIp9ozNDSESqWy/x8GBgYwMDCgtZKEEFJUjAoJpzAQQmwD\n0Oe2i5XL5VjqY5EFCq+9mPDai0la127F6CbAW0hUKhVZVBsjIYToQAhhteMaACClnGm6DoQQQpqx\nIpiOEEKInVijSRA9/O53wH/8B7B3rzo+7DDgYx8DOvkmkALz7LPAffcBIyPqePZs4MorgSOPNFsv\nG+CnoUDs2QP82Z8BW7Y0p3/+88Dll5upEyGmmZgAli4FNm9uTh8dBW6/3UiVrILmpgLx7rutAgIA\nXnpJf10IsYXx8VYBAQAbN+qvi41QSBSIiYn26ZYMcCPECF7vv1d7KRoUEgXCqzFQSJAiw86TPxQS\nBcKrMbDHRIoMNQl/KCQKBBsDIa2w8+QPhUSBoFpNSCsUBv5wCGyBYI+JkODkvV1Uq1VUq9VJz6OQ\nKBB0XBPSSlE7T84Zta1dT4LoheYmQlphu/CHQqJA0HFNSCtsF/5QSBQImpsIaYWahD8UEgWiqLZX\nQvxwCgOnaZ7tQkEhUSCcjaHkePJsDKTION//jo726UWGQqJAOF9659TgVKtJkXG2ixK/iC3wlhQI\n9pgIacXZSWK7aIVCokB4NQZqEqTIsPPkD4VEgfBqDBQSpMiw8+QPhUSBoFpNSCtsF/5QSBQILwcd\ne0ykyHBAhz8UEgWCtldCWqEm4Q+FREFhYyBEwc6TPxQSBYJqNSGtME7CH96SAsEeEyGt0NzkD4VE\ngeBQP0JaYefJHwqJAsHGQEgr7Dz5QyFRINgYCGnFL8iUbYPLlxYKE3ESmzYBDz0EbNmijqdNAy66\nCJg/P70yCQmDe3ZkIRppUjZPH15EKCQKhAlz0/e+Bzz5ZHPac88B992XXpmEhGEyIVF0aG4qECZG\ncaxe3Zr25pvplUdIWJzvvxBca8UNNYkCYSJOol0jY8OLx6pVwL33Ajt3quMZM4BPfQo4+miz9coq\nbiHhNC9Rk6CQKBQmNAm/dbWLbuuNyo03Ar/5TXPa228Dd95ppj5Zp525qd1vRcWouUkIcbYQQjo3\nk/XJOyZGN/kJCRKNd94JlkaCQXOTP6Y1iScBXCGl/IkQ4ngArwkh3pNSzjFcr1xiwnHtlTeFRHTa\n3bu83c/hYeDXv25c14IFwCGHpFOWW5PgDMnNGBUSUkoJ4Ce1wwW1v5TdKWGTJjEx0VwHEpy8+3mG\nh4HzzwdGRhppQihz2sknJ1+en08iT/c1KsZHNwkhyjUz08MAIKWca7hKucVEnAQ1ieTJu5B44YVm\nAQGo9+XZZ9Mpz90uaG5qxriQkFIOSSkFgK8DgBDidefvQ0NDqFQq+7dqtWqknnnAJnMTG1908i5g\n9+0Llx4XOq79Me2T2I+U8hYhxE1omJ0AAOVyGYODg4ZqlS9sGt1EIRGddvc0T/dT92AHZ750XLdi\nenTT94UQX6vt/y8AAsCYyTrlGVviJNIuM+/k3dykW/v0MzfxPTWvSZwH4GghxM21Yyml7DVZoTxj\nkybBxhedvI9u0t2xcGsSOsrMEqZHNx1jsvyi4TfbpY4yg6QTf7xmJs3T/dRtomSchD+mNQmiEdOO\n61KpccweWjSKoJlF1STWrwc2bpw8/xNPVLMRj48DL78M/Pa3jd/c5qZf/xqYObN9Ph0dwEknAV1d\nk5eZZSgkMsavfgXcdhuwbZv/eR0dwDnnANddB1x/PfD008CePc2/1xkfBz76Ue+8urqAyy4DPvvZ\neHXv7GzUgT20aBRhIEAUQfjTnwK33BIs/6lTgQcfBK6+Gnjrrebf3HES117rn9ecOcDDD+dbUFBI\nZIwf/Qh4771g5z7yiIpUffzx1t+mT1cvdv2jPeYzXGBsTAUy/cmfqMnkwmDCxJVn3KaR+n0sgpDw\nu8Zly4Lnv3s3cP/9rQICAObOBbZuDd7G3nsPeO014NRTg5efNSgkMsaOHeHO37ChNW3+fODcc9VH\n+847mzUMLyYm1KyjSQmJtD5qe/YAP/sZ8MYb6rhUAs44A1i8OJ3ydOMe059W7IBJovixxscb+wsW\ntH9P33oL2LxZ7Ts7RT09ymx0+OHA5Zerc+66y19bf/31xiy8WX0G1Wo1UNwZhUSGefBBpe66+dzn\nlC0VaH6BzzgDuPlmoLtb9UI/8xngiiv8X/JLLwV+9zu1H+XDrntE1WOPAd/9bnPaww8rwXHooemU\nqRO30K0/uyJoEn7ap/P6v/KV9j37G29U2jXQLFQOPxz4p39qHM+eDdx0k38dBwaAFSsmr5fNDAwM\nYGBgAAAgfKZkppDIGM4XsqdHbW6cMRBOATBlSuv5U6aozYu4H3bd5qZ2CxpJCaxZkz8hUXJFOeVl\n+vUomoRbw2qHM93ZLvJwz9LE+LQcJDpeL7czPW5jiDsc0Nl4dQTwFcGxWyevgV9R5hhzvudeQsKr\nXXid70eRpu6gkMgYQV5I50vvVKujNIYkNQkd48/zKAycFGHGUq+ORZKahJ9GFgQKCWItftGhdbzU\n6iiNwSuvILgbj45eb941iSLMWBrFROkWnu1I0txUJBMVhUQOSdLcFEeTMDFxWt6FhPue5rFH66UV\nBBUSQdYpoZAIDoVExsiSJuE33UFaH7S8TyhYZE3C7/riaBJR2oWTvLxbXlBIZIwgQsIWx7XfPP26\nfRJ5+YAWQZOIKyR0j27Ky333gkIiw+iwvSZlbtLV68373EZ+juu8XKOTKD6JIKOb4g7oyPt9d8I4\niYwR5IX06rGbNjfpaFgmNIlVq4Dlyxv356ijgLPPbh6ZkxQ0N03+Pzo0iSL5JCgkckhajuuwH3YT\nHzTdjuvhYeDTn26d2uS664CLL06+PD/tLC892iijm+KYYemT8IfmpowR1nGdpFodZwisaU0iLd58\ns/3cV6+/3pqWBIyTaI/z3fQa3ZSk47pImgSFRMbQPbopjk+iCOYmk+sxF83cFFeTSMvcRE2CWIXJ\n0U1xNAkT5iaTEd46ri+vjuu4cRI6HNdFgrcnh6QVJxFXk9AdcW1yrigdmlLRNIkkHddB4iqCkhfh\n7AWFRIbRoUkkZW4yESehY2pyk5pE0RzXcYWE1/tHc5M/FBIZI84Ef0UzN+mYmly3kAg6yV2WifIM\ng0zwxziJaFBIZIwsOa5Nj27SoUnQ3JQ8UZ5hkPeccRLRoJDIGGEd13HV6qR8EqY1CZqbsoMOTYJD\nYINDIZExwkZcm3Rcm9Yk8u64ZpxE+9/CahIMpvOHQiLD6DY3ZW1ajjzOOlvkOAmbpuVwQiFBrCVr\ns8AyTiLZ8ooWJ+FHELNqWo7rvEMhkTHCjm6yKeJad89eh7nJC2oS0YmrSYSdloNDYP2hkMgYcXwS\nuofAmoiTMOm4NqEp5dFxndaiQ5zgLxqcBTbDhJ3gL66Q2L4deP9973Nnzmz03nfsALZsac7Hmde2\nbf55zZoVrfGajJPo7GxM9sc4iehEeYZxIq5pbvKHQiJjmFy+9Ac/UJsXs2cD//iPwGOPAXfd5W8/\nv/56/3IPPxz4539WgicMJuMkdAslmpsaxBkCGwWam4i16J7g78ADg5/7/vtKQNxzT2uD7usL98F/\n+23gqaeCn1/HpLlJt1DKq7kprOCV0ux6EnmHmkSG0aFJXHwx8PLLwOrV3ueMjAC7d6v90dHm9RU+\n8AG1XXWVEhLvvANs2OCd165dKg+gkWcYTMZJhNEktm4FNm6cPP/+fqC3V+W3Zo3a6ri1s9WrvXvI\nQqgV86ZMmbxM03g9wyDDjf3eccZJRINCImPoDqY76CDgjjv8z7n3XuC229qX9+//3nzuj3/sn9f3\nv680EaDZnxIUk3ESQYXE8uXAV78a7PqmTVP34/bb1f85cWsS3/ymf17lMvDAAypPmwmrDQb1L3B0\nUzSMKlpCiEeEENKx5cSqapYkp+WIU14UnD3HKELCZJxEUHPT448Hv7Zdu4Cf/7xVQADqo18uB8sH\nAIaGgOefD36+KcLGSQQVEkmu/V4kIWFakxgGsF5KOV8IcSWAqhDiFSnlSYbrZS1hHddB0uOSpBof\nV0iYNDcF1Vyc5rhDDgFmzGg9Z9OmxuiwkZHmMo49FpgzB7jyysY92rTJu7wNG9RoMyDaPdVNWMEb\ntCMUxKEdFI5u0oSU8grH/o+EEFUA/eZqZD9hHXRB0uOSZCRrnGlAgGw4rp3X9aUvAWed1XrOHXeo\n0V1A8z098MCGOa5O3dTnxTe+ATzxhNrX1et9/HFlhqz7l7zo7gYuu0zdg9tuA155BXjvvcbvznv6\n1lvqXDdBAumA8J0qojCtSexHCPF4bffrRitiOXE0ibSERJK23iQ1CR1DUqNMRhdE28ryjKXj48Df\n/i2wc2ew82+6SQ1SePjh1t+c/pM9e/wHUADNz8BNkhp2kcxNVshQIcTlAM4BsEdKOYmblEwGzU0K\nW+Mkwk4hkbWlNsfGggsIQAmI9etb08tl4E//FDj55OB5XXSR929pdZ7yLiSMaxJCiOMB3A1ASim7\n3b8PDQ2hUqnsPx4YGMDAwIDGGtqLjeYm05qEkyyYm7yERJL3VHev11lGT0/DbOZmYKDhK3Fe40UX\nKbNSf796H+68E1i7dvL3Yfp0YO5c79+TNDfRJ6EJIYQA8BoASCnbPqpyuYzBwUGt9bKZsBP8BUmP\niy1TkwPZiJMwaW7SLSQ6OoBjjml/njNmw3mNs2cDRx/dOBYCOOKI+PVKsvNUJHOTaU1iqL4jhKjf\n6hEp5VRD9bGeOI7rtIREko7rJM1NtsZJ5H3G0iDvqPs3HRHQNDc1U61WUa1WJz3P9Oim2SbLzyu2\nOK6jUIQ4iSDmpiS1M5OmkaBDUuNORBmnLlm7p0nhNN0LnwuywnFNguHusdgYJxE3SKkIcRK6zU1O\nqEn41yMoeRASQaGQyCG2OK7jCom8xkmENTfF7WWbNDf5kdYILi/SMsNm1dwUFAqJDBGlhxYkPS42\njW7KWpxE2NFNWXNcB50mI8k1p73wiqHwi60IQt6FhGnHNYmIX0M67rhw6XFJK05i167m6Fs306ap\nYY+AiuwdHgb27m387vwA79zpn9eMGWq21bBEEUpB7hEd18lTqajFrJyLYXV1AWeeGT4vjm4iVhL0\nZfzwh9UUBy++2EirVIDjj0+nXmlpEs8/D5x3nve5U6YA11wDHHoo8PWvK6HixPnR/sUv1OZFby/w\nne8AixeHq2/W1mPWTVDHtQ5NYupUNVniqlWNZ3DEEWqtk7AUySdBIZEhggoJIdTHLuwHLypJahKz\nZgU/d+9e4Kc/VQ3dLSBKJWD+/OB5jYwADz4Y755F0SR0mJuc2OqT0LUIUFdXuAhuL6hJeCAELpIS\nP0urMiQ4NvVkkoyTWLBATcWwbJl3b3zfPmDzZrU/Nqa2On19ygR1ySXAH/8x8NJLgF8s5t69agEg\nIP4iR1GGwIZdj5nmJqKbQEJCCGwCUN5/KLAawFFSwqJPVf4J2vh0k2SchBBqZtQvfcn7nA0bgAsv\nVPsTE80f0RtuAM44o3H8rW/5l7dyJfDnf672ncImKGkF0+VFk7ApTiJJiqRJBH3lygAerR9IiaN9\nziUpYevLqNts4P6IxBGeXV2Nfec6D0GJGyeR99FN1CSyT+DHISUuTLMiJBw29bZ0N3b3CmNxhES3\nY0rJKEJCh7kpyTgJ3djkuE4SahJtqJmcIASOFwJcZtQwNjUk3Y3dba+PE+Xt1CSimJuixEnoNjdl\nIZgua5oEhUQrt6Lhk3gNgABwcyo1Ip7Y+jKa1CSkDL8mspO0NAm/Z+XUDHQMgXVCc1Py2NoukyKQ\n41pKfAXAV1KuC5kEWx3XbvNPu/Qk8dMk4vgkknRcB9UkGEynSGvalLSwqf2lTSCZLQSkEPia4/h6\nIZBz+WkftvZYdJsN3A7iJM1N+/Z5b+4PWT29jltIBMkn7+amKD6JLGgSRRISgeMkpMQtjv1vC4Eb\n06kS8SILmkTcOImw5cV1XLsXvvn93/c+t1RSq6Z98YvA5z7XHNEONH/wh4b882r3P+6y6uTVcU1z\nUzYI47i+3rF/QzrVIUGxSUhk2XEtBDBzZrBzJyaAhx4CHnigVUAAak3mMOV3dzcLKXe96mRZk/Aj\ny3ESTigkFPsA3FgzO0kA36ylEUPY1JB0Cwk/x3WUMv/yL9UEf6WS9+ZkaKj5uFQCPvQhFeD36U+r\neaD88iqV1OSEn/1sME0i7hodTmwyN2VZk7Cp/aVNUMd1pxB4BcCJtaRXpcRJ6VWLtMPWHovJxWPi\nOq4BNYXHJZf4n/PJTwJvvdUos87FFwPXXdc4vuoqtcUlyUkTs+aTyMIHuEhDYH2FhBC4WkrcUTMv\nPVTb6r/dICX+T9oVJA1sfRlNmpviOq6Donvtg7yYm4rguLa1XSbFZJrEDwHcAWVeageFRAyGh5UT\n9JVXGmmnnQbcfnvz+P06WXBc6/hgO3E3UB3ai25tKWtThRfB3FQkfB+HlBBCQAD4Nykh3JumOmrj\n5ZfVDKRLlqjt3HOBe+9Nr7wnnlBl1u3qUqo1FP7zPyf/X5uERJJrB4ctLwlzUxB0j+DKS5yEH1l2\nXFOTcCAlpBDwWfolP9x1F/DGG81pP/wh8IlPAD09yZfnXgOhzs6d7dNt1SSCBISlVZ4uc5NJ53xe\nZ4E1qYFmkWeeAR591HtZ34MOUotvJU3gOAkhsEFKHJp8Fexh27bWtD17VJBVGkLCK8o0Lz2TLMRJ\nBMWkuSmv60kkeY26iXtPX3sN+Pa3gXffVcddXWpwxF/8Rfvzd+8GvvENtTiWF/394esRhDCv+CH1\nIbCOobC5wuthpzVlQNjysqZJZCFOIigmHddOsuC49irb77esmZucRLmn99yjrBY7d6ptyxagWgW2\nb29//ubN/gIiTYJqEo+lWgtL0N2zD1uerRqGbiGR5AR/QTGpSQRJ94OO6+SJK3jbCYOJCWWCnjGj\n9TdnGbNnA9de23pOb2/4egRhsiGwZwP4JdSsrxLAEVJiXTpVMY9uTcIr3yDl2dTbSrLXG7Y8KeOv\nhhe2TB293iQFbxYc11mOk4hC2A6i8/xp0/StXw9Mbm76FRoCQgBYm3aFTBLnox0FmpuiIYR+p6du\n+3lagtdWn0SQdJuI+9zjtH3d92fS4mrDXUsA+tOvjlloboqGbk3CXaaJ1fB0lhckPWheNgkJE+9N\nGkS5p2E7pCY7iGFmgV1nU+81DWx3XDux6Vno1iTqZdbvk25Hsu44Ca96BCUJITE+3hiyXSoBBxwQ\nvmw3Jt6bNIhyT8N2BE2O/ppUSLhHMTmP8xZQlyVNwqaGZMJsYNLclOXRTVFYuRK45ho1wqbO4sXA\n3/1d+7yLoEmk5ZMIkm6VualdlHWeI651m3nyEidhk7kpj3ESTkxoEo8+2iwgAGD5cmDNmvbnR3Fc\nO7GpA+RF3HuapQ5iBmS2PnSbm7zImvA49FBg7tzW9EolvTK9pnRgnIQ/Ud4hr/H5XulFc1zrNjfp\nvj+BfRJFwKS5SYhGOTb2JvwolVRw0NNPqwh1ADjsMH1CwuSEe1mLk7Cp15tlc1Nc4jiuKSQM4n65\n68c64iQ6Oho9YhtHOExGXx9w/vn6yvPq2esoT0ecRJDFiIKSVq837gAL297hMKQ1BNZGx7VxmS2E\nkPXNdF28FqjXMbopbHlZbmBJYIsmkdZzWLBATdjmJkoQle4PWhHMTU50CN5MDIFNka0AdgCYZ7oi\nXj17HeYmp5AIUl7RhYTuOAndQqmrC3jwQbWWdr28/n5lxgtLWuamuObZLJubkrynQawWaZibqtUq\nqtXqpOcZFxJSyllCiMdhgZCI07OPW16n40lkzXFtApMR0LqciL29wKJFyeapw8ka9IOW5dFNTuLe\n0yAd0jTe8YGBAQwMDNTy9M40AzJbH84H4fxo26JJUEg00G2q0D26KUlMahJ+9+fww1vTOjraj5Sz\njSTjJIJ0SDm6yYehoSFUHMNknNIvabx6QDqERJDybHZc6ybJaSuCUOT1mNOaiPLSS4HRUWDtWnXc\n0QGcc46a5dR2krynYTuIRfRJ+FIulzE4OKilLK+PNh3X9mFy5tksaxJRiGNu8uOAA4C/+qtodTJN\nkvc0iKnZ2ojrokFzU3YwuWQqNQn/vIqm8ca9p2GtCIUzNzmHvtb290kpjdeLQ2DtRvf152UVtShk\nabhmVghr2rZ6gr+0kVJa8xrZHifBxtfAFk0iC88hreGaRdYkkrynYUc2Fi6YzibixC3oKI/mpga6\nfRJFNjdR421F9z01aW7KwCuuD9vjJJzktfEFxeS62hQS/nkVoTOj+54WeloOm6DjOjvQ3BQc3WP6\ni2BuikuW4iQoJBwwTiI76I6TMBFxnQZ5t5/rIi3Ba2Pbz9grni6Mk8gOJs1NOspLEpPmpizcn7jE\nvaeMk8gQWTI3FaHx+XHaaa1plYqeiOsg6TaRVnRwkTszSd5TxklkCJOOax3l5YlrrgFOPx3YskUd\nT50KnHFGeuVRk1CE/aCR9mTpnlJIONAdJxHHIVh0OjrSFQpusjxjaZHs57rQHSdBc5MlhLW9xoWN\nLztk2dzkRIe5qQjvqW4THuMkLMGk+YejRuymyAvk0HHdSpIT/DFOIiNIab8m4SSvjc9W8mJu0h0d\nnIX7ExcdmgTNTRbgftBxG1YQwo5wIOYosiYRJ06CtCdL2lkGXnE9uG1+NsZJFK2HZhN50SSikKUP\nmi6S1M7ouM4Ibpuf7ohrTsthN3RcK9iZURQpTiJjr3h6uB+C1zQMaZXJxmc3jJNQhO08ZeH+RKFI\nsScUEjX8NAkb4yTy2vhsZeHC1rRSCTj+eP11CUuR1mPOClmKk2AwXQ33Q9DtuNYxmopE58wzgX/4\nB2DVKnVcKgGLFgEHHWS2XkEwOaY/r5icWZdCwhCmHddcT8JuSiU1Dcjpp5uuSXjouE6XvK8nQSFR\nI6rj+o03gKefbl5joB3TpwN/9EfAjBnAk08Ca9YA777b+N35orz5JlCttubx/vvNdSQkLHRcJ0Mc\n7cwvJovmJo2sXw88+6z37/PnA7/3e8DYGPDUU8CmTY3f3C/2c88BO3a05rF5M/DjHwev0zPPAOec\nA3znO62/OR/82rXthQQhUdA9XNOrbKJwPwPbtbPcColVq4Dvfc//nGuvVQJg2bLmdLe56Re/UFtc\nXn0VmDu3Nb27GzjvPOC++4Kbtk48MX59SDHgtBzJE1eTqBN0JCU1CUO8+KLa3Bx9NHDsseHyKpeB\niy9uTd+zB7jrLrXvVjMXLwZOOEE5RY85Rmkl//Vfk5c1ezbwh38Yrn6kuNBxnTzOe/r668C//Iv3\nuYcdpmYs3rNHmZqdZmN3Xs8/r6wbblasaOxTSCTEvHnAJz/Zmr5uHfDf/632pWx+0S+8UH3sL7wQ\nmDULmDJF+Q4m46ijlIBo9/B27/YWEmedBVxwQeN44cL2Qy0JiUOSjmsus6twXteKFc0f8XZ84QvK\nf+m2SLitFsuWtVo2TJNbIXHccWpz89hjDSExMdEsJP76r4G+vsbxpZfGr4d7lFSW10cm2YdxEslw\nzDHhzn/xReC3v21NP/bY8FaLBQvCnR+X3AoJL9w9obRfaL/yKCSIDrh8afJUKsB3v9veXF3nnXeA\n5cvVvrvtX3ABcPDBwPnnK+tFqQSsXj15ucceq9/UXDgh4XYSpd2z9ysvrw2I2AUd18kjBHD22Wrz\nYvnyZiHhbPtXXtk8iMVpdraNwvVl3eaftHv2NDcR09BxbQbdHdK0yFBVk8HdYNLu2bvLK0Ivi9hF\nko7rsOtJZOljmDR+HdIstf3CmZt09+zdQsIZmV3kBkTMsHKlsqV7cdBBaqSelMDDDwNDQ8DISON3\npybxzDPAtm2teaxd29jP0scwafw6pFlq+4UWEjp69kKorV4OfRJEN873bP16tfnx7rsNIeHGKSRe\ne01tQcsuGrpN22GpVquoBpjawYKq6sWE+ccrorLIDYjo45RTwr1rq1e3H655zDEqADQMp54a7vw8\n4e6Q2tb2BwYGMDg4iMHBQd/zCqdJmHAmOcukuYno5pBDgPvvB/7nf7zP2bChETXs/qB96lPAkUeq\n4M8ZM4C771Zmq8mYPx847bRYVc80eXFcF05I6DY31cusCwcKCWKCI45QmxcrVzYLCSd/8Adq+pg6\nJ5zQfEzakxfHtfHPlBBilxBC1raUVm5wltfY37dPv7nJKSSy9KKQfMN4nuRx3zcKiQgIIcoApgJ4\nCsCHVZJ4Jc0yvRYTqjuY0y6TmgSxkbyMxLEJmpuS4VUAkFJ+HACEEBJAqpNgm+jVO1+I8XE9ZRIS\nhryYRmwiL/fUtDzrcx1PAEj19nl9sNOU7HRcE9vxG4nD9zQaedHOMlTVZPAy/ejSJOiTIDbiNo3Y\nNqY/i9geJxEU01Uddh2XADSNrRgaGkKlUtm/BQn+8MOEuYmOa2I7fjMR8D2Nhu1xEkEx7ZM4DcBa\nIcQvAXwVytTUFMNZLpcnDfYIgwknMh3XxHbyYhqxibw4ro1WVUq5DsAIgCUAXlBJ0ojjmkKCFJm8\nOFltIi/31LQmASnlVJ3lmZ794eXIAAAKSUlEQVQiI6sqJ8k3eTGN2ISfnydL97RwfVmamwhpxW9O\nM76n0TARk5UGhXv8jJMgpJW89HptwrTVIikKJyQYJ0FIK4yTSJ68jGos3OO3KU6CjY/YAuMkkicv\n7T5j1Y2PTXEShNgC4ySSJy9m5sIJCTquCWmFcRLJkxczc8aqGx/TcRK6/CCEhCEvY/ptIi+dw4xV\nNz4mVMC8OLBIfmGcRPLkpd0XWkjoUqlNlElIGOi4Th5qEjlAV2/Jbe/VUSYhYaDjOnnouM4oJnr1\nXi9F1noUJL/QcZ08XsF0WbufGatufLwekC5zk64yCQmDl7YL8D2NSl7MzBmrbnxMqHpZUy9J8aC5\nKXm8NImsUTghQU2CkFZobkqevJiZM1bd+Jh4cBQSxHYYJ5E8eWn3GatufLwekK7RTbrKJCQMjJNI\nnry0ewqJSdKzWiYhYWCcRPLkpd1nrLrxMSHd89KjIPmFjuvkMWG1SIPCCQlqEoS04jcSJ2sfNVug\n4zqj0HFNSCt+2i6FRDTy0u4zVt34mHjhp01rTSuVgO5u/XUhpB1ewoACIjp5MTN3mq5AWry88WU8\ntOohrB9ej66OLggIjO0bQ6fswrqTBGRpDF0j89C9fSHGZqzEjoPX46qfN85z/k/Y/Xl987CwvBAr\nh1Zi/fB6bFvYhU1jArvHxoB9XejoEDjy2DFc82S8fOPUMa18s1BH5ts+380HL8ToASuxp3c9sE/9\nhs6xWO0ii/chqXyniC6sWyggO5q/Ne+X431r3HWc1zcPnzjuEzj54JNT+ZYK6Y7Bt4xKpSIHBwdD\n/c/LG1/Grc/eigN7DsTo+CiWr1sOADixfCJeHXoVm7cAE2sXY19pB3bPeg59OxbhtFOm463x5vOi\n7C+evxg7xnbguQ3PYdFhizC9a3pL+XnNNwt1ZL7e+T74/HOYtnURxN7p2PWB5ZAApu46EQcvtLO+\ntue7cuhVrFkDHLBlMfZ1qG/NtK2L0Nc7HT3HJtPGjpp1FIZHh7F1dCu+vOjLkQWFEAJSyrY6Ti6F\nxNKnlmLryFa89N5LWLttLfZO7AUksGvvLkzrmgZIoENMAQCMjo+gd0ovhEDb88LuT+lQ+Y7sVfkC\nxck3C3Vkvt75btg0ArGvF5CALO0FAEx07MLBH7CzvtbnO2Uaht4HMDEFkMBExwhKE72YMgWYOSuZ\nNtY/sx8AcMqcU3Bg74FYetbSUN/KOn5CIpc+ifXD69HX0wcAGB0fRafoRGepU5mbavt7JkaxZ2IU\nPZ3dGNs36nle2P3RcZVXd0f3/v2i5JuFOjJf73wx0Q1ZGoUsjQKyE0J2Qpbsra/1+ZY6Mb23c/89\nFRPdQMcounuTa2N1+nr6sH54fajvZLVaRaVSQaVS8T0vlz6JeX3zsHVkKwCgp7Nnf2+gu6Mb43Ic\nkCodaN9rcJ4Xdr/I+WahjszXO1+UmjUJCUBIe+ubhXx7pwEHHNADKdtbLZKoIwAMjw5jXt+8oJ9I\nAMDAwAAGBgYAKE3Ci46lS5eGylg31Wp1af1CgjKrZxaWrV2G/pn96J/Zj407N6K7sxsfOfQj2D62\nHd2d3VhyxBLMmTYHm3ZvwumHn44Tyie0PS/sfpHzzUIdma93vr95ZxOmbTkd3TtOwHj3RoiJbszY\n/hHMmmtnfTOT75FLcPB0le9HE25jH5z7QczsmYmto1vxmQ9+BnOmz4n0nb3hhhuwdOnSG9r9lkuf\nBOA9uilrIySylm8W6sh82+f7f3+wEMPdzaObOnvGcOH5dta3qPmmMbqpcI5rQkh4Pv5xYMeO5rQZ\nM4Bly8zUh+ijcI5rQkh42pmlsxYdTJKHrwAhBAAwc2ZrWl+f/noQu6CQIIQAAK6+Gpg3TwmLmTPV\n/tVXm64VMU0uh8ASQsKzZInaCHFCTYIQQognxoSEEGJMCCFr29um6kEIIcQbk5rE6wBWGCyfEELI\nJBjzSUgpTwUAIYTdgRqEEFJgcu+TqFarpqtgDF57MeG1F5O0rj1VIeHwObi30cn/WzE0NLR/psJK\npRL6RvClKSa89mLCa0+eVM1NXmHeYSiXy+C0HIQQYgbjczfVfBLvSCkP9/mdEEJIilg3wZ8QYgxA\nlzMtCc2DEEJIchjXJAghhNhL7kc3EUIIiU6uhYQQYpdjRNWE6fqkhRDiEdfosYla+g2u9OtN1zUt\nhBDl+nXWjn/puvYPma5jGgghvua6zk1Fee5CiAnXdZ6d1+fuvCZHWtvnLBTO9HWxys6ruUkIUQaw\nCcBTAK4B8AKAlVLKk0zWKw2EEHcDOFNKOV8IcSWAKoCVABYCkFLKUk1wiLz6ferXByjfVq0xjUsp\np9T2pZQyd52i2rVNSCk7ah/EcwDcjJw/dyHE/QAuBXCulPL/1e7DPgAdyOFzF0JsAbADwLz6s3Re\nn/M5CyF2A+iFUgI2AjgozvPP/M3z4VUAkFJ+XEq5AoAEcKLZKqWDlPIKKeX82v6Pasn9tb/3uP7m\nDiHEA1ACYnvtuN4gjq79Har9niuEEF+o7XYCgJRyhZTyllpa7p97jY8IIebX9uvxV7l77lLKWQB+\n0+ands+5F0pQSinlHGB/RzISedYkxgB0OaTuOICOvPWo3AghHofqTf5vAD90XL+A6nHm7vprPaq3\nAcwA0AfgVgBfdlz7IwAuyNu1CyHeAnCEK/kyAP9akOe+X3uskevnXm/bTk2i3XOutYfNUsrZ9fPg\nE2YwGXnWJAqHEOJyKAGxR0p5h/M3mdPegBBiBACklPNM18UAU2p/Nzo+hP/qPCHHz/16KAHx9wBO\nqCV/0VyNzJLmc86zkBh2HZegTE65RAhxPIC7od6Xbkf63c6/OaQH2N9bqi+2+eVaWt0MschAvXTw\nBABIKefWjrfUfyjAc/8bAJBSfkFK+Toa/ogiPPf9+Dxn98KzT0YuI6cdjfqLshbArwB8Fcpx/ZqU\nMnd+ibqqCTQHJHo5tgxVM3WEENsA9BXQcb1CSvlh58gX5Py5CyF+B+BgABcDeATq/ZdQ2kUun3s7\ncxPouI6OlHIdgBEAS6AEhMyjgKgxVN9xDHvbDTXKpf7BFAC+ZaqCBngKQKfjw/lRg3VJkzUAPuS4\nzr9BAZ67Q3t6GLUOEtQzfgo5fO616zmnvl/zsXo952m1vxMADgLwTqyy86pJEEIIiU9uNQlCCCHx\noZAghBDiCYUEIYQQTygkCCGEeEIhQcgkCCFuck2Y1rSZrh8hacLRTYSEIG9j7wmZDL7ohMSgrkkI\nIba10TLGHfvLa+d933XOkH8JhJiFQoKQZPkJgNNr+/UJJSWAj9XSPg81+ZoA8G8AZuuvIiHBobmJ\nkBC4zU31mTidU4I4zntfSlkWQgwDmOGYLsTNFVLKn2i7CEJC0Gm6AoTkmPr6Bm7BsERKuUx3ZQiJ\nAs1NhOjnl/Wd2poQhFgLhQQherkVtUnZaqYn96JBhFgFfRKEEEI8oSZBCCHEEwoJQgghnlBIEEII\n8YRCghBCiCcUEoQQQjyhkCCEEOIJhQQhhBBPKCQIIYR48v8BNQO1iM0WXbcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqVanaFHL9D2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eNMWDODNlg4O",
        "colab_type": "text"
      },
      "source": [
        "# RLgraph"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vobaHBUzliF_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install rlgraph gym pygame > /dev/null 2>&1\n",
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install pyvirtualdisplay > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aQmw0THVei0t",
        "colab_type": "code",
        "outputId": "c8833db2-2ed0-44cd-e54d-c17e0a276773",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import os\n",
        "os.mkdir(\"configs\")\n",
        "import urllib.request \n",
        "urllib.request.urlretrieve(\"https://raw.githubusercontent.com/rlgraph/rlgraph/master/examples/configs/dqn_cartpole.json\", \"configs/dqn_cartpole.json\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('configs/dqn_cartpole.json', <http.client.HTTPMessage at 0x7f953b5ccd30>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h0SPaywOljtF",
        "colab_type": "code",
        "outputId": "9c54df73-8e1e-42c6-b42c-cab6f4a5ac97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "from pyvirtualdisplay import Display\n",
        "from gym.wrappers.monitor import Monitor\n",
        "display = Display(visible=0, size=(1400, 900))\n",
        "display.start()\n",
        "\n",
        "import numpy as np\n",
        "from rlgraph.agents import DQNAgent\n",
        "from rlgraph.environments import OpenAIGymEnv\n",
        "from rlgraph.execution import SingleThreadedWorker\n",
        "\n",
        "environment = OpenAIGymEnv('CartPole-v0', monitor=\".\", monitor_video=1, visualize=True)\n",
        "\n",
        "# Create from .json file or dict, see agent API for all\n",
        "# possible configuration parameters.\n",
        "agent = DQNAgent.from_file(\n",
        "  \"configs/dqn_cartpole.json\",\n",
        "  state_space=environment.state_space, \n",
        "  action_space=environment.action_space\n",
        ")\n",
        "\n",
        "episode_returns = []\n",
        "\n",
        "def episode_finished_callback(episode_return, duration, timesteps, **kwargs):\n",
        "  episode_returns.append(episode_return)\n",
        "  if len(episode_returns) % 10 == 0:\n",
        "    print(\"Episode {} finished: reward={:.2f}, average reward={:.2f}.\".format(\n",
        "      len(episode_returns), episode_return, np.mean(episode_returns[-10:])\n",
        "    ))\n",
        "\n",
        "worker = SingleThreadedWorker(env_spec=lambda: environment, agent=agent, render=True, worker_executes_preprocessing=False,\n",
        "                              episode_finish_callback=episode_finished_callback)\n",
        "print(\"Starting workload, this will take some time for the agents to build.\")\n",
        "\n",
        "# Use exploration is true for training, false for evaluation.\n",
        "worker.execute_timesteps(1000, use_exploration=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0628 19:17:14.603792 140280332130176 abstractdisplay.py:151] xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:14:WARNING:xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:14.928444 140280332130176 agent.py:100] Parsed state space definition: Floatbox((4,) <class 'numpy.float32'> )\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:14:INFO:Parsed state space definition: Floatbox((4,) <class 'numpy.float32'> )\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:14.933475 140280332130176 agent.py:103] Parsed action space definition: Intbox(() <class 'numpy.int32'> )\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:14:INFO:Parsed action space definition: Intbox(() <class 'numpy.int32'> )\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:14.948281 140280332130176 agent.py:127] No preprocessing required.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:14:INFO:No preprocessing required.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:15.003520 140280332130176 graph_executor.py:66] Execution spec is: {'mode': 'single', 'distributed_spec': None, 'disable_monitoring': False, 'gpu_spec': {'gpus_enabled': False, 'max_usable_gpus': 0, 'fake_gpus_if_necessary': False, 'cuda_devices': None, 'per_process_gpu_memory_fraction': None, 'allow_memory_growth': False}, 'device_strategy': 'default', 'default_device': None, 'device_map': {}, 'session_config': {'type': 'monitored-training-session', 'allow_soft_placement': True, 'log_device_placement': False}, 'seed': None, 'enable_profiler': False, 'profiler_frequency': 1000, 'enable_timeline': False, 'timeline_frequency': 1}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Expose API sync from synchronizable to dueling-policy code:\n",
            "@rlgraph_api(component=self, must_be_complete=False, ok_to_overwrite=False)\n",
            "def sync(self, values_, ):\n",
            "\treturn getattr(self.sub_components['synchronizable'], 'sync')(values_)\n",
            "\n",
            "19-06-28 19:17:15:INFO:Execution spec is: {'mode': 'single', 'distributed_spec': None, 'disable_monitoring': False, 'gpu_spec': {'gpus_enabled': False, 'max_usable_gpus': 0, 'fake_gpus_if_necessary': False, 'cuda_devices': None, 'per_process_gpu_memory_fraction': None, 'allow_memory_growth': False}, 'device_strategy': 'default', 'default_device': None, 'device_map': {}, 'session_config': {'type': 'monitored-training-session', 'allow_soft_placement': True, 'log_device_placement': False}, 'seed': None, 'enable_profiler': False, 'profiler_frequency': 1000, 'enable_timeline': False, 'timeline_frequency': 1}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0628 19:17:15.006790 140280332130176 tensorflow_executor.py:136] `device_map` given for device-strategy=`default`. Map will be ignored. Use device-strategy=`custom` together with a `device_map`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:15:WARNING:`device_map` given for device-strategy=`default`. Map will be ignored. Use device-strategy=`custom` together with a `device_map`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:15.009473 140280332130176 tensorflow_executor.py:139] Initializing graph executor with default device strategy. Backend will assign all visible devices.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:15:INFO:Initializing graph executor with default device strategy. Backend will assign all visible devices.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:15.013883 140280332130176 tensorflow_executor.py:141] GPUs enabled: False. Usable GPUs: None\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:15:INFO:GPUs enabled: False. Usable GPUs: None\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:15.049511 140280332130176 graph_executor.py:244] Components created: 50\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:15:INFO:Components created: 50\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:25.299105 140280332130176 meta_graph_builder.py:134] Meta-graph build completed in 10.247604421000005 s.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:25:INFO:Meta-graph build completed in 10.247604421000005 s.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:25.303957 140280332130176 meta_graph_builder.py:138] Meta-graph op-records generated: 809\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:25:INFO:Meta-graph op-records generated: 809\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:32.666053 140280332130176 graph_builder.py:210] Computation-Graph build completed in 7.356028490999961 s (87 iterations).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:32:INFO:Computation-Graph build completed in 7.356028490999961 s (87 iterations).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:32.694278 140280332130176 graph_builder.py:214] Actual graph ops generated: 1221\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:32:INFO:Actual graph ops generated: 1221\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:32.696989 140280332130176 graph_builder.py:217] Number of trainable parameters: 238\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:32:INFO:Number of trainable parameters: 238\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:32.700151 140280332130176 tensorflow_executor.py:727] Checking if all visible devices are in use for strategy: default. Available devices are: ['/device:CPU:0'].\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:32:INFO:Checking if all visible devices are in use for strategy: default. Available devices are: ['/device:CPU:0'].\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0628 19:17:32.705060 140280332130176 tensorflow_executor.py:731] Warning: Device /device:CPU:0 is usable but has not been assigned.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:32:WARNING:Warning: Device /device:CPU:0 is usable but has not been assigned.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:33.017920 140280332130176 tensorflow_executor.py:575] Setting up singular monitored session for non-distributed mode. Session config: allow_soft_placement: true\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:33:INFO:Setting up singular monitored session for non-distributed mode. Session config: allow_soft_placement: true\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:33.089520 140280332130176 monitored_session.py:240] Graph was finalized.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:33:INFO:Graph was finalized.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:33.288555 140280332130176 session_manager.py:500] Running local_init_op.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:33:INFO:Running local_init_op.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:33.323627 140280332130176 session_manager.py:502] Done running local_init_op.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:33:INFO:Done running local_init_op.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:33.394548 140280332130176 single_threaded_worker.py:39] Initialized single-threaded executor with 1 environments 'OpenAIGym(<Monitor<TimeLimit<CartPoleEnv<CartPole-v0>>>>)' and Agent 'DQNAgent(doubleQ=True duelingQ=True)'\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:33:INFO:Initialized single-threaded executor with 1 environments 'OpenAIGym(<Monitor<TimeLimit<CartPoleEnv<CartPole-v0>>>>)' and Agent 'DQNAgent(doubleQ=True duelingQ=True)'\n",
            "Starting workload, this will take some time for the agents to build.\n",
            "Episode 10 finished: reward=19.00, average reward=19.50.\n",
            "Episode 20 finished: reward=112.00, average reward=41.00.\n",
            "Episode 30 finished: reward=10.00, average reward=27.20.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.724617 140280332130176 single_threaded_worker.py:381] Finished execution in 22.326781784000104 s\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Finished execution in 22.326781784000104 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.733809 140280332130176 single_threaded_worker.py:384] Time steps (actions) executed: 1000 (44.78925846431766 ops/s)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Time steps (actions) executed: 1000 (44.78925846431766 ops/s)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.736529 140280332130176 single_threaded_worker.py:387] Env frames executed (incl. action repeats): 1000 (44.78925846431766 frames/s)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Env frames executed (incl. action repeats): 1000 (44.78925846431766 frames/s)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.742660 140280332130176 single_threaded_worker.py:390] Episodes finished: 35 (94.05744277506707 episodes/min)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Episodes finished: 35 (94.05744277506707 episodes/min)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.746781 140280332130176 single_threaded_worker.py:391] Mean episode runtime: 0.44233926240000465s\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Mean episode runtime: 0.44233926240000465s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.750730 140280332130176 single_threaded_worker.py:392] Mean episode reward: 28.114285714285714\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Mean episode reward: 28.114285714285714\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.754754 140280332130176 single_threaded_worker.py:394] Mean episode reward (last 10 episodes): 16.6\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Mean episode reward (last 10 episodes): 16.6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.758997 140280332130176 single_threaded_worker.py:396] Max. episode reward: 124.0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Max. episode reward: 124.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "I0628 19:17:55.763466 140280332130176 single_threaded_worker.py:397] Final episode reward: 40.0\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "19-06-28 19:17:55:INFO:Final episode reward: 40.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'env_frames': 1000,\n",
              " 'env_frames_per_second': 44.78925846431766,\n",
              " 'episodes_executed': 35,\n",
              " 'episodes_per_minute': 94.05744277506707,\n",
              " 'final_episode_reward': 40.0,\n",
              " 'max_episode_reward': 124.0,\n",
              " 'mean_episode_reward': 28.114285714285714,\n",
              " 'mean_episode_reward_last_10_episodes': 16.6,\n",
              " 'mean_episode_runtime': 0.44233926240000465,\n",
              " 'ops_per_second': 44.78925846431766,\n",
              " 'runtime': 22.326781784000104,\n",
              " 'timesteps_executed': 1000}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0pcN67FFeFZb",
        "colab_type": "code",
        "outputId": "4da78f31-c170-41db-bb54-fccbefd79445",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        }
      },
      "source": [
        "show_vid(next(glob.iglob(\"*.mp4\")))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<video alt=\"test\" autoplay \n",
              "              loop controls style=\"height: 400px;\">\n",
              "              <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAEDVtZGF0AAACrgYF//+q3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE1MiByMjg1NCBlOWE1OTAzIC0gSC4yNjQvTVBFRy00IEFWQyBjb2RlYyAtIENvcHlsZWZ0IDIwMDMtMjAxNyAtIGh0dHA6Ly93d3cudmlkZW9sYW4ub3JnL3gyNjQuaHRtbCAtIG9wdGlvbnM6IGNhYmFjPTEgcmVmPTMgZGVibG9jaz0xOjA6MCBhbmFseXNlPTB4MzoweDExMyBtZT1oZXggc3VibWU9NyBwc3k9MSBwc3lfcmQ9MS4wMDowLjAwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTE2IGNocm9tYV9tZT0xIHRyZWxsaXM9MSA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29mZnNldD0tMiB0aHJlYWRzPTMgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5yPTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2ludHJhPTAgYmZyYW1lcz0zIGJfcHlyYW1pZD0yIGJfYWRhcHQ9MSBiX2JpYXM9MCBkaXJlY3Q9MSB3ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNjZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NDAgcmM9Y3JmIG1idHJlZT0xIGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0xLjQwIGFxPTE6MS4wMACAAAAByGWIhAAz//727L4FNf2f0JcRLMXaSnA+KqSAgHc0wAAAAwAAAwAAFgn0I7DkqgN3QAAAHGAFBCwCPCVC2EhH2Olf8H/B4S4Aki/BnAW3OE3W6w+rQR7xDuK6XMGpy0V8AvgqgoQRY2pzK29v9pyNGBcghdlOeD9La2aQIoUX0IhDZXesNNqKKgnAJJriJNSS69RlhdowBuPvrP9JfAvksAT+EpCL4t1hxVKIVOB+Ay5AXi1X6TuQcvaaM0VF/4m9k4j/8uW1DlTxtoZLDSXYBpyPabyBfBxLLpwJ9xSLOWIG0PdA5F+s6kL3fqswmwApqr6UkE796J5IsMfZ7VqLzFjQAkQ7O/UgVI4eJbvIzXSAAs+9Iv8f+SGg1Wd9ZMgyBreNuzg1yxzDDN9w+/GEiEW93/cn23T/Yn8bqyQhHpmotqvZZHsHQXMTgyegRGP7jr9pYwm4IV6IiRNdbBuyaxkJzAAAGFDBtfd0/Otc3rAKgF9Ko2AGopxUpqjcA3DvTzF56yESYAERBgpkhaIvYl343+HE5w+pk84vLirbPGgAtus9Nqs9TK+pu8OT4MkMnJRHY5oAMYAE1SagAAADAAADAAADAAAPeQAAANJBmiRsQz/+nhAAAEV1ov1VpQA4xKy2TRmFbZAz01m54lJGq4vuN6t4r/2LP5eJb7nfgXxoZjCbdyt0exxi0np5xEpTCaSAXAgTSK5xilwa5hTZwp66sosixGpnyXmitQQeno2LLgZy0Pyq+PAy5FjhjTu5BakywJ/FK9VaQD2StBT5Rq1P/qADNcLnKk6DAIxDtdSiHo6y3Ta31cUVoU5aHzXC8z5mpzMCILyEMInAvWDvuV5+yOhcaNPda0Nj4SxTG/zNPeMN++2qZi9OpzPyuM0AAABSQZ5CeIR/AAAWvkuwQntWYvHYIPdpp7Hy0ngAbDd3Ag8Rf5SwqLTzeI3C3ipZrgJ97gIs8M650tWlxZEvSZCMYHpE4bQA1xQKs9oH56Mvg36E7QAAADcBnmF0R/8AAAMARY//dfgjkpAUM/gxjBviNmpETFRsfxDNmCIq6ANNT0tKFBNwjmMDiy9GWA+YAAAAOgGeY2pH/wAAI5ksd2sEnlanbFSDAorFo4AL6nfeoBDCJjQFjeRKNT77aWmDgT6EcQ/Mjb9yW9PCAVsAAADXQZpoSahBaJlMCGf//p4QAABFY8lThkAbZnWeIk1DFo2T7hZ6Jb3GEu2b/3Nt/5gspTpYCf68BDVOVtMCa/A/6bI2cLw9NrG/U7wjQ4fLqVvMcUSMHRwQiS7TVgDsMaX9MYmdSX/aauB9ylBQpbSRh86RnkVo/KvMXlSnRtiwP/g1Uaz0n6JJa4RiL9ipcZAbH+FXqT4uIG3Mj9z9/+hDg1TK5MDoCgjrCla+R40RwbP/zw87Wpxfywet/KkxXAkNxSTDZF2GTxrDleuJiNLbald6xJSg8e0AAABSQZ6GRREsI/8AABa1N6pg5rHPd0Wlu56tWtKhqrfasIAqudr8qxukLsviIZiHdLfOLXpseC9CfpxqsO22XiPU/6XZOwhhPv7vdXtdesvVKoC7gQAAAEEBnqV0R/8AACPDF2QkGv/1IqliKDsv4YNMK6wEOqpRfk8pAAnF5lUHlPEUE4icxX6N4/eAAAADAADDm20rUqgMCQAAADwBnqdqR/8AAA2COh0IrK0uG3rjwILVVCqnprpc6RAn9YJ4LWfsODNAkhnoAAADAAADAAd9pMw+9aMsB0wAAACCQZqsSahBbJlMCGf//p4QAABFRSWfI0+qiLFwk1y9db8DUyMAAh3fV/pg0gqTcbW+PVFJ7Uq0Lqku4NuY+yiD++8Jc5AZ4rxo5ircYeSdAuj652MsXhLfLGwT4C+/Oz603QBjrdfUjtYEZ6OnYzvuAAADAAADAAADACnZN3UtUywGzAAAAExBnspFFSwj/wAAFrxHFaNSBesAFdquA/VicCDi7iHWhfwJwS7zbF4WcAAJFujho/dmTptTl0SdnDWvzbAAAAMAAwCW6mVmYStGWAppAAAAOwGe6XRH/wAAI7zqFX5jW9f+r9b02jfQG56E7qjEv2Wy800a7tr+augz5g4IAAADAAADA36GMlCmWAesAAAAOwGe62pH/wAAI7PUQAcPQjNeOaAlcUo1fH+KIhCboA/2Lxj6KP65f30J0FpR8qbQ1lH2kmb9jMNXpcCAAAAAb0Ga8EmoQWyZTAhf//6MsAAARhDNHIAHFrg4/FIt++D+/zLfv++8Z9uNHTvM8JU0/blnnrS2g2eBXgdWTAcLgOvDobkeLpkclCkKOCjcgeRtHZ/AeNg2ruHmsHIbEVvTl1mKhEkIA1k1Yn5mh3quVQAAAD9Bnw5FFSwj/wAAFrxDzFPv7gufICCYQbOVfyKaO5KUJ326+vF4JkSXgAP0+4nk33fq0TNP7Dl8PB4jv8rPSpMAAAAoAZ8tdEf/AAANf09pmhnp9XY6AXAeUO07yIxNj+C7mR4Hx1pEeD9MqQAAADQBny9qR/8AACO/BCZuneBbFy6O/sRGD/gD3mBTVocPF2w5dgATlcROp1eNObSDcrPApksoAAAAqkGbNEmoQWyZTAhf//6MsAAARgEMYMAIx9xIBg2/sFFR7RcFB82JNddF5mC+Ja3VRqgtQ0fNt968R8MinKkyfmgBlA2YzIMYxLMW/9dsMHTkF+invfPYtHWrndubO+sSAUs1TAQDDH+dDo4L7ndR45Z227/yTYi3X74HbIS/vsDKwdFQ77NycQaLp4hMjomHZzgWWeMAjLH4LW7x7B54fFsDJAwJb9Cn9LBCAAAAakGfUkUVLCP/AAAWrS/d4mKULFStCQQX15xC0rQZsbHJXng8VbmhXvoXKuaDp6le3bvHQ40yr+GgJQFDgMVYofyLHoQDIbukFJABL816zlJYHzxbFqoWJBh4flQUtALrq5x+9CTGczERE6cAAABIAZ9xdEf/AAAjqc7FkRmzQiNArzTwMU/jpai10gsLYqLT1OtJ1h96AEtM+XIub6Kx51sCn5J2tQ3aXoSf69qVoipBTPdCShJgAAAASAGfc2pH/wAAI72WhBDHx4kyTrSXg7IievYspMonPAuZle3Buca/+CUI2SLfVSHjgAJaFs9aQRR9XP823puE6n3pktlhkVJdeAAAAIlBm3hJqEFsmUwIX//+jLAAAEZ6b0GvfafgARf/XLtUfjF/MxYTinz+Wm9/FFvAwBE8e43g9S1wioByz+g5s7av2cMaCxKTEpb1FNhxE3/4M7iHhfdKL5xQOOXTb4o4XR2+VTMnMfhSfOehzWyuv/tBEJdyU2JfRF9oHfK023qKwY0wuXykuD3jQQAAAF1Bn5ZFFSwj/wAAFrKesZk0+ty57EYzr0SxWeQsqqCnPV/wNwcXbiPE00A+lqASMmpnpTLvO0jwACRbonG9fADX1/Gf7wZxRgAKmpoXibxhRBjeN2AEu2e6jGcA6YAAAAAvAZ+1dEf/AAAjwTQOzHvwfhToWhhRdDIM6atLNoQi3zrccwQnOOMzHu6a1qzsPSEAAAA0AZ+3akf/AAAjwj8/VWVdcHcsK8EflsjBzp1ifd6Z/cvdl5BcD4ScHrXcHlFl2nFXJWc9IQAAAIxBm7xJqEFsmUwIV//+OEAAAQXHNIo1L/+/03WS+bhwM4u+HIE27j/zgAErr8j0iRqsQXxPeWif4rCWDMAqS4AGNGPjGFsJMDEUiEDUTazRvGL0Zx29zft7Jo5ZpKp8fWfwCA3Zqe1CDt64qCgJNQx/KY8GqzOUevx0Jzbw2qrPTUubgjjAUq84lWl/wAAAAGNBn9pFFSwj/wAAFjA4Sv4ABF9I2DzWdkmkiubTDpaBogoCr8gnYOVmiumW31oclpdAXtKuEDklFXC0M8V+vgLBbzP4ThsHjGfPR6FAdKm9EyqlK5HU3whY7MRD/FrTYt3UATcAAAA1AZ/5dEf/AAAio4Irq93WU8Ybn5vAMpq09er9j+bj5Y1ARXbrquK76upF+OQF5IK8U03A2YAAAAA7AZ/7akf/AAAirj5CgBGK67URZfm5hoeBniWv4Ps/IygIGzuoAJpaBaGUq6bvUlAFwguihACk8saghJkAAACBQZvgSahBbJlMCP/8hAAAD4n/TXMyYArag3NoCCGXoeQ2z8ezY0zmqvLh9vUfz6+qI1agXFj5wRqtOsoUkJ+oLetv2muxDg79FNvDbCtY/5/J+tuouIXxwXspCKFBxVWUSV5QJPHv6UJGHglG8PsNtjA3UHplE6+V6JtSTuPRFHMZAAAAWEGeHkUVLCP/AAAWGlIJVY6RGbibwWwtJan6yCyiRUPLW/5B5ij8kb1r32ioXOfJiKDr4bS7jm75KHqgBJaWwJJzRCm326Gn82Moolqvca8QGsCE0Q+WopIAAAA8AZ49dEf/AAAiwLj6SjafPz1ojprIIGeE703/7yVUFpOY5RY1ydwRFqt3t0aWbVG+LZfHFJVuxfI/AytmAAAARQGeP2pH/wAAInqKxoQAr01P+GWh2F77eddY/YmedDDONfcugBo6BcrttL9ZyB3ORp5/I+uZ0sS61LnnUEHNkvgaC+/2IwAABJ9tb292AAAAbG12aGQAAAAAAAAAAAAAAAAAAAPoAAAClAABAAABAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAADyXRyYWsAAABcdGtoZAAAAAMAAAAAAAAAAAAAAAEAAAAAAAAClAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAEAAAAACWAAAAZAAAAAAACRlZHRzAAAAHGVsc3QAAAAAAAAAAQAAApQAAAIAAAEAAAAAA0FtZGlhAAAAIG1kaGQAAAAAAAAAAAAAAAAAADIAAAAhAFXEAAAAAAAtaGRscgAAAAAAAAAAdmlkZQAAAAAAAAAAAAAAAFZpZGVvSGFuZGxlcgAAAALsbWluZgAAABR2bWhkAAAAAQAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAACrHN0YmwAAACYc3RzZAAAAAAAAAABAAAAiGF2YzEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAACWAGQAEgAAABIAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAY//8AAAAyYXZjQwFkAB//4QAZZ2QAH6zZQJgz5eEAAAMAAQAAAwBkDxgxlgEABmjr48siwAAAABhzdHRzAAAAAAAAAAEAAAAhAAABAAAAABRzdHNzAAAAAAAAAAEAAAABAAABGGN0dHMAAAAAAAAAIQAAAAEAAAIAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAIQAAAAEAAACYc3RzegAAAAAAAAAAAAAAIQAABH4AAADWAAAAVgAAADsAAAA+AAAA2wAAAFYAAABFAAAAQAAAAIYAAABQAAAAPwAAAD8AAABzAAAAQwAAACwAAAA4AAAArgAAAG4AAABMAAAATAAAAI0AAABhAAAAMwAAADgAAACQAAAAZwAAADkAAAA/AAAAhQAAAFwAAABAAAAASQAAABRzdGNvAAAAAAAAAAEAAAAwAAAAYnVkdGEAAABabWV0YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAtaWxzdAAAACWpdG9vAAAAHWRhdGEAAAABAAAAAExhdmY1Ny44My4xMDA=\" type=\"video/mp4\" />\n",
              "           </video>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8fTIEPMmm7VT",
        "colab_type": "text"
      },
      "source": [
        "# simple_rl"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5SGZUh_ee1Jm",
        "colab_type": "code",
        "outputId": "2fe25bf3-9ece-4e95-9dcb-07f4014bb3ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        }
      },
      "source": [
        "!pip install gym pygame > /dev/null 2>&1\n",
        "!apt-get install -y xvfb python-opengl ffmpeg python-pyglet > /dev/null 2>&1\n",
        "!pip install pyvirtualdisplay > /dev/null 2>&1\n",
        "!pip install simple_rl > /dev/null 2>&1"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling matplotlib-3.1.0:\n",
            "  Successfully uninstalled matplotlib-3.1.0\n",
            "Collecting matplotlib\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/da/83/d989ee20c78117c737ab40e0318ea221f1aed4e3f5a40b4f93541b369b93/matplotlib-3.1.0-cp36-cp36m-manylinux1_x86_64.whl (13.1MB)\n",
            "\u001b[K     |████████████████████████████████| 13.1MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.16.4)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.4.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.5.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib) (41.0.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib) (1.12.0)\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: matplotlib\n",
            "Successfully installed matplotlib-3.1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCzUdCCcf75f",
        "colab_type": "code",
        "outputId": "fb127718-021f-4372-f741-7b7dd9206935",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 511
        }
      },
      "source": [
        "from simple_rl.agents import QLearningAgent, RandomAgent, RMaxAgent\n",
        "from simple_rl.tasks import GridWorldMDP\n",
        "from simple_rl.run_experiments import run_agents_on_mdp\n",
        "\n",
        "# Setup MDP.\n",
        "mdp = GridWorldMDP(width=4, height=3, init_loc=(1, 1), goal_locs=[(4, 3)], lava_locs=[(4, 2)], gamma=0.95, walls=[(2, 2)], slip_prob=0.05)\n",
        "\n",
        "# Setup Agents.\n",
        "ql_agent = QLearningAgent(actions=mdp.get_actions())\n",
        "rmax_agent = RMaxAgent(actions=mdp.get_actions())\n",
        "rand_agent = RandomAgent(actions=mdp.get_actions())\n",
        "\n",
        "# Run experiment and make plot.\n",
        "run_agents_on_mdp([ql_agent, rmax_agent, rand_agent], mdp, instances=5, episodes=50, steps=10)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-ddb6a06cc11c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Agg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magents\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mQLearningAgent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRandomAgent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRMaxAgent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGridWorldMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_experiments\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrun_agents_on_mdp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;31m# Imports.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperiments\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmdp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplanning\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_experiments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/abstraction/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Classes.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAbstractionWrapperClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAbstractionWrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAbstractValueIterationClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAbstractValueIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_abs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStateAbstractionClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mStateAbstraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_abs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mProbStateAbstractionClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mProbStateAbstraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/abstraction/AbstractValueIterationClass.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# Other imports.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmake_mdp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_abs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mActionAbstractionClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mActionAbstraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabstraction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_abs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStateAbstractionClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mStateAbstraction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/utils/make_mdp.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m# Other imports.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mChainMDP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mGridWorldMDP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTaxiOOMDP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRandomMDP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFourRoomMDP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHanoiMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrid_world\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGridWorldMDPClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmake_grid_world_from_file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmdp\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMDPDistribution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/tasks/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombo_lock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mComboLockMDPClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mComboLockMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfour_room\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFourRoomMDPClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFourRoomMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGatherMDPClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGatherMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGatherStateClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGatherState\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrid_game\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGridGameMDPClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGridGameMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/tasks/gather/GatherMDPClass.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;31m# Other imports.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmdp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmarkov_game\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMarkovGameMDPClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mMarkovGameMDP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGatherStateClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGatherState\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msimple_rl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGatherStateClass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mGatherAgent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/simple_rl/tasks/gather/GatherStateClass.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'TkAgg'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m \u001b[0;31m# NOTE: for debugging\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/cbook/deprecation.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    305\u001b[0m                 f\"for the old name will be dropped %(removal)s.\")\n\u001b[1;32m    306\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m     \u001b[0;31m# wrapper() must keep the same documented signature as func(): if we\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36muse\u001b[0;34m(backend, warn, force)\u001b[0m\n\u001b[1;32m   1295\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mforce\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1296\u001b[0m             \u001b[0;32mfrom\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mswitch_backend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1297\u001b[0;31m             \u001b[0mswitch_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1298\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1299\u001b[0m         \u001b[0;31m# Finally if pyplot is not imported update both rcParams and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/matplotlib/pyplot.py\u001b[0m in \u001b[0;36mswitch_backend\u001b[0;34m(newbackend)\u001b[0m\n\u001b[1;32m    228\u001b[0m                 \u001b[0;34m\"Cannot load backend {!r} which requires the {!r} interactive \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m                 \"framework, as {!r} is currently running\".format(\n\u001b[0;32m--> 230\u001b[0;31m                     newbackend, required_framework, current_framework))\n\u001b[0m\u001b[1;32m    231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m     \u001b[0mrcParams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'backend'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrcParamsDefault\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'backend'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnewbackend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mImportError\u001b[0m: Cannot load backend 'TkAgg' which requires the 'tk' interactive framework, as 'headless' is currently running",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    }
  ]
}