A2C.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RL_DQN_new.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "ddE_fR4j_-jR",
        "vZq4VjDqVrms",
        "c61fq6FMVv0A",
        "QN2SUkuJn7W1",
        "Q9jNon_ptVrX",
        "nAibF3uqtY1c"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RBqYjRFyngvS"
      },
      "source": [
        "## Install libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXLEWcW9_mGv"
      },
      "source": [
        "%%capture\n",
        "!sudo apt update && sudo apt install xvfb\n",
        "!pip install gym-notebook-wrapper stable-baselines[mpi] box2d box2d-kengz pyvirtualdisplay pyglet\n",
        "!pip install wandb\n",
        "!pip install pyvirtualdisplay -qq\n",
        "!pip install folium==0.2.1\n",
        "!apt-get install -y python-opengl ffmpeg -qq"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5BmHLRuAXyL"
      },
      "source": [
        "%%capture\n",
        "!git clone https://github.com/openai/gym.git\n",
        "%cd gym\n",
        "!pip install -e .\n",
        "!pip install stable-baselines[mpi]\n",
        "!pip install stable-baselines3[extra]\n",
        "!pip install tensorflow==1.15"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ddE_fR4j_-jR"
      },
      "source": [
        "## Import useful libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RtC2hX1EcRXE"
      },
      "source": [
        "import math\n",
        "import random\n",
        "import numpy as np\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "from collections import namedtuple\n",
        "from itertools import count\n",
        "from PIL import Image\n",
        "import torch\n",
        "import gnwrapper\n",
        "import gym\n",
        "from gym import logger as gymlogger\n",
        "from gym.wrappers import Monitor\n",
        "gymlogger.set_level(30)\n",
        "import glob\n",
        "import io\n",
        "import os\n",
        "import cv2\n",
        "import base64\n",
        "from collections import deque\n",
        "from datetime import datetime\n",
        "from IPython.display import HTML\n",
        "from IPython import display as ipythondisplay\n",
        "from pyvirtualdisplay import Display\n",
        "import wandb\n",
        "import time\n",
        "\n",
        "is_ipython = 'inline' in matplotlib.get_backend()\n",
        "if is_ipython:\n",
        "    from IPython import display\n",
        "\n",
        "# if gpu is to be used\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vZq4VjDqVrms"
      },
      "source": [
        "## Get a modified vsersion of Car Racing from github official site "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nRplH8vdVpw0",
        "outputId": "ee9675b5-71be-4afc-a88b-8117e1b6d005"
      },
      "source": [
        "!git clone https://github.com/AGiannoutsos/car_racer_gym.git\n",
        "%cd car_racer_gym\n",
        "\n",
        "from car_racing import CarRacingDiscrete, CarRacing "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'car_racer_gym'...\n",
            "remote: Enumerating objects: 42, done.\u001b[K\n",
            "remote: Counting objects: 100% (42/42), done.\u001b[K\n",
            "remote: Compressing objects: 100% (42/42), done.\u001b[K\n",
            "remote: Total 42 (delta 17), reused 0 (delta 0), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (42/42), done.\n",
            "/content/car_racer_gym\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c61fq6FMVv0A"
      },
      "source": [
        "## Quick example with CarRacing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-3RPjFbySdb6",
        "outputId": "7a17b1ac-db01-4a71-8fa1-fbfc5f62f290"
      },
      "source": [
        "# from gym.envs.box2d import CarRacing\n",
        "env = gnwrapper.Animation(CarRacingDiscrete())\n",
        "env = CarRacingDiscrete()\n",
        "\n",
        "env.reset()\n",
        "env.render()\n",
        "im = env.render(\"state_pixels\")\n",
        "\n",
        "def state_image_preprocess(state_image):\n",
        "    # crop image\n",
        "    # state_image = state_image[0:84, :, :]\n",
        "    state_image = state_image.transpose((2,0,1))\n",
        "    # to torch\n",
        "    state_image = np.ascontiguousarray(state_image, dtype=np.float32) / 255\n",
        "    state_image = torch.from_numpy(state_image)\n",
        "    return state_image.unsqueeze(0).to(device)\n",
        "\n",
        "# plt.imshow(im)\n",
        "state_image_preprocess(im).shape\n",
        "plt.imshow(state_image_preprocess(im).cpu().squeeze(0).permute(1, 2, 0).numpy())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Track generation: 1063..1333 -> 270-tiles track\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fe20033ff10>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcLElEQVR4nO2de4yddZnHP885M9OZTlt6ofcZoZSqFBYW07iAtw3oLrhGiFGDIaZZMSSG3aKSKLB/mPWvNTEIf2xMGllDNgZ1KS6EGF2smCi7FlupKC2lpdVehtI29DJTOjNnznn2j/c9w5nTc3nPe788n+Rk5pz39ry/5/2+v+d3F1XFMIz8U0raAMMw4sHEbhgFwcRuGAXBxG4YBcHEbhgFwcRuGAUhkNhF5FYR2SciB0TkgbCMMgwjfMRvO7uIlIHXgI8BR4HfAZ9T1T3hmWcYRlj0BTj2/cABVT0IICI/BG4H2op9cPGgLlyzMMAljYtQkAmBqaQN8Ukf6EKFcojnLJOPAmoVqPV2yPjYOJNnJqXVtiBiXwscafh+FPib5p1E5B7gHoAFqxbwqf/8VIBLGhcxDeX/K1M6OPfpVlVEWvo8VegyZebDM7DY4wHdbkmAhcB8D/umCeUde+vB9lng7d5O89Tnn2q7LYjYPaGqW4GtAMs3Lre+uTGRtNAje9kMAfO67NMf/mWBuYIM+3yN5xXeEXyIBBH7MWC04fuI+5thRCN0AQZwBN9I2CLsdP00n68LQUo2vwM2iMg6ERkA7gSeCccsI0niGBwV6jWyFK4niO+cXVVnROSfgJ/jVIn8h6q+EpplRmJEkSs3h/VJFzN6Iq7IIWICldlV9afAT0OyxcggXsvmiYm7HkAEuXy3Y1u9DMJ4QfTj1E9UgZmA5yIfDRRGgqQ6h64LLuxKtWZand/LC6IbQzitFPM97OuByGvjjXjI6iQkPdtdz+UEJ6vqJKoo3kNhnbPTeZpfUiFd08SeA1Q1kqaaOOjJdsVpd57EyfWGIzMrWSIKlkzsacaL0wUURfJQg+SFqvsZ6LKflzJzu/J80PJ2p+MTrOwzsaeRMk45zUuNyhTRdSLJMh5flBcRhhjjLlp4xMSeRko4YaoX75Q97md4I8cBkj0mUSA4TSZ+2zr6mPvQhdF8ZPinud+6Fz+k0Gcm9igoAwsIL7yO+YHJyiCa2Gjut97rMSnBxN6OPvznzPUhlkEdHkNlTithm9ADkOLedib2VpRwcuZuNb7tqLcBByWGh8aEHTIpTs78i91PD6oy0VV8pezNn4uQPeo0TZnP/JJ/sQ9y8ZDIbgjBytudHg5psT3BypzMCx3CSbekfBbjiyT/Yi/j1IzH+Ux7mU2ll/2N6EnKZzH63gbCGEZByK3YszowpCdivMXMTWiRVrT+p8O9+kgGL2mXW7HHVhYN+/ns5XwxhoBxpGeRfNZxLIOPZPCSdrkVuyd6cVK7faOcl6wAGV2kmM/mkP8Kuk704vQkKtG8XlOzGwJ3DGeDkmafJUCxxd6Mlz7QUQ5/9HtKVbSWTbH3ugjCRWTUZ0lQ7DC+GS99oKMc/mj0jvnMM5kXe0/hq5/MT9uEmj1d1meum9HMuhuRFznMZy3JvNh7qsH184aWNjWnPV22h50bH5ac5CjNRF7rnkefhXDezIs9d+RU4LkmbJ81Rwf9OAOzhoJdyyroDCNtNAu63/1Mu5+qv9Nazt4LKS6PFY4ZnFlmp+nslzz4LKS5703svRD1YgN5eDDjYgpnSeMJOjffmc9mMbE3E5fz/KwiYsyl5n7y5rOI7qeYYu+UmCY4I2lskYgANDeNmKCNApJ/sU/i1F724yy8kCWh56SbZqFIsc9yG8bP9tKaAS7QvdY2LaSgU01Sg2qyOpgnKz7LrdgzO7ea1/XJGg8RoVQqhXbPSaVdkXwWugk2nj2HtPBpqVQKVexGyKTELfkvsydBQuU2Ecme4CXiMe1eSXFZOyxM7H7o9mB4Cev8PFhdpi8WEcrlso8TJ4eWlKrf/p89XYhU+ixOTOx+SGp8dKdpy7KWo9eJqyk0hT6LGyuzR0GCUanfGu2wa8IzV7OeMXP90FXsIjIqIs+LyB4ReUVE7nN/Xyoiz4nIfvfvkujNzQgJvs395vCNx4Uh1MxFGhkz1w9ecvYZ4H5V3QjcANwrIhuBB4DtqroB2O5+N3JA5oRqeKKr2FX1DVX9vfv/OLAXWAvcDjzu7vY4cEdURvohc2FknYyaHQbmswAmhN2pRkQuB64HdgArVfUNd9NxYGWbY+4RkZ0isnPy9GQvlwtEZnOnjMxB3g0/wjWfBTAhzE41IrIA2AZ8WVXPNW5Tx7Mtb1NVt6rqJlXdNLhk0OvlDAitHJlEjplZ4QYlxbftSewi0o8j9B+o6lPuz2+KyGp3+2rgRDQmpgwvuglDW9r0NwBxCi+VoXgGfRYFXmrjBXgM2KuqDzdsegbY7P6/GXg6fPMSoJujvOgmDG1J098eSUp0qczRM+KzWSJynZdONR8APg/8UUR2u789BPwb8GMRuRv4C/DZaEyMmbie1VY9skLsshlEdKqaTtEmTcQ+myWpyStU9TcdLn9LuOaknDAdm+JpqXIl9IL4zAvWXbYX/PSfjmmARSrLyh6IfBBMin3WlnbXD2iTiT1MEnrzq2pqK4W60rYdJyaylFuXgWGcmZfqMzD1gIk9bfjIVRTtbbmiFJHViGQOYUcC7c5VwhF7Dadfa49it4EwQYnjWc2BHiKjhjOHfC85XVZ9FnCEoIk9KHHUxAa4Ri5yzk7MAOfcz4zHY1Lus6gwsTeibf5vt0/Qa8SAiEQu+MRfKErnVWEy5rOWhFC3YWX2RqTN/+32CXqNmIiyKS0TbfIZ9NlF2FpvPVB/M8a5XFABSL3QjVmKI/YKTrluAt9L3rYk7BeHvYiip6A+y63YLypH1nBqbC/QuXzXK0l2lYzxIYujXB5b2T8FPuvYmchHMtgiEXknxluMIz2L5LOO/SJ8JIMtEmEYxiwm9ijoFlE1b0+6y6iRnM9i9LuJvZkwEr9TRNWqa2Vcc6e3IPE28rSQlM9i9Lu1szcTdeKnrFhaiHJyUMJOoqBNwD6PNbGngXrO0ZyDJD3U0qVdx5lMdKiJiiA+U+BtnD79fq/ttWtwAyb2NNBuOqMedJRUl9hCFgOa55prVZ7vdnwFmA7TqO6Y2OOm8c3vNefusjhg5sezp51mn4HTZ8NvzgyO2GPGxJ4kXnNuD2FhZnPYrJldD92ncELxDGFi94vf8nQKy+SFobGcXSFY7uqjzJw0JnYvtGt6ad6n3e9BRtB1Oj5ruWKcdPPZFM44iSDnzxgmdi8EmXc8yuGVFhXMpYL3NJkhk4INgom9V5IIvS3c704NJ6f2mk5hDobKCMUWey8iqu8bpui8Xt+E7o0CCrgXit1dthcR1St2wgz9Wl2/1fkLFm4a0VBssfdKGDm7n7XkLGc3QsDEHjdec3PDCBkTezNJCM9ybiMGTOyQjMAtNzdixsQO3qaQjvKahhEDJvYgxJU7WxRghICJPQi9tNF7pdW+FgUYIVDMTjX1mULimg6q25RHjX+b/++G5fqGRzIv9p5nS6l3qywB84F50djVSNcllS/gf2z0FJRmSkg5m9m/lpRaCru+5XEWnsyLvWeH1MciC+EJvUvu2nXt9GkcwfthGqQmlErJlMgaRdGLQOr7akmpSfrEnjehQw7EnjiKI9YK/kPqBGYtCYtGUfQikDyKKe14FruIlIGdwDFV/YSIrAN+CCwDdgGfV9WYZ9VKCdPAeNJG9EYcYWoeQ+Es00vsdx+wt+H7t4DvqOqVwGng7jANi436TJ2TPj9TZHLWElvOqXh4EruIjAD/AHzP/S7AzcCT7i6PA3dEYWAsTAFngTM+P5PxmxwnmZ3fzpiD1zD+EeBrwEL3+zLgjKrW87SjwNpWB4rIPcA9AAtWLfBvaRRUmz7GHOphuOXQ+aBrzi4inwBOqOouPxdQ1a2quklVNw0uGfRzimioT9R/Bv814SnEby7c6rioRG6RQjJ4ydk/AHxSRD4ODAKLgEeBxSLS5+buI8Cx6MyMiBzm6H4FGmfubZFCMnTN2VX1QVUdUdXLgTuBX6rqXcDzwKfd3TYDT0dmpWEYgQnSE+PrwFdF5ABOGf6xcEwyoiTMEFpVLSTPED11qlHVXwG/cv8/CLw/fJOMKAkzhLZwPFvYqDfDKAjWXTZn+O21FmdvN1WFIdCFCouBciyXLTwm9pwRZm18VC8AEaG2okb1+qozGClFLbJ5xsSeE6KqKIusAq4PWAAMRHN642JM7Dkgi+uza9YMzgFWQZcDOgmnXc6cdJNZ0tcvIib2nNOuzG3NZsXDxG4YBSHzYo8jHAx6jayHrH7sT/qei+6zVmRe7FmYhCHrIbMf+5O+56L7rBWZF7sxl6BDXPOYoxkOJvac0WuOVBd3/Ti/ve+M9GNiLzjtes4FPYeRPkzsBSBt4rVIIBlM7DmkWUxRiDeIYC0SSAYTew7JQguFET8mdsMoCCb2AmBlZANM7IXAQm4DbIhrPlDLvY3umNhzgKqitYyJPWPm5gEL47NOVkWTVbszjIndMAqCid0wCoKJ3TAKgondMAqCid0wCoKJ3TAKgondMAqCid0wCoKJ3TAKgondMApCbsVepIEhYa+bllTaFcpnId+rl/PldiBMYYZ1CpSkBCHebixpJxe/pArjM8K/Vy/ny63Yi0SpVELK2RKKlpQq1aTNKBQm9pyQtVzRlmyOH09ldhFZLCJPisirIrJXRG4UkaUi8pyI7Hf/LonaWMMw/OO1gu5R4Geq+l7gOmAv8ACwXVU3ANvd74ZhpJSuYheRS4APA48BqOq0qp4Bbgced3d7HLgjKiM7Yau4Gq0wn12Ml5x9HXAS+L6IvCQi3xORYWClqr7h7nMcWNnqYBG5R0R2isjOydOT4Vg99/yhnzPsa6SxPB33wxz39fLos6B4EXsf8D7gu6p6PXCeppBdHU+29KaqblXVTaq6aXDJYFB7jZCI+2HOo3iyhhexHwWOquoO9/uTOOJ/U0RWA7h/T0RjomEYYdBV7Kp6HDgiIu9xf7oF2AM8A2x2f9sMPB2JhYZRdMrAPGDQw6dDAOW1nf2fgR+IyABwEPhHnBfFj0XkbuAvwGd93IaRQlTVwu40MQAs8rhvuf0mT2JX1d3AphabbvFoghEzQQRrQg8Zwcka/SZruYfjQ8jZjYxQF3mvgu3l5WA5f4/0AwvxL/aQhquZ2HNGKxF6EWcv4i2k0IPcchlH8AmPMTWxF4AkxJmr3L8PmI9/sZYJdVSiX3Ir9lw9bB5IY4+vXm1Krc/KwBDtK7+UxMWsKNLFiNyKPZUPTUTUtIbUsnW/rV4EkfmshNN05Tdn7qOzmLuZHcPLoJvQIcdiLwwKWsvgKq61GK9VBhaQ3NOekvewid0oDp1El4JQPGpyOwedYfSEl1DcD21HjcSP5eyGgyoD09P0Vyodd6v09zM9MAAFqhMB/Of6KUomE7sBgKhyxcGDrH/99Y77HbjySl5797vRvIm9AGG8id2YZfjcOZaPjVFibtOwAlWcOrU3VqxIyrxoybnQwcRuuNSA/wV+BbwX+ChOaxXAFPALYB/OmIwBCqGNbOEhMsltBV0aO5mkGQV2A0/giH66Yds08Bt32x+Irr4psz5Lgdkq3Y3IrdiL1KkmL0TmsxowCbwNzERw/kazExK+l041uRW7YcxSBcbdTxRibyTFeYyV2bNCvdasGSW0V/bw8DDLly9n4eQkpYkJcMPqkgiLFixg+eAgw8PD4VwsCTq1eYdRG18/R0pr9k3sWWEIZ+RVM1M4wycDIiKsX7+ekZERrjp8mL5du8Btc+/r6+Pqa66hb3SUefPm5bOIFMYtSdNfv0T0sjCxx0mQHLhM6wEZ1YDndRERhoaGGBoaYv6pU3M6zYgIw/Pns3jx4uAXyjKtRBiFMCN6l5rY42IerXNmLwjmqTTQSoRJBDk+XzD2CMVFH11n/0yUhmYvUUWamsFEdc4+uekuG1du3Qvdrt9qmwebTexeCSrWEMrVUbP22DFWHT/O0rfeolx9ZznlcrXKukOHWHzmDMdXreLY2rUJWhkyacmtg+LBZhO7V/pxxkRH/SAklKuIKmuPHePal1++KFevi/3yP/+ZP1x3HWNr1uSvb3wQwvaZldljZoC5FV/1PqJJhHwxvgBE9aJLzX7Pag+3ZnLmM6+Y2FtRAoZ5p3N4J+KoiU3ZQ5N5CuozE3s7SrRu0gr6xk7hG7/OW8AhnNLKpbxz+1XgFM6Knm+Riq7g8ZIGn9UTPYAdJvZeCer0pB+aNlSB/8FZxfNGnPW96i2Fk8CPgR3AKHA5qb2NaEjDzbbqf9+jXdY33pjlJPAqMIYj/jpV4Ji77SQFzNnTRqu6Iw+Y2IMS9pNvSoqegvost2KPbWx0kl0ltf4n+nuNI5I1n83dpxe8PAO5LbPncrBGM1L/E/xeBbge+CvgPTgtjXUGgA8CK+m+XkIgG8xnc/bp7bS2SIThkRJwE47Ymxsi5gG3An8HvAz8nsxErvnE+saniG7OaN4eQrNKGIxfcglvdukKO7FoUUzWxExSPvMj3GY7PB5vYo+CXlceSUH0qiIcvOIKjoyOdtyv0t+fz66ySfksxqZcE3vcpFUnIlQGBqgMDHTft2ik1Wc9ktva+EyhTX+bfzfSRwZ9ZmJPA+2mM8pJjpJLMugzT2IXka+IyCsi8icReUJEBkVknYjsEJEDIvIjEbH4zwva5v9ux6Q4x8g9SfsspPN0FbuIrAW2AJtU9Rqc2dDuBL4FfEdVrwROA3eHY1KB8JoL+OweaURAEj4L6Txew/g+YEhE+nDGR7wB3Aw86W5/HLgjHJMygt+3bQoWFCgsijMb73mcZW68pn9OfNZV7Kp6DPg2cBhH5GeBXcAZVa1PuX8UyNFcRU20cnCrSSxa7dfp4fDyxu50fIYfvERQnFVhzuGIvv5bq/3akWGfeQnjlwC3A+uANTjTOtzq9QIico+I7BSRnZOnJ30bmiheHNwubIuyHdVCe380Cq5APvMSxn8UOKSqJ1W1AjwFfABY7Ib1ACM4oyAvQlW3quomVd00uGQwFKMTJYk3s+Xg2SOFPvMi9sPADSIyX5yRCrcAe4DngU+7+2wGno7GxISo8c6i5I0hephvZj9lRiNZwvJZAi8DL2X2HTgVcb8H/ugesxX4OvBVETkALAMei9DOeKkBEzi1E/WynRB+E1i7+b+9/GYkQ1g+Czq1mQ88dZdV1W8A32j6+SDwfn+XzQAV99PYeyCMHNbPAgCWsydL2nzm89zWgy5uvOYMRnrIic9M7GnAcu7skUGfxTrqbercFK8/93qclwzOEM6yT2mlglObcjhpQ3rkPE59SFLjLgdxfJszps5Mtd0msc37BZTKJe0fzsCiZ41k4Q1eYe50sFmg7H6SSt8s+NUHlYkKtWqt5d3FKnYRyWBJxzCyhaq2FLuV2Q2jIJjYDaMgmNgNoyCY2A2jIJjYDaMgmNgNoyAkOpX04sWLWbp0KTMzM0xOTlKpVBgfH6dWq3HVVVcxMjIyu+/p06d5+eWXUVWuvvpqli1bhoggIoyNjbFnzx6qVaexub+/n6uvvpqVK1fy+uuvc+DAgaRu0TBSQ6Ji37BhAzfeeCMTExMcPXqU8fFxXnnlFSqVCps3b+auu+6a3XfHjh1s2bKFSqXCli1b+NCHPkR/fz/lcplt27bx0EMPMTExAcDw8DD33nsvt912G4888ggPP/wwtVotqds0jFSQaBi/aNEiRkdHufTSSxGR2ZwZYGZmhunpaaamprhw4QJTU1OoKqrK9PQ0k5OTzMzMzObujajq7HGVSiXu2zKMVJJYDzoR4Utf+hL3338/L7zwAt/85jc5ceIE58+fR1XZuHEjo6Ojs8I+e/Ys+/fvR1VZv349S5Ys4bLLLmPNmjXs2bOH7du3Mz09DUBfXx/r169n2bJlHD16lMOHs9Zx3DD8064HXaJh/CWXXMK73vUu9uzZw/j4OOfOnZvddvDgQcbGxpicnOTtt9+ec9zevXsREd566y1Onz7N2NgY1WoVEWHAXb5o//797Nu3L9b7MYw0k9q13ryu1V0qlWb3HRkZYfPmzSxcuJAnnniC3bt3R2miYWSK1Iq9TifRN24TES699FI+85nPsGrVKn7729+a2A2jgcTErqocPnyYX//61xw6dIg1a9Ywb9483nzzTarVKjfddBPXXnstlUqF6elpxsbGeP7556lWq3zkIx9hdHSU1atXs3z5cl588UVeffVV4J2c3mtkYBhFIdGcfd++fWzbto3p6Wk2bNjAihUrmJiY4MKFC9x+++184QtfAJwXwwsvvMDu3bupVCp88Ytf5Oabb6a/v5++vj7mz5/PT37yk1mRN4b2hmE4JCr2s2fPcuTIEarV6mxF3MzMDLVajcOHD7Nr167ZfV977TWmp6eZmZnhwIEDLFmyhHK5TKlU4tChQ1SrVSYmJnjppZdYunQpp06dSvDODCN9JDp5xcDAAPPmzQOgVqtRq9Vm29OXLVvGokWLZve9cOECJ0+eRFVZvnw5Q0ND9XMyPj7OqVOn6O/vZ8WKFZTLZU6dOsX58+djuzfDSAvtmt5sphrDyBk2U41hFBwTu2EUBBO7YRQEE7thFAQTu2EUBBO7YRQEE7thFIS4e9CdwlnlK2vd2y4lezZDNu02m4NxWbsNsXaqARCRnaq6KdaLBiSLNkM27Tabo8PCeMMoCCZ2wygISYh9awLXDEoWbYZs2m02R0TsZXbDMJLBwnjDKAgmdsMoCLGJXURuFZF9InJARB6I67q9IiKjIvK8iOwRkVdE5D7396Ui8pyI7Hf/Lkna1mZEpCwiL4nIs+73dSKyw03zH4nIQNI2NiIii0XkSRF5VUT2isiNGUnnr7jPxp9E5AkRGUx7WkNMYheRMvDvwG3ARuBzIrIxjmv7YAa4X1U3AjcA97q2PgBsV9UNwHb3e9q4D9jb8P1bwHdU9UrgNHB3Ila151HgZ6r6XuA6HNtTnc4ishbYAmxS1WuAMnAn6U9rZpdUivID3Aj8vOH7g8CDcVw7BNufBj4G7ANWu7+tBvYlbVuTnSM44rgZeBYQnF5dfa18kPQHuAQ4hFtJ3PB72tN5LXAEWIrTA/VZ4O/TnNb1T1xhfD2B6hx1f0s1InI5cD2wA1ipqm+4m44DKxMyqx2PAF8D6itYLgPOqOqM+z1tab4OOAl83y16fE9Ehkl5OqvqMeDbwGHgDeAssIt0pzVgFXRtEZEFwDbgy6p6rnGbOq/v1LRZisgngBOquqvrzumhD3gf8F1VvR5nzMSckD1t6Qzg1iHcjvOyWgMMA7cmapRH4hL7MWC04fuI+1sqEZF+HKH/QFWfcn9+U0RWu9tXAyeSsq8FHwA+KSJ/Bn6IE8o/CiwWkfpgp7Sl+VHgqKrucL8/iSP+NKczwEeBQ6p6UlUrwFM46Z/mtAbiE/vvgA1ujeUAToXGMzFduyfEWV3iMWCvqj7csOkZYLP7/2acsnwqUNUHVXVEVS/HSdtfqupdwPPAp93d0mbzceCIiLzH/ekWYA8pTmeXw8ANIjLffVbqdqc2rWeJsWLj48BrwOvAvyRdWdHBzg/ihI4vA7vdz8dxysDbgf3AL4ClSdvaxv6/BZ51/78CeBE4APwXMC9p+5ps/Wtgp5vW/w0syUI6A/8KvAr8CfhPYF7a01pVrbusYRQFq6AzjIJgYjeMgmBiN4yCYGI3jIJgYjeMgmBiN4yCYGI3jILw//pOmHoXvb8OAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QN2SUkuJn7W1"
      },
      "source": [
        "## Custom Callback Class"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q9jNon_ptVrX"
      },
      "source": [
        "### DQN callback"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S5ayf9GSMSUs"
      },
      "source": [
        "from stable_baselines3.common.callbacks import BaseCallback\n",
        "\n",
        "class DQNCustomCallback(BaseCallback):\n",
        "    \"\"\"\n",
        "    A custom callback that derives from ``BaseCallback``.\n",
        "\n",
        "    :param verbose: (int) Verbosity level 0: not output 1: info 2: debug\n",
        "    \"\"\"\n",
        "    def __init__(self, verbose=0):\n",
        "        super(DQNCustomCallback, self).__init__(verbose)\n",
        "        # Those variables will be accessible in the callback\n",
        "        # (they are defined in the base class)\n",
        "        # The RL model\n",
        "        # self.model = None  # type: BaseAlgorithm\n",
        "        # An alias for self.model.get_env(), the environment used for training\n",
        "        # self.training_env = None  # type: Union[gym.Env, VecEnv, None]\n",
        "        # Number of time the callback was called\n",
        "        # self.n_calls = 0  # type: int\n",
        "        # self.num_timesteps = 0  # type: int\n",
        "        # local and global variables\n",
        "        # self.locals = None  # type: Dict[str, Any]\n",
        "        # self.globals = None  # type: Dict[str, Any]\n",
        "        # The logger object, used to report things in the terminal\n",
        "        # self.logger = None  # stable_baselines3.common.logger\n",
        "        # # Sometimes, for event callback, it is useful\n",
        "        # # to have access to the parent object\n",
        "        # self.parent = None  # type: Optional[BaseCallback]\n",
        "        self.episodes = 0\n",
        "        self.total_episode_reward = 0\n",
        "\n",
        "    def _on_training_start(self) -> None:\n",
        "        \"\"\"\n",
        "        This method is called before the first rollout starts.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_rollout_start(self) -> None:\n",
        "        \"\"\"\n",
        "        A rollout is the collection of environment interaction\n",
        "        using the current policy.\n",
        "        This event is triggered before collecting new samples.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_step(self) -> bool:\n",
        "        # update commulative reward to log at the end of every episode\n",
        "        self.total_episode_reward += self.locals[\"reward\"]\n",
        "        # at the end of every episode\n",
        "        if self.locals[\"done\"][0]:\n",
        "\n",
        "            # log the reward value if its time to not log 2 times\n",
        "            if self.episodes % self.locals[\"log_interval\"] != 0: \n",
        "                wandb.log({\"reward_per_episode\": self.total_episode_reward})\n",
        "\n",
        "            # if log interval has passed\n",
        "            if self.episodes % self.locals[\"log_interval\"] == 0:\n",
        "                # log at wandb and print the last video\n",
        "                # Save your model and optimizer\n",
        "                self.model.save(MODEL_SAVE_NAME)\n",
        "                # Save as artifact for version control.\n",
        "                artifact = wandb.Artifact(MODEL_SAVE_NAME, type='model')\n",
        "                artifact.add_file(MODEL_SAVE_NAME+\".zip\")\n",
        "                wandb.log_artifact(artifact)\n",
        "                wandb.log({\"reward_per_episode\": self.total_episode_reward})\n",
        "\n",
        "\n",
        "                mp4list = glob.glob('video/*.mp4')\n",
        "                print(mp4list)\n",
        "                if len(mp4list) > 0:\n",
        "                    print(len(mp4list))\n",
        "                    mp4 = mp4list[-1]\n",
        "                    video = io.open(mp4, 'r+b').read()\n",
        "                    encoded = base64.b64encode(video)\n",
        "\n",
        "                    # log gameplay video in wandb\n",
        "                    wandb.log({\"gameplays\": wandb.Video(mp4, fps=4, format=\"gif\")})\n",
        "\n",
        "                    # display gameplay video\n",
        "                    ipythondisplay.clear_output(wait=True)\n",
        "                    ipythondisplay.display(HTML(data='''<video alt=\"\" autoplay \n",
        "                                loop controls style=\"height: 400px;\">\n",
        "                                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "                            </video>'''.format(encoded.decode('ascii'))))\n",
        "                    print(\"Episode \", self.episodes)\n",
        "            self.episodes += 1\n",
        "            self.total_episode_reward = 0\n",
        "\n",
        "        \n",
        "        return True\n",
        "\n",
        "    def _on_rollout_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before updating the policy.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_training_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before exiting the `learn()` method.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nAibF3uqtY1c"
      },
      "source": [
        "### PPO callback"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-mL6SAtttdwK"
      },
      "source": [
        "class PPOCustomCallback(BaseCallback):\n",
        "    \"\"\"\n",
        "    A custom callback that derives from ``BaseCallback``.\n",
        "\n",
        "    :param verbose: (int) Verbosity level 0: not output 1: info 2: debug\n",
        "    \"\"\"\n",
        "    def __init__(self, verbose=0):\n",
        "        super(PPOCustomCallback, self).__init__(verbose)\n",
        "        # Those variables will be accessible in the callback\n",
        "        # (they are defined in the base class)\n",
        "        # The RL model\n",
        "        # self.model = None  # type: BaseAlgorithm\n",
        "        # An alias for self.model.get_env(), the environment used for training\n",
        "        # self.training_env = None  # type: Union[gym.Env, VecEnv, None]\n",
        "        # Number of time the callback was called\n",
        "        # self.n_calls = 0  # type: int\n",
        "        # self.num_timesteps = 0  # type: int\n",
        "        # local and global variables\n",
        "        # self.locals = None  # type: Dict[str, Any]\n",
        "        # self.globals = None  # type: Dict[str, Any]\n",
        "        # The logger object, used to report things in the terminal\n",
        "        # self.logger = None  # stable_baselines3.common.logger\n",
        "        # # Sometimes, for event callback, it is useful\n",
        "        # # to have access to the parent object\n",
        "        # self.parent = None  # type: Optional[BaseCallback]\n",
        "        self.episodes = 0\n",
        "        self.total_episode_reward = 0\n",
        "\n",
        "    def _on_training_start(self) -> None:\n",
        "        \"\"\"\n",
        "        This method is called before the first rollout starts.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_rollout_start(self) -> None:\n",
        "        \"\"\"\n",
        "        A rollout is the collection of environment interaction\n",
        "        using the current policy.\n",
        "        This event is triggered before collecting new samples.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_step(self) -> bool:\n",
        "        # update commulative reward to log at the end of every episode\n",
        "        self.total_episode_reward += np.mean(self.locals[\"rewards\"])\n",
        "        # at the end of every episode\n",
        "        # print(self.locals[\"dones\"])\n",
        "        if self.locals[\"dones\"].all():\n",
        "\n",
        "            # log the reward value if its time to not log 2 times\n",
        "            if self.episodes % self.locals[\"log_interval\"] != 0: \n",
        "                wandb.log({\"reward_per_episode\": self.total_episode_reward})\n",
        "\n",
        "            # if log interval has passed\n",
        "            if self.episodes % self.locals[\"log_interval\"] == 0:\n",
        "                # log at wandb and print the last video\n",
        "                # Save your model and optimizer\n",
        "                self.model.save(MODEL_SAVE_NAME)\n",
        "                # Save as artifact for version control.\n",
        "                artifact = wandb.Artifact(MODEL_SAVE_NAME, type='model')\n",
        "                artifact.add_file(MODEL_SAVE_NAME+\".zip\")\n",
        "                wandb.log_artifact(artifact)\n",
        "                wandb.log({\"reward_per_episode\": self.total_episode_reward})\n",
        "\n",
        "\n",
        "                mp4list = glob.glob('video/*.mp4')\n",
        "                print(mp4list)\n",
        "                if len(mp4list) > 0:\n",
        "                    print(len(mp4list))\n",
        "                    mp4 = mp4list[-1]\n",
        "                    video = io.open(mp4, 'r+b').read()\n",
        "                    encoded = base64.b64encode(video)\n",
        "\n",
        "                    # log gameplay video in wandb\n",
        "                    wandb.log({\"gameplays\": wandb.Video(mp4, fps=4, format=\"gif\")})\n",
        "\n",
        "                    # display gameplay video\n",
        "                    ipythondisplay.clear_output(wait=True)\n",
        "                    ipythondisplay.display(HTML(data='''<video alt=\"\" autoplay \n",
        "                                loop controls style=\"height: 400px;\">\n",
        "                                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "                            </video>'''.format(encoded.decode('ascii'))))\n",
        "                    print(\"Episode \", self.episodes)\n",
        "            self.episodes += 1\n",
        "            self.total_episode_reward = 0\n",
        "\n",
        "        \n",
        "        return True\n",
        "\n",
        "    def _on_rollout_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before updating the policy.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_training_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before exiting the `learn()` method.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "class PPO2CustomCallbackNenvs(BaseCallback):\n",
        "    \"\"\"\n",
        "    A custom callback that derives from ``BaseCallback``.\n",
        "\n",
        "    :param verbose: (int) Verbosity level 0: not output 1: info 2: debug\n",
        "    \"\"\"\n",
        "    def __init__(self, verbose=0):\n",
        "        super(PPO2CustomCallbackNenvs, self).__init__(verbose)\n",
        "        # Those variables will be accessible in the callback\n",
        "        # (they are defined in the base class)\n",
        "        # The RL model\n",
        "        # self.model = None  # type: BaseAlgorithm\n",
        "        # An alias for self.model.get_env(), the environment used for training\n",
        "        # self.training_env = None  # type: Union[gym.Env, VecEnv, None]\n",
        "        # Number of time the callback was called\n",
        "        # self.n_calls = 0  # type: int\n",
        "        # self.num_timesteps = 0  # type: int\n",
        "        # local and global variables\n",
        "        # self.locals = None  # type: Dict[str, Any]\n",
        "        # self.globals = None  # type: Dict[str, Any]\n",
        "        # The logger object, used to report things in the terminal\n",
        "        # self.logger = None  # stable_baselines3.common.logger\n",
        "        # # Sometimes, for event callback, it is useful\n",
        "        # # to have access to the parent object\n",
        "        # self.parent = None  # type: Optional[BaseCallback]\n",
        "        self.episodes = 0\n",
        "        self.total_episode_reward = np.array([0,0], dtype=np.float32)\n",
        "\n",
        "    def _on_training_start(self) -> None:\n",
        "        \"\"\"\n",
        "        This method is called before the first rollout starts.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_rollout_start(self) -> None:\n",
        "        \"\"\"\n",
        "        A rollout is the collection of environment interaction\n",
        "        using the current policy.\n",
        "        This event is triggered before collecting new samples.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_step(self) -> bool:\n",
        "        # update commulative reward to log at the end of every episode\n",
        "        self.total_episode_reward += self.locals[\"rewards\"]\n",
        "        # at the end of every episode\n",
        "        # print(self.locals[\"dones\"])\n",
        "        if np.array(self.locals[\"mb_dones\"][-1]).any():\n",
        "            print(np.array(self.locals[\"mb_dones\"][-1]))\n",
        "        \n",
        "        if np.array(self.locals[\"mb_dones\"][-1]).any():\n",
        "\n",
        "            # log the reward value if its time to not log 2 times\n",
        "            if self.episodes % self.locals[\"log_interval\"] != 0: \n",
        "                wandb.log({\"reward_per_episode\": np.mean(self.total_episode_reward[np.array(self.locals[\"mb_dones\"][-1])==True]) })\n",
        "\n",
        "            # if log interval has passed\n",
        "            if self.episodes % self.locals[\"log_interval\"] == 0:\n",
        "                # log at wandb and print the last video\n",
        "                # Save your model and optimizer\n",
        "                self.model.save(MODEL_SAVE_NAME)\n",
        "                # Save as artifact for version control.\n",
        "                artifact = wandb.Artifact(MODEL_SAVE_NAME, type='model')\n",
        "                artifact.add_file(MODEL_SAVE_NAME+\".zip\")\n",
        "                wandb.log_artifact(artifact)\n",
        "                wandb.log({\"reward_per_episode\": np.mean(self.total_episode_reward[np.array(self.locals[\"mb_dones\"][-1])==True]) })\n",
        "\n",
        "\n",
        "                mp4list = glob.glob('video/*.mp4')\n",
        "                print(mp4list)\n",
        "                if len(mp4list) > 0:\n",
        "                    print(len(mp4list))\n",
        "                    mp4 = mp4list[-1]\n",
        "                    video = io.open(mp4, 'r+b').read()\n",
        "                    encoded = base64.b64encode(video)\n",
        "\n",
        "                    # log gameplay video in wandb\n",
        "                    wandb.log({\"gameplays\": wandb.Video(mp4, fps=4, format=\"gif\")})\n",
        "\n",
        "                    # display gameplay video\n",
        "                    ipythondisplay.clear_output(wait=True)\n",
        "                    ipythondisplay.display(HTML(data='''<video alt=\"\" autoplay \n",
        "                                loop controls style=\"height: 400px;\">\n",
        "                                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "                            </video>'''.format(encoded.decode('ascii'))))\n",
        "                    print(\"Episode \", self.episodes)\n",
        "            self.episodes += 1\n",
        "            self.total_episode_reward[np.array(self.locals[\"mb_dones\"][-1])==True] = 0\n",
        "\n",
        "        \n",
        "        return True\n",
        "\n",
        "    def _on_rollout_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before updating the policy.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_training_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before exiting the `learn()` method.\n",
        "        \"\"\"\n",
        "        pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "plxIQTXnJ5MH"
      },
      "source": [
        "### A2C callack"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KrwF9azGJ93u",
        "outputId": "a62b6a8d-a0a7-4c60-a208-250b9f29420c"
      },
      "source": [
        "from stable_baselines.common.callbacks import BaseCallback\n",
        "\n",
        "class A2CCustomCallbackNenvs(BaseCallback):\n",
        "    \"\"\"\n",
        "    A custom callback that derives from ``BaseCallback``.\n",
        "\n",
        "    :param verbose: (int) Verbosity level 0: not output 1: info 2: debug\n",
        "    \"\"\"\n",
        "    def __init__(self, verbose=0):\n",
        "        super(A2CCustomCallbackNenvs, self).__init__(verbose)\n",
        "        # Those variables will be accessible in the callback\n",
        "        # (they are defined in the base class)\n",
        "        # The RL model\n",
        "        # self.model = None  # type: BaseAlgorithm\n",
        "        # An alias for self.model.get_env(), the environment used for training\n",
        "        # self.training_env = None  # type: Union[gym.Env, VecEnv, None]\n",
        "        # Number of time the callback was called\n",
        "        # self.n_calls = 0  # type: int\n",
        "        # self.num_timesteps = 0  # type: int\n",
        "        # local and global variables\n",
        "        # self.locals = None  # type: Dict[str, Any]\n",
        "        # self.globals = None  # type: Dict[str, Any]\n",
        "        # The logger object, used to report things in the terminal\n",
        "        # self.logger = None  # stable_baselines3.common.logger\n",
        "        # # Sometimes, for event callback, it is useful\n",
        "        # # to have access to the parent object\n",
        "        # self.parent = None  # type: Optional[BaseCallback]\n",
        "        self.episodes = 0\n",
        "        self.total_episode_reward = np.array([0,0], dtype=np.float32)\n",
        "\n",
        "    def _on_training_start(self) -> None:\n",
        "        \"\"\"\n",
        "        This method is called before the first rollout starts.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_rollout_start(self) -> None:\n",
        "        \"\"\"\n",
        "        A rollout is the collection of environment interaction\n",
        "        using the current policy.\n",
        "        This event is triggered before collecting new samples.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_step(self) -> bool:\n",
        "        # update commulative reward to log at the end of every episode\n",
        "        self.total_episode_reward += self.locals[\"rewards\"]\n",
        "        # at the end of every episode\n",
        "        # print(self.locals[\"dones\"])\n",
        "        if self.locals[\"dones\"].any():\n",
        "            print(self.locals[\"dones\"])\n",
        "        \n",
        "        if self.locals[\"dones\"].any():\n",
        "\n",
        "            # log the reward value if its time to not log 2 times\n",
        "            if self.episodes % self.locals[\"log_interval\"] != 0: \n",
        "                wandb.log({\"reward_per_episode\": np.mean(self.total_episode_reward[self.locals[\"dones\"]==True]) })\n",
        "\n",
        "            # if log interval has passed\n",
        "            if self.episodes % self.locals[\"log_interval\"] == 0:\n",
        "                # log at wandb and print the last video\n",
        "                # Save your model and optimizer\n",
        "                self.model.save(MODEL_SAVE_NAME)\n",
        "                # Save as artifact for version control.\n",
        "                artifact = wandb.Artifact(MODEL_SAVE_NAME, type='model')\n",
        "                artifact.add_file(MODEL_SAVE_NAME+\".zip\")\n",
        "                wandb.log_artifact(artifact)\n",
        "                wandb.log({\"reward_per_episode\": np.mean(self.total_episode_reward[self.locals[\"dones\"]==True]) })\n",
        "\n",
        "\n",
        "                mp4list = glob.glob('video/*.mp4')\n",
        "                print(mp4list)\n",
        "                if len(mp4list) > 0:\n",
        "                    print(len(mp4list))\n",
        "                    mp4 = mp4list[-1]\n",
        "                    mp4 = max(mp4list, key=os.path.getctime)\n",
        "                    video = io.open(mp4, 'r+b').read()\n",
        "                    encoded = base64.b64encode(video)\n",
        "\n",
        "                    # log gameplay video in wandb\n",
        "                    wandb.log({\"gameplays\": wandb.Video(mp4, fps=4, format=\"gif\")})\n",
        "\n",
        "                    # display gameplay video\n",
        "                    ipythondisplay.clear_output(wait=True)\n",
        "                    ipythondisplay.display(HTML(data='''<video alt=\"\" autoplay \n",
        "                                loop controls style=\"height: 400px;\">\n",
        "                                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "                            </video>'''.format(encoded.decode('ascii'))))\n",
        "                    print(\"Episode \", self.episodes)\n",
        "                    print(mp4)\n",
        "            self.episodes += 1\n",
        "            self.total_episode_reward[self.locals[\"dones\"]==True] = 0\n",
        "\n",
        "        \n",
        "        return True\n",
        "\n",
        "    def _on_rollout_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before updating the policy.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_training_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before exiting the `learn()` method.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "class A2CCustomCallback(BaseCallback):\n",
        "    \"\"\"\n",
        "    A custom callback that derives from ``BaseCallback``.\n",
        "\n",
        "    :param verbose: (int) Verbosity level 0: not output 1: info 2: debug\n",
        "    \"\"\"\n",
        "    def __init__(self, verbose=0):\n",
        "        super(A2CCustomCallback, self).__init__(verbose)\n",
        "        # Those variables will be accessible in the callback\n",
        "        # (they are defined in the base class)\n",
        "        # The RL model\n",
        "        # self.model = None  # type: BaseAlgorithm\n",
        "        # An alias for self.model.get_env(), the environment used for training\n",
        "        # self.training_env = None  # type: Union[gym.Env, VecEnv, None]\n",
        "        # Number of time the callback was called\n",
        "        # self.n_calls = 0  # type: int\n",
        "        # self.num_timesteps = 0  # type: int\n",
        "        # local and global variables\n",
        "        # self.locals = None  # type: Dict[str, Any]\n",
        "        # self.globals = None  # type: Dict[str, Any]\n",
        "        # The logger object, used to report things in the terminal\n",
        "        # self.logger = None  # stable_baselines3.common.logger\n",
        "        # # Sometimes, for event callback, it is useful\n",
        "        # # to have access to the parent object\n",
        "        # self.parent = None  # type: Optional[BaseCallback]\n",
        "        self.episodes = 0\n",
        "        self.total_episode_reward = 0\n",
        "\n",
        "    def _on_training_start(self) -> None:\n",
        "        \"\"\"\n",
        "        This method is called before the first rollout starts.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_rollout_start(self) -> None:\n",
        "        \"\"\"\n",
        "        A rollout is the collection of environment interaction\n",
        "        using the current policy.\n",
        "        This event is triggered before collecting new samples.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_step(self) -> bool:\n",
        "        # update commulative reward to log at the end of every episode\n",
        "        self.total_episode_reward += np.mean(self.locals[\"rewards\"])\n",
        "        # at the end of every episode\n",
        "        # print(self.locals[\"dones\"])\n",
        "        if self.locals[\"dones\"].all():\n",
        "\n",
        "            # log the reward value if its time to not log 2 times\n",
        "            if self.episodes % self.locals[\"log_interval\"] != 0: \n",
        "                wandb.log({\"reward_per_episode\": self.total_episode_reward})\n",
        "\n",
        "            # if log interval has passed\n",
        "            if self.episodes % self.locals[\"log_interval\"] == 0:\n",
        "                # log at wandb and print the last video\n",
        "                # Save your model and optimizer\n",
        "                self.model.save(MODEL_SAVE_NAME)\n",
        "                # Save as artifact for version control.\n",
        "                artifact = wandb.Artifact(MODEL_SAVE_NAME, type='model')\n",
        "                artifact.add_file(MODEL_SAVE_NAME+\".zip\")\n",
        "                wandb.log_artifact(artifact)\n",
        "                wandb.log({\"reward_per_episode\": self.total_episode_reward})\n",
        "\n",
        "\n",
        "                mp4list = glob.glob('video/*.mp4')\n",
        "                print(mp4list)\n",
        "                if len(mp4list) > 0:\n",
        "                    print(len(mp4list))\n",
        "                    mp4 = mp4list[-1]\n",
        "                    mp4 = max(mp4list, key=os.path.getctime)\n",
        "                    video = io.open(mp4, 'r+b').read()\n",
        "                    encoded = base64.b64encode(video)\n",
        "\n",
        "                    # log gameplay video in wandb\n",
        "                    wandb.log({\"gameplays\": wandb.Video(mp4, fps=4, format=\"gif\")})\n",
        "\n",
        "                    # display gameplay video\n",
        "                    ipythondisplay.clear_output(wait=True)\n",
        "                    ipythondisplay.display(HTML(data='''<video alt=\"\" autoplay \n",
        "                                loop controls style=\"height: 400px;\">\n",
        "                                <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "                            </video>'''.format(encoded.decode('ascii'))))\n",
        "                    print(\"Episode \", self.episodes)\n",
        "                    print(mp4)\n",
        "            self.episodes += 1\n",
        "            self.total_episode_reward = 0\n",
        "\n",
        "        \n",
        "        return True\n",
        "\n",
        "    def _on_rollout_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before updating the policy.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n",
        "    def _on_training_end(self) -> None:\n",
        "        \"\"\"\n",
        "        This event is triggered before exiting the `learn()` method.\n",
        "        \"\"\"\n",
        "        pass\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yJi0aHY7Abkn"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NgYsw2vnKsr1"
      },
      "source": [
        "### A2C"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VSDhKYVSKsGO"
      },
      "source": [
        "from stable_baselines.common.policies import CnnPolicy, CnnLstmPolicy\n",
        "from stable_baselines.common import make_vec_env\n",
        "from stable_baselines import A2C\n",
        "\n",
        "\n",
        "NUM_OF_STEPS        = 5000\n",
        "NUM_OF_EPISODES     = 400\n",
        "LOG_INTERVAL        = 5\n",
        "BUFFER_SIZE         = 10000\n",
        "LEARNING_STARTS     = 1000\n",
        "WANDB_ID            = \"andreas16\"\n",
        "WNDB_NAME           = \"ANDREAS16\"\n",
        "LOAD_SAVED_MODEL    = True\n",
        "MODEL_SAVE_NAME     = \"andreas_a2c_16\"\n",
        "SAVED_MODEL_VERSION = \"latest\"\n",
        "\n",
        "# wnadb api key: 00d5bfbd342bb73d5aaf4f2833436d20457ef040\n",
        "os.environ[\"WANDB_ENTITY\"]  = \"andreas_giannoutsos\"\n",
        "os.environ[\"WANDB_PROJECT\"] = \"gym_car_racer\"\n",
        "os.environ[\"WANDB_RESUME\"]  = \"allow\"\n",
        "wandb.init(resume=WANDB_ID)\n",
        "wandb.run.name = WNDB_NAME\n",
        "\n",
        "class MonitorCustom(gym.wrappers.Monitor):\n",
        "    def __init__(self, env):\n",
        "        super(MonitorCustom, self).__init__(env, './video', force=True)\n",
        "\n",
        "env = make_vec_env(CarRacing, n_envs=1, wrapper_class=MonitorCustom, env_kwargs={\"total_episode_steps\":NUM_OF_STEPS})\n",
        "\n",
        "# Load model\n",
        "if LOAD_SAVED_MODEL:\n",
        "    try:\n",
        "        model_artifact = wandb.use_artifact(MODEL_SAVE_NAME+':'+SAVED_MODEL_VERSION, type='model')\n",
        "        artifact_dir = model_artifact.download()\n",
        "        A2Cmodel = A2C.load(artifact_dir+\"/\"+MODEL_SAVE_NAME, env=env)\n",
        "        print(\"LOAD SAVED PPΟ MODEL\")\n",
        "\n",
        "    except:\n",
        "        print(\"NO MODEL FOUND\")\n",
        "else:\n",
        "    if 'A2Cmodel' not in globals():\n",
        "        # A2Cmodel = A2C(CnnPolicy, env, verbose=0, n_steps=5, policy_kwargs={\"optimizer_class\":RMSpropTFLike}, vf_coef=0.25, ent_coef=0.01)\n",
        "        # A2Cmodel = A2C(CnnPolicy, env, verbose=0, ent_coef=0.05, max_grad_norm=0.5, learning_rate=0.00005, n_steps=16, gamma=0.99)\n",
        "        A2Cmodel = A2C(CnnPolicy, env, verbose=0)\n",
        "        print(\"INITIALIZE NEW A2C MODEL\")\n",
        "    else:\n",
        "        A2Cmodel = A2C.load(MODEL_SAVE_NAME, env=env)\n",
        "        print(\"CONTINUE A2C MODEL TRAINING\")\n",
        "\n",
        "\n",
        "# Train model\n",
        "A2Cmodel.learn(total_timesteps=NUM_OF_STEPS*NUM_OF_EPISODES, log_interval=LOG_INTERVAL, callback=A2CCustomCallback())\n",
        "A2Cmodel.save(MODEL_SAVE_NAME)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a7J6QN1x065c"
      },
      "source": [
        "### PPO"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHa1SAwLTUzG"
      },
      "source": [
        "from stable_baselines3 import PPO\n",
        "from stable_baselines3.ppo import CnnPolicy\n",
        "from stable_baselines3.common.env_util import make_vec_env\n",
        "from stable_baselines3.common.vec_env import SubprocVecEnv\n",
        "\n",
        "\n",
        "\n",
        "NUM_OF_STEPS        = 1000\n",
        "NUM_OF_EPISODES     = 10\n",
        "LOG_INTERVAL        = 2\n",
        "BUFFER_SIZE         = 10000\n",
        "LEARNING_STARTS     = 1000\n",
        "WANDB_ID            = \"test_logging4\"\n",
        "WNDB_NAME           = \"test_logging4\"\n",
        "LOAD_SAVED_MODEL    = False\n",
        "MODEL_SAVE_NAME     = \"test_modell2\"\n",
        "SAVED_MODEL_VERSION = \"latest\"\n",
        "\n",
        "# wnadb api key: 00d5bfbd342bb73d5aaf4f2833436d20457ef040\n",
        "os.environ[\"WANDB_ENTITY\"]  = \"andreas_giannoutsos\"\n",
        "os.environ[\"WANDB_PROJECT\"] = \"gym_car_racer\"\n",
        "os.environ[\"WANDB_RESUME\"]  = \"allow\"\n",
        "wandb.init(resume=WANDB_ID)\n",
        "wandb.run.name = WNDB_NAME\n",
        "\n",
        "class MonitorCustom(gym.wrappers.Monitor):\n",
        "    def __init__(self, env):\n",
        "        super(MonitorCustom, self).__init__(env, './video', force=True)\n",
        "\n",
        "env = make_vec_env(CarRacing, n_envs=1, wrapper_class=MonitorCustom, env_kwargs={\"total_episode_steps\":NUM_OF_STEPS})\n",
        "\n",
        "\n",
        "# Load model\n",
        "if LOAD_SAVED_MODEL:\n",
        "    try:\n",
        "        model_artifact = wandb.use_artifact(MODEL_SAVE_NAME+':'+SAVED_MODEL_VERSION, type='model')\n",
        "        artifact_dir = model_artifact.download()\n",
        "        PPOmodel = PPO.load(artifact_dir+\"/\"+MODEL_SAVE_NAME, env=env)\n",
        "        print(\"LOAD SAVED PPΟ MODEL\")\n",
        "\n",
        "    except:\n",
        "        print(\"NO MODEL FOUND\")\n",
        "else:\n",
        "    if 'PPOmodel' not in globals():\n",
        "        PPOmodel = PPO(CnnPolicy, env, verbose=1)\n",
        "        print(\"INITIALIZE NEW PPO MODEL\")\n",
        "    else:\n",
        "        PPOmodel = PPO.load(MODEL_SAVE_NAME, env=env)\n",
        "        print(\"CONTINUE PPO MODEL TRAINING\")\n",
        "\n",
        "\n",
        "# Train model\n",
        "PPOmodel.learn(total_timesteps=NUM_OF_STEPS*NUM_OF_EPISODES, log_interval=LOG_INTERVAL, callback=PPOCustomCallback())\n",
        "PPOmodel.save(MODEL_SAVE_NAME)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aSBBoq8W08vN"
      },
      "source": [
        "### DQN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gVcpDbXiEPD7"
      },
      "source": [
        "from stable_baselines3 import DQN\n",
        "from stable_baselines3.dqn import CnnPolicy\n",
        "\n",
        "NUM_OF_STEPS        = 100\n",
        "NUM_OF_EPISODES     = 4\n",
        "LOG_INTERVAL        = 2\n",
        "BUFFER_SIZE         = 100000\n",
        "LEARNING_STARTS     = 1000\n",
        "WANDB_ID            = \"test_logging2\"\n",
        "WNDB_NAME           = \"test_logging2\"\n",
        "LOAD_SAVED_MODEL    = False\n",
        "MODEL_SAVE_NAME     = \"test_modell\"\n",
        "SAVED_MODEL_VERSION = \"latest\"\n",
        "\n",
        "# wnadb api key: 00d5bfbd342bb73d5aaf4f2833436d20457ef040\n",
        "os.environ[\"WANDB_ENTITY\"]  = \"andreas_giannoutsos\"\n",
        "os.environ[\"WANDB_PROJECT\"] = \"gym_car_racer\"\n",
        "os.environ[\"WANDB_RESUME\"]  = \"allow\"\n",
        "wandb.init(resume=WANDB_ID)\n",
        "wandb.run.name = WNDB_NAME\n",
        "\n",
        "# env = CarRacingDiscrete(NUM_OF_STEPS)\n",
        "env = gym.wrappers.Monitor(CarRacingDiscrete(NUM_OF_STEPS), './video', force=True)\n",
        "\n",
        "# Load model\n",
        "if LOAD_SAVED_MODEL:\n",
        "    try:\n",
        "        model_artifact = wandb.use_artifact(MODEL_SAVE_NAME+':'+SAVED_MODEL_VERSION, type='model')\n",
        "        artifact_dir = model_artifact.download()\n",
        "        DQNmodel = DQN.load(artifact_dir+\"/\"+MODEL_SAVE_NAME, env=env)\n",
        "        print(\"LOAD SAVED DQN MODEL\")\n",
        "    except:\n",
        "        print(\"NO MODEL FOUND\")\n",
        "else:    \n",
        "    if 'DQNmodel' not in globals():\n",
        "        DQNmodel = DQN(CnnPolicy, env, verbose=1, buffer_size=BUFFER_SIZE, learning_starts=LEARNING_STARTS)\n",
        "        print(\"INITIALIZE NEW DQN MODEL\")\n",
        "    else:\n",
        "        DQNmodel = DQN.load(MODEL_SAVE_NAME, env=env)\n",
        "        print(\"CONTINUE DQN MODEL TRAINING\")\n",
        "\n",
        "# Train model\n",
        "DQNmodel.learn(total_timesteps=NUM_OF_STEPS*NUM_OF_EPISODES, log_interval=LOG_INTERVAL, callback=DQNCustomCallback())\n",
        "DQNmodel.save(MODEL_SAVE_NAME)\n",
        "# del DQNmodel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MgnCizqTa10-"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Zs_IpmTa4p2"
      },
      "source": [
        "\n",
        "from stable_baselines3 import DQN\n",
        "\n",
        "def evaluate_version(model, env, version_name, version, video_path):\n",
        "    # get version of model\n",
        "    MODEL_SAVE_NAME     = version_name\n",
        "    SAVED_MODEL_VERSION = version\n",
        "    # wnadb api key: 00d5bfbd342bb73d5aaf4f2833436d20457ef040\n",
        "    os.environ[\"WANDB_ENTITY\"]  = \"andreas_giannoutsos\"\n",
        "    os.environ[\"WANDB_PROJECT\"] = \"gym_car_racer\"\n",
        "    os.environ[\"WANDB_RESUME\"]  = \"allow\"\n",
        "    wandb.init()\n",
        "    model_artifact = wandb.use_artifact(MODEL_SAVE_NAME+':'+SAVED_MODEL_VERSION, type='model')\n",
        "    artifact_dir = model_artifact.download()\n",
        "    loaded_model = model.load(artifact_dir+\"/\"+MODEL_SAVE_NAME)\n",
        "\n",
        "    # play model\n",
        "    env = gym.wrappers.Monitor(env, video_path, force=True)\n",
        "    obs = env.reset()\n",
        "    done = False\n",
        "    while not done:\n",
        "        action, _states = loaded_model.predict(obs, deterministic=True)\n",
        "        obs, reward, done, info = env.step(action)\n",
        "    env.close()\n",
        "\n",
        "    # display video\n",
        "    mp4list = glob.glob(video_path+'/*.mp4')\n",
        "    print(mp4list)\n",
        "    if len(mp4list) > 0:\n",
        "        print(len(mp4list))\n",
        "        mp4 = mp4list[-1]\n",
        "        video = io.open(mp4, 'r+b').read()\n",
        "        encoded = base64.b64encode(video)\n",
        "\n",
        "        # log gameplay video in wandb\n",
        "        wandb.log({\"gameplays\": wandb.Video(mp4, fps=4, format=\"gif\")})\n",
        "\n",
        "        # display gameplay video\n",
        "        ipythondisplay.clear_output(wait=True)\n",
        "        ipythondisplay.display(HTML(data='''<video alt=\"\" autoplay \n",
        "                    loop controls style=\"height: 400px;\">\n",
        "                    <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\" />\n",
        "                </video>'''.format(encoded.decode('ascii'))))\n",
        "        print(\"Video path: \", mp4)\n",
        "\n",
        "\n",
        "from stable_baselines3 import DQN\n",
        "from stable_baselines import PPO2\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}