policy-iteration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import sys\n",
    "import matplotlib \n",
    "import random\n",
    "#matplotlib.use(\"TkAgg\")\n",
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "import envs\n",
    "import pickle\n",
    "#import gridworld_env\n",
    "from gym import wrappers, logger\n",
    "import numpy as np\n",
    "import copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rappel sur la structure retournée par envx.getMPD()  :\n",
    "\n",
    "envx.getMPD() return (states,P)\n",
    "print(type(envx.getMDP())) = <class 'tuple'>\n",
    "\n",
    "states:\n",
    "print(type(envx.getMDP()[0])) = <class 'dict'> with 11 keys\n",
    "(key, value) = (state's name, num) = <, int>\n",
    "\n",
    "P: (state's name, dict(action's num))\n",
    "<state's name: 0:(proba de la transition, état destination (name's key), reward, done)\n",
    "               1:(proba de la transition, état destination (name's key), reward, done)\n",
    "               2:(proba de la transition, état destination (name's key), reward, done)\n",
    "               3:(proba de la transition, état destination (name's key), reward, done)>\n",
    ">               \n",
    "type(envx.getMDP()[1]) = dict\n",
    "\n",
    "En vraie, une observation de l'etat est une observation partielle de l'état complet de l'univers (!= d'une observation totale). Le paramètre d'observation partielle peut etre réduit à un paramètre pleinement observé avec l'utilisation d'historique (l'état devient une accumulation d'état précédents).\n",
    "Néansmois, il n'est pas courant de ne pas accumler toute l'histoire. soit seules les dernieres observations h sont cumulées => Chaine de Markov.\n",
    "Par un abus de langages, l'observation = Etat\n",
    "Ici l'état est une observation 6*6, certaines cases sont accessibles par le robot. Le robot = agent = 2 (par déduction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Etant donné, que les données retournées par envx.getMDP() sont peu lisibles et moins facile a manipuler, on se propose de re-structurer des données. De sorte d'avoir un dictionnaire organiser de la façon suivante:\n",
    "proba = {state's num : action's num : {next_state: proba}*}\n",
    "rewards = {state's num : action's num : {next_state: reward}*}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#transformation\n",
    "def transform(envx, norm=True):\n",
    "    \"\"\"return proba sous le format decrit plus haut et rewards\n",
    "    \"\"\"\n",
    "    \n",
    "    #version 1\n",
    "    \"\"\"    proba = {}\n",
    "    for state_name, rest in envx.getMDP()[1].items():     \n",
    "            actions = {}\n",
    "            for action in rest.keys():\n",
    "                transitions = {}   \n",
    "                for p in rest[action]:\n",
    "                    transitions[envx.getMDP()[0][p[1]]] = p[0]\n",
    "                actions[action] = transitions\n",
    "            proba[envx.getMDP()[0][state_name]] = actions\n",
    "    \"\"\"\n",
    "    #version 2\n",
    "    proba = { envx.getMDP()[0][state_name]: \n",
    "    { \n",
    "        action : \n",
    "        {\n",
    "            envx.getMDP()[0][p[1]] :  \n",
    "            p[0]  for p in rest[action] \n",
    "        } \n",
    "        for action in rest.keys()\n",
    "    }\n",
    "    for state_name, rest in envx.getMDP()[1].items()\n",
    "   }\n",
    "    #normalisation des probas\n",
    "    if norm:\n",
    "        #version 1\n",
    "        for s in proba:\n",
    "            for a in proba[s]:\n",
    "                tmp = np.sum(list(proba[s][a].values()))\n",
    "                for k, v in proba[s][a].items():\n",
    "                    proba[s][a][k] = v/tmp         \n",
    "        #vesrion 2\n",
    "        #proba = {state: {a: {proba[state][a][s]: j/np.sum(list(proba[state][a].values())) \n",
    "        #     for s, j in proba[state][a].items()}for a in proba[state]}for state, v in proba.items()}\n",
    "        #version 2               \n",
    "    rewards = { envx.getMDP()[0][state_name]: \n",
    "    { \n",
    "        action : \n",
    "        {\n",
    "            envx.getMDP()[0][p[1]] :  \n",
    "            p[2]  for p in rest[action] \n",
    "        } \n",
    "        for action in rest.keys()\n",
    "    }\n",
    "    for state_name, rest in envx.getMDP()[1].items()\n",
    "   }\n",
    "    return proba, rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PlicyIteration(object):\n",
    "    \n",
    "    def __init__(self, envx, plan, discount=1, eps=.0001, robot=2):\n",
    "        self.eps = eps\n",
    "        self.envx = envx\n",
    "        self.robot = robot \n",
    "        self.plan = plan\n",
    "        self.discount = discount\n",
    "        self.policy_stable = False\n",
    "        self.action_space = envx.action_space \n",
    "        self.proba, self.rewards = transform(envx)\n",
    "        #self.values : dict mieux que array car ils manquent certains etats\n",
    "        self.values = {state : 0 for state in envx.getMDP()[0].values()}\n",
    "        # <!> les observations sont dans un fichier binaires,  sér\n",
    "        self.observation_to_state = {str(pickle.loads(list(envx.getMDP()[0].keys())[state])): \n",
    "                        state  for state in envx.getMDP()[0].values()}\n",
    "        actions_possibles = set([v for actions in envx.getMDP()[1].values() for v in list(actions.keys())])\n",
    "        self.policy = {state : random.choice(list(actions_possibles)) for state in range(len(envx.getMDP()[0].values()))} \n",
    "        \n",
    "    \n",
    "    def check_policy(self, state):\n",
    "        if state not in self.policy_dico.keys():\n",
    "            #ce cas n'est pas arrivé\n",
    "            self.policy_dico[state] = None\n",
    "        return self.policy_dico[state]\n",
    "    \n",
    "    def get_state_value(self, current_state, action):\n",
    "        return  np.sum([prob*(self.rewards[current_state][action][next_state] + \\\n",
    "        self.discount*self.values[next_state]) for next_state, prob in self.proba[current_state][action].items()])\n",
    "        \n",
    "    def policy_evaluation(self, nb_iter=100000):\n",
    "        delta = np.inf\n",
    "        while delta > self.eps and nb_iter:\n",
    "            delta = 0\n",
    "            for current_state in self.proba:\n",
    "                v_old = self.values[current_state]\n",
    "                action = self.policy[current_state]\n",
    "                tmp = 0\n",
    "                self.values[current_state]= self.get_state_value(current_state, action)\n",
    "                delta = max(delta, abs(v_old - self.values[current_state]))      \n",
    "            nb_iter -=1\n",
    "            \n",
    "    def best_action(self, current_state):\n",
    "        values = [self.get_state_value(current_state, action) for action in self.proba[current_state]]\n",
    "        action =  np.argmax(values)\n",
    "        value = values[int(action)]\n",
    "        return action\n",
    "    \n",
    "    def policy_improvement(self):\n",
    "        self.policy_stable = True \n",
    "        for state in self.proba:\n",
    "            current_action = self.policy[state]\n",
    "            self.policy[state] = self.best_action(state)\n",
    "            if current_action != self.policy[state]:\n",
    "                self.policy_stable = False\n",
    "                  \n",
    "    def fit(self, nb_iter=1000): \n",
    "        \n",
    "        while not self.policy_stable and nb_iter: \n",
    "            self.policy_evaluation()\n",
    "            self.policy_improvement()\n",
    "            nb_iter -=1\n",
    "\n",
    "\n",
    "    def plt_values(self):\n",
    "        grid_size = pickle.loads(list(self.envx.getMDP()[0].keys())[5]).shape\n",
    "        self.grid = np.zeros(grid_size)\n",
    "        \n",
    "        for state_name, state_num in self.envx.getMDP()[0].items():\n",
    "            state_serial = np.array(pickle.loads(state_name))\n",
    "            x, y  = np.where(state_serial == self.robot)\n",
    "            self.grid[x, y] = self.values[state_num]\n",
    "        print(self.values)\n",
    "        plt.title(str(self.plan))\n",
    "        plt.imshow(self.grid / self.grid.max(), cmap ='gray')\n",
    "        plt.show()\n",
    "        \n",
    "\n",
    "    def act(self, observation, reward, done):\n",
    "        return self.policy[self.observation_to_state[str(observation)]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1 1 1 1 1 1\\n', '1 0 0 0 3 1\\n', '1 0 1 0 5 1\\n', '1 0 0 0 2 1\\n', '1 1 1 1 1 1\\n', '1 1 1 1 1 1']\n",
      "['1 1 1 1 1 1\\n', '1 0 0 0 3 1\\n', '1 0 1 0 5 1\\n', '1 0 0 0 2 1\\n', '1 1 1 1 1 1\\n', '1 1 1 1 1 1']\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "outdir = 'gridworld-v0/random-agent-results'\n",
    "envx = gym.make('gridworld-v0')\n",
    "env = wrappers.Monitor(envx, directory=outdir, force=True, video_callable=False)\n",
    "env.seed(0)\n",
    "episode_count = 100\n",
    "reward = 0\n",
    "done = False\n",
    "envx.verbose = True\n",
    "plan =   {0: -3, 3: 1 , 4: -1, 5: -1, 6: -1}\n",
    "envx.setPlan(\"gridworldPlans/plan0.txt\", plan )\n",
    "\n",
    "agent = PlicyIteration(envx, plan)\n",
    "#np.random.seed(5)\n",
    "agent.fit()\n",
    "rsum=0\n",
    "\n",
    "for i in range(episode_count):\n",
    "    ob = env.reset()\n",
    "\n",
    "    if i % 100 == 0 and i > 0:\n",
    "        envx.verbose = True\n",
    "    else:\n",
    "        envx.verbose = False\n",
    "\n",
    "    if envx.verbose:\n",
    "            #envx.render(1)\n",
    "            1\n",
    "    j = 0\n",
    "    #print(str(ob))\n",
    "    while True:\n",
    "\n",
    "        action = agent.act(ob, reward, done)\n",
    "        ob, reward, done, _ = env.step(action)\n",
    "        rsum+=reward\n",
    "        j += 1\n",
    "        if envx.verbose:\n",
    "            #envx.render()\n",
    "            1\n",
    "        if done:\n",
    "            #print(str(i)+\" rsum=\"+str(rsum)+\", \"+str(j)+\" actions\")\n",
    "            rsum=0\n",
    "            break\n",
    "\n",
    "print(\"done\")\n",
    "env.close()\n",
    "\n",
    "#agent.plt_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: -2.163653366676953, 1: 0, 2: -5.472880299387934, 3: -1.9466957606131134, 4: 0.0059226932643687835, 5: 0, 6: -3.3690773067371693, 7: -7.540952308049711, 8: -10.915952308793802, 9: -12.411001021247987, 10: -8.847880299472694}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAEICAYAAAByNDmmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADxFJREFUeJzt3X2MHPVhxvHvgzGYQihQbyi1DSYhhRYUIDohKlKSEHDNS4BWEYI0KVUoVqWmIW0TQiJVIv0jatOKokZNU/PS0kKwqAAlpQkvEliU1LycwRiMoUXgEpcQnyEGTAnB+OkfO6eczy/7Nnuz9+vzkVa+m53Zee68z/5mZvdmZJuIKNNeTQeIiOFJwSMKloJHFCwFjyhYCh5RsBQ8omApeETBZlXBJR0r6TFJr0n6tabzREwn6XJJP5Z0l6SfazrPrCo48GngOeAg26smJ0o6QdJqSf9b/XtCvyuQ9BFJT0jaIullSbdLWtDlsvMlfb9aboukVZJO6XHd90l6VdKGAX6Gf5BkSUcN8Bh9Z5H0YUnbJW2dcru4iSzV8i1J36r+T34s6aYBshxXlXezpJ0+JWb7a8BC4BhgSb/rqctsK/ghwHrb2ycnSNoH+DZwI3AwcAPw7Wp6P54CfsP2QcAvAf8F/F2Xy26l/SLUqrL8BfCvkvbucvk3gOuBL/SUeApJHwTe2+/yNWZ50fYBU243NJjlNuAl4Ajg3cBfDZDlbeAW4JLdzWD7DeB54BcGWE8tZlvB9wa2T5v24Wr61bbfsv03gIDT+lmB7R/ZfnHKpHeArkZC2z+x/Uz1AqRq2YNpvzB1s/zDtv+Z9lZKz6oXkq8Dn+ln+Tqz1GmQLJKWAIuAL9h+1fbbth8bIMsztq8D1nWYdTvt52WjZk3BJR0CjAEvTLvrWGCtd/xQ/dpqOpI+IWltj+s6XNIW4E3g88DXelx+LfAT4DvAtbY3VdM/WD3usPwRcL/tnX5eSd+Q9I0hrnu6d0v6kaTnJf21pP0bynIy8AxwQ7Xr9IikD03JcoWkO4aw3h8AHxpgS7Ietkf+BvwhYOBBYO60+/4UWDFt2k3AlTWs9xDgi8DJfSw7D7gIuLiPZU8HNvS4zCLgWeDnq+8NHFXD76CfLL8I/CrtAeRI4H7g7xvKsrz6XVwCzAUuBLYA8wfMclS7Pru9/720dwveBsYG/dn7vc2KEdz214HDaD9xzpt291bgwGnTDgRe7/S4kn59ykGgnTa5bL/Cz/bpe9rccntz/WbgCknH97Jsn64G/sz2q70uKOnLU34P3xw0iO2XbD9le7vt54HLgY83kYX2VtgG29e5vXm+gvbo2vHgp6TfnpLlez2u9zLgEeBA2+O9x67HrCg4tJ80wCraI8NU64D3S9KUae+n8z4Stv/dPzsIdOxuZtub9oGZ6S8i3ZoLvKfPZXvxUeAvJb0k6aVq2ipJn+i0oO2vTvk9/P4Qspn2MYnOM9afZW21/p7ZvmlKljN7XPxXgDttv9nPuusyawpeeQuYvk+zkvbBrM9K2lfS5AGme/tZgaTfknS0pL0ktYCrgMeq0RxJV0pauZtlT672s/eRtJ+kLwKHAg91ue69JM2j/aIgSfOm7sNJWinpyt0s/svA8cAJ1Q3gY8Dt3ay7zizV22SHq20R8Oe03+noy4C/l9uBgyVdLGmOpI8DC4Dv95lFVZZ9qu/nSdp3F7POpf18bdRsK/h2pmW2/VPgfOB3aO9bfRo4v5o+uZnVcTSfYgFwJ+1N/Ceqdf7mlPsXsfsnx77A3wIvA/8DnAWc7eqo/OQuwR7WfSrtTcrvAodXX9/dzbptb6o2jV+qtnYANk+OIJK+2eMmb99ZgA/Q3tp6A/gP4Engs5N3zmSW6oX5XNoHS18FrgDOs725yvLlHje/j6jWP/mcepP2Qbzp5rDzOz4zTtUBgVlB0leBE4Fzbb/dUIY1wEdtvzzD610I/Ivtxj/Blyx7Jukg2p+n+D3b3200yywr+Htof+DhWOBjth9sOFLEDiR9nvbWwn3A79pudDN9VhU8Inoz2/bBI6IHQ/ko3fz587148eJhPHREABs2bGDz5s0d33ocSsEXL17M+Hhj7+1HFG9sbKyr+bKJHlGwFDyiYCl4RMFS8IiCpeARBUvBIwqWgkcULAWPKFgKHlGwFDyiYCl4RMFS8IiCpeARBeuq4JKWSnpG0rOSrhh2qIioR8eCS5pD+0SCZ9I+ZfFFkqafujgiRlA3I/hJwLO2n6vOVLqCnS8+EBEjqJuCL6B9JYhJG6tpO5C0TNK4pPGJiYm68kXEALop+K5OC7Or6yIvtz1me6zVag2eLCIG1k3BN9I+sfykhcCLu5k3IkZINwV/BHifpCOry8VcSPuyuBEx4jqedNH2tup6X3fRvhzL9bZ7uRRQRDSkq7OqVpdfafQSLBHRu3ySLaJgKXhEwVLwiIKl4BEFS8EjCpaCRxQsBY8oWAoeUbAUPKJgKXhEwVLwiIKl4BEF6+qPTWa7Sy+9tOkIO7nggguajrCDM844o+kIMQQZwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgnUsuKTrJW2S9ORMBIqI+nQzgv8jsHTIOSJiCDoW3Pb9wCszkCUialbbPrikZZLGJY1PTEzU9bARMYDaCm57ue0x22OtVquuh42IAeQoekTBUvCIgnXzNtnNwCrgaEkbJV0y/FgRUYeO50W3fdFMBImI+mUTPaJgKXhEwVLwiIKl4BEFS8EjCpaCRxQsBY8oWAoeUbAUPKJgKXhEwVLwiIKl4BEF6/jHJiW4++67m46wk2uuuabpCDt4+umnm46wg2OOOabpCEXICB5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMK1s3FBxdJuk/SeknrJF02E8EiYnDd/D34NuBPbD8q6V3Aakn32H5qyNkiYkAdR3DbP7T9aPX168B6YMGwg0XE4HraB5e0GDgReGgX9y2TNC5pfGJiop50ETGQrgsu6QDgVuBztl+bfr/t5bbHbI+1Wq06M0ZEn7oquKS5tMt9k+3bhhspIurSzVF0AdcB621fNfxIEVGXbkbwU4BPAadJWlPdzhpyroioQce3yWw/AGgGskREzfJJtoiCpeARBUvBIwqWgkcULAWPKFgKHlGwFDyiYCl4RMFS8IiCpeARBUvBIwqWgkcUrJtzss16L7zwQtMRdnLWWaP1B3lLlixpOsIOtm3b1nSEnRx33HFNR+hZRvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsG6uLjpP0sOSHpe0TtJXZiJYRAyum78Hfws4zfbW6jrhD0j6nu0Hh5wtIgbUzdVFDWytvp1b3TzMUBFRj672wSXNkbQG2ATcY/uhXcyzTNK4pPGJiYm6c0ZEH7oquO13bJ8ALAROkrTTuWtsL7c9Znus1WrVnTMi+tDTUXTbW4CVwNKhpImIWnVzFL0l6aDq6/2A04Gnhx0sIgbXzVH0w4AbJM2h/YJwi+07hhsrIurQzVH0tcCJM5AlImqWT7JFFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omDd/DXZrNc+61TE/z8ZwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBui64pDmSHpOUCw9GzBK9jOCXAeuHFSQi6tdVwSUtBM4Grh1unIioU7cj+NXA5cD23c0gaZmkcUnjExMTtYSLiMF0LLikc4BNtlfvaT7by22P2R5rtVq1BYyI/nUzgp8CnCtpA7ACOE3SjUNNFRG16Fhw21+yvdD2YuBC4F7bnxx6sogYWN4HjyhYT6dNtr0SWDmUJBFRu4zgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYCh5RsBQ8omApeETBUvCIgqXgEQVLwSMKloJHFCwFjyhYVxcfrK4N/jrwDrDN9tgwQ0VEPXq5uuhHbG8eWpKIqF020SMK1m3BDdwtabWkZbuaQdIySeOSxicmJupLGBF967bgp9j+AHAm8AeSTp0+g+3ltsdsj7VarVpDRkR/uiq47RerfzcBtwMnDTNURNSjY8El7S/pXZNfA0uAJ4cdLCIG181R9EOB2yVNzv8t23cONVVE1KJjwW0/Bxw/A1kiomZ5myyiYCl4RMFS8IiCpeARBUvBIwqWgkcULAWPKFgKHlGwFDyiYCl4RMFS8IiCpeARBZPt+h9UmgD+u4aHmg+M0nngkmfPRi0PjF6muvIcYbvjmVWGUvC6SBofpTO4Js+ejVoeGL1MM50nm+gRBUvBIwo26gVf3nSAaZJnz0YtD4xephnNM9L74BExmFEfwSNiACl4RMFGsuCSlkp6RtKzkq4YgTzXS9okaSROFy1pkaT7JK2XtE7SZQ3nmSfpYUmPV3m+0mSeSZLmSHpM0h1NZ4H2RTwlPSFpjaTxGVnnqO2DS5oD/CdwBrAReAS4yPZTDWY6FdgK/JPt45rKMSXPYcBhth+tzlm/Gji/qd+R2ufU3t/2VklzgQeAy2w/2ESeKbn+GBgDDrR9TpNZqjwbgLGZvIjnKI7gJwHP2n7O9k+BFcB5TQayfT/wSpMZprL9Q9uPVl+/DqwHFjSYx7a3Vt/OrW6NjhySFgJnA9c2maNpo1jwBcAPpny/kQafvKNO0mLgROChhnPMkbQG2ATcY7vRPMDVwOXA9oZzTNXxIp51G8WCaxfTRms/YkRIOgC4Ffic7deazGL7HdsnAAuBkyQ1tisj6Rxgk+3VTWXYjY4X8azbKBZ8I7BoyvcLgRcbyjKyqn3dW4GbbN/WdJ5JtrcAK4GlDcY4BTi32uddAZwm6cYG8wDNXMRzFAv+CPA+SUdK2ge4EPhOw5lGSnVQ6zpgve2rRiBPS9JB1df7AacDTzeVx/aXbC+0vZj28+de259sKg80dxHPkSu47W3AZ4C7aB88usX2uiYzSboZWAUcLWmjpEuazEN7hPoU7ZFpTXU7q8E8hwH3SVpL+wX6Htsj8dbUCDkUeEDS48DDwL/NxEU8R+5tsoioz8iN4BFRnxQ8omApeETBUvCIgqXgEQVLwSMKloJHFOz/AMiEyCMXUdShAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "agent.plt_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}