Module 5 Assignment/Slutz_Eric_Assignment5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Module Five Assignment: Cartpole Problem\n",
    "Review the code in this notebook and in the score_logger.py file in the *scores* folder (directory). Once you have reviewed the code, return to this notebook and select **Cell** and then **Run All** from the menu bar to run this code. The code takes several minutes to run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import random  \n",
    "import gym  \n",
    "import numpy as np  \n",
    "from collections import deque  \n",
    "from keras.models import Sequential  \n",
    "from keras.layers import Dense  \n",
    "from keras.optimizers import Adam  \n",
    "from scores.score_logger import ScoreLogger\n",
    "\n",
    "  \n",
    "ENV_NAME = \"CartPole-v1\"  \n",
    "  \n",
    "GAMMA = 0.95  \n",
    "LEARNING_RATE = 0.001  \n",
    "  \n",
    "MEMORY_SIZE = 1000000  \n",
    "BATCH_SIZE = 20  \n",
    "  \n",
    "EXPLORATION_MAX = 1.0  \n",
    "EXPLORATION_MIN = 0.01  \n",
    "EXPLORATION_DECAY = 0.995  \n",
    "  \n",
    "  \n",
    "class DQNSolver:  \n",
    "  \n",
    "    def __init__(self, observation_space, action_space):  \n",
    "        self.exploration_rate = EXPLORATION_MAX  \n",
    "  \n",
    "        self.action_space = action_space  \n",
    "        self.memory = deque(maxlen=MEMORY_SIZE)  \n",
    "  \n",
    "        self.model = Sequential()  \n",
    "        self.model.add(Dense(24, input_shape=(observation_space,), activation=\"relu\"))  \n",
    "        self.model.add(Dense(24, activation=\"relu\"))  \n",
    "        self.model.add(Dense(self.action_space, activation=\"linear\"))  \n",
    "        self.model.compile(loss=\"mse\", optimizer=Adam(lr=LEARNING_RATE))  \n",
    "  \n",
    "    def remember(self, state, action, reward, next_state, done):  \n",
    "        self.memory.append((state, action, reward, next_state, done))  \n",
    "  \n",
    "    def act(self, state):  \n",
    "        if np.random.rand() < self.exploration_rate:  \n",
    "            return random.randrange(self.action_space)  \n",
    "        q_values = self.model.predict(state)  \n",
    "        return np.argmax(q_values[0])  \n",
    "  \n",
    "    def experience_replay(self):  \n",
    "        if len(self.memory) < BATCH_SIZE:  \n",
    "            return  \n",
    "        batch = random.sample(self.memory, BATCH_SIZE)  \n",
    "        for state, action, reward, state_next, terminal in batch:  \n",
    "            q_update = reward  \n",
    "            if not terminal:  \n",
    "                q_update = (reward + GAMMA * np.amax(self.model.predict(state_next)[0]))  \n",
    "            q_values = self.model.predict(state)  \n",
    "            q_values[0][action] = q_update  \n",
    "            self.model.fit(state, q_values, verbose=0)  \n",
    "        self.exploration_rate *= EXPLORATION_DECAY  \n",
    "        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)  \n",
    "  \n",
    "  \n",
    "def cartpole():  \n",
    "    env = gym.make(ENV_NAME)  \n",
    "    score_logger = ScoreLogger(ENV_NAME)  \n",
    "    observation_space = env.observation_space.shape[0]  \n",
    "    action_space = env.action_space.n  \n",
    "    dqn_solver = DQNSolver(observation_space, action_space)  \n",
    "    run = 0  \n",
    "    while True:  \n",
    "        run += 1  \n",
    "        state = env.reset()  \n",
    "        state = np.reshape(state, [1, observation_space])  \n",
    "        step = 0  \n",
    "        while True:  \n",
    "            step += 1  \n",
    "            #env.render()  \n",
    "            action = dqn_solver.act(state)  \n",
    "            state_next, reward, terminal, info = env.step(action)  \n",
    "            reward = reward if not terminal else -reward  \n",
    "            state_next = np.reshape(state_next, [1, observation_space])  \n",
    "            dqn_solver.remember(state, action, reward, state_next, terminal)  \n",
    "            state = state_next  \n",
    "            if terminal:  \n",
    "                print (\"Run: \" + str(run) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(step))  \n",
    "                score_logger.add_score(step, run)  \n",
    "                break  \n",
    "            dqn_solver.experience_replay()  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 1, exploration: 0.985074875, score: 23\n",
      "Scores: (min: 23, avg: 23, max: 23)\n",
      "\n",
      "Run: 2, exploration: 0.9137248860125932, score: 16\n",
      "Scores: (min: 16, avg: 19.5, max: 23)\n",
      "\n",
      "Run: 3, exploration: 0.8433051360508336, score: 17\n",
      "Scores: (min: 16, avg: 18.666666666666668, max: 23)\n",
      "\n",
      "Run: 4, exploration: 0.8020760579717637, score: 11\n",
      "Scores: (min: 11, avg: 16.75, max: 23)\n",
      "\n",
      "Run: 5, exploration: 0.6832098777212641, score: 33\n",
      "Scores: (min: 11, avg: 20, max: 33)\n",
      "\n",
      "Run: 6, exploration: 0.653073201944699, score: 10\n",
      "Scores: (min: 10, avg: 18.333333333333332, max: 33)\n",
      "\n",
      "Run: 7, exploration: 0.6242658676435396, score: 10\n",
      "Scores: (min: 10, avg: 17.142857142857142, max: 33)\n",
      "\n",
      "Run: 8, exploration: 0.6027415843082742, score: 8\n",
      "Scores: (min: 8, avg: 16, max: 33)\n",
      "\n",
      "Run: 9, exploration: 0.5647174463480732, score: 14\n",
      "Scores: (min: 8, avg: 15.777777777777779, max: 33)\n",
      "\n",
      "Run: 10, exploration: 0.5398075216808175, score: 10\n",
      "Scores: (min: 8, avg: 15.2, max: 33)\n",
      "\n",
      "Run: 11, exploration: 0.5082950737585841, score: 13\n",
      "Scores: (min: 8, avg: 15, max: 33)\n",
      "\n",
      "Run: 12, exploration: 0.457510005540005, score: 22\n",
      "Scores: (min: 8, avg: 15.583333333333334, max: 33)\n",
      "\n",
      "Run: 13, exploration: 0.4351424010585501, score: 11\n",
      "Scores: (min: 8, avg: 15.23076923076923, max: 33)\n",
      "\n",
      "Run: 14, exploration: 0.4159480862733536, score: 10\n",
      "Scores: (min: 8, avg: 14.857142857142858, max: 33)\n",
      "\n",
      "Run: 15, exploration: 0.3897078735047413, score: 14\n",
      "Scores: (min: 8, avg: 14.8, max: 33)\n",
      "\n",
      "Run: 16, exploration: 0.3706551064126331, score: 11\n",
      "Scores: (min: 8, avg: 14.5625, max: 33)\n",
      "\n",
      "Run: 17, exploration: 0.34901730169741024, score: 13\n",
      "Scores: (min: 8, avg: 14.470588235294118, max: 33)\n",
      "\n",
      "Run: 18, exploration: 0.33195389135223546, score: 11\n",
      "Scores: (min: 8, avg: 14.277777777777779, max: 33)\n",
      "\n",
      "Run: 19, exploration: 0.3141460853680822, score: 12\n",
      "Scores: (min: 8, avg: 14.157894736842104, max: 33)\n",
      "\n",
      "Run: 20, exploration: 0.29580711868545667, score: 13\n",
      "Scores: (min: 8, avg: 14.1, max: 33)\n",
      "\n",
      "Run: 21, exploration: 0.2827589419554058, score: 10\n",
      "Scores: (min: 8, avg: 13.904761904761905, max: 33)\n",
      "\n",
      "Run: 22, exploration: 0.26759021970270175, score: 12\n",
      "Scores: (min: 8, avg: 13.818181818181818, max: 33)\n",
      "\n",
      "Run: 23, exploration: 0.25450776877280124, score: 11\n",
      "Scores: (min: 8, avg: 13.695652173913043, max: 33)\n",
      "\n",
      "Run: 24, exploration: 0.23965031961336, score: 13\n",
      "Scores: (min: 8, avg: 13.666666666666666, max: 33)\n",
      "\n",
      "Run: 25, exploration: 0.22566020663225933, score: 13\n",
      "Scores: (min: 8, avg: 13.64, max: 33)\n",
      "\n",
      "Run: 26, exploration: 0.2178795886667409, score: 8\n",
      "Scores: (min: 8, avg: 13.423076923076923, max: 33)\n",
      "\n",
      "Run: 27, exploration: 0.20826882814336947, score: 10\n",
      "Scores: (min: 8, avg: 13.296296296296296, max: 33)\n",
      "\n",
      "Run: 28, exploration: 0.2000824143909432, score: 9\n",
      "Scores: (min: 8, avg: 13.142857142857142, max: 33)\n",
      "\n",
      "Run: 29, exploration: 0.192217783647157, score: 9\n",
      "Scores: (min: 8, avg: 13, max: 33)\n",
      "\n",
      "Run: 30, exploration: 0.15111286553822956, score: 49\n",
      "Scores: (min: 8, avg: 14.2, max: 49)\n",
      "\n",
      "Run: 31, exploration: 0.12490462201997637, score: 39\n",
      "Scores: (min: 8, avg: 15, max: 49)\n",
      "\n",
      "Run: 32, exploration: 0.09770335251664321, score: 50\n",
      "Scores: (min: 8, avg: 16.09375, max: 50)\n",
      "\n",
      "Run: 33, exploration: 0.07875931641927113, score: 44\n",
      "Scores: (min: 8, avg: 16.939393939393938, max: 50)\n",
      "\n",
      "Run: 34, exploration: 0.06844601986126451, score: 29\n",
      "Scores: (min: 8, avg: 17.294117647058822, max: 50)\n",
      "\n",
      "Run: 35, exploration: 0.05462441922520914, score: 46\n",
      "Scores: (min: 8, avg: 18.114285714285714, max: 50)\n",
      "\n",
      "Run: 36, exploration: 0.04652918187562211, score: 33\n",
      "Scores: (min: 8, avg: 18.52777777777778, max: 50)\n",
      "\n",
      "Run: 37, exploration: 0.04447676004441063, score: 10\n",
      "Scores: (min: 8, avg: 18.2972972972973, max: 50)\n",
      "\n",
      "Run: 38, exploration: 0.04272851397543923, score: 9\n",
      "Scores: (min: 8, avg: 18.05263157894737, max: 50)\n",
      "\n",
      "Run: 39, exploration: 0.03963364364733769, score: 16\n",
      "Scores: (min: 8, avg: 18, max: 50)\n",
      "\n",
      "Run: 40, exploration: 0.036033175291307735, score: 20\n",
      "Scores: (min: 8, avg: 18.05, max: 50)\n",
      "\n",
      "Run: 41, exploration: 0.033423255248208356, score: 16\n",
      "Scores: (min: 8, avg: 18, max: 50)\n",
      "\n",
      "Run: 42, exploration: 0.030235026982422884, score: 21\n",
      "Scores: (min: 8, avg: 18.071428571428573, max: 50)\n",
      "\n",
      "Run: 43, exploration: 0.028186002814352063, score: 15\n",
      "Scores: (min: 8, avg: 18, max: 50)\n",
      "\n",
      "Run: 44, exploration: 0.024618221284683196, score: 28\n",
      "Scores: (min: 8, avg: 18.227272727272727, max: 50)\n",
      "\n",
      "Run: 45, exploration: 0.02171869316274056, score: 26\n",
      "Scores: (min: 8, avg: 18.4, max: 50)\n",
      "\n",
      "Run: 46, exploration: 0.020246819920648168, score: 15\n",
      "Scores: (min: 8, avg: 18.32608695652174, max: 50)\n",
      "\n",
      "Run: 47, exploration: 0.018042124582040707, score: 24\n",
      "Scores: (min: 8, avg: 18.4468085106383, max: 50)\n",
      "\n",
      "Run: 48, exploration: 0.016651638319386028, score: 17\n",
      "Scores: (min: 8, avg: 18.416666666666668, max: 50)\n",
      "\n",
      "Run: 49, exploration: 0.014838425699981627, score: 24\n",
      "Scores: (min: 8, avg: 18.53061224489796, max: 50)\n",
      "\n",
      "Run: 50, exploration: 0.013090759526015528, score: 26\n",
      "Scores: (min: 8, avg: 18.68, max: 50)\n",
      "\n",
      "Run: 51, exploration: 0.012576201611761997, score: 9\n",
      "Scores: (min: 8, avg: 18.49019607843137, max: 50)\n",
      "\n",
      "Run: 52, exploration: 0.011491188384178622, score: 19\n",
      "Scores: (min: 8, avg: 18.5, max: 50)\n",
      "\n",
      "Run: 53, exploration: 0.01, score: 49\n",
      "Scores: (min: 8, avg: 19.07547169811321, max: 50)\n",
      "\n",
      "Run: 54, exploration: 0.01, score: 34\n",
      "Scores: (min: 8, avg: 19.35185185185185, max: 50)\n",
      "\n",
      "Run: 55, exploration: 0.01, score: 20\n",
      "Scores: (min: 8, avg: 19.363636363636363, max: 50)\n",
      "\n",
      "Run: 56, exploration: 0.01, score: 47\n",
      "Scores: (min: 8, avg: 19.857142857142858, max: 50)\n",
      "\n",
      "Run: 57, exploration: 0.01, score: 27\n",
      "Scores: (min: 8, avg: 19.982456140350877, max: 50)\n",
      "\n",
      "Run: 58, exploration: 0.01, score: 35\n",
      "Scores: (min: 8, avg: 20.24137931034483, max: 50)\n",
      "\n",
      "Run: 59, exploration: 0.01, score: 26\n",
      "Scores: (min: 8, avg: 20.338983050847457, max: 50)\n",
      "\n",
      "Run: 60, exploration: 0.01, score: 28\n",
      "Scores: (min: 8, avg: 20.466666666666665, max: 50)\n",
      "\n",
      "Run: 61, exploration: 0.01, score: 30\n",
      "Scores: (min: 8, avg: 20.62295081967213, max: 50)\n",
      "\n",
      "Run: 62, exploration: 0.01, score: 38\n",
      "Scores: (min: 8, avg: 20.903225806451612, max: 50)\n",
      "\n",
      "Run: 63, exploration: 0.01, score: 31\n",
      "Scores: (min: 8, avg: 21.063492063492063, max: 50)\n",
      "\n",
      "Run: 64, exploration: 0.01, score: 20\n",
      "Scores: (min: 8, avg: 21.046875, max: 50)\n",
      "\n",
      "Run: 65, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 21.046153846153846, max: 50)\n",
      "\n",
      "Run: 66, exploration: 0.01, score: 22\n",
      "Scores: (min: 8, avg: 21.060606060606062, max: 50)\n",
      "\n",
      "Run: 67, exploration: 0.01, score: 20\n",
      "Scores: (min: 8, avg: 21.044776119402986, max: 50)\n",
      "\n",
      "Run: 68, exploration: 0.01, score: 23\n",
      "Scores: (min: 8, avg: 21.073529411764707, max: 50)\n",
      "\n",
      "Run: 69, exploration: 0.01, score: 26\n",
      "Scores: (min: 8, avg: 21.144927536231883, max: 50)\n",
      "\n",
      "Run: 70, exploration: 0.01, score: 28\n",
      "Scores: (min: 8, avg: 21.242857142857144, max: 50)\n",
      "\n",
      "Run: 71, exploration: 0.01, score: 24\n",
      "Scores: (min: 8, avg: 21.281690140845072, max: 50)\n",
      "\n",
      "Run: 72, exploration: 0.01, score: 23\n",
      "Scores: (min: 8, avg: 21.305555555555557, max: 50)\n",
      "\n",
      "Run: 73, exploration: 0.01, score: 19\n",
      "Scores: (min: 8, avg: 21.273972602739725, max: 50)\n",
      "\n",
      "Run: 74, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 21.18918918918919, max: 50)\n",
      "\n",
      "Run: 75, exploration: 0.01, score: 18\n",
      "Scores: (min: 8, avg: 21.14666666666667, max: 50)\n",
      "\n",
      "Run: 76, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 21.06578947368421, max: 50)\n",
      "\n",
      "Run: 77, exploration: 0.01, score: 24\n",
      "Scores: (min: 8, avg: 21.103896103896105, max: 50)\n",
      "\n",
      "Run: 78, exploration: 0.01, score: 20\n",
      "Scores: (min: 8, avg: 21.08974358974359, max: 50)\n",
      "\n",
      "Run: 79, exploration: 0.01, score: 16\n",
      "Scores: (min: 8, avg: 21.025316455696203, max: 50)\n",
      "\n",
      "Run: 80, exploration: 0.01, score: 18\n",
      "Scores: (min: 8, avg: 20.9875, max: 50)\n",
      "\n",
      "Run: 81, exploration: 0.01, score: 22\n",
      "Scores: (min: 8, avg: 21, max: 50)\n",
      "\n",
      "Run: 82, exploration: 0.01, score: 25\n",
      "Scores: (min: 8, avg: 21.048780487804876, max: 50)\n",
      "\n",
      "Run: 83, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 21.048192771084338, max: 50)\n",
      "\n",
      "Run: 84, exploration: 0.01, score: 19\n",
      "Scores: (min: 8, avg: 21.023809523809526, max: 50)\n",
      "\n",
      "Run: 85, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 20.95294117647059, max: 50)\n",
      "\n",
      "Run: 86, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 20.88372093023256, max: 50)\n",
      "\n",
      "Run: 87, exploration: 0.01, score: 18\n",
      "Scores: (min: 8, avg: 20.850574712643677, max: 50)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 88, exploration: 0.01, score: 18\n",
      "Scores: (min: 8, avg: 20.818181818181817, max: 50)\n",
      "\n",
      "Run: 89, exploration: 0.01, score: 25\n",
      "Scores: (min: 8, avg: 20.865168539325843, max: 50)\n",
      "\n",
      "Run: 90, exploration: 0.01, score: 27\n",
      "Scores: (min: 8, avg: 20.933333333333334, max: 50)\n",
      "\n",
      "Run: 91, exploration: 0.01, score: 25\n",
      "Scores: (min: 8, avg: 20.978021978021978, max: 50)\n",
      "\n",
      "Run: 92, exploration: 0.01, score: 20\n",
      "Scores: (min: 8, avg: 20.967391304347824, max: 50)\n",
      "\n",
      "Run: 93, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 20.967741935483872, max: 50)\n",
      "\n",
      "Run: 94, exploration: 0.01, score: 19\n",
      "Scores: (min: 8, avg: 20.9468085106383, max: 50)\n",
      "\n",
      "Run: 95, exploration: 0.01, score: 22\n",
      "Scores: (min: 8, avg: 20.957894736842107, max: 50)\n",
      "\n",
      "Run: 96, exploration: 0.01, score: 43\n",
      "Scores: (min: 8, avg: 21.1875, max: 50)\n",
      "\n",
      "Run: 97, exploration: 0.01, score: 24\n",
      "Scores: (min: 8, avg: 21.216494845360824, max: 50)\n",
      "\n",
      "Run: 98, exploration: 0.01, score: 18\n",
      "Scores: (min: 8, avg: 21.183673469387756, max: 50)\n",
      "\n",
      "Run: 99, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 21.181818181818183, max: 50)\n",
      "\n",
      "Run: 100, exploration: 0.01, score: 52\n",
      "Scores: (min: 8, avg: 21.49, max: 52)\n",
      "\n",
      "Run: 101, exploration: 0.01, score: 44\n",
      "Scores: (min: 8, avg: 21.7, max: 52)\n",
      "\n",
      "Run: 102, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 21.69, max: 52)\n",
      "\n",
      "Run: 103, exploration: 0.01, score: 40\n",
      "Scores: (min: 8, avg: 21.92, max: 52)\n",
      "\n",
      "Run: 104, exploration: 0.01, score: 59\n",
      "Scores: (min: 8, avg: 22.4, max: 59)\n",
      "\n",
      "Run: 105, exploration: 0.01, score: 41\n",
      "Scores: (min: 8, avg: 22.48, max: 59)\n",
      "\n",
      "Run: 106, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 22.49, max: 59)\n",
      "\n",
      "Run: 107, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 22.51, max: 59)\n",
      "\n",
      "Run: 108, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 22.56, max: 59)\n",
      "\n",
      "Run: 109, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 22.53, max: 59)\n",
      "\n",
      "Run: 110, exploration: 0.01, score: 26\n",
      "Scores: (min: 8, avg: 22.69, max: 59)\n",
      "\n",
      "Run: 111, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 22.77, max: 59)\n",
      "\n",
      "Run: 112, exploration: 0.01, score: 85\n",
      "Scores: (min: 8, avg: 23.4, max: 85)\n",
      "\n",
      "Run: 113, exploration: 0.01, score: 80\n",
      "Scores: (min: 8, avg: 24.09, max: 85)\n",
      "\n",
      "Run: 114, exploration: 0.01, score: 62\n",
      "Scores: (min: 8, avg: 24.61, max: 85)\n",
      "\n",
      "Run: 115, exploration: 0.01, score: 22\n",
      "Scores: (min: 8, avg: 24.69, max: 85)\n",
      "\n",
      "Run: 116, exploration: 0.01, score: 20\n",
      "Scores: (min: 8, avg: 24.78, max: 85)\n",
      "\n",
      "Run: 117, exploration: 0.01, score: 62\n",
      "Scores: (min: 8, avg: 25.27, max: 85)\n",
      "\n",
      "Run: 118, exploration: 0.01, score: 69\n",
      "Scores: (min: 8, avg: 25.85, max: 85)\n",
      "\n",
      "Run: 119, exploration: 0.01, score: 56\n",
      "Scores: (min: 8, avg: 26.29, max: 85)\n",
      "\n",
      "Run: 120, exploration: 0.01, score: 51\n",
      "Scores: (min: 8, avg: 26.67, max: 85)\n",
      "\n",
      "Run: 121, exploration: 0.01, score: 17\n",
      "Scores: (min: 8, avg: 26.74, max: 85)\n",
      "\n",
      "Run: 122, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 26.83, max: 85)\n",
      "\n",
      "Run: 123, exploration: 0.01, score: 21\n",
      "Scores: (min: 8, avg: 26.93, max: 85)\n",
      "\n",
      "Run: 124, exploration: 0.01, score: 89\n",
      "Scores: (min: 8, avg: 27.69, max: 89)\n",
      "\n",
      "Run: 125, exploration: 0.01, score: 31\n",
      "Scores: (min: 8, avg: 27.87, max: 89)\n",
      "\n",
      "Run: 126, exploration: 0.01, score: 81\n",
      "Scores: (min: 9, avg: 28.6, max: 89)\n",
      "\n",
      "Run: 127, exploration: 0.01, score: 106\n",
      "Scores: (min: 9, avg: 29.56, max: 106)\n",
      "\n",
      "Run: 128, exploration: 0.01, score: 159\n",
      "Scores: (min: 9, avg: 31.06, max: 159)\n",
      "\n",
      "Run: 129, exploration: 0.01, score: 143\n",
      "Scores: (min: 9, avg: 32.4, max: 159)\n",
      "\n",
      "Run: 130, exploration: 0.01, score: 170\n",
      "Scores: (min: 9, avg: 33.61, max: 170)\n",
      "\n",
      "Run: 131, exploration: 0.01, score: 175\n",
      "Scores: (min: 9, avg: 34.97, max: 175)\n",
      "\n",
      "Run: 132, exploration: 0.01, score: 152\n",
      "Scores: (min: 9, avg: 35.99, max: 175)\n",
      "\n",
      "Run: 133, exploration: 0.01, score: 158\n",
      "Scores: (min: 9, avg: 37.13, max: 175)\n",
      "\n",
      "Run: 134, exploration: 0.01, score: 152\n",
      "Scores: (min: 9, avg: 38.36, max: 175)\n",
      "\n",
      "Run: 135, exploration: 0.01, score: 151\n",
      "Scores: (min: 9, avg: 39.41, max: 175)\n",
      "\n",
      "Run: 136, exploration: 0.01, score: 216\n",
      "Scores: (min: 9, avg: 41.24, max: 216)\n",
      "\n",
      "Run: 137, exploration: 0.01, score: 17\n",
      "Scores: (min: 9, avg: 41.31, max: 216)\n",
      "\n",
      "Run: 138, exploration: 0.01, score: 171\n",
      "Scores: (min: 9, avg: 42.93, max: 216)\n",
      "\n",
      "Run: 139, exploration: 0.01, score: 9\n",
      "Scores: (min: 9, avg: 42.86, max: 216)\n",
      "\n",
      "Run: 140, exploration: 0.01, score: 9\n",
      "Scores: (min: 9, avg: 42.75, max: 216)\n",
      "\n",
      "Run: 141, exploration: 0.01, score: 10\n",
      "Scores: (min: 9, avg: 42.69, max: 216)\n",
      "\n",
      "Run: 142, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 42.56, max: 216)\n",
      "\n",
      "Run: 143, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 42.5, max: 216)\n",
      "\n",
      "Run: 144, exploration: 0.01, score: 166\n",
      "Scores: (min: 8, avg: 43.88, max: 216)\n",
      "\n",
      "Run: 145, exploration: 0.01, score: 138\n",
      "Scores: (min: 8, avg: 45, max: 216)\n",
      "\n",
      "Run: 146, exploration: 0.01, score: 170\n",
      "Scores: (min: 8, avg: 46.55, max: 216)\n",
      "\n",
      "Run: 147, exploration: 0.01, score: 176\n",
      "Scores: (min: 8, avg: 48.07, max: 216)\n",
      "\n",
      "Run: 148, exploration: 0.01, score: 57\n",
      "Scores: (min: 8, avg: 48.47, max: 216)\n",
      "\n",
      "Run: 149, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 48.33, max: 216)\n",
      "\n",
      "Run: 150, exploration: 0.01, score: 150\n",
      "Scores: (min: 8, avg: 49.57, max: 216)\n",
      "\n",
      "Run: 151, exploration: 0.01, score: 219\n",
      "Scores: (min: 8, avg: 51.67, max: 219)\n",
      "\n",
      "Run: 152, exploration: 0.01, score: 184\n",
      "Scores: (min: 8, avg: 53.32, max: 219)\n",
      "\n",
      "Run: 153, exploration: 0.01, score: 164\n",
      "Scores: (min: 8, avg: 54.47, max: 219)\n",
      "\n",
      "Run: 154, exploration: 0.01, score: 139\n",
      "Scores: (min: 8, avg: 55.52, max: 219)\n",
      "\n",
      "Run: 155, exploration: 0.01, score: 229\n",
      "Scores: (min: 8, avg: 57.61, max: 229)\n",
      "\n",
      "Run: 156, exploration: 0.01, score: 149\n",
      "Scores: (min: 8, avg: 58.63, max: 229)\n",
      "\n",
      "Run: 157, exploration: 0.01, score: 190\n",
      "Scores: (min: 8, avg: 60.26, max: 229)\n",
      "\n",
      "Run: 158, exploration: 0.01, score: 166\n",
      "Scores: (min: 8, avg: 61.57, max: 229)\n",
      "\n",
      "Run: 159, exploration: 0.01, score: 197\n",
      "Scores: (min: 8, avg: 63.28, max: 229)\n",
      "\n",
      "Run: 160, exploration: 0.01, score: 161\n",
      "Scores: (min: 8, avg: 64.61, max: 229)\n",
      "\n",
      "Run: 161, exploration: 0.01, score: 157\n",
      "Scores: (min: 8, avg: 65.88, max: 229)\n",
      "\n",
      "Run: 162, exploration: 0.01, score: 257\n",
      "Scores: (min: 8, avg: 68.07, max: 257)\n",
      "\n",
      "Run: 163, exploration: 0.01, score: 191\n",
      "Scores: (min: 8, avg: 69.67, max: 257)\n",
      "\n",
      "Run: 164, exploration: 0.01, score: 210\n",
      "Scores: (min: 8, avg: 71.57, max: 257)\n",
      "\n",
      "Run: 165, exploration: 0.01, score: 190\n",
      "Scores: (min: 8, avg: 73.26, max: 257)\n",
      "\n",
      "Run: 166, exploration: 0.01, score: 156\n",
      "Scores: (min: 8, avg: 74.6, max: 257)\n",
      "\n",
      "Run: 167, exploration: 0.01, score: 170\n",
      "Scores: (min: 8, avg: 76.1, max: 257)\n",
      "\n",
      "Run: 168, exploration: 0.01, score: 44\n",
      "Scores: (min: 8, avg: 76.31, max: 257)\n",
      "\n",
      "Run: 169, exploration: 0.01, score: 164\n",
      "Scores: (min: 8, avg: 77.69, max: 257)\n",
      "\n",
      "Run: 170, exploration: 0.01, score: 283\n",
      "Scores: (min: 8, avg: 80.24, max: 283)\n",
      "\n",
      "Run: 171, exploration: 0.01, score: 183\n",
      "Scores: (min: 8, avg: 81.83, max: 283)\n",
      "\n",
      "Run: 172, exploration: 0.01, score: 215\n",
      "Scores: (min: 8, avg: 83.75, max: 283)\n",
      "\n",
      "Run: 173, exploration: 0.01, score: 182\n",
      "Scores: (min: 8, avg: 85.38, max: 283)\n",
      "\n",
      "Run: 174, exploration: 0.01, score: 194\n",
      "Scores: (min: 8, avg: 87.17, max: 283)\n",
      "\n",
      "Run: 175, exploration: 0.01, score: 163\n",
      "Scores: (min: 8, avg: 88.62, max: 283)\n",
      "\n",
      "Run: 176, exploration: 0.01, score: 475\n",
      "Scores: (min: 8, avg: 93.22, max: 475)\n",
      "\n",
      "Run: 177, exploration: 0.01, score: 192\n",
      "Scores: (min: 8, avg: 94.9, max: 475)\n",
      "\n",
      "Run: 178, exploration: 0.01, score: 189\n",
      "Scores: (min: 8, avg: 96.59, max: 475)\n",
      "\n",
      "Run: 179, exploration: 0.01, score: 374\n",
      "Scores: (min: 8, avg: 100.17, max: 475)\n",
      "\n",
      "Run: 180, exploration: 0.01, score: 195\n",
      "Scores: (min: 8, avg: 101.94, max: 475)\n",
      "\n",
      "Run: 181, exploration: 0.01, score: 183\n",
      "Scores: (min: 8, avg: 103.55, max: 475)\n",
      "\n",
      "Run: 182, exploration: 0.01, score: 202\n",
      "Scores: (min: 8, avg: 105.32, max: 475)\n",
      "\n",
      "Run: 183, exploration: 0.01, score: 483\n",
      "Scores: (min: 8, avg: 109.94, max: 483)\n",
      "\n",
      "Run: 184, exploration: 0.01, score: 272\n",
      "Scores: (min: 8, avg: 112.47, max: 483)\n",
      "\n",
      "Run: 185, exploration: 0.01, score: 27\n",
      "Scores: (min: 8, avg: 112.59, max: 483)\n",
      "\n",
      "Run: 186, exploration: 0.01, score: 209\n",
      "Scores: (min: 8, avg: 114.53, max: 483)\n",
      "\n",
      "Run: 187, exploration: 0.01, score: 196\n",
      "Scores: (min: 8, avg: 116.31, max: 483)\n",
      "\n",
      "Run: 188, exploration: 0.01, score: 157\n",
      "Scores: (min: 8, avg: 117.7, max: 483)\n",
      "\n",
      "Run: 189, exploration: 0.01, score: 85\n",
      "Scores: (min: 8, avg: 118.3, max: 483)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 190, exploration: 0.01, score: 225\n",
      "Scores: (min: 8, avg: 120.28, max: 483)\n",
      "\n",
      "Run: 191, exploration: 0.01, score: 171\n",
      "Scores: (min: 8, avg: 121.74, max: 483)\n",
      "\n",
      "Run: 192, exploration: 0.01, score: 150\n",
      "Scores: (min: 8, avg: 123.04, max: 483)\n",
      "\n",
      "Run: 193, exploration: 0.01, score: 141\n",
      "Scores: (min: 8, avg: 124.24, max: 483)\n",
      "\n",
      "Run: 194, exploration: 0.01, score: 165\n",
      "Scores: (min: 8, avg: 125.7, max: 483)\n",
      "\n",
      "Run: 195, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 125.6, max: 483)\n",
      "\n",
      "Run: 196, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 125.27, max: 483)\n",
      "\n",
      "Run: 197, exploration: 0.01, score: 169\n",
      "Scores: (min: 8, avg: 126.72, max: 483)\n",
      "\n",
      "Run: 198, exploration: 0.01, score: 230\n",
      "Scores: (min: 8, avg: 128.84, max: 483)\n",
      "\n",
      "Run: 199, exploration: 0.01, score: 148\n",
      "Scores: (min: 8, avg: 130.11, max: 483)\n",
      "\n",
      "Run: 200, exploration: 0.01, score: 170\n",
      "Scores: (min: 8, avg: 131.29, max: 483)\n",
      "\n",
      "Run: 201, exploration: 0.01, score: 141\n",
      "Scores: (min: 8, avg: 132.26, max: 483)\n",
      "\n",
      "Run: 202, exploration: 0.01, score: 184\n",
      "Scores: (min: 8, avg: 133.95, max: 483)\n",
      "\n",
      "Run: 203, exploration: 0.01, score: 174\n",
      "Scores: (min: 8, avg: 135.29, max: 483)\n",
      "\n",
      "Run: 204, exploration: 0.01, score: 225\n",
      "Scores: (min: 8, avg: 136.95, max: 483)\n",
      "\n",
      "Run: 205, exploration: 0.01, score: 253\n",
      "Scores: (min: 8, avg: 139.07, max: 483)\n",
      "\n",
      "Run: 206, exploration: 0.01, score: 248\n",
      "Scores: (min: 8, avg: 141.44, max: 483)\n",
      "\n",
      "Run: 207, exploration: 0.01, score: 162\n",
      "Scores: (min: 8, avg: 142.94, max: 483)\n",
      "\n",
      "Run: 208, exploration: 0.01, score: 173\n",
      "Scores: (min: 8, avg: 144.54, max: 483)\n",
      "\n",
      "Run: 209, exploration: 0.01, score: 174\n",
      "Scores: (min: 8, avg: 146.17, max: 483)\n",
      "\n",
      "Run: 210, exploration: 0.01, score: 153\n",
      "Scores: (min: 8, avg: 147.44, max: 483)\n",
      "\n",
      "Run: 211, exploration: 0.01, score: 161\n",
      "Scores: (min: 8, avg: 148.84, max: 483)\n",
      "\n",
      "Run: 212, exploration: 0.01, score: 142\n",
      "Scores: (min: 8, avg: 149.41, max: 483)\n",
      "\n",
      "Run: 213, exploration: 0.01, score: 213\n",
      "Scores: (min: 8, avg: 150.74, max: 483)\n",
      "\n",
      "Run: 214, exploration: 0.01, score: 197\n",
      "Scores: (min: 8, avg: 152.09, max: 483)\n",
      "\n",
      "Run: 215, exploration: 0.01, score: 201\n",
      "Scores: (min: 8, avg: 153.88, max: 483)\n",
      "\n",
      "Run: 216, exploration: 0.01, score: 122\n",
      "Scores: (min: 8, avg: 154.9, max: 483)\n",
      "\n",
      "Run: 217, exploration: 0.01, score: 287\n",
      "Scores: (min: 8, avg: 157.15, max: 483)\n",
      "\n",
      "Run: 218, exploration: 0.01, score: 229\n",
      "Scores: (min: 8, avg: 158.75, max: 483)\n",
      "\n",
      "Run: 219, exploration: 0.01, score: 165\n",
      "Scores: (min: 8, avg: 159.84, max: 483)\n",
      "\n",
      "Run: 220, exploration: 0.01, score: 154\n",
      "Scores: (min: 8, avg: 160.87, max: 483)\n",
      "\n",
      "Run: 221, exploration: 0.01, score: 192\n",
      "Scores: (min: 8, avg: 162.62, max: 483)\n",
      "\n",
      "Run: 222, exploration: 0.01, score: 223\n",
      "Scores: (min: 8, avg: 164.64, max: 483)\n",
      "\n",
      "Run: 223, exploration: 0.01, score: 159\n",
      "Scores: (min: 8, avg: 166.02, max: 483)\n",
      "\n",
      "Run: 224, exploration: 0.01, score: 222\n",
      "Scores: (min: 8, avg: 167.35, max: 483)\n",
      "\n",
      "Run: 225, exploration: 0.01, score: 163\n",
      "Scores: (min: 8, avg: 168.67, max: 483)\n",
      "\n",
      "Run: 226, exploration: 0.01, score: 173\n",
      "Scores: (min: 8, avg: 169.59, max: 483)\n",
      "\n",
      "Run: 227, exploration: 0.01, score: 149\n",
      "Scores: (min: 8, avg: 170.02, max: 483)\n",
      "\n",
      "Run: 228, exploration: 0.01, score: 176\n",
      "Scores: (min: 8, avg: 170.19, max: 483)\n",
      "\n",
      "Run: 229, exploration: 0.01, score: 216\n",
      "Scores: (min: 8, avg: 170.92, max: 483)\n",
      "\n",
      "Run: 230, exploration: 0.01, score: 166\n",
      "Scores: (min: 8, avg: 170.88, max: 483)\n",
      "\n",
      "Run: 231, exploration: 0.01, score: 204\n",
      "Scores: (min: 8, avg: 171.17, max: 483)\n",
      "\n",
      "Run: 232, exploration: 0.01, score: 268\n",
      "Scores: (min: 8, avg: 172.33, max: 483)\n",
      "\n",
      "Run: 233, exploration: 0.01, score: 277\n",
      "Scores: (min: 8, avg: 173.52, max: 483)\n",
      "\n",
      "Run: 234, exploration: 0.01, score: 177\n",
      "Scores: (min: 8, avg: 173.77, max: 483)\n",
      "\n",
      "Run: 235, exploration: 0.01, score: 319\n",
      "Scores: (min: 8, avg: 175.45, max: 483)\n",
      "\n",
      "Run: 236, exploration: 0.01, score: 180\n",
      "Scores: (min: 8, avg: 175.09, max: 483)\n",
      "\n",
      "Run: 237, exploration: 0.01, score: 281\n",
      "Scores: (min: 8, avg: 177.73, max: 483)\n",
      "\n",
      "Run: 238, exploration: 0.01, score: 221\n",
      "Scores: (min: 8, avg: 178.23, max: 483)\n",
      "\n",
      "Run: 239, exploration: 0.01, score: 154\n",
      "Scores: (min: 8, avg: 179.68, max: 483)\n",
      "\n",
      "Run: 240, exploration: 0.01, score: 101\n",
      "Scores: (min: 8, avg: 180.6, max: 483)\n",
      "\n",
      "Run: 241, exploration: 0.01, score: 187\n",
      "Scores: (min: 8, avg: 182.37, max: 483)\n",
      "\n",
      "Run: 242, exploration: 0.01, score: 191\n",
      "Scores: (min: 9, avg: 184.2, max: 483)\n",
      "\n",
      "Run: 243, exploration: 0.01, score: 231\n",
      "Scores: (min: 10, avg: 186.42, max: 483)\n",
      "\n",
      "Run: 244, exploration: 0.01, score: 213\n",
      "Scores: (min: 10, avg: 186.89, max: 483)\n",
      "\n",
      "Run: 245, exploration: 0.01, score: 209\n",
      "Scores: (min: 10, avg: 187.6, max: 483)\n",
      "\n",
      "Run: 246, exploration: 0.01, score: 175\n",
      "Scores: (min: 10, avg: 187.65, max: 483)\n",
      "\n",
      "Run: 247, exploration: 0.01, score: 138\n",
      "Scores: (min: 10, avg: 187.27, max: 483)\n",
      "\n",
      "Run: 248, exploration: 0.01, score: 228\n",
      "Scores: (min: 10, avg: 188.98, max: 483)\n",
      "\n",
      "Run: 249, exploration: 0.01, score: 123\n",
      "Scores: (min: 10, avg: 190.11, max: 483)\n",
      "\n",
      "Run: 250, exploration: 0.01, score: 131\n",
      "Scores: (min: 10, avg: 189.92, max: 483)\n",
      "\n",
      "Run: 251, exploration: 0.01, score: 229\n",
      "Scores: (min: 10, avg: 190.02, max: 483)\n",
      "\n",
      "Run: 252, exploration: 0.01, score: 200\n",
      "Scores: (min: 10, avg: 190.18, max: 483)\n",
      "\n",
      "Run: 253, exploration: 0.01, score: 180\n",
      "Scores: (min: 10, avg: 190.34, max: 483)\n",
      "\n",
      "Run: 254, exploration: 0.01, score: 127\n",
      "Scores: (min: 10, avg: 190.22, max: 483)\n",
      "\n",
      "Run: 255, exploration: 0.01, score: 163\n",
      "Scores: (min: 10, avg: 189.56, max: 483)\n",
      "\n",
      "Run: 256, exploration: 0.01, score: 188\n",
      "Scores: (min: 10, avg: 189.95, max: 483)\n",
      "\n",
      "Run: 257, exploration: 0.01, score: 173\n",
      "Scores: (min: 10, avg: 189.78, max: 483)\n",
      "\n",
      "Run: 258, exploration: 0.01, score: 209\n",
      "Scores: (min: 10, avg: 190.21, max: 483)\n",
      "\n",
      "Run: 259, exploration: 0.01, score: 213\n",
      "Scores: (min: 10, avg: 190.37, max: 483)\n",
      "\n",
      "Run: 260, exploration: 0.01, score: 189\n",
      "Scores: (min: 10, avg: 190.65, max: 483)\n",
      "\n",
      "Run: 261, exploration: 0.01, score: 183\n",
      "Scores: (min: 10, avg: 190.91, max: 483)\n",
      "\n",
      "Run: 262, exploration: 0.01, score: 156\n",
      "Scores: (min: 10, avg: 189.9, max: 483)\n",
      "\n",
      "Run: 263, exploration: 0.01, score: 196\n",
      "Scores: (min: 10, avg: 189.95, max: 483)\n",
      "\n",
      "Run: 264, exploration: 0.01, score: 157\n",
      "Scores: (min: 10, avg: 189.42, max: 483)\n",
      "\n",
      "Run: 265, exploration: 0.01, score: 135\n",
      "Scores: (min: 10, avg: 188.87, max: 483)\n",
      "\n",
      "Run: 266, exploration: 0.01, score: 139\n",
      "Scores: (min: 10, avg: 188.7, max: 483)\n",
      "\n",
      "Run: 267, exploration: 0.01, score: 153\n",
      "Scores: (min: 10, avg: 188.53, max: 483)\n",
      "\n",
      "Run: 268, exploration: 0.01, score: 123\n",
      "Scores: (min: 10, avg: 189.32, max: 483)\n",
      "\n",
      "Run: 269, exploration: 0.01, score: 159\n",
      "Scores: (min: 10, avg: 189.27, max: 483)\n",
      "\n",
      "Run: 270, exploration: 0.01, score: 128\n",
      "Scores: (min: 10, avg: 187.72, max: 483)\n",
      "\n",
      "Run: 271, exploration: 0.01, score: 158\n",
      "Scores: (min: 10, avg: 187.47, max: 483)\n",
      "\n",
      "Run: 272, exploration: 0.01, score: 144\n",
      "Scores: (min: 10, avg: 186.76, max: 483)\n",
      "\n",
      "Run: 273, exploration: 0.01, score: 135\n",
      "Scores: (min: 10, avg: 186.29, max: 483)\n",
      "\n",
      "Run: 274, exploration: 0.01, score: 125\n",
      "Scores: (min: 10, avg: 185.6, max: 483)\n",
      "\n",
      "Run: 275, exploration: 0.01, score: 135\n",
      "Scores: (min: 10, avg: 185.32, max: 483)\n",
      "\n",
      "Run: 276, exploration: 0.01, score: 142\n",
      "Scores: (min: 10, avg: 181.99, max: 483)\n",
      "\n",
      "Run: 277, exploration: 0.01, score: 165\n",
      "Scores: (min: 10, avg: 181.72, max: 483)\n",
      "\n",
      "Run: 278, exploration: 0.01, score: 162\n",
      "Scores: (min: 10, avg: 181.45, max: 483)\n",
      "\n",
      "Run: 279, exploration: 0.01, score: 267\n",
      "Scores: (min: 10, avg: 180.38, max: 483)\n",
      "\n",
      "Run: 280, exploration: 0.01, score: 148\n",
      "Scores: (min: 10, avg: 179.91, max: 483)\n",
      "\n",
      "Run: 281, exploration: 0.01, score: 264\n",
      "Scores: (min: 10, avg: 180.72, max: 483)\n",
      "\n",
      "Run: 282, exploration: 0.01, score: 220\n",
      "Scores: (min: 10, avg: 180.9, max: 483)\n",
      "\n",
      "Run: 283, exploration: 0.01, score: 188\n",
      "Scores: (min: 10, avg: 177.95, max: 319)\n",
      "\n",
      "Run: 284, exploration: 0.01, score: 155\n",
      "Scores: (min: 10, avg: 176.78, max: 319)\n",
      "\n",
      "Run: 285, exploration: 0.01, score: 177\n",
      "Scores: (min: 10, avg: 178.28, max: 319)\n",
      "\n",
      "Run: 286, exploration: 0.01, score: 217\n",
      "Scores: (min: 10, avg: 178.36, max: 319)\n",
      "\n",
      "Run: 287, exploration: 0.01, score: 223\n",
      "Scores: (min: 10, avg: 178.63, max: 319)\n",
      "\n",
      "Run: 288, exploration: 0.01, score: 158\n",
      "Scores: (min: 10, avg: 178.64, max: 319)\n",
      "\n",
      "Run: 289, exploration: 0.01, score: 298\n",
      "Scores: (min: 10, avg: 180.77, max: 319)\n",
      "\n",
      "Run: 290, exploration: 0.01, score: 192\n",
      "Scores: (min: 10, avg: 180.44, max: 319)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 291, exploration: 0.01, score: 181\n",
      "Scores: (min: 10, avg: 180.54, max: 319)\n",
      "\n",
      "Run: 292, exploration: 0.01, score: 196\n",
      "Scores: (min: 10, avg: 181, max: 319)\n",
      "\n",
      "Run: 293, exploration: 0.01, score: 147\n",
      "Scores: (min: 10, avg: 181.06, max: 319)\n",
      "\n",
      "Run: 294, exploration: 0.01, score: 218\n",
      "Scores: (min: 10, avg: 181.59, max: 319)\n",
      "\n",
      "Run: 295, exploration: 0.01, score: 241\n",
      "Scores: (min: 10, avg: 183.88, max: 319)\n",
      "\n",
      "Run: 296, exploration: 0.01, score: 189\n",
      "Scores: (min: 101, avg: 185.67, max: 319)\n",
      "\n",
      "Run: 297, exploration: 0.01, score: 130\n",
      "Scores: (min: 101, avg: 185.28, max: 319)\n",
      "\n",
      "Run: 298, exploration: 0.01, score: 119\n",
      "Scores: (min: 101, avg: 184.17, max: 319)\n",
      "\n",
      "Run: 299, exploration: 0.01, score: 167\n",
      "Scores: (min: 101, avg: 184.36, max: 319)\n",
      "\n",
      "Run: 300, exploration: 0.01, score: 196\n",
      "Scores: (min: 101, avg: 184.62, max: 319)\n",
      "\n",
      "Run: 301, exploration: 0.01, score: 323\n",
      "Scores: (min: 101, avg: 186.44, max: 323)\n",
      "\n",
      "Run: 302, exploration: 0.01, score: 153\n",
      "Scores: (min: 101, avg: 186.13, max: 323)\n",
      "\n",
      "Run: 303, exploration: 0.01, score: 244\n",
      "Scores: (min: 101, avg: 186.83, max: 323)\n",
      "\n",
      "Run: 304, exploration: 0.01, score: 217\n",
      "Scores: (min: 101, avg: 186.75, max: 323)\n",
      "\n",
      "Run: 305, exploration: 0.01, score: 208\n",
      "Scores: (min: 101, avg: 186.3, max: 323)\n",
      "\n",
      "Run: 306, exploration: 0.01, score: 120\n",
      "Scores: (min: 101, avg: 185.02, max: 323)\n",
      "\n",
      "Run: 307, exploration: 0.01, score: 185\n",
      "Scores: (min: 101, avg: 185.25, max: 323)\n",
      "\n",
      "Run: 308, exploration: 0.01, score: 180\n",
      "Scores: (min: 101, avg: 185.32, max: 323)\n",
      "\n",
      "Run: 309, exploration: 0.01, score: 244\n",
      "Scores: (min: 101, avg: 186.02, max: 323)\n",
      "\n",
      "Run: 310, exploration: 0.01, score: 162\n",
      "Scores: (min: 101, avg: 186.11, max: 323)\n",
      "\n",
      "Run: 311, exploration: 0.01, score: 132\n",
      "Scores: (min: 101, avg: 185.82, max: 323)\n",
      "\n",
      "Run: 312, exploration: 0.01, score: 145\n",
      "Scores: (min: 101, avg: 185.85, max: 323)\n",
      "\n",
      "Run: 313, exploration: 0.01, score: 124\n",
      "Scores: (min: 101, avg: 184.96, max: 323)\n",
      "\n",
      "Run: 314, exploration: 0.01, score: 175\n",
      "Scores: (min: 101, avg: 184.74, max: 323)\n",
      "\n",
      "Run: 315, exploration: 0.01, score: 413\n",
      "Scores: (min: 101, avg: 186.86, max: 413)\n",
      "\n",
      "Run: 316, exploration: 0.01, score: 116\n",
      "Scores: (min: 101, avg: 186.8, max: 413)\n",
      "\n",
      "Run: 317, exploration: 0.01, score: 163\n",
      "Scores: (min: 101, avg: 185.56, max: 413)\n",
      "\n",
      "Run: 318, exploration: 0.01, score: 119\n",
      "Scores: (min: 101, avg: 184.46, max: 413)\n",
      "\n",
      "Run: 319, exploration: 0.01, score: 342\n",
      "Scores: (min: 101, avg: 186.23, max: 413)\n",
      "\n",
      "Run: 320, exploration: 0.01, score: 206\n",
      "Scores: (min: 101, avg: 186.75, max: 413)\n",
      "\n",
      "Run: 321, exploration: 0.01, score: 304\n",
      "Scores: (min: 101, avg: 187.87, max: 413)\n",
      "\n",
      "Run: 322, exploration: 0.01, score: 70\n",
      "Scores: (min: 70, avg: 186.34, max: 413)\n",
      "\n",
      "Run: 323, exploration: 0.01, score: 133\n",
      "Scores: (min: 70, avg: 186.08, max: 413)\n",
      "\n",
      "Run: 324, exploration: 0.01, score: 148\n",
      "Scores: (min: 70, avg: 185.34, max: 413)\n",
      "\n",
      "Run: 325, exploration: 0.01, score: 161\n",
      "Scores: (min: 70, avg: 185.32, max: 413)\n",
      "\n",
      "Run: 326, exploration: 0.01, score: 165\n",
      "Scores: (min: 70, avg: 185.24, max: 413)\n",
      "\n",
      "Run: 327, exploration: 0.01, score: 139\n",
      "Scores: (min: 70, avg: 185.14, max: 413)\n",
      "\n",
      "Run: 328, exploration: 0.01, score: 315\n",
      "Scores: (min: 70, avg: 186.53, max: 413)\n",
      "\n",
      "Run: 329, exploration: 0.01, score: 328\n",
      "Scores: (min: 70, avg: 187.65, max: 413)\n",
      "\n",
      "Run: 330, exploration: 0.01, score: 221\n",
      "Scores: (min: 70, avg: 188.2, max: 413)\n",
      "\n",
      "Run: 331, exploration: 0.01, score: 234\n",
      "Scores: (min: 70, avg: 188.5, max: 413)\n",
      "\n",
      "Run: 332, exploration: 0.01, score: 193\n",
      "Scores: (min: 70, avg: 187.75, max: 413)\n",
      "\n",
      "Run: 333, exploration: 0.01, score: 202\n",
      "Scores: (min: 70, avg: 187, max: 413)\n",
      "\n",
      "Run: 334, exploration: 0.01, score: 194\n",
      "Scores: (min: 70, avg: 187.17, max: 413)\n",
      "\n",
      "Run: 335, exploration: 0.01, score: 209\n",
      "Scores: (min: 70, avg: 186.07, max: 413)\n",
      "\n",
      "Run: 336, exploration: 0.01, score: 217\n",
      "Scores: (min: 70, avg: 186.44, max: 413)\n",
      "\n",
      "Run: 337, exploration: 0.01, score: 176\n",
      "Scores: (min: 70, avg: 185.39, max: 413)\n",
      "\n",
      "Run: 338, exploration: 0.01, score: 192\n",
      "Scores: (min: 70, avg: 185.1, max: 413)\n",
      "\n",
      "Run: 339, exploration: 0.01, score: 40\n",
      "Scores: (min: 40, avg: 183.96, max: 413)\n",
      "\n",
      "Run: 340, exploration: 0.01, score: 9\n",
      "Scores: (min: 9, avg: 183.04, max: 413)\n",
      "\n",
      "Run: 341, exploration: 0.01, score: 9\n",
      "Scores: (min: 9, avg: 181.26, max: 413)\n",
      "\n",
      "Run: 342, exploration: 0.01, score: 10\n",
      "Scores: (min: 9, avg: 179.45, max: 413)\n",
      "\n",
      "Run: 343, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 177.22, max: 413)\n",
      "\n",
      "Run: 344, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 175.19, max: 413)\n",
      "\n",
      "Run: 345, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 173.2, max: 413)\n",
      "\n",
      "Run: 346, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 171.55, max: 413)\n",
      "\n",
      "Run: 347, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 170.29, max: 413)\n",
      "\n",
      "Run: 348, exploration: 0.01, score: 159\n",
      "Scores: (min: 8, avg: 169.6, max: 413)\n",
      "\n",
      "Run: 349, exploration: 0.01, score: 191\n",
      "Scores: (min: 8, avg: 170.28, max: 413)\n",
      "\n",
      "Run: 350, exploration: 0.01, score: 197\n",
      "Scores: (min: 8, avg: 170.94, max: 413)\n",
      "\n",
      "Run: 351, exploration: 0.01, score: 170\n",
      "Scores: (min: 8, avg: 170.35, max: 413)\n",
      "\n",
      "Run: 352, exploration: 0.01, score: 177\n",
      "Scores: (min: 8, avg: 170.12, max: 413)\n",
      "\n",
      "Run: 353, exploration: 0.01, score: 149\n",
      "Scores: (min: 8, avg: 169.81, max: 413)\n",
      "\n",
      "Run: 354, exploration: 0.01, score: 263\n",
      "Scores: (min: 8, avg: 171.17, max: 413)\n",
      "\n",
      "Run: 355, exploration: 0.01, score: 187\n",
      "Scores: (min: 8, avg: 171.41, max: 413)\n",
      "\n",
      "Run: 356, exploration: 0.01, score: 184\n",
      "Scores: (min: 8, avg: 171.37, max: 413)\n",
      "\n",
      "Run: 357, exploration: 0.01, score: 265\n",
      "Scores: (min: 8, avg: 172.29, max: 413)\n",
      "\n",
      "Run: 358, exploration: 0.01, score: 126\n",
      "Scores: (min: 8, avg: 171.46, max: 413)\n",
      "\n",
      "Run: 359, exploration: 0.01, score: 147\n",
      "Scores: (min: 8, avg: 170.8, max: 413)\n",
      "\n",
      "Run: 360, exploration: 0.01, score: 173\n",
      "Scores: (min: 8, avg: 170.64, max: 413)\n",
      "\n",
      "Run: 361, exploration: 0.01, score: 74\n",
      "Scores: (min: 8, avg: 169.55, max: 413)\n",
      "\n",
      "Run: 362, exploration: 0.01, score: 289\n",
      "Scores: (min: 8, avg: 170.88, max: 413)\n",
      "\n",
      "Run: 363, exploration: 0.01, score: 336\n",
      "Scores: (min: 8, avg: 172.28, max: 413)\n",
      "\n",
      "Run: 364, exploration: 0.01, score: 247\n",
      "Scores: (min: 8, avg: 173.18, max: 413)\n",
      "\n",
      "Run: 365, exploration: 0.01, score: 274\n",
      "Scores: (min: 8, avg: 174.57, max: 413)\n",
      "\n",
      "Run: 366, exploration: 0.01, score: 276\n",
      "Scores: (min: 8, avg: 175.94, max: 413)\n",
      "\n",
      "Run: 367, exploration: 0.01, score: 227\n",
      "Scores: (min: 8, avg: 176.68, max: 413)\n",
      "\n",
      "Run: 368, exploration: 0.01, score: 197\n",
      "Scores: (min: 8, avg: 177.42, max: 413)\n",
      "\n",
      "Run: 369, exploration: 0.01, score: 292\n",
      "Scores: (min: 8, avg: 178.75, max: 413)\n",
      "\n",
      "Run: 370, exploration: 0.01, score: 359\n",
      "Scores: (min: 8, avg: 181.06, max: 413)\n",
      "\n",
      "Run: 371, exploration: 0.01, score: 93\n",
      "Scores: (min: 8, avg: 180.41, max: 413)\n",
      "\n",
      "Run: 372, exploration: 0.01, score: 406\n",
      "Scores: (min: 8, avg: 183.03, max: 413)\n",
      "\n",
      "Run: 373, exploration: 0.01, score: 133\n",
      "Scores: (min: 8, avg: 183.01, max: 413)\n",
      "\n",
      "Run: 374, exploration: 0.01, score: 191\n",
      "Scores: (min: 8, avg: 183.67, max: 413)\n",
      "\n",
      "Run: 375, exploration: 0.01, score: 259\n",
      "Scores: (min: 8, avg: 184.91, max: 413)\n",
      "\n",
      "Run: 376, exploration: 0.01, score: 152\n",
      "Scores: (min: 8, avg: 185.01, max: 413)\n",
      "\n",
      "Run: 377, exploration: 0.01, score: 177\n",
      "Scores: (min: 8, avg: 185.13, max: 413)\n",
      "\n",
      "Run: 378, exploration: 0.01, score: 220\n",
      "Scores: (min: 8, avg: 185.71, max: 413)\n",
      "\n",
      "Run: 379, exploration: 0.01, score: 190\n",
      "Scores: (min: 8, avg: 184.94, max: 413)\n",
      "\n",
      "Run: 380, exploration: 0.01, score: 153\n",
      "Scores: (min: 8, avg: 184.99, max: 413)\n",
      "\n",
      "Run: 381, exploration: 0.01, score: 213\n",
      "Scores: (min: 8, avg: 184.48, max: 413)\n",
      "\n",
      "Run: 382, exploration: 0.01, score: 142\n",
      "Scores: (min: 8, avg: 183.7, max: 413)\n",
      "\n",
      "Run: 383, exploration: 0.01, score: 334\n",
      "Scores: (min: 8, avg: 185.16, max: 413)\n",
      "\n",
      "Run: 384, exploration: 0.01, score: 200\n",
      "Scores: (min: 8, avg: 185.61, max: 413)\n",
      "\n",
      "Run: 385, exploration: 0.01, score: 239\n",
      "Scores: (min: 8, avg: 186.23, max: 413)\n",
      "\n",
      "Run: 386, exploration: 0.01, score: 150\n",
      "Scores: (min: 8, avg: 185.56, max: 413)\n",
      "\n",
      "Run: 387, exploration: 0.01, score: 273\n",
      "Scores: (min: 8, avg: 186.06, max: 413)\n",
      "\n",
      "Run: 388, exploration: 0.01, score: 163\n",
      "Scores: (min: 8, avg: 186.11, max: 413)\n",
      "\n",
      "Run: 389, exploration: 0.01, score: 156\n",
      "Scores: (min: 8, avg: 184.69, max: 413)\n",
      "\n",
      "Run: 390, exploration: 0.01, score: 142\n",
      "Scores: (min: 8, avg: 184.19, max: 413)\n",
      "\n",
      "Run: 391, exploration: 0.01, score: 267\n",
      "Scores: (min: 8, avg: 185.05, max: 413)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 392, exploration: 0.01, score: 228\n",
      "Scores: (min: 8, avg: 185.37, max: 413)\n",
      "\n",
      "Run: 393, exploration: 0.01, score: 223\n",
      "Scores: (min: 8, avg: 186.13, max: 413)\n",
      "\n",
      "Run: 394, exploration: 0.01, score: 305\n",
      "Scores: (min: 8, avg: 187, max: 413)\n",
      "\n",
      "Run: 395, exploration: 0.01, score: 243\n",
      "Scores: (min: 8, avg: 187.02, max: 413)\n",
      "\n",
      "Run: 396, exploration: 0.01, score: 146\n",
      "Scores: (min: 8, avg: 186.59, max: 413)\n",
      "\n",
      "Run: 397, exploration: 0.01, score: 147\n",
      "Scores: (min: 8, avg: 186.76, max: 413)\n",
      "\n",
      "Run: 398, exploration: 0.01, score: 189\n",
      "Scores: (min: 8, avg: 187.46, max: 413)\n",
      "\n",
      "Run: 399, exploration: 0.01, score: 145\n",
      "Scores: (min: 8, avg: 187.24, max: 413)\n",
      "\n",
      "Run: 400, exploration: 0.01, score: 128\n",
      "Scores: (min: 8, avg: 186.56, max: 413)\n",
      "\n",
      "Run: 401, exploration: 0.01, score: 183\n",
      "Scores: (min: 8, avg: 185.16, max: 413)\n",
      "\n",
      "Run: 402, exploration: 0.01, score: 210\n",
      "Scores: (min: 8, avg: 185.73, max: 413)\n",
      "\n",
      "Run: 403, exploration: 0.01, score: 231\n",
      "Scores: (min: 8, avg: 185.6, max: 413)\n",
      "\n",
      "Run: 404, exploration: 0.01, score: 205\n",
      "Scores: (min: 8, avg: 185.48, max: 413)\n",
      "\n",
      "Run: 405, exploration: 0.01, score: 229\n",
      "Scores: (min: 8, avg: 185.69, max: 413)\n",
      "\n",
      "Run: 406, exploration: 0.01, score: 500\n",
      "Scores: (min: 8, avg: 189.49, max: 500)\n",
      "\n",
      "Run: 407, exploration: 0.01, score: 245\n",
      "Scores: (min: 8, avg: 190.09, max: 500)\n",
      "\n",
      "Run: 408, exploration: 0.01, score: 231\n",
      "Scores: (min: 8, avg: 190.6, max: 500)\n",
      "\n",
      "Run: 409, exploration: 0.01, score: 290\n",
      "Scores: (min: 8, avg: 191.06, max: 500)\n",
      "\n",
      "Run: 410, exploration: 0.01, score: 408\n",
      "Scores: (min: 8, avg: 193.52, max: 500)\n",
      "\n",
      "Run: 411, exploration: 0.01, score: 165\n",
      "Scores: (min: 8, avg: 193.85, max: 500)\n",
      "\n",
      "Run: 412, exploration: 0.01, score: 382\n",
      "Scores: (min: 8, avg: 196.22, max: 500)\n",
      "\n",
      "Solved in 312 runs, 412 total runs.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'exit' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-2-e23913c48dc8>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mcartpole\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32m<ipython-input-1-bea6327a4fd8>\u001B[0m in \u001B[0;36mcartpole\u001B[1;34m()\u001B[0m\n\u001B[0;32m     84\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mterminal\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     85\u001B[0m                 \u001B[0mprint\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"Run: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", exploration: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexploration_rate\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", score: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 86\u001B[1;33m                 \u001B[0mscore_logger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0madd_score\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     87\u001B[0m                 \u001B[1;32mbreak\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     88\u001B[0m             \u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexperience_replay\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mU:\\Jupyter\\Cartpole\\scores\\score_logger.py\u001B[0m in \u001B[0;36madd_score\u001B[1;34m(self, score, run)\u001B[0m\n\u001B[0;32m     52\u001B[0m             \u001B[1;31m#               show_trend=False,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     53\u001B[0m             \u001B[1;31m#               show_legend=False)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 54\u001B[1;33m             \u001B[0mexit\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     55\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     56\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_save_png\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0minput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0moutput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mx_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maverage_of_n_last\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_goal\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_trend\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_legend\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'exit' is not defined"
     ]
    }
   ],
   "source": [
    "cartpole()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: If the code is running properly, you should begin to see output appearing above this code block. It will take several minutes, so it is recommended that you let this code run in the background while completing other work. When the code has finished, it will print output saying, \"Solved in _ runs, _ total runs.\"\n",
    "\n",
    "You may see an error about not having an exit command. This error does not affect the program's functionality and results from the steps taken to convert the code from Python 2.x to Python 3. Please disregard this error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modified cartpole: increased gamma to 0.995\n",
    "\n",
    "ENV_NAME = \"CartPole-v1\"  \n",
    "  \n",
    "GAMMA = 0.995  \n",
    "LEARNING_RATE = 0.001  \n",
    "  \n",
    "MEMORY_SIZE = 1000000  \n",
    "BATCH_SIZE = 20  \n",
    "  \n",
    "EXPLORATION_MAX = 1.0  \n",
    "EXPLORATION_MIN = 0.01  \n",
    "EXPLORATION_DECAY = 0.995  \n",
    "  \n",
    "  \n",
    "class DQNSolver:  \n",
    "  \n",
    "    def __init__(self, observation_space, action_space):  \n",
    "        self.exploration_rate = EXPLORATION_MAX  \n",
    "  \n",
    "        self.action_space = action_space  \n",
    "        self.memory = deque(maxlen=MEMORY_SIZE)  \n",
    "  \n",
    "        self.model = Sequential()  \n",
    "        self.model.add(Dense(24, input_shape=(observation_space,), activation=\"relu\"))  \n",
    "        self.model.add(Dense(24, activation=\"relu\"))  \n",
    "        self.model.add(Dense(self.action_space, activation=\"linear\"))  \n",
    "        self.model.compile(loss=\"mse\", optimizer=Adam(lr=LEARNING_RATE))  \n",
    "  \n",
    "    def remember(self, state, action, reward, next_state, done):  \n",
    "        self.memory.append((state, action, reward, next_state, done))  \n",
    "  \n",
    "    def act(self, state):  \n",
    "        if np.random.rand() < self.exploration_rate:  \n",
    "            return random.randrange(self.action_space)  \n",
    "        q_values = self.model.predict(state)  \n",
    "        return np.argmax(q_values[0])  \n",
    "  \n",
    "    def experience_replay(self):  \n",
    "        if len(self.memory) < BATCH_SIZE:  \n",
    "            return  \n",
    "        batch = random.sample(self.memory, BATCH_SIZE)  \n",
    "        for state, action, reward, state_next, terminal in batch:  \n",
    "            q_update = reward  \n",
    "            if not terminal:  \n",
    "                q_update = (reward + GAMMA * np.amax(self.model.predict(state_next)[0]))  \n",
    "            q_values = self.model.predict(state)  \n",
    "            q_values[0][action] = q_update  \n",
    "            self.model.fit(state, q_values, verbose=0)  \n",
    "        self.exploration_rate *= EXPLORATION_DECAY  \n",
    "        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)  \n",
    "  \n",
    "  \n",
    "def cartpole():  \n",
    "    env = gym.make(ENV_NAME)  \n",
    "    score_logger = ScoreLogger(ENV_NAME)  \n",
    "    observation_space = env.observation_space.shape[0]  \n",
    "    action_space = env.action_space.n  \n",
    "    dqn_solver = DQNSolver(observation_space, action_space)  \n",
    "    run = 0  \n",
    "    while True:  \n",
    "        run += 1  \n",
    "        state = env.reset()  \n",
    "        state = np.reshape(state, [1, observation_space])  \n",
    "        step = 0  \n",
    "        while True:  \n",
    "            step += 1  \n",
    "            #env.render()  \n",
    "            action = dqn_solver.act(state)  \n",
    "            state_next, reward, terminal, info = env.step(action)  \n",
    "            reward = reward if not terminal else -reward  \n",
    "            state_next = np.reshape(state_next, [1, observation_space])  \n",
    "            dqn_solver.remember(state, action, reward, state_next, terminal)  \n",
    "            state = state_next  \n",
    "            if terminal:  \n",
    "                print (\"Run: \" + str(run) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(step))  \n",
    "                score_logger.add_score(step, run)  \n",
    "                break  \n",
    "            dqn_solver.experience_replay()  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 1, exploration: 1.0, score: 12\n",
      "Scores: (min: 12, avg: 12, max: 12)\n",
      "\n",
      "Run: 2, exploration: 0.9703725093562657, score: 14\n",
      "Scores: (min: 12, avg: 13, max: 14)\n",
      "\n",
      "Run: 3, exploration: 0.8433051360508336, score: 29\n",
      "Scores: (min: 12, avg: 18.333333333333332, max: 29)\n",
      "\n",
      "Run: 4, exploration: 0.697046600835495, score: 39\n",
      "Scores: (min: 12, avg: 23.5, max: 39)\n",
      "\n",
      "Run: 5, exploration: 0.6057704364907278, score: 29\n",
      "Scores: (min: 12, avg: 24.6, max: 39)\n",
      "\n",
      "Run: 6, exploration: 0.500708706245853, score: 39\n",
      "Scores: (min: 12, avg: 27, max: 39)\n",
      "\n",
      "Run: 7, exploration: 0.39561243860243744, score: 48\n",
      "Scores: (min: 12, avg: 30, max: 48)\n",
      "\n",
      "Run: 8, exploration: 0.3386767948568688, score: 32\n",
      "Scores: (min: 12, avg: 30.25, max: 48)\n",
      "\n",
      "Run: 9, exploration: 0.27714603575484437, score: 41\n",
      "Scores: (min: 12, avg: 31.444444444444443, max: 48)\n",
      "\n",
      "Run: 10, exploration: 0.17829525136613786, score: 89\n",
      "Scores: (min: 12, avg: 37.2, max: 89)\n",
      "\n",
      "Run: 11, exploration: 0.14811319969530845, score: 38\n",
      "Scores: (min: 12, avg: 37.27272727272727, max: 89)\n",
      "\n",
      "Run: 12, exploration: 0.1176130407830293, score: 47\n",
      "Scores: (min: 12, avg: 38.083333333333336, max: 89)\n",
      "\n",
      "Run: 13, exploration: 0.10018300685223579, score: 33\n",
      "Scores: (min: 12, avg: 37.69230769230769, max: 89)\n",
      "\n",
      "Run: 14, exploration: 0.07642587550895225, score: 55\n",
      "Scores: (min: 12, avg: 38.92857142857143, max: 89)\n",
      "\n",
      "Run: 15, exploration: 0.050162503748802344, score: 85\n",
      "Scores: (min: 12, avg: 42, max: 89)\n",
      "\n",
      "Run: 16, exploration: 0.03308985827710748, score: 84\n",
      "Scores: (min: 12, avg: 44.625, max: 89)\n",
      "\n",
      "Run: 17, exploration: 0.025497353542912736, score: 53\n",
      "Scores: (min: 12, avg: 45.11764705882353, max: 89)\n",
      "\n",
      "Run: 18, exploration: 0.020760672449809284, score: 42\n",
      "Scores: (min: 12, avg: 44.94444444444444, max: 89)\n",
      "\n",
      "Run: 19, exploration: 0.01, score: 150\n",
      "Scores: (min: 12, avg: 50.473684210526315, max: 150)\n",
      "\n",
      "Run: 20, exploration: 0.01, score: 96\n",
      "Scores: (min: 12, avg: 52.75, max: 150)\n",
      "\n",
      "Run: 21, exploration: 0.01, score: 122\n",
      "Scores: (min: 12, avg: 56.04761904761905, max: 150)\n",
      "\n",
      "Run: 22, exploration: 0.01, score: 172\n",
      "Scores: (min: 12, avg: 61.31818181818182, max: 172)\n",
      "\n",
      "Run: 23, exploration: 0.01, score: 253\n",
      "Scores: (min: 12, avg: 69.65217391304348, max: 253)\n",
      "\n",
      "Run: 24, exploration: 0.01, score: 189\n",
      "Scores: (min: 12, avg: 74.625, max: 253)\n",
      "\n",
      "Run: 25, exploration: 0.01, score: 169\n",
      "Scores: (min: 12, avg: 78.4, max: 253)\n",
      "\n",
      "Run: 26, exploration: 0.01, score: 210\n",
      "Scores: (min: 12, avg: 83.46153846153847, max: 253)\n",
      "\n",
      "Run: 27, exploration: 0.01, score: 175\n",
      "Scores: (min: 12, avg: 86.85185185185185, max: 253)\n",
      "\n",
      "Run: 28, exploration: 0.01, score: 289\n",
      "Scores: (min: 12, avg: 94.07142857142857, max: 289)\n",
      "\n",
      "Run: 29, exploration: 0.01, score: 200\n",
      "Scores: (min: 12, avg: 97.72413793103448, max: 289)\n",
      "\n",
      "Run: 30, exploration: 0.01, score: 10\n",
      "Scores: (min: 10, avg: 94.8, max: 289)\n",
      "\n",
      "Run: 31, exploration: 0.01, score: 14\n",
      "Scores: (min: 10, avg: 92.19354838709677, max: 289)\n",
      "\n",
      "Run: 32, exploration: 0.01, score: 11\n",
      "Scores: (min: 10, avg: 89.65625, max: 289)\n",
      "\n",
      "Run: 33, exploration: 0.01, score: 9\n",
      "Scores: (min: 9, avg: 87.21212121212122, max: 289)\n",
      "\n",
      "Run: 34, exploration: 0.01, score: 10\n",
      "Scores: (min: 9, avg: 84.94117647058823, max: 289)\n",
      "\n",
      "Run: 35, exploration: 0.01, score: 15\n",
      "Scores: (min: 9, avg: 82.94285714285714, max: 289)\n",
      "\n",
      "Run: 36, exploration: 0.01, score: 158\n",
      "Scores: (min: 9, avg: 85.02777777777777, max: 289)\n",
      "\n",
      "Run: 37, exploration: 0.01, score: 261\n",
      "Scores: (min: 9, avg: 89.78378378378379, max: 289)\n",
      "\n",
      "Run: 38, exploration: 0.01, score: 291\n",
      "Scores: (min: 9, avg: 95.07894736842105, max: 291)\n",
      "\n",
      "Run: 39, exploration: 0.01, score: 280\n",
      "Scores: (min: 9, avg: 99.82051282051282, max: 291)\n",
      "\n",
      "Run: 40, exploration: 0.01, score: 195\n",
      "Scores: (min: 9, avg: 102.2, max: 291)\n",
      "\n",
      "Run: 41, exploration: 0.01, score: 212\n",
      "Scores: (min: 9, avg: 104.8780487804878, max: 291)\n",
      "\n",
      "Run: 42, exploration: 0.01, score: 192\n",
      "Scores: (min: 9, avg: 106.95238095238095, max: 291)\n",
      "\n",
      "Run: 43, exploration: 0.01, score: 225\n",
      "Scores: (min: 9, avg: 109.69767441860465, max: 291)\n",
      "\n",
      "Run: 44, exploration: 0.01, score: 181\n",
      "Scores: (min: 9, avg: 111.31818181818181, max: 291)\n",
      "\n",
      "Run: 45, exploration: 0.01, score: 163\n",
      "Scores: (min: 9, avg: 112.46666666666667, max: 291)\n",
      "\n",
      "Run: 46, exploration: 0.01, score: 182\n",
      "Scores: (min: 9, avg: 113.97826086956522, max: 291)\n",
      "\n",
      "Run: 47, exploration: 0.01, score: 175\n",
      "Scores: (min: 9, avg: 115.27659574468085, max: 291)\n",
      "\n",
      "Run: 48, exploration: 0.01, score: 175\n",
      "Scores: (min: 9, avg: 116.52083333333333, max: 291)\n",
      "\n",
      "Run: 49, exploration: 0.01, score: 197\n",
      "Scores: (min: 9, avg: 118.16326530612245, max: 291)\n",
      "\n",
      "Run: 50, exploration: 0.01, score: 175\n",
      "Scores: (min: 9, avg: 119.3, max: 291)\n",
      "\n",
      "Run: 51, exploration: 0.01, score: 160\n",
      "Scores: (min: 9, avg: 120.09803921568627, max: 291)\n",
      "\n",
      "Run: 52, exploration: 0.01, score: 152\n",
      "Scores: (min: 9, avg: 120.71153846153847, max: 291)\n",
      "\n",
      "Run: 53, exploration: 0.01, score: 194\n",
      "Scores: (min: 9, avg: 122.09433962264151, max: 291)\n",
      "\n",
      "Run: 54, exploration: 0.01, score: 170\n",
      "Scores: (min: 9, avg: 122.98148148148148, max: 291)\n",
      "\n",
      "Run: 55, exploration: 0.01, score: 189\n",
      "Scores: (min: 9, avg: 124.18181818181819, max: 291)\n",
      "\n",
      "Run: 56, exploration: 0.01, score: 177\n",
      "Scores: (min: 9, avg: 125.125, max: 291)\n",
      "\n",
      "Run: 57, exploration: 0.01, score: 318\n",
      "Scores: (min: 9, avg: 128.50877192982455, max: 318)\n",
      "\n",
      "Run: 58, exploration: 0.01, score: 309\n",
      "Scores: (min: 9, avg: 131.6206896551724, max: 318)\n",
      "\n",
      "Run: 59, exploration: 0.01, score: 204\n",
      "Scores: (min: 9, avg: 132.84745762711864, max: 318)\n",
      "\n",
      "Run: 60, exploration: 0.01, score: 243\n",
      "Scores: (min: 9, avg: 134.68333333333334, max: 318)\n",
      "\n",
      "Run: 61, exploration: 0.01, score: 282\n",
      "Scores: (min: 9, avg: 137.0983606557377, max: 318)\n",
      "\n",
      "Run: 62, exploration: 0.01, score: 291\n",
      "Scores: (min: 9, avg: 139.58064516129033, max: 318)\n",
      "\n",
      "Run: 63, exploration: 0.01, score: 250\n",
      "Scores: (min: 9, avg: 141.33333333333334, max: 318)\n",
      "\n",
      "Run: 64, exploration: 0.01, score: 262\n",
      "Scores: (min: 9, avg: 143.21875, max: 318)\n",
      "\n",
      "Run: 65, exploration: 0.01, score: 242\n",
      "Scores: (min: 9, avg: 144.73846153846154, max: 318)\n",
      "\n",
      "Run: 66, exploration: 0.01, score: 260\n",
      "Scores: (min: 9, avg: 146.4848484848485, max: 318)\n",
      "\n",
      "Run: 67, exploration: 0.01, score: 238\n",
      "Scores: (min: 9, avg: 147.8507462686567, max: 318)\n",
      "\n",
      "Run: 68, exploration: 0.01, score: 48\n",
      "Scores: (min: 9, avg: 146.38235294117646, max: 318)\n",
      "\n",
      "Run: 69, exploration: 0.01, score: 10\n",
      "Scores: (min: 9, avg: 144.40579710144928, max: 318)\n",
      "\n",
      "Run: 70, exploration: 0.01, score: 9\n",
      "Scores: (min: 9, avg: 142.47142857142856, max: 318)\n",
      "\n",
      "Run: 71, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 140.57746478873239, max: 318)\n",
      "\n",
      "Run: 72, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 138.76388888888889, max: 318)\n",
      "\n",
      "Run: 73, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 137, max: 318)\n",
      "\n",
      "Run: 74, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 135.25675675675674, max: 318)\n",
      "\n",
      "Run: 75, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 133.58666666666667, max: 318)\n",
      "\n",
      "Run: 76, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 131.94736842105263, max: 318)\n",
      "\n",
      "Run: 77, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 130.36363636363637, max: 318)\n",
      "\n",
      "Run: 78, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 128.82051282051282, max: 318)\n",
      "\n",
      "Run: 79, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 127.29113924050633, max: 318)\n",
      "\n",
      "Run: 80, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 125.8125, max: 318)\n",
      "\n",
      "Run: 81, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 124.37037037037037, max: 318)\n",
      "\n",
      "Run: 82, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 122.96341463414635, max: 318)\n",
      "\n",
      "Run: 83, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 121.59036144578313, max: 318)\n",
      "\n",
      "Run: 84, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 120.25, max: 318)\n",
      "\n",
      "Run: 85, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 118.95294117647059, max: 318)\n",
      "\n",
      "Run: 86, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 117.67441860465117, max: 318)\n",
      "\n",
      "Run: 87, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 116.41379310344827, max: 318)\n",
      "\n",
      "Run: 88, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 115.20454545454545, max: 318)\n",
      "\n",
      "Run: 89, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 114, max: 318)\n",
      "\n",
      "Run: 90, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 112.84444444444445, max: 318)\n",
      "\n",
      "Run: 91, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 111.72527472527473, max: 318)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 92, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 110.6195652173913, max: 318)\n",
      "\n",
      "Run: 93, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 109.53763440860214, max: 318)\n",
      "\n",
      "Run: 94, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 108.46808510638297, max: 318)\n",
      "\n",
      "Run: 95, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 107.42105263157895, max: 318)\n",
      "\n",
      "Run: 96, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 106.40625, max: 318)\n",
      "\n",
      "Run: 97, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 105.41237113402062, max: 318)\n",
      "\n",
      "Run: 98, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 104.42857142857143, max: 318)\n",
      "\n",
      "Run: 99, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 103.47474747474747, max: 318)\n",
      "\n",
      "Run: 100, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 102.55, max: 318)\n",
      "\n",
      "Run: 101, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 102.52, max: 318)\n",
      "\n",
      "Run: 102, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 102.48, max: 318)\n",
      "\n",
      "Run: 103, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 102.29, max: 318)\n",
      "\n",
      "Run: 104, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 101.99, max: 318)\n",
      "\n",
      "Run: 105, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 101.79, max: 318)\n",
      "\n",
      "Run: 106, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 101.49, max: 318)\n",
      "\n",
      "Run: 107, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 101.1, max: 318)\n",
      "\n",
      "Run: 108, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 100.87, max: 318)\n",
      "\n",
      "Run: 109, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 100.55, max: 318)\n",
      "\n",
      "Run: 110, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 99.75, max: 318)\n",
      "\n",
      "Run: 111, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 99.46, max: 318)\n",
      "\n",
      "Run: 112, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 99.09, max: 318)\n",
      "\n",
      "Run: 113, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 98.85, max: 318)\n",
      "\n",
      "Run: 114, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 98.38, max: 318)\n",
      "\n",
      "Run: 115, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 97.63, max: 318)\n",
      "\n",
      "Run: 116, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 96.88, max: 318)\n",
      "\n",
      "Run: 117, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 96.43, max: 318)\n",
      "\n",
      "Run: 118, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 96.11, max: 318)\n",
      "\n",
      "Run: 119, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 94.71, max: 318)\n",
      "\n",
      "Run: 120, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 93.83, max: 318)\n",
      "\n",
      "Run: 121, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 92.7, max: 318)\n",
      "\n",
      "Run: 122, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 91.08, max: 318)\n",
      "\n",
      "Run: 123, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 88.64, max: 318)\n",
      "\n",
      "Run: 124, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 86.85, max: 318)\n",
      "\n",
      "Run: 125, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 85.25, max: 318)\n",
      "\n",
      "Run: 126, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 83.25, max: 318)\n",
      "\n",
      "Run: 127, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 81.6, max: 318)\n",
      "\n",
      "Run: 128, exploration: 0.01, score: 28\n",
      "Scores: (min: 8, avg: 78.99, max: 318)\n",
      "\n",
      "Run: 129, exploration: 0.01, score: 51\n",
      "Scores: (min: 8, avg: 77.5, max: 318)\n",
      "\n",
      "Run: 130, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 77.52, max: 318)\n",
      "\n",
      "Run: 131, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 77.48, max: 318)\n",
      "\n",
      "Run: 132, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 77.46, max: 318)\n",
      "\n",
      "Run: 133, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 77.47, max: 318)\n",
      "\n",
      "Run: 134, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 77.46, max: 318)\n",
      "\n",
      "Run: 135, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 77.41, max: 318)\n",
      "\n",
      "Run: 136, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 75.93, max: 318)\n",
      "\n",
      "Run: 137, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 73.4, max: 318)\n",
      "\n",
      "Run: 138, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 70.58, max: 318)\n",
      "\n",
      "Run: 139, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 67.87, max: 318)\n",
      "\n",
      "Run: 140, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 66.02, max: 318)\n",
      "\n",
      "Run: 141, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 64, max: 318)\n",
      "\n",
      "Run: 142, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 62.17, max: 318)\n",
      "\n",
      "Run: 143, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 60.01, max: 318)\n",
      "\n",
      "Run: 144, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 58.28, max: 318)\n",
      "\n",
      "Run: 145, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 56.73, max: 318)\n",
      "\n",
      "Run: 146, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 55, max: 318)\n",
      "\n",
      "Run: 147, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 53.35, max: 318)\n",
      "\n",
      "Run: 148, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 51.68, max: 318)\n",
      "\n",
      "Run: 149, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 49.8, max: 318)\n",
      "\n",
      "Run: 150, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 48.15, max: 318)\n",
      "\n",
      "Run: 151, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 46.63, max: 318)\n",
      "\n",
      "Run: 152, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 45.2, max: 318)\n",
      "\n",
      "Run: 153, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 43.36, max: 318)\n",
      "\n",
      "Run: 154, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 41.76, max: 318)\n",
      "\n",
      "Run: 155, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 39.97, max: 318)\n",
      "\n",
      "Run: 156, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 38.29, max: 318)\n",
      "\n",
      "Run: 157, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 35.21, max: 309)\n",
      "\n",
      "Run: 158, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 32.21, max: 291)\n",
      "\n",
      "Run: 159, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 30.28, max: 291)\n",
      "\n",
      "Run: 160, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 27.94, max: 291)\n",
      "\n",
      "Run: 161, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 25.22, max: 291)\n",
      "\n",
      "Run: 162, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 22.4, max: 262)\n",
      "\n",
      "Run: 163, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 20, max: 262)\n",
      "\n",
      "Run: 164, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 17.48, max: 260)\n",
      "\n",
      "Run: 165, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 15.16, max: 260)\n",
      "\n",
      "Run: 166, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 12.66, max: 238)\n",
      "\n",
      "Run: 167, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 10.38, max: 51)\n",
      "\n",
      "Run: 168, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 10, max: 51)\n",
      "\n",
      "Run: 169, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.99, max: 51)\n",
      "\n",
      "Run: 170, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 10, max: 51)\n",
      "\n",
      "Run: 171, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 10, max: 51)\n",
      "\n",
      "Run: 172, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.99, max: 51)\n",
      "\n",
      "Run: 173, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 174, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.99, max: 51)\n",
      "\n",
      "Run: 175, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 176, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 177, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 9.96, max: 51)\n",
      "\n",
      "Run: 178, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.95, max: 51)\n",
      "\n",
      "Run: 179, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 9.97, max: 51)\n",
      "\n",
      "Run: 180, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 181, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 182, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 183, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 184, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 185, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 9.98, max: 51)\n",
      "\n",
      "Run: 186, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 9.99, max: 51)\n",
      "\n",
      "Run: 187, exploration: 0.01, score: 8\n",
      "Scores: (min: 8, avg: 9.99, max: 51)\n",
      "\n",
      "Run: 188, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 10.04, max: 51)\n",
      "\n",
      "Run: 189, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 10.05, max: 51)\n",
      "\n",
      "Run: 190, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 10.07, max: 51)\n",
      "\n",
      "Run: 191, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 10.06, max: 51)\n",
      "\n",
      "Run: 192, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 10.06, max: 51)\n",
      "\n",
      "Run: 193, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 10.08, max: 51)\n",
      "\n",
      "Run: 194, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 10.12, max: 51)\n",
      "\n",
      "Run: 195, exploration: 0.01, score: 17\n",
      "Scores: (min: 8, avg: 10.2, max: 51)\n",
      "\n",
      "Run: 196, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 10.22, max: 51)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 197, exploration: 0.01, score: 14\n",
      "Scores: (min: 8, avg: 10.26, max: 51)\n",
      "\n",
      "Run: 198, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 10.3, max: 51)\n",
      "\n",
      "Run: 199, exploration: 0.01, score: 27\n",
      "Scores: (min: 8, avg: 10.47, max: 51)\n",
      "\n",
      "Run: 200, exploration: 0.01, score: 26\n",
      "Scores: (min: 8, avg: 10.62, max: 51)\n",
      "\n",
      "Run: 201, exploration: 0.01, score: 52\n",
      "Scores: (min: 8, avg: 11.05, max: 52)\n",
      "\n",
      "Run: 202, exploration: 0.01, score: 46\n",
      "Scores: (min: 8, avg: 11.41, max: 52)\n",
      "\n",
      "Run: 203, exploration: 0.01, score: 56\n",
      "Scores: (min: 8, avg: 11.87, max: 56)\n",
      "\n",
      "Run: 204, exploration: 0.01, score: 396\n",
      "Scores: (min: 8, avg: 15.74, max: 396)\n",
      "\n",
      "Run: 205, exploration: 0.01, score: 61\n",
      "Scores: (min: 8, avg: 16.26, max: 396)\n",
      "\n",
      "Run: 206, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 16.3, max: 396)\n",
      "\n",
      "Run: 207, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 16.32, max: 396)\n",
      "\n",
      "Run: 208, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 16.34, max: 396)\n",
      "\n",
      "Run: 209, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 16.36, max: 396)\n",
      "\n",
      "Run: 210, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.37, max: 396)\n",
      "\n",
      "Run: 211, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.37, max: 396)\n",
      "\n",
      "Run: 212, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.37, max: 396)\n",
      "\n",
      "Run: 213, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.37, max: 396)\n",
      "\n",
      "Run: 214, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.38, max: 396)\n",
      "\n",
      "Run: 215, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.38, max: 396)\n",
      "\n",
      "Run: 216, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.39, max: 396)\n",
      "\n",
      "Run: 217, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.4, max: 396)\n",
      "\n",
      "Run: 218, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.4, max: 396)\n",
      "\n",
      "Run: 219, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.39, max: 396)\n",
      "\n",
      "Run: 220, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.41, max: 396)\n",
      "\n",
      "Run: 221, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.42, max: 396)\n",
      "\n",
      "Run: 222, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.42, max: 396)\n",
      "\n",
      "Run: 223, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.42, max: 396)\n",
      "\n",
      "Run: 224, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 16.42, max: 396)\n",
      "\n",
      "Run: 225, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 16.44, max: 396)\n",
      "\n",
      "Run: 226, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.43, max: 396)\n",
      "\n",
      "Run: 227, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.42, max: 396)\n",
      "\n",
      "Run: 228, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 16.23, max: 396)\n",
      "\n",
      "Run: 229, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 15.82, max: 396)\n",
      "\n",
      "Run: 230, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 15.8, max: 396)\n",
      "\n",
      "Run: 231, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 15.79, max: 396)\n",
      "\n",
      "Run: 232, exploration: 0.01, score: 9\n",
      "Scores: (min: 8, avg: 15.79, max: 396)\n",
      "\n",
      "Run: 233, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 15.81, max: 396)\n",
      "\n",
      "Run: 234, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 15.83, max: 396)\n",
      "\n",
      "Run: 235, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 15.83, max: 396)\n",
      "\n",
      "Run: 236, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 15.83, max: 396)\n",
      "\n",
      "Run: 237, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 15.88, max: 396)\n",
      "\n",
      "Run: 238, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 15.9, max: 396)\n",
      "\n",
      "Run: 239, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 15.94, max: 396)\n",
      "\n",
      "Run: 240, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 15.97, max: 396)\n",
      "\n",
      "Run: 241, exploration: 0.01, score: 14\n",
      "Scores: (min: 8, avg: 16.01, max: 396)\n",
      "\n",
      "Run: 242, exploration: 0.01, score: 19\n",
      "Scores: (min: 8, avg: 16.11, max: 396)\n",
      "\n",
      "Run: 243, exploration: 0.01, score: 23\n",
      "Scores: (min: 8, avg: 16.25, max: 396)\n",
      "\n",
      "Run: 244, exploration: 0.01, score: 22\n",
      "Scores: (min: 8, avg: 16.39, max: 396)\n",
      "\n",
      "Run: 245, exploration: 0.01, score: 69\n",
      "Scores: (min: 8, avg: 17, max: 396)\n",
      "\n",
      "Run: 246, exploration: 0.01, score: 80\n",
      "Scores: (min: 8, avg: 17.71, max: 396)\n",
      "\n",
      "Run: 247, exploration: 0.01, score: 87\n",
      "Scores: (min: 8, avg: 18.48, max: 396)\n",
      "\n",
      "Run: 248, exploration: 0.01, score: 145\n",
      "Scores: (min: 8, avg: 19.85, max: 396)\n",
      "\n",
      "Run: 249, exploration: 0.01, score: 131\n",
      "Scores: (min: 8, avg: 21.07, max: 396)\n",
      "\n",
      "Run: 250, exploration: 0.01, score: 128\n",
      "Scores: (min: 8, avg: 22.25, max: 396)\n",
      "\n",
      "Run: 251, exploration: 0.01, score: 124\n",
      "Scores: (min: 8, avg: 23.41, max: 396)\n",
      "\n",
      "Run: 252, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 23.47, max: 396)\n",
      "\n",
      "Run: 253, exploration: 0.01, score: 16\n",
      "Scores: (min: 8, avg: 23.53, max: 396)\n",
      "\n",
      "Run: 254, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 23.56, max: 396)\n",
      "\n",
      "Run: 255, exploration: 0.01, score: 14\n",
      "Scores: (min: 8, avg: 23.6, max: 396)\n",
      "\n",
      "Run: 256, exploration: 0.01, score: 14\n",
      "Scores: (min: 8, avg: 23.65, max: 396)\n",
      "\n",
      "Run: 257, exploration: 0.01, score: 15\n",
      "Scores: (min: 8, avg: 23.7, max: 396)\n",
      "\n",
      "Run: 258, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 23.73, max: 396)\n",
      "\n",
      "Run: 259, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 23.75, max: 396)\n",
      "\n",
      "Run: 260, exploration: 0.01, score: 10\n",
      "Scores: (min: 8, avg: 23.76, max: 396)\n",
      "\n",
      "Run: 261, exploration: 0.01, score: 13\n",
      "Scores: (min: 8, avg: 23.79, max: 396)\n",
      "\n",
      "Run: 262, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 23.82, max: 396)\n",
      "\n",
      "Run: 263, exploration: 0.01, score: 14\n",
      "Scores: (min: 8, avg: 23.86, max: 396)\n",
      "\n",
      "Run: 264, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 23.88, max: 396)\n",
      "\n",
      "Run: 265, exploration: 0.01, score: 11\n",
      "Scores: (min: 8, avg: 23.89, max: 396)\n",
      "\n",
      "Run: 266, exploration: 0.01, score: 12\n",
      "Scores: (min: 8, avg: 23.91, max: 396)\n",
      "\n",
      "Run: 267, exploration: 0.01, score: 14\n",
      "Scores: (min: 8, avg: 23.95, max: 396)\n",
      "\n",
      "Run: 268, exploration: 0.01, score: 17\n",
      "Scores: (min: 8, avg: 24.02, max: 396)\n",
      "\n",
      "Run: 269, exploration: 0.01, score: 59\n",
      "Scores: (min: 8, avg: 24.52, max: 396)\n",
      "\n",
      "Run: 270, exploration: 0.01, score: 61\n",
      "Scores: (min: 8, avg: 25.03, max: 396)\n",
      "\n",
      "Run: 271, exploration: 0.01, score: 64\n",
      "Scores: (min: 8, avg: 25.59, max: 396)\n",
      "\n",
      "Run: 272, exploration: 0.01, score: 69\n",
      "Scores: (min: 8, avg: 26.19, max: 396)\n",
      "\n",
      "Run: 273, exploration: 0.01, score: 71\n",
      "Scores: (min: 8, avg: 26.81, max: 396)\n",
      "\n",
      "Run: 274, exploration: 0.01, score: 76\n",
      "Scores: (min: 8, avg: 27.48, max: 396)\n",
      "\n",
      "Run: 275, exploration: 0.01, score: 320\n",
      "Scores: (min: 8, avg: 30.59, max: 396)\n",
      "\n",
      "Run: 276, exploration: 0.01, score: 169\n",
      "Scores: (min: 8, avg: 32.19, max: 396)\n",
      "\n",
      "Run: 277, exploration: 0.01, score: 282\n",
      "Scores: (min: 8, avg: 34.93, max: 396)\n",
      "\n",
      "Run: 278, exploration: 0.01, score: 302\n",
      "Scores: (min: 8, avg: 37.86, max: 396)\n",
      "\n",
      "Run: 279, exploration: 0.01, score: 408\n",
      "Scores: (min: 8, avg: 41.84, max: 408)\n",
      "\n",
      "Run: 280, exploration: 0.01, score: 294\n",
      "Scores: (min: 8, avg: 44.68, max: 408)\n",
      "\n",
      "Run: 281, exploration: 0.01, score: 125\n",
      "Scores: (min: 8, avg: 45.84, max: 408)\n",
      "\n",
      "Run: 282, exploration: 0.01, score: 323\n",
      "Scores: (min: 8, avg: 48.98, max: 408)\n",
      "\n",
      "Run: 283, exploration: 0.01, score: 468\n",
      "Scores: (min: 8, avg: 53.57, max: 468)\n",
      "\n",
      "Run: 284, exploration: 0.01, score: 500\n",
      "Scores: (min: 8, avg: 58.48, max: 500)\n",
      "\n",
      "Run: 285, exploration: 0.01, score: 500\n",
      "Scores: (min: 8, avg: 63.38, max: 500)\n",
      "\n",
      "Run: 286, exploration: 0.01, score: 500\n",
      "Scores: (min: 8, avg: 68.28, max: 500)\n",
      "\n",
      "Run: 287, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 73.2, max: 500)\n",
      "\n",
      "Run: 288, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 78.05, max: 500)\n",
      "\n",
      "Run: 289, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 82.96, max: 500)\n",
      "\n",
      "Run: 290, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 87.84, max: 500)\n",
      "\n",
      "Run: 291, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 92.74, max: 500)\n",
      "\n",
      "Run: 292, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 97.64, max: 500)\n",
      "\n",
      "Run: 293, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 102.52, max: 500)\n",
      "\n",
      "Run: 294, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 107.39, max: 500)\n",
      "\n",
      "Run: 295, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 112.22, max: 500)\n",
      "\n",
      "Run: 296, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 117.1, max: 500)\n",
      "\n",
      "Run: 297, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 121.96, max: 500)\n",
      "\n",
      "Run: 298, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 126.83, max: 500)\n",
      "\n",
      "Run: 299, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 131.56, max: 500)\n",
      "\n",
      "Run: 300, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 136.3, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 301, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 140.78, max: 500)\n",
      "\n",
      "Run: 302, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 145.32, max: 500)\n",
      "\n",
      "Run: 303, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 149.76, max: 500)\n",
      "\n",
      "Run: 304, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 150.8, max: 500)\n",
      "\n",
      "Run: 305, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 155.19, max: 500)\n",
      "\n",
      "Run: 306, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 160.06, max: 500)\n",
      "\n",
      "Run: 307, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 164.95, max: 500)\n",
      "\n",
      "Run: 308, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 169.84, max: 500)\n",
      "\n",
      "Run: 309, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 174.73, max: 500)\n",
      "\n",
      "Run: 310, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 179.63, max: 500)\n",
      "\n",
      "Run: 311, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 184.54, max: 500)\n",
      "\n",
      "Run: 312, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 189.44, max: 500)\n",
      "\n",
      "Run: 313, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 194.35, max: 500)\n",
      "\n",
      "Run: 314, exploration: 0.01, score: 500\n",
      "Scores: (min: 9, avg: 199.26, max: 500)\n",
      "\n",
      "Solved in 214 runs, 314 total runs.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'exit' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-6-e23913c48dc8>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mcartpole\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32m<ipython-input-5-7a5080e87ce8>\u001B[0m in \u001B[0;36mcartpole\u001B[1;34m()\u001B[0m\n\u001B[0;32m     78\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mterminal\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     79\u001B[0m                 \u001B[0mprint\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"Run: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", exploration: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexploration_rate\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", score: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 80\u001B[1;33m                 \u001B[0mscore_logger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0madd_score\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     81\u001B[0m                 \u001B[1;32mbreak\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     82\u001B[0m             \u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexperience_replay\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mU:\\Jupyter\\Cartpole\\scores\\score_logger.py\u001B[0m in \u001B[0;36madd_score\u001B[1;34m(self, score, run)\u001B[0m\n\u001B[0;32m     52\u001B[0m             \u001B[1;31m#               show_trend=False,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     53\u001B[0m             \u001B[1;31m#               show_legend=False)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 54\u001B[1;33m             \u001B[0mexit\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     55\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     56\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_save_png\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0minput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0moutput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mx_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maverage_of_n_last\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_goal\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_trend\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_legend\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'exit' is not defined"
     ]
    }
   ],
   "source": [
    "cartpole()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modified cartpole: increased gamma to 0.995, increased exploration min to 0.5\n",
    "  \n",
    "ENV_NAME = \"CartPole-v1\"  \n",
    "  \n",
    "GAMMA = 0.995  \n",
    "LEARNING_RATE = 0.001 \n",
    "  \n",
    "MEMORY_SIZE = 1000000  \n",
    "BATCH_SIZE = 20  \n",
    "  \n",
    "EXPLORATION_MAX = 1.0  \n",
    "EXPLORATION_MIN = 0.5 \n",
    "EXPLORATION_DECAY = 0.995  \n",
    "  \n",
    "  \n",
    "class DQNSolver:  \n",
    "  \n",
    "    def __init__(self, observation_space, action_space):  \n",
    "        self.exploration_rate = EXPLORATION_MAX  \n",
    "  \n",
    "        self.action_space = action_space  \n",
    "        self.memory = deque(maxlen=MEMORY_SIZE)  \n",
    "  \n",
    "        self.model = Sequential()  \n",
    "        self.model.add(Dense(24, input_shape=(observation_space,), activation=\"relu\"))  \n",
    "        self.model.add(Dense(24, activation=\"relu\"))  \n",
    "        self.model.add(Dense(self.action_space, activation=\"linear\"))  \n",
    "        self.model.compile(loss=\"mse\", optimizer=Adam(lr=LEARNING_RATE))  \n",
    "  \n",
    "    def remember(self, state, action, reward, next_state, done):  \n",
    "        self.memory.append((state, action, reward, next_state, done))  \n",
    "  \n",
    "    def act(self, state):  \n",
    "        if np.random.rand() < self.exploration_rate:  \n",
    "            return random.randrange(self.action_space)  \n",
    "        q_values = self.model.predict(state)  \n",
    "        return np.argmax(q_values[0])  \n",
    "  \n",
    "    def experience_replay(self):  \n",
    "        if len(self.memory) < BATCH_SIZE:  \n",
    "            return  \n",
    "        batch = random.sample(self.memory, BATCH_SIZE)  \n",
    "        for state, action, reward, state_next, terminal in batch:  \n",
    "            q_update = reward  \n",
    "            if not terminal:  \n",
    "                q_update = (reward + GAMMA * np.amax(self.model.predict(state_next)[0]))  \n",
    "            q_values = self.model.predict(state)  \n",
    "            q_values[0][action] = q_update  \n",
    "            self.model.fit(state, q_values, verbose=0)  \n",
    "        self.exploration_rate *= EXPLORATION_DECAY  \n",
    "        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)  \n",
    "  \n",
    "  \n",
    "def cartpole():  \n",
    "    env = gym.make(ENV_NAME)  \n",
    "    score_logger = ScoreLogger(ENV_NAME)  \n",
    "    observation_space = env.observation_space.shape[0]  \n",
    "    action_space = env.action_space.n  \n",
    "    dqn_solver = DQNSolver(observation_space, action_space)  \n",
    "    run = 0  \n",
    "    while True:  \n",
    "        run += 1  \n",
    "        state = env.reset()  \n",
    "        state = np.reshape(state, [1, observation_space])  \n",
    "        step = 0  \n",
    "        while True:  \n",
    "            step += 1  \n",
    "            #env.render()  \n",
    "            action = dqn_solver.act(state)  \n",
    "            state_next, reward, terminal, info = env.step(action)  \n",
    "            reward = reward if not terminal else -reward  \n",
    "            state_next = np.reshape(state_next, [1, observation_space])  \n",
    "            dqn_solver.remember(state, action, reward, state_next, terminal)  \n",
    "            state = state_next  \n",
    "            if terminal:  \n",
    "                print (\"Run: \" + str(run) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(step))  \n",
    "                score_logger.add_score(step, run)  \n",
    "                break  \n",
    "            dqn_solver.experience_replay()  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 1, exploration: 0.9801495006250001, score: 24\n",
      "Scores: (min: 24, avg: 24, max: 24)\n",
      "\n",
      "Run: 2, exploration: 0.8911090557802088, score: 20\n",
      "Scores: (min: 20, avg: 22, max: 24)\n",
      "\n",
      "Run: 3, exploration: 0.7940753492934954, score: 24\n",
      "Scores: (min: 20, avg: 22.666666666666668, max: 24)\n",
      "\n",
      "Run: 4, exploration: 0.7328768546436799, score: 17\n",
      "Scores: (min: 17, avg: 21.25, max: 24)\n",
      "\n",
      "Run: 5, exploration: 0.697046600835495, score: 11\n",
      "Scores: (min: 11, avg: 19.2, max: 24)\n",
      "\n",
      "Run: 6, exploration: 0.6465587967553006, score: 16\n",
      "Scores: (min: 11, avg: 18.666666666666668, max: 24)\n",
      "\n",
      "Run: 7, exploration: 0.6027415843082742, score: 15\n",
      "Scores: (min: 11, avg: 18.142857142857142, max: 24)\n",
      "\n",
      "Run: 8, exploration: 0.5452463540625918, score: 21\n",
      "Scores: (min: 11, avg: 18.5, max: 24)\n",
      "\n",
      "Run: 9, exploration: 0.5211953074858876, score: 10\n",
      "Scores: (min: 10, avg: 17.555555555555557, max: 24)\n",
      "\n",
      "Run: 10, exploration: 0.500708706245853, score: 9\n",
      "Scores: (min: 9, avg: 16.7, max: 24)\n",
      "\n",
      "Run: 11, exploration: 0.5, score: 11\n",
      "Scores: (min: 9, avg: 16.181818181818183, max: 24)\n",
      "\n",
      "Run: 12, exploration: 0.5, score: 10\n",
      "Scores: (min: 9, avg: 15.666666666666666, max: 24)\n",
      "\n",
      "Run: 13, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 16.23076923076923, max: 24)\n",
      "\n",
      "Run: 14, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 15.928571428571429, max: 24)\n",
      "\n",
      "Run: 15, exploration: 0.5, score: 17\n",
      "Scores: (min: 9, avg: 16, max: 24)\n",
      "\n",
      "Run: 16, exploration: 0.5, score: 10\n",
      "Scores: (min: 9, avg: 15.625, max: 24)\n",
      "\n",
      "Run: 17, exploration: 0.5, score: 15\n",
      "Scores: (min: 9, avg: 15.588235294117647, max: 24)\n",
      "\n",
      "Run: 18, exploration: 0.5, score: 37\n",
      "Scores: (min: 9, avg: 16.77777777777778, max: 37)\n",
      "\n",
      "Run: 19, exploration: 0.5, score: 17\n",
      "Scores: (min: 9, avg: 16.789473684210527, max: 37)\n",
      "\n",
      "Run: 20, exploration: 0.5, score: 53\n",
      "Scores: (min: 9, avg: 18.6, max: 53)\n",
      "\n",
      "Run: 21, exploration: 0.5, score: 45\n",
      "Scores: (min: 9, avg: 19.857142857142858, max: 53)\n",
      "\n",
      "Run: 22, exploration: 0.5, score: 53\n",
      "Scores: (min: 9, avg: 21.363636363636363, max: 53)\n",
      "\n",
      "Run: 23, exploration: 0.5, score: 24\n",
      "Scores: (min: 9, avg: 21.47826086956522, max: 53)\n",
      "\n",
      "Run: 24, exploration: 0.5, score: 74\n",
      "Scores: (min: 9, avg: 23.666666666666668, max: 74)\n",
      "\n",
      "Run: 25, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 23.64, max: 74)\n",
      "\n",
      "Run: 26, exploration: 0.5, score: 49\n",
      "Scores: (min: 9, avg: 24.615384615384617, max: 74)\n",
      "\n",
      "Run: 27, exploration: 0.5, score: 97\n",
      "Scores: (min: 9, avg: 27.296296296296298, max: 97)\n",
      "\n",
      "Run: 28, exploration: 0.5, score: 105\n",
      "Scores: (min: 9, avg: 30.071428571428573, max: 105)\n",
      "\n",
      "Run: 29, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 29.82758620689655, max: 105)\n",
      "\n",
      "Run: 30, exploration: 0.5, score: 22\n",
      "Scores: (min: 9, avg: 29.566666666666666, max: 105)\n",
      "\n",
      "Run: 31, exploration: 0.5, score: 92\n",
      "Scores: (min: 9, avg: 31.580645161290324, max: 105)\n",
      "\n",
      "Run: 32, exploration: 0.5, score: 102\n",
      "Scores: (min: 9, avg: 33.78125, max: 105)\n",
      "\n",
      "Run: 33, exploration: 0.5, score: 110\n",
      "Scores: (min: 9, avg: 36.09090909090909, max: 110)\n",
      "\n",
      "Run: 34, exploration: 0.5, score: 200\n",
      "Scores: (min: 9, avg: 40.911764705882355, max: 200)\n",
      "\n",
      "Run: 35, exploration: 0.5, score: 51\n",
      "Scores: (min: 9, avg: 41.2, max: 200)\n",
      "\n",
      "Run: 36, exploration: 0.5, score: 47\n",
      "Scores: (min: 9, avg: 41.361111111111114, max: 200)\n",
      "\n",
      "Run: 37, exploration: 0.5, score: 296\n",
      "Scores: (min: 9, avg: 48.24324324324324, max: 296)\n",
      "\n",
      "Run: 38, exploration: 0.5, score: 157\n",
      "Scores: (min: 9, avg: 51.10526315789474, max: 296)\n",
      "\n",
      "Run: 39, exploration: 0.5, score: 220\n",
      "Scores: (min: 9, avg: 55.43589743589744, max: 296)\n",
      "\n",
      "Run: 40, exploration: 0.5, score: 49\n",
      "Scores: (min: 9, avg: 55.275, max: 296)\n",
      "\n",
      "Run: 41, exploration: 0.5, score: 150\n",
      "Scores: (min: 9, avg: 57.58536585365854, max: 296)\n",
      "\n",
      "Run: 42, exploration: 0.5, score: 32\n",
      "Scores: (min: 9, avg: 56.976190476190474, max: 296)\n",
      "\n",
      "Run: 43, exploration: 0.5, score: 41\n",
      "Scores: (min: 9, avg: 56.604651162790695, max: 296)\n",
      "\n",
      "Run: 44, exploration: 0.5, score: 71\n",
      "Scores: (min: 9, avg: 56.93181818181818, max: 296)\n",
      "\n",
      "Run: 45, exploration: 0.5, score: 141\n",
      "Scores: (min: 9, avg: 58.8, max: 296)\n",
      "\n",
      "Run: 46, exploration: 0.5, score: 130\n",
      "Scores: (min: 9, avg: 60.34782608695652, max: 296)\n",
      "\n",
      "Run: 47, exploration: 0.5, score: 134\n",
      "Scores: (min: 9, avg: 61.91489361702128, max: 296)\n",
      "\n",
      "Run: 48, exploration: 0.5, score: 209\n",
      "Scores: (min: 9, avg: 64.97916666666667, max: 296)\n",
      "\n",
      "Run: 49, exploration: 0.5, score: 20\n",
      "Scores: (min: 9, avg: 64.06122448979592, max: 296)\n",
      "\n",
      "Run: 50, exploration: 0.5, score: 181\n",
      "Scores: (min: 9, avg: 66.4, max: 296)\n",
      "\n",
      "Run: 51, exploration: 0.5, score: 44\n",
      "Scores: (min: 9, avg: 65.96078431372548, max: 296)\n",
      "\n",
      "Run: 52, exploration: 0.5, score: 140\n",
      "Scores: (min: 9, avg: 67.38461538461539, max: 296)\n",
      "\n",
      "Run: 53, exploration: 0.5, score: 57\n",
      "Scores: (min: 9, avg: 67.18867924528301, max: 296)\n",
      "\n",
      "Run: 54, exploration: 0.5, score: 241\n",
      "Scores: (min: 9, avg: 70.4074074074074, max: 296)\n",
      "\n",
      "Run: 55, exploration: 0.5, score: 198\n",
      "Scores: (min: 9, avg: 72.72727272727273, max: 296)\n",
      "\n",
      "Run: 56, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 71.83928571428571, max: 296)\n",
      "\n",
      "Run: 57, exploration: 0.5, score: 172\n",
      "Scores: (min: 9, avg: 73.59649122807018, max: 296)\n",
      "\n",
      "Run: 58, exploration: 0.5, score: 219\n",
      "Scores: (min: 9, avg: 76.10344827586206, max: 296)\n",
      "\n",
      "Run: 59, exploration: 0.5, score: 131\n",
      "Scores: (min: 9, avg: 77.03389830508475, max: 296)\n",
      "\n",
      "Run: 60, exploration: 0.5, score: 169\n",
      "Scores: (min: 9, avg: 78.56666666666666, max: 296)\n",
      "\n",
      "Run: 61, exploration: 0.5, score: 239\n",
      "Scores: (min: 9, avg: 81.19672131147541, max: 296)\n",
      "\n",
      "Run: 62, exploration: 0.5, score: 171\n",
      "Scores: (min: 9, avg: 82.64516129032258, max: 296)\n",
      "\n",
      "Run: 63, exploration: 0.5, score: 100\n",
      "Scores: (min: 9, avg: 82.92063492063492, max: 296)\n",
      "\n",
      "Run: 64, exploration: 0.5, score: 79\n",
      "Scores: (min: 9, avg: 82.859375, max: 296)\n",
      "\n",
      "Run: 65, exploration: 0.5, score: 155\n",
      "Scores: (min: 9, avg: 83.96923076923076, max: 296)\n",
      "\n",
      "Run: 66, exploration: 0.5, score: 172\n",
      "Scores: (min: 9, avg: 85.3030303030303, max: 296)\n",
      "\n",
      "Run: 67, exploration: 0.5, score: 32\n",
      "Scores: (min: 9, avg: 84.50746268656717, max: 296)\n",
      "\n",
      "Run: 68, exploration: 0.5, score: 243\n",
      "Scores: (min: 9, avg: 86.83823529411765, max: 296)\n",
      "\n",
      "Run: 69, exploration: 0.5, score: 65\n",
      "Scores: (min: 9, avg: 86.52173913043478, max: 296)\n",
      "\n",
      "Run: 70, exploration: 0.5, score: 213\n",
      "Scores: (min: 9, avg: 88.32857142857142, max: 296)\n",
      "\n",
      "Run: 71, exploration: 0.5, score: 72\n",
      "Scores: (min: 9, avg: 88.09859154929578, max: 296)\n",
      "\n",
      "Run: 72, exploration: 0.5, score: 143\n",
      "Scores: (min: 9, avg: 88.86111111111111, max: 296)\n",
      "\n",
      "Run: 73, exploration: 0.5, score: 13\n",
      "Scores: (min: 9, avg: 87.82191780821918, max: 296)\n",
      "\n",
      "Run: 74, exploration: 0.5, score: 69\n",
      "Scores: (min: 9, avg: 87.56756756756756, max: 296)\n",
      "\n",
      "Run: 75, exploration: 0.5, score: 206\n",
      "Scores: (min: 9, avg: 89.14666666666666, max: 296)\n",
      "\n",
      "Run: 76, exploration: 0.5, score: 128\n",
      "Scores: (min: 9, avg: 89.65789473684211, max: 296)\n",
      "\n",
      "Run: 77, exploration: 0.5, score: 117\n",
      "Scores: (min: 9, avg: 90.01298701298701, max: 296)\n",
      "\n",
      "Run: 78, exploration: 0.5, score: 16\n",
      "Scores: (min: 9, avg: 89.06410256410257, max: 296)\n",
      "\n",
      "Run: 79, exploration: 0.5, score: 16\n",
      "Scores: (min: 9, avg: 88.13924050632912, max: 296)\n",
      "\n",
      "Run: 80, exploration: 0.5, score: 98\n",
      "Scores: (min: 9, avg: 88.2625, max: 296)\n",
      "\n",
      "Run: 81, exploration: 0.5, score: 108\n",
      "Scores: (min: 9, avg: 88.50617283950618, max: 296)\n",
      "\n",
      "Run: 82, exploration: 0.5, score: 168\n",
      "Scores: (min: 9, avg: 89.47560975609755, max: 296)\n",
      "\n",
      "Run: 83, exploration: 0.5, score: 50\n",
      "Scores: (min: 9, avg: 89, max: 296)\n",
      "\n",
      "Run: 84, exploration: 0.5, score: 230\n",
      "Scores: (min: 9, avg: 90.67857142857143, max: 296)\n",
      "\n",
      "Run: 85, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 89.88235294117646, max: 296)\n",
      "\n",
      "Run: 86, exploration: 0.5, score: 86\n",
      "Scores: (min: 9, avg: 89.83720930232558, max: 296)\n",
      "\n",
      "Run: 87, exploration: 0.5, score: 212\n",
      "Scores: (min: 9, avg: 91.24137931034483, max: 296)\n",
      "\n",
      "Run: 88, exploration: 0.5, score: 120\n",
      "Scores: (min: 9, avg: 91.56818181818181, max: 296)\n",
      "\n",
      "Run: 89, exploration: 0.5, score: 35\n",
      "Scores: (min: 9, avg: 90.93258426966293, max: 296)\n",
      "\n",
      "Run: 90, exploration: 0.5, score: 159\n",
      "Scores: (min: 9, avg: 91.68888888888888, max: 296)\n",
      "\n",
      "Run: 91, exploration: 0.5, score: 63\n",
      "Scores: (min: 9, avg: 91.37362637362638, max: 296)\n",
      "\n",
      "Run: 92, exploration: 0.5, score: 211\n",
      "Scores: (min: 9, avg: 92.67391304347827, max: 296)\n",
      "\n",
      "Run: 93, exploration: 0.5, score: 73\n",
      "Scores: (min: 9, avg: 92.46236559139786, max: 296)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 94, exploration: 0.5, score: 78\n",
      "Scores: (min: 9, avg: 92.30851063829788, max: 296)\n",
      "\n",
      "Run: 95, exploration: 0.5, score: 55\n",
      "Scores: (min: 9, avg: 91.91578947368421, max: 296)\n",
      "\n",
      "Run: 96, exploration: 0.5, score: 142\n",
      "Scores: (min: 9, avg: 92.4375, max: 296)\n",
      "\n",
      "Run: 97, exploration: 0.5, score: 145\n",
      "Scores: (min: 9, avg: 92.97938144329896, max: 296)\n",
      "\n",
      "Run: 98, exploration: 0.5, score: 198\n",
      "Scores: (min: 9, avg: 94.05102040816327, max: 296)\n",
      "\n",
      "Run: 99, exploration: 0.5, score: 231\n",
      "Scores: (min: 9, avg: 95.43434343434343, max: 296)\n",
      "\n",
      "Run: 100, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 94.74, max: 296)\n",
      "\n",
      "Run: 101, exploration: 0.5, score: 232\n",
      "Scores: (min: 9, avg: 96.82, max: 296)\n",
      "\n",
      "Run: 102, exploration: 0.5, score: 90\n",
      "Scores: (min: 9, avg: 97.52, max: 296)\n",
      "\n",
      "Run: 103, exploration: 0.5, score: 31\n",
      "Scores: (min: 9, avg: 97.59, max: 296)\n",
      "\n",
      "Run: 104, exploration: 0.5, score: 196\n",
      "Scores: (min: 9, avg: 99.38, max: 296)\n",
      "\n",
      "Run: 105, exploration: 0.5, score: 212\n",
      "Scores: (min: 9, avg: 101.39, max: 296)\n",
      "\n",
      "Run: 106, exploration: 0.5, score: 114\n",
      "Scores: (min: 9, avg: 102.37, max: 296)\n",
      "\n",
      "Run: 107, exploration: 0.5, score: 202\n",
      "Scores: (min: 9, avg: 104.24, max: 296)\n",
      "\n",
      "Run: 108, exploration: 0.5, score: 431\n",
      "Scores: (min: 9, avg: 108.34, max: 431)\n",
      "\n",
      "Run: 109, exploration: 0.5, score: 231\n",
      "Scores: (min: 9, avg: 110.55, max: 431)\n",
      "\n",
      "Run: 110, exploration: 0.5, score: 500\n",
      "Scores: (min: 10, avg: 115.46, max: 500)\n",
      "\n",
      "Run: 111, exploration: 0.5, score: 29\n",
      "Scores: (min: 10, avg: 115.64, max: 500)\n",
      "\n",
      "Run: 112, exploration: 0.5, score: 201\n",
      "Scores: (min: 10, avg: 117.55, max: 500)\n",
      "\n",
      "Run: 113, exploration: 0.5, score: 57\n",
      "Scores: (min: 10, avg: 117.89, max: 500)\n",
      "\n",
      "Run: 114, exploration: 0.5, score: 315\n",
      "Scores: (min: 10, avg: 120.92, max: 500)\n",
      "\n",
      "Run: 115, exploration: 0.5, score: 218\n",
      "Scores: (min: 10, avg: 122.93, max: 500)\n",
      "\n",
      "Run: 116, exploration: 0.5, score: 208\n",
      "Scores: (min: 13, avg: 124.91, max: 500)\n",
      "\n",
      "Run: 117, exploration: 0.5, score: 169\n",
      "Scores: (min: 13, avg: 126.45, max: 500)\n",
      "\n",
      "Run: 118, exploration: 0.5, score: 107\n",
      "Scores: (min: 13, avg: 127.15, max: 500)\n",
      "\n",
      "Run: 119, exploration: 0.5, score: 19\n",
      "Scores: (min: 13, avg: 127.17, max: 500)\n",
      "\n",
      "Run: 120, exploration: 0.5, score: 20\n",
      "Scores: (min: 13, avg: 126.84, max: 500)\n",
      "\n",
      "Run: 121, exploration: 0.5, score: 73\n",
      "Scores: (min: 13, avg: 127.12, max: 500)\n",
      "\n",
      "Run: 122, exploration: 0.5, score: 175\n",
      "Scores: (min: 13, avg: 128.34, max: 500)\n",
      "\n",
      "Run: 123, exploration: 0.5, score: 91\n",
      "Scores: (min: 13, avg: 129.01, max: 500)\n",
      "\n",
      "Run: 124, exploration: 0.5, score: 111\n",
      "Scores: (min: 13, avg: 129.38, max: 500)\n",
      "\n",
      "Run: 125, exploration: 0.5, score: 116\n",
      "Scores: (min: 13, avg: 130.31, max: 500)\n",
      "\n",
      "Run: 126, exploration: 0.5, score: 87\n",
      "Scores: (min: 13, avg: 130.69, max: 500)\n",
      "\n",
      "Run: 127, exploration: 0.5, score: 330\n",
      "Scores: (min: 13, avg: 133.02, max: 500)\n",
      "\n",
      "Run: 128, exploration: 0.5, score: 88\n",
      "Scores: (min: 13, avg: 132.85, max: 500)\n",
      "\n",
      "Run: 129, exploration: 0.5, score: 237\n",
      "Scores: (min: 13, avg: 134.99, max: 500)\n",
      "\n",
      "Run: 130, exploration: 0.5, score: 50\n",
      "Scores: (min: 13, avg: 135.27, max: 500)\n",
      "\n",
      "Run: 131, exploration: 0.5, score: 85\n",
      "Scores: (min: 13, avg: 135.2, max: 500)\n",
      "\n",
      "Run: 132, exploration: 0.5, score: 392\n",
      "Scores: (min: 13, avg: 138.1, max: 500)\n",
      "\n",
      "Run: 133, exploration: 0.5, score: 114\n",
      "Scores: (min: 13, avg: 138.14, max: 500)\n",
      "\n",
      "Run: 134, exploration: 0.5, score: 499\n",
      "Scores: (min: 13, avg: 141.13, max: 500)\n",
      "\n",
      "Run: 135, exploration: 0.5, score: 444\n",
      "Scores: (min: 13, avg: 145.06, max: 500)\n",
      "\n",
      "Run: 136, exploration: 0.5, score: 253\n",
      "Scores: (min: 13, avg: 147.12, max: 500)\n",
      "\n",
      "Run: 137, exploration: 0.5, score: 154\n",
      "Scores: (min: 13, avg: 145.7, max: 500)\n",
      "\n",
      "Run: 138, exploration: 0.5, score: 160\n",
      "Scores: (min: 13, avg: 145.73, max: 500)\n",
      "\n",
      "Run: 139, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 148.53, max: 500)\n",
      "\n",
      "Run: 140, exploration: 0.5, score: 94\n",
      "Scores: (min: 13, avg: 148.98, max: 500)\n",
      "\n",
      "Run: 141, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 152.48, max: 500)\n",
      "\n",
      "Run: 142, exploration: 0.5, score: 247\n",
      "Scores: (min: 13, avg: 154.63, max: 500)\n",
      "\n",
      "Run: 143, exploration: 0.5, score: 214\n",
      "Scores: (min: 13, avg: 156.36, max: 500)\n",
      "\n",
      "Run: 144, exploration: 0.5, score: 426\n",
      "Scores: (min: 13, avg: 159.91, max: 500)\n",
      "\n",
      "Run: 145, exploration: 0.5, score: 23\n",
      "Scores: (min: 13, avg: 158.73, max: 500)\n",
      "\n",
      "Run: 146, exploration: 0.5, score: 237\n",
      "Scores: (min: 13, avg: 159.8, max: 500)\n",
      "\n",
      "Run: 147, exploration: 0.5, score: 101\n",
      "Scores: (min: 13, avg: 159.47, max: 500)\n",
      "\n",
      "Run: 148, exploration: 0.5, score: 104\n",
      "Scores: (min: 13, avg: 158.42, max: 500)\n",
      "\n",
      "Run: 149, exploration: 0.5, score: 194\n",
      "Scores: (min: 13, avg: 160.16, max: 500)\n",
      "\n",
      "Run: 150, exploration: 0.5, score: 15\n",
      "Scores: (min: 13, avg: 158.5, max: 500)\n",
      "\n",
      "Run: 151, exploration: 0.5, score: 163\n",
      "Scores: (min: 13, avg: 159.69, max: 500)\n",
      "\n",
      "Run: 152, exploration: 0.5, score: 260\n",
      "Scores: (min: 13, avg: 160.89, max: 500)\n",
      "\n",
      "Run: 153, exploration: 0.5, score: 122\n",
      "Scores: (min: 13, avg: 161.54, max: 500)\n",
      "\n",
      "Run: 154, exploration: 0.5, score: 249\n",
      "Scores: (min: 13, avg: 161.62, max: 500)\n",
      "\n",
      "Run: 155, exploration: 0.5, score: 155\n",
      "Scores: (min: 13, avg: 161.19, max: 500)\n",
      "\n",
      "Run: 156, exploration: 0.5, score: 55\n",
      "Scores: (min: 13, avg: 161.51, max: 500)\n",
      "\n",
      "Run: 157, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 164.79, max: 500)\n",
      "\n",
      "Run: 158, exploration: 0.5, score: 76\n",
      "Scores: (min: 13, avg: 163.36, max: 500)\n",
      "\n",
      "Run: 159, exploration: 0.5, score: 148\n",
      "Scores: (min: 13, avg: 163.53, max: 500)\n",
      "\n",
      "Run: 160, exploration: 0.5, score: 168\n",
      "Scores: (min: 13, avg: 163.52, max: 500)\n",
      "\n",
      "Run: 161, exploration: 0.5, score: 183\n",
      "Scores: (min: 13, avg: 162.96, max: 500)\n",
      "\n",
      "Run: 162, exploration: 0.5, score: 135\n",
      "Scores: (min: 13, avg: 162.6, max: 500)\n",
      "\n",
      "Run: 163, exploration: 0.5, score: 50\n",
      "Scores: (min: 13, avg: 162.1, max: 500)\n",
      "\n",
      "Run: 164, exploration: 0.5, score: 413\n",
      "Scores: (min: 13, avg: 165.44, max: 500)\n",
      "\n",
      "Run: 165, exploration: 0.5, score: 139\n",
      "Scores: (min: 13, avg: 165.28, max: 500)\n",
      "\n",
      "Run: 166, exploration: 0.5, score: 86\n",
      "Scores: (min: 13, avg: 164.42, max: 500)\n",
      "\n",
      "Run: 167, exploration: 0.5, score: 261\n",
      "Scores: (min: 13, avg: 166.71, max: 500)\n",
      "\n",
      "Run: 168, exploration: 0.5, score: 352\n",
      "Scores: (min: 13, avg: 167.8, max: 500)\n",
      "\n",
      "Run: 169, exploration: 0.5, score: 320\n",
      "Scores: (min: 13, avg: 170.35, max: 500)\n",
      "\n",
      "Run: 170, exploration: 0.5, score: 43\n",
      "Scores: (min: 13, avg: 168.65, max: 500)\n",
      "\n",
      "Run: 171, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 172.93, max: 500)\n",
      "\n",
      "Run: 172, exploration: 0.5, score: 82\n",
      "Scores: (min: 13, avg: 172.32, max: 500)\n",
      "\n",
      "Run: 173, exploration: 0.5, score: 99\n",
      "Scores: (min: 15, avg: 173.18, max: 500)\n",
      "\n",
      "Run: 174, exploration: 0.5, score: 500\n",
      "Scores: (min: 15, avg: 177.49, max: 500)\n",
      "\n",
      "Run: 175, exploration: 0.5, score: 57\n",
      "Scores: (min: 15, avg: 176, max: 500)\n",
      "\n",
      "Run: 176, exploration: 0.5, score: 48\n",
      "Scores: (min: 15, avg: 175.2, max: 500)\n",
      "\n",
      "Run: 177, exploration: 0.5, score: 500\n",
      "Scores: (min: 15, avg: 179.03, max: 500)\n",
      "\n",
      "Run: 178, exploration: 0.5, score: 46\n",
      "Scores: (min: 15, avg: 179.33, max: 500)\n",
      "\n",
      "Run: 179, exploration: 0.5, score: 266\n",
      "Scores: (min: 15, avg: 181.83, max: 500)\n",
      "\n",
      "Run: 180, exploration: 0.5, score: 245\n",
      "Scores: (min: 15, avg: 183.3, max: 500)\n",
      "\n",
      "Run: 181, exploration: 0.5, score: 230\n",
      "Scores: (min: 15, avg: 184.52, max: 500)\n",
      "\n",
      "Run: 182, exploration: 0.5, score: 97\n",
      "Scores: (min: 15, avg: 183.81, max: 500)\n",
      "\n",
      "Run: 183, exploration: 0.5, score: 181\n",
      "Scores: (min: 15, avg: 185.12, max: 500)\n",
      "\n",
      "Run: 184, exploration: 0.5, score: 252\n",
      "Scores: (min: 15, avg: 185.34, max: 500)\n",
      "\n",
      "Run: 185, exploration: 0.5, score: 426\n",
      "Scores: (min: 15, avg: 189.37, max: 500)\n",
      "\n",
      "Run: 186, exploration: 0.5, score: 88\n",
      "Scores: (min: 15, avg: 189.39, max: 500)\n",
      "\n",
      "Run: 187, exploration: 0.5, score: 240\n",
      "Scores: (min: 15, avg: 189.67, max: 500)\n",
      "\n",
      "Run: 188, exploration: 0.5, score: 500\n",
      "Scores: (min: 15, avg: 193.47, max: 500)\n",
      "\n",
      "Run: 189, exploration: 0.5, score: 64\n",
      "Scores: (min: 15, avg: 193.76, max: 500)\n",
      "\n",
      "Run: 190, exploration: 0.5, score: 329\n",
      "Scores: (min: 15, avg: 195.46, max: 500)\n",
      "\n",
      "Solved in 90 runs, 190 total runs.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'exit' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-10-e23913c48dc8>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mcartpole\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32m<ipython-input-9-f6dccadf18c1>\u001B[0m in \u001B[0;36mcartpole\u001B[1;34m()\u001B[0m\n\u001B[0;32m     78\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mterminal\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     79\u001B[0m                 \u001B[0mprint\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"Run: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", exploration: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexploration_rate\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", score: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 80\u001B[1;33m                 \u001B[0mscore_logger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0madd_score\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     81\u001B[0m                 \u001B[1;32mbreak\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     82\u001B[0m             \u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexperience_replay\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mU:\\Jupyter\\Cartpole\\scores\\score_logger.py\u001B[0m in \u001B[0;36madd_score\u001B[1;34m(self, score, run)\u001B[0m\n\u001B[0;32m     52\u001B[0m             \u001B[1;31m#               show_trend=False,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     53\u001B[0m             \u001B[1;31m#               show_legend=False)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 54\u001B[1;33m             \u001B[0mexit\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     55\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     56\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_save_png\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0minput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0moutput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mx_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maverage_of_n_last\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_goal\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_trend\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_legend\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'exit' is not defined"
     ]
    }
   ],
   "source": [
    "cartpole()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modified cartpole: increased gamma to 0.995, increased exploration min to 0.5, decreased exploration decay to 0.8\n",
    "  \n",
    "ENV_NAME = \"CartPole-v1\"  \n",
    "  \n",
    "GAMMA = 0.995  \n",
    "LEARNING_RATE = 0.001 \n",
    "  \n",
    "MEMORY_SIZE = 1000000  \n",
    "BATCH_SIZE = 20  \n",
    "  \n",
    "EXPLORATION_MAX = 1.0  \n",
    "EXPLORATION_MIN = 0.5 \n",
    "EXPLORATION_DECAY = 0.8  \n",
    "  \n",
    "  \n",
    "class DQNSolver:  \n",
    "  \n",
    "    def __init__(self, observation_space, action_space):  \n",
    "        self.exploration_rate = EXPLORATION_MAX  \n",
    "  \n",
    "        self.action_space = action_space  \n",
    "        self.memory = deque(maxlen=MEMORY_SIZE)  \n",
    "  \n",
    "        self.model = Sequential()  \n",
    "        self.model.add(Dense(24, input_shape=(observation_space,), activation=\"relu\"))  \n",
    "        self.model.add(Dense(24, activation=\"relu\"))  \n",
    "        self.model.add(Dense(self.action_space, activation=\"linear\"))  \n",
    "        self.model.compile(loss=\"mse\", optimizer=Adam(lr=LEARNING_RATE))  \n",
    "  \n",
    "    def remember(self, state, action, reward, next_state, done):  \n",
    "        self.memory.append((state, action, reward, next_state, done))  \n",
    "  \n",
    "    def act(self, state):  \n",
    "        if np.random.rand() < self.exploration_rate:  \n",
    "            return random.randrange(self.action_space)  \n",
    "        q_values = self.model.predict(state)  \n",
    "        return np.argmax(q_values[0])  \n",
    "  \n",
    "    def experience_replay(self):  \n",
    "        if len(self.memory) < BATCH_SIZE:  \n",
    "            return  \n",
    "        batch = random.sample(self.memory, BATCH_SIZE)  \n",
    "        for state, action, reward, state_next, terminal in batch:  \n",
    "            q_update = reward  \n",
    "            if not terminal:  \n",
    "                q_update = (reward + GAMMA * np.amax(self.model.predict(state_next)[0]))  \n",
    "            q_values = self.model.predict(state)  \n",
    "            q_values[0][action] = q_update  \n",
    "            self.model.fit(state, q_values, verbose=0)  \n",
    "        self.exploration_rate *= EXPLORATION_DECAY  \n",
    "        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)  \n",
    "  \n",
    "  \n",
    "def cartpole():  \n",
    "    env = gym.make(ENV_NAME)  \n",
    "    score_logger = ScoreLogger(ENV_NAME)  \n",
    "    observation_space = env.observation_space.shape[0]  \n",
    "    action_space = env.action_space.n  \n",
    "    dqn_solver = DQNSolver(observation_space, action_space)  \n",
    "    run = 0  \n",
    "    while True:  \n",
    "        run += 1  \n",
    "        state = env.reset()  \n",
    "        state = np.reshape(state, [1, observation_space])  \n",
    "        step = 0  \n",
    "        while True:  \n",
    "            step += 1  \n",
    "            #env.render()  \n",
    "            action = dqn_solver.act(state)  \n",
    "            state_next, reward, terminal, info = env.step(action)  \n",
    "            reward = reward if not terminal else -reward  \n",
    "            state_next = np.reshape(state_next, [1, observation_space])  \n",
    "            dqn_solver.remember(state, action, reward, state_next, terminal)  \n",
    "            state = state_next  \n",
    "            if terminal:  \n",
    "                print (\"Run: \" + str(run) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(step))  \n",
    "                score_logger.add_score(step, run)  \n",
    "                break  \n",
    "            dqn_solver.experience_replay()  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 1, exploration: 1.0, score: 14\n",
      "Scores: (min: 14, avg: 14, max: 14)\n",
      "\n",
      "Run: 2, exploration: 0.5, score: 28\n",
      "Scores: (min: 14, avg: 21, max: 28)\n",
      "\n",
      "Run: 3, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 17.666666666666668, max: 28)\n",
      "\n",
      "Run: 4, exploration: 0.5, score: 13\n",
      "Scores: (min: 11, avg: 16.5, max: 28)\n",
      "\n",
      "Run: 5, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 15.4, max: 28)\n",
      "\n",
      "Run: 6, exploration: 0.5, score: 20\n",
      "Scores: (min: 11, avg: 16.166666666666668, max: 28)\n",
      "\n",
      "Run: 7, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 15.428571428571429, max: 28)\n",
      "\n",
      "Run: 8, exploration: 0.5, score: 15\n",
      "Scores: (min: 11, avg: 15.375, max: 28)\n",
      "\n",
      "Run: 9, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 14.88888888888889, max: 28)\n",
      "\n",
      "Run: 10, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 14.5, max: 28)\n",
      "\n",
      "Run: 11, exploration: 0.5, score: 18\n",
      "Scores: (min: 11, avg: 14.818181818181818, max: 28)\n",
      "\n",
      "Run: 12, exploration: 0.5, score: 10\n",
      "Scores: (min: 10, avg: 14.416666666666666, max: 28)\n",
      "\n",
      "Run: 13, exploration: 0.5, score: 13\n",
      "Scores: (min: 10, avg: 14.307692307692308, max: 28)\n",
      "\n",
      "Run: 14, exploration: 0.5, score: 39\n",
      "Scores: (min: 10, avg: 16.071428571428573, max: 39)\n",
      "\n",
      "Run: 15, exploration: 0.5, score: 12\n",
      "Scores: (min: 10, avg: 15.8, max: 39)\n",
      "\n",
      "Run: 16, exploration: 0.5, score: 20\n",
      "Scores: (min: 10, avg: 16.0625, max: 39)\n",
      "\n",
      "Run: 17, exploration: 0.5, score: 22\n",
      "Scores: (min: 10, avg: 16.41176470588235, max: 39)\n",
      "\n",
      "Run: 18, exploration: 0.5, score: 22\n",
      "Scores: (min: 10, avg: 16.72222222222222, max: 39)\n",
      "\n",
      "Run: 19, exploration: 0.5, score: 10\n",
      "Scores: (min: 10, avg: 16.36842105263158, max: 39)\n",
      "\n",
      "Run: 20, exploration: 0.5, score: 14\n",
      "Scores: (min: 10, avg: 16.25, max: 39)\n",
      "\n",
      "Run: 21, exploration: 0.5, score: 14\n",
      "Scores: (min: 10, avg: 16.142857142857142, max: 39)\n",
      "\n",
      "Run: 22, exploration: 0.5, score: 21\n",
      "Scores: (min: 10, avg: 16.363636363636363, max: 39)\n",
      "\n",
      "Run: 23, exploration: 0.5, score: 34\n",
      "Scores: (min: 10, avg: 17.130434782608695, max: 39)\n",
      "\n",
      "Run: 24, exploration: 0.5, score: 30\n",
      "Scores: (min: 10, avg: 17.666666666666668, max: 39)\n",
      "\n",
      "Run: 25, exploration: 0.5, score: 15\n",
      "Scores: (min: 10, avg: 17.56, max: 39)\n",
      "\n",
      "Run: 26, exploration: 0.5, score: 12\n",
      "Scores: (min: 10, avg: 17.346153846153847, max: 39)\n",
      "\n",
      "Run: 27, exploration: 0.5, score: 23\n",
      "Scores: (min: 10, avg: 17.555555555555557, max: 39)\n",
      "\n",
      "Run: 28, exploration: 0.5, score: 97\n",
      "Scores: (min: 10, avg: 20.392857142857142, max: 97)\n",
      "\n",
      "Run: 29, exploration: 0.5, score: 28\n",
      "Scores: (min: 10, avg: 20.655172413793103, max: 97)\n",
      "\n",
      "Run: 30, exploration: 0.5, score: 33\n",
      "Scores: (min: 10, avg: 21.066666666666666, max: 97)\n",
      "\n",
      "Run: 31, exploration: 0.5, score: 20\n",
      "Scores: (min: 10, avg: 21.032258064516128, max: 97)\n",
      "\n",
      "Run: 32, exploration: 0.5, score: 53\n",
      "Scores: (min: 10, avg: 22.03125, max: 97)\n",
      "\n",
      "Run: 33, exploration: 0.5, score: 61\n",
      "Scores: (min: 10, avg: 23.21212121212121, max: 97)\n",
      "\n",
      "Run: 34, exploration: 0.5, score: 34\n",
      "Scores: (min: 10, avg: 23.529411764705884, max: 97)\n",
      "\n",
      "Run: 35, exploration: 0.5, score: 48\n",
      "Scores: (min: 10, avg: 24.228571428571428, max: 97)\n",
      "\n",
      "Run: 36, exploration: 0.5, score: 32\n",
      "Scores: (min: 10, avg: 24.444444444444443, max: 97)\n",
      "\n",
      "Run: 37, exploration: 0.5, score: 53\n",
      "Scores: (min: 10, avg: 25.216216216216218, max: 97)\n",
      "\n",
      "Run: 38, exploration: 0.5, score: 47\n",
      "Scores: (min: 10, avg: 25.789473684210527, max: 97)\n",
      "\n",
      "Run: 39, exploration: 0.5, score: 39\n",
      "Scores: (min: 10, avg: 26.128205128205128, max: 97)\n",
      "\n",
      "Run: 40, exploration: 0.5, score: 131\n",
      "Scores: (min: 10, avg: 28.75, max: 131)\n",
      "\n",
      "Run: 41, exploration: 0.5, score: 115\n",
      "Scores: (min: 10, avg: 30.853658536585368, max: 131)\n",
      "\n",
      "Run: 42, exploration: 0.5, score: 83\n",
      "Scores: (min: 10, avg: 32.095238095238095, max: 131)\n",
      "\n",
      "Run: 43, exploration: 0.5, score: 74\n",
      "Scores: (min: 10, avg: 33.06976744186046, max: 131)\n",
      "\n",
      "Run: 44, exploration: 0.5, score: 108\n",
      "Scores: (min: 10, avg: 34.77272727272727, max: 131)\n",
      "\n",
      "Run: 45, exploration: 0.5, score: 230\n",
      "Scores: (min: 10, avg: 39.111111111111114, max: 230)\n",
      "\n",
      "Run: 46, exploration: 0.5, score: 65\n",
      "Scores: (min: 10, avg: 39.67391304347826, max: 230)\n",
      "\n",
      "Run: 47, exploration: 0.5, score: 122\n",
      "Scores: (min: 10, avg: 41.42553191489362, max: 230)\n",
      "\n",
      "Run: 48, exploration: 0.5, score: 284\n",
      "Scores: (min: 10, avg: 46.479166666666664, max: 284)\n",
      "\n",
      "Run: 49, exploration: 0.5, score: 42\n",
      "Scores: (min: 10, avg: 46.38775510204081, max: 284)\n",
      "\n",
      "Run: 50, exploration: 0.5, score: 84\n",
      "Scores: (min: 10, avg: 47.14, max: 284)\n",
      "\n",
      "Run: 51, exploration: 0.5, score: 25\n",
      "Scores: (min: 10, avg: 46.705882352941174, max: 284)\n",
      "\n",
      "Run: 52, exploration: 0.5, score: 105\n",
      "Scores: (min: 10, avg: 47.82692307692308, max: 284)\n",
      "\n",
      "Run: 53, exploration: 0.5, score: 310\n",
      "Scores: (min: 10, avg: 52.77358490566038, max: 310)\n",
      "\n",
      "Run: 54, exploration: 0.5, score: 15\n",
      "Scores: (min: 10, avg: 52.074074074074076, max: 310)\n",
      "\n",
      "Run: 55, exploration: 0.5, score: 201\n",
      "Scores: (min: 10, avg: 54.78181818181818, max: 310)\n",
      "\n",
      "Run: 56, exploration: 0.5, score: 25\n",
      "Scores: (min: 10, avg: 54.25, max: 310)\n",
      "\n",
      "Run: 57, exploration: 0.5, score: 283\n",
      "Scores: (min: 10, avg: 58.26315789473684, max: 310)\n",
      "\n",
      "Run: 58, exploration: 0.5, score: 230\n",
      "Scores: (min: 10, avg: 61.224137931034484, max: 310)\n",
      "\n",
      "Run: 59, exploration: 0.5, score: 318\n",
      "Scores: (min: 10, avg: 65.57627118644068, max: 318)\n",
      "\n",
      "Run: 60, exploration: 0.5, score: 195\n",
      "Scores: (min: 10, avg: 67.73333333333333, max: 318)\n",
      "\n",
      "Run: 61, exploration: 0.5, score: 339\n",
      "Scores: (min: 10, avg: 72.18032786885246, max: 339)\n",
      "\n",
      "Run: 62, exploration: 0.5, score: 87\n",
      "Scores: (min: 10, avg: 72.41935483870968, max: 339)\n",
      "\n",
      "Run: 63, exploration: 0.5, score: 117\n",
      "Scores: (min: 10, avg: 73.12698412698413, max: 339)\n",
      "\n",
      "Run: 64, exploration: 0.5, score: 34\n",
      "Scores: (min: 10, avg: 72.515625, max: 339)\n",
      "\n",
      "Run: 65, exploration: 0.5, score: 213\n",
      "Scores: (min: 10, avg: 74.67692307692307, max: 339)\n",
      "\n",
      "Run: 66, exploration: 0.5, score: 70\n",
      "Scores: (min: 10, avg: 74.60606060606061, max: 339)\n",
      "\n",
      "Run: 67, exploration: 0.5, score: 88\n",
      "Scores: (min: 10, avg: 74.80597014925372, max: 339)\n",
      "\n",
      "Run: 68, exploration: 0.5, score: 218\n",
      "Scores: (min: 10, avg: 76.91176470588235, max: 339)\n",
      "\n",
      "Run: 69, exploration: 0.5, score: 151\n",
      "Scores: (min: 10, avg: 77.98550724637681, max: 339)\n",
      "\n",
      "Run: 70, exploration: 0.5, score: 125\n",
      "Scores: (min: 10, avg: 78.65714285714286, max: 339)\n",
      "\n",
      "Run: 71, exploration: 0.5, score: 204\n",
      "Scores: (min: 10, avg: 80.4225352112676, max: 339)\n",
      "\n",
      "Run: 72, exploration: 0.5, score: 142\n",
      "Scores: (min: 10, avg: 81.27777777777777, max: 339)\n",
      "\n",
      "Run: 73, exploration: 0.5, score: 242\n",
      "Scores: (min: 10, avg: 83.47945205479452, max: 339)\n",
      "\n",
      "Run: 74, exploration: 0.5, score: 128\n",
      "Scores: (min: 10, avg: 84.08108108108108, max: 339)\n",
      "\n",
      "Run: 75, exploration: 0.5, score: 95\n",
      "Scores: (min: 10, avg: 84.22666666666667, max: 339)\n",
      "\n",
      "Run: 76, exploration: 0.5, score: 105\n",
      "Scores: (min: 10, avg: 84.5, max: 339)\n",
      "\n",
      "Run: 77, exploration: 0.5, score: 163\n",
      "Scores: (min: 10, avg: 85.51948051948052, max: 339)\n",
      "\n",
      "Run: 78, exploration: 0.5, score: 44\n",
      "Scores: (min: 10, avg: 84.98717948717949, max: 339)\n",
      "\n",
      "Run: 79, exploration: 0.5, score: 136\n",
      "Scores: (min: 10, avg: 85.63291139240506, max: 339)\n",
      "\n",
      "Run: 80, exploration: 0.5, score: 203\n",
      "Scores: (min: 10, avg: 87.1, max: 339)\n",
      "\n",
      "Run: 81, exploration: 0.5, score: 237\n",
      "Scores: (min: 10, avg: 88.95061728395062, max: 339)\n",
      "\n",
      "Run: 82, exploration: 0.5, score: 162\n",
      "Scores: (min: 10, avg: 89.84146341463415, max: 339)\n",
      "\n",
      "Run: 83, exploration: 0.5, score: 274\n",
      "Scores: (min: 10, avg: 92.06024096385542, max: 339)\n",
      "\n",
      "Run: 84, exploration: 0.5, score: 19\n",
      "Scores: (min: 10, avg: 91.19047619047619, max: 339)\n",
      "\n",
      "Run: 85, exploration: 0.5, score: 25\n",
      "Scores: (min: 10, avg: 90.41176470588235, max: 339)\n",
      "\n",
      "Run: 86, exploration: 0.5, score: 212\n",
      "Scores: (min: 10, avg: 91.82558139534883, max: 339)\n",
      "\n",
      "Run: 87, exploration: 0.5, score: 64\n",
      "Scores: (min: 10, avg: 91.50574712643679, max: 339)\n",
      "\n",
      "Run: 88, exploration: 0.5, score: 20\n",
      "Scores: (min: 10, avg: 90.69318181818181, max: 339)\n",
      "\n",
      "Run: 89, exploration: 0.5, score: 24\n",
      "Scores: (min: 10, avg: 89.9438202247191, max: 339)\n",
      "\n",
      "Run: 90, exploration: 0.5, score: 99\n",
      "Scores: (min: 10, avg: 90.04444444444445, max: 339)\n",
      "\n",
      "Run: 91, exploration: 0.5, score: 241\n",
      "Scores: (min: 10, avg: 91.7032967032967, max: 339)\n",
      "\n",
      "Run: 92, exploration: 0.5, score: 148\n",
      "Scores: (min: 10, avg: 92.31521739130434, max: 339)\n",
      "\n",
      "Run: 93, exploration: 0.5, score: 204\n",
      "Scores: (min: 10, avg: 93.51612903225806, max: 339)\n",
      "\n",
      "Run: 94, exploration: 0.5, score: 248\n",
      "Scores: (min: 10, avg: 95.15957446808511, max: 339)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 95, exploration: 0.5, score: 53\n",
      "Scores: (min: 10, avg: 94.71578947368421, max: 339)\n",
      "\n",
      "Run: 96, exploration: 0.5, score: 78\n",
      "Scores: (min: 10, avg: 94.54166666666667, max: 339)\n",
      "\n",
      "Run: 97, exploration: 0.5, score: 66\n",
      "Scores: (min: 10, avg: 94.24742268041237, max: 339)\n",
      "\n",
      "Run: 98, exploration: 0.5, score: 248\n",
      "Scores: (min: 10, avg: 95.81632653061224, max: 339)\n",
      "\n",
      "Run: 99, exploration: 0.5, score: 32\n",
      "Scores: (min: 10, avg: 95.17171717171718, max: 339)\n",
      "\n",
      "Run: 100, exploration: 0.5, score: 22\n",
      "Scores: (min: 10, avg: 94.44, max: 339)\n",
      "\n",
      "Run: 101, exploration: 0.5, score: 88\n",
      "Scores: (min: 10, avg: 95.18, max: 339)\n",
      "\n",
      "Run: 102, exploration: 0.5, score: 83\n",
      "Scores: (min: 10, avg: 95.73, max: 339)\n",
      "\n",
      "Run: 103, exploration: 0.5, score: 133\n",
      "Scores: (min: 10, avg: 96.95, max: 339)\n",
      "\n",
      "Run: 104, exploration: 0.5, score: 178\n",
      "Scores: (min: 10, avg: 98.6, max: 339)\n",
      "\n",
      "Run: 105, exploration: 0.5, score: 71\n",
      "Scores: (min: 10, avg: 99.2, max: 339)\n",
      "\n",
      "Run: 106, exploration: 0.5, score: 207\n",
      "Scores: (min: 10, avg: 101.07, max: 339)\n",
      "\n",
      "Run: 107, exploration: 0.5, score: 115\n",
      "Scores: (min: 10, avg: 102.11, max: 339)\n",
      "\n",
      "Run: 108, exploration: 0.5, score: 370\n",
      "Scores: (min: 10, avg: 105.66, max: 370)\n",
      "\n",
      "Run: 109, exploration: 0.5, score: 126\n",
      "Scores: (min: 10, avg: 106.81, max: 370)\n",
      "\n",
      "Run: 110, exploration: 0.5, score: 122\n",
      "Scores: (min: 10, avg: 107.92, max: 370)\n",
      "\n",
      "Run: 111, exploration: 0.5, score: 124\n",
      "Scores: (min: 10, avg: 108.98, max: 370)\n",
      "\n",
      "Run: 112, exploration: 0.5, score: 171\n",
      "Scores: (min: 10, avg: 110.59, max: 370)\n",
      "\n",
      "Run: 113, exploration: 0.5, score: 171\n",
      "Scores: (min: 10, avg: 112.17, max: 370)\n",
      "\n",
      "Run: 114, exploration: 0.5, score: 54\n",
      "Scores: (min: 10, avg: 112.32, max: 370)\n",
      "\n",
      "Run: 115, exploration: 0.5, score: 91\n",
      "Scores: (min: 10, avg: 113.11, max: 370)\n",
      "\n",
      "Run: 116, exploration: 0.5, score: 184\n",
      "Scores: (min: 10, avg: 114.75, max: 370)\n",
      "\n",
      "Run: 117, exploration: 0.5, score: 127\n",
      "Scores: (min: 10, avg: 115.8, max: 370)\n",
      "\n",
      "Run: 118, exploration: 0.5, score: 237\n",
      "Scores: (min: 10, avg: 117.95, max: 370)\n",
      "\n",
      "Run: 119, exploration: 0.5, score: 369\n",
      "Scores: (min: 12, avg: 121.54, max: 370)\n",
      "\n",
      "Run: 120, exploration: 0.5, score: 196\n",
      "Scores: (min: 12, avg: 123.36, max: 370)\n",
      "\n",
      "Run: 121, exploration: 0.5, score: 433\n",
      "Scores: (min: 12, avg: 127.55, max: 433)\n",
      "\n",
      "Run: 122, exploration: 0.5, score: 211\n",
      "Scores: (min: 12, avg: 129.45, max: 433)\n",
      "\n",
      "Run: 123, exploration: 0.5, score: 79\n",
      "Scores: (min: 12, avg: 129.9, max: 433)\n",
      "\n",
      "Run: 124, exploration: 0.5, score: 155\n",
      "Scores: (min: 12, avg: 131.15, max: 433)\n",
      "\n",
      "Run: 125, exploration: 0.5, score: 222\n",
      "Scores: (min: 12, avg: 133.22, max: 433)\n",
      "\n",
      "Run: 126, exploration: 0.5, score: 169\n",
      "Scores: (min: 15, avg: 134.79, max: 433)\n",
      "\n",
      "Run: 127, exploration: 0.5, score: 187\n",
      "Scores: (min: 15, avg: 136.43, max: 433)\n",
      "\n",
      "Run: 128, exploration: 0.5, score: 243\n",
      "Scores: (min: 15, avg: 137.89, max: 433)\n",
      "\n",
      "Run: 129, exploration: 0.5, score: 181\n",
      "Scores: (min: 15, avg: 139.42, max: 433)\n",
      "\n",
      "Run: 130, exploration: 0.5, score: 146\n",
      "Scores: (min: 15, avg: 140.55, max: 433)\n",
      "\n",
      "Run: 131, exploration: 0.5, score: 69\n",
      "Scores: (min: 15, avg: 141.04, max: 433)\n",
      "\n",
      "Run: 132, exploration: 0.5, score: 54\n",
      "Scores: (min: 15, avg: 141.05, max: 433)\n",
      "\n",
      "Run: 133, exploration: 0.5, score: 280\n",
      "Scores: (min: 15, avg: 143.24, max: 433)\n",
      "\n",
      "Run: 134, exploration: 0.5, score: 41\n",
      "Scores: (min: 15, avg: 143.31, max: 433)\n",
      "\n",
      "Run: 135, exploration: 0.5, score: 87\n",
      "Scores: (min: 15, avg: 143.7, max: 433)\n",
      "\n",
      "Run: 136, exploration: 0.5, score: 239\n",
      "Scores: (min: 15, avg: 145.77, max: 433)\n",
      "\n",
      "Run: 137, exploration: 0.5, score: 42\n",
      "Scores: (min: 15, avg: 145.66, max: 433)\n",
      "\n",
      "Run: 138, exploration: 0.5, score: 205\n",
      "Scores: (min: 15, avg: 147.24, max: 433)\n",
      "\n",
      "Run: 139, exploration: 0.5, score: 58\n",
      "Scores: (min: 15, avg: 147.43, max: 433)\n",
      "\n",
      "Run: 140, exploration: 0.5, score: 190\n",
      "Scores: (min: 15, avg: 148.02, max: 433)\n",
      "\n",
      "Run: 141, exploration: 0.5, score: 64\n",
      "Scores: (min: 15, avg: 147.51, max: 433)\n",
      "\n",
      "Run: 142, exploration: 0.5, score: 93\n",
      "Scores: (min: 15, avg: 147.61, max: 433)\n",
      "\n",
      "Run: 143, exploration: 0.5, score: 340\n",
      "Scores: (min: 15, avg: 150.27, max: 433)\n",
      "\n",
      "Run: 144, exploration: 0.5, score: 23\n",
      "Scores: (min: 15, avg: 149.42, max: 433)\n",
      "\n",
      "Run: 145, exploration: 0.5, score: 267\n",
      "Scores: (min: 15, avg: 149.79, max: 433)\n",
      "\n",
      "Run: 146, exploration: 0.5, score: 62\n",
      "Scores: (min: 15, avg: 149.76, max: 433)\n",
      "\n",
      "Run: 147, exploration: 0.5, score: 72\n",
      "Scores: (min: 15, avg: 149.26, max: 433)\n",
      "\n",
      "Run: 148, exploration: 0.5, score: 30\n",
      "Scores: (min: 15, avg: 146.72, max: 433)\n",
      "\n",
      "Run: 149, exploration: 0.5, score: 31\n",
      "Scores: (min: 15, avg: 146.61, max: 433)\n",
      "\n",
      "Run: 150, exploration: 0.5, score: 296\n",
      "Scores: (min: 15, avg: 148.73, max: 433)\n",
      "\n",
      "Run: 151, exploration: 0.5, score: 50\n",
      "Scores: (min: 15, avg: 148.98, max: 433)\n",
      "\n",
      "Run: 152, exploration: 0.5, score: 294\n",
      "Scores: (min: 15, avg: 150.87, max: 433)\n",
      "\n",
      "Run: 153, exploration: 0.5, score: 52\n",
      "Scores: (min: 15, avg: 148.29, max: 433)\n",
      "\n",
      "Run: 154, exploration: 0.5, score: 13\n",
      "Scores: (min: 13, avg: 148.27, max: 433)\n",
      "\n",
      "Run: 155, exploration: 0.5, score: 360\n",
      "Scores: (min: 13, avg: 149.86, max: 433)\n",
      "\n",
      "Run: 156, exploration: 0.5, score: 49\n",
      "Scores: (min: 13, avg: 150.1, max: 433)\n",
      "\n",
      "Run: 157, exploration: 0.5, score: 197\n",
      "Scores: (min: 13, avg: 149.24, max: 433)\n",
      "\n",
      "Run: 158, exploration: 0.5, score: 143\n",
      "Scores: (min: 13, avg: 148.37, max: 433)\n",
      "\n",
      "Run: 159, exploration: 0.5, score: 55\n",
      "Scores: (min: 13, avg: 145.74, max: 433)\n",
      "\n",
      "Run: 160, exploration: 0.5, score: 151\n",
      "Scores: (min: 13, avg: 145.3, max: 433)\n",
      "\n",
      "Run: 161, exploration: 0.5, score: 213\n",
      "Scores: (min: 13, avg: 144.04, max: 433)\n",
      "\n",
      "Run: 162, exploration: 0.5, score: 309\n",
      "Scores: (min: 13, avg: 146.26, max: 433)\n",
      "\n",
      "Run: 163, exploration: 0.5, score: 212\n",
      "Scores: (min: 13, avg: 147.21, max: 433)\n",
      "\n",
      "Run: 164, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 146.98, max: 433)\n",
      "\n",
      "Run: 165, exploration: 0.5, score: 180\n",
      "Scores: (min: 11, avg: 146.65, max: 433)\n",
      "\n",
      "Run: 166, exploration: 0.5, score: 95\n",
      "Scores: (min: 11, avg: 146.9, max: 433)\n",
      "\n",
      "Run: 167, exploration: 0.5, score: 59\n",
      "Scores: (min: 11, avg: 146.61, max: 433)\n",
      "\n",
      "Run: 168, exploration: 0.5, score: 24\n",
      "Scores: (min: 11, avg: 144.67, max: 433)\n",
      "\n",
      "Run: 169, exploration: 0.5, score: 153\n",
      "Scores: (min: 11, avg: 144.69, max: 433)\n",
      "\n",
      "Run: 170, exploration: 0.5, score: 209\n",
      "Scores: (min: 11, avg: 145.53, max: 433)\n",
      "\n",
      "Run: 171, exploration: 0.5, score: 70\n",
      "Scores: (min: 11, avg: 144.19, max: 433)\n",
      "\n",
      "Run: 172, exploration: 0.5, score: 98\n",
      "Scores: (min: 11, avg: 143.75, max: 433)\n",
      "\n",
      "Run: 173, exploration: 0.5, score: 46\n",
      "Scores: (min: 11, avg: 141.79, max: 433)\n",
      "\n",
      "Run: 174, exploration: 0.5, score: 179\n",
      "Scores: (min: 11, avg: 142.3, max: 433)\n",
      "\n",
      "Run: 175, exploration: 0.5, score: 160\n",
      "Scores: (min: 11, avg: 142.95, max: 433)\n",
      "\n",
      "Run: 176, exploration: 0.5, score: 13\n",
      "Scores: (min: 11, avg: 142.03, max: 433)\n",
      "\n",
      "Run: 177, exploration: 0.5, score: 170\n",
      "Scores: (min: 11, avg: 142.1, max: 433)\n",
      "\n",
      "Run: 178, exploration: 0.5, score: 158\n",
      "Scores: (min: 11, avg: 143.24, max: 433)\n",
      "\n",
      "Run: 179, exploration: 0.5, score: 75\n",
      "Scores: (min: 11, avg: 142.63, max: 433)\n",
      "\n",
      "Run: 180, exploration: 0.5, score: 220\n",
      "Scores: (min: 11, avg: 142.8, max: 433)\n",
      "\n",
      "Run: 181, exploration: 0.5, score: 170\n",
      "Scores: (min: 11, avg: 142.13, max: 433)\n",
      "\n",
      "Run: 182, exploration: 0.5, score: 87\n",
      "Scores: (min: 11, avg: 141.38, max: 433)\n",
      "\n",
      "Run: 183, exploration: 0.5, score: 288\n",
      "Scores: (min: 11, avg: 141.52, max: 433)\n",
      "\n",
      "Run: 184, exploration: 0.5, score: 38\n",
      "Scores: (min: 11, avg: 141.71, max: 433)\n",
      "\n",
      "Run: 185, exploration: 0.5, score: 162\n",
      "Scores: (min: 11, avg: 143.08, max: 433)\n",
      "\n",
      "Run: 186, exploration: 0.5, score: 233\n",
      "Scores: (min: 11, avg: 143.29, max: 433)\n",
      "\n",
      "Run: 187, exploration: 0.5, score: 220\n",
      "Scores: (min: 11, avg: 144.85, max: 433)\n",
      "\n",
      "Run: 188, exploration: 0.5, score: 226\n",
      "Scores: (min: 11, avg: 146.91, max: 433)\n",
      "\n",
      "Run: 189, exploration: 0.5, score: 121\n",
      "Scores: (min: 11, avg: 147.88, max: 433)\n",
      "\n",
      "Run: 190, exploration: 0.5, score: 213\n",
      "Scores: (min: 11, avg: 149.02, max: 433)\n",
      "\n",
      "Run: 191, exploration: 0.5, score: 133\n",
      "Scores: (min: 11, avg: 147.94, max: 433)\n",
      "\n",
      "Run: 192, exploration: 0.5, score: 243\n",
      "Scores: (min: 11, avg: 148.89, max: 433)\n",
      "\n",
      "Run: 193, exploration: 0.5, score: 158\n",
      "Scores: (min: 11, avg: 148.43, max: 433)\n",
      "\n",
      "Run: 194, exploration: 0.5, score: 157\n",
      "Scores: (min: 11, avg: 147.52, max: 433)\n",
      "\n",
      "Run: 195, exploration: 0.5, score: 274\n",
      "Scores: (min: 11, avg: 149.73, max: 433)\n",
      "\n",
      "Run: 196, exploration: 0.5, score: 68\n",
      "Scores: (min: 11, avg: 149.63, max: 433)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 197, exploration: 0.5, score: 120\n",
      "Scores: (min: 11, avg: 150.17, max: 433)\n",
      "\n",
      "Run: 198, exploration: 0.5, score: 17\n",
      "Scores: (min: 11, avg: 147.86, max: 433)\n",
      "\n",
      "Run: 199, exploration: 0.5, score: 61\n",
      "Scores: (min: 11, avg: 148.15, max: 433)\n",
      "\n",
      "Run: 200, exploration: 0.5, score: 240\n",
      "Scores: (min: 11, avg: 150.33, max: 433)\n",
      "\n",
      "Run: 201, exploration: 0.5, score: 69\n",
      "Scores: (min: 11, avg: 150.14, max: 433)\n",
      "\n",
      "Run: 202, exploration: 0.5, score: 225\n",
      "Scores: (min: 11, avg: 151.56, max: 433)\n",
      "\n",
      "Run: 203, exploration: 0.5, score: 149\n",
      "Scores: (min: 11, avg: 151.72, max: 433)\n",
      "\n",
      "Run: 204, exploration: 0.5, score: 226\n",
      "Scores: (min: 11, avg: 152.2, max: 433)\n",
      "\n",
      "Run: 205, exploration: 0.5, score: 115\n",
      "Scores: (min: 11, avg: 152.64, max: 433)\n",
      "\n",
      "Run: 206, exploration: 0.5, score: 40\n",
      "Scores: (min: 11, avg: 150.97, max: 433)\n",
      "\n",
      "Run: 207, exploration: 0.5, score: 99\n",
      "Scores: (min: 11, avg: 150.81, max: 433)\n",
      "\n",
      "Run: 208, exploration: 0.5, score: 72\n",
      "Scores: (min: 11, avg: 147.83, max: 433)\n",
      "\n",
      "Run: 209, exploration: 0.5, score: 199\n",
      "Scores: (min: 11, avg: 148.56, max: 433)\n",
      "\n",
      "Run: 210, exploration: 0.5, score: 97\n",
      "Scores: (min: 11, avg: 148.31, max: 433)\n",
      "\n",
      "Run: 211, exploration: 0.5, score: 195\n",
      "Scores: (min: 11, avg: 149.02, max: 433)\n",
      "\n",
      "Run: 212, exploration: 0.5, score: 46\n",
      "Scores: (min: 11, avg: 147.77, max: 433)\n",
      "\n",
      "Run: 213, exploration: 0.5, score: 61\n",
      "Scores: (min: 11, avg: 146.67, max: 433)\n",
      "\n",
      "Run: 214, exploration: 0.5, score: 76\n",
      "Scores: (min: 11, avg: 146.89, max: 433)\n",
      "\n",
      "Run: 215, exploration: 0.5, score: 206\n",
      "Scores: (min: 11, avg: 148.04, max: 433)\n",
      "\n",
      "Run: 216, exploration: 0.5, score: 250\n",
      "Scores: (min: 11, avg: 148.7, max: 433)\n",
      "\n",
      "Run: 217, exploration: 0.5, score: 103\n",
      "Scores: (min: 11, avg: 148.46, max: 433)\n",
      "\n",
      "Run: 218, exploration: 0.5, score: 185\n",
      "Scores: (min: 11, avg: 147.94, max: 433)\n",
      "\n",
      "Run: 219, exploration: 0.5, score: 176\n",
      "Scores: (min: 11, avg: 146.01, max: 433)\n",
      "\n",
      "Run: 220, exploration: 0.5, score: 392\n",
      "Scores: (min: 11, avg: 147.97, max: 433)\n",
      "\n",
      "Run: 221, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 143.75, max: 392)\n",
      "\n",
      "Run: 222, exploration: 0.5, score: 180\n",
      "Scores: (min: 11, avg: 143.44, max: 392)\n",
      "\n",
      "Run: 223, exploration: 0.5, score: 25\n",
      "Scores: (min: 11, avg: 142.9, max: 392)\n",
      "\n",
      "Run: 224, exploration: 0.5, score: 56\n",
      "Scores: (min: 11, avg: 141.91, max: 392)\n",
      "\n",
      "Run: 225, exploration: 0.5, score: 231\n",
      "Scores: (min: 11, avg: 142, max: 392)\n",
      "\n",
      "Run: 226, exploration: 0.5, score: 261\n",
      "Scores: (min: 11, avg: 142.92, max: 392)\n",
      "\n",
      "Run: 227, exploration: 0.5, score: 170\n",
      "Scores: (min: 11, avg: 142.75, max: 392)\n",
      "\n",
      "Run: 228, exploration: 0.5, score: 67\n",
      "Scores: (min: 11, avg: 140.99, max: 392)\n",
      "\n",
      "Run: 229, exploration: 0.5, score: 234\n",
      "Scores: (min: 11, avg: 141.52, max: 392)\n",
      "\n",
      "Run: 230, exploration: 0.5, score: 46\n",
      "Scores: (min: 11, avg: 140.52, max: 392)\n",
      "\n",
      "Run: 231, exploration: 0.5, score: 250\n",
      "Scores: (min: 11, avg: 142.33, max: 392)\n",
      "\n",
      "Run: 232, exploration: 0.5, score: 40\n",
      "Scores: (min: 11, avg: 142.19, max: 392)\n",
      "\n",
      "Run: 233, exploration: 0.5, score: 20\n",
      "Scores: (min: 11, avg: 139.59, max: 392)\n",
      "\n",
      "Run: 234, exploration: 0.5, score: 120\n",
      "Scores: (min: 11, avg: 140.38, max: 392)\n",
      "\n",
      "Run: 235, exploration: 0.5, score: 15\n",
      "Scores: (min: 11, avg: 139.66, max: 392)\n",
      "\n",
      "Run: 236, exploration: 0.5, score: 179\n",
      "Scores: (min: 11, avg: 139.06, max: 392)\n",
      "\n",
      "Run: 237, exploration: 0.5, score: 304\n",
      "Scores: (min: 11, avg: 141.68, max: 392)\n",
      "\n",
      "Run: 238, exploration: 0.5, score: 87\n",
      "Scores: (min: 11, avg: 140.5, max: 392)\n",
      "\n",
      "Run: 239, exploration: 0.5, score: 214\n",
      "Scores: (min: 11, avg: 142.06, max: 392)\n",
      "\n",
      "Run: 240, exploration: 0.5, score: 194\n",
      "Scores: (min: 11, avg: 142.1, max: 392)\n",
      "\n",
      "Run: 241, exploration: 0.5, score: 194\n",
      "Scores: (min: 11, avg: 143.4, max: 392)\n",
      "\n",
      "Run: 242, exploration: 0.5, score: 363\n",
      "Scores: (min: 11, avg: 146.1, max: 392)\n",
      "\n",
      "Run: 243, exploration: 0.5, score: 185\n",
      "Scores: (min: 11, avg: 144.55, max: 392)\n",
      "\n",
      "Run: 244, exploration: 0.5, score: 62\n",
      "Scores: (min: 11, avg: 144.94, max: 392)\n",
      "\n",
      "Run: 245, exploration: 0.5, score: 212\n",
      "Scores: (min: 11, avg: 144.39, max: 392)\n",
      "\n",
      "Run: 246, exploration: 0.5, score: 20\n",
      "Scores: (min: 11, avg: 143.97, max: 392)\n",
      "\n",
      "Run: 247, exploration: 0.5, score: 25\n",
      "Scores: (min: 11, avg: 143.5, max: 392)\n",
      "\n",
      "Run: 248, exploration: 0.5, score: 149\n",
      "Scores: (min: 11, avg: 144.69, max: 392)\n",
      "\n",
      "Run: 249, exploration: 0.5, score: 208\n",
      "Scores: (min: 11, avg: 146.46, max: 392)\n",
      "\n",
      "Run: 250, exploration: 0.5, score: 254\n",
      "Scores: (min: 11, avg: 146.04, max: 392)\n",
      "\n",
      "Run: 251, exploration: 0.5, score: 298\n",
      "Scores: (min: 11, avg: 148.52, max: 392)\n",
      "\n",
      "Run: 252, exploration: 0.5, score: 83\n",
      "Scores: (min: 11, avg: 146.41, max: 392)\n",
      "\n",
      "Run: 253, exploration: 0.5, score: 56\n",
      "Scores: (min: 11, avg: 146.45, max: 392)\n",
      "\n",
      "Run: 254, exploration: 0.5, score: 127\n",
      "Scores: (min: 11, avg: 147.59, max: 392)\n",
      "\n",
      "Run: 255, exploration: 0.5, score: 30\n",
      "Scores: (min: 11, avg: 144.29, max: 392)\n",
      "\n",
      "Run: 256, exploration: 0.5, score: 44\n",
      "Scores: (min: 11, avg: 144.24, max: 392)\n",
      "\n",
      "Run: 257, exploration: 0.5, score: 275\n",
      "Scores: (min: 11, avg: 145.02, max: 392)\n",
      "\n",
      "Run: 258, exploration: 0.5, score: 105\n",
      "Scores: (min: 11, avg: 144.64, max: 392)\n",
      "\n",
      "Run: 259, exploration: 0.5, score: 133\n",
      "Scores: (min: 11, avg: 145.42, max: 392)\n",
      "\n",
      "Run: 260, exploration: 0.5, score: 292\n",
      "Scores: (min: 11, avg: 146.83, max: 392)\n",
      "\n",
      "Run: 261, exploration: 0.5, score: 195\n",
      "Scores: (min: 11, avg: 146.65, max: 392)\n",
      "\n",
      "Run: 262, exploration: 0.5, score: 141\n",
      "Scores: (min: 11, avg: 144.97, max: 392)\n",
      "\n",
      "Run: 263, exploration: 0.5, score: 25\n",
      "Scores: (min: 11, avg: 143.1, max: 392)\n",
      "\n",
      "Run: 264, exploration: 0.5, score: 62\n",
      "Scores: (min: 11, avg: 143.61, max: 392)\n",
      "\n",
      "Run: 265, exploration: 0.5, score: 206\n",
      "Scores: (min: 11, avg: 143.87, max: 392)\n",
      "\n",
      "Run: 266, exploration: 0.5, score: 100\n",
      "Scores: (min: 11, avg: 143.92, max: 392)\n",
      "\n",
      "Run: 267, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 143.44, max: 392)\n",
      "\n",
      "Run: 268, exploration: 0.5, score: 27\n",
      "Scores: (min: 11, avg: 143.47, max: 392)\n",
      "\n",
      "Run: 269, exploration: 0.5, score: 75\n",
      "Scores: (min: 11, avg: 142.69, max: 392)\n",
      "\n",
      "Run: 270, exploration: 0.5, score: 55\n",
      "Scores: (min: 11, avg: 141.15, max: 392)\n",
      "\n",
      "Run: 271, exploration: 0.5, score: 33\n",
      "Scores: (min: 11, avg: 140.78, max: 392)\n",
      "\n",
      "Run: 272, exploration: 0.5, score: 80\n",
      "Scores: (min: 11, avg: 140.6, max: 392)\n",
      "\n",
      "Run: 273, exploration: 0.5, score: 167\n",
      "Scores: (min: 11, avg: 141.81, max: 392)\n",
      "\n",
      "Run: 274, exploration: 0.5, score: 103\n",
      "Scores: (min: 11, avg: 141.05, max: 392)\n",
      "\n",
      "Run: 275, exploration: 0.5, score: 97\n",
      "Scores: (min: 11, avg: 140.42, max: 392)\n",
      "\n",
      "Run: 276, exploration: 0.5, score: 145\n",
      "Scores: (min: 11, avg: 141.74, max: 392)\n",
      "\n",
      "Run: 277, exploration: 0.5, score: 223\n",
      "Scores: (min: 11, avg: 142.27, max: 392)\n",
      "\n",
      "Run: 278, exploration: 0.5, score: 216\n",
      "Scores: (min: 11, avg: 142.85, max: 392)\n",
      "\n",
      "Run: 279, exploration: 0.5, score: 227\n",
      "Scores: (min: 11, avg: 144.37, max: 392)\n",
      "\n",
      "Run: 280, exploration: 0.5, score: 187\n",
      "Scores: (min: 11, avg: 144.04, max: 392)\n",
      "\n",
      "Run: 281, exploration: 0.5, score: 22\n",
      "Scores: (min: 11, avg: 142.56, max: 392)\n",
      "\n",
      "Run: 282, exploration: 0.5, score: 224\n",
      "Scores: (min: 11, avg: 143.93, max: 392)\n",
      "\n",
      "Run: 283, exploration: 0.5, score: 18\n",
      "Scores: (min: 11, avg: 141.23, max: 392)\n",
      "\n",
      "Run: 284, exploration: 0.5, score: 250\n",
      "Scores: (min: 11, avg: 143.35, max: 392)\n",
      "\n",
      "Run: 285, exploration: 0.5, score: 308\n",
      "Scores: (min: 11, avg: 144.81, max: 392)\n",
      "\n",
      "Run: 286, exploration: 0.5, score: 56\n",
      "Scores: (min: 11, avg: 143.04, max: 392)\n",
      "\n",
      "Run: 287, exploration: 0.5, score: 183\n",
      "Scores: (min: 11, avg: 142.67, max: 392)\n",
      "\n",
      "Run: 288, exploration: 0.5, score: 143\n",
      "Scores: (min: 11, avg: 141.84, max: 392)\n",
      "\n",
      "Run: 289, exploration: 0.5, score: 98\n",
      "Scores: (min: 11, avg: 141.61, max: 392)\n",
      "\n",
      "Run: 290, exploration: 0.5, score: 204\n",
      "Scores: (min: 11, avg: 141.52, max: 392)\n",
      "\n",
      "Run: 291, exploration: 0.5, score: 355\n",
      "Scores: (min: 11, avg: 143.74, max: 392)\n",
      "\n",
      "Run: 292, exploration: 0.5, score: 132\n",
      "Scores: (min: 11, avg: 142.63, max: 392)\n",
      "\n",
      "Run: 293, exploration: 0.5, score: 413\n",
      "Scores: (min: 11, avg: 145.18, max: 413)\n",
      "\n",
      "Run: 294, exploration: 0.5, score: 16\n",
      "Scores: (min: 11, avg: 143.77, max: 413)\n",
      "\n",
      "Run: 295, exploration: 0.5, score: 149\n",
      "Scores: (min: 11, avg: 142.52, max: 413)\n",
      "\n",
      "Run: 296, exploration: 0.5, score: 27\n",
      "Scores: (min: 11, avg: 142.11, max: 413)\n",
      "\n",
      "Run: 297, exploration: 0.5, score: 262\n",
      "Scores: (min: 11, avg: 143.53, max: 413)\n",
      "\n",
      "Run: 298, exploration: 0.5, score: 42\n",
      "Scores: (min: 11, avg: 143.78, max: 413)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 299, exploration: 0.5, score: 249\n",
      "Scores: (min: 11, avg: 145.66, max: 413)\n",
      "\n",
      "Run: 300, exploration: 0.5, score: 89\n",
      "Scores: (min: 11, avg: 144.15, max: 413)\n",
      "\n",
      "Run: 301, exploration: 0.5, score: 19\n",
      "Scores: (min: 11, avg: 143.65, max: 413)\n",
      "\n",
      "Run: 302, exploration: 0.5, score: 156\n",
      "Scores: (min: 11, avg: 142.96, max: 413)\n",
      "\n",
      "Run: 303, exploration: 0.5, score: 151\n",
      "Scores: (min: 11, avg: 142.98, max: 413)\n",
      "\n",
      "Run: 304, exploration: 0.5, score: 78\n",
      "Scores: (min: 11, avg: 141.5, max: 413)\n",
      "\n",
      "Run: 305, exploration: 0.5, score: 58\n",
      "Scores: (min: 11, avg: 140.93, max: 413)\n",
      "\n",
      "Run: 306, exploration: 0.5, score: 111\n",
      "Scores: (min: 11, avg: 141.64, max: 413)\n",
      "\n",
      "Run: 307, exploration: 0.5, score: 171\n",
      "Scores: (min: 11, avg: 142.36, max: 413)\n",
      "\n",
      "Run: 308, exploration: 0.5, score: 51\n",
      "Scores: (min: 11, avg: 142.15, max: 413)\n",
      "\n",
      "Run: 309, exploration: 0.5, score: 47\n",
      "Scores: (min: 11, avg: 140.63, max: 413)\n",
      "\n",
      "Run: 310, exploration: 0.5, score: 185\n",
      "Scores: (min: 11, avg: 141.51, max: 413)\n",
      "\n",
      "Run: 311, exploration: 0.5, score: 63\n",
      "Scores: (min: 11, avg: 140.19, max: 413)\n",
      "\n",
      "Run: 312, exploration: 0.5, score: 166\n",
      "Scores: (min: 11, avg: 141.39, max: 413)\n",
      "\n",
      "Run: 313, exploration: 0.5, score: 159\n",
      "Scores: (min: 11, avg: 142.37, max: 413)\n",
      "\n",
      "Run: 314, exploration: 0.5, score: 151\n",
      "Scores: (min: 11, avg: 143.12, max: 413)\n",
      "\n",
      "Run: 315, exploration: 0.5, score: 74\n",
      "Scores: (min: 11, avg: 141.8, max: 413)\n",
      "\n",
      "Run: 316, exploration: 0.5, score: 190\n",
      "Scores: (min: 11, avg: 141.2, max: 413)\n",
      "\n",
      "Run: 317, exploration: 0.5, score: 152\n",
      "Scores: (min: 11, avg: 141.69, max: 413)\n",
      "\n",
      "Run: 318, exploration: 0.5, score: 235\n",
      "Scores: (min: 11, avg: 142.19, max: 413)\n",
      "\n",
      "Run: 319, exploration: 0.5, score: 19\n",
      "Scores: (min: 11, avg: 140.62, max: 413)\n",
      "\n",
      "Run: 320, exploration: 0.5, score: 52\n",
      "Scores: (min: 11, avg: 137.22, max: 413)\n",
      "\n",
      "Run: 321, exploration: 0.5, score: 202\n",
      "Scores: (min: 11, avg: 139.13, max: 413)\n",
      "\n",
      "Run: 322, exploration: 0.5, score: 268\n",
      "Scores: (min: 11, avg: 140.01, max: 413)\n",
      "\n",
      "Run: 323, exploration: 0.5, score: 12\n",
      "Scores: (min: 11, avg: 139.88, max: 413)\n",
      "\n",
      "Run: 324, exploration: 0.5, score: 14\n",
      "Scores: (min: 11, avg: 139.46, max: 413)\n",
      "\n",
      "Run: 325, exploration: 0.5, score: 124\n",
      "Scores: (min: 11, avg: 138.39, max: 413)\n",
      "\n",
      "Run: 326, exploration: 0.5, score: 299\n",
      "Scores: (min: 11, avg: 138.77, max: 413)\n",
      "\n",
      "Run: 327, exploration: 0.5, score: 256\n",
      "Scores: (min: 11, avg: 139.63, max: 413)\n",
      "\n",
      "Run: 328, exploration: 0.5, score: 197\n",
      "Scores: (min: 11, avg: 140.93, max: 413)\n",
      "\n",
      "Run: 329, exploration: 0.5, score: 75\n",
      "Scores: (min: 11, avg: 139.34, max: 413)\n",
      "\n",
      "Run: 330, exploration: 0.5, score: 22\n",
      "Scores: (min: 11, avg: 139.1, max: 413)\n",
      "\n",
      "Run: 331, exploration: 0.5, score: 88\n",
      "Scores: (min: 11, avg: 137.48, max: 413)\n",
      "\n",
      "Run: 332, exploration: 0.5, score: 247\n",
      "Scores: (min: 11, avg: 139.55, max: 413)\n",
      "\n",
      "Run: 333, exploration: 0.5, score: 184\n",
      "Scores: (min: 11, avg: 141.19, max: 413)\n",
      "\n",
      "Run: 334, exploration: 0.5, score: 220\n",
      "Scores: (min: 11, avg: 142.19, max: 413)\n",
      "\n",
      "Run: 335, exploration: 0.5, score: 21\n",
      "Scores: (min: 11, avg: 142.25, max: 413)\n",
      "\n",
      "Run: 336, exploration: 0.5, score: 222\n",
      "Scores: (min: 11, avg: 142.68, max: 413)\n",
      "\n",
      "Run: 337, exploration: 0.5, score: 241\n",
      "Scores: (min: 11, avg: 142.05, max: 413)\n",
      "\n",
      "Run: 338, exploration: 0.5, score: 183\n",
      "Scores: (min: 11, avg: 143.01, max: 413)\n",
      "\n",
      "Run: 339, exploration: 0.5, score: 290\n",
      "Scores: (min: 11, avg: 143.77, max: 413)\n",
      "\n",
      "Run: 340, exploration: 0.5, score: 36\n",
      "Scores: (min: 11, avg: 142.19, max: 413)\n",
      "\n",
      "Run: 341, exploration: 0.5, score: 130\n",
      "Scores: (min: 11, avg: 141.55, max: 413)\n",
      "\n",
      "Run: 342, exploration: 0.5, score: 89\n",
      "Scores: (min: 11, avg: 138.81, max: 413)\n",
      "\n",
      "Run: 343, exploration: 0.5, score: 225\n",
      "Scores: (min: 11, avg: 139.21, max: 413)\n",
      "\n",
      "Run: 344, exploration: 0.5, score: 137\n",
      "Scores: (min: 11, avg: 139.96, max: 413)\n",
      "\n",
      "Run: 345, exploration: 0.5, score: 146\n",
      "Scores: (min: 11, avg: 139.3, max: 413)\n",
      "\n",
      "Run: 346, exploration: 0.5, score: 100\n",
      "Scores: (min: 11, avg: 140.1, max: 413)\n",
      "\n",
      "Run: 347, exploration: 0.5, score: 227\n",
      "Scores: (min: 11, avg: 142.12, max: 413)\n",
      "\n",
      "Run: 348, exploration: 0.5, score: 51\n",
      "Scores: (min: 11, avg: 141.14, max: 413)\n",
      "\n",
      "Run: 349, exploration: 0.5, score: 16\n",
      "Scores: (min: 11, avg: 139.22, max: 413)\n",
      "\n",
      "Run: 350, exploration: 0.5, score: 152\n",
      "Scores: (min: 11, avg: 138.2, max: 413)\n",
      "\n",
      "Run: 351, exploration: 0.5, score: 155\n",
      "Scores: (min: 11, avg: 136.77, max: 413)\n",
      "\n",
      "Run: 352, exploration: 0.5, score: 334\n",
      "Scores: (min: 11, avg: 139.28, max: 413)\n",
      "\n",
      "Run: 353, exploration: 0.5, score: 62\n",
      "Scores: (min: 11, avg: 139.34, max: 413)\n",
      "\n",
      "Run: 354, exploration: 0.5, score: 62\n",
      "Scores: (min: 11, avg: 138.69, max: 413)\n",
      "\n",
      "Run: 355, exploration: 0.5, score: 53\n",
      "Scores: (min: 11, avg: 138.92, max: 413)\n",
      "\n",
      "Run: 356, exploration: 0.5, score: 281\n",
      "Scores: (min: 11, avg: 141.29, max: 413)\n",
      "\n",
      "Run: 357, exploration: 0.5, score: 230\n",
      "Scores: (min: 11, avg: 140.84, max: 413)\n",
      "\n",
      "Run: 358, exploration: 0.5, score: 17\n",
      "Scores: (min: 11, avg: 139.96, max: 413)\n",
      "\n",
      "Run: 359, exploration: 0.5, score: 202\n",
      "Scores: (min: 11, avg: 140.65, max: 413)\n",
      "\n",
      "Run: 360, exploration: 0.5, score: 90\n",
      "Scores: (min: 11, avg: 138.63, max: 413)\n",
      "\n",
      "Run: 361, exploration: 0.5, score: 65\n",
      "Scores: (min: 11, avg: 137.33, max: 413)\n",
      "\n",
      "Run: 362, exploration: 0.5, score: 158\n",
      "Scores: (min: 11, avg: 137.5, max: 413)\n",
      "\n",
      "Run: 363, exploration: 0.5, score: 129\n",
      "Scores: (min: 11, avg: 138.54, max: 413)\n",
      "\n",
      "Run: 364, exploration: 0.5, score: 74\n",
      "Scores: (min: 11, avg: 138.66, max: 413)\n",
      "\n",
      "Run: 365, exploration: 0.5, score: 35\n",
      "Scores: (min: 11, avg: 136.95, max: 413)\n",
      "\n",
      "Run: 366, exploration: 0.5, score: 113\n",
      "Scores: (min: 11, avg: 137.08, max: 413)\n",
      "\n",
      "Run: 367, exploration: 0.5, score: 81\n",
      "Scores: (min: 12, avg: 137.78, max: 413)\n",
      "\n",
      "Run: 368, exploration: 0.5, score: 236\n",
      "Scores: (min: 12, avg: 139.87, max: 413)\n",
      "\n",
      "Run: 369, exploration: 0.5, score: 46\n",
      "Scores: (min: 12, avg: 139.58, max: 413)\n",
      "\n",
      "Run: 370, exploration: 0.5, score: 134\n",
      "Scores: (min: 12, avg: 140.37, max: 413)\n",
      "\n",
      "Run: 371, exploration: 0.5, score: 70\n",
      "Scores: (min: 12, avg: 140.74, max: 413)\n",
      "\n",
      "Run: 372, exploration: 0.5, score: 317\n",
      "Scores: (min: 12, avg: 143.11, max: 413)\n",
      "\n",
      "Run: 373, exploration: 0.5, score: 170\n",
      "Scores: (min: 12, avg: 143.14, max: 413)\n",
      "\n",
      "Run: 374, exploration: 0.5, score: 34\n",
      "Scores: (min: 12, avg: 142.45, max: 413)\n",
      "\n",
      "Run: 375, exploration: 0.5, score: 131\n",
      "Scores: (min: 12, avg: 142.79, max: 413)\n",
      "\n",
      "Run: 376, exploration: 0.5, score: 94\n",
      "Scores: (min: 12, avg: 142.28, max: 413)\n",
      "\n",
      "Run: 377, exploration: 0.5, score: 199\n",
      "Scores: (min: 12, avg: 142.04, max: 413)\n",
      "\n",
      "Run: 378, exploration: 0.5, score: 97\n",
      "Scores: (min: 12, avg: 140.85, max: 413)\n",
      "\n",
      "Run: 379, exploration: 0.5, score: 116\n",
      "Scores: (min: 12, avg: 139.74, max: 413)\n",
      "\n",
      "Run: 380, exploration: 0.5, score: 72\n",
      "Scores: (min: 12, avg: 138.59, max: 413)\n",
      "\n",
      "Run: 381, exploration: 0.5, score: 203\n",
      "Scores: (min: 12, avg: 140.4, max: 413)\n",
      "\n",
      "Run: 382, exploration: 0.5, score: 68\n",
      "Scores: (min: 12, avg: 138.84, max: 413)\n",
      "\n",
      "Run: 383, exploration: 0.5, score: 19\n",
      "Scores: (min: 12, avg: 138.85, max: 413)\n",
      "\n",
      "Run: 384, exploration: 0.5, score: 40\n",
      "Scores: (min: 12, avg: 136.75, max: 413)\n",
      "\n",
      "Run: 385, exploration: 0.5, score: 28\n",
      "Scores: (min: 12, avg: 133.95, max: 413)\n",
      "\n",
      "Run: 386, exploration: 0.5, score: 172\n",
      "Scores: (min: 12, avg: 135.11, max: 413)\n",
      "\n",
      "Run: 387, exploration: 0.5, score: 30\n",
      "Scores: (min: 12, avg: 133.58, max: 413)\n",
      "\n",
      "Run: 388, exploration: 0.5, score: 115\n",
      "Scores: (min: 12, avg: 133.3, max: 413)\n",
      "\n",
      "Run: 389, exploration: 0.5, score: 220\n",
      "Scores: (min: 12, avg: 134.52, max: 413)\n",
      "\n",
      "Run: 390, exploration: 0.5, score: 223\n",
      "Scores: (min: 12, avg: 134.71, max: 413)\n",
      "\n",
      "Run: 391, exploration: 0.5, score: 39\n",
      "Scores: (min: 12, avg: 131.55, max: 413)\n",
      "\n",
      "Run: 392, exploration: 0.5, score: 27\n",
      "Scores: (min: 12, avg: 130.5, max: 413)\n",
      "\n",
      "Run: 393, exploration: 0.5, score: 223\n",
      "Scores: (min: 12, avg: 128.6, max: 334)\n",
      "\n",
      "Run: 394, exploration: 0.5, score: 57\n",
      "Scores: (min: 12, avg: 129.01, max: 334)\n",
      "\n",
      "Run: 395, exploration: 0.5, score: 158\n",
      "Scores: (min: 12, avg: 129.1, max: 334)\n",
      "\n",
      "Run: 396, exploration: 0.5, score: 80\n",
      "Scores: (min: 12, avg: 129.63, max: 334)\n",
      "\n",
      "Run: 397, exploration: 0.5, score: 126\n",
      "Scores: (min: 12, avg: 128.27, max: 334)\n",
      "\n",
      "Run: 398, exploration: 0.5, score: 290\n",
      "Scores: (min: 12, avg: 130.75, max: 334)\n",
      "\n",
      "Run: 399, exploration: 0.5, score: 52\n",
      "Scores: (min: 12, avg: 128.78, max: 334)\n",
      "\n",
      "Run: 400, exploration: 0.5, score: 229\n",
      "Scores: (min: 12, avg: 130.18, max: 334)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 401, exploration: 0.5, score: 336\n",
      "Scores: (min: 12, avg: 133.35, max: 336)\n",
      "\n",
      "Run: 402, exploration: 0.5, score: 208\n",
      "Scores: (min: 12, avg: 133.87, max: 336)\n",
      "\n",
      "Run: 403, exploration: 0.5, score: 158\n",
      "Scores: (min: 12, avg: 133.94, max: 336)\n",
      "\n",
      "Run: 404, exploration: 0.5, score: 113\n",
      "Scores: (min: 12, avg: 134.29, max: 336)\n",
      "\n",
      "Run: 405, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 138.71, max: 500)\n",
      "\n",
      "Run: 406, exploration: 0.5, score: 400\n",
      "Scores: (min: 12, avg: 141.6, max: 500)\n",
      "\n",
      "Run: 407, exploration: 0.5, score: 39\n",
      "Scores: (min: 12, avg: 140.28, max: 500)\n",
      "\n",
      "Run: 408, exploration: 0.5, score: 145\n",
      "Scores: (min: 12, avg: 141.22, max: 500)\n",
      "\n",
      "Run: 409, exploration: 0.5, score: 158\n",
      "Scores: (min: 12, avg: 142.33, max: 500)\n",
      "\n",
      "Run: 410, exploration: 0.5, score: 229\n",
      "Scores: (min: 12, avg: 142.77, max: 500)\n",
      "\n",
      "Run: 411, exploration: 0.5, score: 281\n",
      "Scores: (min: 12, avg: 144.95, max: 500)\n",
      "\n",
      "Run: 412, exploration: 0.5, score: 308\n",
      "Scores: (min: 12, avg: 146.37, max: 500)\n",
      "\n",
      "Run: 413, exploration: 0.5, score: 237\n",
      "Scores: (min: 12, avg: 147.15, max: 500)\n",
      "\n",
      "Run: 414, exploration: 0.5, score: 174\n",
      "Scores: (min: 12, avg: 147.38, max: 500)\n",
      "\n",
      "Run: 415, exploration: 0.5, score: 245\n",
      "Scores: (min: 12, avg: 149.09, max: 500)\n",
      "\n",
      "Run: 416, exploration: 0.5, score: 437\n",
      "Scores: (min: 12, avg: 151.56, max: 500)\n",
      "\n",
      "Run: 417, exploration: 0.5, score: 363\n",
      "Scores: (min: 12, avg: 153.67, max: 500)\n",
      "\n",
      "Run: 418, exploration: 0.5, score: 81\n",
      "Scores: (min: 12, avg: 152.13, max: 500)\n",
      "\n",
      "Run: 419, exploration: 0.5, score: 95\n",
      "Scores: (min: 12, avg: 152.89, max: 500)\n",
      "\n",
      "Run: 420, exploration: 0.5, score: 404\n",
      "Scores: (min: 12, avg: 156.41, max: 500)\n",
      "\n",
      "Run: 421, exploration: 0.5, score: 122\n",
      "Scores: (min: 12, avg: 155.61, max: 500)\n",
      "\n",
      "Run: 422, exploration: 0.5, score: 360\n",
      "Scores: (min: 12, avg: 156.53, max: 500)\n",
      "\n",
      "Run: 423, exploration: 0.5, score: 18\n",
      "Scores: (min: 14, avg: 156.59, max: 500)\n",
      "\n",
      "Run: 424, exploration: 0.5, score: 78\n",
      "Scores: (min: 16, avg: 157.23, max: 500)\n",
      "\n",
      "Run: 425, exploration: 0.5, score: 183\n",
      "Scores: (min: 16, avg: 157.82, max: 500)\n",
      "\n",
      "Run: 426, exploration: 0.5, score: 19\n",
      "Scores: (min: 16, avg: 155.02, max: 500)\n",
      "\n",
      "Run: 427, exploration: 0.5, score: 93\n",
      "Scores: (min: 16, avg: 153.39, max: 500)\n",
      "\n",
      "Run: 428, exploration: 0.5, score: 62\n",
      "Scores: (min: 16, avg: 152.04, max: 500)\n",
      "\n",
      "Run: 429, exploration: 0.5, score: 189\n",
      "Scores: (min: 16, avg: 153.18, max: 500)\n",
      "\n",
      "Run: 430, exploration: 0.5, score: 146\n",
      "Scores: (min: 16, avg: 154.42, max: 500)\n",
      "\n",
      "Run: 431, exploration: 0.5, score: 81\n",
      "Scores: (min: 16, avg: 154.35, max: 500)\n",
      "\n",
      "Run: 432, exploration: 0.5, score: 77\n",
      "Scores: (min: 16, avg: 152.65, max: 500)\n",
      "\n",
      "Run: 433, exploration: 0.5, score: 59\n",
      "Scores: (min: 16, avg: 151.4, max: 500)\n",
      "\n",
      "Run: 434, exploration: 0.5, score: 19\n",
      "Scores: (min: 16, avg: 149.39, max: 500)\n",
      "\n",
      "Run: 435, exploration: 0.5, score: 246\n",
      "Scores: (min: 16, avg: 151.64, max: 500)\n",
      "\n",
      "Run: 436, exploration: 0.5, score: 35\n",
      "Scores: (min: 16, avg: 149.77, max: 500)\n",
      "\n",
      "Run: 437, exploration: 0.5, score: 436\n",
      "Scores: (min: 16, avg: 151.72, max: 500)\n",
      "\n",
      "Run: 438, exploration: 0.5, score: 127\n",
      "Scores: (min: 16, avg: 151.16, max: 500)\n",
      "\n",
      "Run: 439, exploration: 0.5, score: 33\n",
      "Scores: (min: 16, avg: 148.59, max: 500)\n",
      "\n",
      "Run: 440, exploration: 0.5, score: 52\n",
      "Scores: (min: 16, avg: 148.75, max: 500)\n",
      "\n",
      "Run: 441, exploration: 0.5, score: 107\n",
      "Scores: (min: 16, avg: 148.52, max: 500)\n",
      "\n",
      "Run: 442, exploration: 0.5, score: 94\n",
      "Scores: (min: 16, avg: 148.57, max: 500)\n",
      "\n",
      "Run: 443, exploration: 0.5, score: 162\n",
      "Scores: (min: 16, avg: 147.94, max: 500)\n",
      "\n",
      "Run: 444, exploration: 0.5, score: 292\n",
      "Scores: (min: 16, avg: 149.49, max: 500)\n",
      "\n",
      "Run: 445, exploration: 0.5, score: 182\n",
      "Scores: (min: 16, avg: 149.85, max: 500)\n",
      "\n",
      "Run: 446, exploration: 0.5, score: 27\n",
      "Scores: (min: 16, avg: 149.12, max: 500)\n",
      "\n",
      "Run: 447, exploration: 0.5, score: 53\n",
      "Scores: (min: 16, avg: 147.38, max: 500)\n",
      "\n",
      "Run: 448, exploration: 0.5, score: 443\n",
      "Scores: (min: 16, avg: 151.3, max: 500)\n",
      "\n",
      "Run: 449, exploration: 0.5, score: 278\n",
      "Scores: (min: 17, avg: 153.92, max: 500)\n",
      "\n",
      "Run: 450, exploration: 0.5, score: 173\n",
      "Scores: (min: 17, avg: 154.13, max: 500)\n",
      "\n",
      "Run: 451, exploration: 0.5, score: 184\n",
      "Scores: (min: 17, avg: 154.42, max: 500)\n",
      "\n",
      "Run: 452, exploration: 0.5, score: 294\n",
      "Scores: (min: 17, avg: 154.02, max: 500)\n",
      "\n",
      "Run: 453, exploration: 0.5, score: 495\n",
      "Scores: (min: 17, avg: 158.35, max: 500)\n",
      "\n",
      "Run: 454, exploration: 0.5, score: 75\n",
      "Scores: (min: 17, avg: 158.48, max: 500)\n",
      "\n",
      "Run: 455, exploration: 0.5, score: 12\n",
      "Scores: (min: 12, avg: 158.07, max: 500)\n",
      "\n",
      "Run: 456, exploration: 0.5, score: 131\n",
      "Scores: (min: 12, avg: 156.57, max: 500)\n",
      "\n",
      "Run: 457, exploration: 0.5, score: 207\n",
      "Scores: (min: 12, avg: 156.34, max: 500)\n",
      "\n",
      "Run: 458, exploration: 0.5, score: 305\n",
      "Scores: (min: 12, avg: 159.22, max: 500)\n",
      "\n",
      "Run: 459, exploration: 0.5, score: 214\n",
      "Scores: (min: 12, avg: 159.34, max: 500)\n",
      "\n",
      "Run: 460, exploration: 0.5, score: 219\n",
      "Scores: (min: 12, avg: 160.63, max: 500)\n",
      "\n",
      "Run: 461, exploration: 0.5, score: 13\n",
      "Scores: (min: 12, avg: 160.11, max: 500)\n",
      "\n",
      "Run: 462, exploration: 0.5, score: 192\n",
      "Scores: (min: 12, avg: 160.45, max: 500)\n",
      "\n",
      "Run: 463, exploration: 0.5, score: 78\n",
      "Scores: (min: 12, avg: 159.94, max: 500)\n",
      "\n",
      "Run: 464, exploration: 0.5, score: 138\n",
      "Scores: (min: 12, avg: 160.58, max: 500)\n",
      "\n",
      "Run: 465, exploration: 0.5, score: 383\n",
      "Scores: (min: 12, avg: 164.06, max: 500)\n",
      "\n",
      "Run: 466, exploration: 0.5, score: 113\n",
      "Scores: (min: 12, avg: 164.06, max: 500)\n",
      "\n",
      "Run: 467, exploration: 0.5, score: 35\n",
      "Scores: (min: 12, avg: 163.6, max: 500)\n",
      "\n",
      "Run: 468, exploration: 0.5, score: 99\n",
      "Scores: (min: 12, avg: 162.23, max: 500)\n",
      "\n",
      "Run: 469, exploration: 0.5, score: 447\n",
      "Scores: (min: 12, avg: 166.24, max: 500)\n",
      "\n",
      "Run: 470, exploration: 0.5, score: 194\n",
      "Scores: (min: 12, avg: 166.84, max: 500)\n",
      "\n",
      "Run: 471, exploration: 0.5, score: 50\n",
      "Scores: (min: 12, avg: 166.64, max: 500)\n",
      "\n",
      "Run: 472, exploration: 0.5, score: 52\n",
      "Scores: (min: 12, avg: 163.99, max: 500)\n",
      "\n",
      "Run: 473, exploration: 0.5, score: 123\n",
      "Scores: (min: 12, avg: 163.52, max: 500)\n",
      "\n",
      "Run: 474, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 168.18, max: 500)\n",
      "\n",
      "Run: 475, exploration: 0.5, score: 137\n",
      "Scores: (min: 12, avg: 168.24, max: 500)\n",
      "\n",
      "Run: 476, exploration: 0.5, score: 39\n",
      "Scores: (min: 12, avg: 167.69, max: 500)\n",
      "\n",
      "Run: 477, exploration: 0.5, score: 257\n",
      "Scores: (min: 12, avg: 168.27, max: 500)\n",
      "\n",
      "Run: 478, exploration: 0.5, score: 344\n",
      "Scores: (min: 12, avg: 170.74, max: 500)\n",
      "\n",
      "Run: 479, exploration: 0.5, score: 89\n",
      "Scores: (min: 12, avg: 170.47, max: 500)\n",
      "\n",
      "Run: 480, exploration: 0.5, score: 285\n",
      "Scores: (min: 12, avg: 172.6, max: 500)\n",
      "\n",
      "Run: 481, exploration: 0.5, score: 75\n",
      "Scores: (min: 12, avg: 171.32, max: 500)\n",
      "\n",
      "Run: 482, exploration: 0.5, score: 193\n",
      "Scores: (min: 12, avg: 172.57, max: 500)\n",
      "\n",
      "Run: 483, exploration: 0.5, score: 122\n",
      "Scores: (min: 12, avg: 173.6, max: 500)\n",
      "\n",
      "Run: 484, exploration: 0.5, score: 286\n",
      "Scores: (min: 12, avg: 176.06, max: 500)\n",
      "\n",
      "Run: 485, exploration: 0.5, score: 164\n",
      "Scores: (min: 12, avg: 177.42, max: 500)\n",
      "\n",
      "Run: 486, exploration: 0.5, score: 35\n",
      "Scores: (min: 12, avg: 176.05, max: 500)\n",
      "\n",
      "Run: 487, exploration: 0.5, score: 172\n",
      "Scores: (min: 12, avg: 177.47, max: 500)\n",
      "\n",
      "Run: 488, exploration: 0.5, score: 38\n",
      "Scores: (min: 12, avg: 176.7, max: 500)\n",
      "\n",
      "Run: 489, exploration: 0.5, score: 84\n",
      "Scores: (min: 12, avg: 175.34, max: 500)\n",
      "\n",
      "Run: 490, exploration: 0.5, score: 12\n",
      "Scores: (min: 12, avg: 173.23, max: 500)\n",
      "\n",
      "Run: 491, exploration: 0.5, score: 309\n",
      "Scores: (min: 12, avg: 175.93, max: 500)\n",
      "\n",
      "Run: 492, exploration: 0.5, score: 88\n",
      "Scores: (min: 12, avg: 176.54, max: 500)\n",
      "\n",
      "Run: 493, exploration: 0.5, score: 267\n",
      "Scores: (min: 12, avg: 176.98, max: 500)\n",
      "\n",
      "Run: 494, exploration: 0.5, score: 162\n",
      "Scores: (min: 12, avg: 178.03, max: 500)\n",
      "\n",
      "Run: 495, exploration: 0.5, score: 75\n",
      "Scores: (min: 12, avg: 177.2, max: 500)\n",
      "\n",
      "Run: 496, exploration: 0.5, score: 391\n",
      "Scores: (min: 12, avg: 180.31, max: 500)\n",
      "\n",
      "Run: 497, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 184.05, max: 500)\n",
      "\n",
      "Run: 498, exploration: 0.5, score: 107\n",
      "Scores: (min: 12, avg: 182.22, max: 500)\n",
      "\n",
      "Run: 499, exploration: 0.5, score: 212\n",
      "Scores: (min: 12, avg: 183.82, max: 500)\n",
      "\n",
      "Run: 500, exploration: 0.5, score: 82\n",
      "Scores: (min: 12, avg: 182.35, max: 500)\n",
      "\n",
      "Run: 501, exploration: 0.5, score: 307\n",
      "Scores: (min: 12, avg: 182.06, max: 500)\n",
      "\n",
      "Run: 502, exploration: 0.5, score: 310\n",
      "Scores: (min: 12, avg: 183.08, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 503, exploration: 0.5, score: 156\n",
      "Scores: (min: 12, avg: 183.06, max: 500)\n",
      "\n",
      "Run: 504, exploration: 0.5, score: 18\n",
      "Scores: (min: 12, avg: 182.11, max: 500)\n",
      "\n",
      "Run: 505, exploration: 0.5, score: 87\n",
      "Scores: (min: 12, avg: 177.98, max: 500)\n",
      "\n",
      "Run: 506, exploration: 0.5, score: 57\n",
      "Scores: (min: 12, avg: 174.55, max: 500)\n",
      "\n",
      "Run: 507, exploration: 0.5, score: 228\n",
      "Scores: (min: 12, avg: 176.44, max: 500)\n",
      "\n",
      "Run: 508, exploration: 0.5, score: 139\n",
      "Scores: (min: 12, avg: 176.38, max: 500)\n",
      "\n",
      "Run: 509, exploration: 0.5, score: 19\n",
      "Scores: (min: 12, avg: 174.99, max: 500)\n",
      "\n",
      "Run: 510, exploration: 0.5, score: 93\n",
      "Scores: (min: 12, avg: 173.63, max: 500)\n",
      "\n",
      "Run: 511, exploration: 0.5, score: 22\n",
      "Scores: (min: 12, avg: 171.04, max: 500)\n",
      "\n",
      "Run: 512, exploration: 0.5, score: 33\n",
      "Scores: (min: 12, avg: 168.29, max: 500)\n",
      "\n",
      "Run: 513, exploration: 0.5, score: 463\n",
      "Scores: (min: 12, avg: 170.55, max: 500)\n",
      "\n",
      "Run: 514, exploration: 0.5, score: 399\n",
      "Scores: (min: 12, avg: 172.8, max: 500)\n",
      "\n",
      "Run: 515, exploration: 0.5, score: 71\n",
      "Scores: (min: 12, avg: 171.06, max: 500)\n",
      "\n",
      "Run: 516, exploration: 0.5, score: 44\n",
      "Scores: (min: 12, avg: 167.13, max: 500)\n",
      "\n",
      "Run: 517, exploration: 0.5, score: 18\n",
      "Scores: (min: 12, avg: 163.68, max: 500)\n",
      "\n",
      "Run: 518, exploration: 0.5, score: 152\n",
      "Scores: (min: 12, avg: 164.39, max: 500)\n",
      "\n",
      "Run: 519, exploration: 0.5, score: 21\n",
      "Scores: (min: 12, avg: 163.65, max: 500)\n",
      "\n",
      "Run: 520, exploration: 0.5, score: 204\n",
      "Scores: (min: 12, avg: 161.65, max: 500)\n",
      "\n",
      "Run: 521, exploration: 0.5, score: 352\n",
      "Scores: (min: 12, avg: 163.95, max: 500)\n",
      "\n",
      "Run: 522, exploration: 0.5, score: 165\n",
      "Scores: (min: 12, avg: 162, max: 500)\n",
      "\n",
      "Run: 523, exploration: 0.5, score: 197\n",
      "Scores: (min: 12, avg: 163.79, max: 500)\n",
      "\n",
      "Run: 524, exploration: 0.5, score: 36\n",
      "Scores: (min: 12, avg: 163.37, max: 500)\n",
      "\n",
      "Run: 525, exploration: 0.5, score: 143\n",
      "Scores: (min: 12, avg: 162.97, max: 500)\n",
      "\n",
      "Run: 526, exploration: 0.5, score: 46\n",
      "Scores: (min: 12, avg: 163.24, max: 500)\n",
      "\n",
      "Run: 527, exploration: 0.5, score: 128\n",
      "Scores: (min: 12, avg: 163.59, max: 500)\n",
      "\n",
      "Run: 528, exploration: 0.5, score: 209\n",
      "Scores: (min: 12, avg: 165.06, max: 500)\n",
      "\n",
      "Run: 529, exploration: 0.5, score: 175\n",
      "Scores: (min: 12, avg: 164.92, max: 500)\n",
      "\n",
      "Run: 530, exploration: 0.5, score: 349\n",
      "Scores: (min: 12, avg: 166.95, max: 500)\n",
      "\n",
      "Run: 531, exploration: 0.5, score: 28\n",
      "Scores: (min: 12, avg: 166.42, max: 500)\n",
      "\n",
      "Run: 532, exploration: 0.5, score: 436\n",
      "Scores: (min: 12, avg: 170.01, max: 500)\n",
      "\n",
      "Run: 533, exploration: 0.5, score: 400\n",
      "Scores: (min: 12, avg: 173.42, max: 500)\n",
      "\n",
      "Run: 534, exploration: 0.5, score: 239\n",
      "Scores: (min: 12, avg: 175.62, max: 500)\n",
      "\n",
      "Run: 535, exploration: 0.5, score: 334\n",
      "Scores: (min: 12, avg: 176.5, max: 500)\n",
      "\n",
      "Run: 536, exploration: 0.5, score: 169\n",
      "Scores: (min: 12, avg: 177.84, max: 500)\n",
      "\n",
      "Run: 537, exploration: 0.5, score: 53\n",
      "Scores: (min: 12, avg: 174.01, max: 500)\n",
      "\n",
      "Run: 538, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 172.85, max: 500)\n",
      "\n",
      "Run: 539, exploration: 0.5, score: 132\n",
      "Scores: (min: 11, avg: 173.84, max: 500)\n",
      "\n",
      "Run: 540, exploration: 0.5, score: 12\n",
      "Scores: (min: 11, avg: 173.44, max: 500)\n",
      "\n",
      "Run: 541, exploration: 0.5, score: 77\n",
      "Scores: (min: 11, avg: 173.14, max: 500)\n",
      "\n",
      "Run: 542, exploration: 0.5, score: 49\n",
      "Scores: (min: 11, avg: 172.69, max: 500)\n",
      "\n",
      "Run: 543, exploration: 0.5, score: 157\n",
      "Scores: (min: 11, avg: 172.64, max: 500)\n",
      "\n",
      "Run: 544, exploration: 0.5, score: 236\n",
      "Scores: (min: 11, avg: 172.08, max: 500)\n",
      "\n",
      "Run: 545, exploration: 0.5, score: 157\n",
      "Scores: (min: 11, avg: 171.83, max: 500)\n",
      "\n",
      "Run: 546, exploration: 0.5, score: 294\n",
      "Scores: (min: 11, avg: 174.5, max: 500)\n",
      "\n",
      "Run: 547, exploration: 0.5, score: 157\n",
      "Scores: (min: 11, avg: 175.54, max: 500)\n",
      "\n",
      "Run: 548, exploration: 0.5, score: 220\n",
      "Scores: (min: 11, avg: 173.31, max: 500)\n",
      "\n",
      "Run: 549, exploration: 0.5, score: 110\n",
      "Scores: (min: 11, avg: 171.63, max: 500)\n",
      "\n",
      "Run: 550, exploration: 0.5, score: 151\n",
      "Scores: (min: 11, avg: 171.41, max: 500)\n",
      "\n",
      "Run: 551, exploration: 0.5, score: 275\n",
      "Scores: (min: 11, avg: 172.32, max: 500)\n",
      "\n",
      "Run: 552, exploration: 0.5, score: 49\n",
      "Scores: (min: 11, avg: 169.87, max: 500)\n",
      "\n",
      "Run: 553, exploration: 0.5, score: 126\n",
      "Scores: (min: 11, avg: 166.18, max: 500)\n",
      "\n",
      "Run: 554, exploration: 0.5, score: 253\n",
      "Scores: (min: 11, avg: 167.96, max: 500)\n",
      "\n",
      "Run: 555, exploration: 0.5, score: 101\n",
      "Scores: (min: 11, avg: 168.85, max: 500)\n",
      "\n",
      "Run: 556, exploration: 0.5, score: 438\n",
      "Scores: (min: 11, avg: 171.92, max: 500)\n",
      "\n",
      "Run: 557, exploration: 0.5, score: 276\n",
      "Scores: (min: 11, avg: 172.61, max: 500)\n",
      "\n",
      "Run: 558, exploration: 0.5, score: 19\n",
      "Scores: (min: 11, avg: 169.75, max: 500)\n",
      "\n",
      "Run: 559, exploration: 0.5, score: 281\n",
      "Scores: (min: 11, avg: 170.42, max: 500)\n",
      "\n",
      "Run: 560, exploration: 0.5, score: 162\n",
      "Scores: (min: 11, avg: 169.85, max: 500)\n",
      "\n",
      "Run: 561, exploration: 0.5, score: 36\n",
      "Scores: (min: 11, avg: 170.08, max: 500)\n",
      "\n",
      "Run: 562, exploration: 0.5, score: 57\n",
      "Scores: (min: 11, avg: 168.73, max: 500)\n",
      "\n",
      "Run: 563, exploration: 0.5, score: 500\n",
      "Scores: (min: 11, avg: 172.95, max: 500)\n",
      "\n",
      "Run: 564, exploration: 0.5, score: 323\n",
      "Scores: (min: 11, avg: 174.8, max: 500)\n",
      "\n",
      "Run: 565, exploration: 0.5, score: 113\n",
      "Scores: (min: 11, avg: 172.1, max: 500)\n",
      "\n",
      "Run: 566, exploration: 0.5, score: 144\n",
      "Scores: (min: 11, avg: 172.41, max: 500)\n",
      "\n",
      "Run: 567, exploration: 0.5, score: 213\n",
      "Scores: (min: 11, avg: 174.19, max: 500)\n",
      "\n",
      "Run: 568, exploration: 0.5, score: 357\n",
      "Scores: (min: 11, avg: 176.77, max: 500)\n",
      "\n",
      "Run: 569, exploration: 0.5, score: 101\n",
      "Scores: (min: 11, avg: 173.31, max: 500)\n",
      "\n",
      "Run: 570, exploration: 0.5, score: 394\n",
      "Scores: (min: 11, avg: 175.31, max: 500)\n",
      "\n",
      "Run: 571, exploration: 0.5, score: 500\n",
      "Scores: (min: 11, avg: 179.81, max: 500)\n",
      "\n",
      "Run: 572, exploration: 0.5, score: 62\n",
      "Scores: (min: 11, avg: 179.91, max: 500)\n",
      "\n",
      "Run: 573, exploration: 0.5, score: 127\n",
      "Scores: (min: 11, avg: 179.95, max: 500)\n",
      "\n",
      "Run: 574, exploration: 0.5, score: 18\n",
      "Scores: (min: 11, avg: 175.13, max: 500)\n",
      "\n",
      "Run: 575, exploration: 0.5, score: 342\n",
      "Scores: (min: 11, avg: 177.18, max: 500)\n",
      "\n",
      "Run: 576, exploration: 0.5, score: 184\n",
      "Scores: (min: 11, avg: 178.63, max: 500)\n",
      "\n",
      "Run: 577, exploration: 0.5, score: 56\n",
      "Scores: (min: 11, avg: 176.62, max: 500)\n",
      "\n",
      "Run: 578, exploration: 0.5, score: 31\n",
      "Scores: (min: 11, avg: 173.49, max: 500)\n",
      "\n",
      "Run: 579, exploration: 0.5, score: 289\n",
      "Scores: (min: 11, avg: 175.49, max: 500)\n",
      "\n",
      "Run: 580, exploration: 0.5, score: 155\n",
      "Scores: (min: 11, avg: 174.19, max: 500)\n",
      "\n",
      "Run: 581, exploration: 0.5, score: 133\n",
      "Scores: (min: 11, avg: 174.77, max: 500)\n",
      "\n",
      "Run: 582, exploration: 0.5, score: 190\n",
      "Scores: (min: 11, avg: 174.74, max: 500)\n",
      "\n",
      "Run: 583, exploration: 0.5, score: 50\n",
      "Scores: (min: 11, avg: 174.02, max: 500)\n",
      "\n",
      "Run: 584, exploration: 0.5, score: 48\n",
      "Scores: (min: 11, avg: 171.64, max: 500)\n",
      "\n",
      "Run: 585, exploration: 0.5, score: 50\n",
      "Scores: (min: 11, avg: 170.5, max: 500)\n",
      "\n",
      "Run: 586, exploration: 0.5, score: 44\n",
      "Scores: (min: 11, avg: 170.59, max: 500)\n",
      "\n",
      "Run: 587, exploration: 0.5, score: 106\n",
      "Scores: (min: 11, avg: 169.93, max: 500)\n",
      "\n",
      "Run: 588, exploration: 0.5, score: 21\n",
      "Scores: (min: 11, avg: 169.76, max: 500)\n",
      "\n",
      "Run: 589, exploration: 0.5, score: 119\n",
      "Scores: (min: 11, avg: 170.11, max: 500)\n",
      "\n",
      "Run: 590, exploration: 0.5, score: 143\n",
      "Scores: (min: 11, avg: 171.42, max: 500)\n",
      "\n",
      "Run: 591, exploration: 0.5, score: 500\n",
      "Scores: (min: 11, avg: 173.33, max: 500)\n",
      "\n",
      "Run: 592, exploration: 0.5, score: 109\n",
      "Scores: (min: 11, avg: 173.54, max: 500)\n",
      "\n",
      "Run: 593, exploration: 0.5, score: 78\n",
      "Scores: (min: 11, avg: 171.65, max: 500)\n",
      "\n",
      "Run: 594, exploration: 0.5, score: 369\n",
      "Scores: (min: 11, avg: 173.72, max: 500)\n",
      "\n",
      "Run: 595, exploration: 0.5, score: 103\n",
      "Scores: (min: 11, avg: 174, max: 500)\n",
      "\n",
      "Run: 596, exploration: 0.5, score: 127\n",
      "Scores: (min: 11, avg: 171.36, max: 500)\n",
      "\n",
      "Run: 597, exploration: 0.5, score: 294\n",
      "Scores: (min: 11, avg: 169.3, max: 500)\n",
      "\n",
      "Run: 598, exploration: 0.5, score: 187\n",
      "Scores: (min: 11, avg: 170.1, max: 500)\n",
      "\n",
      "Run: 599, exploration: 0.5, score: 29\n",
      "Scores: (min: 11, avg: 168.27, max: 500)\n",
      "\n",
      "Run: 600, exploration: 0.5, score: 72\n",
      "Scores: (min: 11, avg: 168.17, max: 500)\n",
      "\n",
      "Run: 601, exploration: 0.5, score: 182\n",
      "Scores: (min: 11, avg: 166.92, max: 500)\n",
      "\n",
      "Run: 602, exploration: 0.5, score: 82\n",
      "Scores: (min: 11, avg: 164.64, max: 500)\n",
      "\n",
      "Run: 603, exploration: 0.5, score: 313\n",
      "Scores: (min: 11, avg: 166.21, max: 500)\n",
      "\n",
      "Run: 604, exploration: 0.5, score: 292\n",
      "Scores: (min: 11, avg: 168.95, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 605, exploration: 0.5, score: 173\n",
      "Scores: (min: 11, avg: 169.81, max: 500)\n",
      "\n",
      "Run: 606, exploration: 0.5, score: 152\n",
      "Scores: (min: 11, avg: 170.76, max: 500)\n",
      "\n",
      "Run: 607, exploration: 0.5, score: 84\n",
      "Scores: (min: 11, avg: 169.32, max: 500)\n",
      "\n",
      "Run: 608, exploration: 0.5, score: 26\n",
      "Scores: (min: 11, avg: 168.19, max: 500)\n",
      "\n",
      "Run: 609, exploration: 0.5, score: 297\n",
      "Scores: (min: 11, avg: 170.97, max: 500)\n",
      "\n",
      "Run: 610, exploration: 0.5, score: 190\n",
      "Scores: (min: 11, avg: 171.94, max: 500)\n",
      "\n",
      "Run: 611, exploration: 0.5, score: 228\n",
      "Scores: (min: 11, avg: 174, max: 500)\n",
      "\n",
      "Run: 612, exploration: 0.5, score: 91\n",
      "Scores: (min: 11, avg: 174.58, max: 500)\n",
      "\n",
      "Run: 613, exploration: 0.5, score: 147\n",
      "Scores: (min: 11, avg: 171.42, max: 500)\n",
      "\n",
      "Run: 614, exploration: 0.5, score: 135\n",
      "Scores: (min: 11, avg: 168.78, max: 500)\n",
      "\n",
      "Run: 615, exploration: 0.5, score: 411\n",
      "Scores: (min: 11, avg: 172.18, max: 500)\n",
      "\n",
      "Run: 616, exploration: 0.5, score: 119\n",
      "Scores: (min: 11, avg: 172.93, max: 500)\n",
      "\n",
      "Run: 617, exploration: 0.5, score: 80\n",
      "Scores: (min: 11, avg: 173.55, max: 500)\n",
      "\n",
      "Run: 618, exploration: 0.5, score: 102\n",
      "Scores: (min: 11, avg: 173.05, max: 500)\n",
      "\n",
      "Run: 619, exploration: 0.5, score: 88\n",
      "Scores: (min: 11, avg: 173.72, max: 500)\n",
      "\n",
      "Run: 620, exploration: 0.5, score: 274\n",
      "Scores: (min: 11, avg: 174.42, max: 500)\n",
      "\n",
      "Run: 621, exploration: 0.5, score: 281\n",
      "Scores: (min: 11, avg: 173.71, max: 500)\n",
      "\n",
      "Run: 622, exploration: 0.5, score: 95\n",
      "Scores: (min: 11, avg: 173.01, max: 500)\n",
      "\n",
      "Run: 623, exploration: 0.5, score: 174\n",
      "Scores: (min: 11, avg: 172.78, max: 500)\n",
      "\n",
      "Run: 624, exploration: 0.5, score: 21\n",
      "Scores: (min: 11, avg: 172.63, max: 500)\n",
      "\n",
      "Run: 625, exploration: 0.5, score: 286\n",
      "Scores: (min: 11, avg: 174.06, max: 500)\n",
      "\n",
      "Run: 626, exploration: 0.5, score: 22\n",
      "Scores: (min: 11, avg: 173.82, max: 500)\n",
      "\n",
      "Run: 627, exploration: 0.5, score: 198\n",
      "Scores: (min: 11, avg: 174.52, max: 500)\n",
      "\n",
      "Run: 628, exploration: 0.5, score: 116\n",
      "Scores: (min: 11, avg: 173.59, max: 500)\n",
      "\n",
      "Run: 629, exploration: 0.5, score: 456\n",
      "Scores: (min: 11, avg: 176.4, max: 500)\n",
      "\n",
      "Run: 630, exploration: 0.5, score: 245\n",
      "Scores: (min: 11, avg: 175.36, max: 500)\n",
      "\n",
      "Run: 631, exploration: 0.5, score: 304\n",
      "Scores: (min: 11, avg: 178.12, max: 500)\n",
      "\n",
      "Run: 632, exploration: 0.5, score: 99\n",
      "Scores: (min: 11, avg: 174.75, max: 500)\n",
      "\n",
      "Run: 633, exploration: 0.5, score: 396\n",
      "Scores: (min: 11, avg: 174.71, max: 500)\n",
      "\n",
      "Run: 634, exploration: 0.5, score: 192\n",
      "Scores: (min: 11, avg: 174.24, max: 500)\n",
      "\n",
      "Run: 635, exploration: 0.5, score: 337\n",
      "Scores: (min: 11, avg: 174.27, max: 500)\n",
      "\n",
      "Run: 636, exploration: 0.5, score: 45\n",
      "Scores: (min: 11, avg: 173.03, max: 500)\n",
      "\n",
      "Run: 637, exploration: 0.5, score: 64\n",
      "Scores: (min: 11, avg: 173.14, max: 500)\n",
      "\n",
      "Run: 638, exploration: 0.5, score: 225\n",
      "Scores: (min: 12, avg: 175.28, max: 500)\n",
      "\n",
      "Run: 639, exploration: 0.5, score: 13\n",
      "Scores: (min: 12, avg: 174.09, max: 500)\n",
      "\n",
      "Run: 640, exploration: 0.5, score: 245\n",
      "Scores: (min: 13, avg: 176.42, max: 500)\n",
      "\n",
      "Run: 641, exploration: 0.5, score: 129\n",
      "Scores: (min: 13, avg: 176.94, max: 500)\n",
      "\n",
      "Run: 642, exploration: 0.5, score: 41\n",
      "Scores: (min: 13, avg: 176.86, max: 500)\n",
      "\n",
      "Run: 643, exploration: 0.5, score: 123\n",
      "Scores: (min: 13, avg: 176.52, max: 500)\n",
      "\n",
      "Run: 644, exploration: 0.5, score: 83\n",
      "Scores: (min: 13, avg: 174.99, max: 500)\n",
      "\n",
      "Run: 645, exploration: 0.5, score: 67\n",
      "Scores: (min: 13, avg: 174.09, max: 500)\n",
      "\n",
      "Run: 646, exploration: 0.5, score: 73\n",
      "Scores: (min: 13, avg: 171.88, max: 500)\n",
      "\n",
      "Run: 647, exploration: 0.5, score: 140\n",
      "Scores: (min: 13, avg: 171.71, max: 500)\n",
      "\n",
      "Run: 648, exploration: 0.5, score: 224\n",
      "Scores: (min: 13, avg: 171.75, max: 500)\n",
      "\n",
      "Run: 649, exploration: 0.5, score: 22\n",
      "Scores: (min: 13, avg: 170.87, max: 500)\n",
      "\n",
      "Run: 650, exploration: 0.5, score: 264\n",
      "Scores: (min: 13, avg: 172, max: 500)\n",
      "\n",
      "Run: 651, exploration: 0.5, score: 160\n",
      "Scores: (min: 13, avg: 170.85, max: 500)\n",
      "\n",
      "Run: 652, exploration: 0.5, score: 180\n",
      "Scores: (min: 13, avg: 172.16, max: 500)\n",
      "\n",
      "Run: 653, exploration: 0.5, score: 85\n",
      "Scores: (min: 13, avg: 171.75, max: 500)\n",
      "\n",
      "Run: 654, exploration: 0.5, score: 394\n",
      "Scores: (min: 13, avg: 173.16, max: 500)\n",
      "\n",
      "Run: 655, exploration: 0.5, score: 36\n",
      "Scores: (min: 13, avg: 172.51, max: 500)\n",
      "\n",
      "Run: 656, exploration: 0.5, score: 198\n",
      "Scores: (min: 13, avg: 170.11, max: 500)\n",
      "\n",
      "Run: 657, exploration: 0.5, score: 21\n",
      "Scores: (min: 13, avg: 167.56, max: 500)\n",
      "\n",
      "Run: 658, exploration: 0.5, score: 100\n",
      "Scores: (min: 13, avg: 168.37, max: 500)\n",
      "\n",
      "Run: 659, exploration: 0.5, score: 160\n",
      "Scores: (min: 13, avg: 167.16, max: 500)\n",
      "\n",
      "Run: 660, exploration: 0.5, score: 59\n",
      "Scores: (min: 13, avg: 166.13, max: 500)\n",
      "\n",
      "Run: 661, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 170.77, max: 500)\n",
      "\n",
      "Run: 662, exploration: 0.5, score: 125\n",
      "Scores: (min: 13, avg: 171.45, max: 500)\n",
      "\n",
      "Run: 663, exploration: 0.5, score: 158\n",
      "Scores: (min: 13, avg: 168.03, max: 500)\n",
      "\n",
      "Run: 664, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 169.8, max: 500)\n",
      "\n",
      "Run: 665, exploration: 0.5, score: 402\n",
      "Scores: (min: 13, avg: 172.69, max: 500)\n",
      "\n",
      "Run: 666, exploration: 0.5, score: 252\n",
      "Scores: (min: 13, avg: 173.77, max: 500)\n",
      "\n",
      "Run: 667, exploration: 0.5, score: 235\n",
      "Scores: (min: 13, avg: 173.99, max: 500)\n",
      "\n",
      "Run: 668, exploration: 0.5, score: 232\n",
      "Scores: (min: 13, avg: 172.74, max: 500)\n",
      "\n",
      "Run: 669, exploration: 0.5, score: 62\n",
      "Scores: (min: 13, avg: 172.35, max: 500)\n",
      "\n",
      "Run: 670, exploration: 0.5, score: 326\n",
      "Scores: (min: 13, avg: 171.67, max: 500)\n",
      "\n",
      "Run: 671, exploration: 0.5, score: 233\n",
      "Scores: (min: 13, avg: 169, max: 500)\n",
      "\n",
      "Run: 672, exploration: 0.5, score: 53\n",
      "Scores: (min: 13, avg: 168.91, max: 500)\n",
      "\n",
      "Run: 673, exploration: 0.5, score: 159\n",
      "Scores: (min: 13, avg: 169.23, max: 500)\n",
      "\n",
      "Run: 674, exploration: 0.5, score: 334\n",
      "Scores: (min: 13, avg: 172.39, max: 500)\n",
      "\n",
      "Run: 675, exploration: 0.5, score: 77\n",
      "Scores: (min: 13, avg: 169.74, max: 500)\n",
      "\n",
      "Run: 676, exploration: 0.5, score: 87\n",
      "Scores: (min: 13, avg: 168.77, max: 500)\n",
      "\n",
      "Run: 677, exploration: 0.5, score: 40\n",
      "Scores: (min: 13, avg: 168.61, max: 500)\n",
      "\n",
      "Run: 678, exploration: 0.5, score: 74\n",
      "Scores: (min: 13, avg: 169.04, max: 500)\n",
      "\n",
      "Run: 679, exploration: 0.5, score: 108\n",
      "Scores: (min: 13, avg: 167.23, max: 500)\n",
      "\n",
      "Run: 680, exploration: 0.5, score: 326\n",
      "Scores: (min: 13, avg: 168.94, max: 500)\n",
      "\n",
      "Run: 681, exploration: 0.5, score: 91\n",
      "Scores: (min: 13, avg: 168.52, max: 500)\n",
      "\n",
      "Run: 682, exploration: 0.5, score: 499\n",
      "Scores: (min: 13, avg: 171.61, max: 500)\n",
      "\n",
      "Run: 683, exploration: 0.5, score: 171\n",
      "Scores: (min: 13, avg: 172.82, max: 500)\n",
      "\n",
      "Run: 684, exploration: 0.5, score: 209\n",
      "Scores: (min: 13, avg: 174.43, max: 500)\n",
      "\n",
      "Run: 685, exploration: 0.5, score: 92\n",
      "Scores: (min: 13, avg: 174.85, max: 500)\n",
      "\n",
      "Run: 686, exploration: 0.5, score: 263\n",
      "Scores: (min: 13, avg: 177.04, max: 500)\n",
      "\n",
      "Run: 687, exploration: 0.5, score: 183\n",
      "Scores: (min: 13, avg: 177.81, max: 500)\n",
      "\n",
      "Run: 688, exploration: 0.5, score: 205\n",
      "Scores: (min: 13, avg: 179.65, max: 500)\n",
      "\n",
      "Run: 689, exploration: 0.5, score: 229\n",
      "Scores: (min: 13, avg: 180.75, max: 500)\n",
      "\n",
      "Run: 690, exploration: 0.5, score: 71\n",
      "Scores: (min: 13, avg: 180.03, max: 500)\n",
      "\n",
      "Run: 691, exploration: 0.5, score: 56\n",
      "Scores: (min: 13, avg: 175.59, max: 500)\n",
      "\n",
      "Run: 692, exploration: 0.5, score: 35\n",
      "Scores: (min: 13, avg: 174.85, max: 500)\n",
      "\n",
      "Run: 693, exploration: 0.5, score: 221\n",
      "Scores: (min: 13, avg: 176.28, max: 500)\n",
      "\n",
      "Run: 694, exploration: 0.5, score: 29\n",
      "Scores: (min: 13, avg: 172.88, max: 500)\n",
      "\n",
      "Run: 695, exploration: 0.5, score: 282\n",
      "Scores: (min: 13, avg: 174.67, max: 500)\n",
      "\n",
      "Run: 696, exploration: 0.5, score: 231\n",
      "Scores: (min: 13, avg: 175.71, max: 500)\n",
      "\n",
      "Run: 697, exploration: 0.5, score: 103\n",
      "Scores: (min: 13, avg: 173.8, max: 500)\n",
      "\n",
      "Run: 698, exploration: 0.5, score: 267\n",
      "Scores: (min: 13, avg: 174.6, max: 500)\n",
      "\n",
      "Run: 699, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 179.31, max: 500)\n",
      "\n",
      "Run: 700, exploration: 0.5, score: 356\n",
      "Scores: (min: 13, avg: 182.15, max: 500)\n",
      "\n",
      "Run: 701, exploration: 0.5, score: 228\n",
      "Scores: (min: 13, avg: 182.61, max: 500)\n",
      "\n",
      "Run: 702, exploration: 0.5, score: 26\n",
      "Scores: (min: 13, avg: 182.05, max: 500)\n",
      "\n",
      "Run: 703, exploration: 0.5, score: 273\n",
      "Scores: (min: 13, avg: 181.65, max: 500)\n",
      "\n",
      "Run: 704, exploration: 0.5, score: 101\n",
      "Scores: (min: 13, avg: 179.74, max: 500)\n",
      "\n",
      "Run: 705, exploration: 0.5, score: 30\n",
      "Scores: (min: 13, avg: 178.31, max: 500)\n",
      "\n",
      "Run: 706, exploration: 0.5, score: 274\n",
      "Scores: (min: 13, avg: 179.53, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 707, exploration: 0.5, score: 245\n",
      "Scores: (min: 13, avg: 181.14, max: 500)\n",
      "\n",
      "Run: 708, exploration: 0.5, score: 119\n",
      "Scores: (min: 13, avg: 182.07, max: 500)\n",
      "\n",
      "Run: 709, exploration: 0.5, score: 61\n",
      "Scores: (min: 13, avg: 179.71, max: 500)\n",
      "\n",
      "Run: 710, exploration: 0.5, score: 41\n",
      "Scores: (min: 13, avg: 178.22, max: 500)\n",
      "\n",
      "Run: 711, exploration: 0.5, score: 83\n",
      "Scores: (min: 13, avg: 176.77, max: 500)\n",
      "\n",
      "Run: 712, exploration: 0.5, score: 203\n",
      "Scores: (min: 13, avg: 177.89, max: 500)\n",
      "\n",
      "Run: 713, exploration: 0.5, score: 34\n",
      "Scores: (min: 13, avg: 176.76, max: 500)\n",
      "\n",
      "Run: 714, exploration: 0.5, score: 99\n",
      "Scores: (min: 13, avg: 176.4, max: 500)\n",
      "\n",
      "Run: 715, exploration: 0.5, score: 121\n",
      "Scores: (min: 13, avg: 173.5, max: 500)\n",
      "\n",
      "Run: 716, exploration: 0.5, score: 37\n",
      "Scores: (min: 13, avg: 172.68, max: 500)\n",
      "\n",
      "Run: 717, exploration: 0.5, score: 45\n",
      "Scores: (min: 13, avg: 172.33, max: 500)\n",
      "\n",
      "Run: 718, exploration: 0.5, score: 54\n",
      "Scores: (min: 13, avg: 171.85, max: 500)\n",
      "\n",
      "Run: 719, exploration: 0.5, score: 28\n",
      "Scores: (min: 13, avg: 171.25, max: 500)\n",
      "\n",
      "Run: 720, exploration: 0.5, score: 76\n",
      "Scores: (min: 13, avg: 169.27, max: 500)\n",
      "\n",
      "Run: 721, exploration: 0.5, score: 131\n",
      "Scores: (min: 13, avg: 167.77, max: 500)\n",
      "\n",
      "Run: 722, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 171.82, max: 500)\n",
      "\n",
      "Run: 723, exploration: 0.5, score: 279\n",
      "Scores: (min: 13, avg: 172.87, max: 500)\n",
      "\n",
      "Run: 724, exploration: 0.5, score: 206\n",
      "Scores: (min: 13, avg: 174.72, max: 500)\n",
      "\n",
      "Run: 725, exploration: 0.5, score: 14\n",
      "Scores: (min: 13, avg: 172, max: 500)\n",
      "\n",
      "Run: 726, exploration: 0.5, score: 167\n",
      "Scores: (min: 13, avg: 173.45, max: 500)\n",
      "\n",
      "Run: 727, exploration: 0.5, score: 146\n",
      "Scores: (min: 13, avg: 172.93, max: 500)\n",
      "\n",
      "Run: 728, exploration: 0.5, score: 81\n",
      "Scores: (min: 13, avg: 172.58, max: 500)\n",
      "\n",
      "Run: 729, exploration: 0.5, score: 39\n",
      "Scores: (min: 13, avg: 168.41, max: 500)\n",
      "\n",
      "Run: 730, exploration: 0.5, score: 72\n",
      "Scores: (min: 13, avg: 166.68, max: 500)\n",
      "\n",
      "Run: 731, exploration: 0.5, score: 11\n",
      "Scores: (min: 11, avg: 163.75, max: 500)\n",
      "\n",
      "Run: 732, exploration: 0.5, score: 221\n",
      "Scores: (min: 11, avg: 164.97, max: 500)\n",
      "\n",
      "Run: 733, exploration: 0.5, score: 43\n",
      "Scores: (min: 11, avg: 161.44, max: 500)\n",
      "\n",
      "Run: 734, exploration: 0.5, score: 449\n",
      "Scores: (min: 11, avg: 164.01, max: 500)\n",
      "\n",
      "Run: 735, exploration: 0.5, score: 87\n",
      "Scores: (min: 11, avg: 161.51, max: 500)\n",
      "\n",
      "Run: 736, exploration: 0.5, score: 103\n",
      "Scores: (min: 11, avg: 162.09, max: 500)\n",
      "\n",
      "Run: 737, exploration: 0.5, score: 124\n",
      "Scores: (min: 11, avg: 162.69, max: 500)\n",
      "\n",
      "Run: 738, exploration: 0.5, score: 61\n",
      "Scores: (min: 11, avg: 161.05, max: 500)\n",
      "\n",
      "Run: 739, exploration: 0.5, score: 106\n",
      "Scores: (min: 11, avg: 161.98, max: 500)\n",
      "\n",
      "Run: 740, exploration: 0.5, score: 48\n",
      "Scores: (min: 11, avg: 160.01, max: 500)\n",
      "\n",
      "Run: 741, exploration: 0.5, score: 196\n",
      "Scores: (min: 11, avg: 160.68, max: 500)\n",
      "\n",
      "Run: 742, exploration: 0.5, score: 127\n",
      "Scores: (min: 11, avg: 161.54, max: 500)\n",
      "\n",
      "Run: 743, exploration: 0.5, score: 46\n",
      "Scores: (min: 11, avg: 160.77, max: 500)\n",
      "\n",
      "Run: 744, exploration: 0.5, score: 162\n",
      "Scores: (min: 11, avg: 161.56, max: 500)\n",
      "\n",
      "Run: 745, exploration: 0.5, score: 43\n",
      "Scores: (min: 11, avg: 161.32, max: 500)\n",
      "\n",
      "Run: 746, exploration: 0.5, score: 108\n",
      "Scores: (min: 11, avg: 161.67, max: 500)\n",
      "\n",
      "Run: 747, exploration: 0.5, score: 156\n",
      "Scores: (min: 11, avg: 161.83, max: 500)\n",
      "\n",
      "Run: 748, exploration: 0.5, score: 137\n",
      "Scores: (min: 11, avg: 160.96, max: 500)\n",
      "\n",
      "Run: 749, exploration: 0.5, score: 216\n",
      "Scores: (min: 11, avg: 162.9, max: 500)\n",
      "\n",
      "Run: 750, exploration: 0.5, score: 142\n",
      "Scores: (min: 11, avg: 161.68, max: 500)\n",
      "\n",
      "Run: 751, exploration: 0.5, score: 164\n",
      "Scores: (min: 11, avg: 161.72, max: 500)\n",
      "\n",
      "Run: 752, exploration: 0.5, score: 27\n",
      "Scores: (min: 11, avg: 160.19, max: 500)\n",
      "\n",
      "Run: 753, exploration: 0.5, score: 232\n",
      "Scores: (min: 11, avg: 161.66, max: 500)\n",
      "\n",
      "Run: 754, exploration: 0.5, score: 131\n",
      "Scores: (min: 11, avg: 159.03, max: 500)\n",
      "\n",
      "Run: 755, exploration: 0.5, score: 71\n",
      "Scores: (min: 11, avg: 159.38, max: 500)\n",
      "\n",
      "Run: 756, exploration: 0.5, score: 296\n",
      "Scores: (min: 11, avg: 160.36, max: 500)\n",
      "\n",
      "Run: 757, exploration: 0.5, score: 187\n",
      "Scores: (min: 11, avg: 162.02, max: 500)\n",
      "\n",
      "Run: 758, exploration: 0.5, score: 181\n",
      "Scores: (min: 11, avg: 162.83, max: 500)\n",
      "\n",
      "Run: 759, exploration: 0.5, score: 143\n",
      "Scores: (min: 11, avg: 162.66, max: 500)\n",
      "\n",
      "Run: 760, exploration: 0.5, score: 67\n",
      "Scores: (min: 11, avg: 162.74, max: 500)\n",
      "\n",
      "Run: 761, exploration: 0.5, score: 176\n",
      "Scores: (min: 11, avg: 159.5, max: 500)\n",
      "\n",
      "Run: 762, exploration: 0.5, score: 226\n",
      "Scores: (min: 11, avg: 160.51, max: 500)\n",
      "\n",
      "Run: 763, exploration: 0.5, score: 60\n",
      "Scores: (min: 11, avg: 159.53, max: 500)\n",
      "\n",
      "Run: 764, exploration: 0.5, score: 127\n",
      "Scores: (min: 11, avg: 155.8, max: 500)\n",
      "\n",
      "Run: 765, exploration: 0.5, score: 258\n",
      "Scores: (min: 11, avg: 154.36, max: 500)\n",
      "\n",
      "Run: 766, exploration: 0.5, score: 13\n",
      "Scores: (min: 11, avg: 151.97, max: 500)\n",
      "\n",
      "Run: 767, exploration: 0.5, score: 137\n",
      "Scores: (min: 11, avg: 150.99, max: 500)\n",
      "\n",
      "Run: 768, exploration: 0.5, score: 225\n",
      "Scores: (min: 11, avg: 150.92, max: 500)\n",
      "\n",
      "Run: 769, exploration: 0.5, score: 267\n",
      "Scores: (min: 11, avg: 152.97, max: 500)\n",
      "\n",
      "Run: 770, exploration: 0.5, score: 116\n",
      "Scores: (min: 11, avg: 150.87, max: 500)\n",
      "\n",
      "Run: 771, exploration: 0.5, score: 45\n",
      "Scores: (min: 11, avg: 148.99, max: 500)\n",
      "\n",
      "Run: 772, exploration: 0.5, score: 339\n",
      "Scores: (min: 11, avg: 151.85, max: 500)\n",
      "\n",
      "Run: 773, exploration: 0.5, score: 92\n",
      "Scores: (min: 11, avg: 151.18, max: 500)\n",
      "\n",
      "Run: 774, exploration: 0.5, score: 132\n",
      "Scores: (min: 11, avg: 149.16, max: 500)\n",
      "\n",
      "Run: 775, exploration: 0.5, score: 306\n",
      "Scores: (min: 11, avg: 151.45, max: 500)\n",
      "\n",
      "Run: 776, exploration: 0.5, score: 303\n",
      "Scores: (min: 11, avg: 153.61, max: 500)\n",
      "\n",
      "Run: 777, exploration: 0.5, score: 458\n",
      "Scores: (min: 11, avg: 157.79, max: 500)\n",
      "\n",
      "Run: 778, exploration: 0.5, score: 402\n",
      "Scores: (min: 11, avg: 161.07, max: 500)\n",
      "\n",
      "Run: 779, exploration: 0.5, score: 289\n",
      "Scores: (min: 11, avg: 162.88, max: 500)\n",
      "\n",
      "Run: 780, exploration: 0.5, score: 309\n",
      "Scores: (min: 11, avg: 162.71, max: 500)\n",
      "\n",
      "Run: 781, exploration: 0.5, score: 100\n",
      "Scores: (min: 11, avg: 162.8, max: 500)\n",
      "\n",
      "Run: 782, exploration: 0.5, score: 355\n",
      "Scores: (min: 11, avg: 161.36, max: 500)\n",
      "\n",
      "Run: 783, exploration: 0.5, score: 500\n",
      "Scores: (min: 11, avg: 164.65, max: 500)\n",
      "\n",
      "Run: 784, exploration: 0.5, score: 145\n",
      "Scores: (min: 11, avg: 164.01, max: 500)\n",
      "\n",
      "Run: 785, exploration: 0.5, score: 126\n",
      "Scores: (min: 11, avg: 164.35, max: 500)\n",
      "\n",
      "Run: 786, exploration: 0.5, score: 236\n",
      "Scores: (min: 11, avg: 164.08, max: 500)\n",
      "\n",
      "Run: 787, exploration: 0.5, score: 124\n",
      "Scores: (min: 11, avg: 163.49, max: 500)\n",
      "\n",
      "Run: 788, exploration: 0.5, score: 227\n",
      "Scores: (min: 11, avg: 163.71, max: 500)\n",
      "\n",
      "Run: 789, exploration: 0.5, score: 191\n",
      "Scores: (min: 11, avg: 163.33, max: 500)\n",
      "\n",
      "Run: 790, exploration: 0.5, score: 345\n",
      "Scores: (min: 11, avg: 166.07, max: 500)\n",
      "\n",
      "Run: 791, exploration: 0.5, score: 32\n",
      "Scores: (min: 11, avg: 165.83, max: 500)\n",
      "\n",
      "Run: 792, exploration: 0.5, score: 342\n",
      "Scores: (min: 11, avg: 168.9, max: 500)\n",
      "\n",
      "Run: 793, exploration: 0.5, score: 17\n",
      "Scores: (min: 11, avg: 166.86, max: 500)\n",
      "\n",
      "Run: 794, exploration: 0.5, score: 191\n",
      "Scores: (min: 11, avg: 168.48, max: 500)\n",
      "\n",
      "Run: 795, exploration: 0.5, score: 132\n",
      "Scores: (min: 11, avg: 166.98, max: 500)\n",
      "\n",
      "Run: 796, exploration: 0.5, score: 53\n",
      "Scores: (min: 11, avg: 165.2, max: 500)\n",
      "\n",
      "Run: 797, exploration: 0.5, score: 24\n",
      "Scores: (min: 11, avg: 164.41, max: 500)\n",
      "\n",
      "Run: 798, exploration: 0.5, score: 49\n",
      "Scores: (min: 11, avg: 162.23, max: 500)\n",
      "\n",
      "Run: 799, exploration: 0.5, score: 95\n",
      "Scores: (min: 11, avg: 158.18, max: 500)\n",
      "\n",
      "Run: 800, exploration: 0.5, score: 96\n",
      "Scores: (min: 11, avg: 155.58, max: 500)\n",
      "\n",
      "Run: 801, exploration: 0.5, score: 135\n",
      "Scores: (min: 11, avg: 154.65, max: 500)\n",
      "\n",
      "Run: 802, exploration: 0.5, score: 29\n",
      "Scores: (min: 11, avg: 154.68, max: 500)\n",
      "\n",
      "Run: 803, exploration: 0.5, score: 344\n",
      "Scores: (min: 11, avg: 155.39, max: 500)\n",
      "\n",
      "Run: 804, exploration: 0.5, score: 56\n",
      "Scores: (min: 11, avg: 154.94, max: 500)\n",
      "\n",
      "Run: 805, exploration: 0.5, score: 91\n",
      "Scores: (min: 11, avg: 155.55, max: 500)\n",
      "\n",
      "Run: 806, exploration: 0.5, score: 137\n",
      "Scores: (min: 11, avg: 154.18, max: 500)\n",
      "\n",
      "Run: 807, exploration: 0.5, score: 18\n",
      "Scores: (min: 11, avg: 151.91, max: 500)\n",
      "\n",
      "Run: 808, exploration: 0.5, score: 65\n",
      "Scores: (min: 11, avg: 151.37, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 809, exploration: 0.5, score: 127\n",
      "Scores: (min: 11, avg: 152.03, max: 500)\n",
      "\n",
      "Run: 810, exploration: 0.5, score: 500\n",
      "Scores: (min: 11, avg: 156.62, max: 500)\n",
      "\n",
      "Run: 811, exploration: 0.5, score: 117\n",
      "Scores: (min: 11, avg: 156.96, max: 500)\n",
      "\n",
      "Run: 812, exploration: 0.5, score: 500\n",
      "Scores: (min: 11, avg: 159.93, max: 500)\n",
      "\n",
      "Run: 813, exploration: 0.5, score: 77\n",
      "Scores: (min: 11, avg: 160.36, max: 500)\n",
      "\n",
      "Run: 814, exploration: 0.5, score: 57\n",
      "Scores: (min: 11, avg: 159.94, max: 500)\n",
      "\n",
      "Run: 815, exploration: 0.5, score: 48\n",
      "Scores: (min: 11, avg: 159.21, max: 500)\n",
      "\n",
      "Run: 816, exploration: 0.5, score: 12\n",
      "Scores: (min: 11, avg: 158.96, max: 500)\n",
      "\n",
      "Run: 817, exploration: 0.5, score: 75\n",
      "Scores: (min: 11, avg: 159.26, max: 500)\n",
      "\n",
      "Run: 818, exploration: 0.5, score: 117\n",
      "Scores: (min: 11, avg: 159.89, max: 500)\n",
      "\n",
      "Run: 819, exploration: 0.5, score: 183\n",
      "Scores: (min: 11, avg: 161.44, max: 500)\n",
      "\n",
      "Run: 820, exploration: 0.5, score: 126\n",
      "Scores: (min: 11, avg: 161.94, max: 500)\n",
      "\n",
      "Run: 821, exploration: 0.5, score: 270\n",
      "Scores: (min: 11, avg: 163.33, max: 500)\n",
      "\n",
      "Run: 822, exploration: 0.5, score: 39\n",
      "Scores: (min: 11, avg: 158.72, max: 500)\n",
      "\n",
      "Run: 823, exploration: 0.5, score: 335\n",
      "Scores: (min: 11, avg: 159.28, max: 500)\n",
      "\n",
      "Run: 824, exploration: 0.5, score: 234\n",
      "Scores: (min: 11, avg: 159.56, max: 500)\n",
      "\n",
      "Run: 825, exploration: 0.5, score: 201\n",
      "Scores: (min: 11, avg: 161.43, max: 500)\n",
      "\n",
      "Run: 826, exploration: 0.5, score: 180\n",
      "Scores: (min: 11, avg: 161.56, max: 500)\n",
      "\n",
      "Run: 827, exploration: 0.5, score: 172\n",
      "Scores: (min: 11, avg: 161.82, max: 500)\n",
      "\n",
      "Run: 828, exploration: 0.5, score: 65\n",
      "Scores: (min: 11, avg: 161.66, max: 500)\n",
      "\n",
      "Run: 829, exploration: 0.5, score: 208\n",
      "Scores: (min: 11, avg: 163.35, max: 500)\n",
      "\n",
      "Run: 830, exploration: 0.5, score: 63\n",
      "Scores: (min: 11, avg: 163.26, max: 500)\n",
      "\n",
      "Run: 831, exploration: 0.5, score: 74\n",
      "Scores: (min: 12, avg: 163.89, max: 500)\n",
      "\n",
      "Run: 832, exploration: 0.5, score: 113\n",
      "Scores: (min: 12, avg: 162.81, max: 500)\n",
      "\n",
      "Run: 833, exploration: 0.5, score: 283\n",
      "Scores: (min: 12, avg: 165.21, max: 500)\n",
      "\n",
      "Run: 834, exploration: 0.5, score: 83\n",
      "Scores: (min: 12, avg: 161.55, max: 500)\n",
      "\n",
      "Run: 835, exploration: 0.5, score: 269\n",
      "Scores: (min: 12, avg: 163.37, max: 500)\n",
      "\n",
      "Run: 836, exploration: 0.5, score: 241\n",
      "Scores: (min: 12, avg: 164.75, max: 500)\n",
      "\n",
      "Run: 837, exploration: 0.5, score: 291\n",
      "Scores: (min: 12, avg: 166.42, max: 500)\n",
      "\n",
      "Run: 838, exploration: 0.5, score: 125\n",
      "Scores: (min: 12, avg: 167.06, max: 500)\n",
      "\n",
      "Run: 839, exploration: 0.5, score: 121\n",
      "Scores: (min: 12, avg: 167.21, max: 500)\n",
      "\n",
      "Run: 840, exploration: 0.5, score: 262\n",
      "Scores: (min: 12, avg: 169.35, max: 500)\n",
      "\n",
      "Run: 841, exploration: 0.5, score: 58\n",
      "Scores: (min: 12, avg: 167.97, max: 500)\n",
      "\n",
      "Run: 842, exploration: 0.5, score: 105\n",
      "Scores: (min: 12, avg: 167.75, max: 500)\n",
      "\n",
      "Run: 843, exploration: 0.5, score: 30\n",
      "Scores: (min: 12, avg: 167.59, max: 500)\n",
      "\n",
      "Run: 844, exploration: 0.5, score: 155\n",
      "Scores: (min: 12, avg: 167.52, max: 500)\n",
      "\n",
      "Run: 845, exploration: 0.5, score: 289\n",
      "Scores: (min: 12, avg: 169.98, max: 500)\n",
      "\n",
      "Run: 846, exploration: 0.5, score: 42\n",
      "Scores: (min: 12, avg: 169.32, max: 500)\n",
      "\n",
      "Run: 847, exploration: 0.5, score: 114\n",
      "Scores: (min: 12, avg: 168.9, max: 500)\n",
      "\n",
      "Run: 848, exploration: 0.5, score: 26\n",
      "Scores: (min: 12, avg: 167.79, max: 500)\n",
      "\n",
      "Run: 849, exploration: 0.5, score: 17\n",
      "Scores: (min: 12, avg: 165.8, max: 500)\n",
      "\n",
      "Run: 850, exploration: 0.5, score: 124\n",
      "Scores: (min: 12, avg: 165.62, max: 500)\n",
      "\n",
      "Run: 851, exploration: 0.5, score: 129\n",
      "Scores: (min: 12, avg: 165.27, max: 500)\n",
      "\n",
      "Run: 852, exploration: 0.5, score: 25\n",
      "Scores: (min: 12, avg: 165.25, max: 500)\n",
      "\n",
      "Run: 853, exploration: 0.5, score: 327\n",
      "Scores: (min: 12, avg: 166.2, max: 500)\n",
      "\n",
      "Run: 854, exploration: 0.5, score: 78\n",
      "Scores: (min: 12, avg: 165.67, max: 500)\n",
      "\n",
      "Run: 855, exploration: 0.5, score: 97\n",
      "Scores: (min: 12, avg: 165.93, max: 500)\n",
      "\n",
      "Run: 856, exploration: 0.5, score: 48\n",
      "Scores: (min: 12, avg: 163.45, max: 500)\n",
      "\n",
      "Run: 857, exploration: 0.5, score: 402\n",
      "Scores: (min: 12, avg: 165.6, max: 500)\n",
      "\n",
      "Run: 858, exploration: 0.5, score: 115\n",
      "Scores: (min: 12, avg: 164.94, max: 500)\n",
      "\n",
      "Run: 859, exploration: 0.5, score: 92\n",
      "Scores: (min: 12, avg: 164.43, max: 500)\n",
      "\n",
      "Run: 860, exploration: 0.5, score: 59\n",
      "Scores: (min: 12, avg: 164.35, max: 500)\n",
      "\n",
      "Run: 861, exploration: 0.5, score: 188\n",
      "Scores: (min: 12, avg: 164.47, max: 500)\n",
      "\n",
      "Run: 862, exploration: 0.5, score: 220\n",
      "Scores: (min: 12, avg: 164.41, max: 500)\n",
      "\n",
      "Run: 863, exploration: 0.5, score: 331\n",
      "Scores: (min: 12, avg: 167.12, max: 500)\n",
      "\n",
      "Run: 864, exploration: 0.5, score: 96\n",
      "Scores: (min: 12, avg: 166.81, max: 500)\n",
      "\n",
      "Run: 865, exploration: 0.5, score: 362\n",
      "Scores: (min: 12, avg: 167.85, max: 500)\n",
      "\n",
      "Run: 866, exploration: 0.5, score: 441\n",
      "Scores: (min: 12, avg: 172.13, max: 500)\n",
      "\n",
      "Run: 867, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 175.76, max: 500)\n",
      "\n",
      "Run: 868, exploration: 0.5, score: 15\n",
      "Scores: (min: 12, avg: 173.66, max: 500)\n",
      "\n",
      "Run: 869, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 175.99, max: 500)\n",
      "\n",
      "Run: 870, exploration: 0.5, score: 18\n",
      "Scores: (min: 12, avg: 175.01, max: 500)\n",
      "\n",
      "Run: 871, exploration: 0.5, score: 24\n",
      "Scores: (min: 12, avg: 174.8, max: 500)\n",
      "\n",
      "Run: 872, exploration: 0.5, score: 312\n",
      "Scores: (min: 12, avg: 174.53, max: 500)\n",
      "\n",
      "Run: 873, exploration: 0.5, score: 449\n",
      "Scores: (min: 12, avg: 178.1, max: 500)\n",
      "\n",
      "Run: 874, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 181.78, max: 500)\n",
      "\n",
      "Run: 875, exploration: 0.5, score: 182\n",
      "Scores: (min: 12, avg: 180.54, max: 500)\n",
      "\n",
      "Run: 876, exploration: 0.5, score: 75\n",
      "Scores: (min: 12, avg: 178.26, max: 500)\n",
      "\n",
      "Run: 877, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 178.68, max: 500)\n",
      "\n",
      "Run: 878, exploration: 0.5, score: 46\n",
      "Scores: (min: 12, avg: 175.12, max: 500)\n",
      "\n",
      "Run: 879, exploration: 0.5, score: 78\n",
      "Scores: (min: 12, avg: 173.01, max: 500)\n",
      "\n",
      "Run: 880, exploration: 0.5, score: 421\n",
      "Scores: (min: 12, avg: 174.13, max: 500)\n",
      "\n",
      "Run: 881, exploration: 0.5, score: 262\n",
      "Scores: (min: 12, avg: 175.75, max: 500)\n",
      "\n",
      "Run: 882, exploration: 0.5, score: 63\n",
      "Scores: (min: 12, avg: 172.83, max: 500)\n",
      "\n",
      "Run: 883, exploration: 0.5, score: 177\n",
      "Scores: (min: 12, avg: 169.6, max: 500)\n",
      "\n",
      "Run: 884, exploration: 0.5, score: 53\n",
      "Scores: (min: 12, avg: 168.68, max: 500)\n",
      "\n",
      "Run: 885, exploration: 0.5, score: 56\n",
      "Scores: (min: 12, avg: 167.98, max: 500)\n",
      "\n",
      "Run: 886, exploration: 0.5, score: 418\n",
      "Scores: (min: 12, avg: 169.8, max: 500)\n",
      "\n",
      "Run: 887, exploration: 0.5, score: 497\n",
      "Scores: (min: 12, avg: 173.53, max: 500)\n",
      "\n",
      "Run: 888, exploration: 0.5, score: 166\n",
      "Scores: (min: 12, avg: 172.92, max: 500)\n",
      "\n",
      "Run: 889, exploration: 0.5, score: 224\n",
      "Scores: (min: 12, avg: 173.25, max: 500)\n",
      "\n",
      "Run: 890, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 174.8, max: 500)\n",
      "\n",
      "Run: 891, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 179.48, max: 500)\n",
      "\n",
      "Run: 892, exploration: 0.5, score: 500\n",
      "Scores: (min: 12, avg: 181.06, max: 500)\n",
      "\n",
      "Run: 893, exploration: 0.5, score: 89\n",
      "Scores: (min: 12, avg: 181.78, max: 500)\n",
      "\n",
      "Run: 894, exploration: 0.5, score: 29\n",
      "Scores: (min: 12, avg: 180.16, max: 500)\n",
      "\n",
      "Run: 895, exploration: 0.5, score: 219\n",
      "Scores: (min: 12, avg: 181.03, max: 500)\n",
      "\n",
      "Run: 896, exploration: 0.5, score: 14\n",
      "Scores: (min: 12, avg: 180.64, max: 500)\n",
      "\n",
      "Run: 897, exploration: 0.5, score: 269\n",
      "Scores: (min: 12, avg: 183.09, max: 500)\n",
      "\n",
      "Run: 898, exploration: 0.5, score: 67\n",
      "Scores: (min: 12, avg: 183.27, max: 500)\n",
      "\n",
      "Run: 899, exploration: 0.5, score: 115\n",
      "Scores: (min: 12, avg: 183.47, max: 500)\n",
      "\n",
      "Run: 900, exploration: 0.5, score: 372\n",
      "Scores: (min: 12, avg: 186.23, max: 500)\n",
      "\n",
      "Run: 901, exploration: 0.5, score: 73\n",
      "Scores: (min: 12, avg: 185.61, max: 500)\n",
      "\n",
      "Run: 902, exploration: 0.5, score: 246\n",
      "Scores: (min: 12, avg: 187.78, max: 500)\n",
      "\n",
      "Run: 903, exploration: 0.5, score: 14\n",
      "Scores: (min: 12, avg: 184.48, max: 500)\n",
      "\n",
      "Run: 904, exploration: 0.5, score: 169\n",
      "Scores: (min: 12, avg: 185.61, max: 500)\n",
      "\n",
      "Run: 905, exploration: 0.5, score: 62\n",
      "Scores: (min: 12, avg: 185.32, max: 500)\n",
      "\n",
      "Run: 906, exploration: 0.5, score: 292\n",
      "Scores: (min: 12, avg: 186.87, max: 500)\n",
      "\n",
      "Run: 907, exploration: 0.5, score: 220\n",
      "Scores: (min: 12, avg: 188.89, max: 500)\n",
      "\n",
      "Run: 908, exploration: 0.5, score: 13\n",
      "Scores: (min: 12, avg: 188.37, max: 500)\n",
      "\n",
      "Run: 909, exploration: 0.5, score: 32\n",
      "Scores: (min: 12, avg: 187.42, max: 500)\n",
      "\n",
      "Run: 910, exploration: 0.5, score: 171\n",
      "Scores: (min: 12, avg: 184.13, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 911, exploration: 0.5, score: 115\n",
      "Scores: (min: 12, avg: 184.11, max: 500)\n",
      "\n",
      "Run: 912, exploration: 0.5, score: 49\n",
      "Scores: (min: 12, avg: 179.6, max: 500)\n",
      "\n",
      "Run: 913, exploration: 0.5, score: 239\n",
      "Scores: (min: 12, avg: 181.22, max: 500)\n",
      "\n",
      "Run: 914, exploration: 0.5, score: 52\n",
      "Scores: (min: 12, avg: 181.17, max: 500)\n",
      "\n",
      "Run: 915, exploration: 0.5, score: 190\n",
      "Scores: (min: 12, avg: 182.59, max: 500)\n",
      "\n",
      "Run: 916, exploration: 0.5, score: 220\n",
      "Scores: (min: 13, avg: 184.67, max: 500)\n",
      "\n",
      "Run: 917, exploration: 0.5, score: 29\n",
      "Scores: (min: 13, avg: 184.21, max: 500)\n",
      "\n",
      "Run: 918, exploration: 0.5, score: 238\n",
      "Scores: (min: 13, avg: 185.42, max: 500)\n",
      "\n",
      "Run: 919, exploration: 0.5, score: 102\n",
      "Scores: (min: 13, avg: 184.61, max: 500)\n",
      "\n",
      "Run: 920, exploration: 0.5, score: 105\n",
      "Scores: (min: 13, avg: 184.4, max: 500)\n",
      "\n",
      "Run: 921, exploration: 0.5, score: 319\n",
      "Scores: (min: 13, avg: 184.89, max: 500)\n",
      "\n",
      "Run: 922, exploration: 0.5, score: 317\n",
      "Scores: (min: 13, avg: 187.67, max: 500)\n",
      "\n",
      "Run: 923, exploration: 0.5, score: 25\n",
      "Scores: (min: 13, avg: 184.57, max: 500)\n",
      "\n",
      "Run: 924, exploration: 0.5, score: 22\n",
      "Scores: (min: 13, avg: 182.45, max: 500)\n",
      "\n",
      "Run: 925, exploration: 0.5, score: 346\n",
      "Scores: (min: 13, avg: 183.9, max: 500)\n",
      "\n",
      "Run: 926, exploration: 0.5, score: 158\n",
      "Scores: (min: 13, avg: 183.68, max: 500)\n",
      "\n",
      "Run: 927, exploration: 0.5, score: 322\n",
      "Scores: (min: 13, avg: 185.18, max: 500)\n",
      "\n",
      "Run: 928, exploration: 0.5, score: 191\n",
      "Scores: (min: 13, avg: 186.44, max: 500)\n",
      "\n",
      "Run: 929, exploration: 0.5, score: 113\n",
      "Scores: (min: 13, avg: 185.49, max: 500)\n",
      "\n",
      "Run: 930, exploration: 0.5, score: 196\n",
      "Scores: (min: 13, avg: 186.82, max: 500)\n",
      "\n",
      "Run: 931, exploration: 0.5, score: 112\n",
      "Scores: (min: 13, avg: 187.2, max: 500)\n",
      "\n",
      "Run: 932, exploration: 0.5, score: 283\n",
      "Scores: (min: 13, avg: 188.9, max: 500)\n",
      "\n",
      "Run: 933, exploration: 0.5, score: 124\n",
      "Scores: (min: 13, avg: 187.31, max: 500)\n",
      "\n",
      "Run: 934, exploration: 0.5, score: 233\n",
      "Scores: (min: 13, avg: 188.81, max: 500)\n",
      "\n",
      "Run: 935, exploration: 0.5, score: 180\n",
      "Scores: (min: 13, avg: 187.92, max: 500)\n",
      "\n",
      "Run: 936, exploration: 0.5, score: 240\n",
      "Scores: (min: 13, avg: 187.91, max: 500)\n",
      "\n",
      "Run: 937, exploration: 0.5, score: 276\n",
      "Scores: (min: 13, avg: 187.76, max: 500)\n",
      "\n",
      "Run: 938, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 191.51, max: 500)\n",
      "\n",
      "Run: 939, exploration: 0.5, score: 180\n",
      "Scores: (min: 13, avg: 192.1, max: 500)\n",
      "\n",
      "Run: 940, exploration: 0.5, score: 47\n",
      "Scores: (min: 13, avg: 189.95, max: 500)\n",
      "\n",
      "Run: 941, exploration: 0.5, score: 49\n",
      "Scores: (min: 13, avg: 189.86, max: 500)\n",
      "\n",
      "Run: 942, exploration: 0.5, score: 40\n",
      "Scores: (min: 13, avg: 189.21, max: 500)\n",
      "\n",
      "Run: 943, exploration: 0.5, score: 129\n",
      "Scores: (min: 13, avg: 190.2, max: 500)\n",
      "\n",
      "Run: 944, exploration: 0.5, score: 90\n",
      "Scores: (min: 13, avg: 189.55, max: 500)\n",
      "\n",
      "Run: 945, exploration: 0.5, score: 154\n",
      "Scores: (min: 13, avg: 188.2, max: 500)\n",
      "\n",
      "Run: 946, exploration: 0.5, score: 144\n",
      "Scores: (min: 13, avg: 189.22, max: 500)\n",
      "\n",
      "Run: 947, exploration: 0.5, score: 38\n",
      "Scores: (min: 13, avg: 188.46, max: 500)\n",
      "\n",
      "Run: 948, exploration: 0.5, score: 318\n",
      "Scores: (min: 13, avg: 191.38, max: 500)\n",
      "\n",
      "Run: 949, exploration: 0.5, score: 161\n",
      "Scores: (min: 13, avg: 192.82, max: 500)\n",
      "\n",
      "Run: 950, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 196.58, max: 500)\n",
      "\n",
      "Solved in 850 runs, 950 total runs.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'exit' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-12-e23913c48dc8>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mcartpole\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32m<ipython-input-11-c51df9c70f0a>\u001B[0m in \u001B[0;36mcartpole\u001B[1;34m()\u001B[0m\n\u001B[0;32m     78\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mterminal\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     79\u001B[0m                 \u001B[0mprint\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"Run: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", exploration: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexploration_rate\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", score: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 80\u001B[1;33m                 \u001B[0mscore_logger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0madd_score\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     81\u001B[0m                 \u001B[1;32mbreak\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     82\u001B[0m             \u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexperience_replay\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mU:\\Jupyter\\Cartpole\\scores\\score_logger.py\u001B[0m in \u001B[0;36madd_score\u001B[1;34m(self, score, run)\u001B[0m\n\u001B[0;32m     52\u001B[0m             \u001B[1;31m#               show_trend=False,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     53\u001B[0m             \u001B[1;31m#               show_legend=False)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 54\u001B[1;33m             \u001B[0mexit\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     55\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     56\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_save_png\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0minput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0moutput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mx_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maverage_of_n_last\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_goal\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_trend\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_legend\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'exit' is not defined"
     ]
    }
   ],
   "source": [
    "cartpole()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Modified cartpole: increased gamma to 0.995, increased exploration min to 0.5, increased learning rate to 0.01\n",
    "\n",
    "import random  \n",
    "import gym  \n",
    "import numpy as np  \n",
    "from collections import deque  \n",
    "from keras.models import Sequential  \n",
    "from keras.layers import Dense  \n",
    "from keras.optimizers import Adam  \n",
    "from scores.score_logger import ScoreLogger\n",
    "\n",
    "\n",
    "ENV_NAME = \"CartPole-v1\"\n",
    "\n",
    "GAMMA = 0.995\n",
    "LEARNING_RATE = 0.01\n",
    "\n",
    "MEMORY_SIZE = 1000000\n",
    "BATCH_SIZE = 20\n",
    "\n",
    "EXPLORATION_MAX = 1.0\n",
    "EXPLORATION_MIN = 0.5\n",
    "EXPLORATION_DECAY = 0.995\n",
    "\n",
    "\n",
    "class DQNSolver:\n",
    "\n",
    "    def __init__(self, observation_space, action_space):\n",
    "        self.exploration_rate = EXPLORATION_MAX\n",
    "\n",
    "        self.action_space = action_space\n",
    "        self.memory = deque(maxlen=MEMORY_SIZE)\n",
    "\n",
    "        self.model = Sequential()\n",
    "        self.model.add(Dense(24, input_shape=(observation_space,), activation=\"relu\"))\n",
    "        self.model.add(Dense(24, activation=\"relu\"))\n",
    "        self.model.add(Dense(self.action_space, activation=\"linear\"))\n",
    "        self.model.compile(loss=\"mse\", optimizer=Adam(lr=LEARNING_RATE))\n",
    "\n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "\n",
    "    def act(self, state):\n",
    "        if np.random.rand() < self.exploration_rate:\n",
    "            return random.randrange(self.action_space)\n",
    "        q_values = self.model.predict(state)\n",
    "        return np.argmax(q_values[0])\n",
    "\n",
    "    def experience_replay(self):\n",
    "        if len(self.memory) < BATCH_SIZE:\n",
    "            return\n",
    "        batch = random.sample(self.memory, BATCH_SIZE)\n",
    "        for state, action, reward, state_next, terminal in batch:\n",
    "            q_update = reward\n",
    "            if not terminal:\n",
    "                q_update = (reward + GAMMA * np.amax(self.model.predict(state_next)[0]))\n",
    "            q_values = self.model.predict(state)\n",
    "            q_values[0][action] = q_update\n",
    "            self.model.fit(state, q_values, verbose=0)\n",
    "        self.exploration_rate *= EXPLORATION_DECAY\n",
    "        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)\n",
    "\n",
    "\n",
    "def cartpole():\n",
    "    env = gym.make(ENV_NAME)\n",
    "    score_logger = ScoreLogger(ENV_NAME)\n",
    "    observation_space = env.observation_space.shape[0]\n",
    "    action_space = env.action_space.n\n",
    "    dqn_solver = DQNSolver(observation_space, action_space)\n",
    "    run = 0\n",
    "    while True:\n",
    "        run += 1\n",
    "        state = env.reset()\n",
    "        state = np.reshape(state, [1, observation_space])\n",
    "        step = 0\n",
    "        while True:\n",
    "            step += 1\n",
    "            #env.render()\n",
    "            action = dqn_solver.act(state)\n",
    "            state_next, reward, terminal, info = env.step(action)\n",
    "            reward = reward if not terminal else -reward\n",
    "            state_next = np.reshape(state_next, [1, observation_space])\n",
    "            dqn_solver.remember(state, action, reward, state_next, terminal)\n",
    "            state = state_next\n",
    "            if terminal:\n",
    "                print (\"Run: \" + str(run) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(step))\n",
    "                score_logger.add_score(step, run)\n",
    "                break\n",
    "            dqn_solver.experience_replay()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 1, exploration: 1.0, score: 18\n",
      "Scores: (min: 18, avg: 18, max: 18)\n",
      "\n",
      "Run: 2, exploration: 0.8734200960253871, score: 29\n",
      "Scores: (min: 18, avg: 23.5, max: 29)\n",
      "\n",
      "Run: 3, exploration: 0.8224322824348486, score: 13\n",
      "Scores: (min: 13, avg: 20, max: 29)\n",
      "\n",
      "Run: 4, exploration: 0.7292124703704616, score: 25\n",
      "Scores: (min: 13, avg: 21.25, max: 29)\n",
      "\n",
      "Run: 5, exploration: 0.697046600835495, score: 10\n",
      "Scores: (min: 10, avg: 19, max: 29)\n",
      "\n",
      "Run: 6, exploration: 0.6696478204705644, score: 9\n",
      "Scores: (min: 9, avg: 17.333333333333332, max: 29)\n",
      "\n",
      "Run: 7, exploration: 0.6305556603555866, score: 13\n",
      "Scores: (min: 9, avg: 16.714285714285715, max: 29)\n",
      "\n",
      "Run: 8, exploration: 0.5997278763867329, score: 11\n",
      "Scores: (min: 9, avg: 16, max: 29)\n",
      "\n",
      "Run: 9, exploration: 0.5704072587541458, score: 11\n",
      "Scores: (min: 9, avg: 15.444444444444445, max: 29)\n",
      "\n",
      "Run: 10, exploration: 0.5371084840724134, score: 13\n",
      "Scores: (min: 9, avg: 15.2, max: 29)\n",
      "\n",
      "Run: 11, exploration: 0.5, score: 22\n",
      "Scores: (min: 9, avg: 15.818181818181818, max: 29)\n",
      "\n",
      "Run: 12, exploration: 0.5, score: 10\n",
      "Scores: (min: 9, avg: 15.333333333333334, max: 29)\n",
      "\n",
      "Run: 13, exploration: 0.5, score: 11\n",
      "Scores: (min: 9, avg: 15, max: 29)\n",
      "\n",
      "Run: 14, exploration: 0.5, score: 8\n",
      "Scores: (min: 8, avg: 14.5, max: 29)\n",
      "\n",
      "Run: 15, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.4, max: 29)\n",
      "\n",
      "Run: 16, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.1875, max: 29)\n",
      "\n",
      "Run: 17, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.058823529411764, max: 29)\n",
      "\n",
      "Run: 18, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14, max: 29)\n",
      "\n",
      "Run: 19, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 13.842105263157896, max: 29)\n",
      "\n",
      "Run: 20, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 13.85, max: 29)\n",
      "\n",
      "Run: 21, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 13.857142857142858, max: 29)\n",
      "\n",
      "Run: 22, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 13.772727272727273, max: 29)\n",
      "\n",
      "Run: 23, exploration: 0.5, score: 26\n",
      "Scores: (min: 8, avg: 14.304347826086957, max: 29)\n",
      "\n",
      "Run: 24, exploration: 0.5, score: 17\n",
      "Scores: (min: 8, avg: 14.416666666666666, max: 29)\n",
      "\n",
      "Run: 25, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 14.4, max: 29)\n",
      "\n",
      "Run: 26, exploration: 0.5, score: 20\n",
      "Scores: (min: 8, avg: 14.615384615384615, max: 29)\n",
      "\n",
      "Run: 27, exploration: 0.5, score: 9\n",
      "Scores: (min: 8, avg: 14.407407407407407, max: 29)\n",
      "\n",
      "Run: 28, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.285714285714286, max: 29)\n",
      "\n",
      "Run: 29, exploration: 0.5, score: 21\n",
      "Scores: (min: 8, avg: 14.517241379310345, max: 29)\n",
      "\n",
      "Run: 30, exploration: 0.5, score: 9\n",
      "Scores: (min: 8, avg: 14.333333333333334, max: 29)\n",
      "\n",
      "Run: 31, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.258064516129032, max: 29)\n",
      "\n",
      "Run: 32, exploration: 0.5, score: 15\n",
      "Scores: (min: 8, avg: 14.28125, max: 29)\n",
      "\n",
      "Run: 33, exploration: 0.5, score: 27\n",
      "Scores: (min: 8, avg: 14.666666666666666, max: 29)\n",
      "\n",
      "Run: 34, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.558823529411764, max: 29)\n",
      "\n",
      "Run: 35, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.428571428571429, max: 29)\n",
      "\n",
      "Run: 36, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.38888888888889, max: 29)\n",
      "\n",
      "Run: 37, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.35135135135135, max: 29)\n",
      "\n",
      "Run: 38, exploration: 0.5, score: 16\n",
      "Scores: (min: 8, avg: 14.394736842105264, max: 29)\n",
      "\n",
      "Run: 39, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.35897435897436, max: 29)\n",
      "\n",
      "Run: 40, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.275, max: 29)\n",
      "\n",
      "Run: 41, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.170731707317072, max: 29)\n",
      "\n",
      "Run: 42, exploration: 0.5, score: 24\n",
      "Scores: (min: 8, avg: 14.404761904761905, max: 29)\n",
      "\n",
      "Run: 43, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.30232558139535, max: 29)\n",
      "\n",
      "Run: 44, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.204545454545455, max: 29)\n",
      "\n",
      "Run: 45, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.177777777777777, max: 29)\n",
      "\n",
      "Run: 46, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.108695652173912, max: 29)\n",
      "\n",
      "Run: 47, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.042553191489361, max: 29)\n",
      "\n",
      "Run: 48, exploration: 0.5, score: 16\n",
      "Scores: (min: 8, avg: 14.083333333333334, max: 29)\n",
      "\n",
      "Run: 49, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 14.081632653061224, max: 29)\n",
      "\n",
      "Run: 50, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 14.08, max: 29)\n",
      "\n",
      "Run: 51, exploration: 0.5, score: 20\n",
      "Scores: (min: 8, avg: 14.196078431372548, max: 29)\n",
      "\n",
      "Run: 52, exploration: 0.5, score: 30\n",
      "Scores: (min: 8, avg: 14.5, max: 30)\n",
      "\n",
      "Run: 53, exploration: 0.5, score: 47\n",
      "Scores: (min: 8, avg: 15.11320754716981, max: 47)\n",
      "\n",
      "Run: 54, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 15.074074074074074, max: 47)\n",
      "\n",
      "Run: 55, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 15, max: 47)\n",
      "\n",
      "Run: 56, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.946428571428571, max: 47)\n",
      "\n",
      "Run: 57, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.87719298245614, max: 47)\n",
      "\n",
      "Run: 58, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.810344827586206, max: 47)\n",
      "\n",
      "Run: 59, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.728813559322035, max: 47)\n",
      "\n",
      "Run: 60, exploration: 0.5, score: 16\n",
      "Scores: (min: 8, avg: 14.75, max: 47)\n",
      "\n",
      "Run: 61, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.672131147540984, max: 47)\n",
      "\n",
      "Run: 62, exploration: 0.5, score: 9\n",
      "Scores: (min: 8, avg: 14.580645161290322, max: 47)\n",
      "\n",
      "Run: 63, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.53968253968254, max: 47)\n",
      "\n",
      "Run: 64, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.46875, max: 47)\n",
      "\n",
      "Run: 65, exploration: 0.5, score: 16\n",
      "Scores: (min: 8, avg: 14.492307692307692, max: 47)\n",
      "\n",
      "Run: 66, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.424242424242424, max: 47)\n",
      "\n",
      "Run: 67, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.388059701492537, max: 47)\n",
      "\n",
      "Run: 68, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.338235294117647, max: 47)\n",
      "\n",
      "Run: 69, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.318840579710145, max: 47)\n",
      "\n",
      "Run: 70, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 14.314285714285715, max: 47)\n",
      "\n",
      "Run: 71, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.295774647887324, max: 47)\n",
      "\n",
      "Run: 72, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.277777777777779, max: 47)\n",
      "\n",
      "Run: 73, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.232876712328768, max: 47)\n",
      "\n",
      "Run: 74, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.18918918918919, max: 47)\n",
      "\n",
      "Run: 75, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.173333333333334, max: 47)\n",
      "\n",
      "Run: 76, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.144736842105264, max: 47)\n",
      "\n",
      "Run: 77, exploration: 0.5, score: 24\n",
      "Scores: (min: 8, avg: 14.272727272727273, max: 47)\n",
      "\n",
      "Run: 78, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.243589743589743, max: 47)\n",
      "\n",
      "Run: 79, exploration: 0.5, score: 15\n",
      "Scores: (min: 8, avg: 14.253164556962025, max: 47)\n",
      "\n",
      "Run: 80, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.2, max: 47)\n",
      "\n",
      "Run: 81, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.185185185185185, max: 47)\n",
      "\n",
      "Run: 82, exploration: 0.5, score: 20\n",
      "Scores: (min: 8, avg: 14.25609756097561, max: 47)\n",
      "\n",
      "Run: 83, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.240963855421686, max: 47)\n",
      "\n",
      "Run: 84, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.214285714285714, max: 47)\n",
      "\n",
      "Run: 85, exploration: 0.5, score: 26\n",
      "Scores: (min: 8, avg: 14.352941176470589, max: 47)\n",
      "\n",
      "Run: 86, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 14.337209302325581, max: 47)\n",
      "\n",
      "Run: 87, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 14.298850574712644, max: 47)\n",
      "\n",
      "Run: 88, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.25, max: 47)\n",
      "\n",
      "Run: 89, exploration: 0.5, score: 12\n",
      "Scores: (min: 8, avg: 14.224719101123595, max: 47)\n",
      "\n",
      "Run: 90, exploration: 0.5, score: 25\n",
      "Scores: (min: 8, avg: 14.344444444444445, max: 47)\n",
      "\n",
      "Run: 91, exploration: 0.5, score: 20\n",
      "Scores: (min: 8, avg: 14.406593406593407, max: 47)\n",
      "\n",
      "Run: 92, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.358695652173912, max: 47)\n",
      "\n",
      "Run: 93, exploration: 0.5, score: 23\n",
      "Scores: (min: 8, avg: 14.451612903225806, max: 47)\n",
      "\n",
      "Run: 94, exploration: 0.5, score: 10\n",
      "Scores: (min: 8, avg: 14.404255319148936, max: 47)\n",
      "\n",
      "Run: 95, exploration: 0.5, score: 21\n",
      "Scores: (min: 8, avg: 14.473684210526315, max: 47)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 96, exploration: 0.5, score: 25\n",
      "Scores: (min: 8, avg: 14.583333333333334, max: 47)\n",
      "\n",
      "Run: 97, exploration: 0.5, score: 17\n",
      "Scores: (min: 8, avg: 14.608247422680412, max: 47)\n",
      "\n",
      "Run: 98, exploration: 0.5, score: 49\n",
      "Scores: (min: 8, avg: 14.959183673469388, max: 49)\n",
      "\n",
      "Run: 99, exploration: 0.5, score: 33\n",
      "Scores: (min: 8, avg: 15.141414141414142, max: 49)\n",
      "\n",
      "Run: 100, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 15.12, max: 49)\n",
      "\n",
      "Run: 101, exploration: 0.5, score: 38\n",
      "Scores: (min: 8, avg: 15.32, max: 49)\n",
      "\n",
      "Run: 102, exploration: 0.5, score: 14\n",
      "Scores: (min: 8, avg: 15.17, max: 49)\n",
      "\n",
      "Run: 103, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 15.15, max: 49)\n",
      "\n",
      "Run: 104, exploration: 0.5, score: 21\n",
      "Scores: (min: 8, avg: 15.11, max: 49)\n",
      "\n",
      "Run: 105, exploration: 0.5, score: 15\n",
      "Scores: (min: 8, avg: 15.16, max: 49)\n",
      "\n",
      "Run: 106, exploration: 0.5, score: 46\n",
      "Scores: (min: 8, avg: 15.53, max: 49)\n",
      "\n",
      "Run: 107, exploration: 0.5, score: 19\n",
      "Scores: (min: 8, avg: 15.59, max: 49)\n",
      "\n",
      "Run: 108, exploration: 0.5, score: 13\n",
      "Scores: (min: 8, avg: 15.61, max: 49)\n",
      "\n",
      "Run: 109, exploration: 0.5, score: 27\n",
      "Scores: (min: 8, avg: 15.77, max: 49)\n",
      "\n",
      "Run: 110, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 15.75, max: 49)\n",
      "\n",
      "Run: 111, exploration: 0.5, score: 24\n",
      "Scores: (min: 8, avg: 15.77, max: 49)\n",
      "\n",
      "Run: 112, exploration: 0.5, score: 9\n",
      "Scores: (min: 8, avg: 15.76, max: 49)\n",
      "\n",
      "Run: 113, exploration: 0.5, score: 11\n",
      "Scores: (min: 8, avg: 15.76, max: 49)\n",
      "\n",
      "Run: 114, exploration: 0.5, score: 13\n",
      "Scores: (min: 9, avg: 15.81, max: 49)\n",
      "\n",
      "Run: 115, exploration: 0.5, score: 19\n",
      "Scores: (min: 9, avg: 15.87, max: 49)\n",
      "\n",
      "Run: 116, exploration: 0.5, score: 11\n",
      "Scores: (min: 9, avg: 15.87, max: 49)\n",
      "\n",
      "Run: 117, exploration: 0.5, score: 19\n",
      "Scores: (min: 9, avg: 15.94, max: 49)\n",
      "\n",
      "Run: 118, exploration: 0.5, score: 14\n",
      "Scores: (min: 9, avg: 15.95, max: 49)\n",
      "\n",
      "Run: 119, exploration: 0.5, score: 80\n",
      "Scores: (min: 9, avg: 16.64, max: 80)\n",
      "\n",
      "Run: 120, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 16.62, max: 80)\n",
      "\n",
      "Run: 121, exploration: 0.5, score: 20\n",
      "Scores: (min: 9, avg: 16.68, max: 80)\n",
      "\n",
      "Run: 122, exploration: 0.5, score: 14\n",
      "Scores: (min: 9, avg: 16.7, max: 80)\n",
      "\n",
      "Run: 123, exploration: 0.5, score: 18\n",
      "Scores: (min: 9, avg: 16.62, max: 80)\n",
      "\n",
      "Run: 124, exploration: 0.5, score: 22\n",
      "Scores: (min: 9, avg: 16.67, max: 80)\n",
      "\n",
      "Run: 125, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 16.65, max: 80)\n",
      "\n",
      "Run: 126, exploration: 0.5, score: 9\n",
      "Scores: (min: 9, avg: 16.54, max: 80)\n",
      "\n",
      "Run: 127, exploration: 0.5, score: 18\n",
      "Scores: (min: 9, avg: 16.63, max: 80)\n",
      "\n",
      "Run: 128, exploration: 0.5, score: 11\n",
      "Scores: (min: 9, avg: 16.63, max: 80)\n",
      "\n",
      "Run: 129, exploration: 0.5, score: 9\n",
      "Scores: (min: 9, avg: 16.51, max: 80)\n",
      "\n",
      "Run: 130, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 16.54, max: 80)\n",
      "\n",
      "Run: 131, exploration: 0.5, score: 38\n",
      "Scores: (min: 9, avg: 16.8, max: 80)\n",
      "\n",
      "Run: 132, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 16.77, max: 80)\n",
      "\n",
      "Run: 133, exploration: 0.5, score: 19\n",
      "Scores: (min: 9, avg: 16.69, max: 80)\n",
      "\n",
      "Run: 134, exploration: 0.5, score: 25\n",
      "Scores: (min: 9, avg: 16.83, max: 80)\n",
      "\n",
      "Run: 135, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 16.99, max: 80)\n",
      "\n",
      "Run: 136, exploration: 0.5, score: 11\n",
      "Scores: (min: 9, avg: 16.97, max: 80)\n",
      "\n",
      "Run: 137, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 16.96, max: 80)\n",
      "\n",
      "Run: 138, exploration: 0.5, score: 9\n",
      "Scores: (min: 9, avg: 16.89, max: 80)\n",
      "\n",
      "Run: 139, exploration: 0.5, score: 46\n",
      "Scores: (min: 9, avg: 17.22, max: 80)\n",
      "\n",
      "Run: 140, exploration: 0.5, score: 13\n",
      "Scores: (min: 9, avg: 17.24, max: 80)\n",
      "\n",
      "Run: 141, exploration: 0.5, score: 16\n",
      "Scores: (min: 9, avg: 17.3, max: 80)\n",
      "\n",
      "Run: 142, exploration: 0.5, score: 45\n",
      "Scores: (min: 9, avg: 17.51, max: 80)\n",
      "\n",
      "Run: 143, exploration: 0.5, score: 22\n",
      "Scores: (min: 9, avg: 17.63, max: 80)\n",
      "\n",
      "Run: 144, exploration: 0.5, score: 15\n",
      "Scores: (min: 9, avg: 17.68, max: 80)\n",
      "\n",
      "Run: 145, exploration: 0.5, score: 20\n",
      "Scores: (min: 9, avg: 17.75, max: 80)\n",
      "\n",
      "Run: 146, exploration: 0.5, score: 27\n",
      "Scores: (min: 9, avg: 17.91, max: 80)\n",
      "\n",
      "Run: 147, exploration: 0.5, score: 22\n",
      "Scores: (min: 9, avg: 18.02, max: 80)\n",
      "\n",
      "Run: 148, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 18.09, max: 80)\n",
      "\n",
      "Run: 149, exploration: 0.5, score: 11\n",
      "Scores: (min: 9, avg: 18.06, max: 80)\n",
      "\n",
      "Run: 150, exploration: 0.5, score: 56\n",
      "Scores: (min: 9, avg: 18.48, max: 80)\n",
      "\n",
      "Run: 151, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 18.54, max: 80)\n",
      "\n",
      "Run: 152, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 18.5, max: 80)\n",
      "\n",
      "Run: 153, exploration: 0.5, score: 34\n",
      "Scores: (min: 9, avg: 18.37, max: 80)\n",
      "\n",
      "Run: 154, exploration: 0.5, score: 68\n",
      "Scores: (min: 9, avg: 18.92, max: 80)\n",
      "\n",
      "Run: 155, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 19.04, max: 80)\n",
      "\n",
      "Run: 156, exploration: 0.5, score: 46\n",
      "Scores: (min: 9, avg: 19.38, max: 80)\n",
      "\n",
      "Run: 157, exploration: 0.5, score: 36\n",
      "Scores: (min: 9, avg: 19.63, max: 80)\n",
      "\n",
      "Run: 158, exploration: 0.5, score: 73\n",
      "Scores: (min: 9, avg: 20.25, max: 80)\n",
      "\n",
      "Run: 159, exploration: 0.5, score: 61\n",
      "Scores: (min: 9, avg: 20.76, max: 80)\n",
      "\n",
      "Run: 160, exploration: 0.5, score: 47\n",
      "Scores: (min: 9, avg: 21.07, max: 80)\n",
      "\n",
      "Run: 161, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 21.2, max: 80)\n",
      "\n",
      "Run: 162, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 21.37, max: 80)\n",
      "\n",
      "Run: 163, exploration: 0.5, score: 15\n",
      "Scores: (min: 9, avg: 21.4, max: 80)\n",
      "\n",
      "Run: 164, exploration: 0.5, score: 59\n",
      "Scores: (min: 9, avg: 21.89, max: 80)\n",
      "\n",
      "Run: 165, exploration: 0.5, score: 50\n",
      "Scores: (min: 9, avg: 22.23, max: 80)\n",
      "\n",
      "Run: 166, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 22.36, max: 80)\n",
      "\n",
      "Run: 167, exploration: 0.5, score: 18\n",
      "Scores: (min: 9, avg: 22.42, max: 80)\n",
      "\n",
      "Run: 168, exploration: 0.5, score: 18\n",
      "Scores: (min: 9, avg: 22.49, max: 80)\n",
      "\n",
      "Run: 169, exploration: 0.5, score: 25\n",
      "Scores: (min: 9, avg: 22.61, max: 80)\n",
      "\n",
      "Run: 170, exploration: 0.5, score: 15\n",
      "Scores: (min: 9, avg: 22.62, max: 80)\n",
      "\n",
      "Run: 171, exploration: 0.5, score: 38\n",
      "Scores: (min: 9, avg: 22.87, max: 80)\n",
      "\n",
      "Run: 172, exploration: 0.5, score: 62\n",
      "Scores: (min: 9, avg: 23.36, max: 80)\n",
      "\n",
      "Run: 173, exploration: 0.5, score: 9\n",
      "Scores: (min: 9, avg: 23.34, max: 80)\n",
      "\n",
      "Run: 174, exploration: 0.5, score: 20\n",
      "Scores: (min: 9, avg: 23.43, max: 80)\n",
      "\n",
      "Run: 175, exploration: 0.5, score: 29\n",
      "Scores: (min: 9, avg: 23.59, max: 80)\n",
      "\n",
      "Run: 176, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 23.73, max: 80)\n",
      "\n",
      "Run: 177, exploration: 0.5, score: 61\n",
      "Scores: (min: 9, avg: 24.1, max: 80)\n",
      "\n",
      "Run: 178, exploration: 0.5, score: 18\n",
      "Scores: (min: 9, avg: 24.16, max: 80)\n",
      "\n",
      "Run: 179, exploration: 0.5, score: 10\n",
      "Scores: (min: 9, avg: 24.11, max: 80)\n",
      "\n",
      "Run: 180, exploration: 0.5, score: 20\n",
      "Scores: (min: 9, avg: 24.21, max: 80)\n",
      "\n",
      "Run: 181, exploration: 0.5, score: 70\n",
      "Scores: (min: 9, avg: 24.78, max: 80)\n",
      "\n",
      "Run: 182, exploration: 0.5, score: 62\n",
      "Scores: (min: 9, avg: 25.2, max: 80)\n",
      "\n",
      "Run: 183, exploration: 0.5, score: 71\n",
      "Scores: (min: 9, avg: 25.78, max: 80)\n",
      "\n",
      "Run: 184, exploration: 0.5, score: 59\n",
      "Scores: (min: 9, avg: 26.25, max: 80)\n",
      "\n",
      "Run: 185, exploration: 0.5, score: 69\n",
      "Scores: (min: 9, avg: 26.68, max: 80)\n",
      "\n",
      "Run: 186, exploration: 0.5, score: 56\n",
      "Scores: (min: 9, avg: 27.11, max: 80)\n",
      "\n",
      "Run: 187, exploration: 0.5, score: 67\n",
      "Scores: (min: 9, avg: 27.67, max: 80)\n",
      "\n",
      "Run: 188, exploration: 0.5, score: 33\n",
      "Scores: (min: 9, avg: 27.9, max: 80)\n",
      "\n",
      "Run: 189, exploration: 0.5, score: 14\n",
      "Scores: (min: 9, avg: 27.92, max: 80)\n",
      "\n",
      "Run: 190, exploration: 0.5, score: 39\n",
      "Scores: (min: 9, avg: 28.06, max: 80)\n",
      "\n",
      "Run: 191, exploration: 0.5, score: 87\n",
      "Scores: (min: 9, avg: 28.73, max: 87)\n",
      "\n",
      "Run: 192, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 28.86, max: 87)\n",
      "\n",
      "Run: 193, exploration: 0.5, score: 70\n",
      "Scores: (min: 9, avg: 29.33, max: 87)\n",
      "\n",
      "Run: 194, exploration: 0.5, score: 55\n",
      "Scores: (min: 9, avg: 29.78, max: 87)\n",
      "\n",
      "Run: 195, exploration: 0.5, score: 86\n",
      "Scores: (min: 9, avg: 30.43, max: 87)\n",
      "\n",
      "Run: 196, exploration: 0.5, score: 71\n",
      "Scores: (min: 9, avg: 30.89, max: 87)\n",
      "\n",
      "Run: 197, exploration: 0.5, score: 56\n",
      "Scores: (min: 9, avg: 31.28, max: 87)\n",
      "\n",
      "Run: 198, exploration: 0.5, score: 15\n",
      "Scores: (min: 9, avg: 30.94, max: 87)\n",
      "\n",
      "Run: 199, exploration: 0.5, score: 74\n",
      "Scores: (min: 9, avg: 31.35, max: 87)\n",
      "\n",
      "Run: 200, exploration: 0.5, score: 28\n",
      "Scores: (min: 9, avg: 31.5, max: 87)\n",
      "\n",
      "Run: 201, exploration: 0.5, score: 26\n",
      "Scores: (min: 9, avg: 31.38, max: 87)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 202, exploration: 0.5, score: 87\n",
      "Scores: (min: 9, avg: 32.11, max: 87)\n",
      "\n",
      "Run: 203, exploration: 0.5, score: 117\n",
      "Scores: (min: 9, avg: 33.17, max: 117)\n",
      "\n",
      "Run: 204, exploration: 0.5, score: 34\n",
      "Scores: (min: 9, avg: 33.3, max: 117)\n",
      "\n",
      "Run: 205, exploration: 0.5, score: 65\n",
      "Scores: (min: 9, avg: 33.8, max: 117)\n",
      "\n",
      "Run: 206, exploration: 0.5, score: 47\n",
      "Scores: (min: 9, avg: 33.81, max: 117)\n",
      "\n",
      "Run: 207, exploration: 0.5, score: 111\n",
      "Scores: (min: 9, avg: 34.73, max: 117)\n",
      "\n",
      "Run: 208, exploration: 0.5, score: 105\n",
      "Scores: (min: 9, avg: 35.65, max: 117)\n",
      "\n",
      "Run: 209, exploration: 0.5, score: 123\n",
      "Scores: (min: 9, avg: 36.61, max: 123)\n",
      "\n",
      "Run: 210, exploration: 0.5, score: 131\n",
      "Scores: (min: 9, avg: 37.81, max: 131)\n",
      "\n",
      "Run: 211, exploration: 0.5, score: 144\n",
      "Scores: (min: 9, avg: 39.01, max: 144)\n",
      "\n",
      "Run: 212, exploration: 0.5, score: 178\n",
      "Scores: (min: 9, avg: 40.7, max: 178)\n",
      "\n",
      "Run: 213, exploration: 0.5, score: 101\n",
      "Scores: (min: 9, avg: 41.6, max: 178)\n",
      "\n",
      "Run: 214, exploration: 0.5, score: 122\n",
      "Scores: (min: 9, avg: 42.69, max: 178)\n",
      "\n",
      "Run: 215, exploration: 0.5, score: 80\n",
      "Scores: (min: 9, avg: 43.3, max: 178)\n",
      "\n",
      "Run: 216, exploration: 0.5, score: 500\n",
      "Scores: (min: 9, avg: 48.19, max: 500)\n",
      "\n",
      "Run: 217, exploration: 0.5, score: 154\n",
      "Scores: (min: 9, avg: 49.54, max: 500)\n",
      "\n",
      "Run: 218, exploration: 0.5, score: 311\n",
      "Scores: (min: 9, avg: 52.51, max: 500)\n",
      "\n",
      "Run: 219, exploration: 0.5, score: 56\n",
      "Scores: (min: 9, avg: 52.27, max: 500)\n",
      "\n",
      "Run: 220, exploration: 0.5, score: 91\n",
      "Scores: (min: 9, avg: 53.06, max: 500)\n",
      "\n",
      "Run: 221, exploration: 0.5, score: 32\n",
      "Scores: (min: 9, avg: 53.18, max: 500)\n",
      "\n",
      "Run: 222, exploration: 0.5, score: 185\n",
      "Scores: (min: 9, avg: 54.89, max: 500)\n",
      "\n",
      "Run: 223, exploration: 0.5, score: 58\n",
      "Scores: (min: 9, avg: 55.29, max: 500)\n",
      "\n",
      "Run: 224, exploration: 0.5, score: 193\n",
      "Scores: (min: 9, avg: 57, max: 500)\n",
      "\n",
      "Run: 225, exploration: 0.5, score: 139\n",
      "Scores: (min: 9, avg: 58.27, max: 500)\n",
      "\n",
      "Run: 226, exploration: 0.5, score: 262\n",
      "Scores: (min: 9, avg: 60.8, max: 500)\n",
      "\n",
      "Run: 227, exploration: 0.5, score: 232\n",
      "Scores: (min: 9, avg: 62.94, max: 500)\n",
      "\n",
      "Run: 228, exploration: 0.5, score: 150\n",
      "Scores: (min: 9, avg: 64.33, max: 500)\n",
      "\n",
      "Run: 229, exploration: 0.5, score: 79\n",
      "Scores: (min: 9, avg: 65.03, max: 500)\n",
      "\n",
      "Run: 230, exploration: 0.5, score: 355\n",
      "Scores: (min: 9, avg: 68.46, max: 500)\n",
      "\n",
      "Run: 231, exploration: 0.5, score: 72\n",
      "Scores: (min: 9, avg: 68.8, max: 500)\n",
      "\n",
      "Run: 232, exploration: 0.5, score: 111\n",
      "Scores: (min: 9, avg: 69.79, max: 500)\n",
      "\n",
      "Run: 233, exploration: 0.5, score: 206\n",
      "Scores: (min: 9, avg: 71.66, max: 500)\n",
      "\n",
      "Run: 234, exploration: 0.5, score: 21\n",
      "Scores: (min: 9, avg: 71.62, max: 500)\n",
      "\n",
      "Run: 235, exploration: 0.5, score: 376\n",
      "Scores: (min: 9, avg: 75.12, max: 500)\n",
      "\n",
      "Run: 236, exploration: 0.5, score: 263\n",
      "Scores: (min: 9, avg: 77.64, max: 500)\n",
      "\n",
      "Run: 237, exploration: 0.5, score: 56\n",
      "Scores: (min: 9, avg: 78.08, max: 500)\n",
      "\n",
      "Run: 238, exploration: 0.5, score: 325\n",
      "Scores: (min: 9, avg: 81.24, max: 500)\n",
      "\n",
      "Run: 239, exploration: 0.5, score: 147\n",
      "Scores: (min: 9, avg: 82.25, max: 500)\n",
      "\n",
      "Run: 240, exploration: 0.5, score: 286\n",
      "Scores: (min: 9, avg: 84.98, max: 500)\n",
      "\n",
      "Run: 241, exploration: 0.5, score: 128\n",
      "Scores: (min: 9, avg: 86.1, max: 500)\n",
      "\n",
      "Run: 242, exploration: 0.5, score: 173\n",
      "Scores: (min: 9, avg: 87.38, max: 500)\n",
      "\n",
      "Run: 243, exploration: 0.5, score: 227\n",
      "Scores: (min: 9, avg: 89.43, max: 500)\n",
      "\n",
      "Run: 244, exploration: 0.5, score: 23\n",
      "Scores: (min: 9, avg: 89.51, max: 500)\n",
      "\n",
      "Run: 245, exploration: 0.5, score: 231\n",
      "Scores: (min: 9, avg: 91.62, max: 500)\n",
      "\n",
      "Run: 246, exploration: 0.5, score: 261\n",
      "Scores: (min: 9, avg: 93.96, max: 500)\n",
      "\n",
      "Run: 247, exploration: 0.5, score: 213\n",
      "Scores: (min: 9, avg: 95.87, max: 500)\n",
      "\n",
      "Run: 248, exploration: 0.5, score: 260\n",
      "Scores: (min: 9, avg: 98.24, max: 500)\n",
      "\n",
      "Run: 249, exploration: 0.5, score: 157\n",
      "Scores: (min: 9, avg: 99.7, max: 500)\n",
      "\n",
      "Run: 250, exploration: 0.5, score: 199\n",
      "Scores: (min: 9, avg: 101.13, max: 500)\n",
      "\n",
      "Run: 251, exploration: 0.5, score: 242\n",
      "Scores: (min: 9, avg: 103.29, max: 500)\n",
      "\n",
      "Run: 252, exploration: 0.5, score: 27\n",
      "Scores: (min: 9, avg: 103.3, max: 500)\n",
      "\n",
      "Run: 253, exploration: 0.5, score: 101\n",
      "Scores: (min: 9, avg: 103.97, max: 500)\n",
      "\n",
      "Run: 254, exploration: 0.5, score: 247\n",
      "Scores: (min: 9, avg: 105.76, max: 500)\n",
      "\n",
      "Run: 255, exploration: 0.5, score: 94\n",
      "Scores: (min: 9, avg: 106.47, max: 500)\n",
      "\n",
      "Run: 256, exploration: 0.5, score: 25\n",
      "Scores: (min: 9, avg: 106.26, max: 500)\n",
      "\n",
      "Run: 257, exploration: 0.5, score: 103\n",
      "Scores: (min: 9, avg: 106.93, max: 500)\n",
      "\n",
      "Run: 258, exploration: 0.5, score: 149\n",
      "Scores: (min: 9, avg: 107.69, max: 500)\n",
      "\n",
      "Run: 259, exploration: 0.5, score: 49\n",
      "Scores: (min: 9, avg: 107.57, max: 500)\n",
      "\n",
      "Run: 260, exploration: 0.5, score: 115\n",
      "Scores: (min: 9, avg: 108.25, max: 500)\n",
      "\n",
      "Run: 261, exploration: 0.5, score: 197\n",
      "Scores: (min: 9, avg: 109.99, max: 500)\n",
      "\n",
      "Run: 262, exploration: 0.5, score: 268\n",
      "Scores: (min: 9, avg: 112.41, max: 500)\n",
      "\n",
      "Run: 263, exploration: 0.5, score: 293\n",
      "Scores: (min: 9, avg: 115.19, max: 500)\n",
      "\n",
      "Run: 264, exploration: 0.5, score: 58\n",
      "Scores: (min: 9, avg: 115.18, max: 500)\n",
      "\n",
      "Run: 265, exploration: 0.5, score: 59\n",
      "Scores: (min: 9, avg: 115.27, max: 500)\n",
      "\n",
      "Run: 266, exploration: 0.5, score: 12\n",
      "Scores: (min: 9, avg: 115.16, max: 500)\n",
      "\n",
      "Run: 267, exploration: 0.5, score: 68\n",
      "Scores: (min: 9, avg: 115.66, max: 500)\n",
      "\n",
      "Run: 268, exploration: 0.5, score: 72\n",
      "Scores: (min: 9, avg: 116.2, max: 500)\n",
      "\n",
      "Run: 269, exploration: 0.5, score: 98\n",
      "Scores: (min: 9, avg: 116.93, max: 500)\n",
      "\n",
      "Run: 270, exploration: 0.5, score: 38\n",
      "Scores: (min: 9, avg: 117.16, max: 500)\n",
      "\n",
      "Run: 271, exploration: 0.5, score: 209\n",
      "Scores: (min: 9, avg: 118.87, max: 500)\n",
      "\n",
      "Run: 272, exploration: 0.5, score: 100\n",
      "Scores: (min: 9, avg: 119.25, max: 500)\n",
      "\n",
      "Run: 273, exploration: 0.5, score: 161\n",
      "Scores: (min: 10, avg: 120.77, max: 500)\n",
      "\n",
      "Run: 274, exploration: 0.5, score: 86\n",
      "Scores: (min: 10, avg: 121.43, max: 500)\n",
      "\n",
      "Run: 275, exploration: 0.5, score: 222\n",
      "Scores: (min: 10, avg: 123.36, max: 500)\n",
      "\n",
      "Run: 276, exploration: 0.5, score: 87\n",
      "Scores: (min: 10, avg: 123.97, max: 500)\n",
      "\n",
      "Run: 277, exploration: 0.5, score: 181\n",
      "Scores: (min: 10, avg: 125.17, max: 500)\n",
      "\n",
      "Run: 278, exploration: 0.5, score: 194\n",
      "Scores: (min: 10, avg: 126.93, max: 500)\n",
      "\n",
      "Run: 279, exploration: 0.5, score: 217\n",
      "Scores: (min: 12, avg: 129, max: 500)\n",
      "\n",
      "Run: 280, exploration: 0.5, score: 114\n",
      "Scores: (min: 12, avg: 129.94, max: 500)\n",
      "\n",
      "Run: 281, exploration: 0.5, score: 218\n",
      "Scores: (min: 12, avg: 131.42, max: 500)\n",
      "\n",
      "Run: 282, exploration: 0.5, score: 121\n",
      "Scores: (min: 12, avg: 132.01, max: 500)\n",
      "\n",
      "Run: 283, exploration: 0.5, score: 202\n",
      "Scores: (min: 12, avg: 133.32, max: 500)\n",
      "\n",
      "Run: 284, exploration: 0.5, score: 99\n",
      "Scores: (min: 12, avg: 133.72, max: 500)\n",
      "\n",
      "Run: 285, exploration: 0.5, score: 217\n",
      "Scores: (min: 12, avg: 135.2, max: 500)\n",
      "\n",
      "Run: 286, exploration: 0.5, score: 68\n",
      "Scores: (min: 12, avg: 135.32, max: 500)\n",
      "\n",
      "Run: 287, exploration: 0.5, score: 13\n",
      "Scores: (min: 12, avg: 134.78, max: 500)\n",
      "\n",
      "Run: 288, exploration: 0.5, score: 207\n",
      "Scores: (min: 12, avg: 136.52, max: 500)\n",
      "\n",
      "Run: 289, exploration: 0.5, score: 234\n",
      "Scores: (min: 12, avg: 138.72, max: 500)\n",
      "\n",
      "Run: 290, exploration: 0.5, score: 32\n",
      "Scores: (min: 12, avg: 138.65, max: 500)\n",
      "\n",
      "Run: 291, exploration: 0.5, score: 68\n",
      "Scores: (min: 12, avg: 138.46, max: 500)\n",
      "\n",
      "Run: 292, exploration: 0.5, score: 178\n",
      "Scores: (min: 12, avg: 140.01, max: 500)\n",
      "\n",
      "Run: 293, exploration: 0.5, score: 68\n",
      "Scores: (min: 12, avg: 139.99, max: 500)\n",
      "\n",
      "Run: 294, exploration: 0.5, score: 187\n",
      "Scores: (min: 12, avg: 141.31, max: 500)\n",
      "\n",
      "Run: 295, exploration: 0.5, score: 39\n",
      "Scores: (min: 12, avg: 140.84, max: 500)\n",
      "\n",
      "Run: 296, exploration: 0.5, score: 144\n",
      "Scores: (min: 12, avg: 141.57, max: 500)\n",
      "\n",
      "Run: 297, exploration: 0.5, score: 207\n",
      "Scores: (min: 12, avg: 143.08, max: 500)\n",
      "\n",
      "Run: 298, exploration: 0.5, score: 331\n",
      "Scores: (min: 12, avg: 146.24, max: 500)\n",
      "\n",
      "Run: 299, exploration: 0.5, score: 261\n",
      "Scores: (min: 12, avg: 148.11, max: 500)\n",
      "\n",
      "Run: 300, exploration: 0.5, score: 130\n",
      "Scores: (min: 12, avg: 149.13, max: 500)\n",
      "\n",
      "Run: 301, exploration: 0.5, score: 310\n",
      "Scores: (min: 12, avg: 151.97, max: 500)\n",
      "\n",
      "Run: 302, exploration: 0.5, score: 171\n",
      "Scores: (min: 12, avg: 152.81, max: 500)\n",
      "\n",
      "Run: 303, exploration: 0.5, score: 67\n",
      "Scores: (min: 12, avg: 152.31, max: 500)\n",
      "\n",
      "Run: 304, exploration: 0.5, score: 20\n",
      "Scores: (min: 12, avg: 152.17, max: 500)\n",
      "\n",
      "Run: 305, exploration: 0.5, score: 213\n",
      "Scores: (min: 12, avg: 153.65, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 306, exploration: 0.5, score: 35\n",
      "Scores: (min: 12, avg: 153.53, max: 500)\n",
      "\n",
      "Run: 307, exploration: 0.5, score: 75\n",
      "Scores: (min: 12, avg: 153.17, max: 500)\n",
      "\n",
      "Run: 308, exploration: 0.5, score: 81\n",
      "Scores: (min: 12, avg: 152.93, max: 500)\n",
      "\n",
      "Run: 309, exploration: 0.5, score: 258\n",
      "Scores: (min: 12, avg: 154.28, max: 500)\n",
      "\n",
      "Run: 310, exploration: 0.5, score: 245\n",
      "Scores: (min: 12, avg: 155.42, max: 500)\n",
      "\n",
      "Run: 311, exploration: 0.5, score: 54\n",
      "Scores: (min: 12, avg: 154.52, max: 500)\n",
      "\n",
      "Run: 312, exploration: 0.5, score: 103\n",
      "Scores: (min: 12, avg: 153.77, max: 500)\n",
      "\n",
      "Run: 313, exploration: 0.5, score: 224\n",
      "Scores: (min: 12, avg: 155, max: 500)\n",
      "\n",
      "Run: 314, exploration: 0.5, score: 441\n",
      "Scores: (min: 12, avg: 158.19, max: 500)\n",
      "\n",
      "Run: 315, exploration: 0.5, score: 273\n",
      "Scores: (min: 12, avg: 160.12, max: 500)\n",
      "\n",
      "Run: 316, exploration: 0.5, score: 164\n",
      "Scores: (min: 12, avg: 156.76, max: 441)\n",
      "\n",
      "Run: 317, exploration: 0.5, score: 256\n",
      "Scores: (min: 12, avg: 157.78, max: 441)\n",
      "\n",
      "Run: 318, exploration: 0.5, score: 30\n",
      "Scores: (min: 12, avg: 154.97, max: 441)\n",
      "\n",
      "Run: 319, exploration: 0.5, score: 22\n",
      "Scores: (min: 12, avg: 154.63, max: 441)\n",
      "\n",
      "Run: 320, exploration: 0.5, score: 127\n",
      "Scores: (min: 12, avg: 154.99, max: 441)\n",
      "\n",
      "Run: 321, exploration: 0.5, score: 228\n",
      "Scores: (min: 12, avg: 156.95, max: 441)\n",
      "\n",
      "Run: 322, exploration: 0.5, score: 60\n",
      "Scores: (min: 12, avg: 155.7, max: 441)\n",
      "\n",
      "Run: 323, exploration: 0.5, score: 22\n",
      "Scores: (min: 12, avg: 155.34, max: 441)\n",
      "\n",
      "Run: 324, exploration: 0.5, score: 78\n",
      "Scores: (min: 12, avg: 154.19, max: 441)\n",
      "\n",
      "Run: 325, exploration: 0.5, score: 115\n",
      "Scores: (min: 12, avg: 153.95, max: 441)\n",
      "\n",
      "Run: 326, exploration: 0.5, score: 109\n",
      "Scores: (min: 12, avg: 152.42, max: 441)\n",
      "\n",
      "Run: 327, exploration: 0.5, score: 57\n",
      "Scores: (min: 12, avg: 150.67, max: 441)\n",
      "\n",
      "Run: 328, exploration: 0.5, score: 64\n",
      "Scores: (min: 12, avg: 149.81, max: 441)\n",
      "\n",
      "Run: 329, exploration: 0.5, score: 244\n",
      "Scores: (min: 12, avg: 151.46, max: 441)\n",
      "\n",
      "Run: 330, exploration: 0.5, score: 202\n",
      "Scores: (min: 12, avg: 149.93, max: 441)\n",
      "\n",
      "Run: 331, exploration: 0.5, score: 131\n",
      "Scores: (min: 12, avg: 150.52, max: 441)\n",
      "\n",
      "Run: 332, exploration: 0.5, score: 249\n",
      "Scores: (min: 12, avg: 151.9, max: 441)\n",
      "\n",
      "Run: 333, exploration: 0.5, score: 307\n",
      "Scores: (min: 12, avg: 152.91, max: 441)\n",
      "\n",
      "Run: 334, exploration: 0.5, score: 216\n",
      "Scores: (min: 12, avg: 154.86, max: 441)\n",
      "\n",
      "Run: 335, exploration: 0.5, score: 73\n",
      "Scores: (min: 12, avg: 151.83, max: 441)\n",
      "\n",
      "Run: 336, exploration: 0.5, score: 53\n",
      "Scores: (min: 12, avg: 149.73, max: 441)\n",
      "\n",
      "Run: 337, exploration: 0.5, score: 242\n",
      "Scores: (min: 12, avg: 151.59, max: 441)\n",
      "\n",
      "Run: 338, exploration: 0.5, score: 233\n",
      "Scores: (min: 12, avg: 150.67, max: 441)\n",
      "\n",
      "Run: 339, exploration: 0.5, score: 87\n",
      "Scores: (min: 12, avg: 150.07, max: 441)\n",
      "\n",
      "Run: 340, exploration: 0.5, score: 262\n",
      "Scores: (min: 12, avg: 149.83, max: 441)\n",
      "\n",
      "Run: 341, exploration: 0.5, score: 347\n",
      "Scores: (min: 12, avg: 152.02, max: 441)\n",
      "\n",
      "Run: 342, exploration: 0.5, score: 19\n",
      "Scores: (min: 12, avg: 150.48, max: 441)\n",
      "\n",
      "Run: 343, exploration: 0.5, score: 207\n",
      "Scores: (min: 12, avg: 150.28, max: 441)\n",
      "\n",
      "Run: 344, exploration: 0.5, score: 77\n",
      "Scores: (min: 12, avg: 150.82, max: 441)\n",
      "\n",
      "Run: 345, exploration: 0.5, score: 34\n",
      "Scores: (min: 12, avg: 148.85, max: 441)\n",
      "\n",
      "Run: 346, exploration: 0.5, score: 243\n",
      "Scores: (min: 12, avg: 148.67, max: 441)\n",
      "\n",
      "Run: 347, exploration: 0.5, score: 130\n",
      "Scores: (min: 12, avg: 147.84, max: 441)\n",
      "\n",
      "Run: 348, exploration: 0.5, score: 126\n",
      "Scores: (min: 12, avg: 146.5, max: 441)\n",
      "\n",
      "Run: 349, exploration: 0.5, score: 44\n",
      "Scores: (min: 12, avg: 145.37, max: 441)\n",
      "\n",
      "Run: 350, exploration: 0.5, score: 351\n",
      "Scores: (min: 12, avg: 146.89, max: 441)\n",
      "\n",
      "Run: 351, exploration: 0.5, score: 122\n",
      "Scores: (min: 12, avg: 145.69, max: 441)\n",
      "\n",
      "Run: 352, exploration: 0.5, score: 57\n",
      "Scores: (min: 12, avg: 145.99, max: 441)\n",
      "\n",
      "Run: 353, exploration: 0.5, score: 175\n",
      "Scores: (min: 12, avg: 146.73, max: 441)\n",
      "\n",
      "Run: 354, exploration: 0.5, score: 142\n",
      "Scores: (min: 12, avg: 145.68, max: 441)\n",
      "\n",
      "Run: 355, exploration: 0.5, score: 135\n",
      "Scores: (min: 12, avg: 146.09, max: 441)\n",
      "\n",
      "Run: 356, exploration: 0.5, score: 153\n",
      "Scores: (min: 12, avg: 147.37, max: 441)\n",
      "\n",
      "Run: 357, exploration: 0.5, score: 273\n",
      "Scores: (min: 12, avg: 149.07, max: 441)\n",
      "\n",
      "Run: 358, exploration: 0.5, score: 88\n",
      "Scores: (min: 12, avg: 148.46, max: 441)\n",
      "\n",
      "Run: 359, exploration: 0.5, score: 188\n",
      "Scores: (min: 12, avg: 149.85, max: 441)\n",
      "\n",
      "Run: 360, exploration: 0.5, score: 188\n",
      "Scores: (min: 12, avg: 150.58, max: 441)\n",
      "\n",
      "Run: 361, exploration: 0.5, score: 222\n",
      "Scores: (min: 12, avg: 150.83, max: 441)\n",
      "\n",
      "Run: 362, exploration: 0.5, score: 232\n",
      "Scores: (min: 12, avg: 150.47, max: 441)\n",
      "\n",
      "Run: 363, exploration: 0.5, score: 149\n",
      "Scores: (min: 12, avg: 149.03, max: 441)\n",
      "\n",
      "Run: 364, exploration: 0.5, score: 67\n",
      "Scores: (min: 12, avg: 149.12, max: 441)\n",
      "\n",
      "Run: 365, exploration: 0.5, score: 66\n",
      "Scores: (min: 12, avg: 149.19, max: 441)\n",
      "\n",
      "Run: 366, exploration: 0.5, score: 210\n",
      "Scores: (min: 13, avg: 151.17, max: 441)\n",
      "\n",
      "Run: 367, exploration: 0.5, score: 71\n",
      "Scores: (min: 13, avg: 151.2, max: 441)\n",
      "\n",
      "Run: 368, exploration: 0.5, score: 181\n",
      "Scores: (min: 13, avg: 152.29, max: 441)\n",
      "\n",
      "Run: 369, exploration: 0.5, score: 326\n",
      "Scores: (min: 13, avg: 154.57, max: 441)\n",
      "\n",
      "Run: 370, exploration: 0.5, score: 53\n",
      "Scores: (min: 13, avg: 154.72, max: 441)\n",
      "\n",
      "Run: 371, exploration: 0.5, score: 184\n",
      "Scores: (min: 13, avg: 154.47, max: 441)\n",
      "\n",
      "Run: 372, exploration: 0.5, score: 291\n",
      "Scores: (min: 13, avg: 156.38, max: 441)\n",
      "\n",
      "Run: 373, exploration: 0.5, score: 164\n",
      "Scores: (min: 13, avg: 156.41, max: 441)\n",
      "\n",
      "Run: 374, exploration: 0.5, score: 151\n",
      "Scores: (min: 13, avg: 157.06, max: 441)\n",
      "\n",
      "Run: 375, exploration: 0.5, score: 212\n",
      "Scores: (min: 13, avg: 156.96, max: 441)\n",
      "\n",
      "Run: 376, exploration: 0.5, score: 133\n",
      "Scores: (min: 13, avg: 157.42, max: 441)\n",
      "\n",
      "Run: 377, exploration: 0.5, score: 68\n",
      "Scores: (min: 13, avg: 156.29, max: 441)\n",
      "\n",
      "Run: 378, exploration: 0.5, score: 156\n",
      "Scores: (min: 13, avg: 155.91, max: 441)\n",
      "\n",
      "Run: 379, exploration: 0.5, score: 97\n",
      "Scores: (min: 13, avg: 154.71, max: 441)\n",
      "\n",
      "Run: 380, exploration: 0.5, score: 39\n",
      "Scores: (min: 13, avg: 153.96, max: 441)\n",
      "\n",
      "Run: 381, exploration: 0.5, score: 187\n",
      "Scores: (min: 13, avg: 153.65, max: 441)\n",
      "\n",
      "Run: 382, exploration: 0.5, score: 217\n",
      "Scores: (min: 13, avg: 154.61, max: 441)\n",
      "\n",
      "Run: 383, exploration: 0.5, score: 143\n",
      "Scores: (min: 13, avg: 154.02, max: 441)\n",
      "\n",
      "Run: 384, exploration: 0.5, score: 102\n",
      "Scores: (min: 13, avg: 154.05, max: 441)\n",
      "\n",
      "Run: 385, exploration: 0.5, score: 59\n",
      "Scores: (min: 13, avg: 152.47, max: 441)\n",
      "\n",
      "Run: 386, exploration: 0.5, score: 103\n",
      "Scores: (min: 13, avg: 152.82, max: 441)\n",
      "\n",
      "Run: 387, exploration: 0.5, score: 156\n",
      "Scores: (min: 19, avg: 154.25, max: 441)\n",
      "\n",
      "Run: 388, exploration: 0.5, score: 142\n",
      "Scores: (min: 19, avg: 153.6, max: 441)\n",
      "\n",
      "Run: 389, exploration: 0.5, score: 269\n",
      "Scores: (min: 19, avg: 153.95, max: 441)\n",
      "\n",
      "Run: 390, exploration: 0.5, score: 76\n",
      "Scores: (min: 19, avg: 154.39, max: 441)\n",
      "\n",
      "Run: 391, exploration: 0.5, score: 31\n",
      "Scores: (min: 19, avg: 154.02, max: 441)\n",
      "\n",
      "Run: 392, exploration: 0.5, score: 209\n",
      "Scores: (min: 19, avg: 154.33, max: 441)\n",
      "\n",
      "Run: 393, exploration: 0.5, score: 146\n",
      "Scores: (min: 19, avg: 155.11, max: 441)\n",
      "\n",
      "Run: 394, exploration: 0.5, score: 334\n",
      "Scores: (min: 19, avg: 156.58, max: 441)\n",
      "\n",
      "Run: 395, exploration: 0.5, score: 276\n",
      "Scores: (min: 19, avg: 158.95, max: 441)\n",
      "\n",
      "Run: 396, exploration: 0.5, score: 295\n",
      "Scores: (min: 19, avg: 160.46, max: 441)\n",
      "\n",
      "Run: 397, exploration: 0.5, score: 76\n",
      "Scores: (min: 19, avg: 159.15, max: 441)\n",
      "\n",
      "Run: 398, exploration: 0.5, score: 19\n",
      "Scores: (min: 19, avg: 156.03, max: 441)\n",
      "\n",
      "Run: 399, exploration: 0.5, score: 80\n",
      "Scores: (min: 19, avg: 154.22, max: 441)\n",
      "\n",
      "Run: 400, exploration: 0.5, score: 151\n",
      "Scores: (min: 19, avg: 154.43, max: 441)\n",
      "\n",
      "Run: 401, exploration: 0.5, score: 86\n",
      "Scores: (min: 19, avg: 152.19, max: 441)\n",
      "\n",
      "Run: 402, exploration: 0.5, score: 215\n",
      "Scores: (min: 19, avg: 152.63, max: 441)\n",
      "\n",
      "Run: 403, exploration: 0.5, score: 285\n",
      "Scores: (min: 19, avg: 154.81, max: 441)\n",
      "\n",
      "Run: 404, exploration: 0.5, score: 383\n",
      "Scores: (min: 19, avg: 158.44, max: 441)\n",
      "\n",
      "Run: 405, exploration: 0.5, score: 155\n",
      "Scores: (min: 19, avg: 157.86, max: 441)\n",
      "\n",
      "Run: 406, exploration: 0.5, score: 210\n",
      "Scores: (min: 19, avg: 159.61, max: 441)\n",
      "\n",
      "Run: 407, exploration: 0.5, score: 66\n",
      "Scores: (min: 19, avg: 159.52, max: 441)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 408, exploration: 0.5, score: 305\n",
      "Scores: (min: 19, avg: 161.76, max: 441)\n",
      "\n",
      "Run: 409, exploration: 0.5, score: 244\n",
      "Scores: (min: 19, avg: 161.62, max: 441)\n",
      "\n",
      "Run: 410, exploration: 0.5, score: 28\n",
      "Scores: (min: 19, avg: 159.45, max: 441)\n",
      "\n",
      "Run: 411, exploration: 0.5, score: 384\n",
      "Scores: (min: 19, avg: 162.75, max: 441)\n",
      "\n",
      "Run: 412, exploration: 0.5, score: 318\n",
      "Scores: (min: 19, avg: 164.9, max: 441)\n",
      "\n",
      "Run: 413, exploration: 0.5, score: 51\n",
      "Scores: (min: 19, avg: 163.17, max: 441)\n",
      "\n",
      "Run: 414, exploration: 0.5, score: 15\n",
      "Scores: (min: 15, avg: 158.91, max: 384)\n",
      "\n",
      "Run: 415, exploration: 0.5, score: 137\n",
      "Scores: (min: 15, avg: 157.55, max: 384)\n",
      "\n",
      "Run: 416, exploration: 0.5, score: 19\n",
      "Scores: (min: 15, avg: 156.1, max: 384)\n",
      "\n",
      "Run: 417, exploration: 0.5, score: 350\n",
      "Scores: (min: 15, avg: 157.04, max: 384)\n",
      "\n",
      "Run: 418, exploration: 0.5, score: 238\n",
      "Scores: (min: 15, avg: 159.12, max: 384)\n",
      "\n",
      "Run: 419, exploration: 0.5, score: 251\n",
      "Scores: (min: 15, avg: 161.41, max: 384)\n",
      "\n",
      "Run: 420, exploration: 0.5, score: 394\n",
      "Scores: (min: 15, avg: 164.08, max: 394)\n",
      "\n",
      "Run: 421, exploration: 0.5, score: 290\n",
      "Scores: (min: 15, avg: 164.7, max: 394)\n",
      "\n",
      "Run: 422, exploration: 0.5, score: 205\n",
      "Scores: (min: 15, avg: 166.15, max: 394)\n",
      "\n",
      "Run: 423, exploration: 0.5, score: 170\n",
      "Scores: (min: 15, avg: 167.63, max: 394)\n",
      "\n",
      "Run: 424, exploration: 0.5, score: 158\n",
      "Scores: (min: 15, avg: 168.43, max: 394)\n",
      "\n",
      "Run: 425, exploration: 0.5, score: 28\n",
      "Scores: (min: 15, avg: 167.56, max: 394)\n",
      "\n",
      "Run: 426, exploration: 0.5, score: 35\n",
      "Scores: (min: 15, avg: 166.82, max: 394)\n",
      "\n",
      "Run: 427, exploration: 0.5, score: 112\n",
      "Scores: (min: 15, avg: 167.37, max: 394)\n",
      "\n",
      "Run: 428, exploration: 0.5, score: 70\n",
      "Scores: (min: 15, avg: 167.43, max: 394)\n",
      "\n",
      "Run: 429, exploration: 0.5, score: 86\n",
      "Scores: (min: 15, avg: 165.85, max: 394)\n",
      "\n",
      "Run: 430, exploration: 0.5, score: 211\n",
      "Scores: (min: 15, avg: 165.94, max: 394)\n",
      "\n",
      "Run: 431, exploration: 0.5, score: 454\n",
      "Scores: (min: 15, avg: 169.17, max: 454)\n",
      "\n",
      "Run: 432, exploration: 0.5, score: 83\n",
      "Scores: (min: 15, avg: 167.51, max: 454)\n",
      "\n",
      "Run: 433, exploration: 0.5, score: 257\n",
      "Scores: (min: 15, avg: 167.01, max: 454)\n",
      "\n",
      "Run: 434, exploration: 0.5, score: 20\n",
      "Scores: (min: 15, avg: 165.05, max: 454)\n",
      "\n",
      "Run: 435, exploration: 0.5, score: 23\n",
      "Scores: (min: 15, avg: 164.55, max: 454)\n",
      "\n",
      "Run: 436, exploration: 0.5, score: 48\n",
      "Scores: (min: 15, avg: 164.5, max: 454)\n",
      "\n",
      "Run: 437, exploration: 0.5, score: 420\n",
      "Scores: (min: 15, avg: 166.28, max: 454)\n",
      "\n",
      "Run: 438, exploration: 0.5, score: 242\n",
      "Scores: (min: 15, avg: 166.37, max: 454)\n",
      "\n",
      "Run: 439, exploration: 0.5, score: 205\n",
      "Scores: (min: 15, avg: 167.55, max: 454)\n",
      "\n",
      "Run: 440, exploration: 0.5, score: 321\n",
      "Scores: (min: 15, avg: 168.14, max: 454)\n",
      "\n",
      "Run: 441, exploration: 0.5, score: 134\n",
      "Scores: (min: 15, avg: 166.01, max: 454)\n",
      "\n",
      "Run: 442, exploration: 0.5, score: 257\n",
      "Scores: (min: 15, avg: 168.39, max: 454)\n",
      "\n",
      "Run: 443, exploration: 0.5, score: 373\n",
      "Scores: (min: 15, avg: 170.05, max: 454)\n",
      "\n",
      "Run: 444, exploration: 0.5, score: 100\n",
      "Scores: (min: 15, avg: 170.28, max: 454)\n",
      "\n",
      "Run: 445, exploration: 0.5, score: 31\n",
      "Scores: (min: 15, avg: 170.25, max: 454)\n",
      "\n",
      "Run: 446, exploration: 0.5, score: 22\n",
      "Scores: (min: 15, avg: 168.04, max: 454)\n",
      "\n",
      "Run: 447, exploration: 0.5, score: 42\n",
      "Scores: (min: 15, avg: 167.16, max: 454)\n",
      "\n",
      "Run: 448, exploration: 0.5, score: 243\n",
      "Scores: (min: 15, avg: 168.33, max: 454)\n",
      "\n",
      "Run: 449, exploration: 0.5, score: 130\n",
      "Scores: (min: 15, avg: 169.19, max: 454)\n",
      "\n",
      "Run: 450, exploration: 0.5, score: 25\n",
      "Scores: (min: 15, avg: 165.93, max: 454)\n",
      "\n",
      "Run: 451, exploration: 0.5, score: 137\n",
      "Scores: (min: 15, avg: 166.08, max: 454)\n",
      "\n",
      "Run: 452, exploration: 0.5, score: 172\n",
      "Scores: (min: 15, avg: 167.23, max: 454)\n",
      "\n",
      "Run: 453, exploration: 0.5, score: 342\n",
      "Scores: (min: 15, avg: 168.9, max: 454)\n",
      "\n",
      "Run: 454, exploration: 0.5, score: 103\n",
      "Scores: (min: 15, avg: 168.51, max: 454)\n",
      "\n",
      "Run: 455, exploration: 0.5, score: 16\n",
      "Scores: (min: 15, avg: 167.32, max: 454)\n",
      "\n",
      "Run: 456, exploration: 0.5, score: 103\n",
      "Scores: (min: 15, avg: 166.82, max: 454)\n",
      "\n",
      "Run: 457, exploration: 0.5, score: 61\n",
      "Scores: (min: 15, avg: 164.7, max: 454)\n",
      "\n",
      "Run: 458, exploration: 0.5, score: 35\n",
      "Scores: (min: 15, avg: 164.17, max: 454)\n",
      "\n",
      "Run: 459, exploration: 0.5, score: 64\n",
      "Scores: (min: 15, avg: 162.93, max: 454)\n",
      "\n",
      "Run: 460, exploration: 0.5, score: 21\n",
      "Scores: (min: 15, avg: 161.26, max: 454)\n",
      "\n",
      "Run: 461, exploration: 0.5, score: 212\n",
      "Scores: (min: 15, avg: 161.16, max: 454)\n",
      "\n",
      "Run: 462, exploration: 0.5, score: 27\n",
      "Scores: (min: 15, avg: 159.11, max: 454)\n",
      "\n",
      "Run: 463, exploration: 0.5, score: 97\n",
      "Scores: (min: 15, avg: 158.59, max: 454)\n",
      "\n",
      "Run: 464, exploration: 0.5, score: 108\n",
      "Scores: (min: 15, avg: 159, max: 454)\n",
      "\n",
      "Run: 465, exploration: 0.5, score: 102\n",
      "Scores: (min: 15, avg: 159.36, max: 454)\n",
      "\n",
      "Run: 466, exploration: 0.5, score: 27\n",
      "Scores: (min: 15, avg: 157.53, max: 454)\n",
      "\n",
      "Run: 467, exploration: 0.5, score: 329\n",
      "Scores: (min: 15, avg: 160.11, max: 454)\n",
      "\n",
      "Run: 468, exploration: 0.5, score: 127\n",
      "Scores: (min: 15, avg: 159.57, max: 454)\n",
      "\n",
      "Run: 469, exploration: 0.5, score: 132\n",
      "Scores: (min: 15, avg: 157.63, max: 454)\n",
      "\n",
      "Run: 470, exploration: 0.5, score: 141\n",
      "Scores: (min: 15, avg: 158.51, max: 454)\n",
      "\n",
      "Run: 471, exploration: 0.5, score: 82\n",
      "Scores: (min: 15, avg: 157.49, max: 454)\n",
      "\n",
      "Run: 472, exploration: 0.5, score: 126\n",
      "Scores: (min: 15, avg: 155.84, max: 454)\n",
      "\n",
      "Run: 473, exploration: 0.5, score: 69\n",
      "Scores: (min: 15, avg: 154.89, max: 454)\n",
      "\n",
      "Run: 474, exploration: 0.5, score: 196\n",
      "Scores: (min: 15, avg: 155.34, max: 454)\n",
      "\n",
      "Run: 475, exploration: 0.5, score: 87\n",
      "Scores: (min: 15, avg: 154.09, max: 454)\n",
      "\n",
      "Run: 476, exploration: 0.5, score: 59\n",
      "Scores: (min: 15, avg: 153.35, max: 454)\n",
      "\n",
      "Run: 477, exploration: 0.5, score: 274\n",
      "Scores: (min: 15, avg: 155.41, max: 454)\n",
      "\n",
      "Run: 478, exploration: 0.5, score: 121\n",
      "Scores: (min: 15, avg: 155.06, max: 454)\n",
      "\n",
      "Run: 479, exploration: 0.5, score: 87\n",
      "Scores: (min: 15, avg: 154.96, max: 454)\n",
      "\n",
      "Run: 480, exploration: 0.5, score: 360\n",
      "Scores: (min: 15, avg: 158.17, max: 454)\n",
      "\n",
      "Run: 481, exploration: 0.5, score: 60\n",
      "Scores: (min: 15, avg: 156.9, max: 454)\n",
      "\n",
      "Run: 482, exploration: 0.5, score: 36\n",
      "Scores: (min: 15, avg: 155.09, max: 454)\n",
      "\n",
      "Run: 483, exploration: 0.5, score: 188\n",
      "Scores: (min: 15, avg: 155.54, max: 454)\n",
      "\n",
      "Run: 484, exploration: 0.5, score: 431\n",
      "Scores: (min: 15, avg: 158.83, max: 454)\n",
      "\n",
      "Run: 485, exploration: 0.5, score: 119\n",
      "Scores: (min: 15, avg: 159.43, max: 454)\n",
      "\n",
      "Run: 486, exploration: 0.5, score: 182\n",
      "Scores: (min: 15, avg: 160.22, max: 454)\n",
      "\n",
      "Run: 487, exploration: 0.5, score: 371\n",
      "Scores: (min: 15, avg: 162.37, max: 454)\n",
      "\n",
      "Run: 488, exploration: 0.5, score: 99\n",
      "Scores: (min: 15, avg: 161.94, max: 454)\n",
      "\n",
      "Run: 489, exploration: 0.5, score: 212\n",
      "Scores: (min: 15, avg: 161.37, max: 454)\n",
      "\n",
      "Run: 490, exploration: 0.5, score: 68\n",
      "Scores: (min: 15, avg: 161.29, max: 454)\n",
      "\n",
      "Run: 491, exploration: 0.5, score: 263\n",
      "Scores: (min: 15, avg: 163.61, max: 454)\n",
      "\n",
      "Run: 492, exploration: 0.5, score: 267\n",
      "Scores: (min: 15, avg: 164.19, max: 454)\n",
      "\n",
      "Run: 493, exploration: 0.5, score: 39\n",
      "Scores: (min: 15, avg: 163.12, max: 454)\n",
      "\n",
      "Run: 494, exploration: 0.5, score: 232\n",
      "Scores: (min: 15, avg: 162.1, max: 454)\n",
      "\n",
      "Run: 495, exploration: 0.5, score: 65\n",
      "Scores: (min: 15, avg: 159.99, max: 454)\n",
      "\n",
      "Run: 496, exploration: 0.5, score: 386\n",
      "Scores: (min: 15, avg: 160.9, max: 454)\n",
      "\n",
      "Run: 497, exploration: 0.5, score: 91\n",
      "Scores: (min: 15, avg: 161.05, max: 454)\n",
      "\n",
      "Run: 498, exploration: 0.5, score: 287\n",
      "Scores: (min: 15, avg: 163.73, max: 454)\n",
      "\n",
      "Run: 499, exploration: 0.5, score: 362\n",
      "Scores: (min: 15, avg: 166.55, max: 454)\n",
      "\n",
      "Run: 500, exploration: 0.5, score: 308\n",
      "Scores: (min: 15, avg: 168.12, max: 454)\n",
      "\n",
      "Run: 501, exploration: 0.5, score: 337\n",
      "Scores: (min: 15, avg: 170.63, max: 454)\n",
      "\n",
      "Run: 502, exploration: 0.5, score: 90\n",
      "Scores: (min: 15, avg: 169.38, max: 454)\n",
      "\n",
      "Run: 503, exploration: 0.5, score: 347\n",
      "Scores: (min: 15, avg: 170, max: 454)\n",
      "\n",
      "Run: 504, exploration: 0.5, score: 124\n",
      "Scores: (min: 15, avg: 167.41, max: 454)\n",
      "\n",
      "Run: 505, exploration: 0.5, score: 277\n",
      "Scores: (min: 15, avg: 168.63, max: 454)\n",
      "\n",
      "Run: 506, exploration: 0.5, score: 112\n",
      "Scores: (min: 15, avg: 167.65, max: 454)\n",
      "\n",
      "Run: 507, exploration: 0.5, score: 428\n",
      "Scores: (min: 15, avg: 171.27, max: 454)\n",
      "\n",
      "Run: 508, exploration: 0.5, score: 63\n",
      "Scores: (min: 15, avg: 168.85, max: 454)\n",
      "\n",
      "Run: 509, exploration: 0.5, score: 47\n",
      "Scores: (min: 15, avg: 166.88, max: 454)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 510, exploration: 0.5, score: 181\n",
      "Scores: (min: 15, avg: 168.41, max: 454)\n",
      "\n",
      "Run: 511, exploration: 0.5, score: 45\n",
      "Scores: (min: 15, avg: 165.02, max: 454)\n",
      "\n",
      "Run: 512, exploration: 0.5, score: 144\n",
      "Scores: (min: 15, avg: 163.28, max: 454)\n",
      "\n",
      "Run: 513, exploration: 0.5, score: 173\n",
      "Scores: (min: 15, avg: 164.5, max: 454)\n",
      "\n",
      "Run: 514, exploration: 0.5, score: 122\n",
      "Scores: (min: 16, avg: 165.57, max: 454)\n",
      "\n",
      "Run: 515, exploration: 0.5, score: 287\n",
      "Scores: (min: 16, avg: 167.07, max: 454)\n",
      "\n",
      "Run: 516, exploration: 0.5, score: 244\n",
      "Scores: (min: 16, avg: 169.32, max: 454)\n",
      "\n",
      "Run: 517, exploration: 0.5, score: 275\n",
      "Scores: (min: 16, avg: 168.57, max: 454)\n",
      "\n",
      "Run: 518, exploration: 0.5, score: 145\n",
      "Scores: (min: 16, avg: 167.64, max: 454)\n",
      "\n",
      "Run: 519, exploration: 0.5, score: 25\n",
      "Scores: (min: 16, avg: 165.38, max: 454)\n",
      "\n",
      "Run: 520, exploration: 0.5, score: 51\n",
      "Scores: (min: 16, avg: 161.95, max: 454)\n",
      "\n",
      "Run: 521, exploration: 0.5, score: 13\n",
      "Scores: (min: 13, avg: 159.18, max: 454)\n",
      "\n",
      "Run: 522, exploration: 0.5, score: 117\n",
      "Scores: (min: 13, avg: 158.3, max: 454)\n",
      "\n",
      "Run: 523, exploration: 0.5, score: 213\n",
      "Scores: (min: 13, avg: 158.73, max: 454)\n",
      "\n",
      "Run: 524, exploration: 0.5, score: 76\n",
      "Scores: (min: 13, avg: 157.91, max: 454)\n",
      "\n",
      "Run: 525, exploration: 0.5, score: 84\n",
      "Scores: (min: 13, avg: 158.47, max: 454)\n",
      "\n",
      "Run: 526, exploration: 0.5, score: 172\n",
      "Scores: (min: 13, avg: 159.84, max: 454)\n",
      "\n",
      "Run: 527, exploration: 0.5, score: 45\n",
      "Scores: (min: 13, avg: 159.17, max: 454)\n",
      "\n",
      "Run: 528, exploration: 0.5, score: 276\n",
      "Scores: (min: 13, avg: 161.23, max: 454)\n",
      "\n",
      "Run: 529, exploration: 0.5, score: 159\n",
      "Scores: (min: 13, avg: 161.96, max: 454)\n",
      "\n",
      "Run: 530, exploration: 0.5, score: 86\n",
      "Scores: (min: 13, avg: 160.71, max: 454)\n",
      "\n",
      "Run: 531, exploration: 0.5, score: 71\n",
      "Scores: (min: 13, avg: 156.88, max: 431)\n",
      "\n",
      "Run: 532, exploration: 0.5, score: 113\n",
      "Scores: (min: 13, avg: 157.18, max: 431)\n",
      "\n",
      "Run: 533, exploration: 0.5, score: 150\n",
      "Scores: (min: 13, avg: 156.11, max: 431)\n",
      "\n",
      "Run: 534, exploration: 0.5, score: 219\n",
      "Scores: (min: 13, avg: 158.1, max: 431)\n",
      "\n",
      "Run: 535, exploration: 0.5, score: 171\n",
      "Scores: (min: 13, avg: 159.58, max: 431)\n",
      "\n",
      "Run: 536, exploration: 0.5, score: 206\n",
      "Scores: (min: 13, avg: 161.16, max: 431)\n",
      "\n",
      "Run: 537, exploration: 0.5, score: 116\n",
      "Scores: (min: 13, avg: 158.12, max: 431)\n",
      "\n",
      "Run: 538, exploration: 0.5, score: 69\n",
      "Scores: (min: 13, avg: 156.39, max: 431)\n",
      "\n",
      "Run: 539, exploration: 0.5, score: 68\n",
      "Scores: (min: 13, avg: 155.02, max: 431)\n",
      "\n",
      "Run: 540, exploration: 0.5, score: 137\n",
      "Scores: (min: 13, avg: 153.18, max: 431)\n",
      "\n",
      "Run: 541, exploration: 0.5, score: 16\n",
      "Scores: (min: 13, avg: 152, max: 431)\n",
      "\n",
      "Run: 542, exploration: 0.5, score: 127\n",
      "Scores: (min: 13, avg: 150.7, max: 431)\n",
      "\n",
      "Run: 543, exploration: 0.5, score: 139\n",
      "Scores: (min: 13, avg: 148.36, max: 431)\n",
      "\n",
      "Run: 544, exploration: 0.5, score: 188\n",
      "Scores: (min: 13, avg: 149.24, max: 431)\n",
      "\n",
      "Run: 545, exploration: 0.5, score: 96\n",
      "Scores: (min: 13, avg: 149.89, max: 431)\n",
      "\n",
      "Run: 546, exploration: 0.5, score: 198\n",
      "Scores: (min: 13, avg: 151.65, max: 431)\n",
      "\n",
      "Run: 547, exploration: 0.5, score: 145\n",
      "Scores: (min: 13, avg: 152.68, max: 431)\n",
      "\n",
      "Run: 548, exploration: 0.5, score: 173\n",
      "Scores: (min: 13, avg: 151.98, max: 431)\n",
      "\n",
      "Run: 549, exploration: 0.5, score: 37\n",
      "Scores: (min: 13, avg: 151.05, max: 431)\n",
      "\n",
      "Run: 550, exploration: 0.5, score: 270\n",
      "Scores: (min: 13, avg: 153.5, max: 431)\n",
      "\n",
      "Run: 551, exploration: 0.5, score: 52\n",
      "Scores: (min: 13, avg: 152.65, max: 431)\n",
      "\n",
      "Run: 552, exploration: 0.5, score: 358\n",
      "Scores: (min: 13, avg: 154.51, max: 431)\n",
      "\n",
      "Run: 553, exploration: 0.5, score: 272\n",
      "Scores: (min: 13, avg: 153.81, max: 431)\n",
      "\n",
      "Run: 554, exploration: 0.5, score: 88\n",
      "Scores: (min: 13, avg: 153.66, max: 431)\n",
      "\n",
      "Run: 555, exploration: 0.5, score: 17\n",
      "Scores: (min: 13, avg: 153.67, max: 431)\n",
      "\n",
      "Run: 556, exploration: 0.5, score: 373\n",
      "Scores: (min: 13, avg: 156.37, max: 431)\n",
      "\n",
      "Run: 557, exploration: 0.5, score: 78\n",
      "Scores: (min: 13, avg: 156.54, max: 431)\n",
      "\n",
      "Run: 558, exploration: 0.5, score: 119\n",
      "Scores: (min: 13, avg: 157.38, max: 431)\n",
      "\n",
      "Run: 559, exploration: 0.5, score: 82\n",
      "Scores: (min: 13, avg: 157.56, max: 431)\n",
      "\n",
      "Run: 560, exploration: 0.5, score: 74\n",
      "Scores: (min: 13, avg: 158.09, max: 431)\n",
      "\n",
      "Run: 561, exploration: 0.5, score: 98\n",
      "Scores: (min: 13, avg: 156.95, max: 431)\n",
      "\n",
      "Run: 562, exploration: 0.5, score: 117\n",
      "Scores: (min: 13, avg: 157.85, max: 431)\n",
      "\n",
      "Run: 563, exploration: 0.5, score: 439\n",
      "Scores: (min: 13, avg: 161.27, max: 439)\n",
      "\n",
      "Run: 564, exploration: 0.5, score: 399\n",
      "Scores: (min: 13, avg: 164.18, max: 439)\n",
      "\n",
      "Run: 565, exploration: 0.5, score: 294\n",
      "Scores: (min: 13, avg: 166.1, max: 439)\n",
      "\n",
      "Run: 566, exploration: 0.5, score: 85\n",
      "Scores: (min: 13, avg: 166.68, max: 439)\n",
      "\n",
      "Run: 567, exploration: 0.5, score: 26\n",
      "Scores: (min: 13, avg: 163.65, max: 439)\n",
      "\n",
      "Run: 568, exploration: 0.5, score: 49\n",
      "Scores: (min: 13, avg: 162.87, max: 439)\n",
      "\n",
      "Run: 569, exploration: 0.5, score: 41\n",
      "Scores: (min: 13, avg: 161.96, max: 439)\n",
      "\n",
      "Run: 570, exploration: 0.5, score: 16\n",
      "Scores: (min: 13, avg: 160.71, max: 439)\n",
      "\n",
      "Run: 571, exploration: 0.5, score: 44\n",
      "Scores: (min: 13, avg: 160.33, max: 439)\n",
      "\n",
      "Run: 572, exploration: 0.5, score: 17\n",
      "Scores: (min: 13, avg: 159.24, max: 439)\n",
      "\n",
      "Run: 573, exploration: 0.5, score: 387\n",
      "Scores: (min: 13, avg: 162.42, max: 439)\n",
      "\n",
      "Run: 574, exploration: 0.5, score: 255\n",
      "Scores: (min: 13, avg: 163.01, max: 439)\n",
      "\n",
      "Run: 575, exploration: 0.5, score: 421\n",
      "Scores: (min: 13, avg: 166.35, max: 439)\n",
      "\n",
      "Run: 576, exploration: 0.5, score: 225\n",
      "Scores: (min: 13, avg: 168.01, max: 439)\n",
      "\n",
      "Run: 577, exploration: 0.5, score: 206\n",
      "Scores: (min: 13, avg: 167.33, max: 439)\n",
      "\n",
      "Run: 578, exploration: 0.5, score: 43\n",
      "Scores: (min: 13, avg: 166.55, max: 439)\n",
      "\n",
      "Run: 579, exploration: 0.5, score: 397\n",
      "Scores: (min: 13, avg: 169.65, max: 439)\n",
      "\n",
      "Run: 580, exploration: 0.5, score: 359\n",
      "Scores: (min: 13, avg: 169.64, max: 439)\n",
      "\n",
      "Run: 581, exploration: 0.5, score: 396\n",
      "Scores: (min: 13, avg: 173, max: 439)\n",
      "\n",
      "Run: 582, exploration: 0.5, score: 56\n",
      "Scores: (min: 13, avg: 173.2, max: 439)\n",
      "\n",
      "Run: 583, exploration: 0.5, score: 239\n",
      "Scores: (min: 13, avg: 173.71, max: 439)\n",
      "\n",
      "Run: 584, exploration: 0.5, score: 394\n",
      "Scores: (min: 13, avg: 173.34, max: 439)\n",
      "\n",
      "Run: 585, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 177.15, max: 500)\n",
      "\n",
      "Run: 586, exploration: 0.5, score: 356\n",
      "Scores: (min: 13, avg: 178.89, max: 500)\n",
      "\n",
      "Run: 587, exploration: 0.5, score: 112\n",
      "Scores: (min: 13, avg: 176.3, max: 500)\n",
      "\n",
      "Run: 588, exploration: 0.5, score: 282\n",
      "Scores: (min: 13, avg: 178.13, max: 500)\n",
      "\n",
      "Run: 589, exploration: 0.5, score: 312\n",
      "Scores: (min: 13, avg: 179.13, max: 500)\n",
      "\n",
      "Run: 590, exploration: 0.5, score: 261\n",
      "Scores: (min: 13, avg: 181.06, max: 500)\n",
      "\n",
      "Run: 591, exploration: 0.5, score: 59\n",
      "Scores: (min: 13, avg: 179.02, max: 500)\n",
      "\n",
      "Run: 592, exploration: 0.5, score: 251\n",
      "Scores: (min: 13, avg: 178.86, max: 500)\n",
      "\n",
      "Run: 593, exploration: 0.5, score: 293\n",
      "Scores: (min: 13, avg: 181.4, max: 500)\n",
      "\n",
      "Run: 594, exploration: 0.5, score: 23\n",
      "Scores: (min: 13, avg: 179.31, max: 500)\n",
      "\n",
      "Run: 595, exploration: 0.5, score: 102\n",
      "Scores: (min: 13, avg: 179.68, max: 500)\n",
      "\n",
      "Run: 596, exploration: 0.5, score: 459\n",
      "Scores: (min: 13, avg: 180.41, max: 500)\n",
      "\n",
      "Run: 597, exploration: 0.5, score: 267\n",
      "Scores: (min: 13, avg: 182.17, max: 500)\n",
      "\n",
      "Run: 598, exploration: 0.5, score: 79\n",
      "Scores: (min: 13, avg: 180.09, max: 500)\n",
      "\n",
      "Run: 599, exploration: 0.5, score: 46\n",
      "Scores: (min: 13, avg: 176.93, max: 500)\n",
      "\n",
      "Run: 600, exploration: 0.5, score: 133\n",
      "Scores: (min: 13, avg: 175.18, max: 500)\n",
      "\n",
      "Run: 601, exploration: 0.5, score: 32\n",
      "Scores: (min: 13, avg: 172.13, max: 500)\n",
      "\n",
      "Run: 602, exploration: 0.5, score: 145\n",
      "Scores: (min: 13, avg: 172.68, max: 500)\n",
      "\n",
      "Run: 603, exploration: 0.5, score: 33\n",
      "Scores: (min: 13, avg: 169.54, max: 500)\n",
      "\n",
      "Run: 604, exploration: 0.5, score: 252\n",
      "Scores: (min: 13, avg: 170.82, max: 500)\n",
      "\n",
      "Run: 605, exploration: 0.5, score: 59\n",
      "Scores: (min: 13, avg: 168.64, max: 500)\n",
      "\n",
      "Run: 606, exploration: 0.5, score: 327\n",
      "Scores: (min: 13, avg: 170.79, max: 500)\n",
      "\n",
      "Run: 607, exploration: 0.5, score: 47\n",
      "Scores: (min: 13, avg: 166.98, max: 500)\n",
      "\n",
      "Run: 608, exploration: 0.5, score: 210\n",
      "Scores: (min: 13, avg: 168.45, max: 500)\n",
      "\n",
      "Run: 609, exploration: 0.5, score: 200\n",
      "Scores: (min: 13, avg: 169.98, max: 500)\n",
      "\n",
      "Run: 610, exploration: 0.5, score: 500\n",
      "Scores: (min: 13, avg: 173.17, max: 500)\n",
      "\n",
      "Run: 611, exploration: 0.5, score: 36\n",
      "Scores: (min: 13, avg: 173.08, max: 500)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run: 612, exploration: 0.5, score: 265\n",
      "Scores: (min: 13, avg: 174.29, max: 500)\n",
      "\n",
      "Run: 613, exploration: 0.5, score: 458\n",
      "Scores: (min: 13, avg: 177.14, max: 500)\n",
      "\n",
      "Run: 614, exploration: 0.5, score: 149\n",
      "Scores: (min: 13, avg: 177.41, max: 500)\n",
      "\n",
      "Run: 615, exploration: 0.5, score: 455\n",
      "Scores: (min: 13, avg: 179.09, max: 500)\n",
      "\n",
      "Run: 616, exploration: 0.5, score: 255\n",
      "Scores: (min: 13, avg: 179.2, max: 500)\n",
      "\n",
      "Run: 617, exploration: 0.5, score: 205\n",
      "Scores: (min: 13, avg: 178.5, max: 500)\n",
      "\n",
      "Run: 618, exploration: 0.5, score: 89\n",
      "Scores: (min: 13, avg: 177.94, max: 500)\n",
      "\n",
      "Run: 619, exploration: 0.5, score: 128\n",
      "Scores: (min: 13, avg: 178.97, max: 500)\n",
      "\n",
      "Run: 620, exploration: 0.5, score: 73\n",
      "Scores: (min: 13, avg: 179.19, max: 500)\n",
      "\n",
      "Run: 621, exploration: 0.5, score: 134\n",
      "Scores: (min: 16, avg: 180.4, max: 500)\n",
      "\n",
      "Run: 622, exploration: 0.5, score: 500\n",
      "Scores: (min: 16, avg: 184.23, max: 500)\n",
      "\n",
      "Run: 623, exploration: 0.5, score: 259\n",
      "Scores: (min: 16, avg: 184.69, max: 500)\n",
      "\n",
      "Run: 624, exploration: 0.5, score: 94\n",
      "Scores: (min: 16, avg: 184.87, max: 500)\n",
      "\n",
      "Run: 625, exploration: 0.5, score: 208\n",
      "Scores: (min: 16, avg: 186.11, max: 500)\n",
      "\n",
      "Run: 626, exploration: 0.5, score: 15\n",
      "Scores: (min: 15, avg: 184.54, max: 500)\n",
      "\n",
      "Run: 627, exploration: 0.5, score: 82\n",
      "Scores: (min: 15, avg: 184.91, max: 500)\n",
      "\n",
      "Run: 628, exploration: 0.5, score: 98\n",
      "Scores: (min: 15, avg: 183.13, max: 500)\n",
      "\n",
      "Run: 629, exploration: 0.5, score: 317\n",
      "Scores: (min: 15, avg: 184.71, max: 500)\n",
      "\n",
      "Run: 630, exploration: 0.5, score: 82\n",
      "Scores: (min: 15, avg: 184.67, max: 500)\n",
      "\n",
      "Run: 631, exploration: 0.5, score: 162\n",
      "Scores: (min: 15, avg: 185.58, max: 500)\n",
      "\n",
      "Run: 632, exploration: 0.5, score: 201\n",
      "Scores: (min: 15, avg: 186.46, max: 500)\n",
      "\n",
      "Run: 633, exploration: 0.5, score: 98\n",
      "Scores: (min: 15, avg: 185.94, max: 500)\n",
      "\n",
      "Run: 634, exploration: 0.5, score: 22\n",
      "Scores: (min: 15, avg: 183.97, max: 500)\n",
      "\n",
      "Run: 635, exploration: 0.5, score: 101\n",
      "Scores: (min: 15, avg: 183.27, max: 500)\n",
      "\n",
      "Run: 636, exploration: 0.5, score: 199\n",
      "Scores: (min: 15, avg: 183.2, max: 500)\n",
      "\n",
      "Run: 637, exploration: 0.5, score: 250\n",
      "Scores: (min: 15, avg: 184.54, max: 500)\n",
      "\n",
      "Run: 638, exploration: 0.5, score: 500\n",
      "Scores: (min: 15, avg: 188.85, max: 500)\n",
      "\n",
      "Run: 639, exploration: 0.5, score: 67\n",
      "Scores: (min: 15, avg: 188.84, max: 500)\n",
      "\n",
      "Run: 640, exploration: 0.5, score: 36\n",
      "Scores: (min: 15, avg: 187.83, max: 500)\n",
      "\n",
      "Run: 641, exploration: 0.5, score: 17\n",
      "Scores: (min: 15, avg: 187.84, max: 500)\n",
      "\n",
      "Run: 642, exploration: 0.5, score: 62\n",
      "Scores: (min: 15, avg: 187.19, max: 500)\n",
      "\n",
      "Run: 643, exploration: 0.5, score: 174\n",
      "Scores: (min: 15, avg: 187.54, max: 500)\n",
      "\n",
      "Run: 644, exploration: 0.5, score: 45\n",
      "Scores: (min: 15, avg: 186.11, max: 500)\n",
      "\n",
      "Run: 645, exploration: 0.5, score: 308\n",
      "Scores: (min: 15, avg: 188.23, max: 500)\n",
      "\n",
      "Run: 646, exploration: 0.5, score: 340\n",
      "Scores: (min: 15, avg: 189.65, max: 500)\n",
      "\n",
      "Run: 647, exploration: 0.5, score: 98\n",
      "Scores: (min: 15, avg: 189.18, max: 500)\n",
      "\n",
      "Run: 648, exploration: 0.5, score: 64\n",
      "Scores: (min: 15, avg: 188.09, max: 500)\n",
      "\n",
      "Run: 649, exploration: 0.5, score: 187\n",
      "Scores: (min: 15, avg: 189.59, max: 500)\n",
      "\n",
      "Run: 650, exploration: 0.5, score: 129\n",
      "Scores: (min: 15, avg: 188.18, max: 500)\n",
      "\n",
      "Run: 651, exploration: 0.5, score: 175\n",
      "Scores: (min: 15, avg: 189.41, max: 500)\n",
      "\n",
      "Run: 652, exploration: 0.5, score: 64\n",
      "Scores: (min: 15, avg: 186.47, max: 500)\n",
      "\n",
      "Run: 653, exploration: 0.5, score: 285\n",
      "Scores: (min: 15, avg: 186.6, max: 500)\n",
      "\n",
      "Run: 654, exploration: 0.5, score: 68\n",
      "Scores: (min: 15, avg: 186.4, max: 500)\n",
      "\n",
      "Run: 655, exploration: 0.5, score: 90\n",
      "Scores: (min: 15, avg: 187.13, max: 500)\n",
      "\n",
      "Run: 656, exploration: 0.5, score: 500\n",
      "Scores: (min: 15, avg: 188.4, max: 500)\n",
      "\n",
      "Run: 657, exploration: 0.5, score: 207\n",
      "Scores: (min: 15, avg: 189.69, max: 500)\n",
      "\n",
      "Run: 658, exploration: 0.5, score: 281\n",
      "Scores: (min: 15, avg: 191.31, max: 500)\n",
      "\n",
      "Run: 659, exploration: 0.5, score: 168\n",
      "Scores: (min: 15, avg: 192.17, max: 500)\n",
      "\n",
      "Run: 660, exploration: 0.5, score: 359\n",
      "Scores: (min: 15, avg: 195.02, max: 500)\n",
      "\n",
      "Solved in 560 runs, 660 total runs.\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'exit' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-4-e23913c48dc8>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mcartpole\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m",
      "\u001B[1;32m<ipython-input-3-f5609eb78d77>\u001B[0m in \u001B[0;36mcartpole\u001B[1;34m()\u001B[0m\n\u001B[0;32m     85\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mterminal\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     86\u001B[0m                 \u001B[0mprint\u001B[0m \u001B[1;33m(\u001B[0m\u001B[1;34m\"Run: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", exploration: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexploration_rate\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\", score: \"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 87\u001B[1;33m                 \u001B[0mscore_logger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0madd_score\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstep\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mrun\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     88\u001B[0m                 \u001B[1;32mbreak\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     89\u001B[0m             \u001B[0mdqn_solver\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mexperience_replay\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mU:\\Jupyter\\Cartpole\\scores\\score_logger.py\u001B[0m in \u001B[0;36madd_score\u001B[1;34m(self, score, run)\u001B[0m\n\u001B[0;32m     52\u001B[0m             \u001B[1;31m#               show_trend=False,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     53\u001B[0m             \u001B[1;31m#               show_legend=False)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 54\u001B[1;33m             \u001B[0mexit\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     55\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     56\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_save_png\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0minput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0moutput_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mx_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_label\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maverage_of_n_last\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_goal\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_trend\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mshow_legend\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'exit' is not defined"
     ]
    }
   ],
   "source": [
    "cartpole()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis\n",
    "\n",
    "## Explain how reinforcement learning concepts apply to the cartpole problem\n",
    "\n",
    "In reinforcement learning the goal of the agent is to navigate an unknown environment by maximizing the reward it receives for its actions.  The goal of this program is to \"create an agent that is capable of learning through trial and error and ultimately solving the cartpole problem.\" (Surma, 2021)  The various state values get represented by \"cart position, cart velocity, pole angle, and the velocity of the tip of the pole\" while the action values are \"moving left or moving right.\" (Kurban, 2022)  This particular problem uses the Deep Q-Learning (DQN) reinforcement learning algorithm. (Surma, 2021)\n",
    "\n",
    "## Analyze how experience replay is applied to the cartpole problem\n",
    "\n",
    "Experience replay utilizes previous experiences for training.  According to an article from Rita Kurban,\n",
    "\n",
    ">\"Experience replay stores the agent’s experiences in memory. Batches of experiences are randomly sampled from memory and are used to train the neural network. Such learning consists of two phases — gaining experience and updating the model. The size of the replay controls the number of experiences that are used for the network update. Memory is an array that stores the agent’s state, reward, and action, as well as whether the action finished the game and the next state.\" (Surma, 2021)\n",
    "\n",
    "Along with this, a discount factor comes into play.  The discount factor works by being \"multiplied by future rewards to dampen these rewards’ effect on the agent. It is designed to make future rewards worth less than immediate rewards.\" (Surma, 2021)\n",
    "\n",
    "## Analyze how neural networks are used in deep Q-learning\n",
    "\n",
    "With the cartpole problem a simple neural network is used to predict and update the Q values for the agent.  \"The network takes the agent’s state as an input and returns the 𝑄 values for each of the actions. The maximum 𝑄 value is selected by the agent to perform the next action.\" (Surma, 2021)  The neural network makes the Q-Learning algorithm more efficient by estimating the Q values, allowing it to scale and run faster.  By adjusting the learning rate for the algorithm and increasing it from `0.001` to `0.01`, it almost doubled the number of runs needed to reach the end state.\n",
    "\n",
    "## References\n",
    "\n",
    "Kurban, R. (2022, December 12). Deep Q Learning for the CartPole - Towards Data Science. Medium. https://towardsdatascience.com/deep-q-learning-for-the-cartpole-44d761085c2f\n",
    "\n",
    "Surma, G. (2021, October 13). Cartpole - Introduction to Reinforcement Learning (DQN - Deep Q-Learning). Medium. https://gsurma.medium.com/cartpole-introduction-to-reinforcement-learning-ed0eb5b58288\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}