Optuna_TPU_xla.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OpNRIaHgotGW"
      },
      "source": [
        "# PostreSQL"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "34fde02ShZfK"
      },
      "source": [
        "## Install PostgreSQL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DMdEGztYhN_m"
      },
      "source": [
        "!apt install postgresql postgresql-contrib &>log"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BF08QNZBhSPV",
        "outputId": "ddc14ca0-f301-4350-ca09-442f9e952e09"
      },
      "source": [
        "!service postgresql start"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " * Starting PostgreSQL 14 database server\n",
            "   ...done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6haSRCaKhzO2",
        "outputId": "8116989c-54be-4e0b-a2ce-468442f27c60"
      },
      "source": [
        "!sudo -u postgres psql -c \"CREATE USER root WITH SUPERUSER\""
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CREATE ROLE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XaE71oqSiQfY"
      },
      "source": [
        "## Set PotgreSQL connection"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lzYIwdech8O1"
      },
      "source": [
        "%load_ext sql"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vs0HHGjhh-U2"
      },
      "source": [
        "%config SqlMagic.feedback=False"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sIimeyKxiDNL"
      },
      "source": [
        "%config SqlMagic.autopandas=True"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wxvXxiw6iLA4"
      },
      "source": [
        "%sql postgresql+psycopg2://@/postgres"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mqyec9vk2-Ng"
      },
      "source": [
        "# XLA"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torch~=2.0.0 https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-2.0-cp310-cp310-linux_x86_64.whl"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "OGuMcC2QurLT",
        "outputId": "3cd8b7b2-2f4f-466d-c415-28bcb04eb60d"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torch-xla==2.0\n",
            "  Downloading https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-2.0-cp310-cp310-linux_x86_64.whl (162.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.9/162.9 MB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch~=2.0.0 in /usr/local/lib/python3.10/dist-packages (2.0.1+cu118)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch~=2.0.0) (3.12.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch~=2.0.0) (4.7.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch~=2.0.0) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch~=2.0.0) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch~=2.0.0) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch~=2.0.0) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch~=2.0.0) (3.27.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch~=2.0.0) (16.0.6)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from torch-xla==2.0) (1.4.0)\n",
            "Collecting cloud-tpu-client>=0.10.0 (from torch-xla==2.0)\n",
            "  Downloading cloud_tpu_client-0.10-py3-none-any.whl (7.4 kB)\n",
            "Collecting google-api-python-client==1.8.0 (from cloud-tpu-client>=0.10.0->torch-xla==2.0)\n",
            "  Downloading google_api_python_client-1.8.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.7/57.7 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: oauth2client in /usr/local/lib/python3.10/dist-packages (from cloud-tpu-client>=0.10.0->torch-xla==2.0) (4.1.3)\n",
            "Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (0.22.0)\n",
            "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (2.17.3)\n",
            "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (0.1.0)\n",
            "Collecting google-api-core<2dev,>=1.13.0 (from google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0)\n",
            "  Downloading google_api_core-1.34.0-py3-none-any.whl (120 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.2/120.2 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (1.16.0)\n",
            "Collecting uritemplate<4dev,>=3.0.0 (from google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0)\n",
            "  Downloading uritemplate-3.0.1-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch~=2.0.0) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch~=2.0.0) (1.3.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.10/dist-packages (from oauth2client->cloud-tpu-client>=0.10.0->torch-xla==2.0) (0.5.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.10/dist-packages (from oauth2client->cloud-tpu-client>=0.10.0->torch-xla==2.0) (0.3.0)\n",
            "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from oauth2client->cloud-tpu-client>=0.10.0->torch-xla==2.0) (4.9)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (1.60.0)\n",
            "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<4.0.0dev,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (3.20.3)\n",
            "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (2.31.0)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.4.1->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (5.3.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2<1dev,>=0.9.2->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (3.1.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0dev,>=2.18.0->google-api-core<2dev,>=1.13.0->google-api-python-client==1.8.0->cloud-tpu-client>=0.10.0->torch-xla==2.0) (2023.7.22)\n",
            "Installing collected packages: uritemplate, google-api-core, google-api-python-client, cloud-tpu-client, torch-xla\n",
            "  Attempting uninstall: uritemplate\n",
            "    Found existing installation: uritemplate 4.1.1\n",
            "    Uninstalling uritemplate-4.1.1:\n",
            "      Successfully uninstalled uritemplate-4.1.1\n",
            "  Attempting uninstall: google-api-core\n",
            "    Found existing installation: google-api-core 2.11.1\n",
            "    Uninstalling google-api-core-2.11.1:\n",
            "      Successfully uninstalled google-api-core-2.11.1\n",
            "  Attempting uninstall: google-api-python-client\n",
            "    Found existing installation: google-api-python-client 2.84.0\n",
            "    Uninstalling google-api-python-client-2.84.0:\n",
            "      Successfully uninstalled google-api-python-client-2.84.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "pydrive2 1.6.3 requires google-api-python-client>=1.12.5, but you have google-api-python-client 1.8.0 which is incompatible.\n",
            "earthengine-api 0.1.364 requires google-api-python-client>=1.12.1, but you have google-api-python-client 1.8.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed cloud-tpu-client-0.10 google-api-core-1.34.0 google-api-python-client-1.8.0 torch-xla-2.0.0.dev20230516+colab uritemplate-3.0.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "google"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V1MdKlVu3F1D"
      },
      "source": [
        "import torch_xla\n",
        "import torch_xla.core.xla_model as xm"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XcdkxCPUeX_b"
      },
      "source": [
        "# Optuna simple distributed"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GHWhB8eap0D5"
      },
      "source": [
        "## Install Optuna"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MCHzWX7MeFqG"
      },
      "source": [
        "%pip install optuna"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pA0Z1zI0p5S1"
      },
      "source": [
        "## Import Optuna"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EskBAcpNeXea"
      },
      "source": [
        "import optuna"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ceqeybeDqAOm"
      },
      "source": [
        "## Define objective function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0qiRoiogFY9"
      },
      "source": [
        "def objective(trial):\n",
        "   x = trial.suggest_uniform('x', -10, 10)\n",
        "   return (x - 2) ** 2"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sy_I61IEqMpP"
      },
      "source": [
        "## Create a study"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TG9m6xELgUVb",
        "outputId": "b90b2e19-50d5-4ca4-8381-665081457c43"
      },
      "source": [
        "study = optuna.create_study(study_name=\"distributed-example\", storage=\"postgresql+psycopg2://@/postgres\")"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:30:14,327] A new study created in RDB with name: distributed-example\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IiU2_xOgqRL1"
      },
      "source": [
        "## Define a parallel distribution function and load created study to every XLA device"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rz9g-Uqm6MbL"
      },
      "source": [
        "def para_fn(device):\n",
        "  loaded_study = optuna.load_study(study_name=\"distributed-example\", storage=\"postgresql+psycopg2://@/postgres\")\n",
        "  loaded_study.optimize(objective, n_trials=2)\n",
        "  params = loaded_study.best_params\n",
        "  print('\\rDevice {}\\t Best value is: {}'.format(device, params))"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ThXmUxvIqgHH"
      },
      "source": [
        "## Wrap a parallel distribution function into a run fuction and launch it on XLA cores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KZHpZCCf5PrB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "69a093d7-08aa-4c66-bcfb-4740cdc88bed"
      },
      "source": [
        "from joblib import Parallel, delayed\n",
        "\n",
        "def run(fold):\n",
        "    device = xm.xla_device(fold + 1)\n",
        "    para_fn(device)\n",
        "Parallel(n_jobs=8, backend=\"threading\")(delayed(run)(i) for i in range(8))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-14-8163ab530af6>:2: FutureWarning: suggest_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float instead.\n",
            "  x = trial.suggest_uniform('x', -10, 10)\n",
            "[I 2023-09-01 17:30:38,952] Trial 0 finished with value: 0.7687857186901794 and parameters: {'x': 2.8768042647536447}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:38,964] Trial 2 finished with value: 0.1466573575614657 and parameters: {'x': 1.6170413108944182}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:38,984] Trial 4 finished with value: 0.20183881431615192 and parameters: {'x': 1.550735251420555}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:38,989] Trial 5 finished with value: 7.04566794369932 and parameters: {'x': 4.654367710717436}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:38,996] Trial 1 finished with value: 59.407433691413246 and parameters: {'x': -5.707621792188123}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,037] Trial 3 finished with value: 82.87114347075187 and parameters: {'x': -7.10335891145416}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,091] Trial 7 finished with value: 100.08852456428637 and parameters: {'x': -8.00442524907285}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,108] Trial 6 finished with value: 133.93467721599123 and parameters: {'x': -9.573015044317156}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,415] Trial 10 finished with value: 27.35328333493115 and parameters: {'x': -3.2300366475705644}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,429] Trial 8 finished with value: 57.32330067075658 and parameters: {'x': -5.5712152704012174}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,446] Trial 9 finished with value: 85.54146090766729 and parameters: {'x': -7.248862681847282}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,504] Trial 13 finished with value: 16.834536957395798 and parameters: {'x': 6.102991220731017}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,516] Trial 11 finished with value: 18.206749285004793 and parameters: {'x': 6.2669367566211704}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,533] Trial 12 finished with value: 0.6197983458270507 and parameters: {'x': 2.7872727264595483}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,538] Trial 14 finished with value: 2.9346945217561844 and parameters: {'x': 3.713095012471925}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:30:39,565] Trial 15 finished with value: 2.32702931414346 and parameters: {'x': 3.5254603613806097}. Best is trial 2 with value: 0.1466573575614657.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:3\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:8\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:2\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:6\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:4\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:7\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:1\t Best value is: {'x': 1.6170413108944182}\n",
            "\rDevice xla:5\t Best value is: {'x': 1.6170413108944182}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[None, None, None, None, None, None, None, None]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oxutSrt1o5b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "outputId": "8f374029-64b8-4101-b3f5-865c502b480d"
      },
      "source": [
        "df = %sql SELECT * FROM pg_catalog.pg_tables\n",
        "df"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " * postgresql+psycopg2://@/postgres\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            schemaname                tablename tableowner tablespace  \\\n",
              "0               public             version_info       root       None   \n",
              "1               public                  studies       root       None   \n",
              "2               public         study_directions       root       None   \n",
              "3               public    study_user_attributes       root       None   \n",
              "4               public  study_system_attributes       root       None   \n",
              "..                 ...                      ...        ...        ...   \n",
              "74          pg_catalog           pg_largeobject   postgres       None   \n",
              "75  information_schema                sql_parts   postgres       None   \n",
              "76  information_schema  sql_implementation_info   postgres       None   \n",
              "77  information_schema             sql_features   postgres       None   \n",
              "78  information_schema               sql_sizing   postgres       None   \n",
              "\n",
              "    hasindexes  hasrules  hastriggers  rowsecurity  \n",
              "0         True     False        False        False  \n",
              "1         True     False         True        False  \n",
              "2         True     False         True        False  \n",
              "3         True     False         True        False  \n",
              "4         True     False         True        False  \n",
              "..         ...       ...          ...          ...  \n",
              "74        True     False        False        False  \n",
              "75       False     False        False        False  \n",
              "76       False     False        False        False  \n",
              "77       False     False        False        False  \n",
              "78       False     False        False        False  \n",
              "\n",
              "[79 rows x 8 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-db55ac9b-e5f5-48e2-94b5-1e16004250a1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>schemaname</th>\n",
              "      <th>tablename</th>\n",
              "      <th>tableowner</th>\n",
              "      <th>tablespace</th>\n",
              "      <th>hasindexes</th>\n",
              "      <th>hasrules</th>\n",
              "      <th>hastriggers</th>\n",
              "      <th>rowsecurity</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>public</td>\n",
              "      <td>version_info</td>\n",
              "      <td>root</td>\n",
              "      <td>None</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>public</td>\n",
              "      <td>studies</td>\n",
              "      <td>root</td>\n",
              "      <td>None</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>public</td>\n",
              "      <td>study_directions</td>\n",
              "      <td>root</td>\n",
              "      <td>None</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>public</td>\n",
              "      <td>study_user_attributes</td>\n",
              "      <td>root</td>\n",
              "      <td>None</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>public</td>\n",
              "      <td>study_system_attributes</td>\n",
              "      <td>root</td>\n",
              "      <td>None</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>pg_catalog</td>\n",
              "      <td>pg_largeobject</td>\n",
              "      <td>postgres</td>\n",
              "      <td>None</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>information_schema</td>\n",
              "      <td>sql_parts</td>\n",
              "      <td>postgres</td>\n",
              "      <td>None</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>information_schema</td>\n",
              "      <td>sql_implementation_info</td>\n",
              "      <td>postgres</td>\n",
              "      <td>None</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>information_schema</td>\n",
              "      <td>sql_features</td>\n",
              "      <td>postgres</td>\n",
              "      <td>None</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78</th>\n",
              "      <td>information_schema</td>\n",
              "      <td>sql_sizing</td>\n",
              "      <td>postgres</td>\n",
              "      <td>None</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>79 rows × 8 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-db55ac9b-e5f5-48e2-94b5-1e16004250a1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-db55ac9b-e5f5-48e2-94b5-1e16004250a1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-db55ac9b-e5f5-48e2-94b5-1e16004250a1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7263f78e-e82d-483f-adb5-40a603f9726f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7263f78e-e82d-483f-adb5-40a603f9726f')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7263f78e-e82d-483f-adb5-40a603f9726f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ucXz92e25W0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d39485dc-9fa4-4f9d-f8d3-52322af2833f"
      },
      "source": [
        "loaded_study = optuna.load_study(study_name=\"distributed-example\", storage=\"postgresql+psycopg2://@/postgres\")\n",
        "\n",
        "loaded_study.optimize(objective, n_trials=5)\n",
        "loaded_study.best_params"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-14-8163ab530af6>:2: FutureWarning: suggest_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float instead.\n",
            "  x = trial.suggest_uniform('x', -10, 10)\n",
            "[I 2023-09-01 17:31:04,752] Trial 16 finished with value: 51.9160097986074 and parameters: {'x': 9.205276524784278}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:04,966] Trial 17 finished with value: 3.6128943971861562 and parameters: {'x': 0.09923846914291845}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:05,045] Trial 18 finished with value: 5.206980420229954 and parameters: {'x': -0.28188089527695404}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:05,125] Trial 19 finished with value: 5.121540722752247 and parameters: {'x': -0.2630821290338199}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:05,201] Trial 20 finished with value: 1.7398269082838214 and parameters: {'x': 0.6809750160501806}. Best is trial 2 with value: 0.1466573575614657.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'x': 1.6170413108944182}"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1Xlf9zKz9VSd",
        "outputId": "6a8e8516-49e3-4f74-f52b-342e2eb9b06e"
      },
      "source": [
        "study.optimize(objective, n_trials=5)\n",
        "study.best_params"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-14-8163ab530af6>:2: FutureWarning: suggest_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float instead.\n",
            "  x = trial.suggest_uniform('x', -10, 10)\n",
            "[I 2023-09-01 17:31:08,890] Trial 21 finished with value: 0.4358065214367236 and parameters: {'x': 1.33984356290594}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:08,965] Trial 22 finished with value: 17.919815888223717 and parameters: {'x': -2.2331803514879587}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:09,041] Trial 23 finished with value: 0.47173644489352967 and parameters: {'x': 1.3131692749348427}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:09,117] Trial 24 finished with value: 18.331893517637816 and parameters: {'x': -2.281576055337312}. Best is trial 2 with value: 0.1466573575614657.\n",
            "[I 2023-09-01 17:31:09,192] Trial 25 finished with value: 0.07510307285428505 and parameters: {'x': 1.7259506014341848}. Best is trial 25 with value: 0.07510307285428505.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'x': 1.7259506014341848}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F7GMhjvyt17h"
      },
      "source": [
        "## Optuna with dataset in a parameter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D7AQBPK0t-CQ",
        "outputId": "64293343-c1e5-4591-8e0e-924f49b1e986"
      },
      "source": [
        "import os\n",
        "\n",
        "import optuna\n",
        "from optuna.trial import TrialState\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch.utils.data\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "\n",
        "DEVICE = torch.device(\"cpu\")\n",
        "BATCHSIZE = 128\n",
        "CLASSES = 10\n",
        "DIR = os.getcwd()\n",
        "EPOCHS = 10\n",
        "LOG_INTERVAL = 10\n",
        "N_TRAIN_EXAMPLES = BATCHSIZE * 30\n",
        "N_VALID_EXAMPLES = BATCHSIZE * 10\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "        datasets.FashionMNIST(DIR, train=True, download=True, transform=transforms.ToTensor()),\n",
        "        batch_size=BATCHSIZE,\n",
        "        shuffle=True,\n",
        "    )\n",
        "valid_loader = torch.utils.data.DataLoader(\n",
        "        datasets.FashionMNIST(DIR, train=False, transform=transforms.ToTensor()),\n",
        "        batch_size=BATCHSIZE,\n",
        "        shuffle=True,\n",
        "    )"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to /content/FashionMNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 26421880/26421880 [00:01<00:00, 15018323.69it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /content/FashionMNIST/raw/train-images-idx3-ubyte.gz to /content/FashionMNIST/raw\n",
            "\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to /content/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 29515/29515 [00:00<00:00, 265988.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /content/FashionMNIST/raw/train-labels-idx1-ubyte.gz to /content/FashionMNIST/raw\n",
            "\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to /content/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4422102/4422102 [00:00<00:00, 5073205.72it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /content/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to /content/FashionMNIST/raw\n",
            "\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to /content/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 5148/5148 [00:00<00:00, 10954985.79it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting /content/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to /content/FashionMNIST/raw\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vE2ZUcY8uwCm"
      },
      "source": [
        "def define_model(trial):\n",
        "    # We optimize the number of layers, hidden units and dropout ratio in each layer.\n",
        "    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n",
        "    layers = []\n",
        "\n",
        "    in_features = 28 * 28\n",
        "    for i in range(n_layers):\n",
        "        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 128)\n",
        "        layers.append(nn.Linear(in_features, out_features))\n",
        "        layers.append(nn.ReLU())\n",
        "        p = trial.suggest_float(\"dropout_l{}\".format(i), 0.2, 0.5)\n",
        "        layers.append(nn.Dropout(p))\n",
        "\n",
        "        in_features = out_features\n",
        "    layers.append(nn.Linear(in_features, CLASSES))\n",
        "    layers.append(nn.LogSoftmax(dim=1))\n",
        "\n",
        "    return nn.Sequential(*layers)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HomIA7kjuKRJ"
      },
      "source": [
        "class ObjectivePytorch(object):\n",
        "    def __init__(self, train_loader, valid_loader):\n",
        "        self.train_loader = train_loader\n",
        "        self.valid_loader = valid_loader\n",
        "\n",
        "    def __call__(self, trial):\n",
        "        # Generate the model.\n",
        "        model = define_model(trial).to(DEVICE)\n",
        "\n",
        "        # Generate the optimizers.\n",
        "        optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\", \"SGD\"])\n",
        "        lr = trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n",
        "        optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr)\n",
        "\n",
        "        # Training of the model.\n",
        "        for epoch in range(EPOCHS):\n",
        "            model.train()\n",
        "            for batch_idx, (data, target) in enumerate(train_loader):\n",
        "                # Limiting training data for faster epochs.\n",
        "                if batch_idx * BATCHSIZE >= N_TRAIN_EXAMPLES:\n",
        "                    break\n",
        "\n",
        "                data, target = data.view(data.size(0), -1).to(DEVICE), target.to(DEVICE)\n",
        "\n",
        "                optimizer.zero_grad()\n",
        "                output = model(data)\n",
        "                loss = F.nll_loss(output, target)\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "            # Validation of the model.\n",
        "            model.eval()\n",
        "            correct = 0\n",
        "            with torch.no_grad():\n",
        "                for batch_idx, (data, target) in enumerate(valid_loader):\n",
        "                    # Limiting validation data.\n",
        "                    if batch_idx * BATCHSIZE >= N_VALID_EXAMPLES:\n",
        "                        break\n",
        "                    data, target = data.view(data.size(0), -1).to(DEVICE), target.to(DEVICE)\n",
        "                    output = model(data)\n",
        "                    # Get the index of the max log-probability.\n",
        "                    pred = output.argmax(dim=1, keepdim=True)\n",
        "                    correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "\n",
        "            accuracy = correct / min(len(valid_loader.dataset), N_VALID_EXAMPLES)\n",
        "\n",
        "            trial.report(accuracy, epoch)\n",
        "\n",
        "            # Handle pruning based on the intermediate value.\n",
        "            if trial.should_prune():\n",
        "                raise optuna.exceptions.TrialPruned()\n",
        "\n",
        "        return accuracy"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9vSSmzDAvXy9",
        "outputId": "c2ff3547-f6ae-44a4-f547-02107303bf6f"
      },
      "source": [
        "objective_pytorch = ObjectivePytorch(train_loader, valid_loader)\n",
        "study = optuna.create_study(direction=\"maximize\", study_name=\"distributed-example-advanced-in-memory-dataset\", storage=\"postgresql+psycopg2://@/postgres\", load_if_exists=True)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:31:31,341] A new study created in RDB with name: distributed-example-advanced-in-memory-dataset\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_z6t87twc5N"
      },
      "source": [
        "def para_fn(device):\n",
        "  loaded_study = optuna.load_study(study_name=\"distributed-example-advanced-in-memory-dataset\", storage=\"postgresql+psycopg2://@/postgres\")\n",
        "\n",
        "  loaded_study.optimize(objective_pytorch, n_trials=2)\n",
        "\n",
        "  trial = loaded_study.best_trial\n",
        "  params = loaded_study.best_params\n",
        "  pruned_trials = loaded_study.get_trials(deepcopy=False, states=[TrialState.PRUNED])\n",
        "  complete_trials = loaded_study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])\n",
        "\n",
        "  print('\\rDevice {}\\t Best trial is: {}\\t Best params are: {}\\t Finished trials: {}\\t Pruned trials: {}\\t Complete trials: {}'\n",
        "  .format(device, trial.values[0], params, len(loaded_study.trials), pruned_trials, complete_trials))"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tJxtlaZ7w5sB",
        "outputId": "8fc2fc4b-dc36-48db-f696-86439c2a6101"
      },
      "source": [
        "from joblib import Parallel, delayed\n",
        "\n",
        "def run(fold):\n",
        "    device = xm.xla_device(fold + 1)\n",
        "    DEVICE = device\n",
        "    para_fn(device)\n",
        "\n",
        "Parallel(n_jobs=8, backend=\"threading\")(delayed(run)(i) for i in range(8))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:33:09,623] Trial 7 finished with value: 0.71015625 and parameters: {'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}. Best is trial 7 with value: 0.71015625.\n",
            "[I 2023-09-01 17:33:10,517] Trial 3 finished with value: 0.6328125 and parameters: {'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}. Best is trial 7 with value: 0.71015625.\n",
            "[I 2023-09-01 17:33:12,029] Trial 4 finished with value: 0.31875 and parameters: {'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}. Best is trial 7 with value: 0.71015625.\n",
            "[I 2023-09-01 17:33:13,381] Trial 6 finished with value: 0.2859375 and parameters: {'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}. Best is trial 7 with value: 0.71015625.\n",
            "[I 2023-09-01 17:33:14,002] Trial 1 finished with value: 0.6671875 and parameters: {'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}. Best is trial 7 with value: 0.71015625.\n",
            "[I 2023-09-01 17:33:16,140] Trial 2 finished with value: 0.7921875 and parameters: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}. Best is trial 2 with value: 0.7921875.\n",
            "[I 2023-09-01 17:33:16,949] Trial 5 finished with value: 0.6390625 and parameters: {'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}. Best is trial 2 with value: 0.7921875.\n",
            "[I 2023-09-01 17:33:22,130] Trial 0 pruned. \n",
            "[I 2023-09-01 17:33:23,640] Trial 11 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:8\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:33:26,334] Trial 14 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:3\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:33:30,342] Trial 15 pruned. \n",
            "[I 2023-09-01 17:33:30,376] Trial 10 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:5\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=10, state=TrialState.PRUNED, values=[0.3515625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 12, 600342), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 348784), params={'n_layers': 2, 'n_units_l0': 18, 'dropout_l0': 0.26104392141606003, 'n_units_l1': 89, 'dropout_l1': 0.4037778213341777, 'optimizer': 'Adam', 'lr': 0.04188946586599837}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.3265625, 1: 0.3515625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=37, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.1296875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 22, 191617), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 316390), params={'n_layers': 3, 'n_units_l0': 120, 'dropout_l0': 0.24312511617931362, 'n_units_l1': 106, 'dropout_l1': 0.2423427696033538, 'n_units_l2': 57, 'dropout_l2': 0.29666260976044695, 'optimizer': 'SGD', 'lr': 0.03245987394763857}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1296875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=42, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None)]\n",
            "\rDevice xla:2\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=10, state=TrialState.PRUNED, values=[0.3515625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 12, 600342), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 348784), params={'n_layers': 2, 'n_units_l0': 18, 'dropout_l0': 0.26104392141606003, 'n_units_l1': 89, 'dropout_l1': 0.4037778213341777, 'optimizer': 'Adam', 'lr': 0.04188946586599837}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.3265625, 1: 0.3515625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=37, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.1296875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 22, 191617), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 316390), params={'n_layers': 3, 'n_units_l0': 120, 'dropout_l0': 0.24312511617931362, 'n_units_l1': 106, 'dropout_l1': 0.2423427696033538, 'n_units_l2': 57, 'dropout_l2': 0.29666260976044695, 'optimizer': 'SGD', 'lr': 0.03245987394763857}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1296875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=42, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:34:05,183] Trial 9 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:7\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=9, state=TrialState.PRUNED, values=[0.6546875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 10, 748463), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 5, 171329), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.47955028522456417, 'n_units_l1': 108, 'dropout_l1': 0.46355242390821655, 'optimizer': 'RMSprop', 'lr': 0.00010031943778976585}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.57890625, 1: 0.59140625, 2: 0.59609375, 3: 0.634375, 4: 0.63515625, 5: 0.6109375, 6: 0.63359375, 7: 0.6546875, 8: 0.6546875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=36, value=None), FrozenTrial(number=10, state=TrialState.PRUNED, values=[0.3515625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 12, 600342), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 348784), params={'n_layers': 2, 'n_units_l0': 18, 'dropout_l0': 0.26104392141606003, 'n_units_l1': 89, 'dropout_l1': 0.4037778213341777, 'optimizer': 'Adam', 'lr': 0.04188946586599837}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.3265625, 1: 0.3515625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=37, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.1296875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 22, 191617), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 316390), params={'n_layers': 3, 'n_units_l0': 120, 'dropout_l0': 0.24312511617931362, 'n_units_l1': 106, 'dropout_l1': 0.2423427696033538, 'n_units_l2': 57, 'dropout_l2': 0.29666260976044695, 'optimizer': 'SGD', 'lr': 0.03245987394763857}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1296875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=42, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:34:08,478] Trial 8 finished with value: 0.78984375 and parameters: {'n_layers': 2, 'n_units_l0': 72, 'dropout_l0': 0.23958282532733863, 'n_units_l1': 39, 'dropout_l1': 0.23396772564314208, 'optimizer': 'RMSprop', 'lr': 0.002271963957991449}. Best is trial 2 with value: 0.7921875.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:4\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=9, state=TrialState.PRUNED, values=[0.6546875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 10, 748463), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 5, 171329), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.47955028522456417, 'n_units_l1': 108, 'dropout_l1': 0.46355242390821655, 'optimizer': 'RMSprop', 'lr': 0.00010031943778976585}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.57890625, 1: 0.59140625, 2: 0.59609375, 3: 0.634375, 4: 0.63515625, 5: 0.6109375, 6: 0.63359375, 7: 0.6546875, 8: 0.6546875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=36, value=None), FrozenTrial(number=10, state=TrialState.PRUNED, values=[0.3515625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 12, 600342), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 348784), params={'n_layers': 2, 'n_units_l0': 18, 'dropout_l0': 0.26104392141606003, 'n_units_l1': 89, 'dropout_l1': 0.4037778213341777, 'optimizer': 'Adam', 'lr': 0.04188946586599837}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.3265625, 1: 0.3515625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=37, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.1296875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 22, 191617), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 316390), params={'n_layers': 3, 'n_units_l0': 120, 'dropout_l0': 0.24312511617931362, 'n_units_l1': 106, 'dropout_l1': 0.2423427696033538, 'n_units_l2': 57, 'dropout_l2': 0.29666260976044695, 'optimizer': 'SGD', 'lr': 0.03245987394763857}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1296875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=42, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None), FrozenTrial(number=8, state=TrialState.COMPLETE, values=[0.78984375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 9, 873445), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 8, 450798), params={'n_layers': 2, 'n_units_l0': 72, 'dropout_l0': 0.23958282532733863, 'n_units_l1': 39, 'dropout_l1': 0.23396772564314208, 'optimizer': 'RMSprop', 'lr': 0.002271963957991449}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.68125, 1: 0.66796875, 2: 0.70078125, 3: 0.7484375, 4: 0.71875, 5: 0.76484375, 6: 0.81015625, 7: 0.8, 8: 0.81015625, 9: 0.78984375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=35, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:34:08,842] Trial 12 finished with value: 0.6875 and parameters: {'n_layers': 1, 'n_units_l0': 56, 'dropout_l0': 0.39023607933722204, 'optimizer': 'RMSprop', 'lr': 0.00015529071137695516}. Best is trial 2 with value: 0.7921875.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:6\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=9, state=TrialState.PRUNED, values=[0.6546875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 10, 748463), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 5, 171329), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.47955028522456417, 'n_units_l1': 108, 'dropout_l1': 0.46355242390821655, 'optimizer': 'RMSprop', 'lr': 0.00010031943778976585}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.57890625, 1: 0.59140625, 2: 0.59609375, 3: 0.634375, 4: 0.63515625, 5: 0.6109375, 6: 0.63359375, 7: 0.6546875, 8: 0.6546875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=36, value=None), FrozenTrial(number=10, state=TrialState.PRUNED, values=[0.3515625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 12, 600342), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 348784), params={'n_layers': 2, 'n_units_l0': 18, 'dropout_l0': 0.26104392141606003, 'n_units_l1': 89, 'dropout_l1': 0.4037778213341777, 'optimizer': 'Adam', 'lr': 0.04188946586599837}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.3265625, 1: 0.3515625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=37, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.1296875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 22, 191617), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 316390), params={'n_layers': 3, 'n_units_l0': 120, 'dropout_l0': 0.24312511617931362, 'n_units_l1': 106, 'dropout_l1': 0.2423427696033538, 'n_units_l2': 57, 'dropout_l2': 0.29666260976044695, 'optimizer': 'SGD', 'lr': 0.03245987394763857}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1296875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=42, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None), FrozenTrial(number=8, state=TrialState.COMPLETE, values=[0.78984375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 9, 873445), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 8, 450798), params={'n_layers': 2, 'n_units_l0': 72, 'dropout_l0': 0.23958282532733863, 'n_units_l1': 39, 'dropout_l1': 0.23396772564314208, 'optimizer': 'RMSprop', 'lr': 0.002271963957991449}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.68125, 1: 0.66796875, 2: 0.70078125, 3: 0.7484375, 4: 0.71875, 5: 0.76484375, 6: 0.81015625, 7: 0.8, 8: 0.81015625, 9: 0.78984375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=35, value=None), FrozenTrial(number=12, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 14, 51795), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 8, 818669), params={'n_layers': 1, 'n_units_l0': 56, 'dropout_l0': 0.39023607933722204, 'optimizer': 'RMSprop', 'lr': 0.00015529071137695516}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.6015625, 1: 0.63125, 2: 0.65390625, 3: 0.66484375, 4: 0.66171875, 5: 0.690625, 6: 0.678125, 7: 0.68671875, 8: 0.7015625, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=39, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:34:09,119] Trial 13 finished with value: 0.74375 and parameters: {'n_layers': 1, 'n_units_l0': 103, 'dropout_l0': 0.4657500036263731, 'optimizer': 'SGD', 'lr': 0.03477831735802452}. Best is trial 2 with value: 0.7921875.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:1\t Best trial is: 0.7921875\t Best params are: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=0, state=TrialState.PRUNED, values=[0.47109375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 380959), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 22, 100050), params={'n_layers': 3, 'n_units_l0': 128, 'dropout_l0': 0.38075337002285287, 'n_units_l1': 27, 'dropout_l1': 0.41424000584468035, 'n_units_l2': 118, 'dropout_l2': 0.4670267529469203, 'optimizer': 'Adam', 'lr': 2.3124826261365857e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1734375, 1: 0.22265625, 2: 0.2921875, 3: 0.34765625, 4: 0.4390625, 5: 0.42890625, 6: 0.4421875, 7: 0.4609375, 8: 0.50234375, 9: 0.47109375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=27, value=None), FrozenTrial(number=9, state=TrialState.PRUNED, values=[0.6546875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 10, 748463), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 5, 171329), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.47955028522456417, 'n_units_l1': 108, 'dropout_l1': 0.46355242390821655, 'optimizer': 'RMSprop', 'lr': 0.00010031943778976585}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.57890625, 1: 0.59140625, 2: 0.59609375, 3: 0.634375, 4: 0.63515625, 5: 0.6109375, 6: 0.63359375, 7: 0.6546875, 8: 0.6546875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=36, value=None), FrozenTrial(number=10, state=TrialState.PRUNED, values=[0.3515625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 12, 600342), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 348784), params={'n_layers': 2, 'n_units_l0': 18, 'dropout_l0': 0.26104392141606003, 'n_units_l1': 89, 'dropout_l1': 0.4037778213341777, 'optimizer': 'Adam', 'lr': 0.04188946586599837}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.3265625, 1: 0.3515625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=37, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.09140625], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 13, 421198), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 23, 526149), params={'n_layers': 1, 'n_units_l0': 63, 'dropout_l0': 0.3772797852947176, 'optimizer': 'SGD', 'lr': 0.003589151253802608}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09140625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=38, value=None), FrozenTrial(number=14, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 995064), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 26, 278367), params={'n_layers': 1, 'n_units_l0': 116, 'dropout_l0': 0.2675863166833314, 'optimizer': 'SGD', 'lr': 0.0013547397065666756}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=41, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.1296875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 22, 191617), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 30, 316390), params={'n_layers': 3, 'n_units_l0': 120, 'dropout_l0': 0.24312511617931362, 'n_units_l1': 106, 'dropout_l1': 0.2423427696033538, 'n_units_l2': 57, 'dropout_l2': 0.29666260976044695, 'optimizer': 'SGD', 'lr': 0.03245987394763857}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1296875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=42, value=None)]\t Complete trials: [FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.6671875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 413670), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 888055), params={'n_layers': 2, 'n_units_l0': 71, 'dropout_l0': 0.2553830080945844, 'n_units_l1': 87, 'dropout_l1': 0.29620620836724143, 'optimizer': 'SGD', 'lr': 0.05305456133070553}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2671875, 1: 0.4484375, 2: 0.56640625, 3: 0.63828125, 4: 0.6140625, 5: 0.6578125, 6: 0.63046875, 7: 0.64921875, 8: 0.65390625, 9: 0.6671875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=28, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.7921875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 440437), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 83725), params={'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.22603580095074804, 'optimizer': 'RMSprop', 'lr': 0.010101852213778628}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.67109375, 1: 0.72265625, 2: 0.759375, 3: 0.77265625, 4: 0.7578125, 5: 0.76640625, 6: 0.7734375, 7: 0.7875, 8: 0.8015625, 9: 0.7921875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=29, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.6328125], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 460554), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 10, 224575), params={'n_layers': 1, 'n_units_l0': 8, 'dropout_l0': 0.4915431091499663, 'optimizer': 'SGD', 'lr': 0.014897722552336286}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.22578125, 1: 0.37265625, 2: 0.4609375, 3: 0.46953125, 4: 0.478125, 5: 0.55234375, 6: 0.57265625, 7: 0.5765625, 8: 0.66015625, 9: 0.6328125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=30, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.31875], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 473225), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 11, 721486), params={'n_layers': 1, 'n_units_l0': 10, 'dropout_l0': 0.4809329639591433, 'optimizer': 'Adam', 'lr': 9.78315782444798e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1078125, 1: 0.10078125, 2: 0.10703125, 3: 0.16015625, 4: 0.18203125, 5: 0.2375, 6: 0.24765625, 7: 0.24921875, 8: 0.27890625, 9: 0.31875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=31, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.6390625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 510060), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 16, 861469), params={'n_layers': 1, 'n_units_l0': 101, 'dropout_l0': 0.47769041050527755, 'optimizer': 'Adam', 'lr': 3.301939082910687e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.28046875, 1: 0.33359375, 2: 0.49296875, 3: 0.52421875, 4: 0.6375, 5: 0.628125, 6: 0.63671875, 7: 0.65703125, 8: 0.6578125, 9: 0.6390625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=32, value=None), FrozenTrial(number=6, state=TrialState.COMPLETE, values=[0.2859375], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 520722), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 13, 111454), params={'n_layers': 3, 'n_units_l0': 5, 'dropout_l0': 0.227301731033748, 'n_units_l1': 14, 'dropout_l1': 0.2628317847996494, 'n_units_l2': 110, 'dropout_l2': 0.40007786012259283, 'optimizer': 'RMSprop', 'lr': 0.048350218762468304}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.1859375, 1: 0.18046875, 2: 0.275, 3: 0.24453125, 4: 0.1953125, 5: 0.24765625, 6: 0.1765625, 7: 0.2578125, 8: 0.24921875, 9: 0.2859375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=33, value=None), FrozenTrial(number=7, state=TrialState.COMPLETE, values=[0.71015625], datetime_start=datetime.datetime(2023, 9, 1, 17, 31, 38, 621699), datetime_complete=datetime.datetime(2023, 9, 1, 17, 33, 9, 329526), params={'n_layers': 1, 'n_units_l0': 19, 'dropout_l0': 0.23718902248648108, 'optimizer': 'SGD', 'lr': 0.028647724981487942}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.246875, 1: 0.59140625, 2: 0.6609375, 3: 0.64140625, 4: 0.659375, 5: 0.65859375, 6: 0.66796875, 7: 0.678125, 8: 0.67890625, 9: 0.71015625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=34, value=None), FrozenTrial(number=8, state=TrialState.COMPLETE, values=[0.78984375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 9, 873445), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 8, 450798), params={'n_layers': 2, 'n_units_l0': 72, 'dropout_l0': 0.23958282532733863, 'n_units_l1': 39, 'dropout_l1': 0.23396772564314208, 'optimizer': 'RMSprop', 'lr': 0.002271963957991449}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.68125, 1: 0.66796875, 2: 0.70078125, 3: 0.7484375, 4: 0.71875, 5: 0.76484375, 6: 0.81015625, 7: 0.8, 8: 0.81015625, 9: 0.78984375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=35, value=None), FrozenTrial(number=12, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 14, 51795), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 8, 818669), params={'n_layers': 1, 'n_units_l0': 56, 'dropout_l0': 0.39023607933722204, 'optimizer': 'RMSprop', 'lr': 0.00015529071137695516}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.6015625, 1: 0.63125, 2: 0.65390625, 3: 0.66484375, 4: 0.66171875, 5: 0.690625, 6: 0.678125, 7: 0.68671875, 8: 0.7015625, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=39, value=None), FrozenTrial(number=13, state=TrialState.COMPLETE, values=[0.74375], datetime_start=datetime.datetime(2023, 9, 1, 17, 33, 16, 174244), datetime_complete=datetime.datetime(2023, 9, 1, 17, 34, 9, 91214), params={'n_layers': 1, 'n_units_l0': 103, 'dropout_l0': 0.4657500036263731, 'optimizer': 'SGD', 'lr': 0.03477831735802452}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.575, 1: 0.6, 2: 0.6546875, 3: 0.6484375, 4: 0.628125, 5: 0.6921875, 6: 0.6953125, 7: 0.69921875, 8: 0.70703125, 9: 0.74375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=40, value=None)]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[None, None, None, None, None, None, None, None]"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wsTwoJrhFciD"
      },
      "source": [
        "# Optuna advanced\n",
        "### taken from https://github.com/optuna/optuna/blob/master/examples/pytorch/pytorch_simple.py"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ttfGmNL9FlTK"
      },
      "source": [
        "## Imports and params"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BprU_hWtqc5v"
      },
      "source": [
        "import os\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch.utils.data\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "\n",
        "import optuna\n",
        "from optuna.trial import TrialState\n",
        "\n",
        "DEVICE = torch.device(\"xla\")\n",
        "BATCHSIZE = 64\n",
        "CLASSES = 10\n",
        "DIR = os.getcwd()\n",
        "EPOCHS = 10\n",
        "LOG_INTERVAL = 10\n",
        "N_TRAIN_EXAMPLES = BATCHSIZE * 20\n",
        "N_VALID_EXAMPLES = BATCHSIZE * 5"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1msp53NnFpUc"
      },
      "source": [
        "## Layers & dropout optimization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZWn6UvU0GLAC"
      },
      "source": [
        "def define_model(trial):\n",
        "    # We optimize the number of layers, hidden units and dropout ratio in each layer.\n",
        "    n_layers = trial.suggest_int(\"n_layers\", 1, 3)\n",
        "    layers = []\n",
        "\n",
        "    in_features = 28 * 28\n",
        "    for i in range(n_layers):\n",
        "        out_features = trial.suggest_int(\"n_units_l{}\".format(i), 4, 128)\n",
        "        layers.append(nn.Linear(in_features, out_features))\n",
        "        layers.append(nn.ReLU())\n",
        "        p = trial.suggest_float(\"dropout_l{}\".format(i), 0.2, 0.5)\n",
        "        layers.append(nn.Dropout(p))\n",
        "\n",
        "        in_features = out_features\n",
        "    layers.append(nn.Linear(in_features, CLASSES))\n",
        "    layers.append(nn.LogSoftmax(dim=1))\n",
        "\n",
        "    return nn.Sequential(*layers)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rtguKB-2GZm7"
      },
      "source": [
        "## Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8XzErXkGeZT"
      },
      "source": [
        "def get_mnist():\n",
        "    # Load FashionMNIST dataset.\n",
        "    train_loader = torch.utils.data.DataLoader(\n",
        "        datasets.FashionMNIST(DIR, train=True, download=True, transform=transforms.ToTensor()),\n",
        "        batch_size=BATCHSIZE,\n",
        "        shuffle=True,\n",
        "    )\n",
        "    valid_loader = torch.utils.data.DataLoader(\n",
        "        datasets.FashionMNIST(DIR, train=False, transform=transforms.ToTensor()),\n",
        "        batch_size=BATCHSIZE,\n",
        "        shuffle=True,\n",
        "    )\n",
        "\n",
        "    return train_loader, valid_loader"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wlq7X7Wg379t"
      },
      "source": [
        "## Objective"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JxnOoXiHGmFT"
      },
      "source": [
        "def objective(trial):\n",
        "\n",
        "    # Generate the model.\n",
        "    model = define_model(trial).to(DEVICE)\n",
        "\n",
        "    # Generate the optimizers.\n",
        "    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\", \"SGD\"])\n",
        "    lr = trial.suggest_float(\"lr\", 1e-5, 1e-1, log=True)\n",
        "    optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr)\n",
        "\n",
        "    # Get the FashionMNIST dataset.\n",
        "    train_loader, valid_loader = get_mnist()\n",
        "\n",
        "    # Training of the model.\n",
        "    for epoch in range(EPOCHS):\n",
        "        model.train()\n",
        "        for batch_idx, (data, target) in enumerate(train_loader):\n",
        "            # Limiting training data for faster epochs.\n",
        "            if batch_idx * BATCHSIZE >= N_TRAIN_EXAMPLES:\n",
        "                break\n",
        "            data, target = data.view(data.size(0), -1).to(DEVICE), target.to(DEVICE)\n",
        "            optimizer.zero_grad()\n",
        "            output = model(data)\n",
        "            loss = F.nll_loss(output, target)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "        # Validation of the model.\n",
        "        model.eval()\n",
        "        correct = 0\n",
        "        with torch.no_grad():\n",
        "            for batch_idx, (data, target) in enumerate(valid_loader):\n",
        "                # Limiting validation data.\n",
        "                if batch_idx * BATCHSIZE >= N_VALID_EXAMPLES:\n",
        "                    break\n",
        "                data, target = data.view(data.size(0), -1).to(DEVICE), target.to(DEVICE)\n",
        "                output = model(data)\n",
        "                # Get the index of the max log-probability.\n",
        "                pred = output.argmax(dim=1, keepdim=True)\n",
        "                correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "        accuracy = correct / min(len(valid_loader.dataset), N_VALID_EXAMPLES)\n",
        "        trial.report(accuracy, epoch)\n",
        "\n",
        "        # Handle pruning based on the intermediate value.\n",
        "        if trial.should_prune():\n",
        "            raise optuna.exceptions.TrialPruned()\n",
        "\n",
        "    return accuracy"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t3XLK4Lz4ALG"
      },
      "source": [
        "## Study"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9Q5RHOT33p_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e5097e71-cb84-48ed-d9d3-3773a1468f0a"
      },
      "source": [
        "study = optuna.create_study(direction=\"maximize\", study_name=\"distributed-example-advanced\", storage=\"postgresql+psycopg2://@/postgres\", load_if_exists=True)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:34:53,965] A new study created in RDB with name: distributed-example-advanced\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YeViGpuH4DTt"
      },
      "source": [
        "## Parallel distribution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYhrshim35J0"
      },
      "source": [
        "def para_fn(device):\n",
        "  loaded_study = optuna.load_study(study_name=\"distributed-example-advanced\", storage=\"postgresql+psycopg2://@/postgres\")\n",
        "\n",
        "  loaded_study.optimize(objective, n_trials=2)\n",
        "\n",
        "  trial = loaded_study.best_trial\n",
        "  params = loaded_study.best_params\n",
        "  pruned_trials = loaded_study.get_trials(deepcopy=False, states=[TrialState.PRUNED])\n",
        "  complete_trials = loaded_study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])\n",
        "\n",
        "  print('\\rDevice {}\\t Best trial is: {}\\t Best params are: {}\\t Finished trials: {}\\t Pruned trials: {}\\t Complete trials: {}'\n",
        "  .format(device, trial.values[0], params, len(loaded_study.trials), pruned_trials, complete_trials))"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ADcz5byt6KH2",
        "outputId": "991839d3-0124-4f47-ea95-e2b06db2a9db"
      },
      "source": [
        "from joblib import Parallel, delayed\n",
        "\n",
        "def run(fold):\n",
        "    device = xm.xla_device(fold + 1)\n",
        "    DEVICE = device\n",
        "    para_fn(device)\n",
        "\n",
        "Parallel(n_jobs=8, backend=\"threading\")(delayed(run)(i) for i in range(8))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 17:58:12,702] Trial 0 finished with value: 0.13125 and parameters: {'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}. Best is trial 0 with value: 0.13125.\n",
            "[I 2023-09-01 17:59:33,419] Trial 4 finished with value: 0.6875 and parameters: {'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}. Best is trial 4 with value: 0.6875.\n",
            "[I 2023-09-01 18:00:28,451] Trial 1 finished with value: 0.678125 and parameters: {'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}. Best is trial 4 with value: 0.6875.\n",
            "[I 2023-09-01 18:01:05,995] Trial 5 finished with value: 0.675 and parameters: {'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}. Best is trial 4 with value: 0.6875.\n",
            "[I 2023-09-01 18:07:24,491] Trial 2 finished with value: 0.675 and parameters: {'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}. Best is trial 3 with value: 0.75625.\n",
            "[I 2023-09-01 18:07:24,536] Trial 3 finished with value: 0.75625 and parameters: {'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}. Best is trial 3 with value: 0.75625.\n",
            "[I 2023-09-01 18:08:38,715] Trial 6 pruned. \n",
            "[I 2023-09-01 18:10:45,190] Trial 12 pruned. \n",
            "[I 2023-09-01 18:10:45,625] Trial 7 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:6\t Best trial is: 0.75625\t Best params are: {'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}\t Finished trials: 15\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:11:01,352] Trial 13 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:1\t Best trial is: 0.75625\t Best params are: {'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:11:15,996] Trial 11 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:3\t Best trial is: 0.75625\t Best params are: {'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.54375], datetime_start=datetime.datetime(2023, 9, 1, 18, 1, 6, 81499), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 15, 965655), params={'n_layers': 3, 'n_units_l0': 102, 'dropout_l0': 0.35701494704947756, 'n_units_l1': 36, 'dropout_l1': 0.24470595579197654, 'n_units_l2': 17, 'dropout_l2': 0.29203194244434943, 'optimizer': 'RMSprop', 'lr': 0.006789469664018019}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.390625, 1: 0.54375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=54, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:12:28,623] Trial 15 pruned. \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:4\t Best trial is: 0.75625\t Best params are: {'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.54375], datetime_start=datetime.datetime(2023, 9, 1, 18, 1, 6, 81499), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 15, 965655), params={'n_layers': 3, 'n_units_l0': 102, 'dropout_l0': 0.35701494704947756, 'n_units_l1': 36, 'dropout_l1': 0.24470595579197654, 'n_units_l2': 17, 'dropout_l2': 0.29203194244434943, 'optimizer': 'RMSprop', 'lr': 0.006789469664018019}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.390625, 1: 0.54375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=54, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.25], datetime_start=datetime.datetime(2023, 9, 1, 18, 10, 45, 684066), datetime_complete=datetime.datetime(2023, 9, 1, 18, 12, 28, 594560), params={'n_layers': 2, 'n_units_l0': 7, 'dropout_l0': 0.45641732127419804, 'n_units_l1': 4, 'dropout_l1': 0.32419467850530614, 'optimizer': 'SGD', 'lr': 0.04569886270392929}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.25}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=58, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:24:28,160] Trial 9 finished with value: 0.803125 and parameters: {'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}. Best is trial 9 with value: 0.803125.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:7\t Best trial is: 0.803125\t Best params are: {'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.54375], datetime_start=datetime.datetime(2023, 9, 1, 18, 1, 6, 81499), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 15, 965655), params={'n_layers': 3, 'n_units_l0': 102, 'dropout_l0': 0.35701494704947756, 'n_units_l1': 36, 'dropout_l1': 0.24470595579197654, 'n_units_l2': 17, 'dropout_l2': 0.29203194244434943, 'optimizer': 'RMSprop', 'lr': 0.006789469664018019}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.390625, 1: 0.54375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=54, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.25], datetime_start=datetime.datetime(2023, 9, 1, 18, 10, 45, 684066), datetime_complete=datetime.datetime(2023, 9, 1, 18, 12, 28, 594560), params={'n_layers': 2, 'n_units_l0': 7, 'dropout_l0': 0.45641732127419804, 'n_units_l1': 4, 'dropout_l1': 0.32419467850530614, 'optimizer': 'SGD', 'lr': 0.04569886270392929}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.25}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=58, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None), FrozenTrial(number=9, state=TrialState.COMPLETE, values=[0.803125], datetime_start=datetime.datetime(2023, 9, 1, 17, 59, 33, 510510), datetime_complete=datetime.datetime(2023, 9, 1, 18, 24, 28, 36436), params={'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.63125, 1: 0.653125, 2: 0.7, 3: 0.69375, 4: 0.721875, 5: 0.71875, 6: 0.771875, 7: 0.74375, 8: 0.76875, 9: 0.803125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=52, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:27:08,569] Trial 10 finished with value: 0.7625 and parameters: {'n_layers': 1, 'n_units_l0': 40, 'dropout_l0': 0.2971969506956887, 'optimizer': 'Adam', 'lr': 0.004967695888512498}. Best is trial 9 with value: 0.803125.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:5\t Best trial is: 0.803125\t Best params are: {'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.54375], datetime_start=datetime.datetime(2023, 9, 1, 18, 1, 6, 81499), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 15, 965655), params={'n_layers': 3, 'n_units_l0': 102, 'dropout_l0': 0.35701494704947756, 'n_units_l1': 36, 'dropout_l1': 0.24470595579197654, 'n_units_l2': 17, 'dropout_l2': 0.29203194244434943, 'optimizer': 'RMSprop', 'lr': 0.006789469664018019}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.390625, 1: 0.54375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=54, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.25], datetime_start=datetime.datetime(2023, 9, 1, 18, 10, 45, 684066), datetime_complete=datetime.datetime(2023, 9, 1, 18, 12, 28, 594560), params={'n_layers': 2, 'n_units_l0': 7, 'dropout_l0': 0.45641732127419804, 'n_units_l1': 4, 'dropout_l1': 0.32419467850530614, 'optimizer': 'SGD', 'lr': 0.04569886270392929}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.25}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=58, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None), FrozenTrial(number=9, state=TrialState.COMPLETE, values=[0.803125], datetime_start=datetime.datetime(2023, 9, 1, 17, 59, 33, 510510), datetime_complete=datetime.datetime(2023, 9, 1, 18, 24, 28, 36436), params={'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.63125, 1: 0.653125, 2: 0.7, 3: 0.69375, 4: 0.721875, 5: 0.71875, 6: 0.771875, 7: 0.74375, 8: 0.76875, 9: 0.803125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=52, value=None), FrozenTrial(number=10, state=TrialState.COMPLETE, values=[0.7625], datetime_start=datetime.datetime(2023, 9, 1, 18, 0, 28, 582619), datetime_complete=datetime.datetime(2023, 9, 1, 18, 27, 8, 533645), params={'n_layers': 1, 'n_units_l0': 40, 'dropout_l0': 0.2971969506956887, 'optimizer': 'Adam', 'lr': 0.004967695888512498}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.603125, 1: 0.7, 2: 0.728125, 3: 0.7375, 4: 0.73125, 5: 0.7375, 6: 0.784375, 7: 0.784375, 8: 0.784375, 9: 0.7625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=53, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:31:39,122] Trial 8 finished with value: 0.6875 and parameters: {'n_layers': 3, 'n_units_l0': 49, 'dropout_l0': 0.36236404666585853, 'n_units_l1': 125, 'dropout_l1': 0.4208604251617582, 'n_units_l2': 41, 'dropout_l2': 0.490725851886296, 'optimizer': 'RMSprop', 'lr': 0.00189073705176855}. Best is trial 9 with value: 0.803125.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:2\t Best trial is: 0.803125\t Best params are: {'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.54375], datetime_start=datetime.datetime(2023, 9, 1, 18, 1, 6, 81499), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 15, 965655), params={'n_layers': 3, 'n_units_l0': 102, 'dropout_l0': 0.35701494704947756, 'n_units_l1': 36, 'dropout_l1': 0.24470595579197654, 'n_units_l2': 17, 'dropout_l2': 0.29203194244434943, 'optimizer': 'RMSprop', 'lr': 0.006789469664018019}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.390625, 1: 0.54375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=54, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.25], datetime_start=datetime.datetime(2023, 9, 1, 18, 10, 45, 684066), datetime_complete=datetime.datetime(2023, 9, 1, 18, 12, 28, 594560), params={'n_layers': 2, 'n_units_l0': 7, 'dropout_l0': 0.45641732127419804, 'n_units_l1': 4, 'dropout_l1': 0.32419467850530614, 'optimizer': 'SGD', 'lr': 0.04569886270392929}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.25}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=58, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None), FrozenTrial(number=8, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 58, 55, 910542), datetime_complete=datetime.datetime(2023, 9, 1, 18, 31, 39, 64677), params={'n_layers': 3, 'n_units_l0': 49, 'dropout_l0': 0.36236404666585853, 'n_units_l1': 125, 'dropout_l1': 0.4208604251617582, 'n_units_l2': 41, 'dropout_l2': 0.490725851886296, 'optimizer': 'RMSprop', 'lr': 0.00189073705176855}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.515625, 1: 0.625, 2: 0.5875, 3: 0.64375, 4: 0.665625, 5: 0.703125, 6: 0.69375, 7: 0.675, 8: 0.69375, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=51, value=None), FrozenTrial(number=9, state=TrialState.COMPLETE, values=[0.803125], datetime_start=datetime.datetime(2023, 9, 1, 17, 59, 33, 510510), datetime_complete=datetime.datetime(2023, 9, 1, 18, 24, 28, 36436), params={'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.63125, 1: 0.653125, 2: 0.7, 3: 0.69375, 4: 0.721875, 5: 0.71875, 6: 0.771875, 7: 0.74375, 8: 0.76875, 9: 0.803125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=52, value=None), FrozenTrial(number=10, state=TrialState.COMPLETE, values=[0.7625], datetime_start=datetime.datetime(2023, 9, 1, 18, 0, 28, 582619), datetime_complete=datetime.datetime(2023, 9, 1, 18, 27, 8, 533645), params={'n_layers': 1, 'n_units_l0': 40, 'dropout_l0': 0.2971969506956887, 'optimizer': 'Adam', 'lr': 0.004967695888512498}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.603125, 1: 0.7, 2: 0.728125, 3: 0.7375, 4: 0.73125, 5: 0.7375, 6: 0.784375, 7: 0.784375, 8: 0.784375, 9: 0.7625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=53, value=None)]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[I 2023-09-01 18:32:49,129] Trial 14 finished with value: 0.75 and parameters: {'n_layers': 2, 'n_units_l0': 59, 'dropout_l0': 0.2929078580049342, 'n_units_l1': 74, 'dropout_l1': 0.2469122440347837, 'optimizer': 'RMSprop', 'lr': 0.000754065315933356}. Best is trial 9 with value: 0.803125.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\rDevice xla:8\t Best trial is: 0.803125\t Best params are: {'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}\t Finished trials: 16\t Pruned trials: [FrozenTrial(number=6, state=TrialState.PRUNED, values=[0.259375], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 205690), datetime_complete=datetime.datetime(2023, 9, 1, 18, 8, 38, 573680), params={'n_layers': 3, 'n_units_l0': 57, 'dropout_l0': 0.46140059157792346, 'n_units_l1': 62, 'dropout_l1': 0.4880138355177629, 'n_units_l2': 64, 'dropout_l2': 0.37749645863265213, 'optimizer': 'RMSprop', 'lr': 6.566113322809912e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.2, 1: 0.20625, 2: 0.278125, 3: 0.228125, 4: 0.315625, 5: 0.35625, 6: 0.321875, 7: 0.40625, 8: 0.259375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=49, value=None), FrozenTrial(number=7, state=TrialState.PRUNED, values=[0.10625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 238633), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 540942), params={'n_layers': 3, 'n_units_l0': 96, 'dropout_l0': 0.3257151968038643, 'n_units_l1': 106, 'dropout_l1': 0.4766907730206084, 'n_units_l2': 65, 'dropout_l2': 0.3967265954974123, 'optimizer': 'Adam', 'lr': 1.5135161931953466e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.084375, 1: 0.09375, 2: 0.11875, 3: 0.1, 4: 0.075, 5: 0.084375, 6: 0.09375, 7: 0.0875, 8: 0.10625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=50, value=None), FrozenTrial(number=11, state=TrialState.PRUNED, values=[0.54375], datetime_start=datetime.datetime(2023, 9, 1, 18, 1, 6, 81499), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 15, 965655), params={'n_layers': 3, 'n_units_l0': 102, 'dropout_l0': 0.35701494704947756, 'n_units_l1': 36, 'dropout_l1': 0.24470595579197654, 'n_units_l2': 17, 'dropout_l2': 0.29203194244434943, 'optimizer': 'RMSprop', 'lr': 0.006789469664018019}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.390625, 1: 0.54375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=54, value=None), FrozenTrial(number=12, state=TrialState.PRUNED, values=[0.125], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 617359), datetime_complete=datetime.datetime(2023, 9, 1, 18, 10, 45, 84388), params={'n_layers': 1, 'n_units_l0': 6, 'dropout_l0': 0.31744591718164916, 'optimizer': 'SGD', 'lr': 0.05882403766652752}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=55, value=None), FrozenTrial(number=13, state=TrialState.PRUNED, values=[0.09375], datetime_start=datetime.datetime(2023, 9, 1, 18, 7, 24, 646583), datetime_complete=datetime.datetime(2023, 9, 1, 18, 11, 1, 322221), params={'n_layers': 2, 'n_units_l0': 49, 'dropout_l0': 0.4032786561451569, 'n_units_l1': 112, 'dropout_l1': 0.21694095624123041, 'optimizer': 'SGD', 'lr': 0.0013367487975290849}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.09375}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=56, value=None), FrozenTrial(number=15, state=TrialState.PRUNED, values=[0.25], datetime_start=datetime.datetime(2023, 9, 1, 18, 10, 45, 684066), datetime_complete=datetime.datetime(2023, 9, 1, 18, 12, 28, 594560), params={'n_layers': 2, 'n_units_l0': 7, 'dropout_l0': 0.45641732127419804, 'n_units_l1': 4, 'dropout_l1': 0.32419467850530614, 'optimizer': 'SGD', 'lr': 0.04569886270392929}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.25}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=58, value=None)]\t Complete trials: [FrozenTrial(number=0, state=TrialState.COMPLETE, values=[0.13125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 70703), datetime_complete=datetime.datetime(2023, 9, 1, 17, 58, 12, 422144), params={'n_layers': 2, 'n_units_l0': 88, 'dropout_l0': 0.38544392176223574, 'n_units_l1': 53, 'dropout_l1': 0.3674667972416941, 'optimizer': 'SGD', 'lr': 0.0008110501555179943}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.134375, 1: 0.18125, 2: 0.159375, 3: 0.128125, 4: 0.121875, 5: 0.11875, 6: 0.13125, 7: 0.109375, 8: 0.1, 9: 0.13125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=43, value=None), FrozenTrial(number=1, state=TrialState.COMPLETE, values=[0.678125], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 88860), datetime_complete=datetime.datetime(2023, 9, 1, 18, 0, 28, 279328), params={'n_layers': 1, 'n_units_l0': 108, 'dropout_l0': 0.22655547770644138, 'optimizer': 'RMSprop', 'lr': 5.01903112447575e-05}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.446875, 1: 0.575, 2: 0.584375, 3: 0.65625, 4: 0.615625, 5: 0.6, 6: 0.6625, 7: 0.621875, 8: 0.684375, 9: 0.678125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=44, value=None), FrozenTrial(number=2, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 107229), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 295377), params={'n_layers': 2, 'n_units_l0': 86, 'dropout_l0': 0.4329312441971566, 'n_units_l1': 107, 'dropout_l1': 0.49229643074202123, 'optimizer': 'RMSprop', 'lr': 0.0038315149953758215}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.459375, 1: 0.665625, 2: 0.575, 3: 0.721875, 4: 0.659375, 5: 0.678125, 6: 0.7125, 7: 0.671875, 8: 0.715625, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=45, value=None), FrozenTrial(number=3, state=TrialState.COMPLETE, values=[0.75625], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 148982), datetime_complete=datetime.datetime(2023, 9, 1, 18, 7, 24, 298730), params={'n_layers': 2, 'n_units_l0': 106, 'dropout_l0': 0.4131347793374167, 'n_units_l1': 55, 'dropout_l1': 0.212257947140673, 'optimizer': 'RMSprop', 'lr': 0.005008655700954662}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.45, 1: 0.603125, 2: 0.646875, 3: 0.725, 4: 0.73125, 5: 0.715625, 6: 0.70625, 7: 0.709375, 8: 0.7625, 9: 0.75625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=46, value=None), FrozenTrial(number=4, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 172409), datetime_complete=datetime.datetime(2023, 9, 1, 17, 59, 33, 133895), params={'n_layers': 2, 'n_units_l0': 70, 'dropout_l0': 0.21762821136488056, 'n_units_l1': 114, 'dropout_l1': 0.216719572488067, 'optimizer': 'SGD', 'lr': 0.06751739082213563}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.184375, 1: 0.41875, 2: 0.440625, 3: 0.615625, 4: 0.571875, 5: 0.64375, 6: 0.65, 7: 0.671875, 8: 0.621875, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=47, value=None), FrozenTrial(number=5, state=TrialState.COMPLETE, values=[0.675], datetime_start=datetime.datetime(2023, 9, 1, 17, 34, 58, 189940), datetime_complete=datetime.datetime(2023, 9, 1, 18, 1, 5, 863289), params={'n_layers': 1, 'n_units_l0': 70, 'dropout_l0': 0.2125313712791924, 'optimizer': 'RMSprop', 'lr': 0.03505566571829635}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.553125, 1: 0.59375, 2: 0.5375, 3: 0.6125, 4: 0.69375, 5: 0.709375, 6: 0.61875, 7: 0.7125, 8: 0.6, 9: 0.675}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=48, value=None), FrozenTrial(number=8, state=TrialState.COMPLETE, values=[0.6875], datetime_start=datetime.datetime(2023, 9, 1, 17, 58, 55, 910542), datetime_complete=datetime.datetime(2023, 9, 1, 18, 31, 39, 64677), params={'n_layers': 3, 'n_units_l0': 49, 'dropout_l0': 0.36236404666585853, 'n_units_l1': 125, 'dropout_l1': 0.4208604251617582, 'n_units_l2': 41, 'dropout_l2': 0.490725851886296, 'optimizer': 'RMSprop', 'lr': 0.00189073705176855}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.515625, 1: 0.625, 2: 0.5875, 3: 0.64375, 4: 0.665625, 5: 0.703125, 6: 0.69375, 7: 0.675, 8: 0.69375, 9: 0.6875}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l2': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l2': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=51, value=None), FrozenTrial(number=9, state=TrialState.COMPLETE, values=[0.803125], datetime_start=datetime.datetime(2023, 9, 1, 17, 59, 33, 510510), datetime_complete=datetime.datetime(2023, 9, 1, 18, 24, 28, 36436), params={'n_layers': 1, 'n_units_l0': 121, 'dropout_l0': 0.4390613254642145, 'optimizer': 'RMSprop', 'lr': 0.0004654633567334438}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.63125, 1: 0.653125, 2: 0.7, 3: 0.69375, 4: 0.721875, 5: 0.71875, 6: 0.771875, 7: 0.74375, 8: 0.76875, 9: 0.803125}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=52, value=None), FrozenTrial(number=10, state=TrialState.COMPLETE, values=[0.7625], datetime_start=datetime.datetime(2023, 9, 1, 18, 0, 28, 582619), datetime_complete=datetime.datetime(2023, 9, 1, 18, 27, 8, 533645), params={'n_layers': 1, 'n_units_l0': 40, 'dropout_l0': 0.2971969506956887, 'optimizer': 'Adam', 'lr': 0.004967695888512498}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.603125, 1: 0.7, 2: 0.728125, 3: 0.7375, 4: 0.73125, 5: 0.7375, 6: 0.784375, 7: 0.784375, 8: 0.784375, 9: 0.7625}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=53, value=None), FrozenTrial(number=14, state=TrialState.COMPLETE, values=[0.75], datetime_start=datetime.datetime(2023, 9, 1, 18, 8, 58, 781004), datetime_complete=datetime.datetime(2023, 9, 1, 18, 32, 49, 103122), params={'n_layers': 2, 'n_units_l0': 59, 'dropout_l0': 0.2929078580049342, 'n_units_l1': 74, 'dropout_l1': 0.2469122440347837, 'optimizer': 'RMSprop', 'lr': 0.000754065315933356}, user_attrs={}, system_attrs={}, intermediate_values={0: 0.540625, 1: 0.603125, 2: 0.715625, 3: 0.6875, 4: 0.66875, 5: 0.7, 6: 0.709375, 7: 0.7375, 8: 0.721875, 9: 0.75}, distributions={'n_layers': IntDistribution(high=3, log=False, low=1, step=1), 'n_units_l0': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l0': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'n_units_l1': IntDistribution(high=128, log=False, low=4, step=1), 'dropout_l1': FloatDistribution(high=0.5, log=False, low=0.2, step=None), 'optimizer': CategoricalDistribution(choices=('Adam', 'RMSprop', 'SGD')), 'lr': FloatDistribution(high=0.1, log=True, low=1e-05, step=None)}, trial_id=57, value=None)]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[None, None, None, None, None, None, None, None]"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lm-FjtW9I7Rr"
      },
      "source": [
        "## Further finetuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5p0ba5JSyvb5",
        "outputId": "68a9d38f-2f6f-4867-a08a-894d2332b4b6"
      },
      "source": [
        "fix_optimizer = {}\n",
        "fix_optimizer['optimizer'] = study.best_params['optimizer']\n",
        "study.sampler = optuna.samplers.PartialFixedSampler(fix_optimizer, study.sampler)\n",
        "study.optimize(objective, n_trials=5)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-35-458a4a9edb07>:3: ExperimentalWarning: PartialFixedSampler is experimental (supported from v2.4.0). The interface can change in the future.\n",
            "  study.sampler = optuna.samplers.PartialFixedSampler(fix_optimizer, study.sampler)\n",
            "[I 2023-09-01 18:37:46,843] Trial 16 finished with value: 0.79375 and parameters: {'n_layers': 1, 'n_units_l0': 124, 'dropout_l0': 0.4910243515418116, 'optimizer': 'RMSprop', 'lr': 0.0003375550152258889}. Best is trial 9 with value: 0.803125.\n",
            "[I 2023-09-01 18:43:31,488] Trial 17 finished with value: 0.7625 and parameters: {'n_layers': 1, 'n_units_l0': 122, 'dropout_l0': 0.48835733594152314, 'optimizer': 'RMSprop', 'lr': 0.0004000253932057074}. Best is trial 9 with value: 0.803125.\n",
            "[I 2023-09-01 18:50:09,282] Trial 18 finished with value: 0.696875 and parameters: {'n_layers': 1, 'n_units_l0': 123, 'dropout_l0': 0.4932086029672041, 'optimizer': 'RMSprop', 'lr': 0.00024096186704898929}. Best is trial 9 with value: 0.803125.\n",
            "[I 2023-09-01 18:51:06,936] Trial 19 pruned. \n",
            "[I 2023-09-01 18:51:34,085] Trial 20 pruned. \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bZemnmE40G5P",
        "outputId": "763ddbad-e839-4c15-e8ee-8e083d5a9637"
      },
      "source": [
        "print(\"Best trial:\")\n",
        "trial = study.best_trial\n",
        "\n",
        "print(\"  Value: \", trial.value)\n",
        "print(\"  Params: \")\n",
        "for key, value in trial.params.items():\n",
        "  print(\"    {}: {}\".format(key, value))"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best trial:\n",
            "  Value:  0.803125\n",
            "  Params: \n",
            "    n_layers: 1\n",
            "    n_units_l0: 121\n",
            "    dropout_l0: 0.4390613254642145\n",
            "    optimizer: RMSprop\n",
            "    lr: 0.0004654633567334438\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lSm_uVTB0X3R",
        "outputId": "21b55a71-1ca3-4ed3-b70f-999a7ee290e7"
      },
      "source": [
        "study.best_trial.params.items()"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_items([('n_layers', 1), ('n_units_l0', 121), ('dropout_l0', 0.4390613254642145), ('optimizer', 'RMSprop'), ('lr', 0.0004654633567334438)])"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QVaPsBGCF4GR"
      },
      "source": [
        "## Visualizations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "9xhukHYU1Bix",
        "outputId": "3b5cecb7-9199-4162-f707-e6645cdffc38"
      },
      "source": [
        "optuna.visualization.plot_intermediate_values(study)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"b926da70-58ce-49ee-99fd-71b8fb633e62\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b926da70-58ce-49ee-99fd-71b8fb633e62\")) {                    Plotly.newPlot(                        \"b926da70-58ce-49ee-99fd-71b8fb633e62\",                        [{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial0\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.134375,0.18125,0.159375,0.128125,0.121875,0.11875,0.13125,0.109375,0.1,0.13125],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial1\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.446875,0.575,0.584375,0.65625,0.615625,0.6,0.6625,0.621875,0.684375,0.678125],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial2\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.459375,0.665625,0.575,0.721875,0.659375,0.678125,0.7125,0.671875,0.715625,0.675],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial3\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.45,0.603125,0.646875,0.725,0.73125,0.715625,0.70625,0.709375,0.7625,0.75625],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial4\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.184375,0.41875,0.440625,0.615625,0.571875,0.64375,0.65,0.671875,0.621875,0.6875],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial5\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.553125,0.59375,0.5375,0.6125,0.69375,0.709375,0.61875,0.7125,0.6,0.675],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial6\",\"x\":[0,1,2,3,4,5,6,7,8],\"y\":[0.2,0.20625,0.278125,0.228125,0.315625,0.35625,0.321875,0.40625,0.259375],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial7\",\"x\":[0,1,2,3,4,5,6,7,8],\"y\":[0.084375,0.09375,0.11875,0.1,0.075,0.084375,0.09375,0.0875,0.10625],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial8\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.515625,0.625,0.5875,0.64375,0.665625,0.703125,0.69375,0.675,0.69375,0.6875],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial9\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.63125,0.653125,0.7,0.69375,0.721875,0.71875,0.771875,0.74375,0.76875,0.803125],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial10\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.603125,0.7,0.728125,0.7375,0.73125,0.7375,0.784375,0.784375,0.784375,0.7625],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial11\",\"x\":[0,1],\"y\":[0.390625,0.54375],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial12\",\"x\":[0],\"y\":[0.125],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial13\",\"x\":[0],\"y\":[0.09375],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial14\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.540625,0.603125,0.715625,0.6875,0.66875,0.7,0.709375,0.7375,0.721875,0.75],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial15\",\"x\":[0],\"y\":[0.25],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial16\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.65,0.71875,0.61875,0.71875,0.74375,0.709375,0.69375,0.76875,0.790625,0.79375],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial17\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.64375,0.7125,0.675,0.734375,0.771875,0.75,0.803125,0.75625,0.775,0.7625],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial18\",\"x\":[0,1,2,3,4,5,6,7,8,9],\"y\":[0.684375,0.69375,0.68125,0.7,0.703125,0.765625,0.715625,0.740625,0.74375,0.696875],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial19\",\"x\":[0,1],\"y\":[0.603125,0.60625],\"type\":\"scatter\"},{\"marker\":{\"maxdisplayed\":10},\"mode\":\"lines+markers\",\"name\":\"Trial20\",\"x\":[0],\"y\":[0.478125],\"type\":\"scatter\"}],                        {\"showlegend\":false,\"title\":{\"text\":\"Intermediate Values Plot\"},\"xaxis\":{\"title\":{\"text\":\"Step\"}},\"yaxis\":{\"title\":{\"text\":\"Intermediate Value\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('b926da70-58ce-49ee-99fd-71b8fb633e62');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "NqZi6h6IF9dd",
        "outputId": "bfac2fcb-49b0-4467-b48b-e3fea6b08111"
      },
      "source": [
        "optuna.visualization.plot_param_importances(study)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"5122d70b-5e25-46e9-a9f9-842d7d74b77a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"5122d70b-5e25-46e9-a9f9-842d7d74b77a\")) {                    Plotly.newPlot(                        \"5122d70b-5e25-46e9-a9f9-842d7d74b77a\",                        [{\"cliponaxis\":false,\"hovertemplate\":[\"n_layers (IntDistribution): 0.04942747521430415\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"lr (FloatDistribution): 0.07144369356760678\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"n_units_l0 (IntDistribution): 0.10568009969791338\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"dropout_l0 (FloatDistribution): 0.24569520394540045\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"optimizer (CategoricalDistribution): 0.5277535275747752\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\"],\"marker\":{\"color\":\"rgb(66,146,198)\"},\"orientation\":\"h\",\"text\":[\"0.05\",\"0.07\",\"0.11\",\"0.25\",\"0.53\"],\"textposition\":\"outside\",\"x\":[0.04942747521430415,0.07144369356760678,0.10568009969791338,0.24569520394540045,0.5277535275747752],\"y\":[\"n_layers\",\"lr\",\"n_units_l0\",\"dropout_l0\",\"optimizer\"],\"type\":\"bar\"}],                        {\"showlegend\":false,\"title\":{\"text\":\"Hyperparameter Importances\"},\"xaxis\":{\"title\":{\"text\":\"Importance for Objective Value\"}},\"yaxis\":{\"title\":{\"text\":\"Hyperparameter\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('5122d70b-5e25-46e9-a9f9-842d7d74b77a');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "oq4034q5GAcN",
        "outputId": "60c99a55-b429-480e-d9a3-d4908c82c8a2"
      },
      "source": [
        "optuna.visualization.plot_optimization_history(study)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"604d5587-fc0e-452d-b081-b21b5e838be7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"604d5587-fc0e-452d-b081-b21b5e838be7\")) {                    Plotly.newPlot(                        \"604d5587-fc0e-452d-b081-b21b5e838be7\",                        [{\"mode\":\"markers\",\"name\":\"Objective Value\",\"x\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\"},{\"mode\":\"lines\",\"name\":\"Best Value\",\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],\"y\":[0.13125,0.678125,0.678125,0.75625,0.75625,0.75625,0.75625,0.75625,0.75625,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125,0.803125],\"type\":\"scatter\"},{\"marker\":{\"color\":\"#cccccc\"},\"mode\":\"markers\",\"name\":\"Infeasible Trial\",\"showlegend\":false,\"x\":[],\"y\":[],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Optimization History Plot\"},\"xaxis\":{\"title\":{\"text\":\"Trial\"}},\"yaxis\":{\"title\":{\"text\":\"Objective Value\"}},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('604d5587-fc0e-452d-b081-b21b5e838be7');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 562
        },
        "id": "g_me5H-jGF13",
        "outputId": "23c5381e-6f18-4b53-d02d-81da650cd6b9"
      },
      "source": [
        "optuna.visualization.plot_slice(study)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"604e24af-4070-4fd7-9d23-90bb97a3ac87\" class=\"plotly-graph-div\" style=\"height:525px; width:2700px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"604e24af-4070-4fd7-9d23-90bb97a3ac87\")) {                    Plotly.newPlot(                        \"604e24af-4070-4fd7-9d23-90bb97a3ac87\",                        [{\"marker\":{\"color\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":true},\"mode\":\"markers\",\"showlegend\":false,\"x\":[0.38544392176223574,0.22655547770644138,0.4329312441971566,0.4131347793374167,0.21762821136488056,0.2125313712791924,0.36236404666585853,0.4390613254642145,0.2971969506956887,0.2929078580049342,0.4910243515418116,0.48835733594152314,0.4932086029672041],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\",\"xaxis\":\"x\",\"yaxis\":\"y\"},{\"marker\":{\"color\":[0,2,3,4,8,14],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[0.3674667972416941,0.49229643074202123,0.212257947140673,0.216719572488067,0.4208604251617582,0.2469122440347837],\"y\":[0.13125,0.675,0.75625,0.6875,0.6875,0.75],\"type\":\"scatter\",\"xaxis\":\"x2\",\"yaxis\":\"y2\"},{\"marker\":{\"color\":[8],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[0.490725851886296],\"y\":[0.6875],\"type\":\"scatter\",\"xaxis\":\"x3\",\"yaxis\":\"y3\"},{\"marker\":{\"color\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[0.0008110501555179943,5.01903112447575e-05,0.0038315149953758215,0.005008655700954662,0.06751739082213563,0.03505566571829635,0.00189073705176855,0.0004654633567334438,0.004967695888512498,0.000754065315933356,0.0003375550152258889,0.0004000253932057074,0.00024096186704898929],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\",\"xaxis\":\"x4\",\"yaxis\":\"y4\"},{\"marker\":{\"color\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[2,1,2,2,2,1,3,1,1,2,1,1,1],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\",\"xaxis\":\"x5\",\"yaxis\":\"y5\"},{\"marker\":{\"color\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[88,108,86,106,70,70,49,121,40,59,124,122,123],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\",\"xaxis\":\"x6\",\"yaxis\":\"y6\"},{\"marker\":{\"color\":[0,2,3,4,8,14],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[53,107,55,114,125,74],\"y\":[0.13125,0.675,0.75625,0.6875,0.6875,0.75],\"type\":\"scatter\",\"xaxis\":\"x7\",\"yaxis\":\"y7\"},{\"marker\":{\"color\":[8],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[41],\"y\":[0.6875],\"type\":\"scatter\",\"xaxis\":\"x8\",\"yaxis\":\"y8\"},{\"marker\":{\"color\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5},\"showscale\":false},\"mode\":\"markers\",\"showlegend\":false,\"x\":[\"SGD\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"SGD\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"Adam\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"RMSprop\"],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\",\"xaxis\":\"x9\",\"yaxis\":\"y9\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,0.09135802469135802],\"title\":{\"text\":\"dropout_l0\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Objective Value\"}},\"xaxis2\":{\"anchor\":\"y2\",\"domain\":[0.11358024691358025,0.20493827160493827],\"title\":{\"text\":\"dropout_l1\"}},\"yaxis2\":{\"anchor\":\"x2\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis3\":{\"anchor\":\"y3\",\"domain\":[0.2271604938271605,0.31851851851851853],\"title\":{\"text\":\"dropout_l2\"}},\"yaxis3\":{\"anchor\":\"x3\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis4\":{\"anchor\":\"y4\",\"domain\":[0.34074074074074073,0.43209876543209874],\"title\":{\"text\":\"lr\"},\"type\":\"log\"},\"yaxis4\":{\"anchor\":\"x4\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis5\":{\"anchor\":\"y5\",\"domain\":[0.454320987654321,0.5456790123456791],\"title\":{\"text\":\"n_layers\"}},\"yaxis5\":{\"anchor\":\"x5\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis6\":{\"anchor\":\"y6\",\"domain\":[0.5679012345679012,0.6592592592592592],\"title\":{\"text\":\"n_units_l0\"}},\"yaxis6\":{\"anchor\":\"x6\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis7\":{\"anchor\":\"y7\",\"domain\":[0.6814814814814815,0.7728395061728395],\"title\":{\"text\":\"n_units_l1\"}},\"yaxis7\":{\"anchor\":\"x7\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis8\":{\"anchor\":\"y8\",\"domain\":[0.7950617283950617,0.8864197530864197],\"title\":{\"text\":\"n_units_l2\"}},\"yaxis8\":{\"anchor\":\"x8\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"xaxis9\":{\"anchor\":\"y9\",\"domain\":[0.908641975308642,1.0],\"title\":{\"text\":\"optimizer\"},\"type\":\"category\",\"categoryorder\":\"array\",\"categoryarray\":[\"Adam\",\"RMSprop\",\"SGD\"]},\"yaxis9\":{\"anchor\":\"x9\",\"domain\":[0.0,1.0],\"matches\":\"y\",\"showticklabels\":false},\"title\":{\"text\":\"Slice Plot\"},\"width\":2700},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('604e24af-4070-4fd7-9d23-90bb97a3ac87');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "Qod3YfK1GL_O",
        "outputId": "c8195a3d-b0a3-49f1-8af5-a567a22d4996"
      },
      "source": [
        "optuna.visualization.plot_slice(study, ['optimizer'])"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"87285763-4bdf-4630-b64b-4d5c86747d95\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"87285763-4bdf-4630-b64b-4d5c86747d95\")) {                    Plotly.newPlot(                        \"87285763-4bdf-4630-b64b-4d5c86747d95\",                        [{\"marker\":{\"color\":[0,1,2,3,4,5,8,9,10,14,16,17,18],\"colorbar\":{\"title\":{\"text\":\"Trial\"},\"x\":1.0,\"xpad\":40},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"line\":{\"color\":\"Grey\",\"width\":0.5}},\"mode\":\"markers\",\"showlegend\":false,\"x\":[\"SGD\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"SGD\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"Adam\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"RMSprop\"],\"y\":[0.13125,0.678125,0.675,0.75625,0.6875,0.675,0.6875,0.803125,0.7625,0.75,0.79375,0.7625,0.696875],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Slice Plot\"},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"title\":{\"text\":\"optimizer\"},\"type\":\"category\",\"categoryorder\":\"array\",\"categoryarray\":[\"Adam\",\"RMSprop\",\"SGD\"]},\"yaxis\":{\"title\":{\"text\":\"Objective Value\"}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('87285763-4bdf-4630-b64b-4d5c86747d95');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "XhS--O-iGRHo",
        "outputId": "1f78c6af-70d0-45fc-cd02-aa8a678e5515"
      },
      "source": [
        "optuna.visualization.plot_contour(study, ['lr', 'optimizer'])"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"3859ff6c-f434-46e3-af44-b980c1752ae7\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3859ff6c-f434-46e3-af44-b980c1752ae7\")) {                    Plotly.newPlot(                        \"3859ff6c-f434-46e3-af44-b980c1752ae7\",                        [{\"colorbar\":{\"title\":{\"text\":\"Objective Value\"}},\"colorscale\":[[0.0,\"rgb(247,251,255)\"],[0.125,\"rgb(222,235,247)\"],[0.25,\"rgb(198,219,239)\"],[0.375,\"rgb(158,202,225)\"],[0.5,\"rgb(107,174,214)\"],[0.625,\"rgb(66,146,198)\"],[0.75,\"rgb(33,113,181)\"],[0.875,\"rgb(8,81,156)\"],[1.0,\"rgb(8,48,107)\"]],\"connectgaps\":true,\"contours\":{\"coloring\":\"heatmap\"},\"hoverinfo\":\"none\",\"line\":{\"smoothing\":1.3},\"reversescale\":false,\"x\":[3.500903184634672e-05,5.01903112447575e-05,0.00024096186704898929,0.0003375550152258889,0.0004000253932057074,0.0004654633567334438,0.000754065315933356,0.0008110501555179943,0.00189073705176855,0.0038315149953758215,0.004967695888512498,0.005008655700954662,0.03505566571829635,0.06751739082213563,0.0967955604905009],\"y\":[\"Adam\",\"RMSprop\",\"SGD\"],\"z\":[[null,null,null,null,null,null,null,null,null,null,0.7625,null,null,null,null],[null,0.678125,0.696875,0.79375,0.7625,0.803125,0.75,null,0.6875,0.675,null,0.75625,0.675,null,null],[null,null,null,null,null,null,null,0.13125,null,null,null,null,null,0.6875,null]],\"type\":\"contour\"},{\"marker\":{\"color\":\"black\",\"line\":{\"color\":\"Grey\",\"width\":2.0}},\"mode\":\"markers\",\"showlegend\":false,\"x\":[0.0008110501555179943,5.01903112447575e-05,0.0038315149953758215,0.005008655700954662,0.06751739082213563,0.03505566571829635,0.00189073705176855,0.0004654633567334438,0.004967695888512498,0.000754065315933356,0.0003375550152258889,0.0004000253932057074,0.00024096186704898929],\"y\":[\"SGD\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"SGD\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"Adam\",\"RMSprop\",\"RMSprop\",\"RMSprop\",\"RMSprop\"],\"type\":\"scatter\"}],                        {\"title\":{\"text\":\"Contour Plot\"},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"title\":{\"text\":\"lr\"},\"range\":[-4.4558198992207725,-1.0141445610665243],\"type\":\"log\"},\"yaxis\":{\"title\":{\"text\":\"optimizer\"},\"range\":[-0.1,2.1],\"type\":\"category\"}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('3859ff6c-f434-46e3-af44-b980c1752ae7');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}