train-uxml-adam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version: 1.5.0\n",
      "CUDA GPU 1: GeForce RTX 2080 Ti [Compute Capability: 7.5]\n",
      "PyTorch Lightning version: 0.7.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:root:Logging enabled at DEBUG level.\n"
     ]
    }
   ],
   "source": [
    "# import os\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\" \n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "print(\"PyTorch version:\",torch.__version__)\n",
    "if torch.cuda.is_available():\n",
    "    for i in range(torch.cuda.device_count()):\n",
    "      print(f\"CUDA GPU {i+1}: {torch.cuda.get_device_name(i)} [Compute Capability: {torch.cuda.get_device_capability(0)[0]}.{torch.cuda.get_device_capability(0)[1]}]\")\n",
    "    device = torch.device('cuda')\n",
    "    kwargs = {'num_workers': 8, 'pin_memory': True}\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "    print(\"CUDA GPU is not available. :(\")  \n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "print (\"PyTorch Lightning version:\",pl.__version__)\n",
    "    \n",
    "import scipy.sparse as sp\n",
    "from argparse import Namespace\n",
    "\n",
    "from utilities.custom_lightning import CSVProfiler\n",
    "\n",
    "import logging\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)\n",
    "logging.debug(\"Logging enabled at DEBUG level.\")\n",
    "\n",
    "from constants import (SEED, DATA_DIR, LOG_DIR, TRAIN_DATA_PATH, VAL_DATA_PATH, TEST_DATA_PATH)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "NAME = r'AdamUXML' \n",
    "SAVE_PATH = DATA_DIR+r'/'+NAME+r'.pt'\n",
    "PROFILE_PATH = LOG_DIR+'\\\\'+NAME+'\\\\profile.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Interactions(Dataset):\n",
    "    \"\"\"\n",
    "    Create interactions matrix.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, matrix):\n",
    "        self.matrix = matrix\n",
    "        self.n_users = self.matrix.shape[0]\n",
    "        self.n_items = self.matrix.shape[1]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        row = self.matrix.row[index]\n",
    "        col = self.matrix.col[index]\n",
    "        val = self.matrix.data[index]\n",
    "        return (row, col), val\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.matrix.nnz\n",
    "    \n",
    "interaction = Interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestingCallbacks(pl.Callback):\n",
    "    def on_test_start(self, trainer, pl_module):\n",
    "        global y_hat \n",
    "        y_hat = sp.dok_matrix((hparams.total_users, hparams.total_items), dtype=np.float32) \n",
    "    def on_test_end(self, trainer, pl_module):\n",
    "        logging.debug(f\"Non-zero values in prediction matrix: {y_hat.nnz:,}\")\n",
    "        sp.save_npz(DATA_DIR+NAME+r'-y_hat.npz',y_hat.tocoo())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AdamUXML(pl.LightningModule):\n",
    "    def __init__(self, hparams):\n",
    "        super(AdamUXML, self).__init__() \n",
    "        self.hparams = hparams\n",
    "        self.user_factors = nn.Embedding(hparams.total_users, hparams.n_factors, sparse=hparams.sparse)\n",
    "        self.item_factors = nn.Embedding(hparams.total_items, hparams.n_factors, sparse=hparams.sparse)\n",
    "        self.user_biases = nn.Embedding(hparams.total_users, 1, sparse=hparams.sparse)\n",
    "        self.item_biases = nn.Embedding(hparams.total_items, 1, sparse=hparams.sparse)\n",
    "        self.dropout = nn.Dropout(p=self.hparams.dropout_p)\n",
    "        \n",
    "    def forward(self, users, items):            \n",
    "        \n",
    "        user_factors_users = self.user_factors(users)\n",
    "        item_factors_items = self.item_factors(items)        \n",
    "        predictions = self.user_biases(users)\n",
    "        predictions += self.item_biases(items)\n",
    "        predictions += (self.dropout(user_factors_users) * self.dropout(item_factors_items)).sum(dim=1, keepdim=True)              \n",
    "        return predictions.squeeze()\n",
    "    \n",
    "    def MSELoss(self, logits, labels):\n",
    "        return nn.functional.mse_loss(logits, labels)\n",
    "    \n",
    "    def training_step(self, train_batch, batch_idx):\n",
    "        x, y = train_batch\n",
    "        row, column = x\n",
    "        row = row.long()\n",
    "        column = column.long()\n",
    "        logits = self.forward(row,column)\n",
    "        loss = self.MSELoss(logits, y)\n",
    "        logs = {'train_loss': loss}\n",
    "        return {'loss': loss, 'log': logs}\n",
    "\n",
    "    def validation_step(self, val_batch, batch_idx):\n",
    "        x, y = val_batch\n",
    "        row, column = x\n",
    "        row = row.long()\n",
    "        column = column.long()\n",
    "        logits = self.forward(row,column)                \n",
    "        loss = self.MSELoss(logits, y)\n",
    "        return {'val_loss': loss}\n",
    "    \n",
    "    def validation_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n",
    "        tensorboard_logs = {'val_loss': avg_loss}\n",
    "        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}\n",
    "    \n",
    "    def test_step(self, test_batch, batch_idx):\n",
    "        x, y = test_batch\n",
    "        row, column = x\n",
    "        row = row.long()\n",
    "        column = column.long()\n",
    "        logits = self.forward(row,column)                \n",
    "        loss = self.MSELoss(logits, y)        \n",
    "       \n",
    "        logits_array = logits.cpu().numpy()\n",
    "        r = row.cpu().numpy()\n",
    "        c = column.cpu().numpy()\n",
    "        for i in range(len(logits_array)):\n",
    "            y_hat[r[i],c[i]]=logits_array[i]      \n",
    "        \n",
    "        return {'test_loss': loss}\n",
    "    \n",
    "    def test_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()\n",
    "        tensorboard_logs = {'MSE': avg_loss}\n",
    "        print(f\"Test Mean Squared Error (MSE): {avg_loss}\")                   \n",
    "        \n",
    "        return {'avg_test_loss': avg_loss, 'log': tensorboard_logs}\n",
    "      \n",
    "    def prepare_data(self):\n",
    "        self.train_dataset = sp.load_npz(TRAIN_DATA_PATH)\n",
    "        self.val_dataset = sp.load_npz(VAL_DATA_PATH)\n",
    "        self.test_dataset = sp.load_npz(TEST_DATA_PATH)\n",
    "        \n",
    "    def train_dataloader(self):\n",
    "        return DataLoader(interaction(self.train_dataset), batch_size=self.hparams.batch_size, shuffle=True)\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        return DataLoader(interaction(self.val_dataset), batch_size=self.hparams.batch_size, shuffle=False)\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        return DataLoader(interaction(self.test_dataset), batch_size=self.hparams.batch_size, shuffle=False)\n",
    "    \n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.Adam(self.parameters(),\n",
    "                                     lr=self.hparams.learning_rate,\n",
    "                                     betas=self.hparams.betas,\n",
    "                                     amsgrad=self.hparams.amsgrad)\n",
    "        return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "uxm = sp.load_npz(TRAIN_DATA_PATH)\n",
    "total_users = uxm.shape[0]\n",
    "total_items = uxm.shape[1]\n",
    "del uxm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "INFO:lightning:GPU available: True, used: True\n",
      "No environment variable for node rank defined. Set as 0.\n",
      "WARNING:lightning:No environment variable for node rank defined. Set as 0.\n",
      "CUDA_VISIBLE_DEVICES: [0]\n",
      "INFO:lightning:CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name         | Type      | Params\n",
      "---------------------------------------\n",
      "0 | user_factors | Embedding | 3 M   \n",
      "1 | item_factors | Embedding | 895 K \n",
      "2 | user_biases  | Embedding | 177 K \n",
      "3 | item_biases  | Embedding | 44 K  \n",
      "4 | dropout      | Dropout   | 0     \n",
      "INFO:lightning:\n",
      "  | Name         | Type      | Params\n",
      "---------------------------------------\n",
      "0 | user_factors | Embedding | 3 M   \n",
      "1 | item_factors | Embedding | 895 K \n",
      "2 | user_biases  | Embedding | 177 K \n",
      "3 | item_biases  | Embedding | 44 K  \n",
      "4 | dropout      | Dropout   | 0     \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "157595e7b0924c18a97e59686ae42509",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\u001b[1mProfiler Report\u001b[0m\n",
      "                   mean_duration  total_time\n",
      "on_train_start              0.01        0.01\n",
      "on_epoch_start              0.00        0.00\n",
      "get_train_batch             0.00       24.45\n",
      "on_batch_start              0.00        0.12\n",
      "model_forward               0.00        9.49\n",
      "model_backward              0.00       27.83\n",
      "on_after_backward           0.00        0.03\n",
      "optimizer_step              0.00       10.47\n",
      "on_batch_end                0.00        9.37\n",
      "on_epoch_end                0.00        0.00\n",
      "on_train_end                0.00        0.00\n",
      "\n",
      "Profiler output saved to: C:\\TensorLogs\\AdamUXML\\profile.csv\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hparams = Namespace(**{\n",
    "    'batch_size': 1024,\n",
    "    'learning_rate': 0.001,\n",
    "    'betas': (0.9, 0.999),\n",
    "    'n_factors': 20,\n",
    "    'dropout_p': 0.2, \n",
    "    'sparse': False,\n",
    "    'max_epochs': 5,\n",
    "    'amsgrad': False,\n",
    "    'total_users': total_users,\n",
    "    'total_items': total_items\n",
    "})\n",
    "\n",
    "profiler = CSVProfiler(output_path=PROFILE_PATH,verbose=True)\n",
    "logger = TensorBoardLogger(LOG_DIR, name=NAME)\n",
    "model = AdamUXML(hparams)\n",
    "trainer = pl.Trainer(max_epochs=hparams.max_epochs,\n",
    "                     benchmark=True,\n",
    "                     profiler=profiler,\n",
    "                     logger=logger,\n",
    "                     gpus=1,\n",
    "                     fast_dev_run=False,\n",
    "                     callbacks=[TestingCallbacks()])                \n",
    "trainer.fit(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1e51f2134944706a45ac77929a47129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Testing', layout=Layout(flex='2'), max=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG:root:Non-zero values in prediction matrix: 552,255\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Mean Squared Error (MSE): 4.861913204193115\n",
      "--------------------------------------------------------------------------------\n",
      "TEST RESULTS\n",
      "{'MSE': tensor(4.8619, device='cuda:0'),\n",
      " 'avg_test_loss': tensor(4.8619, device='cuda:0')}\n",
      "--------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainer.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(model.state_dict(), SAVE_PATH)\n",
    "# loaded_model = AdamUXML(hparams)\n",
    "# loaded_model.load_state_dict(torch.load(SAVE_PATH))\n",
    "# loaded_model.eval()\n",
    "# print(\"Model's state_dict:\")\n",
    "# for param_tensor in loaded_model.state_dict():\n",
    "#     print(param_tensor, \"\\t\", loaded_model.state_dict()[param_tensor].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "02dc71560ad04f899a91ae87312460f1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "0431fd1dd77c48a586ae86155881e694": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "12577d9b9055490a9d56f1e5072e1564": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "14f538bc0a0c47feb6dea5e15a7325b7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "description": "Epoch 5: 100%",
       "layout": "IPY_MODEL_e4367483b9ad4f11a0bc89774cba0413",
       "max": 3063,
       "style": "IPY_MODEL_ee4c5fb8f4554696ab00a4289e8e4a9b",
       "value": 3063
      }
     },
     "157595e7b0924c18a97e59686ae42509": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_14f538bc0a0c47feb6dea5e15a7325b7",
        "IPY_MODEL_2789f0a71362489b9f86c0c87bc9df08"
       ],
       "layout": "IPY_MODEL_12577d9b9055490a9d56f1e5072e1564"
      }
     },
     "1e4229e9a8de41289cdaeb059aec86aa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "20804503ef4c48039dc605c7d42fa69b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "23128470cdb3447986faa0cf06c27b7e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "info",
       "description": "Validating: 100%",
       "layout": "IPY_MODEL_02dc71560ad04f899a91ae87312460f1",
       "max": 1,
       "style": "IPY_MODEL_e02c7bb144c24fba97d0e9f83f05dcd5",
       "value": 1
      }
     },
     "2789f0a71362489b9f86c0c87bc9df08": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_492dfd17b3304171987bfa5d398b299b",
       "style": "IPY_MODEL_9505218134144c51a92091c69b05ce99",
       "value": " 3063/3063 [00:19&lt;00:00, 160.37it/s, loss=5.851, v_num=3]"
      }
     },
     "2ca85f84278b462aa04dc34e4e9b172f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "3ca81b0da1194873bf7ffe19cd6fdc4e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "3cb5fac86748470ea14eede7b90599ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "description": "Testing: 100%",
       "layout": "IPY_MODEL_bd213b9f54dd4a2ebfb66a75c37d9b59",
       "max": 1,
       "style": "IPY_MODEL_92ec906a7a0d451e9dbc43260275ba37",
       "value": 1
      }
     },
     "407713ff9d5745db87387db08ba82e5b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "info",
       "description": "Validating: 100%",
       "layout": "IPY_MODEL_2ca85f84278b462aa04dc34e4e9b172f",
       "max": 1,
       "style": "IPY_MODEL_fd581e929c2c444795e575b1ea9310c7",
       "value": 1
      }
     },
     "40d7ba04c05349a2a5c4278b6216371e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "467974d6199f4471be4a09945553c63b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_9af284959f664bc38b78cc81011c9b99",
       "style": "IPY_MODEL_583562e1767c46a998dfe3c52ecb92ad",
       "value": " 540/540 [00:05&lt;00:00, 93.77it/s]"
      }
     },
     "492dfd17b3304171987bfa5d398b299b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "50fc703cddc14ddc9b4c1bdde3535073": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "56a6c3a933de4623be71badd26af2d01": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "583562e1767c46a998dfe3c52ecb92ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "61b9f878cd1a415ca80679b83a8b4504": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "6397ea43f9b742ebb5b587667e3c5841": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_844f5405a2074dc681e2402eabbfad65",
       "style": "IPY_MODEL_8a8288d8e6f540da957a266866001fe2",
       "value": " 541/541 [00:01&lt;00:00, 506.37it/s]"
      }
     },
     "67495a762f38436bad02e2109e923e1a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_c9c8ca5d67374e27a894dc1305824c3c",
       "style": "IPY_MODEL_da91a69c40eb433ea9cac1d50b4b6d84",
       "value": " 2/2 [00:00&lt;00:00, 46.61it/s]"
      }
     },
     "6851333ec93c42c480967fe8d3776305": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_db57f24ed70d45bca641b112eadaae72",
       "style": "IPY_MODEL_ab90f999bc9540fea88b3e7cc3d39749",
       "value": " 541/541 [00:01&lt;00:00, 511.10it/s]"
      }
     },
     "6b1d9b56830d43ea9765cb994018749d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "7afeaa12146246e38c025e18e6a361f7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "844f5405a2074dc681e2402eabbfad65": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "87efb0a210464ea080c667c3d22c925c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_61b9f878cd1a415ca80679b83a8b4504",
       "style": "IPY_MODEL_3ca81b0da1194873bf7ffe19cd6fdc4e",
       "value": " 541/541 [00:01&lt;00:00, 500.82it/s]"
      }
     },
     "88d90741019f4c44b1586851d05b1e75": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "8a8288d8e6f540da957a266866001fe2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "8bc3f1d8ac5d4446900fd301e134dc6e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "info",
       "description": "Validation sanity check: 100%",
       "layout": "IPY_MODEL_ba82a0c5b45341518bc1fc7fb192361b",
       "max": 1,
       "style": "IPY_MODEL_0431fd1dd77c48a586ae86155881e694",
       "value": 1
      }
     },
     "8f08f52015374edb85ac168a2472061e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "info",
       "description": "Validating: 100%",
       "layout": "IPY_MODEL_d3a4723545cd43ae94708c00359fb42f",
       "max": 1,
       "style": "IPY_MODEL_50fc703cddc14ddc9b4c1bdde3535073",
       "value": 1
      }
     },
     "907d953c622f45d6bd06b13fe67d1e7e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "92ec906a7a0d451e9dbc43260275ba37": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "9505218134144c51a92091c69b05ce99": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "9af284959f664bc38b78cc81011c9b99": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "9e6677cceb7e40ca8b7e9ea787c73496": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "aab574e7c28b441694fdeb5589e2f9f5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "ab90f999bc9540fea88b3e7cc3d39749": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b0c704627c5c49ddb704720d004ae35c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "ba82a0c5b45341518bc1fc7fb192361b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "bd213b9f54dd4a2ebfb66a75c37d9b59": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "be6ab0faeb78469582a9d8c6bc08d4d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "info",
       "description": "Validating: 100%",
       "layout": "IPY_MODEL_907d953c622f45d6bd06b13fe67d1e7e",
       "max": 1,
       "style": "IPY_MODEL_1e4229e9a8de41289cdaeb059aec86aa",
       "value": 1
      }
     },
     "c1e51f2134944706a45ac77929a47129": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_3cb5fac86748470ea14eede7b90599ad",
        "IPY_MODEL_467974d6199f4471be4a09945553c63b"
       ],
       "layout": "IPY_MODEL_b0c704627c5c49ddb704720d004ae35c"
      }
     },
     "c89aa46320e44c9f83f8a81af926745b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_d265e733c98a4d5f8555c45dfe00ec4e",
       "style": "IPY_MODEL_7afeaa12146246e38c025e18e6a361f7",
       "value": " 541/541 [00:01&lt;00:00, 500.38it/s]"
      }
     },
     "c9353e29d5914eb882bf06df27ac9c4b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "c9c8ca5d67374e27a894dc1305824c3c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d050e00858754bf3994db832a993d24e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d265e733c98a4d5f8555c45dfe00ec4e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "d2c100281ca9443cbd88fb97364a9663": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "info",
       "description": "Validating: 100%",
       "layout": "IPY_MODEL_88d90741019f4c44b1586851d05b1e75",
       "max": 1,
       "style": "IPY_MODEL_aab574e7c28b441694fdeb5589e2f9f5",
       "value": 1
      }
     },
     "d3a4723545cd43ae94708c00359fb42f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "da91a69c40eb433ea9cac1d50b4b6d84": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "db57f24ed70d45bca641b112eadaae72": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e02c7bb144c24fba97d0e9f83f05dcd5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "e1f0aa670a93497e83361e0af09e6181": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "display": "inline-flex",
       "flex_flow": "row wrap",
       "width": "100%"
      }
     },
     "e4367483b9ad4f11a0bc89774cba0413": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "2"
      }
     },
     "ee4c5fb8f4554696ab00a4289e8e4a9b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     },
     "f92b7186c3a04286abaae2e46b0031e3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_d050e00858754bf3994db832a993d24e",
       "style": "IPY_MODEL_56a6c3a933de4623be71badd26af2d01",
       "value": " 541/541 [00:01&lt;00:00, 501.32it/s]"
      }
     },
     "fd581e929c2c444795e575b1ea9310c7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": "initial"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}