CFNet_v8working.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CFNet v8working.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "kGeu9kx2xZTG",
        "_LrsNBEyarjQ",
        "fpVrFae3bQxt",
        "W1Lc1E-AbvIM",
        "l6Kph10Ab9LY"
      ],
      "authorship_tag": "ABX9TyNDhk/nbAwEWOm+/K98zbKp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AdityaSamantaroy/Recommendation-Engine/blob/master/CFNet_v8working.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J64RgrLPsVPD",
        "colab_type": "text"
      },
      "source": [
        "refactored. tuned. hparam. hitrate has bug. added goals"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MmDrXVsxYcyg",
        "colab_type": "text"
      },
      "source": [
        "# Imports, loading datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7FEcBdnVFAc8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "from zipfile import ZipFile\n",
        "\n",
        "\n",
        "from pathlib import Path\n",
        "from tensorflow import keras\n",
        "# from tf.keras.layers import Embedding\n",
        "# import tensorflow_datasets as tfds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qz6I55nNMFeO",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HFeBM4cIFD_T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "ad4383d5-f1a6-4741-9ac0-99f159f1eb53"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "#mount gdrive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cdostPx4sSo_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "million = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iLPDMK1DF3fT",
        "colab_type": "text"
      },
      "source": [
        "Mount Google Drive and create necessary directories"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8twaAnRXFIQ3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if million == False:\n",
        "\n",
        "  movielens_data_file_url = (\n",
        "      \"http://files.grouplens.org/datasets/movielens/ml-latest-small.zip\"\n",
        "  )\n",
        "\n",
        "  # movielens_data_file_url_1m = (\n",
        "  #     \"http://files.grouplens.org/datasets/movielens/ml-1m.zip\"\n",
        "  # )\n",
        "  # \"http://files.grouplens.org/datasets/movielens/ml-100k.zip\"\n",
        "\n",
        "  base_dir =  \"/content/gdrive/My Drive/Machine Learning/movielens/\" #\"/content/gdrive/My Drive/Machine Learning/movielens1m/\"\n",
        "  if not os.path.isdir(base_dir):\n",
        "    os.makedirs(base_dir)\n",
        "\n",
        "  if not os.path.isfile(base_dir + \"data.zip\"):\n",
        "    dataset_zip = tf.keras.utils.get_file(\n",
        "        fname=base_dir + \"data.zip\", \n",
        "        origin=movielens_data_file_url, \n",
        "        # extract=True,\n",
        "    )\n",
        "    print(dataset_zip)\n",
        "\n",
        "  if not os.path.isdir(base_dir + \"data/\"):\n",
        "    !unzip \"/content/gdrive/My Drive/Machine Learning/movielens/data.zip\" -d \"/content/gdrive/My Drive/Machine Learning/movielens/data\"\n",
        "\n",
        "  root_dir = base_dir +  \"data/ml-latest-small/\" #\"data/ml-1m/\"\n",
        "  ratings_dir = root_dir + \"ratings.csv\"\n",
        "  movies_dir = root_dir + \"movies.csv\"\n",
        "\n",
        "\n",
        "\n",
        "else:\n",
        "  movielens_data_file_url_1m = (\n",
        "      \"http://files.grouplens.org/datasets/movielens/ml-1m.zip\"\n",
        "  )\n",
        "  # movielens_data_file_url = (\n",
        "  #     \"http://files.grouplens.org/datasets/movielens/ml-latest-small.zip\"\n",
        "  # )\n",
        "  # \"http://files.grouplens.org/datasets/movielens/ml-100k.zip\"\n",
        "\n",
        "  base_dir =  \"/content/gdrive/My Drive/Machine Learning/movielens1m/\"#\"/content/gdrive/My Drive/Machine Learning/movielens/\" #\n",
        "  if not os.path.isdir(base_dir):\n",
        "    os.makedirs(base_dir)\n",
        "\n",
        "  if not os.path.isfile(base_dir + \"data.zip\"):\n",
        "    dataset_zip = tf.keras.utils.get_file(\n",
        "        fname=base_dir + \"data.zip\", \n",
        "        origin=movielens_data_file_url_1m, \n",
        "        # extract=True,\n",
        "    )\n",
        "    print(dataset_zip)\n",
        "\n",
        "  if not os.path.isdir(base_dir + \"data/\"):\n",
        "    !unzip \"/content/gdrive/My Drive/Machine Learning/movielens1m/data.zip\" -d \"/content/gdrive/My Drive/Machine Learning/movielens1m/data\"\n",
        "\n",
        "  root_dir = base_dir +  \"data/ml-1m/\"#\"data/ml-latest-small/\" #\n",
        "  ratings_dir = root_dir + \"ratings.dat\"\n",
        "  movies_dir = root_dir + \"movies.dat\"\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ToDscWgrDhT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Irlura-EGAP8",
        "colab_type": "text"
      },
      "source": [
        "Load the datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bik8HodsFMVE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "f558c23c-1e0d-4077-989d-0ff4d105679e"
      },
      "source": [
        "if million == False:\n",
        "  df_ratings = pd.read_csv(ratings_dir)\n",
        "  # df_ratings\n",
        "else:\n",
        "  df_ratings = pd.read_csv(ratings_dir, delimiter='::', header = None)\n",
        "  df_ratings.columns = ['userId', 'movieId', 'rating', 'timestamp']\n",
        "  # df_ratings"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lN6v6GPkuUi9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "outputId": "d7fcec4a-98f2-4b9b-ddf4-fb0dc1381e36"
      },
      "source": [
        "df_ratings"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>userId</th>\n",
              "      <th>movieId</th>\n",
              "      <th>rating</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1193</td>\n",
              "      <td>5</td>\n",
              "      <td>978300760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>661</td>\n",
              "      <td>3</td>\n",
              "      <td>978302109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>914</td>\n",
              "      <td>3</td>\n",
              "      <td>978301968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>3408</td>\n",
              "      <td>4</td>\n",
              "      <td>978300275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>2355</td>\n",
              "      <td>5</td>\n",
              "      <td>978824291</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1000204</th>\n",
              "      <td>6040</td>\n",
              "      <td>1091</td>\n",
              "      <td>1</td>\n",
              "      <td>956716541</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1000205</th>\n",
              "      <td>6040</td>\n",
              "      <td>1094</td>\n",
              "      <td>5</td>\n",
              "      <td>956704887</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1000206</th>\n",
              "      <td>6040</td>\n",
              "      <td>562</td>\n",
              "      <td>5</td>\n",
              "      <td>956704746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1000207</th>\n",
              "      <td>6040</td>\n",
              "      <td>1096</td>\n",
              "      <td>4</td>\n",
              "      <td>956715648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1000208</th>\n",
              "      <td>6040</td>\n",
              "      <td>1097</td>\n",
              "      <td>4</td>\n",
              "      <td>956715569</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000209 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         userId  movieId  rating  timestamp\n",
              "0             1     1193       5  978300760\n",
              "1             1      661       3  978302109\n",
              "2             1      914       3  978301968\n",
              "3             1     3408       4  978300275\n",
              "4             1     2355       5  978824291\n",
              "...         ...      ...     ...        ...\n",
              "1000204    6040     1091       1  956716541\n",
              "1000205    6040     1094       5  956704887\n",
              "1000206    6040      562       5  956704746\n",
              "1000207    6040     1096       4  956715648\n",
              "1000208    6040     1097       4  956715569\n",
              "\n",
              "[1000209 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VcjCyUnzFrx8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "ce3b5cd7-379f-423a-9ab1-6e8b0faec575"
      },
      "source": [
        "if million == False:\n",
        "  df_movies = pd.read_csv(movies_dir)\n",
        "  # df_movies\n",
        "else:\n",
        "  df_movies = pd.read_csv(movies_dir, delimiter='::', header=None)\n",
        "  df_movies.columns = ['movieId', 'title', 'genres']\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  \"\"\"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pAAD6qTgulVB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "outputId": "e09e63ca-d0ca-45d0-970e-f5584da41531"
      },
      "source": [
        "df_movies"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Toy Story (1995)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>Jumanji (1995)</td>\n",
              "      <td>Adventure|Children's|Fantasy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>Grumpier Old Men (1995)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>Waiting to Exhale (1995)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>Father of the Bride Part II (1995)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3878</th>\n",
              "      <td>3948</td>\n",
              "      <td>Meet the Parents (2000)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3879</th>\n",
              "      <td>3949</td>\n",
              "      <td>Requiem for a Dream (2000)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3880</th>\n",
              "      <td>3950</td>\n",
              "      <td>Tigerland (2000)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3881</th>\n",
              "      <td>3951</td>\n",
              "      <td>Two Family House (2000)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3882</th>\n",
              "      <td>3952</td>\n",
              "      <td>Contender, The (2000)</td>\n",
              "      <td>Drama|Thriller</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>3883 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      movieId                               title                        genres\n",
              "0           1                    Toy Story (1995)   Animation|Children's|Comedy\n",
              "1           2                      Jumanji (1995)  Adventure|Children's|Fantasy\n",
              "2           3             Grumpier Old Men (1995)                Comedy|Romance\n",
              "3           4            Waiting to Exhale (1995)                  Comedy|Drama\n",
              "4           5  Father of the Bride Part II (1995)                        Comedy\n",
              "...       ...                                 ...                           ...\n",
              "3878     3948             Meet the Parents (2000)                        Comedy\n",
              "3879     3949          Requiem for a Dream (2000)                         Drama\n",
              "3880     3950                    Tigerland (2000)                         Drama\n",
              "3881     3951             Two Family House (2000)                         Drama\n",
              "3882     3952               Contender, The (2000)                Drama|Thriller\n",
              "\n",
              "[3883 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lMPe7F4JY6lu",
        "colab_type": "text"
      },
      "source": [
        "# Prepare the data\n",
        "use leave k validation split\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_LTQ9wIDs8p",
        "colab_type": "text"
      },
      "source": [
        "Prepare the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hkzZU7SI8zV5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_train_eval(df_ratings, k):\n",
        "  \"\"\"\n",
        "  returns train set and eval set\n",
        "  \"\"\"\n",
        "  # df_ratings['user_mean_bias'] = user_mean_bias.loc[df_ratings['userId'].values].values                   #comment for v1\n",
        "  # df_ratings['movie_mean_bias'] = movie_mean_bias.loc[df_ratings['movieId'].values].values                   #comment for v1\n",
        "\n",
        "  leftOutPredictions =df_ratings.groupby('userId', as_index = False).apply(lambda x: x.iloc[-k:,:])\n",
        "  \n",
        "  # print(leftOutPredictions)\n",
        "\n",
        "  # df_ratings_train = df_ratings[~(df_ratings.isin(leftOutPredictions.values))]\n",
        "  df_ratings_train = df_ratings.groupby('userId', as_index = False).apply(lambda x: x.iloc[:-k,:])#.reset_index(drop = True,inplace =True)\n",
        "  df_ratings_train.index = pd.RangeIndex(len(df_ratings_train.index))\n",
        "\n",
        "  # display(df_ratings_train)\n",
        "\n",
        "  return df_ratings_train, leftOutPredictions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hJ6fqGVQQA82",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def prepare_data_leftout(df_ratings, k=20):\n",
        "  # x = df_ratings[['userId', 'movieId']]\n",
        "  # y = df_ratings[['rating']]\n",
        "\n",
        "  # y_min = min(y.values)\n",
        "  # y_max = max(y.values)\n",
        "\n",
        "  # y = (y - y_min)/(y_max - y_min)\n",
        "\n",
        "  # # global_avg = (global_avg - y_min)/(y_max - y_min)\n",
        "\n",
        "  # user_mean_bias = df_ratings.groupby('userId')['rating'].agg('mean')\n",
        "  # movie_mean_bias = df_ratings.groupby('movieId')['rating'].agg('mean')\n",
        "  # user_mean_bias = (user_mean_bias- y_min)/(y_max - y_min)# - global_avg\n",
        "  # movie_mean_bias = (movie_mean_bias- y_min)/(y_max - y_min)#- global_avg\n",
        "  # # print(user_mean_bias)\n",
        "  # # print(movie_mean_bias)\n",
        "\n",
        "\n",
        "  # split_index = int(0.9*df_ratings.shape[0]) \n",
        "\n",
        "  # x_train, x_test, y_train, y_test = (x.iloc[:split_index,:], x.iloc[split_index:,:], y.iloc[:split_index,:], y.iloc[split_index:,:])\n",
        "\n",
        "  \n",
        "  # x['user_mean_bias'] = user_mean_bias.loc[x['userId'].values].values\n",
        "  # x['movie_mean_bias'] = movie_mean_bias.loc[x['movieId'].values].values\n",
        "\n",
        "  \n",
        "  # x['user_mean_bias'] = user_mean_bias.loc[x['userId'].values].values\n",
        "  # x['movie_mean_bias'] = movie_mean_bias.loc[x['movieId'].values].values\n",
        "\n",
        "  # return x_train, x_test, y_train, y_test, global_avg\n",
        "\n",
        "  #############################################\n",
        "  \n",
        "  y_min = min(df_ratings['rating'].values)\n",
        "  y_max = max(df_ratings['rating'].values)\n",
        "\n",
        "  df_ratings['rating'] = (df_ratings['rating'] - y_min)/ (y_max - y_min)\n",
        "\n",
        "  split_index = int(0.9*df_ratings.shape[0]) \n",
        "  df_ratings_train, df_ratings_test = split_train_eval(df_ratings, k)\n",
        "  # df_ratings_train, df_ratings_test = df_ratings.iloc[:split_index], df_ratings.iloc[split_index:]\n",
        "\n",
        "  global_avg = df_ratings_train['rating'].sum()/df_ratings_train.shape[0] \n",
        "  # print(global_avg)\n",
        "  user_mean_bias = df_ratings_train.groupby('userId')['rating'].agg('mean') - global_avg\n",
        "  movie_mean_bias = df_ratings_train.groupby('movieId')['rating'].agg('mean')  - global_avg\n",
        "\n",
        "  # print(movie_mean_bias)\n",
        "  #########################\n",
        "  # def func(x):\n",
        "  #   x['rating'] = 0\n",
        "  #   return x\n",
        "\n",
        "  # user_test_bias = df_ratings[['userId']].groupby('userId').apply(func)\n",
        "  # movie_test_bias = df_ratings[['movieId']].groupby('movieId').apply(func)\n",
        "  ########################\n",
        "\n",
        "  # user_test_bias['rating'] = 0\n",
        "  # movie_mean_bias['rating'] = 0\n",
        "  # print(  user_test_bias['rating'] )\n",
        "  \n",
        "  # user_mean_bias = user_mean_bias.append(user_test_bias)\n",
        "  # movie_mean_bias = movie_mean_bias.append(movie_test_bias)\n",
        "  \n",
        "  # print(user_mean_bias)\n",
        "\n",
        "  # user_mean_bias = user_mean_bias.reindex(, fill_value=0) #user_test_bias.index\n",
        "  # movie_mean_bias = movie_mean_bias.reindex(range(df_ratings['movieId'].unique().max()+1), fill_value=0) #movie_test_bias.index\n",
        "  \n",
        "  # user_mean_bias = pd.DataFrame(range(df_ratings['userId'].unique().max()+1)).join(user_mean_bias, how = 'outer').fillna(0)\n",
        "  # movie_mean_bias = pd.DataFrame(range(df_ratings['movieId'].unique().max()+1)).join(movie_mean_bias, how = 'outer').fillna(0)\n",
        "  \n",
        "  all_user_mean_bias = pd.Series(np.zeros(df_ratings['userId'].unique().max()+1))#, columns = ['userId']\n",
        "  all_movie_mean_bias = pd.Series(np.zeros(df_ratings['movieId'].unique().max()+1))#, columns = ['movieId']\n",
        "\n",
        "  all_user_mean_bias.update(user_mean_bias)#, overwrite = True\n",
        "  all_movie_mean_bias.update(movie_mean_bias)#, overwrite = True\n",
        "  \n",
        "  user_mean_bias = all_user_mean_bias\n",
        "  movie_mean_bias = all_movie_mean_bias\n",
        "  \n",
        "  # print(user_mean_bias.loc[df_ratings_train['userId'].values].shape, df_ratings_train['user_mean_bias'].shape )\n",
        "\n",
        "  df_ratings_train['user_mean_bias'] = user_mean_bias.loc[df_ratings_train['userId'].values].values\n",
        "  df_ratings_train['movie_mean_bias'] = movie_mean_bias.loc[df_ratings_train['movieId'].values].values\n",
        "\n",
        "  df_ratings_test['user_mean_bias'] = user_mean_bias.loc[df_ratings_test['userId'].values].values\n",
        "  df_ratings_test['movie_mean_bias'] = movie_mean_bias.loc[df_ratings_test['movieId'].values].values\n",
        "\n",
        "  # print(df_ratings_train['user_mean_bias'])\n",
        "\n",
        "  # df_ratings_train, df_ratings_test = df_ratings.iloc[:split_index], df_ratings.iloc[split_index:]\n",
        "\n",
        "  x_train, x_test, y_train, y_test = df_ratings_train[['userId', 'movieId','user_mean_bias','movie_mean_bias']], df_ratings_test[['userId', 'movieId','user_mean_bias','movie_mean_bias']], df_ratings_train[['rating']], df_ratings_test[['rating']]\n",
        "  \n",
        "  return x_train, x_test, y_train, y_test, global_avg"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F7-_u1UpJecU",
        "colab_type": "text"
      },
      "source": [
        "Ensure Data doesnt leak"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_Lb3zRtULmN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train, x_test, y_train, y_test, global_avg = prepare_data_leftout(df_ratings, 10)\n",
        "num_users = df_ratings['userId'].unique().max() +1\n",
        "num_movies = df_ratings['movieId'].unique().max() +1\n",
        "  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j7NWE0bQolv6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "outputId": "78446164-c92a-49ad-f803-008306f2635b"
      },
      "source": [
        "x_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>userId</th>\n",
              "      <th>movieId</th>\n",
              "      <th>user_mean_bias</th>\n",
              "      <th>movie_mean_bias</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1193</td>\n",
              "      <td>0.148187</td>\n",
              "      <td>0.205220</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>661</td>\n",
              "      <td>0.148187</td>\n",
              "      <td>-0.027348</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>914</td>\n",
              "      <td>0.148187</td>\n",
              "      <td>0.146011</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>3408</td>\n",
              "      <td>0.148187</td>\n",
              "      <td>0.073433</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>2355</td>\n",
              "      <td>0.148187</td>\n",
              "      <td>0.071492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>939804</th>\n",
              "      <td>6040</td>\n",
              "      <td>1077</td>\n",
              "      <td>-0.001272</td>\n",
              "      <td>0.088374</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>939805</th>\n",
              "      <td>6040</td>\n",
              "      <td>1079</td>\n",
              "      <td>-0.001272</td>\n",
              "      <td>0.106344</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>939806</th>\n",
              "      <td>6040</td>\n",
              "      <td>549</td>\n",
              "      <td>-0.001272</td>\n",
              "      <td>0.051664</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>939807</th>\n",
              "      <td>6040</td>\n",
              "      <td>2020</td>\n",
              "      <td>-0.001272</td>\n",
              "      <td>0.105144</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>939808</th>\n",
              "      <td>6040</td>\n",
              "      <td>2021</td>\n",
              "      <td>-0.001272</td>\n",
              "      <td>-0.069554</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>939809 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        userId  movieId  user_mean_bias  movie_mean_bias\n",
              "0            1     1193        0.148187         0.205220\n",
              "1            1      661        0.148187        -0.027348\n",
              "2            1      914        0.148187         0.146011\n",
              "3            1     3408        0.148187         0.073433\n",
              "4            1     2355        0.148187         0.071492\n",
              "...        ...      ...             ...              ...\n",
              "939804    6040     1077       -0.001272         0.088374\n",
              "939805    6040     1079       -0.001272         0.106344\n",
              "939806    6040      549       -0.001272         0.051664\n",
              "939807    6040     2020       -0.001272         0.105144\n",
              "939808    6040     2021       -0.001272        -0.069554\n",
              "\n",
              "[939809 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kGeu9kx2xZTG",
        "colab_type": "text"
      },
      "source": [
        "# Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_sFxEWn1xft3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "hparam = {}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DEydgpAxetA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "hparam['lr'] = 0.001\n",
        "hparam['embeddings_initializer'] = 'glorot_uniform'\n",
        "hparam['embedding_regularization'] = 1e-6\n",
        "hparam['embedding_size'] = 30\n",
        "hparam['pure_CF'] = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_LrsNBEyarjQ",
        "colab_type": "text"
      },
      "source": [
        "# Prepare the Models  \n",
        "a. Bias Recommender  \n",
        "b. Bias + Collaborative Filtering (Matrix Factorisation)  \n",
        "c. Pure Collaborative Filtering "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BRHSToYTKdVY",
        "colab_type": "text"
      },
      "source": [
        "Define the models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Fz7PLtbKcqa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import keras.backend as K\n",
        "\n",
        "class CFRecommender(tf.keras.Model):\n",
        "  def __init__(self, num_users, num_movies, global_avg, hparam):\n",
        "    super().__init__()#CFRecommender, self\n",
        "    self.num_users = num_users\n",
        "    self.num_movies = num_movies\n",
        "    # self.embedding_size = \n",
        "    self.global_avg = global_avg\n",
        "    # self.user_mean_bias = user_mean_bias\n",
        "    # self.movie_mean_bias = movie_mean_bias\n",
        "\n",
        "    self.user_embedding = tf.keras.layers.Embedding(self.num_users,hparam['embedding_size'],embeddings_regularizer=keras.regularizers.l2(hparam['embedding_regularization']),embeddings_initializer=hparam['embeddings_initializer'])\n",
        "    self.user_bias = tf.keras.layers.Embedding(self.num_users, 1,embeddings_initializer=hparam['embeddings_initializer']) #, self.embedding_size embeddings_regularizer=keras.regularizers.l2(1e-6),\n",
        "\n",
        "    self.movie_embedding = tf.keras.layers.Embedding(self.num_movies, hparam['embedding_size'],embeddings_regularizer=keras.regularizers.l2(hparam['embedding_regularization']),embeddings_initializer=hparam['embeddings_initializer'])\n",
        "    self.movie_bias = tf.keras.layers.Embedding(self.num_movies, 1,embeddings_initializer=hparam['embeddings_initializer']) #, self.embedding_size\n",
        "\n",
        "  def call(self, inputs): #input: dim1 x 2\n",
        "\n",
        "    user_vector = self.user_embedding(inputs[:,0]) #dim1 x embedding_size\n",
        "    user_bias = self.user_bias(inputs[:,0]) #dim1 x 1\n",
        "\n",
        "    movie_vector = self.movie_embedding(inputs[:,1]) #dim1 x embedding_size\n",
        "    movie_bias = self.movie_bias(inputs[:,1]) #dim1 x 1\n",
        "\n",
        "    mu = self.global_avg\n",
        "\n",
        "    # user_avg = self.user_mean_bias.loc[inputs[:,0]].values\n",
        "    # movie_avg = self.movie_mean_bias.loc[inputs[:,1]].values\n",
        "    user_avg = inputs[:,2] #dim1 x 1\n",
        "    movie_avg = inputs[:,3] #dim1 x 1\n",
        "\n",
        "    \"\"\"\n",
        "    user_vector, movie_vector : (batch_size, embedding_dim)\n",
        "    \"\"\"\n",
        "    dot_user_movie = tf.reduce_sum(tf.math.multiply(user_vector,movie_vector), axis = 1) #tf.math.l2_normalize(user_vector,0) # tf.math.l2_normalize(movie_vector,0)  #dim1 x 1\n",
        "    # print('test0')\n",
        "    # print(K.int_shape(user_avg[:,np.newaxis]), K.int_shape(user_bias))\n",
        "    # print(K.int_shape(dot_user_movie), K.int_shape(tf.math.multiply(user_avg[:,np.newaxis],user_bias)),K.int_shape(tf.math.multiply(movie_avg[:,np.newaxis],movie_bias)) )\n",
        "    # print('test1')\n",
        "    if hparam['pure_CF']:\n",
        "      c_ratings = dot_user_movie\n",
        "    else:\n",
        "      c_ratings = dot_user_movie + tf.math.multiply(user_avg,user_bias) + tf.math.multiply(movie_avg,movie_bias) + mu\n",
        "    # print('test2')\n",
        "    return tf.nn.sigmoid(dot_user_movie)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IFDoX8DBLcMe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# import keras.backend as K\n",
        "\n",
        "class CFpure(tf.keras.Model):\n",
        "  def __init__(self, num_users, num_movies, global_avg, hparam):\n",
        "    super().__init__()#CFRecommender, self\n",
        "    self.num_users = num_users\n",
        "    self.num_movies = num_movies\n",
        "    self.embedding_size = embedding_size\n",
        "    self.global_avg = global_avg\n",
        "    # self.user_mean_bias = user_mean_bias\n",
        "    # self.movie_mean_bias = movie_mean_bias\n",
        "\n",
        "    self.user_embedding = tf.keras.layers.Embedding(self.num_users,self.embedding_size,embeddings_regularizer=keras.regularizers.l2(1e-3),embeddings_initializer=\"glorot_uniform\")\n",
        "    self.user_bias = tf.keras.layers.Embedding(self.num_users, 1,embeddings_initializer=\"glorot_uniform\") #, self.embedding_size embeddings_regularizer=keras.regularizers.l2(1e-6),\n",
        "\n",
        "    self.movie_embedding = tf.keras.layers.Embedding(self.num_movies, self.embedding_size,embeddings_regularizer=keras.regularizers.l2(1e-3),embeddings_initializer=\"glorot_uniform\")\n",
        "    self.movie_bias = tf.keras.layers.Embedding(self.num_movies, 1,embeddings_initializer=\"glorot_uniform\") #, self.embedding_size\n",
        "\n",
        "  def call(self, inputs): #input: dim1 x 2\n",
        "\n",
        "    user_vector = self.user_embedding(inputs[:,0]) #dim1 x embedding_size\n",
        "    user_bias = self.user_bias(inputs[:,0]) #dim1 x 1\n",
        "\n",
        "    movie_vector = self.movie_embedding(inputs[:,1]) #dim1 x embedding_size\n",
        "    movie_bias = self.movie_bias(inputs[:,1]) #dim1 x 1\n",
        "\n",
        "    mu = self.global_avg\n",
        "\n",
        "    # user_avg = self.user_mean_bias.loc[inputs[:,0]].values\n",
        "    # movie_avg = self.movie_mean_bias.loc[inputs[:,1]].values\n",
        "    user_avg = inputs[:,2] #dim1 x 1\n",
        "    movie_avg = inputs[:,3] #dim1 x 1\n",
        "\n",
        "    \"\"\"\n",
        "    user_vector, movie_vector : (batch_size, embedding_dim)\n",
        "    \"\"\"\n",
        "    dot_user_movie = tf.reduce_sum(tf.math.multiply(user_vector,movie_vector), axis = 1) #tf.math.l2_normalize(user_vector,0) # tf.math.l2_normalize(movie_vector,0)  #dim1 x 1\n",
        "    # print('test0')\n",
        "    # print(K.int_shape(user_avg[:,np.newaxis]), K.int_shape(user_bias))\n",
        "    # print(K.int_shape(dot_user_movie), K.int_shape(tf.math.multiply(user_avg[:,np.newaxis],user_bias)),K.int_shape(tf.math.multiply(movie_avg[:,np.newaxis],movie_bias)) )\n",
        "    # print('test1')\n",
        "    c_ratings = dot_user_movie + user_bias + movie_bias #+ tf.math.multiply(user_avg,user_bias) + tf.math.multiply(movie_avg,movie_bias) + mu\n",
        "    # print('test2')\n",
        "    return tf.nn.sigmoid(dot_user_movie)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M_7hnzobPV3K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# Baseline\t0.188850\t0.215598\n",
        "#\tCF\t0.117963\t0.255645\n",
        "\n",
        "# num_users = df_ratings['userId'].unique().max() +1\n",
        "# num_movies = df_ratings['movieId'].unique().max() +1\n",
        "# EMBEDDING_SIZE = 50\n",
        "\n",
        "class BiasRecommender(tf.keras.Model):\n",
        "  def __init__(self, global_avg): #, num_users, num_movies, embedding_size\n",
        "    super().__init__()#CFRecommender, self\n",
        "    # self.num_users = num_users\n",
        "    # self.num_movies = num_movies\n",
        "    # self.embedding_size = embedding_size\n",
        "    self.global_avg = global_avg\n",
        "    # self.user_mean_bias = user_mean_bias\n",
        "    # self.movie_mean_bias = movie_mean_bias\n",
        "\n",
        "    # self.user_embedding = tf.keras.layers.Embedding(self.num_users,self.embedding_size,embeddings_regularizer=keras.regularizers.l2(1e-6),embeddings_initializer=\"he_normal\")\n",
        "    # self.user_bias = tf.keras.layers.Embedding(self.num_users, 1,embeddings_initializer=\"he_normal\") #, self.embedding_size embeddings_regularizer=keras.regularizers.l2(1e-6),\n",
        "\n",
        "    # self.movie_embedding = tf.keras.layers.Embedding(self.num_movies, self.embedding_size,embeddings_regularizer=keras.regularizers.l2(1e-6),embeddings_initializer=\"he_normal\")\n",
        "    # self.movie_bias = tf.keras.layers.Embedding(self.num_movies, 1,embeddings_initializer=\"he_normal\") #, self.embedding_size\n",
        "\n",
        "  def call(self, inputs): #input: dim1 x 2\n",
        "\n",
        "    # user_vector = self.user_embedding(inputs[:,0]) #dim1 x embedding_size\n",
        "    # user_bias = self.user_bias(inputs[:,0]) #dim1 x 1\n",
        "\n",
        "    # movie_vector = self.movie_embedding(inputs[:,1]) #dim1 x embedding_size\n",
        "    # movie_bias = self.movie_bias(inputs[:,1]) #dim1 x 1\n",
        "\n",
        "    mu = self.global_avg\n",
        "\n",
        "    # user_avg = self.user_mean_bias.loc[inputs[:,0]].values\n",
        "    # movie_avg = self.movie_mean_bias.loc[inputs[:,1]].values\n",
        "    user_avg = inputs[:,2] #dim1 x 1\n",
        "    movie_avg = inputs[:,3] #dim1 x 1\n",
        "\n",
        "    bias = (user_avg + movie_avg)/2\n",
        "    \"\"\"\n",
        "    user_vector, movie_vector : (batch_size, embedding_dim)\n",
        "    \"\"\"\n",
        "    # dot_user_movie = tf.reduce_sum(tf.math.multiply(user_vector,movie_vector), axis = 1) #tf.math.l2_normalize(user_vector,0) # tf.math.l2_normalize(movie_vector,0)  #dim1 x 1\n",
        "    # print('test0')\n",
        "    # print(K.int_shape(user_avg[:,np.newaxis]), K.int_shape(user_bias))\n",
        "    # print(K.int_shape(dot_user_movie), K.int_shape(tf.math.multiply(user_avg[:,np.newaxis],user_bias)),K.int_shape(tf.math.multiply(movie_avg[:,np.newaxis],movie_bias)) )\n",
        "    # print('test1')\n",
        "    c_ratings = bias + mu\n",
        "    return c_ratings"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hxf65QB-Q6bC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_users = df_ratings['userId'].unique().max() +1\n",
        "num_movies = df_ratings['movieId'].unique().max() +1\n",
        "# EMBEDDING_SIZE = 50\n",
        "\n",
        "\n",
        "def CFpure(num_users = num_users, num_movies=num_movies, global_avg = global_avg,hparam = hparam): \n",
        "  model = CFRecommender(num_users, num_movies, global_avg, hparam)\n",
        "  model.build(input_shape=(None,4))\n",
        "  model.compile(loss = tf.keras.losses.BinaryCrossentropy(), optimizer = keras.optimizers.Adam(hparam['lr']))\n",
        "  model.summary()\n",
        "\n",
        "  return model\n",
        "\n",
        "def CFmodel(num_users = num_users, num_movies=num_movies, global_avg = global_avg,hparam = hparam): \n",
        "  model = CFRecommender(num_users, num_movies, global_avg, hparam)\n",
        "  model.build(input_shape=(None,4))\n",
        "  model.compile(loss = tf.keras.losses.BinaryCrossentropy(), optimizer = keras.optimizers.Adam(hparam['lr']))\n",
        "  model.summary()\n",
        "\n",
        "  return model\n",
        "\n",
        "def Basemodel(global_avg = global_avg):\n",
        "  model = BiasRecommender(global_avg)\n",
        "  return model\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZzE5PAdxPcs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CFpure()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fpVrFae3bQxt",
        "colab_type": "text"
      },
      "source": [
        "# Tune the CF model's learning rate \n",
        "lr = 0.002"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WIENG6KxTo5i",
        "colab_type": "text"
      },
      "source": [
        "Tune Learning Rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdYPizD6TlNm",
        "colab_type": "code",
        "cellView": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 796
        },
        "outputId": "b0db6cfe-d40c-48ee-b1fe-9555dea0b5ce"
      },
      "source": [
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import keras.backend as K\n",
        "from keras.callbacks import Callback\n",
        "\n",
        "class LRFinder(tf.keras.callbacks.Callback):\n",
        "    \n",
        "    '''\n",
        "    A simple callback for finding the optimal learning rate range for your model + dataset. \n",
        "    \n",
        "    # Usage\n",
        "        ```python\n",
        "            lr_finder = LRFinder(min_lr=1e-5, \n",
        "                                 max_lr=1e-2, \n",
        "                                 steps_per_epoch=np.ceil(epoch_size/batch_size), \n",
        "                                 epochs=3)\n",
        "            model.fit(X_train, Y_train, callbacks=[lr_finder])\n",
        "            \n",
        "            lr_finder.plot_loss()\n",
        "        ```\n",
        "    \n",
        "    # Arguments\n",
        "        min_lr: The lower bound of the learning rate range for the experiment.\n",
        "        max_lr: The upper bound of the learning rate range for the experiment.\n",
        "        steps_per_epoch: Number of mini-batches in the dataset. Calculated as `np.ceil(epoch_size/batch_size)`. \n",
        "        epochs: Number of epochs to run experiment. Usually between 2 and 4 epochs is sufficient. \n",
        "        \n",
        "    # References\n",
        "        Blog post: jeremyjordan.me/nn-learning-rate\n",
        "        Original paper: https://arxiv.org/abs/1506.01186\n",
        "    '''\n",
        "    \n",
        "    def __init__(self, min_lr=1e-5, max_lr=1e-2, steps_per_epoch=None, epochs=None):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.min_lr = min_lr\n",
        "        self.max_lr = max_lr\n",
        "        self.total_iterations = steps_per_epoch * epochs\n",
        "        self.iteration = 0\n",
        "        self.history = {}\n",
        "        \n",
        "    def clr(self):\n",
        "        '''Calculate the learning rate.'''\n",
        "        x = self.iteration / self.total_iterations \n",
        "        return self.min_lr + (self.max_lr-self.min_lr) * x\n",
        "        \n",
        "    def on_train_begin(self, logs=None):\n",
        "        '''Initialize the learning rate to the minimum value at the start of training.'''\n",
        "        logs = logs or {}\n",
        "        K.set_value(self.model.optimizer.lr, self.min_lr)\n",
        "        \n",
        "    def on_batch_end(self, epoch, logs=None):\n",
        "        '''Record previous batch statistics and update the learning rate.'''\n",
        "        logs = logs or {}\n",
        "        self.iteration += 1\n",
        "\n",
        "        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n",
        "        self.history.setdefault('iterations', []).append(self.iteration)\n",
        "\n",
        "        for k, v in logs.items():\n",
        "            self.history.setdefault(k, []).append(v)\n",
        "            \n",
        "        K.set_value(self.model.optimizer.lr, self.clr())\n",
        " \n",
        "    def plot_lr(self):\n",
        "        '''Helper function to quickly inspect the learning rate schedule.'''\n",
        "        plt.plot(self.history['iterations'], self.history['lr'])\n",
        "        plt.yscale('log')\n",
        "        plt.xlabel('Iteration')\n",
        "        plt.ylabel('Learning rate')\n",
        "        plt.show()\n",
        "        \n",
        "    def plot_loss(self):\n",
        "        '''Helper function to quickly observe the learning rate experiment results.'''\n",
        "        plt.plot(self.history['lr'], self.history['loss'])\n",
        "        plt.xscale('log')\n",
        "        plt.xlabel('Learning rate')\n",
        "        plt.ylabel('Loss')\n",
        "        plt.show()\n",
        "\n",
        "batch_size = 130\n",
        "tune_model = CFmodel()\n",
        "\n",
        "lr_finder = LRFinder(min_lr=1e-6, \n",
        "                    max_lr=1e-2, \n",
        "                    steps_per_epoch=np.ceil(x_train.shape[0]/batch_size), \n",
        "                    epochs=1)\n",
        "tune_model.fit(x = x_train, y = y_train, verbose = 2, callbacks = [lr_finder], batch_size = batch_size)\n",
        "\n",
        "lr_finder.plot_loss()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"cf_recommender_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_12 (Embedding)     multiple                  302050    \n",
            "_________________________________________________________________\n",
            "embedding_13 (Embedding)     multiple                  6041      \n",
            "_________________________________________________________________\n",
            "embedding_14 (Embedding)     multiple                  197650    \n",
            "_________________________________________________________________\n",
            "embedding_15 (Embedding)     multiple                  3953      \n",
            "=================================================================\n",
            "Total params: 509,694\n",
            "Trainable params: 509,694\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "WARNING:tensorflow:Layer cf_recommender_3 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "6925/6925 - 44s - loss: 0.6334\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3Sc9X3n8fdHkuW7LF9k8N0GZGIZjAFxiwkxaZN1khbSNKHQpE1Os7Bpl/bkZEMXtt1sym4aesvukrpNTDbpJg0Qmk04auLEaQiXhACxDBjfsHFsjGUuko3xBXzXd/+YRzAej6wZW48eaebzOmeO9fye38x854fxR89lfj9FBGZmZoVqsi7AzMwGJweEmZkV5YAwM7OiHBBmZlaUA8LMzIpyQJiZWVF1WRfQXyZNmhSzZ8/OugwzsyFl1apVOyOiqdi+igmI2bNn097ennUZZmZDiqRtve3zKSYzMyvKAWFmZkU5IMzMrCgHhJmZFeWAMDOzohwQZmZWVMXc5nqqDh09xsMbu45rk3T8dsFzCnYft63C3ifZlERdjaiRqKtN/qwRtb20HfcoKKJw0vbCadyLTepeONN7FOklhJSrW8p9upqkoae9Rj199OZYvPln8onf2n7rsx+/fXy7mWWv6gNi/8Gj3PTNVVmXYb0oOVg4vmPh/vw+fb3mia9RXsjR6/uUVgeF/fvzM/RRC3317+Mz0Ov7lPDaJ/lvOKy2hqaxw5k8dgRTG0fQMqWB86aNY8SwWiw9VR8QDSOH8f0/vrLX/X39lp2/v9zf4iOCY91wtLub7p4/Izh6LOjO35fXdrQ76O7O/XnikU3xf2De3C72Afs4Woqkzojkz+QzdyefLSI3Jt3x1s8njFle3/xxeGu7+P6ehlL7F+6nyH+bcms54TOUWQuF+/vzM5zwnOL76e09T/UzlFkHBfvLrSUC9h48ypau1+ncd5Ajx3I7xw6v44MXTeNTvz6X8aPrsf5X9QExrLaG86aNy7oMMytBd3fQue8Qz3S8xo/Wvcw/P/ECP1jzEt/8xGXMm9KQdXkVxxepzWzIqKkRZ44bwXvmn8kXr1vI9//4SupqavjIV59g48v7si6v4jggzGzImjelgXtuupxhteL3/s8T7D90NOuSKooDwsyGtDmTRvPlj15M575DfPVnW7Iup6I4IMxsyLtw5njed/6Z3PXIFrr2Hcq6nIrhgDCzivCZ95zLwaPd/MNDm7MupWKkGhCSlkjaKGmzpFt76XOdpPWS1km6O6/9ryStTR6/k2adZjb0ndU0hvefP4XvPrmDw0e7sy6nIqQWEJJqgaXAe4EW4AZJLQV9moHbgEURMR/4VNL+fuAiYCFwGfAZSb6HzcxO6rcumsaeA0d4aGNn1qVUhDSPIC4FNkfElog4DNwLXFvQ50ZgaUTsBoiInv+qLcAjEXE0Il4HngGWpFirmVWAd5wziYmj67n/6R1Zl1IR0gyIacD2vO2OpC3fXGCupEclPS6pJwRWA0skjZI0CbgamJFirWZWAepqa/jNC6bykw2d7D14JOtyhrysL1LXAc3AYuAG4C5JjRHxY2A58AvgHuAx4FjhkyXdJKldUntXV1fhbjOrQtcunMrho938aO3LWZcy5KUZEDs4/rf+6Ulbvg6gLSKORMRWYBO5wCAiPh8RCyPi3eSmCNpU+AYRsSwiWiOitampKZUPYWZDy8IZjUwZN4KHN/mXxtOVZkCsBJolzZFUD1wPtBX0uZ/c0QPJqaS5wBZJtZImJu0LgAXAj1Os1cwqhCQumT2BlVtfPWHCQCtPagEREUeBm4EVwAbgvohYJ+l2Sdck3VYAuyStBx4EbomIXcAw4GdJ+zLgo8nrmZn16dI5E+jcd4gXXn0j61KGtFRnc42I5eSuJeS3fTbv5wA+nTzy+xwkdyeTmVnZLp0zAYAntr7KrImjM65m6Mr6IrWZWb87p2kM40cNY+XWV7MuZUhzQJhZxampEa2zJ/DL5x0Qp8MBYWYV6dLZE9i26w069x7MupQhywFhZhWp5zqEjyJOnQPCzCrS/KkNjKqv5Ze+DnHKHBBmVpHqamu4aOZ4B8RpcECYWcW6dM4ENr6yjz1veF6mU+GAMLOKdcnsCURA+zYfRZwKB4SZVawLZzZSVyNWbduddSlDkgPCzCrWiGG1nDN5DBte2pt1KUOSA8LMKlrLlAbWOyBOiQPCzCpay9QGXtl7iJ37D2VdypDjgDCzitYyJbec/foXfRRRLgeEmVW0lqlJQPg0U9kcEGZW0RpH1TOtcaSPIE6BA8LMKt48X6g+JQ4IM6t4LVMb2NK1nwOHj2VdypDigDCzitcypYHugI2v7Mu6lCHFAWFmFW/+VN/JdCocEGZW8aaPH8nY4XWsf2lP1qUMKQ4IM6t4kpg3tcFHEGVyQJhZVWiZ0sCzL+/jWHdkXcqQ4YAws6rQMrWBNw4fY9uu17MuZchwQJhZVXhzyg1/H6JkqQaEpCWSNkraLOnWXvpcJ2m9pHWS7s5r/+ukbYOkOyUpzVrNrLI1nzGGuhr5OkQZ6tJ6YUm1wFLg3UAHsFJSW0Ssz+vTDNwGLIqI3ZImJ+1vBxYBC5KuPwfeCTyUVr1mVtmG1+XWhvARROnSPIK4FNgcEVsi4jBwL3BtQZ8bgaURsRsgIjqT9gBGAPXAcGAY8EqKtZpZFWjxnUxlSTMgpgHb87Y7krZ8c4G5kh6V9LikJQAR8RjwIPBS8lgRERsK30DSTZLaJbV3dXWl8iHMrHK0TGmgc98huvZ5bYhSZH2Rug5oBhYDNwB3SWqUdA4wD5hOLlTeJekdhU+OiGUR0RoRrU1NTQNYtpkNRT1Tf3sJ0tKkGRA7gBl529OTtnwdQFtEHImIrcAmcoHxW8DjEbE/IvYDPwSuSLFWM6sCvpOpPGkGxEqgWdIcSfXA9UBbQZ/7yR09IGkSuVNOW4AXgHdKqpM0jNwF6hNOMZmZlcNrQ5QntYCIiKPAzcAKcv+43xcR6yTdLumapNsKYJek9eSuOdwSEbuA7wC/AtYAq4HVEfGvadVqZtXDa0OULrXbXAEiYjmwvKDts3k/B/Dp5JHf5xjwH9KszcyqU8vUBn767CscOHyMkfW1WZczqGV9kdrMbEB5bYjSOSDMrKp4bYjSOSDMrKp4bYjSOSDMrKp4bYjSOSDMrOp4bYjSOCDMrOp4bYjSOCDMrOr4G9WlcUCYWdXx2hClcUCYWdXx2hClcUCYWVXy2hB9c0CYWVXy2hB9c0CYWVXy2hB9c0CYWVXynUx9c0CYWVXy2hB9c0CYWdXy2hAn54Aws6rVMrWBLV37OXD4WNalDEoOCDOrWl4b4uQcEGZWtbw2xMk5IMysanltiJNzQJhZ1fLaECfngDCzqua1IXrngDCzqua1IXrngDCzquZvVPcu1YCQtETSRkmbJd3aS5/rJK2XtE7S3Unb1ZKeznsclPSBNGs1s+rUszbEOl+HOEFdWi8sqRZYCrwb6ABWSmqLiPV5fZqB24BFEbFb0mSAiHgQWJj0mQBsBn6cVq1mVr2G19VydtMYNr7s70IUSvMI4lJgc0RsiYjDwL3AtQV9bgSWRsRugIjoLPI6HwJ+GBFvpFirmVWxt00Zy7M+xXSCNANiGrA9b7sjacs3F5gr6VFJj0taUuR1rgfuKfYGkm6S1C6pvaurq1+KNrPq87YzG3hxz0H2vHEk61IGlawvUtcBzcBi4AbgLkmNPTslTQHOB1YUe3JELIuI1ohobWpqGoByzawS9awNsc5fmDtOmgGxA5iRtz09acvXAbRFxJGI2ApsIhcYPa4DvhcRjnUzS42n3CguzYBYCTRLmiOpntyporaCPveTO3pA0iRyp5y25O2/gV5OL5mZ9ZdJY4ZzRsNw38lUILWAiIijwM3kTg9tAO6LiHWSbpd0TdJtBbBL0nrgQeCWiNgFIGk2uSOQh9Oq0cysx/yp41j3ok8x5UvtNleAiFgOLC9o+2zezwF8OnkUPvd5TryobWaWivlTG3hoYycHjxxjxLDarMsZFLK+SG1mNijMS9aG2Ny5P+tSBg0HhJkZb025sXaHTzP1cECYmQGzJo6icdQw2rftzrqUQcMBYWZGbm2IS2ZP4MkXHBA9HBBmZokLpo9jS9fr7Dngr16BA8LM7E0XzMhN5LCmw9chwAFhZvamBdNyAbG647WMKxkcHBBmZolxo4YxffxINnhmV8ABYWZ2nLed2eDV5RIlBYSk0ZJqkp/nSrpG0rB0SzMzG3g9F6r3HvSF6lKPIB4BRkiaRm5lt98D/imtoszMsrIguVC91heqSw4IJSu6fRD4h4j4MDA/vbLMzLKxYNo4AFY7IEoPCElXAB8BfpC0eTYrM6s440fXM3PCKJ7xnUwlB8SngNvILd6zTtJZ5KbnNjOrOAtnNPLUCw6IkgIiIh6OiGsi4q+Si9U7I+JPUq7NzCwTC6aP4+W9B+nadyjrUjJV6l1Md0tqkDQaWAusl3RLuqWZmWVjwfTchepqP81U6immlojYC3wA+CEwh9ydTGZmFee8aQ3UCFZvd0CUYljyvYcPAG0RcQSI9MoyM8vOqPo65p4xlqer/E6mUgPiK8DzwGjgEUmzAH/V0Mwq1gXTG1m9/TVyKyNXp1IvUt8ZEdMi4n2Rsw24OuXazMwyc/70cew5cISO3QeyLiUzpV6kHifpi5Lak8ffkTuaMDOrSOcnX5ir5iVISz3F9DVgH3Bd8tgLfD2toszMsnbumWOpqxFrqjgg6krsd3ZE/Hbe9l9IejqNgszMBoMRw2ppPmNsVQdEqUcQByRd2bMhaRHQ54k5SUskbZS0WdKtvfS5TtJ6Sesk3Z3XPlPSjyVtSPbPLrFWM7N+sXDGOFZvf43u7uq8UF3qEcQngW9IGpds7wY+drInSKoFlgLvBjqAlZLaImJ9Xp9mclN4LIqI3ZIm573EN4DPR8S/SRoDdJdYq5lZv7h41gTu+eV2nuvcz7lnjs26nAFX6l1MqyPiAmABsCAiLgTe1cfTLgU2R8SWiDgM3AtcW9DnRmBpROxO3qcTQFILUBcR/5a0709mkzUzGzCXzB4PwMrnX824kmyUtaJcROxNvlEN8Ok+uk8DtudtdyRt+eYCcyU9KulxSUvy2l+T9F1JT0n6m+SIxMxswMycMIqmscNpd0CUTf3w/nVAM7AYuAG4S1Jj0v4O4DPAJcBZwMdPKEC6qefW266urn4ox8zsLZJonTWe9m27sy4lE6cTEH1dtdkBzMjbnp605esgmbojIrYCm8gFRgfwdHJ66ihwP3DRCQVELIuI1ohobWpqOtXPYWbWq9bZE+jYfYCX9lTfF+ZOGhCS9knaW+SxD5jax2uvBJolzZFUD1wPtBX0uZ/c0QOSJpE7tbQleW6jpJ5/9d8FrMfMbIC1zspdh2h/vvqOIk4aEBExNiIaijzGRsRJ74BKfvO/GVgBbADuSxYbul3SNUm3FcAuSevJLUB0S0Tsiohj5E4vPSBpDbnTWXed3kc1MyvfvCkN1NfWVOXU36Xe5npKImI5sLyg7bN5Pwe5i90nXPBO7mBakGZ9ZmZ9qa+rYd7UBlZvr74vzJ3ONQgzs6pw4YxG1uzYw9Fj1fV1LAeEmVkfLpzZyIEjx9j4yr6sSxlQDggzsz5cNDN3ofrJF6rrOoQDwsysD9PHj2Ty2OGsqrIvzDkgzMz6IInW2eNZWWW3ujogzMxK0DprAjteq64vzDkgzMxK0Dq7+r4w54AwMytBy5QGRtXXVtXMrg4IM7MS1NXW0Dp7Ao/9alfWpQwYB4SZWYmuOGsiz3XuZ+f+Q1mXMiAcEGZmJepZQOjJKpn+2wFhZlai86aNY1itWPWCA8LMzPKMGFZLy9RxPLWtOr5R7YAwMyvDRTMbeWbHaxypgon7HBBmZmW4cOZ4Dh7p5tmXKn/iPgeEmVkZLu5ZYW5b5X8fwgFhZlaGaY0jmTFhJCvWvZx1KalzQJiZlencM8by9PbX6O6OrEtJlQPCzKxMi8+dnLsO8XJlX4dwQJiZleldb5sMwC9+tTPjStLlgDAzK9PUxpHMnjiKx7dU9rxMDggzs1NwxdkTeWLrqxyr4OsQDggzs1Nw+VkT2XfwKGt37Mm6lNQ4IMzMTsGV50yiRvCTDa9kXUpqUg0ISUskbZS0WdKtvfS5TtJ6Sesk3Z3XfkzS08mjLc06zczKNXHMcBbOaOSRTV1Zl5KaurReWFItsBR4N9ABrJTUFhHr8/o0A7cBiyJit6TJeS9xICIWplWfmdnpeufcyfyvBzbx6uuHmTC6Puty+l2aRxCXApsjYktEHAbuBa4t6HMjsDQidgNERGeK9ZiZ9aur5k4iAn6+uTJvd00zIKYB2/O2O5K2fHOBuZIelfS4pCV5+0ZIak/aP1DsDSTdlPRp7+qq3MM8MxucFkxvpHHUsIo9zZTaKaYy3r8ZWAxMBx6RdH5EvAbMiogdks4CfippTUT8Kv/JEbEMWAbQ2tpaufeamdmgVFsjrjxnEo9s6iIikJR1Sf0qzSOIHcCMvO3pSVu+DqAtIo5ExFZgE7nAICJ2JH9uAR4CLkyxVjOzU3LV3CY69x2qyGk30gyIlUCzpDmS6oHrgcK7ke4nd/SApEnkTjltkTRe0vC89kXAeszMBpmrmpsAKvI0U2oBERFHgZuBFcAG4L6IWCfpdknXJN1WALskrQceBG6JiF3APKBd0uqk/Y78u5/MzAaLM8eN4Kym0RU57Uaq1yAiYjmwvKDts3k/B/Dp5JHf5xfA+WnWZmbWX945t4lvPfECrx86yujhWV/a7T/+JrWZ2Wl6T8uZHD7aXXG3uzogzMxO08WzxjOqvpZHHRBmZpavvq6Gy+ZM4GfPOSDMzKzAVXOb2Lrzdba/+kbWpfQbB4SZWT+4am7udteHK+h2VweEmVk/OGvSaKaPH+mAMDOz40niqrlNPParXRw+2p11Of3CAWFm1k+uam5i/6GjPPnC7qxL6RcOCDOzfvL2cyYC8O2V2/voOTQ4IMzM+knDiGGMHV7Hhpf2Zl1Kv3BAmJn1o08uPptnX95H596DWZdy2hwQZmb9aPG5udtdH9o49O9mckCYmfWjlikNTB03gp9seCXrUk6bA8LMrB9J4tfmncHPntvJwSPHsi7ntDggzMz62a/Nm8yBI8d47FdDe40IB4SZWT+74uyJjK6vHfKnmRwQZmb9bHhdLe9obuKBDZ3k1kUbmhwQZmYp+PWWM3h570HW7hi634lwQJiZpeDqc5uQ4IFnh+5pJgeEmVkKJo4ZzsIZjXz/mZeG7GkmB4SZWUo+fPEMNnfu55mOPVmXckocEGZmKVly3pnJaabOrEs5JQ4IM7OUTBhdz0Uzx3PnA88NydNMqQaEpCWSNkraLOnWXvpcJ2m9pHWS7i7Y1yCpQ9Lfp1mnmVla3nvemQD8ZMPQO4pILSAk1QJLgfcCLcANkloK+jQDtwGLImI+8KmCl/nvwCNp1WhmlraPXj4LgPvah94aEWkeQVwKbI6ILRFxGLgXuLagz43A0ojYDRARb0aspIuBM4Afp1ijmVmqRgyr5RNXzuGhjZ3s3H8o63LKkmZATAPyI7Mjacs3F5gr6VFJj0taAiCpBvg74DMp1mdmNiCuv2QGR44F/29VR9allCXri9R1QDOwGLgBuEtSI/BHwPKIOOloSrpJUruk9q6uoT/3uplVpuYzxtI6azzfbt8+pC5WpxkQO4AZedvTk7Z8HUBbRByJiK3AJnKBcQVws6Tngb8Ffl/SHYVvEBHLIqI1IlqbmprS+AxmZv3ity+ezpau11n5/O6sSylZmgGxEmiWNEdSPXA90FbQ535yRw9ImkTulNOWiPhIRMyMiNnkTjN9IyKK3gVlZjYUvO/8KQBD6jRTagEREUeBm4EVwAbgvohYJ+l2Sdck3VYAuyStBx4EbomIoT2BuplZEeNGDuPDF0/n+8+8OGQWEtJQOh92Mq2trdHe3p51GWZmvfrF5p387lef4I4Pns/1l87MuhwAJK2KiNZi+7K+SG1mVjUuP2siALd+d82QuFjtgDAzGyA1NeKmq84CYM2OwT+BnwPCzGwA/cerz2HksFq+/ujzWZfSJweEmdkAGjdyGL972Uy+99QONrw0uFebc0CYmQ2wP1x8NgD/+yfPZVzJyTkgzMwG2KQxw3nf+Wfyo3Uvs23X61mX0ysHhJlZBj73m/MB+NCXH8u4kt45IMzMMjC5YQSTxtTTte8Qz+8cnEcRDggzs4x88xOXAfDf2tZlXElxDggzs4zMm9LA7142k4c3ddG2+sWsyzmBA8LMLEO3XzOfC2c28l/vX8srew9mXc5xHBBmZhmqq63hi9ct5NDRY/zpd54ZVFNwOCDMzDI2Z9JobnvvPB7e1MU9vxw8a1c7IMzMBoHfu3wWi86ZyH/53hq+9cS2rMsBHBBmZoNCTY346w9dAMCffW8tL+8p/XpEWqelHBBmZoPEtMaR/HNy6+vlX3igz4vWEcENyx7n5rufSqUeB4SZ2SByZfMkrr9kBgCX/eUDrD3JtOAf//pKHtuyiwNHjqVyFFHX769oZman5Y7fXkB9XQ3feGwbv/GlnwPw/B3vf3P/gcPHmPfZH725/dXfb0VSv9fhJUfNzAapdS/u4f13/vykfTb+jyUMr6s95ffwkqNmZkPQ/KnjeO7z7y2678/fP4/n73j/aYVDX3yKycxsEBtWW3Pc6aWB5CMIMzMrygFhZmZFOSDMzKyoVANC0hJJGyVtlnRrL32uk7Re0jpJdydtsyQ9KenppP2TadZpZmYnSu0itaRaYCnwbqADWCmpLSLW5/VpBm4DFkXEbkmTk10vAVdExCFJY4C1yXMH34TpZmYVKs0jiEuBzRGxJSIOA/cC1xb0uRFYGhG7ASKiM/nzcEQcSvoMT7lOMzMrIs1/eKcB+fPWdiRt+eYCcyU9KulxSUt6dkiaIemZ5DX+qtjRg6SbJLVLau/q6krhI5iZVa+sfzOvA5qBxcANwF2SGgEiYntELADOAT4m6YzCJ0fEsohojYjWpqamASzbzKzypflFuR3AjLzt6Ulbvg7giYg4AmyVtIlcYKzs6RARL0paC7wD+E5vb7Zq1aqdkvInUR8H7CljexKws68PdYoK36u/ntNXn972F2uvhvHqq5/Hq7x+pzNehW0er/Lb8rdPZ7xm9bonIlJ5kAufLcAcoB5YDcwv6LME+L/Jz5PInU6aSC5MRibt44FNwPllvv+yMrfbUxyLZWk8p68+ve0v1l4N49VXP4/XwI1XYZvH6/T+zqU1XqmdYoqIo8DNwApgA3BfRKyTdLuka5JuK4BdktYDDwK3RMQuYB7whKTVwMPA30bEmjJL+Ncyt9N0Ku9VynP66tPb/mLt1TBeffXzeJXX73TGq7DN41V+W+pjVjGzuZ4uSe3Ry4yGdiKPV3k8XuXxeJUnrfHK+iL1YLIs6wKGGI9XeTxe5fF4lSeV8fIRhJmZFeUjCDMzK8oBYWZmRTkgzMysKAdEHyTVSPq8pC9J+ljW9QwFkhZL+pmkL0tanHU9g52k0cmUMb+RdS1DgaR5yd+t70j6w6zrGewkfUDSXZK+Lek95Ty3ogNC0tckdSbfxM5v73Ma8jzXkvvi3hFy3/yuaP00ZgHsB0ZQwWPWT2MF8J+B+9KpcnDpjzGLiA0R8UngOmBRmvVmrZ/G6/6IuBH4JPA7Zb1/Jd/FJOkqcv9QfSMizkvaasl9M/vNacjJzQNVC3yh4CX+IHnsjoivSPpORHxooOrPQj+N2c6I6E7mz/piRHxkoOofSP00VheQmz1gBLlx+/7AVJ+N/hiziOhMvmz7h8A3I+Lugap/oPXXeCXP+zvgWxHxZKnvn+ZcTJmLiEckzS5ofnMacgBJ9wLXRsQXgBMO8SV1AIeTzWPpVTs49MeY5dlNbrr2itRPf78WA6OBFuCApOUR0Z1m3Vnqr79fEdEGtEn6AVCxAdFPf8cE3AH8sJxwgAoPiF4Um4b8spP0/y7wJUnvAB5Js7BBrKwxk/RB4N8BjcDfp1vaoFPWWEXEnwFI+jjJkVeq1Q1O5f79Wgx8kNwvH8tTrWxwKvffsD8Gfh0YJ+mciPhyqW9UjQFRloh4A/hE1nUMJRHxXXLBaiWKiH/KuoahIiIeAh7KuIwhIyLuBO48ledW9EXqXpQyDbkdz2NWOo9V+Txm5Rmw8arGgFgJNEuaI6keuB5oy7imwc5jVjqPVfk8ZuUZsPGq6ICQdA/wGHCupA5Jn+htGvIs6xxMPGal81iVz2NWnqzHq6JvczUzs1NX0UcQZmZ26hwQZmZWlAPCzMyKckCYmVlRDggzMyvKAWFmZkU5IKziSdo/wO/3iwF+v0ZJfzSQ72nVwQFhViZJJ53DLCLePsDv2Qg4IKzfOSCsKkk6W9KPJK1KVr97W9L+m5KekPSUpJ8ka1og6XOSvinpUeCbyfbXJD0kaYukP8l77f3Jn4uT/d+R9KykbyVTLyPpfUnbKkl3SjphHQhJH5fUJumnwAOSxkh6QNKTktZIujbpegdwtqSnJf1N8txbJK2U9Iykv0hzLK1yeTZXq1bLgE9GxHOSLgP+AXgX8HPg8ogISf8e+FPgPyXPaQGujIgDkj4HvA24GhgLbJT0jxFxpOB9LgTmAy8CjwKLJLUDXwGuioityXQKvbkIWBARryZHEb8VEXslTQIel9QG3AqcFxELAZRbVrKZ3LoBIrduwlURUa3T1dspckBY1ZE0Bng78C/JL/Tw1sJG04FvS5oC1ANb857aFhEH8rZ/EBGHgEOSOoEzOHGJ1V9GREfyvk8Ds8mtELYlInpe+x7gpl7K/beIeLWndOAvk1XGusmtC3BGkee8J3k8lWyPIRcYDggriwPCqlEN8FrPb9wFvkRumdS2ZGGaz+Xte72g76G8n49R/P+nUvqcTP57fgRoAi6OiCOSnie3VGkhAV+IiK+U+V5mx/E1CKs6EbEX2Crpw5BbklHSBcnucbw1t/7HUiphI3BW3lKSpS4kPw7oTMLhamBW0r6P3GmuHiuAP0iOlJA0TdLk067aqo6PIKwajFJubfEeXyT32/g/SvpzYBhwL7Ca3BHDv0jaDfwUmPCP/00AAACRSURBVNPfxSTXMP4I+JGk18nN71+KbwH/KmkN0A48m7zeLkmPSlpLbt3hWyTNAx5LTqHtBz4KdPb3Z7HK5um+zTIgaUxE7E/ualoKPBcR/zPruszy+RSTWTZuTC5aryN36sjXC2zQ8RGEmZkV5SMIMzMrygFhZmZFOSDMzKwoB4SZmRXlgDAzs6IcEGZmVtT/B/CwnGfpZYAIAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "go9PxvhQT0DK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# tune_model.compile(loss = tf.keras.losses.BinaryCrossentropy(), optimizer = keras.optimizers.Adam(1e-6)) #0.00003"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W1Lc1E-AbvIM",
        "colab_type": "text"
      },
      "source": [
        "# Define Model training functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XBYQCpsTUaFS",
        "colab_type": "text"
      },
      "source": [
        "Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x6-sILykUZKF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_CFmodel(model, x_aug_train, y_train,x_aug_test, y_test):\n",
        "  BATCH_SIZE = 512\n",
        "  EPOCHS = 20\n",
        "  train_again = True\n",
        "  \n",
        "  callback = tf.keras.callbacks.EarlyStopping(        #check val_loss vs loss which one to consider\n",
        "      monitor='val_loss', min_delta=0, patience=1, verbose=0, mode='auto',\n",
        "      baseline=None, restore_best_weights=False\n",
        "  )\n",
        "  \n",
        "  if train_again:\n",
        "    history = model.fit(x = x_aug_train, y = y_train, batch_size = BATCH_SIZE ,epochs = EPOCHS, verbose = 1, callbacks = [callback], validation_data=(x_aug_test, y_test))\n",
        "\n",
        "  plt.plot(history.history['loss'])\n",
        "  plt.plot(history.history['val_loss'])\n",
        "  plt.title('model loss')\n",
        "  plt.ylabel('loss')\n",
        "  plt.xlabel('eopchs')\n",
        "  plt.legend(['train','test'], loc = \"upper left\")\n",
        "  plt.show()\n",
        "\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hhWBcsJ397Ia",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# train_CFmodel(CFmodel(lr = 0.00005), x_train, y_train,x_test, y_test)\n",
        "# a = Basemodel().predict(x_train)\n",
        "# print(x_train.shape)\n",
        "# print(a.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZoF3mJTUYl7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# train_model(model, x_train, y_train, x_test, y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6Kph10Ab9LY",
        "colab_type": "text"
      },
      "source": [
        "# Load saved model\n",
        "just in case"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L0O7SS_rVjA1",
        "colab_type": "text"
      },
      "source": [
        "Load saved model "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sdveSbL6VdtH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def saved_model():\n",
        "  model = keras.models.load_model(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '6'+ 'refined')\n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "soKtymB3cwHb",
        "colab_type": "text"
      },
      "source": [
        "# RMSE evaluation for the leave one validation set "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QjqM2aAfY6Ub",
        "colab_type": "text"
      },
      "source": [
        "Left out rmse"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B_FAd8y95jyg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## utility function to calculate rmse\n",
        "def rmse(predictions, targets):  \n",
        "  print(np.squeeze(predictions).shape, np.squeeze(targets).shape)\n",
        "  print('')\n",
        "  return np.sqrt(np.mean((np.squeeze(predictions)-np.squeeze(targets))**2))\n",
        "\n",
        "## function to return datafrmae of rmse evalutaion result\n",
        "def rmse_evaluate(model, x_train, x_test, y_train, y_test, model_name ):\n",
        "  BATCH_SIZE_P = 10\n",
        "\n",
        "  ratings_train = model.predict(np.reshape(np.array(x_train.values),(-1,4)), batch_size = BATCH_SIZE_P).flatten() ##remove .iloc for V1 model\n",
        "  ratings_test = model.predict(np.reshape(np.array(x_test.values),(-1,4)), batch_size = BATCH_SIZE_P).flatten() ##remove .iloc for V1 model\n",
        "\n",
        "  rmse_train = rmse(ratings_train, y_train)\n",
        "  rmse_test = rmse(ratings_test, y_test)\n",
        "\n",
        "  data = {'model':[model_name], 'rmse train':[rmse_train], 'rmse test':[rmse_test]}\n",
        "  df = pd.DataFrame(data)\n",
        "\n",
        "  return df\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-Q_SnUFBY2s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def rmse_evaluate_leftout(df_ratings):\n",
        "  # x_train, x_test, y_train, y_test, global_avg = prepare_data(df_ratings)\n",
        "  \n",
        "  ############### prepare data #########\n",
        "  x_train, x_test, y_train, y_test, global_avg = prepare_data_leftout(df_ratings)\n",
        "\n",
        "  ############# get model ##############\n",
        "  num_users = df_ratings['userId'].unique().max() +1\n",
        "  num_movies = df_ratings['movieId'].unique().max() +1\n",
        "  # EMBEDDING_SIZE = 60\n",
        "\n",
        "  lr =  0.002 #2e-2 tune this\n",
        "  \n",
        "  model_CF = CFmodel(num_users = num_users, num_movies=num_movies, global_avg = global_avg,hparam =hparam)\n",
        "  model_base = Basemodel( global_avg = global_avg)\n",
        "  # model_mu = mumodel( global_avg = global_avg)\n",
        "  model_CF_pure = CFpure(num_users = num_users, num_movies=num_movies, global_avg = global_avg,hparam =hparam)\n",
        "  ############ train/load model #############\n",
        "  train_CF = True # True\n",
        "  save_CF = True #True\n",
        "  \n",
        "  if train_CF or not os.path.isdir(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '7'+'1m'):\n",
        "    print(\"CF_Recommender training...\")\n",
        "    model_CF = train_CFmodel(model_CF, x_train, y_train, x_test, y_test)\n",
        "  else:\n",
        "    model_CF = keras.models.load_model(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '7'+'1m')\n",
        "    \n",
        "  if save_CF:\n",
        "    model_CF.save(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '7'+ '1m') #'_mu' + '_bias'+'_meaningful_embeddings_allembeddings_he_initialized'\n",
        " \n",
        " ###############\n",
        "  save_pure = True\n",
        "  train_pure = True\n",
        "  \n",
        "  if train_pure or not os.path.isdir(base_dir + \"Saved Models/\"+'CFpure_Recommender'+'_V_'+ '7'+'1m'):\n",
        "    print(\"CFpure_Recommender training...\")\n",
        "    model_CF_pure = train_CFmodel(model_CF_pure, x_train, y_train, x_test, y_test)\n",
        "  else:\n",
        "    model_CF_pure = keras.models.load_model(base_dir + \"Saved Models/\"+'CFpure_Recommender'+'_V_'+ '7'+'1m')\n",
        "  \n",
        "  if save_pure:\n",
        "    model_CF_pure.save(base_dir + \"Saved Models/\"+'CFpure_Recommender'+'_V_'+ '7'+ '1m') #'_mu' + '_bias'+'_meaningful_embeddings_allembeddings_he_initialized'\n",
        " \n",
        "  # model = keras.models.load_model(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '5'+ '_mu' + '_bias'+'_meaningful_embeddings_allembeddings_he_initialized')\n",
        "  ############ evaluate models #########\n",
        "  # df = rmse_evaluate(model_mu,x_train, x_test, y_train, y_test,\"mu Baseline\")\n",
        "\n",
        "\n",
        "  df = rmse_evaluate(model_base,x_train, x_test, y_train, y_test,\"bias Baseline\")\n",
        "  df = df.append(rmse_evaluate(model_CF,x_train, x_test, y_train, y_test,\"CF\"))\n",
        "  df = df.append(rmse_evaluate(model_CF_pure,x_train, x_test, y_train, y_test,\"CFpure\"))\n",
        "  return(df, model_CF, model_base, model_CF_pure)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kvm0l8N4Y2Is",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "88628e57-4ebf-4c29-c1fc-c6f9ccb808c9"
      },
      "source": [
        "df1, model_CF1, model_base1, model_CF_pure1 = rmse_evaluate_leftout(df_ratings)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"cf_recommender_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_12 (Embedding)     multiple                  181230    \n",
            "_________________________________________________________________\n",
            "embedding_13 (Embedding)     multiple                  6041      \n",
            "_________________________________________________________________\n",
            "embedding_14 (Embedding)     multiple                  118590    \n",
            "_________________________________________________________________\n",
            "embedding_15 (Embedding)     multiple                  3953      \n",
            "=================================================================\n",
            "Total params: 309,814\n",
            "Trainable params: 309,814\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Model: \"cf_recommender_4\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_16 (Embedding)     multiple                  181230    \n",
            "_________________________________________________________________\n",
            "embedding_17 (Embedding)     multiple                  6041      \n",
            "_________________________________________________________________\n",
            "embedding_18 (Embedding)     multiple                  118590    \n",
            "_________________________________________________________________\n",
            "embedding_19 (Embedding)     multiple                  3953      \n",
            "=================================================================\n",
            "Total params: 309,814\n",
            "Trainable params: 309,814\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "CF_Recommender training...\n",
            "Epoch 1/20\n",
            "WARNING:tensorflow:Layer cf_recommender_3 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_13/embeddings:0', 'embedding_15/embeddings:0'] when minimizing the loss.\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.6386 - val_loss: 0.6041\n",
            "Epoch 2/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5987 - val_loss: 0.5924\n",
            "Epoch 3/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5933 - val_loss: 0.5893\n",
            "Epoch 4/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5902 - val_loss: 0.5879\n",
            "Epoch 5/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5875 - val_loss: 0.5870\n",
            "Epoch 6/20\n",
            "1718/1718 [==============================] - 8s 4ms/step - loss: 0.5849 - val_loss: 0.5865\n",
            "Epoch 7/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5824 - val_loss: 0.5863\n",
            "Epoch 8/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5801 - val_loss: 0.5862\n",
            "Epoch 9/20\n",
            "1718/1718 [==============================] - 8s 4ms/step - loss: 0.5780 - val_loss: 0.5867\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xc5Zn3/881M+q9usm25CrbFBsLU2zANoGYQExLDCRkN/ntYvI8y7NpyxN4NmXDlrCb3SybDUkoIYHQQkhInIWEEmwMxIBlY8C9yJZVjCWrd2k01++PcySPxdiWrBnNSLrer9e8PHPqJWPmq/u+z7mPqCrGGGPMQJ5oF2CMMSY2WUAYY4wJyQLCGGNMSBYQxhhjQrKAMMYYE5IFhDHGmJAsIIwJAxH5uYj80yC3PSQiHxvucYyJNAsIY4wxIVlAGGOMCckCwowbbtfOnSLyvoi0ichPRWSCiPxBRFpE5BURyQrafrWI7BCRRhHZICLzgtYtEpGt7n6/BBIHnOsaEdnm7vtnETnnDGu+TUT2i0i9iKwTkcnuchGR/xSRGhFpFpEPROQsd90nRGSnW1uViPzdGf2FmXHPAsKMNzcCVwBzgE8CfwD+H5CH8//D3wKIyBzgKeDL7roXgN+LSLyIxAO/BX4BZAO/co+Lu+8i4BHgdiAHeABYJyIJQylURFYC3wXWAJOAcuBpd/WVwKXuz5HhblPnrvspcLuqpgFnAa8O5bzG9LGAMOPNf6vqUVWtAl4H3lbVd1W1E3gOWORudxPwvKq+rKo9wL8DScDFwIVAHHCfqvao6rPA5qBzrAUeUNW3VbVXVR8Futz9huKzwCOqulVVu4C7gYtEpBDoAdKAYkBUdZeqHnH36wHmi0i6qjao6tYhntcYwALCjD9Hg953hPic6r6fjPMbOwCqGgAqgCnuuio9cabL8qD304Gvud1LjSLSCEx19xuKgTW04rQSpqjqq8APgfuBGhF5UETS3U1vBD4BlIvIayJy0RDPawxgAWHMyVTjfNEDTp8/zpd8FXAEmOIu6zMt6H0F8M+qmhn0SlbVp4ZZQwpOl1UVgKr+QFUXA/NxuprudJdvVtVrgXycrrBnhnheYwALCGNO5hngahG5XETigK/hdBP9GdgE+IG/FZE4EbkBWBK070PAF0XkAncwOUVErhaRtCHW8BTwBRFZ6I5f/AtOl9ghETnfPX4c0AZ0AgF3jOSzIpLhdo01A4Fh/D2YccwCwpgQVHUPcCvw38AxnAHtT6pqt6p2AzcAnwfqccYrfhO0bylwG04XUAOw3912qDW8AnwT+DVOq2UmcLO7Oh0niBpwuqHqgO+56z4HHBKRZuCLOGMZxgyZ2AODjDHGhGItCGOMMSFZQBhjjAnJAsIYY0xIFhDGGGNC8kW7gHDJzc3VwsLCaJdhjDGjypYtW46pal6odWMmIAoLCyktLY12GcYYM6qISPnJ1kW0i0lEVonIHnc2yrtOss0ad+bJHSLy5IB16SJSKSI/jGSdxhhjPipiLQgR8eLME3MFUAlsFpF1qrozaJvZOBOQLVXVBhHJH3CYfwQ2RqpGY4wxJxfJFsQSYL+qlrl3nj4NXDtgm9uA+1W1AUBVa/pWiMhiYALwUgRrNMYYcxKRHIOYgjNpWZ9K4IIB28wBEJE3AS/wD6r6RxHxAP+BM9VByGf3DkZPTw+VlZV0dnae6SFGjcTERAoKCoiLi4t2KcaYMSLag9Q+YDawHCgANorI2TjB8IKqVp44YeaJRGQtztz7TJs27SPrKysrSUtLo7CwkFMdZ7RTVerq6qisrKSoqCja5RhjxohIBkQVzvTIfQrcZcEqcWan7AEOishenMC4CLhERP43zvz88SLSqqonDHSr6oPAgwAlJSUfmVSqs7NzzIcDgIiQk5NDbW1ttEsxxowhkRyD2AzMFpEi9xGNNwPrBmzzW5zWAyKSi9PlVKaqn1XVaapaCPwd8NjAcBissR4OfcbLz2mMGTkRCwhV9QN3AC8Cu4BnVHWHiNwjIqvdzV4E6kRkJ7AeuFNV60IfMTL8vQGONnfS0e0fydMaY0zMi+h9EKr6gqrOUdWZqvrP7rJvqeo6972q6ldVdb6qnq2qT4c4xs9V9Y6IFSlQ09xFY0dPRA7f2NjIj370oyHv94lPfILGxsYIVGSMMYMz7udi8nk8JCd4aemMTAviZAHh95/6fC+88AKZmZkRqckYYwYj2lcxxYT0RB9Hmjrp9geI94U3M++66y4OHDjAwoULiYuLIzExkaysLHbv3s3evXu57rrrqKiooLOzky996UusXbsWOD51SGtrK1dddRXLli3jz3/+M1OmTOF3v/sdSUlJYa3TGGMGGjcB8Z3f72BndXPIdQFVOrp7SfB58HkHHxDzJ6fz7U8uOOU29957L9u3b2fbtm1s2LCBq6++mu3bt/dfjvrII4+QnZ1NR0cH559/PjfeeCM5OTknHGPfvn089dRTPPTQQ6xZs4Zf//rX3HrrrYOu0xhjzsS4CYhT8YggIvgDis8b2XMtWbLkhHsVfvCDH/Dcc88BUFFRwb59+z4SEEVFRSxcuBCAxYsXc+jQocgWaYwxjKOAON1v+lUNHTS0dzN/UjoeT+QuGU1JSel/v2HDBl555RU2bdpEcnIyy5cvD3nXd0JCQv97r9dLR0dHxOozxpg+436Quk9aoo+AKm1hvtw1LS2NlpaWkOuamprIysoiOTmZ3bt389Zbb4X13MYYMxzjpgVxOqkJPjwitHT6SUsM33xGOTk5LF26lLPOOoukpCQmTJjQv27VqlX85Cc/Yd68ecydO5cLL7wwbOc1xpjhEtWPzFAxKpWUlOjABwbt2rWLefPmDfoYB4+10eXvZe6EtFF5Z/JQf15jjBGRLapaEmqddTEFSU/00e0P0OUPRLsUY4yJOguIIGmJTo9bpG6aM8aY0cQCIki8z0uiz0tLZ2Sm3TDGmNHEAmKAtCQfbd299AbGxtiMMcacKQuIAdIS41BVWrusFWGMGd8sIAZIjvfi9QgtHTYOYYwZ3ywgBvCIkJrgo7nLTzguAT7T6b4B7rvvPtrb24ddgzHGnAkLiBDSEuPw9wbo7Okd9rEsIIwxo5XdSR1C3+WuzZ1+kuKH91cUPN33FVdcQX5+Ps888wxdXV1cf/31fOc736GtrY01a9ZQWVlJb28v3/zmNzl69CjV1dWsWLGC3Nxc1q9fH44fzRhjBm38BMQf7oIPPxjUpnHA7B4/IBB3iuldJ54NV917ymMFT/f90ksv8eyzz/LOO++gqqxevZqNGzdSW1vL5MmTef755wFnjqaMjAy+//3vs379enJzcwf5QxpjTPhYF9NJeD0eegOKEr7LXV966SVeeuklFi1axHnnncfu3bvZt28fZ599Ni+//DJf//rXef3118nIyAjbOY0x5kyNnxbEaX7TH8jf7aesppWpWclkpcSHpQRV5e677+b222//yLqtW7fywgsv8I1vfIPLL7+cb33rW2E5pzHGnClrQZxEUpwXn8cz7Luqg6f7/vjHP84jjzxCa2srAFVVVdTU1FBdXU1ycjK33nord955J1u3bv3IvsYYM9LGTwtiiESEtEQfzZ09qOoZz+4aPN33VVddxWc+8xkuuugiAFJTU3n88cfZv38/d955Jx6Ph7i4OH784x8DsHbtWlatWsXkyZNtkNoYM+Jsuu9TaGrvpry+nZl5qaQkxH6W2nTfxpihsum+z1Bqog9BaLbJ+4wx41BEA0JEVonIHhHZLyJ3nWSbNSKyU0R2iMiT7rLpIrJVRLa5y78YyTpPxuvxkJzgtem/jTHjUsT6TUTEC9wPXAFUAptFZJ2q7gzaZjZwN7BUVRtEJN9ddQS4SFW7RCQV2O7uWz3UOoYzfgDOQ4SONHXS7Q8Q74vdBtdY6So0xsSOSH7jLQH2q2qZqnYDTwPXDtjmNuB+VW0AUNUa989uVe1yt0k40zoTExOpq6sb1pdn3/OpY/kZEapKXV0diYmJ0S7FGDOGRHLkdQpQEfS5ErhgwDZzAETkTcAL/IOq/tFdNhV4HpgF3Bmq9SAia4G1ANOmTftIAQUFBVRWVlJbWzusH6SuqZPmI0JOasKwjhNJiYmJFBQURLsMY8wYEu1Lc3zAbGA5UABsFJGzVbVRVSuAc0RkMvBbEXlWVY8G76yqDwIPgnMV08CDx8XFUVRUNOwin/rddp4prWDbt64k8VRTbxhjzBgSyS6mKmBq0OcCd1mwSmCdqvao6kFgL05g9HNbDtuBSyJY6ymtKM6nsyfAW2V10SrBGGNGXCQDYjMwW0SKRCQeuBlYN2Cb3+K0HhCRXJwupzIRKRCRJHd5FrAM2BPBWk/pohk5JMZ5WL+7JlolGGPMiItYQKiqH7gDeBHYBTyjqjtE5B4RWe1u9iJQJyI7gfU4Yw11wDzgbRF5D3gN+HdVHdxUrBGQGOfl4pm5rN9Ta1cLGWPGjYiOQajqC8ALA5Z9K+i9Al91X8HbvAycE8nahmpFcT6v7q7hQG0bs/JTo12OMcZEXOxe2B9jVszNA7BuJmPMuGEBMUgFWcnMmZDK+j0WEMaY8cECYghWFOfzzsH6mL5pzhhjwsUCYghWzs3HH1De2Hcs2qUYY0zEWUAMweLpWaQl+njVxiGMMeOABcQQ+LweLp2Tx4a9tQQCdrmrMWZss4AYopVz86lt6WJHdXO0SzHGmIiygBiiy+bmIYJ1MxljxjwLiCHKTU3gnIJMu9zVGDPmWUCcgZVz83mvspG61q7Tb2yMMaOUBcQZWFmcjyps2DO850wYY0wss4A4Awsmp5OXlsCr1s1kjBnDLCDOgMcjLJ+Tx8a9tfh7A9EuxxhjIsIC4gytLM6npdPPlvKGaJdijDERYQFxhpbNzsXnEetmMsaMWRYQZygtMY7zC7PZsNsGqo0xY5MFxDCsLM5nz9EWqho7ol2KMcaEnQXEMKwozgfsrmpjzNhkATEMM/NSmJadbE+ZM8aMSRYQwyAirJibx58PHKOzpzfa5RhjTFhZQAzTiuJ8OnsCbCqri3YpxhgTVhYQw3ThjBwS4zzWzWSMGXMsIIYpMc7L0pm5vLq7BlV7iJAxZuywgAiDFcX5VDZ0cKC2NdqlGGNM2EQ0IERklYjsEZH9InLXSbZZIyI7RWSHiDzpLlsoIpvcZe+LyE2RrHO47HJXY8xYFLGAEBEvcD9wFTAfuEVE5g/YZjZwN7BUVRcAX3ZXtQN/4S5bBdwnIpmRqnW4pmQmUTwxzQLCGDOmRLIFsQTYr6plqtoNPA1cO2Cb24D7VbUBQFVr3D/3quo+9301UAPkRbDWYVs+N5/SQw00d/ZEuxRjjAmLSAbEFKAi6HOluyzYHGCOiLwpIm+JyKqBBxGRJUA8cCDEurUiUioipbW10Z0TaWVxPv6A8sa+Y1GtwxhjwiXag9Q+YDawHLgFeCi4K0lEJgG/AL6gqh958IKqPqiqJapakpcX3QbGedMySU/0WTeTMWbMiGRAVAFTgz4XuMuCVQLrVLVHVQ8Ce3ECAxFJB54H/l5V34pgnWHh83q4dE4eG/bUEgjY5a7GmNEvkgGxGZgtIkUiEg/cDKwbsM1vcVoPiEguTpdTmbv9c8BjqvpsBGsMq5XF+Rxr7WJ7dVO0SzHGmGGLWECoqh+4A3gR2AU8o6o7ROQeEVntbvYiUCciO4H1wJ2qWgesAS4FPi8i29zXwkjVGi6XzclDxC53NcaMDTJW7v4tKSnR0tLSaJfB9T96k0BA+d0dy6JdijHGnJaIbFHVklDroj1IPeasmJvPe5VN1LZ0RbsUY4wZFguIMFvp3lX92l57FKkxZnSzgAizBZPTyU9LsNldjTGjngVEmIkIy+fmsXFfLT29H7l1wxhjRg0LiAhYWZxPS6efLeUN0S7FGGPOmAVEBCybnUecV6ybyRgzqllAREBqgo8lRdl2P4QxZlSzgIiQFXPz2VfTSkV9e7RLMcaYM2IBESF9DxHasMdaEcaY0ckCIkJm5KYwPSfZupmMMaOWBUSEiAgr5ubz5wN1dPb0RrscY4wZMguICFpRnE+XP8CmA3XRLsUYY4bMAiKCLijKJinOa91MxphRyQIighLjvCydlcuru2sYK7PmGmPGDwuICFtRnEdVYwf7a1qjXYoxxgyJBUSErZjrXO5q3UzGmNHGAiLCJmcmUTwxzQLCGDPqWECMgBXF+ZSWN9Dc2RPtUowxZtAsIEbAyuJ8egPK63uPRbsUY4wZNAuIEbBoaiYZSXHWzWSMGVUsIEaAz+vhsjl5vLa3hkDALnc1xowOFhAjZEVxHsdau/mgqinapRhjzKBYQIyQy+bkI2KXuxpjRo+IBoSIrBKRPSKyX0TuOsk2a0Rkp4jsEJEng5b/UUQaReR/IlnjSMlOiWfR1EzW2/TfxphRImIBISJe4H7gKmA+cIuIzB+wzWzgbmCpqi4Avhy0+nvA5yJVXzSsmJvP+5VN1LZ0RbsUY4w5rUi2IJYA+1W1TFW7gaeBawdscxtwv6o2AKhq/6/XqvonoCWC9Y04e4iQMWY0GVRAiMiXRCRdHD8Vka0icuVpdpsCVAR9rnSXBZsDzBGRN0XkLRFZNfjSR58Fk9OZkJ5g3UzGmFFhsC2I/09Vm4ErgSycrp97w3B+HzAbWA7cAjwkIpmD3VlE1opIqYiU1tbWhqGcyOp7iNDre4/R0xuIdjnGGHNKgw0Icf/8BPALVd0RtOxkqoCpQZ8L3GXBKoF1qtqjqgeBvTiBMSiq+qCqlqhqSV5e3mB3i6rlc/Np6fJTeqgh2qUYY8wpDTYgtojISzgB8aKIpAGn+xV4MzBbRIpEJB64GVg3YJvf4rQeEJFcnC6nskHWNCotm51LnFesm8kYE/MGGxB/BdwFnK+q7UAc8IVT7aCqfuAO4EVgF/CMqu4QkXtEZLW72YtAnYjsBNYDd6pqHYCIvA78CrhcRCpF5OND/NliUmqCjwuKcux+CGNMzPMNcruLgG2q2iYitwLnAf91up1U9QXghQHLvhX0XoGvuq+B+14yyNpGneVz8/in53dRUd/O1OzkaJdjjDEhDbYF8WOgXUTOBb4GHAAei1hVY9xK93JX62YyxsSywQaE3/1t/1rgh6p6P5AWubLGthl5qRTmJFs3kzEmpg02IFpE5G6cy1ufFxEPzjiEOUMrivPZdKCOju7eaJdijDEhDTYgbgK6cO6H+BDnktXvRayqcWDF3Hy6/AE2ldlDhIwxsWlQAeGGwhNAhohcA3Sqqo1BDMMFM7JJjvdaN5MxJmYNdqqNNcA7wKeBNcDbIvKpSBY21iX4vCydlcv63bU4wzvGGBNbBnuZ69/j3ANRAyAiecArwLORKmw8WDE3n5d3HmVfTStzJtiYvzEmtgx2DMITPNMqUDeEfc1JrCh2pgexbiZjTCwa7Jf8H0XkRRH5vIh8HnieATfAmaGblJHEvEnpFhDGmJg02EHqO4EHgXPc14Oq+vVIFjZerCzOY0t5A00dPdEuxRhjTjDobiJV/bWqftV9PRfJosaTFXPz6Q0or++L/enKjTHjyykDQkRaRKQ5xKtFRJpHqsixbNG0LDKT46ybyRgTc055FZOqjo9La+oPQlYhyOkecRF+Xo9w2Zw8XttTSyCgeDwjX4MxxoRiVyId2wc/vhj+9B2I0v0IK+bmU9fWzftVTVE5vzHGhGIBkT0TzrkJ3vhPePUfoxISl83JwyN2uasxJrZYQHg8cPX34by/hNf/A9b/y4iXkJUSz6JpWay3gDDGxBALCHBC4pr7YNHnYOO/wYZ7R7yElcX5fFDVRE1L54if2xhjQrGA6OPxwCd/AAtvhQ3fhQ3/OqKnXz7Xuat6wx673NUYExssIIJ5PLD6B3DuZ2DDv8DGkZvRfP6kdCamJ1o3kzEmZgx2sr7xw+OFa38IGoBX/wnEA5d8LeKnFRFWFOfx+/eO0O0PEO+z7DbGRJd9C4Xi8cJ1P4KzPw1/ugfeuG9ETrtibj6tXX5Ky+tH5HzGGHMq1oI4GY8XrvuJ05J45dtOS2Lp30b0lEtn5RLv9bB+dw0Xz8yN6LmMMeZ0rAVxKl4fXP8gLLgeXv4m/PmHET1dSoKPC2Zk2/0QxpiYYAFxOl4f3PAwzL8OXvp72PSjiJ5uxdx8DtS2cbiuPaLnMcaY04loQIjIKhHZIyL7ReSuk2yzRkR2isgOEXkyaPlfisg+9/WXkazztLw+uPFhmLcaXrwb3n4gYqdaUZwPwPo91oowxkRXxAJCRLzA/cBVwHzgFhGZP2Cb2cDdwFJVXQB82V2eDXwbuABYAnxbRLIiVeugeOPgU49A8TXwh/8L7zwUkdMU5aZQlJti3UzGmKiLZAtiCbBfVctUtRt4Grh2wDa3AferagNA0GNNPw68rKr17rqXgVURrHVwvHHwqZ/B3Kvhhb+DzQ9H5DQr5uazqayO9m5/RI5vjDGDEcmAmAJUBH2udJcFmwPMEZE3ReQtEVk1hH0RkbUiUioipbW1I3QHsi8ePv1zmHMVPP81KH0k7KdYWZxPtz/ApgN1YT+2McYMVrQHqX3AbGA5cAvwkIhkDnZnVX1QVUtUtSQvLy9CJYbgi4c1j8Lsj8P/fAW2/Dyshz+/KIvkeK91MxljoiqSAVEFTA36XOAuC1YJrFPVHlU9COzFCYzB7BtdvgS46Rcw6wr4/Zdg6y/CdugEn5dls3JZv7sGjdIzKowxJpIBsRmYLSJFIhIP3AysG7DNb3FaD4hILk6XUxnwInCliGS5g9NXustiiy8BbnocZl4O6/4PvPtE2A69sjif6qZO9h5tDdsxjTFmKCIWEKrqB+7A+WLfBTyjqjtE5B4RWe1u9iJQJyI7gfXAnapap6r1wD/ihMxm4B53WeyJS4Sbn4QZy+F3fwPbngrLYZfPzccjcNtjpfxi0yE6unvDclxjjBksGStdGCUlJVpaWhq9Ano64Kmboew1uP4BOPemYR9yw54a/utP+3j3cCPZKfH8xUXT+YuLCslOiQ9DwcYYAyKyRVVLQq6zgAij7nZ46iY49IYzRcc5nx72IVWV0vIGHnjtAK/sqiExzsOakqn89bIZTMtJDkPRxpjxzAJiJHW3wZM3QfmbcMNDcPanwnbofUdbeOj1Mp57t4regHLV2ZO4/dIZnFMw6Au/jDHmBBYQI627DZ74NBzeBDf+FM66IayHP9rcyc/ePMQTb5fT0unnohk53H7ZDC6bk4eIhPVcxpixzQIiGrpanZCoeNuZomPBdWE/RUtnD0+/U8FP3zjIh82dFE9MY+2lM/jkuZOJ80b7FhdjzGhgAREtXS3w+KegcjN8+mcwf+BMI+HR7Q/w+/eqeXBjGXuOtjApI5G/WlbEzUumkZpgj/wwxpycBUQ0dbXAL26A6q3OFB3zPhmxU6kqG/bW8sBrB3irrJ60RB+3XjidL1xcSH56YsTOa4wZvSwgoq2zGR6/AarfhTWPQfHVET/lexWNPLixjD9sP4LP4+H6RVO47dIZzMpPjfi5jTGjhwVELOhsgl9cD0fed6bomHvViJy2vK6Nh18/yK+2VNDZE+Bj8ybwxctmUFKYPSLnN8bENguIWNHRCL+4Dj7cDjc/AXM+PmKnrmvt4rFN5Ty26RAN7T2cNy2TtZfO5Mr5E/B47MonY8YrC4hY0tEAj10HNTudKTpmXzGyp+/u5VdbKnjo9TIq6juYkZvCX18ygxvOm0JinHdEazHGRJ8FRKxpr4fHroXaPXDLkzDrYyNegr83wB93fMgDr5XxQVUTuakJfP7i6dx64XQyk20qD2PGCwuIWNReD4+thtq9cMtTMOvyqJShqmwqq+OB18p4bW8tyfFebjp/Kn+1rIiCLJvKw5ixzgIiVrXXw6OroW4f3PI0zFwR1XJ2HWnmoY1lrHuvGgWuOWcSay+dwYLJGVGtyxgTORYQsaytDh79JNQfgM88AzMui3ZFVDd28MgbB3nqncO0dfdyyexcbr90Jktn5dhUHsaMMRYQsa7tmBsSB+Gzv4KiS6JdEQBNHT088XY5P3vzELUtXcyflM7tl83g6rMn4bOpPIwZEywgRoPWWnj0Gmg87IRE4bJoV9Svy9/Lb9+t4sGNZRyobWNKZhKrF05m2axcFk/PsqufjBnFLCBGi9Ya+Pk10FQJtz4L0y+OdkUnCASUP+2u4WdvHuSdg/X4A0qCz8P5hdlcPCuHZbNyWTA5A6/dV2HMqGEBMZq0HIWfXw3N1XDrr2H6RdGuKKTWLj/vHKzjzf11vLn/GLs/bAEgIymOi2bksHR2Lstm5VKYk2zjFsbEMAuI0ablQyckWj6ES74KRcth8kLwxG5XTk1LJ5sOOGHxxr5jVDd1AjAlM4mLZ+awbHYuF8/MJS8tIcqVGmOCWUCMRs1H4JnPOVOFAyRkOOMSRZc6VzrlFUOM/mauqhyqa+fN/cd4c/8x/nygjqaOHgDmTkhj6axcls3OYUlRjk1HbkyUWUCMZq01cHAjHHzN+bPhkLM8Jd8Ji77AyCqMZpWn1BtQdlQ39XdHvXOonm5/AJ9HWDg10w2MXBZOzbQHHRkzwiwgxpKG8uNhcXAjtB51lmdOd8NiORReAmkTolnlKXX29LKlvKG/hfF+VROqkBzv5YKibJbOymXprFyKJ6bZ+IUxEWYBMVapOvM59QXGodedacUB8uYdb11MXwpJmdGt9RSa2nvYVFbXHxhlx9oAyE2N5+KZzmD3xbNybOoPYyLAAmK8CPTCkfeOB0b5JvB3gHhg0sLjgTH1QoiP3S/b6saO/rB4Y38dx1q7ACjMSXa6o2blctHMHJtU0JgwiFpAiMgq4L8AL/Cwqt47YP3nge8BVe6iH6rqw+66fwX6Hr32j6r6y1OdywIiBH8XVJYeD4zKzRDwgzceCpY4YVF0KUxZDN64aFcbkqqyr6aVN/Y5gfFWWR1t3b2IwFmTM9zuqBzOL8y2G/aMOQNRCQgR8QJ7gSuASmAzcIuq7gza5vNAiareMWDfq4EvA1cBCcAG4HJVbT7Z+SwgBqGrFQ6/BQc3OIFx5H1AIS7FuSlvxmVQdBlMOAs8sTlY3NMb4P3KRt7Y53RJvVvRQE+vEu/zUDI9iwuKchirUo4AABPvSURBVDi/MIuF0zJJjrcrpIw5nVMFRCT/D1oC7FfVMreIp4FrgZ2n3MsxH9ioqn7ALyLvA6uAZyJV7LiQkAqzP+a8wJlN9tAbx1sYL33DWZ6U7cwHVXSpcw9GzsyYuaQ2zuth8fRsFk/P5ksfm01bl593DtXz5r5jvHmgjvv+tBdV8HmEBVMyOH96FiWF2ZQUZpGbavdgGDMUkWxBfApYpap/7X7+HHBBcGvBbUF8F6jFaW18RVUrRORK4Ns4rY9k4B3gflX9jwHnWAusBZg2bdri8vLyiPws40Zz9fGro8peg+ZKZ3n6lOOX1BZdBhlTolvnKTR19LD1cAOlh+rZfKiBbRWNdPsDAMzITaGk0AmM8wuz7S5vY4heF9NgAiIHaFXVLhG5HbhJVVe66/4e+DROeNQAm1X1vpOdz7qYwkwV6stOvKS2vc5Zlz0TJp4FmdOcy2szp7vvp8Xc4HeXv5ftVc39gVFaXk9ju3PTXm5qPCXTndbF+YXZzJ+cbvdhmHEnWgFxEfAPqvpx9/PdAKr63ZNs7wXqVfUjT6cRkSeBx1X1hZOdzwIiwgIB5znaB19zuqWO7XNmnu3tOnG7lLyg4HBDI8sNkYypEJcYnfpdgYBSdqyVzYca2Hywns3l9VTUdwCQFOdl0bRMt4WRxaJpWXantxnzohUQPpxuo8txrlLaDHxGVXcEbTNJVY+4768Hvq6qF7phkamqdSJyDvAksNAdkwjJAiIKAgFoq3GCovGwc5d342FoLHf/rIBAz4n7pE4cEBxBYZIxFXwjf+nqh02dlJbXU3qogc2H6tl1pJmAgkdg/uR0SqY7XVLnF2aRnx7dgDMm3KJ5mesngPtwLnN9RFX/WUTuAUpVdZ2IfBdYDfiBeuB/qepuEUkEtrqHaQa+qKrbTnUuC4gYFOh1Jhw8ITTKnbvBGw8705prb9AOAumTjwfIwFZI+pQRuRy3pbOHdw83UnqontLyBt493EhHj1PntOzk/i6p8wuzmJmXauMYZlSzG+VMbOr1Q0u12/ooP94S6QuT5irQwPHtxQPpBSFaIG6YpE+OyIy3Pb0BdlY3s/nQ8VZGXVs3AFnJcSye7oRFSWE2Z0/JIN5n4xhm9LCAMKOTv9sJieDQCA6SliNA0L9fj88ZA0nKhuRsSM4JemUP+DPH2S4+ZciX8PbNVrv5YL0TGuUNHHSnB0nweTh3amZ/YJw3LYuMpNi8CdEYsIAwY5W/y+mmCu62aqtx7u9or3euumqvg44GTgiSYN6EoBDJGhAqOaHDJsSVWrUtXWwpd6+UOlTP9upmegOKiDPFeUlhFounZ7F4WjZTs5OsW8rEDAsIM74Fep1JDPsCo73uxABpr4eO+hPXdzRy0lDxJZ7YGkn6aGulMz6DPU3xlNZ6eLNaeaeyg9Yu5xqL3NQEzpuW6QTG9CzOmpJh04SYqLGAMGaoAr1OSPQHxoAAaW8YECj1bkslNPX4CPiS6JYE2gIJNPl9NPfG0a4JdEkC8UkppKamk5mRQV52Fimp6RCXBHHJ7st9Hz/gc/B7r12SO26oOi3onnbobnWWZU47o0NFa6oNY0YvjxdScpzXYPX6obMxRAulDuluxdvTQVJPO0nd7eT2tNPd2UZbazNd7a34u2rxHGsn8VgXngPdIN1Dr9kb74ZFyoBAOUXQeOOdsRuPz/mZvXFBn91lHh944gZ89rnbekNsH/fR/YOPO5661wIB50u874u8e8D77jboaXP+7HaX97jLu9tOfD/wc/AVgAXnw1+/EvbyLSCMCRevD1JyndcgxLuvPt3+ADuPNLOlvIF3y+vYfugozS3NJEsXmT4/50yI4+z8OObl+piV6SXV0+1++XQc/xLq6XC/SDrcl/u+vc79cuo4vt3AmxxHiniCAiREyHh9IF53O6/z3uNxPov3+DLxuMsHLhvw5wnrJeiYwes9Ic7Ztz5oWaDnNF/sA77ke9qH9nfjS3QunIhLcf7sC/L0Kc77geviU931kyLyn8oCwpgYEe/zsHBqJgunZsKyIsB5NsaW8gYnNA438My2ZvwBp1t4Rl4Oi6dlcd70LBbPzGJWXioezxB+O+/1O194AT/09jjdagH/KV697nZBnwMDP/tDbBN0nuBznOycvT3O5c3a66zXgPMK9H50mb97wLJe57d27R2wT2DAMfvWDzyP+/5k408AiPtF7X5h979SIXXC8c9x7hd435d83/sTvuRTTjxWBC7THg4bgzBmFOno7uX9yka2HG5ga3kjWw83UO/ek5GW6OO8ac7A93nTnCnPbaqQM6QaIpR6nRZPXNKY6iazMQhjxoikeC8XzMjhghnO2EjfPRnBrYz/fMWZ8twjMHdiOounZ/aHxrRsm8F2UESOdzONY9aCMGaMae7sYdvhRraUN7D1sDNVyPFLbOOPtzKmZ3G2XWI77lkLwphxJD0xjkvn5HHpnDwAegPKvpqW/lbG1vIGXtp5FHAerFQ8KY1zCzI51x3/mJmXincoYxlmzLIWhDHj0LHWLraWN/BeZSPvVTTxXmUjLZ1OKyMl3svZBRmcOzWTRVOd4JiYnmhdU2OUtSCMMSfITU3gygUTuXLBRMB5TsbBujbeq2hkW0Uj71U08sgbB+npdX6BzE9L6G9hnFuQydkFGTbH1DhgAWGMweMRZualMjMvlRvOKwCcp/HtOtLCe25gbKts5GW3awpgRl4KC4O6poonpZHgs/GMscQCwhgTUoLPe/y+DFdTRw8fVDpdUu8ebmTjvmP85t0qAOK9HuZNTmeh2z117tRMinJShnZvhokpNgZhjDljqsqRps7+FsZ7FY18UNlEW7czDURaos8dAM9g4dQszi3IsKfyxRgbgzDGRISIMDkzicmZSVx1tjPdQ29AOVDbyrag8YyfvFZGr3sH+OSMxP4WRt94ht3QF5vsv4oxJqy8HmHOhDTmTEhjTclUADp7etlR3cS2iiZnTKOykT9s/xBw7kmbnZ96wqW2cyemEee1J/NFmwWEMSbiEuO8LJ6ezeLp2f3LGtq6+y+z3VbRwJ921/CrLZWAM55RPCmNBZPTWTA5g7OmZFA8Mc1u6hthNgZhjIkJqkplQwfbKhrZXtXE9uomdlQ309jeAzgtk1l5qSyYks5ZbmjMm5RGWqJdbjscNgZhjIl5IsLU7GSmZifzyXMnA05oVDV2sL2qmR1uYLyx7xi/2VrVv19RbgoLJqdz1pSM/hZHdkr8yU5jhsACwhgTs0SEgqxkCrKSWXXWxP7lNc2d7Kh2QmN7VTPbKhr5n/eP9K+fkpkU1D3lhEd+WoLdDT5EFhDGmFEnPz2R/PREVhTn9y9rbO9mZ3Uz293Q2F7dxMu7jtLXi56bGn88MNwuqoKsJAuNU4hoQIjIKuC/AC/wsKreO2D954HvAX3txR+q6sPuun8DrgY8wMvAl3SsDJgYY8IuMzmei2flcvGs40/0a+vys+tIszum0cyO6mYeeK2s/6FL6Ym+/tDo+7Mo1yYr7BOxgBARL3A/cAVQCWwWkXWqunPApr9U1TsG7HsxsBQ4x130BnAZsCFS9Rpjxp6UBB8lhdmUFB6/eqqzp5e9R1vYUX08OB7dVE63PwBAUpyX+ZPTnXGNyRksmJLO7Pw04n3j77LbSLYglgD7VbUMQESeBq4FBgZEKAok4jyyV4A44Ogp9zDGmEFIjPNyTkEm5xQcn0KkpzfAgdpWdrhdUzuqmvnN1ioe21QOOJfdzpmYyoJJGRRPSqN4YjrzJqWRmTy2B8MjGRBTgIqgz5XABSG2u1FELgX2Al9R1QpV3SQi64EjOAHxQ1XdFcFajTHjWJzXQ/HEdIonpnPjYmeywkBAKa9vP37JbVUzL+86yi9Lj3+tTUxPZN6kNIonpVM8MY15k9Ipyk0ZMzf5RXuQ+vfAU6raJSK3A48CK0VkFjAPKHC3e1lELlHV14N3FpG1wFqAadOmjWDZxpixzuMRinJTKMpNOeGy29rWLnYfaWH3h83sPtLCziPNvLH/WP/U6PFeD7PyUymelMa8ienMm5RO8aQ0clMTovnjnJFIBkQVMDXocwHHB6MBUNW6oI8PA//mvr8eeEtVWwFE5A/ARcDrA/Z/EHgQnBvlwlm8McYMJCLkpyWSn5bY/8Q+gG5/gLJjrew+0sIuNzje3H/i/Rq5qQlOa2Oi00VVPCmNWfmpMT1FeiQDYjMwW0SKcILhZuAzwRuIyCRV7bt4eTXQ1410GLhNRL6L08V0GXBfBGs1xpgzFu873kV1HVP6l9e1drHnwxZ2fdjC7iPN7P6w5YQBcZ/7HI6+cY2+VseE9Ni4ZyNiAaGqfhG5A3gR5zLXR1R1h4jcA5Sq6jrgb0VkNeAH6oHPu7s/C6wEPsAZsP6jqv4+UrUaY0wk5KQmcPGshBMuvfX3BjhU186uI8393VSlhxr43bbq/m0yk+P6xzTmucExZ8LIz0VlczEZY0wMaOroYc+HztjGriPN7DrSwp4PW+jocZ6t4REozE1xAsMNj+JJaUzJHN7NfjYXkzHGxLiMpDiWFGWzpOj4PRuBgHK4vt0NDSc8Pqhq4vkPjk8rkpbgY3lxPv99y6Kw12QBYYwxMcrjEQpzUyjMTWHVWZP6l7d2+ftbG7uPtJCWGJmvcgsIY4wZZVITfCyensXi6VkRPc/YuJvDGGNM2FlAGGOMCckCwhhjTEgWEMYYY0KygDDGGBOSBYQxxpiQLCCMMcaEZAFhjDEmpDEzF5OI1ALlwzhELnAsTOWEk9U1NFbX0FhdQzMW65quqnmhVoyZgBguESk92YRV0WR1DY3VNTRW19CMt7qsi8kYY0xIFhDGGGNCsoA47sFoF3ASVtfQWF1DY3UNzbiqy8YgjDHGhGQtCGOMMSFZQBhjjAlp3AeEiKwSkT0isl9E7op2PX1E5BERqRGR7dGupY+ITBWR9SKyU0R2iMiXol0TgIgkisg7IvKeW9d3ol1TMBHxisi7IvI/0a4lmIgcEpEPRGSbiMTMA91FJFNEnhWR3SKyS0QuioGa5rp/T32vZhH5crTrAhCRr7j/7reLyFMikhi2Y4/nMQgR8QJ7gSuASmAzcIuq7oxqYYCIXAq0Ao+p6lnRrgdARCYBk1R1q4ikAVuA66L99yXOE9tTVLVVROKAN4Avqepb0ayrj4h8FSgB0lX1mmjX00dEDgElqhpTN36JyKPA66r6sIjEA8mq2hjtuvq43xtVwAWqOpybc8NRyxScf+/zVbVDRJ4BXlDVn4fj+OO9BbEE2K+qZaraDTwNXBvlmgBQ1Y1AfbTrCKaqR1R1q/u+BdgFTIluVaCOVvdjnPuKid98RKQAuBp4ONq1jAYikgFcCvwUQFW7YykcXJcDB6IdDkF8QJKI+IBkoDpcBx7vATEFqAj6XEkMfOGNBiJSCCwC3o5uJQ63G2cbUAO8rKoxURdwH/B/gUC0CwlBgZdEZIuIrI12Ma4ioBb4mdst97CIpES7qAFuBp6KdhEAqloF/DtwGDgCNKnqS+E6/ngPCHMGRCQV+DXwZVVtjnY9AKraq6oLgQJgiYhEvVtORK4BalR1S7RrOYllqnoecBXwN263ZrT5gPOAH6vqIqANiKWxwXhgNfCraNcCICJZOL0eRcBkIEVEbg3X8cd7QFQBU4M+F7jLzEm4ffy/Bp5Q1d9Eu56B3O6I9cCqaNcCLAVWu339TwMrReTx6JZ0nPvbJ6paAzyH0+UabZVAZVAL8FmcwIgVVwFbVfVotAtxfQw4qKq1qtoD/Aa4OFwHH+8BsRmYLSJF7m8GNwProlxTzHIHg38K7FLV70e7nj4ikicime77JJyLDnZHtypQ1btVtUBVC3H+bb2qqmH77W44RCTFvdAAtwvnSiDqV8yp6odAhYjMdRddDkT9opEgtxAj3Uuuw8CFIpLs/v95Oc7YYFj4wnWg0UhV/SJyB/Ai4AUeUdUdUS4LABF5ClgO5IpIJfBtVf1pdKtiKfA54AO3vx/g/6nqC1GsCWAS8Kh7dYkHeEZVY+qS0hg0AXjO+U7BBzypqn+Mbkn9/g/whPtLWxnwhSjXA/QH6RXA7dGupY+qvi0izwJbAT/wLmGcdmNcX+ZqjDHm5MZ7F5MxxpiTsIAwxhgTkgWEMcaYkCwgjDHGhGQBYYwxJiQLCGOiSESWx9osr8b0sYAwxhgTkgWEMYMgIre6z5zYJiIPuJMD3uI+T2G7iPxr0LatIvKf7hz9fxKRPHf5LBF5xX1uxVYRmenukhr0/IMn3DtiEZF73WdvvC8i/x6FH9uMcxYQxpyGiMwDbgKWuhMC9gKfBf4VWAksBM4XkevcXVKAUlVdALwGfNtd/gRwv6qeizNfzhF3+SLgy8B8YAawVERygOuBBap6DvBPkf0pjfkoCwhjTu9yYDGw2Z1i5HLgK8AGd5I0P86Xf99sqAHgl+77x4Fl7rxHU1T1OQBV7VTVdnebd1S1UlUDwDagEGgCOoGfisgNQN+2xowYCwhjTk+AR1V1ofuaC/zDEPY/3Xw2XUHvewGfGzpLcGYzvQaIlXmSzDhiAWHM6f0J+JSI5AOISDbOb/qXiUiuO0ngLTjdSeD8f/Up9/1ngDfcJ/BV9nVDiUiCiCSf7ITuMzcy3IkQvwKcG4Gfy5hTGtezuRozGKq6U0S+gfP0NQ/QA/wNzoNs1uO0MJ5X1d+5u7ThPLToGzhPuLvJXf454AERucc9xqdPcdo04HfuA+gF+GqYfyxjTstmczUmzESkVVVTo12HMcNlXUzGGGNCshaEMcaYkKwFYYwxJiQLCGOMMSFZQBhjjAnJAsIYY0xIFhDGGGNC+v8Bt2Oc5cict8IAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/gdrive/My Drive/Machine Learning/movielens1m/Saved Models/CFRecommender_V_71m/assets\n",
            "CFpure_Recommender training...\n",
            "Epoch 1/20\n",
            "WARNING:tensorflow:Layer cf_recommender_4 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_17/embeddings:0', 'embedding_19/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_17/embeddings:0', 'embedding_19/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_17/embeddings:0', 'embedding_19/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_17/embeddings:0', 'embedding_19/embeddings:0'] when minimizing the loss.\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.6388 - val_loss: 0.6042\n",
            "Epoch 2/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5993 - val_loss: 0.5930\n",
            "Epoch 3/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5945 - val_loss: 0.5896\n",
            "Epoch 4/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5910 - val_loss: 0.5878\n",
            "Epoch 5/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5877 - val_loss: 0.5869\n",
            "Epoch 6/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5848 - val_loss: 0.5865\n",
            "Epoch 7/20\n",
            "1718/1718 [==============================] - 8s 5ms/step - loss: 0.5822 - val_loss: 0.5862\n",
            "Epoch 8/20\n",
            "1718/1718 [==============================] - 8s 4ms/step - loss: 0.5800 - val_loss: 0.5862\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXzddZ3v8dcnOdm3tkmapE1LSze6BOhCEQqyCTQFi4qyiVedO4D3DncGneEOXB0dmTtXHec6jlfGERBHBwGRClZpyyJFoGxdWLrSpnvSPUvTJXs+94/fr2lIkzRtz8nJ8n4+HueRk99yfp9UyTvf5ff9mbsjIiLSUUK8CxARkb5JASEiIp1SQIiISKcUECIi0ikFhIiIdEoBISIinVJAiESBmf2Hmf3vHh67zcw+caafIxJrCggREemUAkJERDqlgJBBI+zaudfMPjCzI2b2MzMrMLPFZnbIzF4ys6Htjp9vZmvNrMbMXjGzye32TTezVeF5vwZSO1zrejN7Lzz3DTM79zRrvsPMysysyswWmtmIcLuZ2b+Y2T4zqzWz1WY2Ldw3z8zWhbVVmNnfnNY/mAx6CggZbG4ErgYmAp8EFgP/C8gn+O/hLwHMbCLwBHBPuG8R8HszSzazZOBZ4D+BYcBvws8lPHc68ChwF5AL/BRYaGYpp1KomV0JfAe4CSgCtgNPhruvAT4e/hw54TGV4b6fAXe5exYwDXj5VK4rcowCQgab/+fue929AngNeNvd33X3euAZYHp43M3Ac+7+ors3Af8MpAEXAx8DkoAfunuTuz8NLG93jTuBn7r72+7e4u6/ABrC807F54FH3X2VuzcA9wMXmdkYoAnIAs4BzN3Xu/vu8LwmYIqZZbt7tbuvOsXrigAKCBl89rZ7X9fJ95nh+xEEf7ED4O6twE5gZLivwj+60uX2du/PAv467F6qMbMaYFR43qnoWMNhglbCSHd/Gfgx8CCwz8weMrPs8NAbgXnAdjP7k5lddIrXFQEUECJd2UXwix4I+vwJfslXALuBkeG2Y0a3e78T+Ed3H9Lule7uT5xhDRkEXVYVAO7+I3efCUwh6Gq6N9y+3N1vAIYTdIU9dYrXFQEUECJdeQq4zsyuMrMk4K8JuoneAN4EmoG/NLMkM/sMMLvduQ8DXzGzC8PB5Awzu87Msk6xhieAL5vZ+eH4xf8h6BLbZmYXhJ+fBBwB6oHWcIzk82aWE3aN1QKtZ/DvIIOYAkKkE+7+IXA78P+AAwQD2p9090Z3bwQ+A3wJqCIYr/htu3NXAHcQdAFVA2Xhsadaw0vA3wELCFot44Bbwt3ZBEFUTdANVQl8P9z3BWCbmdUCXyEYyxA5ZaYHBomISGfUghARkU4pIEREpFMKCBER6ZQCQkREOhWJdwHRkpeX52PGjIl3GSIi/crKlSsPuHt+Z/sGTECMGTOGFStWxLsMEZF+xcy2d7VPXUwiItKpmAaEmc01sw/D5Yrv6+KYm8Klidea2eMd9mWbWbmZ/TiWdYqIyIli1sVkZokEC4ldDZQDy81sobuva3fMBIIVKue4e7WZDe/wMf8AvBqrGkVEpGuxHIOYDZS5+xYAM3sSuAFY1+6YO4AH3b0awN33HdthZjOBAmAJMOt0CmhqaqK8vJz6+vrT+wn6kdTUVIqLi0lKSop3KSIyQMQyIEYSrGp5TDlwYYdjJgKY2TIgEfh7d19iZgnA/yVYC6fTh7uH591JsPY+o0ePPmF/eXk5WVlZjBkzho8uvDmwuDuVlZWUl5czduzYeJcjIgNEvAepI8AE4HLgVuBhMxsC/HdgkbuXd3eyuz/k7rPcfVZ+/omztOrr68nNzR3Q4QBgZuTm5g6KlpKI9J5YtiAqCNbPP6Y43NZeOcHyxU3AVjPbSBAYFwGXmtl/J3iAS7KZHXb3Tge6uzPQw+GYwfJzikjviWULYjkwwczGhs/wvQVY2OGYZwlaD5hZHkGX0xZ3/7y7j3b3McDfAL88nXDoieaWVvbW1lPX2BKLjxcR6bdiFhDu3gzcDTwPrAeecve1ZvaAmc0PD3seqDSzdcBS4F53r+z8E2Nn36EGauoaY/LZNTU1/Nu//dspnzdv3jxqampiUJGISM8MmOdBzJo1yzveSb1+/XomT5580nO3HjhCQ3MLkwqyot5Vs23bNq6//nrWrFnzke3Nzc1EItHt4evpzysicoyZrXT3TmeKxnuQuk/ISUuisbmV+qbodzPdd999bN68mfPPP58LLriASy+9lPnz5zNlyhQAPvWpTzFz5kymTp3KQw891HbemDFjOHDgANu2bWPy5MnccccdTJ06lWuuuYa6urqo1yki0tGAWYvpZL79+7Ws21Xb6T4HjjY0kxRJIDmx55k5ZUQ23/rk1G6P+e53v8uaNWt47733eOWVV7juuutYs2ZN23TURx99lGHDhlFXV8cFF1zAjTfeSG5u7kc+Y9OmTTzxxBM8/PDD3HTTTSxYsIDbb7+9x3WKiJyOQRMQ3TEgMcFoafHgbowYmj179kfuVfjRj37EM888A8DOnTvZtGnTCQExduxYzj//fABmzpzJtm3bYlukiAiDKCBO9pd+5eEGKmrqmDA8i7Tk2KVERkZG2/tXXnmFl156iTfffJP09HQuv/zyTu9lSElJaXufmJioLiYR6RUagwhlpyVhwMG6pqh+blZWFocOHep038GDBxk6dCjp6els2LCBt956K6rXFhE5E4OmBXEySYkJpKdEOFjXRGFOatQ+Nzc3lzlz5jBt2jTS0tIoKCho2zd37lz+/d//ncmTJzNp0iQ+9rGPRe26IiJnStNc2zlwuIFdNXVMLMgiNSnGgxExoGmuInKqNM21h3LSgpVQo93NJCLSHykg2klKTCAjOaKAEBFBAXGCnLQk6ptaaIjBTXMiIv2JAqKDbHUziYgACogTJEcSSFc3k4iIAqIzOWkR6ppaaGhWN5OIDF4KiE4cm81UG4VWxOku9w3wwx/+kKNHj55xDSIip0MB0YnkSCJpSYkcrGs+489SQIhIf6U7qbuQk57EnoP1NDa3khw5/Rxtv9z31VdfzfDhw3nqqadoaGjg05/+NN/+9rc5cuQIN910E+Xl5bS0tPB3f/d37N27l127dnHFFVeQl5fH0qVLo/jTiYic3OAJiMX3wZ7VPT48z530xhaIJEBXS4AXlkDpd7v9nPbLfb/wwgs8/fTTvPPOO7g78+fP59VXX2X//v2MGDGC5557DgjWaMrJyeEHP/gBS5cuJS8vr8d1i4hEi7qYupBgRkICtLRGbymSF154gRdeeIHp06czY8YMNmzYwKZNmygpKeHFF1/kb//2b3nttdfIycmJ2jVFRE7X4GlBnOQv/c4cqq1nb209kwuzSTqDbqZj3J3777+fu+6664R9q1atYtGiRXzjG9/gqquu4pvf/OYZX09E5EyoBdGNtrWZ6k9/NlP75b6vvfZaHn30UQ4fPgxARUUF+/btY9euXaSnp3P77bdz7733smrVqhPOFRHpbYOnBXEaUpMSSY0kcrCuibzMlJOf0In2y32XlpZy2223cdFFFwGQmZnJY489RllZGffeey8JCQkkJSXxk5/8BIA777yTuXPnMmLECA1Si0iv03LfJ7Gntp59tfVMLsom6RSeVx0PWu5bRE6Vlvs+A9G8aU5EpD9RQJxEaiSBlLCbSURkMBnwAXGmXWhmRk5ahCMNLTS3tEapqugbKF2FItJ3xDQgzGyumX1oZmVmdl8Xx9xkZuvMbK2ZPR5uO8vMVpnZe+H2r5zO9VNTU6msrDzjX545aUk4Tu0ZzGaKJXensrKS1NToPUtbRCRms5jMLBF4ELgaKAeWm9lCd1/X7pgJwP3AHHevNrPh4a7dwEXu3mBmmcCa8Nxdp1JDcXEx5eXl7N+//4x/nsqD9Rzcbac9mynWUlNTKS4ujncZIjKAxHKa62ygzN23AJjZk8ANwLp2x9wBPOju1QDuvi/82tjumBROs6WTlJTE2LFjT+fUEzy7eD0/e20rK79xNTnpSVH5TBGRviyWXUwjgZ3tvi8Pt7U3EZhoZsvM7C0zm3tsh5mNMrMPws/4XmetBzO708xWmNmKaLQSulM6rYjmVufF9Xtjeh0Rkb4i3oPUEWACcDlwK/CwmQ0BcPed7n4uMB74opkVdDzZ3R9y91nuPis/Pz+mhZ5XnMOInFQWr94d0+uIiPQVsQyICmBUu++Lw23tlQML3b3J3bcCGwkCo03YclgDXBrDWk/KzCgtKeK1TQf67GC1iEg0xTIglgMTzGysmSUDtwALOxzzLEHrATPLI+hy2mJmxWaWFm4fClwCfBjDWntkXkkhjS2tvLx+X7xLERGJuZgFhLs3A3cDzwPrgafcfa2ZPWBm88PDngcqzWwdsBS4190rgcnA22b2PvAn4J/dvecPc4iR6aOGUpCdwuI16mYSkYEvpov1ufsiYFGHbd9s996Br4Wv9se8CJwby9pOR0KCUTqtiCfe2cGRhmYyUrTWoYgMXPEepO53SqcV0tDcytIP1c0kIgObAuIUzRozjLzMFBav3hPvUkREYkoBcYoSE4y50wp4ecM+6hpb4l2OiEjMKCBOQ+m0IuqaWvjTRnUzicjApYA4DReOHcbQ9CQWqZtJRAYwBcRpiCQmcO3UQv64fi/1TepmEpGBSQFxmkpLijjS2MLrmw7EuxQRkZhQQJymi8flkpOWxCLdNCciA5QC4jQlJSZw9ZQCXly3l8bmvvukORGR06WAOAPzSgo5VN/Mss3qZhKRgUcBcQbmjM8jKyWiJcBFZEBSQJyBlEgiV00ezgvr9tLUom4mERlYFBBnqLSkiJqjTby1pTLepYiIRJUC4gxdNjGf9ORE3TQnIgOOAuIMpSYlcuU5w3lh7R5aWj3e5YiIRI0CIgrmlRRReaSRd7ZWxbsUEZGoUUBEweWT8klNStCT5kRkQFFAREF6coQrJg1n8Zo9tKqbSUQGCAVElJSWFLH/UAMrd1THuxQRkahQQETJlecMJzmSwCLdNCciA4QCIkoyUyJ8fEI+S9TNJCIDhAIiiuaVFLL7YD3vldfEuxQRkTOmgIiiqyYXkJRoWptJRAYEBUQU5aQlccn4PBav2YO7uplEpH+LaUCY2Vwz+9DMyszsvi6OucnM1pnZWjN7PNx2vpm9GW77wMxujmWd0VRaUkR5dR1rKmrjXYqIyBmJWUCYWSLwIFAKTAFuNbMpHY6ZANwPzHH3qcA94a6jwH8Jt80FfmhmQ2JVazRdM6WASILpSXMi0u/FsgUxGyhz9y3u3gg8CdzQ4Zg7gAfdvRrA3feFXze6+6bw/S5gH5Afw1qjZkh6MheNy2Xx6t3qZhKRfi2WATES2Nnu+/JwW3sTgYlmtszM3jKzuR0/xMxmA8nA5k723WlmK8xsxf79+6NY+pmZV1LEtsqjrN99KN6liIictngPUkeACcDlwK3Aw+27ksysCPhP4MvufsITedz9IXef5e6z8vP7TgPjmikFJBham0lE+rVYBkQFMKrd98XhtvbKgYXu3uTuW4GNBIGBmWUDzwFfd/e3Ylhn1OVmpnDh2FyeUzeTiPRjsQyI5cAEMxtrZsnALcDCDsc8S9B6wMzyCLqctoTHPwP80t2fjmGNMTOvpJAt+4+wad/heJciInJaYhYQ7t4M3A08D6wHnnL3tWb2gJnNDw97Hqg0s3XAUuBed68EbgI+DnzJzN4LX+fHqtZYuHZqIWawWE+aE5F+ygZKF8isWbN8xYoV8S7jI2769zeprW9iyT0fj3cpIiKdMrOV7j6rs33xHqQe0EpLCtmw5xCb96ubSUT6HwVEDM2dVgjAkjXqZhKR/kcBEUNFOWnMGD1Ez4gQkX5JARFj80qKWLurlu2VR+JdiojIKVFAxNi1U4NupsXqZhKRfkYBEWOjhqVzbnGOnhEhIv2OAqIXlE4r4v3yg5RXH413KSIiPaaA6AWlms0kIv2QAqIXjMnLYEpRtsYhRKRfUUD0knklhazcXs2eg/XxLkVEpEcUEL2ktKQIgCVaAlxE+gkFRC8Zl5/JpIIsFqmbSUT6CQVEL5o7rZDl26rYd0jdTCLS9ykgetG8kiLc4fm1e+NdiojISSkgetHEgkzOzs/QTXMi0i8oIHqRmTFvWhFvb62i8nBDvMsREemWAqKXlZYU0tLqvLhO3Uwi0rcpIHrZlKJszspN12wmEenzFBC9zMwonVbEG2UHqDnaGO9yRES6pICIg3klhTSrm0lE+jgFRByUjMxh5JA0rc0kIn2aAiIOgm6mQl7btJ/a+qZ4lyMi0ikFRJyUlhTR1OL8cb26mUSkb1JAxMn0UUMozE5l8Wp1M4lI39SjgDCzvzKzbAv8zMxWmdk1sS5uIEtIMOZOK+SVjfs53NAc73JERE7Q0xbEn7l7LXANMBT4AvDdk51kZnPN7EMzKzOz+7o45iYzW2dma83s8Xbbl5hZjZn9oYc19jvzSopobG5l6YZ98S5FROQEPQ0IC7/OA/7T3de229b5CWaJwINAKTAFuNXMpnQ4ZgJwPzDH3acC97Tb/X2CIBqwZp41lPysFBbrGREi0gf1NCBWmtkLBAHxvJllAa0nOWc2UObuW9y9EXgSuKHDMXcAD7p7NYC7t/0p7e5/BA71sL5+KTHBmDu1kKUb9nO0Ud1MItK39DQg/itwH3CBux8FkoAvn+SckcDOdt+Xh9vamwhMNLNlZvaWmc3tYT0AmNmdZrbCzFbs37//VE7tM0pLCqlrauFPH/bP+kVk4OppQFwEfOjuNWZ2O/AN4GAUrh8BJgCXA7cCD5vZkJ6e7O4Pufssd5+Vn58fhXJ63+wxwxiWkay1mUSkz+lpQPwEOGpm5wF/DWwGfnmScyqAUe2+Lw63tVcOLHT3JnffCmwkCIxBI5KYwLVTC3h5/V7qm1riXY6ISJueBkSzuzvBGMKP3f1BIOsk5ywHJpjZWDNLBm4BFnY45lmC1gNmlkfQ5bSlhzUNGKXTijjS2MKrG9XNJCJ9R08D4pCZ3U8wq+g5M0sgGIfokrs3A3cDzwPrgafcfa2ZPWBm88PDngcqzWwdsBS4190rAczsNeA3wFVmVm5m157qD9dfXDQul5y0JJaom0lE+pBID4+7GbiN4H6IPWY2mmAaarfcfRGwqMO2b7Z778DXwlfHcy/tYW39XlJiAtdMKWDJ2j00NLeQEkmMd0kiIj1rQbj7HuBXQI6ZXQ/Uu/vJxiDkFMwrKeJQfTNvlFXGuxQREaDnS23cBLwDfA64CXjbzD4by8IGm4vH55KVGmHRat00JyJ9Q0+7mL5OcA/EPgAzywdeAp6OVWGDTUokkasnF/DCur38n5ZWkhK1jqKIxFdPfwsltL/LGag8hXOlh0pLijhY18Sbm9XNJCLx19Nf8kvM7Hkz+5KZfQl4jg6Dz3LmLp2QR0ZyotZmEpE+oaeD1PcCDwHnhq+H3P1vY1nYYJSalMiVkwt4fu1emltOttSViEhs9XQMAndfACyIYS0CzJtWyO/f38U726q4eFxevMsRkUGs2xaEmR0ys9pOXofMrLa3ihxMLp80nLSkRD1pTkTirtuAcPcsd8/u5JXl7tm9VeRgkpacyBXn5LNk7R5aWj3e5YjIIKaZSH1Q6bQi9h9qYOX26niXIiKDmAKiD7rinOGkRBJ005yIxJUCog/KTIlw2cR8lqzZQ6u6mUQkThQQAJtehObGeFfxEaUlheyprefdnTXxLkVEBikFxP6N8KvPwWOfgbq+0+d/1eQCkhKNxepmEpE4UUDkT4RP/xR2vAU/uwaqtsa7IgCyU5O4dEI+i9fsIVgVXUSkdykgAM67Gf7Ls3B4HzzyCdi5PN4VAVA6rZCKmjpWV0Tj8d8iIqdGAXHMmEvgz1+ClEz4xfWw9tl4V8TVUwqIJBiLdNOciMSBAqK9vAnw53+EwnPhN1+EZf8KcezeGZKezMXj81i8Zre6mUSk1ykgOsrIgy8uhCmfghe/CX/4KrQ0x62cedMK2V55lHW7tbKJiPQuBURnktLgsz+HS74KK38OT9wM9fH5BX3N1EISE0xrM4lIr1NAdCUhAT7x9/DJf4XNS+HnpXCwvNfLGJaRzIVjh7FotbqZRKR3KSBOZuaX4PanoWZHMMNp9/u9XkJpSRFbDhxh497DvX5tERm8FBA9Me5K+LPnwRLh0VL4cEmvXv7aqQWYobWZRKRXKSB6qmAK3PHHYKbTk7fC2w/12qWHZ6VywZhhLFmjcQgR6T0xDQgzm2tmH5pZmZnd18UxN5nZOjNba2aPt9v+RTPbFL6+GMs6eyyrEL68CCbOhcX3wpL7obWlVy49b1ohH+49RNk+dTOJSO+IWUCYWSLwIFAKTAFuNbMpHY6ZANwPzHH3qcA94fZhwLeAC4HZwLfMbGisaj0lyRlw82Nw4X+Dt/4Nfv0FaDwS88vOnVYEwJI16mYSkd4RyxbEbKDM3be4eyPwJHBDh2PuAB5092oAd98Xbr8WeNHdq8J9LwJzY1jrqUlIhNLvQuk/wcbF8B/XwaG9Mb1kYU4qM88aqruqRaTXxDIgRgI7231fHm5rbyIw0cyWmdlbZjb3FM6Nvwvvglseh/0fwiNXwd51Mb1c6bRC1u2uZduB2LdYRETiPUgdASYAlwO3Ag+b2ZCenmxmd5rZCjNbsX///hiVeBKTSuHLi6GlCR69Fja/HLNLlZYE3UyLNVgtIr0glgFRAYxq931xuK29cmChuze5+1ZgI0Fg9ORc3P0hd5/l7rPy8/OjWvwpGXF+MMMpZ1TwbIlVv4zJZUYOSeO84hwee2s7z75bQV1j7wyQi8jgFMuAWA5MMLOxZpYM3AIs7HDMswStB8wsj6DLaQvwPHCNmQ0NB6evCbf1XTnF8GdLYOxlsPB/wEvfhtbWqF/mb66dhBnc8+v3uOAfX+J/Pv0+b2+p1F3WIhJ1kVh9sLs3m9ndBL/YE4FH3X2tmT0ArHD3hRwPgnVAC3Cvu1cCmNk/EIQMwAPuXhWrWqMmNRtu+zUs+ht4/QdQvQ0+9RNISo3aJS6dkM+r917B21urWLCqnD98sJunVpQzelg6n5kxkhtnFDNqWHrUricig5cNlL88Z82a5StWrIh3GQH3YKnwl74Foy6EW56AjNyYXOpoYzNL1uzh6ZXlvLmlEneYPXYYn51RzLxzi8hMidnfACIyAJjZSnef1ek+BUQMrX0GfnsXZI+Azz8NeeNjermKmjqeWVXOglUVbD1whLSkROZOK+TGGcVcNC6XxASL6fVFpP9RQMTTznfgiVvBW4IpsWddHPNLujurdlTz9MoK/vDBLg7VN1OUk9rWBXV2fmbMaxCR/kEBEW9VW4PZTTXb4YYH4dybeu3S9U0tvLhuLwtWlfPqxv20OkwfPYQbZxTzyXNHkJOe1Gu1iEjfo4DoC45WBctybH8drvg6fPxesN7t8tlbW8+z71awYFU5G/ceJjmSwNVTCvjsjGIunZBHJDHet8WISG9TQPQVzY3BFNgPnoTzbgseRhRJ7vUy3J01FbUsWFXO796roPpoE/lZKXx6etAFNakwq9drEpH4UED0Je7wp+/BK9+BMZfCzf8JafFbh7CxuZWXN+xjwapylm7YR3OrM21kNjfOKOaG80cyLKP3A0xEeo8Coi96/9fwu7+AYWPh87+BoWPiXRGVhxtY+P4unl5ZztpdtUQSjCvPGc6NM4u5YtJwkiPqghIZaBQQfdW21+HJz0NiEtz6JBR3+r9RXGzYU8uCleU88+4uDhxuYFhGMvPPG8FnZxYzdUQ21svjJyISGwqIvuzAJvjVZ+HQHvjMwzBlfrwr+ojmllZe3bSfBSsreHHdXhpbWplUkMWNM0fyqfNHMjw7eneJi0jvU0D0dUcOBPdKlC+Ha/4BLrq712c49cTBo038/oNdLFhVzrs7akgwuGxiPjfOLOYTkwtITUqMd4kicooUEP1BUx088xVY9yzM+jMo/T4k9t1lMjbvPxx2QVWw+2A92akRrg+7oKaPGqIuKJF+QgHRX7S2wssPwOv/AuOvhs/9HFL69pTTllbnzc2VLFhVzuI1u6lvauXsvAxunFnMp6ePZMSQtHiXKCLdUED0Nyv/A/7wNRg+GW57CnL63sP0OnOovonFq/fw9Kpy3tlahRlcPC6XT0wu4NIJeYzLz1TLQqSPUUD0R5tfhqe+CMkZwRLiRefFu6JTsqPyKL99t5zfvbeLreEjUguyU5gzPo9LJ+QxZ1yeBrhF+gAFRH+1d12whlNdddDdNPHaeFd0WnZWHWVZ2QFeLzvAG5srqTrSCMDEgkwuGZ/PJRNymT02V0uTi8SBAqI/O7QHHr8Z9nwApf8Es++Id0VnpLXVWbe7ti0w3tlaRUNzK5EEY8boocwZn8clE3I5r3iI1oYS6QUKiP6u8Qgs+HP4cBF87C+CqbAJA2NKaX1TC6u2V/Na2QGWlR1gdcVB3CErJcKFZ+dyyfhcLpmQz7j8DI1fiMSAAmIgaG2B578Ob/8ERl8MUz8N466A3PF98p6J01VztJE3NlfyehgY2yuPAlCYndo2fnHx+FyGZ2n8QiQaFBADyfJHYNmPgmdLAGSPhLMvh7OvgLMvg8zh8awu6nZWHeX1Y+MXZQeoPtoEwDmFWUF31Pg8Zo8dRobGL0ROiwJiIKraCltegS1LYcufoL4m2D58ahAY466A0RdBysB5etyx8YvXyw7w+qYDvLOtisbmVpISjemjh3LJ+DwumZDHuSNzNH4h0kMKiIGutSUYxN68NAiNHW9BSwMkJMGo2cdbGCOm9+m7s09VfVMLK7dX89qmoDtqza7j4xcfG5cbTKcdn8fZeRq/EOmKAmKwaaoLQmJLGBi7PwAcUrKDZ1CMuyIIjQE2flF1pJE3w/GL18v2s7OqDoAROanh7Kg8Lh6XR35WSpwrFek7FBCD3ZFK2Pqn411SNTuC7dnFYevi8gE5frGj8mhbWCwrq+Rg3fHxi0vG5zFnQh4Xjh1GevLAaVWJnCoFhHxU1dbjrYuO4xfHWhdnXRzcxT1AtLQ663bV8lrZfpaVHWD5tuq28YsZ7cYvSjR+IYOMAkK61toCu98Pw+KVDuMXFx5vYXLR4hYAABKQSURBVAzA8Yvl26raptOuqagFICs1woVjc5kzPpc54/OYMFzrR8nAFreAMLO5wL8CicAj7v7dDvu/BHwfqAg3/djdHwn3fQ+4Ltz+D+7+6+6upYCIkqY62PHm8cBoG7/IgbGXHg+MATh+sazsAG9sDpYDOXb/RV5mChePCwLj4nF5jBqWHudKRaIrLgFhZonARuBqoBxYDtzq7uvaHfMlYJa7393h3OuAe4BSIAV4BbjK3Wu7up4CIkYG6fhFefVR3iirZFkYGPsPNQAwalgac8blcdG4XA14y4DQXUDEss9gNlDm7lvCIp4EbgDWdXtWYArwqrs3A81m9gEwF3gqVsVKFzJyYdpnghdA1ZbjrYsNf4D3Hgu2F0w7HhgDYPyieGg6N12Qzk0XjMLdKdt3OGxhVPLc6t08uXwnAJMKsrhoXNAddeHZw8hOTYpz5SLRE8sWxGeBue7+5+H3XwAubN9aCFsQ3wH2E7Q2vuruO83sGuBbBK2PdOAd4EF3/78drnEncCfA6NGjZ27fvj0mP4t0oUfjF5fB8CkD6oa9llZnTcVB3thcyRubD7B8WxX1Ta0kGJQUD2FOGBgzzxqqx7BKnxevLqaeBEQucNjdG8zsLuBmd78y3Pd14HME4bEPWO7uP+zqeupi6gMaj8LOt4Kw2Lw0uHnvmIzhMOxsGDY2+Dp07PHv04b26/GMhuYW3t1RwxtlB1i2uZL3dtbQ0uokRxKYOXooc8bnctG4PM4r1gwp6XviFRAXAX/v7teG398P4O7f6eL4RKDK3XM62fc48Ji7L+rqegqIPuhIJWxfBpVlQddU9bbga23FR49LzQkDo5PwyCyEhP71S/VwQzPLt1axLAyM9buDobPMlAgXjh3W1iU1qSCLhIT+G4wyMMRrDGI5MMHMxhLMUroFuK1DYUXuvjv8dj6wPtyeCAxx90ozOxc4F3ghhrVKLGTkwpT5J25vqoPq7VC9NQiMqvDr7vdh/e+htfn4sZE0GDqmXetj7PEwyRndJ6feZqZEuOKc4VxxTjBwf+wO72WbgwUH/7hhHwC5Gcl8bFwuc8blMWd8LqOHpWtKrfQpsZ7mOg/4IcE010fd/R/N7AFghbsvNLPvEARDM1AF/Dd332BmqcCq8GNqga+4+3vdXUstiAGipRkO7uwQHlvD77dCc93xYxMikDOq866roWdBUlr8fo5uVNTU8UY44L2s7AD7whlSI4ekhVNq87h4XK4eySq9QjfKycDgHjxhr2rLia2P6q1Qf/Cjx2eNCMNjzIldV6kn9GTGhbuzef+R4P6Lskre3HJ8SZDxwzOZMy4Yv7jo7Fxy0jVDSqJPASGDw9Gqdq2NDuFxeO9Hj03P/WhgHHufUxwMmielxWXg/NiSIG9sDsYvlm+toq6phQSDaSNzgvGLcXlcMGYYacmaISVnTgEh0nD4+CB5x+6r2nLw1o8en5gCaUOCsGj/Sj22rf2+du9TcqI6qN7Y3Mq7O6rbptS+u6OG5lZvewbGBWOGMvOsoUwfNZShGclRu64MHgoIke40NwR3iFdtDWZY1ddAXTXUHfva7n19DTQe7ubDLAiM1I7h0l3YhPsjJ78r+0hDM8u3VfHG5kre3FzJut21tLQG/w2fnZ/BjNFBYMwYPZQJwzM1S0pOSgEhEk3Nje1CpGOQtHvVd9xeA3Tz31tSRuctkxPCZmhbCB0lhdV7G1m5q45VO2tZub267bGsWakRzh81pC00zh89RHd6ywniNc1VZGCKJAdrT53q+lOtrdBQ202QdAidA2Xh+ypoaez0I9OBC8MXCUl4UhqtQ5OpJ5kjrUnU7kqkZnuE+leTWE4ySanpZGZmkZOVRe7QHHKysrGkNIikBq+k1GBqcadfU4OxmfZfNS13QFNAiPSWhISwVTAEGNvz89yDe0dOaJFUQ1N9MPU3/GpN9SQ215HRVE9Gcx3Dm+ppbjzK0SOHqa87QnPDAby6nuSqBhK3N9FsjSTRcvo/UyQ16BrrMlTafY0kB8uwJIavY+8TIpCY3O59UnSPU4idNgWESF9nBsnpwSt7xCmfHgGywxdAa6uzef9hVm6vZtWOat7bXknF/ipSaSTNGpmSn8z5halMLUjinNxkhqc51lwPzfVBUJ30a0Pwvr4Wmvcd39fcAK1N0BK+Wpui+a/UtYRIGB7JwY2VPQmcroKl2y75LvZ1eU43n3Wq5+RNguv+uevPO00KCJFBJiHBmFCQxYSCLG6ZPRqAmqONvLuzhlVhaDy4oYYjH7QAzeRlJjNjdDEzzgrGMkpG5kRnEUL3YMHH1qagC62l+XiAdAyStn2xPq45DLKTTETocldX+7rY3m3r5hTOiVHYKiBEhCHpyVwxaThXTArGVVpanQ/3HGLVjuq20HhhXXAvSSTBmDoimxnhbKmZZw1lxJDTuGvdLPiLPjHSZ+96H+w0i0lEeuTA4Qbe3VHDqh3VrNxezQflNdQ3BfePFGanBvdjjB7CjLOGMnVENikR3cjXH2gWk4icsbzMFK6eUsDVUwoAaGppZf3u2rCFUcPK7dU8tzpYezM5kkDJyJzwnoxgqq3Wlup/1IIQkajZW1vf1iW1cns1aypqaWwJWhkjh6QxbWQ200bkMG1k8NIjW+NPLQgR6RUF2amUlhRRWlIEBA9TWlNRy7s7qnm//CBrKg7y/Nq97Y5PoWRkDlNH5FAShkZBdoqWPe8jFBAiEjMpkURmhrOfjjlU38TaXbWsqTjI2l21rK44yB837Gub2ZmXmdKhpZHNyCFpCo04UECISK/KSk3iY2fn8rGzc9u2HWloZv3uIDRWV9SydtdBXtt0oG2dqaHpSW3dUkFwZOsBS71AASEicZeREmHWmGHMGjOsbVt9U0sQGrtqWVN+kDW7DvLIa1toaglCIys1wrQROZQU5zB1RDYlI3MYk5uhBQqjSAEhIn1SalIi00cPZfro491TDc0tbNxzmNUVQWCsqTjIfyzb1jYQnpkSYcqI7LZWRsnIHM7OzyRRoXFaFBAi0m+kRBIpKQ5aDcc0tbSyae9h1oShsbriII+/s73tHo20pMQwNLLbuqnGD88kKTF6z+0YqDTNVUQGnOaWVrYcOMLq8uMtjbW7ajnaGCxMmBJJ4JyiIDSOzZ6aWJBFcmTwhYaeByEig15Lq7P1wBHW7jrYFhxrK2o51NAMQFKiMakwq2321DmFWUwszBrwz9BQQIiIdKK11dlRdbSta2ptRTDt9mDd8cXvRuSkMikMi3MKs5hYkMW4/MzoLFjYB+hGORGRTiQkGGPyMhiTl8H15wZLqbs75dV1bNx7iA/3HuLDPcHr9bIDbTOoEhOMMbnpTCrMYlJBNpMKM5lUGEy9HUgD4goIEZF2zIxRw9IZNSydqyYXtG1vamll24EjbNhziI17D7FhzyHW7qpl8Zo9bTf5pUQSmFCQ+ZHQmFSQ1W/vDldAiIj0QFJiQttzNNo72tjMpr2H21obG/ce4tVN+1mwqrztmJy0JCYVZDGxXWhMKsgiJ71vj28oIEREzkB6coTzRg3hvFFDPrK96khj0E2153hX1e/e3cWhhh1txxRmB+Mbk8KxjXMKsxg/vO+Mb8Q0IMxsLvCvQCLwiLt/t8P+LwHfByrCTT9290fCff8EXAckAC8Cf+UDZURdRAa8YRnJJywp4u7sOljPxj0fHd94c3Nl281+CQZjcjOYWPDRgfExuelEevnejZgFhJklAg8CVwPlwHIzW+ju6zoc+mt3v7vDuRcDc4Bzw02vA5cBr8SqXhGRWDMzRg5JY+SQNK44Z3jb9uaWVrZVHm1rbRwLkOfXHR/fSI4kMD4/s2367aSCoOVRlJMas/GNWLYgZgNl7r4FwMyeBG4AOgZEZxxIBZIJHsyaBOzt9gwRkX4qkpjA+OGZjB+eyXUUtW2va2yhbN+x8Y1aPtx7mDc2V/LbdyvajslKjXDZxHx+fNuM6NcV9U88biSws9335cCFnRx3o5l9HNgIfNXdd7r7m2a2FNhNEBA/dvf1HU80szuBOwFGjx4d7fpFROIqLfnEpUUAao42snHv4TA0DsXsZr54D1L/HnjC3RvM7C7gF8CVZjYemAwUh8e9aGaXuvtr7U9294eAhyC4Ua4X6xYRiZsh6cnMHjuM2WOHnfzgMxDLEY8KYFS774s5PhgNgLtXuntD+O0jwMzw/aeBt9z9sLsfBhYDF8WwVhER6SCWAbEcmGBmY80sGbgFWNj+ADMravftfOBYN9IO4DIzi5hZEsEA9QldTCIiEjsx62Jy92Yzuxt4nmCa66PuvtbMHgBWuPtC4C/NbD7QDFQBXwpPfxq4ElhNMGC9xN1/H6taRUTkRFqsT0RkEOtusb7Bt/i5iIj0iAJCREQ6pYAQEZFOKSBERKRTA2aQ2sz2A9vP4CPygANRKifW+lOt0L/q7U+1Qv+qtz/VCv2r3jOp9Sx3z+9sx4AJiDNlZiu6Gsnva/pTrdC/6u1PtUL/qrc/1Qr9q95Y1aouJhER6ZQCQkREOqWAOO6heBdwCvpTrdC/6u1PtUL/qrc/1Qr9q96Y1KoxCBER6ZRaECIi0ikFhIiIdGrQB4SZzTWzD82szMzui3c93TGzR81sn5mtiXctJ2Nmo8xsqZmtM7O1ZvZX8a6pO2aWambvmNn7Yb3fjndNJ2NmiWb2rpn9Id61nIyZbTOz1Wb2npn16VU1zWyImT1tZhvMbL2Z9dln0ZjZpPDf9Nir1szuidrnD+YxCDNLJHjU6dUEj0RdDtzq7j15bnavCx/Nehj4pbtPi3c93Qmf9VHk7qvMLAtYCXyqD//bGpDh7ofDZ5C8DvyVu78V59K6ZGZfA2YB2e5+fbzr6Y6ZbQNmuXufv/HMzH4BvObuj4TPskl395p413Uy4e+zCuBCdz+Tm4bbDPYWxGygzN23uHsj8CRwQ5xr6pK7v0rw3Iw+z913u/uq8P0hggc+jYxvVV3zwOHw26Tw1Wf/ejKzYuA6gicxSpSYWQ7wceBnAO7e2B/CIXQVsDla4QAKiJHAznbfl9OHf4n1V2Y2BpgOvB3fSroXdtm8B+wDXnT3vlzvD4H/CbTGu5AecuAFM1tpZnfGu5hujAX2Az8Pu+8eMbOMeBfVQ7cAT0TzAwd7QEiMmVkmsAC4x91r411Pd9y9xd3PJ3h++mwz65PdeGZ2PbDP3VfGu5ZTcIm7zwBKgb8Iu0v7oggwA/iJu08HjgB9emwSIOwKmw/8JpqfO9gDogIY1e774nCbREHYl78A+JW7/zbe9fRU2KWwFJgb71q6MAeYH/brPwlcaWaPxbek7rl7Rfh1H/AMQfduX1QOlLdrPT5NEBh9XSmwyt33RvNDB3tALAcmmNnYMIFvARbGuaYBIRz0/Rmw3t1/EO96TsbM8s1sSPg+jWDiwob4VtU5d7/f3YvdfQzB/2dfdvfb41xWl8wsI5yoQNhdcw3QJ2fiufseYKeZTQo3XQX0yYkVHdxKlLuXIGhODVru3mxmdwPPA4nAo+6+Ns5ldcnMngAuB/LMrBz4lrv/LL5VdWkO8AVgddivD/C/3H1RHGvqThHwi3AmSALwlLv3+emj/UQB8EzwNwMR4HF3XxLfkrr1P4BfhX80bgG+HOd6uhWG7tXAXVH/7ME8zVVERLo22LuYRESkCwoIERHplAJCREQ6pYAQEZFOKSBERKRTCgiRODKzy/vDaqwyOCkgRESkUwoIkR4ws9vD50W8Z2Y/DRf2uzV8xsEaM/teu2MPm9m/hM+V+KOZ5Yfbx5vZS+EzJ1aZ2bjwlMx2zx/4VXgXOmb23fB5Gh+Y2T/H4ceWQU4BIXISZjYZuBmYEy7m1wJ8HvgecCVwPnCBmX0qPCUDWOHuU4E/Ad8Kt/8KeNDdzwMuBnaH26cD9wBTgLOBOWaWC3wamOru5wL/O7Y/pciJFBAiJ3cVMBNYHi4bchXwVeAVd9/v7s0Ev/yPrVDaCvw6fP8YcEm4FtFId38GwN3r3f1oeMw77l7u7q3Ae8AY4CBQD/zMzD4DHDtWpNcoIEROzoBfuPv54WsS8PencP7J1rNpaPe+BYiEoTObYDXR64G+vHaRDFAKCJGT+yPwWTMbDmBmwwj+0r/MzPLCBf5uJehOguC/q8+G728DXg+fqld+rBvKzFLMLL2rC4bP0cgJFzf8KnBeDH4ukW4N6tVcRXrC3deZ2TcInoiWADQBf0HwIJmlBC2M59z9d+EpRwgeOPQNgqfT3Rxu/wLwUzN7IPyMz3Vz2Szgd2aWGn7+16L8Y4mclFZzFYkyMzvs7pnxrkPkTKmLSUREOqUWhIiIdEotCBER6ZQCQkREOqWAEBGRTikgRESkUwoIERHp1P8HR6L2+ITrf90AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/gdrive/My Drive/Machine Learning/movielens1m/Saved Models/CFpure_Recommender_V_71m/assets\n",
            "(879409,) (879409,)\n",
            "\n",
            "(120800,) (120800,)\n",
            "\n",
            "(879409,) (879409,)\n",
            "\n",
            "(120800,) (120800,)\n",
            "\n",
            "(879409,) (879409,)\n",
            "\n",
            "(120800,) (120800,)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3kfEwMhj7nb4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "8e119cc2-3cd6-4888-8bdc-992109830f4c"
      },
      "source": [
        "display(df1) \n",
        "# val_loss instead of loss\n",
        "# hparam['lr'] = 0.001\n",
        "# hparam['embeddings_initializer'] = 'glorot_uniform'\n",
        "# hparam['embedding_regularization'] = 1+e-6\n",
        "# hparam['embedding_size'] = 30\n",
        "# hparam['pure_CF'] = 0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model</th>\n",
              "      <th>rmse train</th>\n",
              "      <th>rmse test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bias Baseline</td>\n",
              "      <td>0.237321</td>\n",
              "      <td>0.244797</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CF</td>\n",
              "      <td>0.194291</td>\n",
              "      <td>0.236018</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CFpure</td>\n",
              "      <td>0.197200</td>\n",
              "      <td>0.236195</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           model  rmse train  rmse test\n",
              "0  bias Baseline    0.237321   0.244797\n",
              "0             CF    0.194291   0.236018\n",
              "0         CFpure    0.197200   0.236195"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TQN7BCGEdmGJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "b5369859-f1bb-4914-bf1e-a7766b5be8cb"
      },
      "source": [
        "display(df1) \n",
        "#loss\n",
        "# hparam['lr'] = 0.001\n",
        "# hparam['embeddings_initializer'] = 'glorot_uniform'\n",
        "# hparam['embedding_regularization'] = 1e-6\n",
        "# hparam['embedding_size'] = 30\n",
        "# hparam['pure_CF'] = 0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model</th>\n",
              "      <th>rmse train</th>\n",
              "      <th>rmse test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bias Baseline</td>\n",
              "      <td>0.237321</td>\n",
              "      <td>0.244797</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CF</td>\n",
              "      <td>0.174360</td>\n",
              "      <td>0.236544</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CFpure</td>\n",
              "      <td>0.174650</td>\n",
              "      <td>0.236769</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           model  rmse train  rmse test\n",
              "0  bias Baseline    0.237321   0.244797\n",
              "0             CF    0.174360   0.236544\n",
              "0         CFpure    0.174650   0.236769"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nufVdeDTZU46",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "5f3e315a-c865-4ff8-90da-f9ba777e1da9"
      },
      "source": [
        "display(df1) #tuned model 0.01,he,40,1e-6,patience 1, loss"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model</th>\n",
              "      <th>rmse train</th>\n",
              "      <th>rmse test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bias Baseline</td>\n",
              "      <td>0.237321</td>\n",
              "      <td>0.244797</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CF</td>\n",
              "      <td>0.183675</td>\n",
              "      <td>0.242563</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CFpure</td>\n",
              "      <td>0.183939</td>\n",
              "      <td>0.243599</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           model  rmse train  rmse test\n",
              "0  bias Baseline    0.237321   0.244797\n",
              "0             CF    0.183675   0.242563\n",
              "0         CFpure    0.183939   0.243599"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2zAb6Yx9NJt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, loss, embedding dim 60,lr 0.002"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WNmxodw1_NG0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, loss, embedding dim 70,lr 0.002"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hybX-m_aB0L2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, loss, embedding dim 60,lr 0.002 ep = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LlUNORgvHI9G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, loss, embedding dim 50,lr 0.002 ep = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKLmzbHafa4V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8oN9y7tT2Mjy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, val_loss, embedding dim 60,lr 0.002 ,ep 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YnX7d_23Kfhk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, val_loss, embedding dim 60,lr 0.002 ,ep 20 reg 1e-3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wPX7tmPtMuMB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "display(df1) #1m, patience 1, loss, embedding dim 60,lr 0.002 ,ep 20 reg 1e-3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NV2TsF_5QFtt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Mb55VRq0t97",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v1z13--YcWRi",
        "colab_type": "text"
      },
      "source": [
        "# Old rmse evaluation\n",
        "\n",
        "doesnt work, remove this code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sEs5ZYmPMHIZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7LHs8_ishDEq",
        "colab_type": "text"
      },
      "source": [
        "Evaluate model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vw9f4niQ7UB-",
        "colab_type": "text"
      },
      "source": [
        "RMSE evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nC__UfK8jgUl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## utility function to calculate rmse\n",
        "def rmse(predictions, targets):  \n",
        "  print(np.squeeze(predictions).shape, np.squeeze(targets).shape)\n",
        "  print('')\n",
        "  return np.sqrt(np.mean((np.squeeze(predictions)-np.squeeze(targets))**2))\n",
        "\n",
        "## function to return datafrmae of rmse evalutaion result\n",
        "def rmse_evaluate(model, x_train, x_test, y_train, y_test, model_name ):\n",
        "  BATCH_SIZE_P = 10\n",
        "\n",
        "  ratings_train = model.predict(np.reshape(np.array(x_train.values),(-1,4)), batch_size = BATCH_SIZE_P).flatten() ##remove .iloc for V1 model\n",
        "  ratings_test = model.predict(np.reshape(np.array(x_test.values),(-1,4)), batch_size = BATCH_SIZE_P).flatten() ##remove .iloc for V1 model\n",
        "\n",
        "  rmse_train = rmse(ratings_train, y_train)\n",
        "  rmse_test = rmse(ratings_test, y_test)\n",
        "\n",
        "  data = {'model':[model_name], 'rmse train':[rmse_train], 'rmse test':[rmse_test]}\n",
        "  df = pd.DataFrame(data)\n",
        "\n",
        "  return df\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s000m--qpED4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def rmse_evaluate_main(df_ratings):\n",
        "  # x_train, x_test, y_train, y_test, global_avg = prepare_data(df_ratings)\n",
        "  \n",
        "  ############### prepare data #########\n",
        "  x_train, x_test, y_train, y_test, global_avg = prepare_data(df_ratings)\n",
        "\n",
        "  ############# get model ##############\n",
        "  num_users = df_ratings['userId'].unique().max() +1\n",
        "  num_movies = df_ratings['movieId'].unique().max() +1\n",
        "  EMBEDDING_SIZE = 50\n",
        "\n",
        "  lr =  0.002 #2e-2 tune this\n",
        "  \n",
        "  model_CF = CFmodel(num_users = num_users, num_movies=num_movies, EMBEDDING_SIZE =EMBEDDING_SIZE, global_avg = global_avg, lr = lr)\n",
        "  model_base = Basemodel( global_avg = global_avg)\n",
        "  # model_mu = mumodel( global_avg = global_avg)\n",
        "  model_CF_pure = CFpure(num_users = num_users, num_movies=num_movies, EMBEDDING_SIZE =EMBEDDING_SIZE, global_avg = global_avg, lr = lr)\n",
        "  ############ train/load model #############\n",
        "  train = True # True\n",
        "  \n",
        "  if train or not os.path.isdir(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '6'+'refined'):\n",
        "    model_CF = train_CFmodel(model_CF, x_train, y_train, x_test, y_test)\n",
        "  else:\n",
        "    model_CF = keras.models.load_model(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '6'+'refined')\n",
        "    \n",
        "  save = True #True\n",
        "  if save:\n",
        "    model_CF.save(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '6'+ 'refined') #'_mu' + '_bias'+'_meaningful_embeddings_allembeddings_he_initialized'\n",
        " \n",
        "  model_CF_pure = train_CFmodel(model_CF_pure, x_train, y_train, x_test, y_test)\n",
        "\n",
        "  # model = keras.models.load_model(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '5'+ '_mu' + '_bias'+'_meaningful_embeddings_allembeddings_he_initialized')\n",
        "  ############ evaluate models #########\n",
        "  # df = rmse_evaluate(model_mu,x_train, x_test, y_train, y_test,\"mu Baseline\")\n",
        "\n",
        "\n",
        "  df = rmse_evaluate(model_base,x_train, x_test, y_train, y_test,\"bias Baseline\")\n",
        "  df = df.append(rmse_evaluate(model_CF,x_train, x_test, y_train, y_test,\"CF\"))\n",
        "  df = df.append(rmse_evaluate(model_CF_pure,x_train, x_test, y_train, y_test,\"CFpure\"))\n",
        "  return(df, model_CF, model_base, model_CF_pure)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MRorrGzfEodG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2887
        },
        "outputId": "a8d4f817-d5b6-4acf-a313-e925b62f7b3c"
      },
      "source": [
        "df, model_CF, model_base, model_CF_pure = rmse_evaluate_main(df_ratings)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:85: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:86: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:88: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:89: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"cf_recommender\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        multiple                  302050    \n",
            "_________________________________________________________________\n",
            "embedding_1 (Embedding)      multiple                  6041      \n",
            "_________________________________________________________________\n",
            "embedding_2 (Embedding)      multiple                  197650    \n",
            "_________________________________________________________________\n",
            "embedding_3 (Embedding)      multiple                  3953      \n",
            "=================================================================\n",
            "Total params: 509,694\n",
            "Trainable params: 509,694\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Model: \"cf_recommender_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_4 (Embedding)      multiple                  302050    \n",
            "_________________________________________________________________\n",
            "embedding_5 (Embedding)      multiple                  6041      \n",
            "_________________________________________________________________\n",
            "embedding_6 (Embedding)      multiple                  197650    \n",
            "_________________________________________________________________\n",
            "embedding_7 (Embedding)      multiple                  3953      \n",
            "=================================================================\n",
            "Total params: 509,694\n",
            "Trainable params: 509,694\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/10\n",
            "WARNING:tensorflow:Layer cf_recommender is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.6158 - val_loss: 0.7102\n",
            "Epoch 2/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5970 - val_loss: 0.7160\n",
            "Epoch 3/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5914 - val_loss: 0.7197\n",
            "Epoch 4/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5883 - val_loss: 0.7215\n",
            "Epoch 5/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5867 - val_loss: 0.7225\n",
            "Epoch 6/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5860 - val_loss: 0.7230\n",
            "Epoch 7/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5854 - val_loss: 0.7234\n",
            "Epoch 8/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5851 - val_loss: 0.7236\n",
            "Epoch 9/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5849 - val_loss: 0.7237\n",
            "Epoch 10/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5847 - val_loss: 0.7238\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3wddZ3/8dc7J0mT9H5F2hRalxa5WiBUsKgICxZEEHUrKCu6u1RX8bbKLvjzinth3V11xXoBZVdFQBYEi6IFFFTk1hQr0FJoKcWmXBrSG23T5vb5/TGT5CSd0pRmetrk/Xw8zuPMfOc7cz7nQOedmTnnO4oIzMzMeisrdQFmZrZvckCYmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEWT+Q9L+S/rmPfVdJ+ss93Y5Z3hwQZmaWyQFhZmaZHBA2aKSndi6R9IikLZK+L+kASb+U9JKkuySNLup/tqQlkjZIukfSYUXLjpH0cLreT4CqXq91lqTF6br3STr6FdZ8kaQVktZJmi9pYtouSV+TtFbSJkmPSjoyXXampKVpbWskffoVfWA26DkgbLB5J3AaMB14G/BL4DPAeJJ/Dx8DkDQduB74RLrsduA2SZWSKoFbgR8BY4D/S7dLuu4xwDXAB4GxwHeB+ZKG7E6hkk4B/g2YAxwIPAPckC4+HXhj+j5Gpn2a0mXfBz4YEcOBI4Hf7M7rmnVyQNhgc2VEvBARa4DfAw9GxB8jYhtwC3BM2u/dwC8i4s6IaAX+E6gGXg+cAFQAX4+I1oi4CVhY9Bpzge9GxIMR0R4RPwC2p+vtjvcC10TEwxGxHbgMOFHSFKAVGA68BlBEPB4Rz6XrtQKHSxoREesj4uHdfF0zwAFhg88LRdPNGfPD0umJJH+xAxARHcBqYFK6bE30HOnymaLpg4FPpaeXNkjaAExO19sdvWvYTHKUMCkifgN8E5gHrJV0laQRadd3AmcCz0j6raQTd/N1zQAHhNnOPEuyoweSc/4kO/k1wHPApLSt00FF06uBf4mIUUWPmoi4fg9rGEpyymoNQER8IyKOAw4nOdV0Sdq+MCLOASaQnAq7cTdf1wxwQJjtzI3AWyWdKqkC+BTJaaL7gPuBNuBjkiokvQOYWbTu1cCHJL0uvZg8VNJbJQ3fzRquBz4gaUZ6/eJfSU6JrZJ0fLr9CmALsA3oSK+RvFfSyPTU2CagYw8+BxvEHBBmGSLiCeAC4ErgRZIL2m+LiJaIaAHeAbwfWEdyveKnRevWAxeRnAJaD6xI++5uDXcBnwNuJjlq+QvgvHTxCJIgWk9yGqoJ+I902V8DqyRtAj5Eci3DbLfJNwwyM7MsPoIwM7NMDggzM8vkgDAzs0wOCDMzy1Re6gL6y7hx42LKlCmlLsPMbL+yaNGiFyNifNayARMQU6ZMob6+vtRlmJntVyQ9s7NlPsVkZmaZHBBmZpbJAWFmZpkGzDWILK2trTQ0NLBt27ZSl5K7qqoqamtrqaioKHUpZjZADOiAaGhoYPjw4UyZMoWeA28OLBFBU1MTDQ0NTJ06tdTlmNkAMaBPMW3bto2xY8cO6HAAkMTYsWMHxZGSme09AzoggAEfDp0Gy/s0s71nQJ9iMrP9RARER/ejo71ovr3n8uJlFK8X3f12aO/dP3Z8zcx1og/b6uxHxjq9prvW65xmJ+1Z7yWK2nttd8REqPtAv/9ncUDkbMOGDVx33XV8+MMf3q31zjzzTK677jpGjRqVU2VWMhHJTq6jFdrTR9d0C3S07WS6PZnv8ejd1jnfuovlGW3tfVxnpzvyjr4vy9rR2ytXe7wDYn+0YcMGvvWtb+0QEG1tbZSX7/zjv/322/MubfDpaIfWZmjdCi1bMqa3QMvWXtNboW37TnbmezC9t3eIZeVFj8Iu5ns/ClA+pHtaBSgrA3U+CkXTZb2W9VpeVgBp95dlLi96UNyu7n47tPfu3/t5Z+sou733dlD365NVh15mmuz2HtvtNZ3zqWUHRM4uvfRSnnrqKWbMmEFFRQVVVVWMHj2aZcuW8eSTT/L2t7+d1atXs23bNj7+8Y8zd+5coHvokM2bN3PGGWdw0kkncd999zFp0iR+9rOfUV1dXeJ3lqPWZtjaBNs29W0H3rq1b9Ntu3sRX1BRk+wcCxVQqEx2koUKKKtI29Lp8iFQGLZje6E8Xa93e0W6rcpdTKfbKF5nlzv3jADwNSp7BXINCEmzgf8GCsD3IuKKXsu/Brw5na0BJkTEKEkzgG+T3FaxneQG8D/Zk1q+dNsSlj67aU82sYPDJ47gC2874mX7XHHFFTz22GMsXryYe+65h7e+9a089thjXV9HveaaaxgzZgzNzc0cf/zxvPOd72Ts2LE9trF8+XKuv/56rr76aubMmcPNN9/MBRdc0K/vJRcRyc59axM0r4Ot6aN5XdK2dV3RsibYuj55bmvu4wukO/DKGqiohoqh6XQNDDuge7qrT9b00HTd4ul0O+VV3rHaoJZbQEgqAPOA04AGYKGk+RGxtLNPRHyyqP9HgWPS2a3A+yJiuaSJwCJJCyJiQ1717i0zZ87s8VuFb3zjG9xyyy0ArF69muXLl+8QEFOnTmXGjBkAHHfccaxatWqv1dslArZv6t6R73Qn3ysE2lt2skFB9SioHgM1Y2HEJHjV0VA9OpmvGQNVI3vu9Lt27p078mrvwM1ylOcRxExgRUSsBJB0A3AOsHQn/c8HvgAQEU92NkbEs5LWAuOBVxwQu/pLf28ZOnRo1/Q999zDXXfdxf33309NTQ0nn3xy5m8ZhgwZ0jVdKBRobu7rX9h91LYd1j0NTcvhxSdh/TPpTn99906/eV1ygTKLytIdfbqzHz0FJh3bPV+8rDMQqkclpz7MbJ+VZ0BMAlYXzTcAr8vqKOlgYCrwm4xlM4FK4KmMZXOBuQAHHXTQnlecg+HDh/PSSy9lLtu4cSOjR4+mpqaGZcuW8cADD+RXSARsaUwC4MXl0LQieX7xSdjwTPdX7gBqxsHQ8clOfdy07p167518TbrjHzIyuTBpZgPKvnKR+jzgpohoL26UdCDwI+DCiOI9WCIirgKuAqirq9snvyc3duxYZs2axZFHHkl1dTUHHHBA17LZs2fzne98h8MOO4xDDz2UE044Yc9fsHUbrFuZHg2kj6bl8OIK2L6xu195FYw9BA58LRz1V0kQjD0keVSN2PM6zGy/p4h89quSTgS+GBFvSecvA4iIf8vo+0fgIxFxX1HbCOAe4F8j4qZdvV5dXV30vmHQ448/zmGHHbYnb2PfFJGc7mnblj62Q9t2Hl+xisN+9a6eRwPDJ8K4Q2DcdBg7rXt6RK3/6jczJC2KiLqsZXkeQSwEpkmaCqwhOUp4T0ZxrwFGA/cXtVUCtwA/7Es4DFgdHdC+vSgEusOAHgdbZenXLCvhjZekQZAeEQwZVrLyzWz/lltARESbpIuBBSRfc70mIpZIuhyoj4j5adfzgBui56HMHOCNwFhJ70/b3h8Ri/Oqt6Qikp1/y+aeQdD7G0BlFcmpoerRyXP5kOS5UJF8m6cpoO4zpXkPZjbg5HoNIiJuB27v1fb5XvNfzFjvWuDaPGsrqYj0R1ybYfuW5LnziEBlUBiSfJWzZkwy3RkG/taPme1F+8pF6oEtOpJf9LZsTh9buq8TFIYkX/msHJb8UKtQ6e/2m9k+wQGRh46OdCiIzbB9cxIOpIFQXpV8TXRIUSCYme2DHBD9oaM9OSroOkLYStdgbOXVMHRs0RGCbwlqZvsHf8/xlehog20bYeMaaHwCnn8E1j0Fm19Iri8MHQ9jXg2vOooNla/iWz+en5xG2s1w+PrXv87WrVtzehNmZi/PAdEX7a3QvAE2NsDaZfD8o8mP0bY0JtcLhh0AY/4iGUto/KEwclIyjlBZeddw36+EA8LMSsmnmLK0t6TXDtLTRl3DRJclg8UNf1Vyyqhi6C5/bFY83Pdpp53GhAkTuPHGG9m+fTvnnnsuX/rSl9iyZQtz5syhoaGB9vZ2Pve5z/HCCy/w7LPP8uY3v5lx48Zx99135/++zcyKDJ6A+OWlyV/+WTpva9jRnt7esPOXyCq6QUqh+2YdnV51FJxxRdYWuxQP933HHXdw00038dBDDxERnH322fzud7+jsbGRiRMn8otf/AJIxmgaOXIkX/3qV7n77rsZN27cnr9/M7Pd5FNMkX7jqG1bcm2h63cI6f0ByqvTr54W6BEOr8Add9zBHXfcwTHHHMOxxx7LsmXLWL58OUcddRR33nkn//RP/8Tvf/97Ro4c2T/vzcxsDwyeI4id/aUfkQxpXTk09xvERASXXXYZH/zgB3dY9vDDD3P77bfz2c9+llNPPZXPf/7zGVswM9t7fAQhwdBxud18pni477e85S1cc801bN68GYA1a9awdu1ann32WWpqarjgggu45JJLePjhh3dY18xsbxs8RxAlUjzc9xlnnMF73vMeTjzxRACGDRvGtddey4oVK7jkkksoKyujoqKCb3/72wDMnTuX2bNnM3HiRF+kNrO9Lrfhvve2QTXc904MtvdrZnvu5Yb79ikmMzPL5IAwM7NMAz4gBsoptF0ZLO/TzPaeAR0QVVVVNDU1DfidZ0TQ1NREVVVVqUsxswFkQH+Lqba2loaGBhobG0tdSu6qqqqora0tdRlmNoAM6ICoqKhg6tSppS7DzGy/NKBPMZmZ2SuXa0BImi3pCUkrJF2asfxrkhanjyclbShadqGk5enjwjzrNDOzHeV2iklSAZgHnAY0AAslzY+IpZ19IuKTRf0/ChyTTo8BvgDUkdyabVG67vq86jUzs57yPIKYCayIiJUR0QLcAJzzMv3PB65Pp98C3BkR69JQuBOYnWOtZmbWS54BMQlYXTTfkLbtQNLBwFTgN7uzrqS5kuol1Q+GbyqZme1N+8pF6vOAmyKifXdWioirIqIuIurGjx+fU2lmZoNTngGxBphcNF+btmU5j+7TS7u7rpmZ5SDPgFgITJM0VVIlSQjM791J0muA0cD9Rc0LgNMljZY0Gjg9bTMzs70kt28xRUSbpItJduwF4JqIWCLpcqA+IjrD4jzghigaDyMi1kn6MknIAFweEevyqtXMzHY0oO8HYWZmL8/3gzAzs93mgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8uUa0BImi3pCUkrJF26kz5zJC2VtETSdUXtX0nbHpf0DUnKs1YzM+upPK8NSyoA84DTgAZgoaT5EbG0qM804DJgVkSslzQhbX89MAs4Ou16L/Am4J686jUzs57yPIKYCayIiJUR0QLcAJzTq89FwLyIWA8QEWvT9gCqgEpgCFABvJBjrWZm1kueATEJWF0035C2FZsOTJf0B0kPSJoNEBH3A3cDz6WPBRHxeI61mplZL7mdYtqN158GnAzUAr+TdBQwDjgsbQO4U9IbIuL3xStLmgvMBTjooIP2Vs1mZoNCnkcQa4DJRfO1aVuxBmB+RLRGxNPAkySBcS7wQERsjojNwC+BE3u/QERcFRF1EVE3fvz4XN6EmdlglWdALASmSZoqqRI4D5jfq8+tJEcPSBpHcsppJfBn4E2SyiVVkFyg9ikmM7O9KLeAiIg24GJgAcnO/caIWCLpcklnp90WAE2SlpJcc7gkIpqAm4CngEeBPwF/iojb8qrVzMx2pIgodQ39oq6uLurr60tdhpnZfkXSooioy1rmX1KbmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllckCYmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllckCYmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllckCYmVkmB4SZmWXKNSAkzZb0hKQVki7dSZ85kpZKWiLpuqL2gyTdIenxdPmUPGs1M7OeyvPasKQCMA84DWgAFkqaHxFLi/pMAy4DZkXEekkTijbxQ+BfIuJOScOAjrxqNTOzHeV5BDETWBERKyOiBbgBOKdXn4uAeRGxHiAi1gJIOhwoj4g70/bNEbE1x1rNzKyXPANiErC6aL4hbSs2HZgu6Q+SHpA0u6h9g6SfSvqjpP9Ij0h6kDRXUr2k+sbGxlzehJnZYFXqi9TlwDTgZOB84GpJo9L2NwCfBo4HXg28v/fKEXFVRNRFRN348eP3Vs1mZoNCnwJC0scljVDi+5IelnT6LlZbA0wumq9N24o1APMjojUingaeJAmMBmBxenqqDbgVOLYvtZqZWf/o6xHE30TEJuB0YDTw18AVu1hnITBN0lRJlcB5wPxefW4lOXpA0jiSU0sr03VHSeo8LDgFWIqZme01fQ0Ipc9nAj+KiCVFbZnSv/wvBhYAjwM3RsQSSZdLOjvttgBokrQUuBu4JCKaIqKd5PTSryU9mr7W1bvzxszMbM8oInbdSfofkgvMU4HXAgXgnog4Lt/y+q6uri7q6+tLXYaZ2X5F0qKIqMta1tffQfwtMANYGRFbJY0BPtBfBZqZ2b6nr6eYTgSeiIgNki4APgtszK8sMzMrtb4GxLeBrZJeC3wKeIrkl85mZjZA9TUg2iK5WHEO8M2ImAcMz68sMzMrtb5eg3hJ0mUkX299g6QyoCK/sszMrNT6egTxbmA7ye8hnif50dt/5FaVmZmVXJ8CIg2FHwMjJZ0FbIsIX4MwMxvA+jrUxhzgIeCvgDnAg5LelWdhZmZWWn29BvH/gOOLhuMeD9wF3JRXYWZmVlp9vQZR1hkOqabdWNfMzPZDfT2C+JWkBcD16fy7gdvzKcnMzPYFfQqIiLhE0juBWWnTVRFxS35lmZlZqfX5ntQRcTNwc461mJnZPuRlA0LSS0DWcK8CIiJG5FKVmZmV3MsGRER4OA0zs0HK30QyM7NMDggzM8vkgDAzs0wOCDMzy5RrQEiaLekJSSskXbqTPnMkLZW0RNJ1vZaNkNQg6Zt51mlmZjvq8+8gdpekAjAPOA1oABZKmh8RS4v6TAMuA2ZFxHpJE3pt5svA7/Kq0czMdi7PI4iZwIqIWBkRLcANJHekK3YRMC8i1gMUj/ck6TjgAOCOHGs0M7OdyDMgJgGri+Yb0rZi04Hpkv4g6QFJswHSO9b9F/Dpl3sBSXMl1Uuqb2xs7MfSzcys1Bepy4FpwMnA+cDVkkYBHwZuj4iGl1s5Iq6KiLqIqBs/fnzuxZqZDSa5XYMA1gCTi+Zr07ZiDcCDEdEKPC3pSZLAOJHk3tcfBoYBlZI2R0TmhW4zM+t/eR5BLASmSZoqqRI4D5jfq8+tJEcPSBpHcsppZUS8NyIOiogpJKeZfuhwMDPbu3ILiIhoAy4GFgCPAzdGxBJJl0s6O+22AGiStBS4G7gkIpryqsnMzPpOEVmDte5/6urqor6+vtRlmJntVyQtioi6rGWlvkhtZmb7KAeEmZllckCYmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllckCYmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllckCYmVkmB4SZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllyjUgJM2W9ISkFZIu3UmfOZKWSloi6bq0bYak+9O2RyS9O886zcxsR+V5bVhSAZgHnAY0AAslzY+IpUV9pgGXAbMiYr2kCemircD7ImK5pInAIkkLImJDXvWamVlPeR5BzARWRMTKiGgBbgDO6dXnImBeRKwHiIi16fOTEbE8nX4WWAuMz7FWMzPrJc+AmASsLppvSNuKTQemS/qDpAckze69EUkzgUrgqYxlcyXVS6pvbGzsx9LNzKzUF6nLgWnAycD5wNWSRnUulHQg8CPgAxHR0XvliLgqIuoiom78eB9gmJn1pzwDYg0wuWi+Nm0r1gDMj4jWiHgaeJIkMJA0AvgF8P8i4oEc6zQzswx5BsRCYJqkqZIqgfOA+b363Epy9ICkcSSnnFam/W8BfhgRN+VYo5mZ7URuARERbcDFwALgceDGiFgi6XJJZ6fdFgBNkpYCdwOXREQTMAd4I/B+SYvTx4y8al2zoTmvTZuZ7bcUEaWuoV/U1dVFfX39bq+36sUtnPmN3/OOYyfx+bOOoLK81JdlzMz2HkmLIqIua9mg3xvWjq7mghMO5toH/sx5V93P8xu3lbokM7N9wqAPiPJCGZ858zC++Z5jWPb8S5x15b08uLKp1GWZmZXcoA+ITmcdPZFbPzKLEVXlvOd7D3LNvU8zUE6/mZm9Eg6IItMPGM6tF8/ilNdM4PKfL+UTP1nM1pa2UpdlZlYSDoheRlRV8N0LjuPTp09n/p+e5R3fuo9nmraUuiwzs73OAZGhrExcfMo0/vcDM3lu4zbeduW93L1sbanLMjPbqxwQL+NN08fz84+eRO3oGv7mBwv5+l1P0tHh6xJmNjg4IHZh8pgabv7713PujEl8/a7lXPTDejY2t5a6LDOz3Dkg+qC6ssB/zXktl59zBL99spGzv3kvy57fVOqyzMxy5YDoI0m878Qp3DD3BJpb2jl33n38bHHvsQfNzAYOB8Ruqpsyhp9/7CSOnDSCj9+wmC//fCmt7TuMRG5mtt9zQLwCE4ZXcd1FJ/D+10/h+/c+zXu/9yCNL20vdVlmZv3KAfEKVRTK+OLZR/D1d8/gkYYNnHXl73n4z+tLXZaZWb9xQOyhtx8ziZ/+/SyGlBd493fv59oHnvEQHWY2IDgg+sHhE0dw28UnMeuQcXz21sf4x5seYVtre6nLMjPbIw6IfjKypoJrLjyej506jf9b1MC7vnMfDeu3lrosM7NXzAHRj8rKxD+cNp3vva+OZ5q28rYr7+X3yxtLXZaZ2SvigMjBXx5+ALddfBIThldx4TUP8a17Vvi6hJntdxwQOZkybii3fOT1vPXoiXzlV0/woWsX8dI2D9FhZvuPXANC0mxJT0haIenSnfSZI2mppCWSritqv1DS8vRxYZ515qWmspxvnDeDz511OHc9vpZz5v2BFWtfKnVZZmZ9kltASCoA84AzgMOB8yUd3qvPNOAyYFZEHAF8Im0fA3wBeB0wE/iCpNF51ZonSfztSVP58d+9jk3NrZzzzT/wy0efK3VZZma7lOcRxExgRUSsjIgW4AbgnF59LgLmRcR6gIjovOnCW4A7I2JduuxOYHaOtebuhFeP5baPnsT0Vw3n73/8MFf8chltHqLDzPZheQbEJGB10XxD2lZsOjBd0h8kPSBp9m6si6S5kuol1Tc27vvfFjpwZDU3zD2B977uIL7z26e48H8eYt2WllKXZWaWqdQXqcuBacDJwPnA1ZJG9XXliLgqIuoiom78+PE5ldi/hpQX+Jdzj+Ir7zqahavW87Yr7+WRhg2lLsvMbAd5BsQaYHLRfG3aVqwBmB8RrRHxNPAkSWD0Zd392py6ydz8odcD8K7v3M+NC1fvYg0zs70rz4BYCEyTNFVSJXAeML9Xn1tJjh6QNI7klNNKYAFwuqTR6cXp09O2AeWo2pHc9tGTmDllDP948yNcevMjPP7cJt/W1Mz2CeV5bTgi2iRdTLJjLwDXRMQSSZcD9RExn+4gWAq0A5dERBOApC+ThAzA5RGxLq9aS2nM0Ep+8Dcz+a87nuBb9zzFDQtXM6KqnLopYzh+yhhmTh3NUZNGUVle6rOBZjbYaKD8wreuri7q6+tLXcYeWbOhmYeebuKhp9fx0NPreKpxCwBDysuYMXkUM6cmoXHswaMZNiS3bDezQUTSooioy1zmgNh3NW3ezsJV61m4ah0LV61jybObaO8ICmXi8ANHdB1h1E0Zw7hhQ0pdrpnthxwQA8Tm7W388c/rWfj0Oh5atY4//nkD29uS31K8evxQZnadlhpD7ehqJJW4YjPb1zkgBqiWtg4eXbMxOcJ4OjnK2LStDYBXjaji+KljmDllNMdPHcP0CcMpK3NgmFlPDohBoqMjeOKFl1i4KrmGsXDVOl7YlNwre2R1BXUHJ2Fx/JQxHDVppC98m9nLBoSvdA4gZWXisANHcNiBI3jfiVOICFava+ahoiOMXy9LRjOpqijjmMmj06OMMRxz0CiG+sK3mRXxHmEAk8RBY2s4aGwN7zquFoDGl7ZTvyq5hrFw1Tq++ZvldAQUysSRE0d0fUvqoDE1TB5Tw8jqihK/CzMrFZ9iGuRe2tbKomfSb0o9vZ7FDRtoaeseRHB4VTmTR9cweUw1k0fXUDu6mslpeNSOrqam0n9jmO3PfIrJdmp4VQUnHzqBkw+dAMD2tnaWv7CZ1eu2snr9Vlava6Zh/VaeatzCb59sZFtrzxFoxw2rZNLoGiZ3BkdRiEwaVe3rHGb7MQeE9TCkvMCRk0Zy5KSROyyLCBo3b6dhfTOr123t8fzomo386rHnaSsaJkRKvk1VOzo9+hjTHSS1o6s5cGQ1BX+zymyf5YCwPpPEhOFVTBhexbEH7Xj/pvaO4PlN22hYt5XVxSGyfisPrGziucVrKD6jWV4mJo6qZvKYampHpaexxtRQmx6RjB8+xL/lMCshB4T1m0KZmDSqmkmjqnldxvKWtg6e29jM6nXN6emr7gD59bK1vLh5+w7bG1FVzsjqCkZWVzAifd7Zo2t5TQXDKsv9uw+zPeSAsL2msryMg8cO5eCxQzOXN7e0s2bD1q4AeWHTNjY2t7KxuY1Nza1sbG5lzfrmtK21x+ms3srEDoEyomrXITOyuoLhVQ4XM3BA2D6kurLAIROGc8iE4bvsGxFsbWnvCovix6aMto3NrazZ0Ny1rLV95+EiwfAh5YysSQKjprKcIeVlDCkvpM9lDKlI5is758vL0unCTqbLGFLRa77X+j6dZvsaB4TtlyQxdEg5Q4eUM3FU9W6tGxE0t7anYdK2y5DZ2tLG5u1tNG1uoaW9g+1t7Wxv7WB7WwctbR1sa2unP74tXlnoDp/Kwo6BUlleRkUhWVZRnjwn06KyUEifey5PntW1bkUh2U5loayoTQwp717W2a+zzV8kGLwcEDboSKKmspyaynIO3PHLWrstImjrCFraktDoDJCW9iQhzygAAAdnSURBVI40SNp7Luua7mB7a3tRv959O2jp7N/awbbWDl7a1kZLW7Lt1vYOWtsimW7rYHt7R4/fsPSXQpmoKKgrNAplorwsee58lO/wXEZZGV39utoLokyd82XJc6F7eUHF82Xd7b1eo6ws2U6Zkv+endNlEur1nNWHomU9+3ZO7952C2l7oaiuzhoLEipjp/325SNHB4TZHpK6d6BDSzzqekTQ3tEZGsH29nZa24PWNFS6wqWtg9b2oKW9nZa2oDVd1tre3a+1PXZoa2nvoKMjCcT2rucO2nvMB23tyXNze3tXn8624n7F2yhu62wfDDoDp1AUPoWyjMDpDKOyHfsdMXEkV55/TL/X5oAwG0Ck5K/08kIZVALsv0OlRAQdAW1FAVQcLh3p8ogggq75joiudbvbs/p0v0bvPr2fX267HRF0dEB70fL2jugK644efbuX9ehXXEdHRr/I2F5H97KDxuzeada+ckCY2T5JEgVBoaxQ6lIGLY+DYGZmmXINCEmzJT0haYWkSzOWv19So6TF6ePvipZ9RdISSY9L+ob25Ss5ZmYDUG6nmCQVgHnAaUADsFDS/IhY2qvrTyLi4l7rvh6YBRydNt0LvAm4J696zcyspzyPIGYCKyJiZUS0ADcA5/Rx3QCqSC6zDSG50vZCLlWamVmmPANiErC6aL4hbevtnZIekXSTpMkAEXE/cDfwXPpYEBGP51irmZn1UuqL1LcBUyLiaOBO4AcAkg4BDgNqSULlFElv6L2ypLmS6iXVNzY27sWyzcwGvjwDYg0wuWi+Nm3rEhFNEdE5hOf3gOPS6XOBByJic0RsBn4JnNj7BSLiqoioi4i68ePH9/sbMDMbzPIMiIXANElTJVUC5wHziztIOrBo9myg8zTSn4E3SSqXVEFygdqnmMzM9qLcvsUUEW2SLgYWAAXgmohYIulyoD4i5gMfk3Q20AasA96frn4TcArwKMkF619FxG0v93qLFi16UdIze1DyOODFPVh/IPFn0ZM/j578eXQbCJ/FwTtboOiPYSgHAEn1O7tx92Djz6Infx49+fPoNtA/i1JfpDYzs32UA8LMzDI5ILpdVeoC9iH+LHry59GTP49uA/qz8DUIMzPL5CMIMzPL5IAwM7NMgz4gdjUk+WAiabKkuyUtTYda/3ipayo1SQVJf5T081LXUmqSRqVjpi1Lh+HfYXSDwUTSJ9N/J49Jul5SValr6m+DOiCKhiQ/AzgcOF/S4aWtqqTagE9FxOHACcBHBvnnAfBx/Cv+Tv9N8qPV1wCvZRB/LpImAR8D6iLiSJIfA59X2qr636AOCPZsSPIBJyKei4iH0+mXSHYAWSPwDgqSaoG3kowTNqhJGgm8Efg+QES0RMSG0lZVcuVAtaRyoAZ4tsT19LvBHhB9HZJ80JE0BTgGeLC0lZTU14F/BDpKXcg+YCrQCPxPesrte5KGlrqoUomINcB/kowb9xywMSLuKG1V/W+wB4RlkDQMuBn4RERsKnU9pSDpLGBtRCwqdS37iHLgWODbEXEMsAUYtNfsJI0mOdswFZgIDJV0QWmr6n+DPSB2OST5YJOOnnsz8OOI+Gmp6ymhWcDZklaRnHo8RdK1pS2ppBqAhojoPKK8iSQwBqu/BJ6OiMaIaAV+Cry+xDX1u8EeELscknwwkSSSc8yPR8RXS11PKUXEZRFRGxFTSP6/+E1EDLi/EPsqIp4HVks6NG06Feh9f/nB5M/ACZJq0n83pzIAL9rnNtz3/mBnQ5KXuKxSmgX8NfCopMVp22ci4vYS1mT7jo8CP07/mFoJfKDE9ZRMRDwo6SbgYZJv//2RATjshofaMDOzTIP9FJOZme2EA8LMzDI5IMzMLJMDwszMMjkgzMwskwPCrIQkneyRYm1f5YAwM7NMDgizPpB0gaSHJC2W9N30PhHnS3o0vR/Avxf13Szpa+m9An4taXzafoikuyT9SdLDkv4iXWVY0X0Wfpz+MhdJV6T35nhE0n+W4G3bIOeAMNsFSYcB7wZmRcQMoB14L/DvwCnADOB4SW9PVxkK1EfEEcBvgS+k7T8G5kXEa0nG7XkubT8G+ATJPUleDcySNBY4FzgiIo4G/jnfd2m2IweE2a6dChwHLEyHIDkV+CRwTzpYWxvJzv+Naf8O4Cfp9LXASZKGA5Mi4haAiNgWEVvTPg9FRENEdACLgSnARmAb8H1J7wA6+5rtNQ4Is10T8IOImJE+DgW+uBvr72o8m+1F0+1AeRo6M0lGTT0L+NVuvJ5Zv3BAmO3ar4F3SZoAIGkMyV/6b5I0Lr117fkkp5Mg+Xf1rnT6PcC96R36GjpPQ0kaIqlmZy+Y3pNjZDpQ4idJbvFptlcN6tFczfoiIpZK+ixwh6QyoBX4CMkNc+4mOcL4RUT8LF1lCzAzXWctyfULSEbK/a6ky9Nt/NXLvOxw4GeSqtLt/0M/vy2zXfJormb9TNLmiBhW6jrM9pRPMZmZWSYfQZiZWSYfQZiZWSYHhJmZZXJAmJlZJgeEmZllckCYmVmm/w+h9Npe4beBiwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
            "INFO:tensorflow:Assets written to: /content/gdrive/My Drive/Machine Learning/movielens1m/Saved Models/CFRecommender_V_6refined/assets\n",
            "Epoch 1/10\n",
            "WARNING:tensorflow:Layer cf_recommender_1 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_5/embeddings:0', 'embedding_7/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_5/embeddings:0', 'embedding_7/embeddings:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_5/embeddings:0', 'embedding_7/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_5/embeddings:0', 'embedding_7/embeddings:0'] when minimizing the loss.\n",
            "1759/1759 [==============================] - 12s 7ms/step - loss: 0.6157 - val_loss: 0.7100\n",
            "Epoch 2/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5971 - val_loss: 0.7160\n",
            "Epoch 3/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5915 - val_loss: 0.7196\n",
            "Epoch 4/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5883 - val_loss: 0.7214\n",
            "Epoch 5/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5867 - val_loss: 0.7224\n",
            "Epoch 6/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5860 - val_loss: 0.7230\n",
            "Epoch 7/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5853 - val_loss: 0.7233\n",
            "Epoch 8/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5849 - val_loss: 0.7235\n",
            "Epoch 9/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5846 - val_loss: 0.7237\n",
            "Epoch 10/10\n",
            "1759/1759 [==============================] - 11s 6ms/step - loss: 0.5844 - val_loss: 0.7237\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgdZZ328e/dWzqdfZcskCiJgCBBmggEFUE0oICIxqCo+M4QfRV1XBhhxmXEmXcYx1EHxQUUZxQBmYAYBjCAggqypBNQSMISQiAdkDTZIGsv5/f+UdXdp7urkw7pyul035/rquuceuqpc37nQOrueqpOlSICMzOzzspKXYCZmfVNDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAw6wWS/kvSP/ew72pJb9vb1zHLmwPCzMwyOSDMzCyTA8IGjHRo50JJf5G0VdJPJE2QdJuklyXdKWlUUf8zJC2TtEnS3ZIOLVp2lKSl6Xq/BKo7vde7JD2crvsnSa9/hTWfL2mlpA2SFkqamLZL0rclrZP0kqRHJB2eLjtN0vK0trWSvvCKvjAb8BwQNtCcDZwCzABOB24D/gEYR/Lv4dMAkmYA1wJ/ly67FbhZUpWkKuAm4OfAaOB/0tclXfco4CrgY8AY4EfAQkmD9qRQSScB/wrMBQ4AngGuSxe/HXhz+jlGpH3Wp8t+AnwsIoYBhwO/25P3NWvlgLCB5rsR8UJErAX+CDwQEQ9FxA7gV8BRab/3A7dExB0R0QR8ExgMHA8cC1QC34mIpohYACwueo/5wI8i4oGIaImI/wZ2puvtiQ8CV0XE0ojYCVwMHCdpKtAEDAMOARQRKyLi+XS9JuAwScMjYmNELN3D9zUDHBA28LxQ9Hx7xvzQ9PlEkr/YAYiIArAGmJQuWxsdr3T5TNHzg4DPp8NLmyRtAqak6+2JzjVsIdlLmBQRvwO+B1wOrJN0haThadezgdOAZyT9XtJxe/i+ZoADwqw7z5Fs6IFkzJ9kI78WeB6YlLa1OrDo+RrgXyJiZNFUExHX7mUNQ0iGrNYCRMRlEXE0cBjJUNOFafviiDgTGE8yFHb9Hr6vGeCAMOvO9cA7JZ0sqRL4PMkw0Z+A+4Bm4NOSKiW9B5hVtO6VwMclvTE9mDxE0jslDdvDGq4FPippZnr84v+RDImtlnRM+vqVwFZgB1BIj5F8UNKIdGjsJaCwF9+DDWAOCLMMEfE4cC7wXeBFkgPap0dEY0Q0Au8BzgM2kByvuLFo3TrgfJIhoI3AyrTvntZwJ/Bl4AaSvZbXAPPSxcNJgmgjyTDUeuDf02UfAlZLegn4OMmxDLM9Jt8wyMzMsngPwszMMjkgzMwskwPCzMwyOSDMzCxTRakL6C1jx46NqVOnlroMM7P9ypIlS16MiHFZy/pNQEydOpW6urpSl2Fmtl+R9Ex3yzzEZGZmmRwQZmaWyQFhZmaZ+s0xiCxNTU3U19ezY8eOUpeSu+rqaiZPnkxlZWWpSzGzfqJfB0R9fT3Dhg1j6tSpdLzwZv8SEaxfv576+nqmTZtW6nLMrJ/o10NMO3bsYMyYMf06HAAkMWbMmAGxp2Rm+06/Dgig34dDq4HyOc1s3+nXQ0xmViIRUGiBKECkj23zhU7zLbte1jYf2X3bpuimfXd9WvbiNdKrYbf9gaau812WsYtlyn6N3S0bNgEOb7steq9xQORs06ZNXHPNNXziE5/Yo/VOO+00rrnmGkaOHJlTZbZPFFqgeSe0NEJLE7Skz5sb07Z0at6ZLm9M+zQVrZdOhebk9Qot7RvKQnP7BrTQ3HED2to/0mWFQqf1WpK2Duu1dHqP5qLXK3q/tg13p41sa18Gym0ERJ/4rJNqHRD7o02bNvH973+/S0A0NzdTUdH913/rrbfmXdrAEZFsYJu2QeM2aNoOTVuTx8atSXvb8+Jl25JlzTu732h33tB3aNuZbixzoHIoq4Cy8vR5eafnFaCyorbWvmVd16uo6rReOZSV7fo92l67rH1qm89YnrlMneZ3tax43bKO810mddO+m+UdPs/uXqOs6K/4jP/f2h6jYxu7WZbZrwfLVP7K/j/ajVwDQtIc4D+BcuDHEXFpp+XfBt6aztYA4yNipKSZwA9I7prVQnJ/31/mWWteLrroIp566ilmzpxJZWUl1dXVjBo1iscee4wnnniCd7/73axZs4YdO3bwmc98hvnz5wPtlw7ZsmULp556KieccAJ/+tOfmDRpEr/+9a8ZPHhwiT9ZTpp3wrYNsG097Nicbry3tW+sOz/P3Ohv67qRj5Y9q0PlUDUEKgdDRTWUV0HFICivhPL0sbKmU1tVsrEt7zR1buu8Tnll2pa1TmvftF/rxtv6LnUeBtp/5RYQksqBy4FTgHpgsaSFEbG8tU9EfLao/6eAo9LZbcCHI+JJSROBJZIWRcSmV1rP125exvLnXnqlq2c6bOJwvnr663bZ59JLL+XRRx/l4Ycf5u677+ad73wnjz76aNvpqFdddRWjR49m+/btHHPMMZx99tmMGTOmw2s8+eSTXHvttVx55ZXMnTuXG264gXPPPbdXP0suWpph+8ZkY99l2pDd1vhyz167vCrZQLduxCtrkmnQMBg6IZ0f3HV5VU3287b5wVA5JN0Y7///wM32Rp57ELOAlRGxCkDSdcCZwPJu+p8DfBUgIp5obYyI5yStA8YBrzgg+opZs2Z1+K3CZZddxq9+9SsA1qxZw5NPPtklIKZNm8bMmTMBOProo1m9evU+q7dNoQA7NnWzYe9mg79jF/+5qoZCzWioGZNMY6enz4vaqkckG+viDXfrxr7co6NmecvzX9kkYE3RfD3wxqyOkg4CpgG/y1g2C6gCntqbYnb3l/6+MmTIkLbnd999N3feeSf33XcfNTU1nHjiiZm/ZRg0aFDb8/LycrZv3967RTVthw2rYP3KZNr4TNeN/vYN3Y+nlw+CIWPbN+4jp7Rv5Dtv9GvGwODRUFndu5/BzHpdX/kzbB6wIKLjQLGkA4CfAx+J6Lp1kjQfmA9w4IEH7os699iwYcN4+eXsYZPNmzczatQoampqeOyxx7j//vvzK6SlGTY90zEI1q+E9U/B5jUd+w4Zl0w1Y2D8IZ029hkb/MoaD8eY9UN5BsRaYErR/OS0Lcs84JPFDZKGA7cA/xgRmVvOiLgCuAKgtra2D5xr1tWYMWOYPXs2hx9+OIMHD2bChAlty+bMmcMPf/hDDj30UF772tdy7LHH7v0bvvR81wBYvxI2roZCU3u/QSNg7MFw0PEw5mAY85rkcfSrk3F8MxvwFJHPdlVSBfAEcDJJMCwGPhARyzr1OwT4DTAt0mIkVQG3ATdHxHd68n61tbXR+YZBK1as4NBDD93bj9L3FJqTs33aph3QvJMVq9Zw6G/e296vfFC64U83/sVTzRj/1W9mSFoSEbVZy3Lbg4iIZkkXAItITnO9KiKWSboEqIuIhWnXecB10TGp5gJvBsZIOi9tOy8iHs6r3j6nUEjOoy8KgLbz8QvNHfu2njpZNQRO+2Z7IAyf7FMizewVy/UYRETcCtzaqe0rneb/KWO9q4Gr86ytT2lpgp0vp+fu72z/tW2xsorkfPzqEek584OS+Yqq5Ec7AOua4NDz9339ZtYv9ZWD1ANLSxM0boGdW5Lz/pt3Ju0qSzb6VUOgYnRRCAxKfuVpZrYPOSD2hV0FQtXQ5HhA1VCfDWRmfYoDIg8dAmFLcgwB2gNh8BgY5EAws77NAdEbdhkIQ5IfhjkQzGw/41NcXomWJti+CTatgXUr4IVHk98ZbN+QXMNn2AEwdga86gg2lY/l+z+/IQmKPQyH73znO2zbti2fz2BmthsOiJ5oaU4CYXNxIDydGQiMORiGvSoNhLK2y32/Eg4IMyslDzFlaWlOhooatySnnxYPGVUOgWGjkl8bVw5uP8W0G8WX+z7llFMYP348119/PTt37uSss87ia1/7Glu3bmXu3LnU19fT0tLCl7/8ZV544QWee+453vrWtzJ27FjuuuuuffDBzczaDZyAuO0i+Osj3SyMjnfTarsklDrdKKWMovsFJnsMp16a9YJtii/3ffvtt7NgwQIefPBBIoIzzjiDP/zhDzQ0NDBx4kRuueUWILlG04gRI/jWt77FXXfdxdixY/f645uZ7SkPMUUhPbC8PTm2AMnvD4rvNVCe3nGLvTvAfPvtt3P77bdz1FFH8YY3vIHHHnuMJ598kiOOOII77riDL37xi/zxj39kxIgRe/+5zMz20sDZg+juL/0I2Lqu/b4Duxky2hsRwcUXX8zHPvaxLsuWLl3Krbfeype+9CVOPvlkvvKVr2S8gpnZvuM9CCm5A9mgobmEQ/Hlvt/xjndw1VVXsWXLFgDWrl3LunXreO6556ipqeHcc8/lwgsvZOnSpV3WNTPb1wbOHkSJFF/u+9RTT+UDH/gAxx13HABDhw7l6quvZuXKlVx44YWUlZVRWVnJD37wAwDmz5/PnDlzmDhxog9Sm9k+l9vlvve1AXW5724MtM9rZntvV5f79hCTmZllckCYmVmmfh8Q/WUIbXcGyuc0s32nXwdEdXU169ev7/cbz4hg/fr1VFdXl7oUM+tH+vVZTJMnT6a+vp6GhoZSl5K76upqJk+eXOoyzKwf6dcBUVlZybRp00pdhpnZfqlfDzGZmdkrl2tASJoj6XFJKyVdlLH825IeTqcnJG0qWvYRSU+m00fyrNPMzLrKbYhJUjlwOXAKUA8slrQwIpa39omIzxb1/xRwVPp8NPBVoBYIYEm67sa86jUzs47y3IOYBayMiFUR0QhcB5y5i/7nANemz98B3BERG9JQuAOYk2OtZmbWSZ4BMQlYUzRfn7Z1IekgYBrwuz1ZV9J8SXWS6gbCmUpmZvtSXzlIPQ9YENF2p54eiYgrIqI2ImrHjRuXU2lmZgNTngGxFphSND85bcsyj/bhpT1d18zMcpBnQCwGpkuaJqmKJAQWdu4k6RBgFHBfUfMi4O2SRkkaBbw9bTMzs30kt7OYIqJZ0gUkG/Zy4KqIWCbpEqAuIlrDYh5wXRRdDyMiNkj6OknIAFwSERvyqtXMzLrq1/eDMDOzXfP9IMzMbI85IMzMLJMDwszMMjkgzMwskwPCzMwyOSDMzCyTA8LMzDI5IMzMLJMDwszMMjkgzMwskwPCzMwyOSDMzCyTA8LMzDI5IMzMLJMDwszMMjkgzMwskwPCzMwyOSDMzCyTA8LMzDLlGhCS5kh6XNJKSRd102eupOWSlkm6pqj9G2nbCkmXSVKetZqZWUcVeb2wpHLgcuAUoB5YLGlhRCwv6jMduBiYHREbJY1P248HZgOvT7veA7wFuDuves3MrKM89yBmASsjYlVENALXAWd26nM+cHlEbASIiHVpewDVQBUwCKgEXsixVjMz6yTPgJgErCmar0/bis0AZki6V9L9kuYARMR9wF3A8+m0KCJWdH4DSfMl1Umqa2hoyOVDmJkNVKU+SF0BTAdOBM4BrpQ0UtLBwKHAZJJQOUnSmzqvHBFXRERtRNSOGzduH5ZtZtb/5RkQa4EpRfOT07Zi9cDCiGiKiKeBJ0gC4yzg/ojYEhFbgNuA43Ks1czMOskzIBYD0yVNk1QFzAMWdupzE8neA5LGkgw5rQKeBd4iqUJSJckB6i5DTGZmlp/cAiIimoELgEUkG/frI2KZpEsknZF2WwSsl7Sc5JjDhRGxHlgAPAU8AvwZ+HNE3JxXrWZm1pUiotQ19Ira2tqoq6srdRlmZvsVSUsiojZrWakPUpuZWR/lgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0y5BoSkOZIel7RS0kXd9JkrabmkZZKuKWo/UNLtklaky6fmWauZmXVUkdcLSyoHLgdOAeqBxZIWRsTyoj7TgYuB2RGxUdL4opf4GfAvEXGHpKFAIa9azcysqzz3IGYBKyNiVUQ0AtcBZ3bqcz5weURsBIiIdQCSDgMqIuKOtH1LRGzLsVYzM+skz4CYBKwpmq9P24rNAGZIulfS/ZLmFLVvknSjpIck/Xu6R9KBpPmS6iTVNTQ05PIhzMwGqlIfpK4ApgMnAucAV0oamba/CfgCcAzwauC8zitHxBURURsRtePGjdtXNZuZDQg9CghJn5E0XImfSFoq6e27WW0tMKVofnLaVqweWBgRTRHxNPAESWDUAw+nw1PNwE3AG3pSq5mZ9Y6e7kH8n4h4CXg7MAr4EHDpbtZZDEyXNE1SFTAPWNipz00kew9IGksytLQqXXekpNbdgpOA5ZiZ2T7T04BQ+nga8POIWFbUlin9y/8CYBGwArg+IpZJukTSGWm3RcB6ScuBu4ALI2J9RLSQDC/9VtIj6XtduScfzMzM9o4iYvedpJ+SHGCeBhwJlAN3R8TR+ZbXc7W1tVFXV1fqMszM9iuSlkREbdaynv4O4m+AmcCqiNgmaTTw0d4q0MzM+p6eDjEdBzweEZsknQt8CdicX1lmZlZqPQ2IHwDbJB0JfB54iuSXzmZm1k/1NCCaIzlYcSbwvYi4HBiWX1lmZlZqPT0G8bKki0lOb32TpDKgMr+yzMys1Hq6B/F+YCfJ7yH+SvKjt3/PrSozMyu5HgVEGgq/AEZIehewIyJ8DMLMrB/r6aU25gIPAu8D5gIPSHpvnoWZmVlp9fQYxD8CxxRdjnsccCewIK/CzMystHp6DKKsNRxS6/dgXTMz2w/1dA/iN5IWAdem8+8Hbs2nJDMz6wt6FBARcaGks4HZadMVEfGr/MoyM7NS6/E9qSPiBuCGHGsxM7M+ZJcBIellIOtyrwIiIobnUpWZmZXcLgMiInw5DTOzAcpnIpmZWSYHhJmZZXJAmJlZJgeEmZllyjUgJM2R9LiklZIu6qbPXEnLJS2TdE2nZcMl1Uv6Xp51mplZVz3+HcSeklQOXA6cAtQDiyUtjIjlRX2mAxcDsyNio6TxnV7m68Af8qrRzMy6l+cexCxgZUSsiohG4DqSO9IVOx+4PCI2AhRf70nS0cAE4PYcazQzs27kGRCTgDVF8/VpW7EZwAxJ90q6X9IcgPSOdf8BfGFXbyBpvqQ6SXUNDQ29WLqZmZX6IHUFMB04ETgHuFLSSOATwK0RUb+rlSPiioiojYjacePG5V6smdlAktsxCGAtMKVofnLaVqweeCAimoCnJT1BEhjHkdz7+hPAUKBK0paIyDzQbWZmvS/PPYjFwHRJ0yRVAfOAhZ363ESy94CksSRDTqsi4oMRcWBETCUZZvqZw8HMbN/KLSAiohm4AFgErACuj4hlki6RdEbabRGwXtJy4C7gwohYn1dNZmbWc4rIuljr/qe2tjbq6upKXYaZ2X5F0pKIqM1aVuqD1GZm1kc5IMzMLJMDwszMMjkgzMwskwPCzMwyOSDMzCyTA8LMzDI5IMzMLJMDwszMMjkgzMwskwPCzMwyOSDMzCyTA8LMzDI5IMzMLJMDwszMMjkgzMwskwPCzMwyOSDMzCyTA8LMzDLlGhCS5kh6XNJKSRd102eupOWSlkm6Jm2bKem+tO0vkt6fZ51mZtZVRV4vLKkcuBw4BagHFktaGBHLi/pMBy4GZkfERknj00XbgA9HxJOSJgJLJC2KiE151WtmZh3luQcxC1gZEasiohG4DjizU5/zgcsjYiNARKxLH5+IiCfT588B64BxOdZqZmad5BkQk4A1RfP1aVuxGcAMSfdKul/SnM4vImkWUAU8lbFsvqQ6SXUNDQ29WLqZmZX6IHUFMB04ETgHuFLSyNaFkg4Afg58NCIKnVeOiCsiojYiaseN8w6GmVlvyjMg1gJTiuYnp23F6oGFEdEUEU8DT5AEBpKGA7cA/xgR9+dYp5mZZcgzIBYD0yVNk1QFzAMWdupzE8neA5LGkgw5rUr7/wr4WUQsyLFGMzPrRm4BERHNwAXAImAFcH1ELJN0iaQz0m6LgPWSlgN3ARdGxHpgLvBm4DxJD6fTzLxqNTOzrhQRpa6hV9TW1kZdXd0rWvf3TzRw3KvHUFVR6kMyZmb7lqQlEVGbtWzAbxGfatjCeT99kA9ceT8vvLSj1OWYmfUZAz4gXjNuKJfNO4rlz7/EOy+7hwdWrS91SWZmfcKADwiA04+cyE2fnM3w6go+8OMH+PEfV9Ffht7MzF4pB0RqxoRh/PqC2bzt0PH88y0ruOCah9iys7nUZZmZlYwDosiw6kp+eO7RXHTqIdz26PO8+/J7WbluS6nLMjMrCQdEJ5L4+Ftew9V/80Y2bm3kzO/dw22PPF/qsszM9jkHRDeOP3gsN3/qBKZPGMb//cVS/vXWFTS3dLnah5lZv+WA2IWJIwfzy48dy4eOPYgf/WEV5/7kARpe3lnqsszM9gkHxG4Mqijn6+8+nP9435E89OwmTv/uPSx5ZmOpyzIzy50DoofOPnoyN37ieKoqyph3xX387L7VPhXWzPo1B8QeeN3EEdx8wQm8afo4vvLrZXzu+j+zvbGl1GWZmeXCAbGHRtRU8uMP1/K5U2Zw08NrOev797L6xa2lLsvMrNc5IF6BsjLx6ZOn89PzjuH5zTs4/Xv3cOfyF0pdlplZr3JA7IUTXzue//3UCRw0poa//Vkd/3H747QUfFzCzPoHB8RemjK6hgUfP565tZP57u9Wct5PH2Tj1sZSl2VmttccEL2gurKcb7z3SC59zxE8sGoD7/ruPfylflOpyzIz2ysOiF40b9aB/M/HjwPgvT+4j+sefLbEFZmZvXIOiF525JSR3PypE3jjq0dz0Y2P8MUFf2FHk0+FNbP9jwMiB6OHVPFfH53FBW89mF/WreF9P7yPNRu2lbosM7M94oDISXmZ+MI7XsuPP1zL6vVbOf179/D7JxpKXZaZWY/lGhCS5kh6XNJKSRd102eupOWSlkm6pqj9I5KeTKeP5Flnnt522ARuvuAEXjW8mvN++iCX/fZJCj4V1sz2A8rrekKSyoEngFOAemAxcE5ELC/qMx24HjgpIjZKGh8R6ySNBuqAWiCAJcDREdHtVfJqa2ujrq4ul8/SG7Y1NvMPNz7CTQ8/x0mHjOfbc2cyoqay1GWZ2QAnaUlE1GYty3MPYhawMiJWRUQjcB1wZqc+5wOXt274I2Jd2v4O4I6I2JAuuwOYk2OtuaupquDb75/JJWe+jj880cDp37uH5c+9VOqyzMy6lWdATALWFM3Xp23FZgAzJN0r6X5Jc/ZgXSTNl1Qnqa6hoe+P70viw8dN5ZcfO5adzS2c9f17uXFpfanLMjPLVOqD1BXAdOBE4BzgSkkje7pyRFwREbURUTtu3LicSux9Rx80mv/91JuYOWUkn7v+z3z5pkdpbPbd6sysb8kzINYCU4rmJ6dtxeqBhRHRFBFPkxyzmN7Ddfdr44YN4hd/+0bmv/nV/Pz+Z3j/Fffx/ObtpS7LzKxNngGxGJguaZqkKmAesLBTn5tI9h6QNJZkyGkVsAh4u6RRkkYBb0/b+pWK8jL+4bRD+f4H38ATf32Z0/7zj/z9gj9zfd0ann5xq29IZGYlVZHXC0dEs6QLSDbs5cBVEbFM0iVAXUQspD0IlgMtwIURsR5A0tdJQgbgkojYkFetpXbaEQcwY8IwvvGbx7h9+QtcX5cclxg7tIrag0ZTO3UUx0wdzWETh1NZXupRQTMbKHI7zXVf6+unufZUoRA81bCFxas3Urd6A4uf2cCaDcnQ0+DKco46cCS1U0dzzNRRHHXgKIYOyi3jzWwA2NVprg6I/cBfN++g7pkN1K3eyOLVG1jx/EsUAsoEh00cTu1BozkmDY3xw6tLXa6Z7UccEP3MyzuaeOjZTckexuqNPLRmIzuakrOgDhxdQ+3UUcyaOpraqaN5zbghSCpxxWbWV+0qIDw+sR8aVl3Jm2eM480zklN7m1oKLHvupTQwNvD7xxu4cWly0teomsq2IanaqaM5fOIIqip8HMPMds97EP1QRPD0i1vbhqTqntnI0y9uBWBQRRkzp4zkmKnJwe83HDSK4dW+5IfZQOUhJqPh5Z0seSYZklq8egPLnnuJlkIgwSGvGt62h3HM1FEcMGJwqcs1s33EAWFdbN3ZzMNrNiV7GKs3svTZjWxrTG5s9Krh1UwdW8OBo5Npyuj256OHVPmYhlk/4mMQ1sWQQRXMPngssw8eC0BzS4EVz7/M4tUbeHTtZp7dsI27H29g3cs7O6w3dFBFGhiDuwTI5FE1Pr5h1o84IAxIftV9xOQRHDF5RIf27Y0t1G/cxrMbtvHM+uRxzYZtrGrYyt2PN7Cz6BpSEkwcMZgpncLjoDFDOHB0DaNqKr33YbYfcUDYLg2uKmf6hGFMnzCsy7JCIXhxy06e2bCNZ4vCo8d7H2lwHDi6hkkjB3vvw6yPcUDYK1ZWJsYPr2b88GqOmTq6y/LtjS2s2dgeHq0B8lTG3keZ4ICivY+DxgzhVcOrGTWkkhGDqxhZU8nIwZWMGFxJhS83YrZPOCAsN4OrypkxYRgzutn7aNiyMwmOor2PZzZs467HG2h4ufv7ZAyrrkgDIwmOEYMrO8yPrKliZGtbTRIwIwZXeg/FbA85IKwkysrEhOHVTOhm72NbYzMvvLSTzdub2LStMX1Mp+2N6fNGNm1vYu3G7WxK++3qdt9DqsqT8KhpD5QR6Z5J8fyomqI9lppKBlWU5/hNmPVdDgjrk2qqKpg2ds/+9ywUgi2NzWwuCpKN25rYvC0NlDRkNqcB8/hLL7cFTfMukqWiTAyqKKOqooxBFeXpY1mnx87t5Qzag367XjdZp7zMB/ht33JAWL9RViaGV1cyvLqSKV13SroVEWxtbEn2SLY1te+tpEGyrbGZnU0FGlsK7Y/NLTQ2F9iZTpu2N6Xz7e2t8zubC/TGz40qykRVVuCUlzGosix9bJ8fVL77cCqeb30+aBd9q8rLqCgTZQ6rAcEBYQOeJIYOqmDooAomj+r9148ImgvRJTiKA6ZrsHTtlz3fcf2XioOqONCakrZdDcHtCQnKJcrL0pWFwhEAAAccSURBVEmivFwd24qXdW7rrn0P1m8NqooyUV5Wlj6q7bE8a1m5KJM6zBcvL+/0GhVlZRlt7cvKypL/f8oEZRISiE7z+/Gp3Q4Is5xJorJcVJaXMWRQaWtpbil0CY7Glpa2oOlJODUXgkIhCb2WCFpa0sdCMhUiaO7U1mFZp7bG5gItUfSarVPxa6bL2tZPX7+4f19WpvYgkYRIAqRtvihQWtvpFDTtgdPapzWQ4LCJI/juOUf1et0OCLMBpKK8jIryMmqqSl1J74oICgHNhQIthY4hUjzfYXkhOjxvXda1PWgpFGgpQEuh0N7ekjwGyXtHQCGCiEifp/Nt9bW3R9peKKTrEkXrJ49J5gWFQvvrtC5v/byt8weOzuf6aQ4IM9vvSaJcUF7mM856k08MNzOzTLkGhKQ5kh6XtFLSRRnLz5PUIOnhdPrbomXfkLRM0gpJl2l/PtJjZrYfym2ISVI5cDlwClAPLJa0MCKWd+r6y4i4oNO6xwOzgdenTfcAbwHuzqteMzPrKM89iFnAyohYFRGNwHXAmT1cN4BqoAoYBFQCL+RSpZmZZcozICYBa4rm69O2zs6W9BdJCyRNAYiI+4C7gOfTaVFErOi8oqT5kuok1TU0NPT+JzAzG8BKfZD6ZmBqRLweuAP4bwBJBwOHApNJQuUkSW/qvHJEXBERtRFRO27cuH1YtplZ/5dnQKwFphTNT07b2kTE+ohovWnAj4Gj0+dnAfdHxJaI2ALcBhyXY61mZtZJngGxGJguaZqkKmAesLC4g6QDimbPAFqHkZ4F3iKpQlIlyQHqLkNMZmaWn9zOYoqIZkkXAIuAcuCqiFgm6RKgLiIWAp+WdAbQDGwAzktXXwCcBDxCcsD6NxFx867eb8mSJS9KemYvSh4LvLgX6/cn/i468vfRkb+Pdv3huziouwWK3rjMZD8gqS4iaktdR1/g76Ijfx8d+fto19+/i1IfpDYzsz7KAWFmZpkcEO2uKHUBfYi/i478fXTk76Ndv/4ufAzCzMwyeQ/CzMwyOSDMzCzTgA+I3V2SfCCRNEXSXZKWp5da/0ypayo1SeWSHpL0v6WupdQkjUyvmfZYehn+AX11A0mfTf+dPCrpWknVpa6ptw3ogCi6JPmpwGHAOZIOK21VJdUMfD4iDgOOBT45wL8PgM/gX/G3+k+SH60eAhzJAP5eJE0CPg3URsThJD8GnlfaqnrfgA4I9u6S5P1ORDwfEUvT5y+TbACyrsA7IEiaDLyT5DphA5qkEcCbgZ8ARERjRGwqbVUlVwEMllQB1ADPlbieXjfQA6KnlyQfcCRNBY4CHihtJSX1HeDvgUKpC+kDpgENwE/TIbcfSxpS6qJKJSLWAt8kuW7c88DmiLi9tFX1voEeEJZB0lDgBuDvIuKlUtdTCpLeBayLiCWlrqWPqADeAPwgIo4CtgID9pidpFEkow3TgInAEEnnlraq3jfQA2K3lyQfaNKr594A/CIibix1PSU0GzhD0mqSoceTJF1d2pJKqh6oj4jWPcoFJIExUL0NeDoiGiKiCbgROL7ENfW6gR4Qu70k+UAiSSRjzCsi4lulrqeUIuLiiJgcEVNJ/r/4XUT0u78Qeyoi/gqskfTatOlkoPP95QeSZ4FjJdWk/25Oph8etM/tct/7g+4uSV7iskppNvAh4BFJD6dt/xARt5awJus7PgX8Iv1jahXw0RLXUzIR8YCkBcBSkrP/HqIfXnbDl9owM7NMA32IyczMuuGAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDArIUkn+kqx1lc5IMzMLJMDwqwHJJ0r6UFJD0v6UXqfiHMkPZLeD+DfivpukfTt9F4Bv5U0Lm0/WNKdkv4saamk16SrDC26z8Iv0l/mIunS9N4cf5H0zRJ8bBvgHBBmuyHpUOD9wOyImAm0AB8E/g04CZgJHCPp3ekqQ4C6iHgd8Hvgq2n7L4DLI+JIkuv2PJ+2HwX8Hck9SV4NzJY0BjgLeF1EvB7453w/pVlXDgiz3TsZOBpYnF6C5GTgs8Dd6cXamkk2/m9O+xeAX6bPrwZOkDQMmBQRvwKIiB0RsS3t82BE1EdEAXgYmApsBnYAP5H0HqC1r9k+44Aw2z0B/x0RM9PptcA/7cH6u7uezc6i5y1ARRo6s0iumvou4Dd78H5mvcIBYbZ7vwXeK2k8gKTRJH/pv0XS2PTWteeQDCdB8u/qvenzDwD3pHfoq28dhpI0SFJNd2+Y3pNjRHqhxM+S3OLTbJ8a0FdzNeuJiFgu6UvA7ZLKgCbgkyQ3zLmLZA/jloj4dbrKVmBWus46kuMXkFwp90eSLklf4327eNthwK8lVaev/7le/lhmu+WruZr1MklbImJoqesw21seYjIzs0zegzAzs0zegzAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NM/x+T2cpV4n+CdgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "(900188,) (900188,)\n",
            "\n",
            "(100021,) (100021,)\n",
            "\n",
            "(900188,) (900188,)\n",
            "\n",
            "(100021,) (100021,)\n",
            "\n",
            "(900188,) (900188,)\n",
            "\n",
            "(100021,) (100021,)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NISJTXXBE6yc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "304385b2-bb9c-4ce2-9519-d5927f251fa9"
      },
      "source": [
        "display(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model</th>\n",
              "      <th>rmse train</th>\n",
              "      <th>rmse test</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bias Baseline</td>\n",
              "      <td>0.237359</td>\n",
              "      <td>0.256709</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CF</td>\n",
              "      <td>0.168583</td>\n",
              "      <td>0.319274</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CFpure</td>\n",
              "      <td>0.168314</td>\n",
              "      <td>0.319254</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           model  rmse train  rmse test\n",
              "0  bias Baseline    0.237359   0.256709\n",
              "0             CF    0.168583   0.319274\n",
              "0         CFpure    0.168314   0.319254"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RGxXS2W8E9Kp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "701a2a8b-6249-48af-82d6-54337cc5da3e"
      },
      "source": [
        "# model_CF.save(base_dir + \"Saved Models/\"+'CFRecommender'+'_V_'+ '6'+ 'refined') #'_mu' + '_bias'+'_meaningful_embeddings_allembeddings_he_initialized'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/gdrive/My Drive/Machine Learning/movielens1m/Saved Models/CFRecommender_V_6refined/assets\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jE_gS_B7ciL2",
        "colab_type": "text"
      },
      "source": [
        "# Explore embeddings and see predictions\n",
        "todo- add tSNE of embeddings classified on genre"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ez_1fSl57Pd9",
        "colab_type": "text"
      },
      "source": [
        "Analyse Embeddings\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZpDFt7RSCPP2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sim(model, df_movies, id=1):\n",
        "\n",
        "  movie_embedding = model.layers[2]\n",
        "  weights = movie_embedding.get_weights()[0]\n",
        "  # print(weights)\n",
        "  # print(weights.shape)\n",
        "\n",
        "\n",
        "  id =id #122912\t #189333 mi #184641 fma #122912 infinity w #593 silence #215 before sunrise #6710+42 dark knight #318 shawshank   #187593 dp #68791 terminator # 4369 ff #122916 thor #1210 starwars\n",
        "  similarity = np.dot(weights,weights[id,:].T)/(np.linalg.norm(weights,axis =1)+0.00000001)/(np.linalg.norm(weights[id,:])+0.00000001)\n",
        "  # print(similarity.shape)\n",
        "  sim = pd.DataFrame(similarity)\n",
        "  sim['movieId'] = np.arange(df_movies['movieId'].unique().max()+1)\n",
        "  sim.columns = ['similarity', 'movieId']\n",
        "  sim = sim.merge(df_movies, on = 'movieId')\n",
        "  sim = sim.sort_values(by = 'similarity', ascending = False)\n",
        "  display(sim[:15])\n",
        "  display(sim[-5:])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PyKoP0n1Cuu8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 844
        },
        "outputId": "049136f4-f264-4f01-e3c5-5d08ae2dbf1d"
      },
      "source": [
        "model_CF = saved_model()\n",
        "sim(model_CF,df_movies,1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1</td>\n",
              "      <td>Toy Story (1995)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3045</th>\n",
              "      <td>0.825308</td>\n",
              "      <td>3114</td>\n",
              "      <td>Toy Story 2 (1999)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2286</th>\n",
              "      <td>0.723963</td>\n",
              "      <td>2355</td>\n",
              "      <td>Bug's Life, A (1998)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>0.652019</td>\n",
              "      <td>364</td>\n",
              "      <td>Lion King, The (1994)</td>\n",
              "      <td>Animation|Children's|Musical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>584</th>\n",
              "      <td>0.610675</td>\n",
              "      <td>588</td>\n",
              "      <td>Aladdin (1992)</td>\n",
              "      <td>Animation|Children's|Comedy|Musical</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2728</th>\n",
              "      <td>0.606673</td>\n",
              "      <td>2797</td>\n",
              "      <td>Big (1988)</td>\n",
              "      <td>Comedy|Fantasy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2182</th>\n",
              "      <td>0.597607</td>\n",
              "      <td>2251</td>\n",
              "      <td>Cabinet of Dr. Ramirez, The (1991)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1636</th>\n",
              "      <td>0.593955</td>\n",
              "      <td>1682</td>\n",
              "      <td>Truman Show, The (1998)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3438</th>\n",
              "      <td>0.589344</td>\n",
              "      <td>3507</td>\n",
              "      <td>Odd Couple, The (1968)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3682</th>\n",
              "      <td>0.586581</td>\n",
              "      <td>3751</td>\n",
              "      <td>Chicken Run (2000)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3312</th>\n",
              "      <td>0.579646</td>\n",
              "      <td>3381</td>\n",
              "      <td>Slaves to the Underground (1997)</td>\n",
              "      <td>Comedy|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1081</th>\n",
              "      <td>0.571245</td>\n",
              "      <td>1097</td>\n",
              "      <td>E.T. the Extra-Terrestrial (1982)</td>\n",
              "      <td>Children's|Drama|Fantasy|Sci-Fi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2012</th>\n",
              "      <td>0.565554</td>\n",
              "      <td>2081</td>\n",
              "      <td>Little Mermaid, The (1989)</td>\n",
              "      <td>Animation|Children's|Comedy|Musical|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1873</th>\n",
              "      <td>0.562943</td>\n",
              "      <td>1942</td>\n",
              "      <td>All the King's Men (1949)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>977</th>\n",
              "      <td>0.562261</td>\n",
              "      <td>989</td>\n",
              "      <td>Schlafes Bruder (Brother of Sleep) (1995)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                                       genres\n",
              "0       1.000000  ...                  Animation|Children's|Comedy\n",
              "3045    0.825308  ...                  Animation|Children's|Comedy\n",
              "2286    0.723963  ...                  Animation|Children's|Comedy\n",
              "360     0.652019  ...                 Animation|Children's|Musical\n",
              "584     0.610675  ...          Animation|Children's|Comedy|Musical\n",
              "2728    0.606673  ...                               Comedy|Fantasy\n",
              "2182    0.597607  ...                                       Comedy\n",
              "1636    0.593955  ...                                        Drama\n",
              "3438    0.589344  ...                                       Comedy\n",
              "3682    0.586581  ...                  Animation|Children's|Comedy\n",
              "3312    0.579646  ...                                 Comedy|Drama\n",
              "1081    0.571245  ...              Children's|Drama|Fantasy|Sci-Fi\n",
              "2012    0.565554  ...  Animation|Children's|Comedy|Musical|Romance\n",
              "1873    0.562943  ...                                        Drama\n",
              "977     0.562261  ...                                        Drama\n",
              "\n",
              "[15 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3596</th>\n",
              "      <td>-0.531322</td>\n",
              "      <td>3665</td>\n",
              "      <td>Curse of the Puppet Master (1998)</td>\n",
              "      <td>Horror|Sci-Fi|Thriller</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2974</th>\n",
              "      <td>-0.539228</td>\n",
              "      <td>3043</td>\n",
              "      <td>Meatballs 4 (1992)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>654</th>\n",
              "      <td>-0.558108</td>\n",
              "      <td>660</td>\n",
              "      <td>August (1996)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>-0.612164</td>\n",
              "      <td>220</td>\n",
              "      <td>Castle Freak (1995)</td>\n",
              "      <td>Horror</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3082</th>\n",
              "      <td>-0.618153</td>\n",
              "      <td>3151</td>\n",
              "      <td>Bat Whispers, The (1930)</td>\n",
              "      <td>Crime|Drama|Mystery</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                  genres\n",
              "3596   -0.531322  ...  Horror|Sci-Fi|Thriller\n",
              "2974   -0.539228  ...                  Comedy\n",
              "654    -0.558108  ...                   Drama\n",
              "218    -0.612164  ...                  Horror\n",
              "3082   -0.618153  ...     Crime|Drama|Mystery\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgcgyXSDMxaK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 691
        },
        "outputId": "e96ed20b-aab2-4c9f-f5ba-9843fb8175de"
      },
      "source": [
        "sim(model_CF_pure,df_movies,215)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>213</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>215</td>\n",
              "      <td>Before Sunrise (1995)</td>\n",
              "      <td>Drama|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2628</th>\n",
              "      <td>0.642365</td>\n",
              "      <td>2697</td>\n",
              "      <td>My Son the Fanatic (1998)</td>\n",
              "      <td>Comedy|Drama|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1257</th>\n",
              "      <td>0.601406</td>\n",
              "      <td>1277</td>\n",
              "      <td>Cyrano de Bergerac (1990)</td>\n",
              "      <td>Action|Drama|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1199</th>\n",
              "      <td>0.588046</td>\n",
              "      <td>1217</td>\n",
              "      <td>Ran (1985)</td>\n",
              "      <td>Drama|War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1866</th>\n",
              "      <td>0.576703</td>\n",
              "      <td>1935</td>\n",
              "      <td>How Green Was My Valley (1941)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2424</th>\n",
              "      <td>0.558626</td>\n",
              "      <td>2493</td>\n",
              "      <td>Harmonists, The (1997)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3401</th>\n",
              "      <td>0.553757</td>\n",
              "      <td>3470</td>\n",
              "      <td>Dersu Uzala (1974)</td>\n",
              "      <td>Adventure|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1831</th>\n",
              "      <td>0.535995</td>\n",
              "      <td>1900</td>\n",
              "      <td>Children of Heaven, The (Bacheha-Ye Aseman) (1...</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2863</th>\n",
              "      <td>0.532223</td>\n",
              "      <td>2932</td>\n",
              "      <td>Days of Heaven (1978)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1115</th>\n",
              "      <td>0.518218</td>\n",
              "      <td>1131</td>\n",
              "      <td>Jean de Florette (1986)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2525</th>\n",
              "      <td>0.516824</td>\n",
              "      <td>2594</td>\n",
              "      <td>Open Your Eyes (Abre los ojos) (1997)</td>\n",
              "      <td>Drama|Romance|Sci-Fi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1824</th>\n",
              "      <td>0.514797</td>\n",
              "      <td>1893</td>\n",
              "      <td>Beyond Silence (1996)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3132</th>\n",
              "      <td>0.512937</td>\n",
              "      <td>3201</td>\n",
              "      <td>Five Easy Pieces (1970)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491</th>\n",
              "      <td>0.511608</td>\n",
              "      <td>495</td>\n",
              "      <td>In the Realm of the Senses (Ai no corrida) (1976)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3463</th>\n",
              "      <td>0.508097</td>\n",
              "      <td>3532</td>\n",
              "      <td>Freedom for Us (� nous la libert� ) (1931)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                genres\n",
              "213     1.000000  ...         Drama|Romance\n",
              "2628    0.642365  ...  Comedy|Drama|Romance\n",
              "1257    0.601406  ...  Action|Drama|Romance\n",
              "1199    0.588046  ...             Drama|War\n",
              "1866    0.576703  ...                 Drama\n",
              "2424    0.558626  ...                 Drama\n",
              "3401    0.553757  ...       Adventure|Drama\n",
              "1831    0.535995  ...                 Drama\n",
              "2863    0.532223  ...                 Drama\n",
              "1115    0.518218  ...                 Drama\n",
              "2525    0.516824  ...  Drama|Romance|Sci-Fi\n",
              "1824    0.514797  ...                 Drama\n",
              "3132    0.512937  ...                 Drama\n",
              "491     0.511608  ...                 Drama\n",
              "3463    0.508097  ...                Comedy\n",
              "\n",
              "[15 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2468</th>\n",
              "      <td>-0.412337</td>\n",
              "      <td>2537</td>\n",
              "      <td>Beyond the Poseidon Adventure (1979)</td>\n",
              "      <td>Adventure</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1600</th>\n",
              "      <td>-0.423138</td>\n",
              "      <td>1646</td>\n",
              "      <td>Rocket Man (1997)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2746</th>\n",
              "      <td>-0.446296</td>\n",
              "      <td>2815</td>\n",
              "      <td>Iron Eagle (1986)</td>\n",
              "      <td>Action|War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3362</th>\n",
              "      <td>-0.452282</td>\n",
              "      <td>3431</td>\n",
              "      <td>Death Wish II (1982)</td>\n",
              "      <td>Action|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3876</th>\n",
              "      <td>-0.472776</td>\n",
              "      <td>3946</td>\n",
              "      <td>Get Carter (2000)</td>\n",
              "      <td>Action|Drama|Thriller</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                 genres\n",
              "2468   -0.412337  ...              Adventure\n",
              "1600   -0.423138  ...                 Comedy\n",
              "2746   -0.446296  ...             Action|War\n",
              "3362   -0.452282  ...           Action|Drama\n",
              "3876   -0.472776  ...  Action|Drama|Thriller\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RofGbpQ5Vmmz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 963
        },
        "outputId": "ea3453b7-fee5-49dc-e596-05a7003a06a9"
      },
      "source": [
        "sim(CFmodel(),df_movies,1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"cf_recommender\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        multiple                  181230    \n",
            "_________________________________________________________________\n",
            "embedding_1 (Embedding)      multiple                  6041      \n",
            "_________________________________________________________________\n",
            "embedding_2 (Embedding)      multiple                  118590    \n",
            "_________________________________________________________________\n",
            "embedding_3 (Embedding)      multiple                  3953      \n",
            "=================================================================\n",
            "Total params: 309,814\n",
            "Trainable params: 309,814\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>1</td>\n",
              "      <td>Toy Story (1995)</td>\n",
              "      <td>Animation|Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>0.622563</td>\n",
              "      <td>27</td>\n",
              "      <td>Now and Then (1995)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2580</th>\n",
              "      <td>0.615697</td>\n",
              "      <td>2649</td>\n",
              "      <td>Son of Frankenstein (1939)</td>\n",
              "      <td>Horror</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1623</th>\n",
              "      <td>0.550291</td>\n",
              "      <td>1669</td>\n",
              "      <td>Tango Lesson, The (1997)</td>\n",
              "      <td>Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3678</th>\n",
              "      <td>0.522904</td>\n",
              "      <td>3747</td>\n",
              "      <td>Jesus' Son (1999)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1624</th>\n",
              "      <td>0.517774</td>\n",
              "      <td>1670</td>\n",
              "      <td>Welcome To Sarajevo (1997)</td>\n",
              "      <td>Drama|War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>0.510233</td>\n",
              "      <td>94</td>\n",
              "      <td>Beautiful Girls (1996)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2597</th>\n",
              "      <td>0.509824</td>\n",
              "      <td>2666</td>\n",
              "      <td>It Conquered the World (1956)</td>\n",
              "      <td>Sci-Fi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2130</th>\n",
              "      <td>0.507234</td>\n",
              "      <td>2199</td>\n",
              "      <td>Phoenix (1998)</td>\n",
              "      <td>Crime|Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>0.502151</td>\n",
              "      <td>1010</td>\n",
              "      <td>Love Bug, The (1969)</td>\n",
              "      <td>Children's|Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2655</th>\n",
              "      <td>0.498880</td>\n",
              "      <td>2724</td>\n",
              "      <td>Runaway Bride (1999)</td>\n",
              "      <td>Comedy|Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3586</th>\n",
              "      <td>0.498448</td>\n",
              "      <td>3655</td>\n",
              "      <td>Blow-Out (La Grande Bouffe) (1973)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1558</th>\n",
              "      <td>0.496263</td>\n",
              "      <td>1599</td>\n",
              "      <td>Steel (1997)</td>\n",
              "      <td>Action</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>971</th>\n",
              "      <td>0.495157</td>\n",
              "      <td>983</td>\n",
              "      <td>Madagascar Skin (1995)</td>\n",
              "      <td>Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3548</th>\n",
              "      <td>0.489274</td>\n",
              "      <td>3617</td>\n",
              "      <td>Road Trip (2000)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                       genres\n",
              "0       1.000000  ...  Animation|Children's|Comedy\n",
              "26      0.622563  ...                        Drama\n",
              "2580    0.615697  ...                       Horror\n",
              "1623    0.550291  ...                      Romance\n",
              "3678    0.522904  ...                        Drama\n",
              "1624    0.517774  ...                    Drama|War\n",
              "92      0.510233  ...                        Drama\n",
              "2597    0.509824  ...                       Sci-Fi\n",
              "2130    0.507234  ...                  Crime|Drama\n",
              "997     0.502151  ...            Children's|Comedy\n",
              "2655    0.498880  ...               Comedy|Romance\n",
              "3586    0.498448  ...                        Drama\n",
              "1558    0.496263  ...                       Action\n",
              "971     0.495157  ...                      Romance\n",
              "3548    0.489274  ...                       Comedy\n",
              "\n",
              "[15 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>movieId</th>\n",
              "      <th>title</th>\n",
              "      <th>genres</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2002</th>\n",
              "      <td>-0.534181</td>\n",
              "      <td>2071</td>\n",
              "      <td>And the Band Played On (1993)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3104</th>\n",
              "      <td>-0.535154</td>\n",
              "      <td>3173</td>\n",
              "      <td>Any Given Sunday (1999)</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3751</th>\n",
              "      <td>-0.560818</td>\n",
              "      <td>3821</td>\n",
              "      <td>Nutty Professor II: The Klumps (2000)</td>\n",
              "      <td>Comedy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>334</th>\n",
              "      <td>-0.568549</td>\n",
              "      <td>338</td>\n",
              "      <td>Virtuosity (1995)</td>\n",
              "      <td>Sci-Fi|Thriller</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>817</th>\n",
              "      <td>-0.665592</td>\n",
              "      <td>828</td>\n",
              "      <td>Adventures of Pinocchio, The (1996)</td>\n",
              "      <td>Adventure|Children's</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                genres\n",
              "2002   -0.534181  ...                 Drama\n",
              "3104   -0.535154  ...                 Drama\n",
              "3751   -0.560818  ...                Comedy\n",
              "334    -0.568549  ...       Sci-Fi|Thriller\n",
              "817    -0.665592  ...  Adventure|Children's\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tFAid3V8xog",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tf.zeros([5]) + 8"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eSj9OrFV0u8C",
        "colab_type": "text"
      },
      "source": [
        "# keras tuner"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xPhJ-fii0wSJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "3e10882f-6d61-45be-9fe0-746a2a180a38"
      },
      "source": [
        "!pip install -q -U keras-tuner\n",
        "import kerastuner as kt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |██████                          | 10kB 29.2MB/s eta 0:00:01\r\u001b[K     |████████████                    | 20kB 3.3MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 30kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 40kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 51kB 3.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 61kB 3.1MB/s \n",
            "\u001b[?25h  Building wheel for keras-tuner (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for terminaltables (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VyBB3k3-2nDt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train, x_test, y_train, y_test, global_avg = prepare_data_leftout(df_ratings, 20)\n",
        "num_users = df_ratings['userId'].unique().max() +1\n",
        "num_movies = df_ratings['movieId'].unique().max() +1\n",
        "  \n",
        "\n",
        "def model_builder(hp):\n",
        "  # model = keras.Sequential()\n",
        "  # model.add(keras.layers.Flatten(input_shape=(28, 28)))\n",
        "  ######################################################\n",
        "  # Tune the number of units in the first Dense layer\n",
        "  # Choose an optimal value between 32-512\n",
        "  # hp_units = hp.Int('units', min_value = 32, max_value = 512, step = 32)\n",
        "  # model.add(keras.layers.Dense(units = hp_units, activation = 'relu'))\n",
        "  # model.add(keras.layers.Dense(10))\n",
        "\n",
        "  hp_embedding = hp.Int('embedding_size', min_value = 30, max_value = 80, step = 10)\n",
        "  hp_lr = hp.Choice('learning_rate', values = [1e-2, 6e-3, 3e-3 ,1e-3, 1e-4]) \n",
        "  hp_emb_init = hp.Choice('embeddings_initializer',values = [\"glorot_uniform\", \"he_uniform\"])\n",
        "  hp_reg = hp.Choice('embedding_regularization', values = [1e-2, 1e-3,1e-4,1e-5,1e-6])\n",
        "  hp_pure = hp.Choice('pure CF model', values = [True, False])\n",
        "  import keras.backend as K\n",
        "  class CFRecommender(tf.keras.Model):\n",
        "    def __init__(self, num_users, num_movies, embedding_size, global_avg,hp_emb_init,hp_reg, hp_pure):\n",
        "      super().__init__()#CFRecommender, self\n",
        "      self.num_users = num_users\n",
        "      self.num_movies = num_movies\n",
        "      self.embedding_size = embedding_size\n",
        "      self.global_avg = global_avg\n",
        "      self.hp_emb_init = hp_emb_init\n",
        "      self.hp_pure = hp_pure\n",
        "      self.hp_reg = hp_reg \n",
        "      # self.user_mean_bias = user_mean_bias\n",
        "      # self.movie_mean_bias = movie_mean_bias\n",
        "\n",
        "      self.user_embedding = tf.keras.layers.Embedding(self.num_users,self.embedding_size,embeddings_regularizer=keras.regularizers.l2(hp_reg),embeddings_initializer=hp_emb_init)\n",
        "      self.user_bias = tf.keras.layers.Embedding(self.num_users, 1,embeddings_initializer=hp_emb_init) #, self.embedding_size embeddings_regularizer=keras.regularizers.l2(1e-6),\n",
        "\n",
        "      self.movie_embedding = tf.keras.layers.Embedding(self.num_movies, self.embedding_size,embeddings_regularizer=keras.regularizers.l2(hp_reg),embeddings_initializer=hp_emb_init)\n",
        "      self.movie_bias = tf.keras.layers.Embedding(self.num_movies, 1,embeddings_initializer=hp_emb_init) #, self.embedding_size\n",
        "\n",
        "    def call(self, inputs): #input: dim1 x 2\n",
        "\n",
        "      user_vector = self.user_embedding(inputs[:,0]) #dim1 x embedding_size\n",
        "      user_bias = self.user_bias(inputs[:,0]) #dim1 x 1\n",
        "\n",
        "      movie_vector = self.movie_embedding(inputs[:,1]) #dim1 x embedding_size\n",
        "      movie_bias = self.movie_bias(inputs[:,1]) #dim1 x 1\n",
        "\n",
        "      mu = self.global_avg\n",
        "\n",
        "      # user_avg = self.user_mean_bias.loc[inputs[:,0]].values\n",
        "      # movie_avg = self.movie_mean_bias.loc[inputs[:,1]].values\n",
        "      user_avg = inputs[:,2] #dim1 x 1\n",
        "      movie_avg = inputs[:,3] #dim1 x 1\n",
        "\n",
        "      \"\"\"\n",
        "      user_vector, movie_vector : (batch_size, embedding_dim)\n",
        "      \"\"\"\n",
        "      dot_user_movie = tf.reduce_sum(tf.math.multiply(user_vector,movie_vector), axis = 1) #tf.math.l2_normalize(user_vector,0) # tf.math.l2_normalize(movie_vector,0)  #dim1 x 1\n",
        "      # print('test0')\n",
        "      # print(K.int_shape(user_avg[:,np.newaxis]), K.int_shape(user_bias))\n",
        "      # print(K.int_shape(dot_user_movie), K.int_shape(tf.math.multiply(user_avg[:,np.newaxis],user_bias)),K.int_shape(tf.math.multiply(movie_avg[:,np.newaxis],movie_bias)) )\n",
        "      # print('test1')\n",
        "      \n",
        "      if hp_pure == True:\n",
        "        c_ratings = dot_user_movie\n",
        "      else:  \n",
        "        c_ratings = dot_user_movie + tf.math.multiply(user_avg,user_bias) + tf.math.multiply(movie_avg,movie_bias) + mu\n",
        "      # print('test2')\n",
        "\n",
        "      return tf.nn.sigmoid(dot_user_movie)\n",
        "\n",
        "\n",
        "  model = CFRecommender(num_users, num_movies, hp_embedding, global_avg,hp_emb_init,hp_reg, hp_pure)\n",
        "  model.build(input_shape=(None,4))\n",
        "  # model.summary()\n",
        "\n",
        "  #######################################################\n",
        "  \n",
        "  # Tune the learning rate for the optimizer \n",
        "  # Choose an optimal value from 0.01, 0.001, or 0.0001\n",
        "  # hp_learning_rate = hp.Choice('learning_rate', values = [1e-2, 1e-3, 1e-4]) \n",
        "  \n",
        "  model.compile(loss = tf.keras.losses.BinaryCrossentropy(), optimizer = keras.optimizers.Adam(hp_lr))#, metrics = ['acc']\n",
        "  \n",
        "  # model.compile(optimizer = keras.optimizers.Adam(learning_rate = hp_learning_rate),\n",
        "  #               loss = keras.losses.SparseCategoricalCrossentropy(from_logits = True), \n",
        "  #               metrics = ['accuracy'])\n",
        "  \n",
        "  return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6QwsTBl3Ku2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "3622091a-e3f0-4727-c667-25caca2b72d8"
      },
      "source": [
        "tuner = kt.Hyperband(model_builder,\n",
        "                     objective = 'val_loss', \n",
        "                     max_epochs = 10,\n",
        "                     factor = 3,\n",
        "                     directory = \"/content/gdrive/My Drive/Machine Learning/movielens1m\"+ 'my_dir',\n",
        "                     project_name = 'intro_to_kt')    "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Reloading Oracle from existing project /content/gdrive/My Drive/Machine Learning/movielens1mmy_dir/intro_to_kt/oracle.json\n",
            "INFO:tensorflow:Reloading Tuner from /content/gdrive/My Drive/Machine Learning/movielens1mmy_dir/intro_to_kt/tuner0.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3mJrPOXA3PWJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# callback = tf.keras.callbacks.EarlyStopping(        #check val_loss vs loss which one to consider\n",
        "#       monitor='loss', min_delta=0, patience=1, verbose=0, mode='auto',\n",
        "#       baseline=None, restore_best_weights=False\n",
        "#   )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNuKxEOYQ5S1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        },
        "outputId": "38c8350f-6456-4112-8a51-ce85bee4a80e"
      },
      "source": [
        "!pip install IPython\n",
        "from IPython.display import clear_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: IPython in /usr/local/lib/python3.6/dist-packages (5.5.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from IPython) (1.0.18)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from IPython) (2.1.3)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.6/dist-packages (from IPython) (49.2.0)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from IPython) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from IPython) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from IPython) (0.8.1)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from IPython) (4.3.3)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from IPython) (4.4.2)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->IPython) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->IPython) (1.15.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->IPython) (0.6.0)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.2->IPython) (0.2.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j40dyTkJRD7u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import IPython\n",
        "import IPython.display"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aRNbMlSoLetB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ClearTrainingOutput(tf.keras.callbacks.Callback):\n",
        "  def on_train_end(*args, **kwargs):\n",
        "    IPython.display.clear_output(wait = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncSFwCeMLfUt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 673
        },
        "outputId": "d2be45cb-dc2b-436c-cf59-d622660e77d0"
      },
      "source": [
        "tuner.search(x_train, y_train, epochs = 10, validation_data = (x_test, y_test), callbacks = [ClearTrainingOutput()] , batch_size = 512)\n",
        "\n",
        "# Get the optimal hyperparameters\n",
        "best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]\n",
        "\n",
        "print(f\"\"\"\n",
        "The hyperparameter search is complete. \n",
        "\n",
        "learning_rate = {best_hps.get('learning_rate')}\n",
        "embedding_size = {best_hps.get('embedding_size')}\n",
        "embeddings_initializer = {best_hps.get('embeddings_initializer')}\n",
        "embedding_regularization = {best_hps.get('embedding_regularization')}\n",
        "pure CF model = {best_hps.get('pure CF model')}\n",
        "\n",
        "\"\"\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:#4527A0\"><h1 style=\"font-size:18px\">Trial complete</h1></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:#4527A0\"><h1 style=\"font-size:18px\">Trial summary</h1></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-Trial ID: a2c5d0393b8578749d9eb99f201f235d</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-Score: 0.6931461691856384</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-Best step: 0</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:#7E57C2\"><h2 style=\"font-size:16px\">Hyperparameters:</h2></span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-embedding_regularization: 0.01</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-embedding_size: 50</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-embeddings_initializer: he_uniform</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-learning_rate: 0.001</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-pure CF model: 0</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-tuner/bracket: 2</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-tuner/epochs: 4</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-tuner/initial_epoch: 2</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:cyan\"> |-tuner/round: 1</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<span style=\"color:blue\"> |-tuner/trial_id: 085af343ced4858743cf2a6d697f4370</span>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 5/10\n",
            "WARNING:tensorflow:Layer cf_recommender is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because its dtype defaults to floatx.\n",
            "\n",
            "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
            "\n",
            "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:432: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
            "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['embedding_1/embeddings:0', 'embedding_3/embeddings:0'] when minimizing the loss.\n",
            "1718/1718 [==============================] - 11s 6ms/step - loss: 0.6160 - val_loss: 0.5947\n",
            "Epoch 6/10\n",
            "1718/1718 [==============================] - 11s 6ms/step - loss: 0.5905 - val_loss: 0.5926\n",
            "Epoch 7/10\n",
            "1015/1718 [================>.............]Buffered data was truncated after reaching the output size limit."
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NOlWrC7GcwEz",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gq8YIvtIN6ah",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "best_hps = tuner.get_best_hyperparameters(num_trials = 1)[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nDUfGopZRkj5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "bc56b7bc-7cb7-495b-ce27-e222cd806451"
      },
      "source": [
        "\n",
        "print(f\"\"\"\n",
        "The hyperparameter search is complete. \n",
        "\n",
        "learning_rate = {best_hps.get('learning_rate')}\n",
        "embedding_size = {best_hps.get('embedding_size')}\n",
        "embeddings_initializer = {best_hps.get('embeddings_initializer')}\n",
        "embedding_regularization = {best_hps.get('embedding_regularization')}\n",
        "pure CF model = {best_hps.get('pure CF model')}\n",
        "\n",
        "\"\"\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "The hyperparameter search is complete. \n",
            "\n",
            "learning_rate = 0.001\n",
            "embedding_size = 30\n",
            "embeddings_initializer = glorot_uniform\n",
            "embedding_regularization = 1e-06\n",
            "pure CF model = 1\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lt8eOxgsRqKi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ovn0RGCu_QNA",
        "colab_type": "text"
      },
      "source": [
        "# Hit Rate on the 1m dataset  \n",
        "Not working. Hit rate is always 0. Find the bug."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mBdVCnWCIeQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def split_train_eval(df_ratings, k):\n",
        "  \"\"\"\n",
        "  returns train set and eval set\n",
        "  \"\"\"\n",
        "  # df_ratings['user_mean_bias'] = user_mean_bias.loc[df_ratings['userId'].values].values                   #comment for v1\n",
        "  # df_ratings['movie_mean_bias'] = movie_mean_bias.loc[df_ratings['movieId'].values].values                   #comment for v1\n",
        "\n",
        "  leftOutPredictions =df_ratings.groupby('userId', as_index = False).apply(lambda x: x.iloc[-k:,:])\n",
        "  \n",
        "  # print(leftOutPredictions)\n",
        "\n",
        "  # df_ratings_train = df_ratings[~(df_ratings.isin(leftOutPredictions.values))]\n",
        "  df_ratings_train = df_ratings.groupby('userId', as_index = False).apply(lambda x: x.iloc[:-k,:])#.reset_index(drop = True,inplace =True)\n",
        "  df_ratings_train.index = pd.RangeIndex(len(df_ratings_train.index))\n",
        "\n",
        "  # display(df_ratings_train)\n",
        "\n",
        "  return df_ratings_train, leftOutPredictions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZNJUGS0lHQK0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def mean_biases(df_ratings, k=1):\n",
        "  y_min = min(df_ratings['rating'].values)\n",
        "  y_max = max(df_ratings['rating'].values)\n",
        "\n",
        "  df_ratings['rating'] = (df_ratings['rating'] - y_min)/ (y_max - y_min)\n",
        "\n",
        "  # split_index = int(0.9*df_ratings.shape[0]) \n",
        "  df_ratings_train, df_ratings_test = split_train_eval(df_ratings, k)\n",
        "  # df_ratings_train, df_ratings_test = df_ratings.iloc[:split_index], df_ratings.iloc[split_index:]\n",
        "\n",
        "  global_avg = df_ratings_train['rating'].sum()/df_ratings_train.shape[0] \n",
        "  # print(global_avg)\n",
        "  user_mean_bias = df_ratings_train.groupby('userId')['rating'].agg('mean') - global_avg\n",
        "  movie_mean_bias = df_ratings_train.groupby('movieId')['rating'].agg('mean')  - global_avg\n",
        "\n",
        "  all_user_mean_bias = pd.Series(np.zeros(df_ratings['userId'].unique().max()+1))#, columns = ['userId']\n",
        "  all_movie_mean_bias = pd.Series(np.zeros(df_ratings['movieId'].unique().max()+1))#, columns = ['movieId']\n",
        "\n",
        "  all_user_mean_bias.update(user_mean_bias)#, overwrite = True\n",
        "  all_movie_mean_bias.update(movie_mean_bias)#, overwrite = True\n",
        "  \n",
        "  user_mean_bias = all_user_mean_bias\n",
        "  movie_mean_bias = all_movie_mean_bias\n",
        "  \n",
        "  return user_mean_bias, movie_mean_bias"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cb5VNu-KCP_R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def prepare_data_hitrate(df_ratings, k=1):\n",
        "\n",
        "  y_min = min(df_ratings['rating'].values)\n",
        "  y_max = max(df_ratings['rating'].values)\n",
        "\n",
        "  df_ratings['rating'] = (df_ratings['rating'] - y_min)/ (y_max - y_min)\n",
        "\n",
        "  split_index = int(0.9*df_ratings.shape[0]) \n",
        "  df_ratings_train, df_ratings_test = split_train_eval(df_ratings, k)\n",
        "  # df_ratings_train, df_ratings_test = df_ratings.iloc[:split_index], df_ratings.iloc[split_index:]\n",
        "\n",
        "  global_avg = df_ratings_train['rating'].sum()/df_ratings_train.shape[0] \n",
        "  # print(global_avg)\n",
        "  user_mean_bias = df_ratings_train.groupby('userId')['rating'].agg('mean') - global_avg\n",
        "  movie_mean_bias = df_ratings_train.groupby('movieId')['rating'].agg('mean')  - global_avg\n",
        "\n",
        "  all_user_mean_bias = pd.Series(np.zeros(df_ratings['userId'].unique().max()+1))#, columns = ['userId']\n",
        "  all_movie_mean_bias = pd.Series(np.zeros(df_ratings['movieId'].unique().max()+1))#, columns = ['movieId']\n",
        "\n",
        "  all_user_mean_bias.update(user_mean_bias)#, overwrite = True\n",
        "  all_movie_mean_bias.update(movie_mean_bias)#, overwrite = True\n",
        "  \n",
        "  user_mean_bias = all_user_mean_bias\n",
        "  movie_mean_bias = all_movie_mean_bias\n",
        "  \n",
        "  df_ratings_train['user_mean_bias'] = user_mean_bias.loc[df_ratings_train['userId'].values].values\n",
        "  df_ratings_train['movie_mean_bias'] = movie_mean_bias.loc[df_ratings_train['movieId'].values].values\n",
        "\n",
        "  df_ratings_test['user_mean_bias'] = user_mean_bias.loc[df_ratings_test['userId'].values].values\n",
        "  df_ratings_test['movie_mean_bias'] = movie_mean_bias.loc[df_ratings_test['movieId'].values].values\n",
        "\n",
        "  # x_train, x_test, y_train, y_test = df_ratings_train[['userId', 'movieId','user_mean_bias','movie_mean_bias']], df_ratings_test[['userId', 'movieId','user_mean_bias','movie_mean_bias']], df_ratings_train[['rating']], df_ratings_test[['rating']]\n",
        "  \n",
        "  # return x_train, x_test, y_train, y_test, global_avg\n",
        "  return df_ratings_train, df_ratings_test"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hx4d_NH4o0ad",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# df_ratings_train, leftOutPredictions = prepare_data_hitrate(df_ratings)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hh_hsl_z7wPC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def HitRate(topNPredicted, leftOutPredictions): \n",
        "  '''\n",
        "  leftOutPredictions : numpy array of the last rating from each user (num_users,3 ie userid, movieid, predictedRating)\n",
        "  topNPredicted : numpy array of the top N predicted movies for each user (num_users, N ie top, 3 ie movied, predictedRating)\n",
        "  '''\n",
        "  hits = 0.0\n",
        "  total = 0.0\n",
        "  # print(leftOutPredictions)\n",
        " # For each left-out rating\n",
        "  for leftOut in leftOutPredictions:\n",
        "      userID = leftOut[0]\n",
        "      # print(userID)\n",
        "      leftOutMovieID = leftOut[1]\n",
        "      # print(\"leftOutMovieID=\",leftOutMovieID)\n",
        "      # print(leftOutMovieID)\n",
        "      # Is it in the predicted top 10 for this user?\n",
        "      hit = False\n",
        "      for predicted in topNPredicted[int(userID)-1]:\n",
        "          movieID, predictedRating = predicted[0], predicted[1]\n",
        "          # print(\"(movieID)=\",int(movieID))\n",
        "          if (int(leftOutMovieID) == int(movieID)):\n",
        "              hit = True\n",
        "              break\n",
        "      if (hit) :\n",
        "          hits += 1.0\n",
        "\n",
        "      total += 1.0\n",
        "\n",
        "  # Compute overall precision\n",
        "  return (hits*1.0)/(total*1.0)\n",
        "\n",
        "# print(\"\\nHit Rate: \", HitRate(topNPredicted, leftOutPredictions))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "okuJAmxTi6uX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def not_watched(model, leftOutPredictions, df_ratings):\n",
        "  # predictions = model.predict(leftOutPrediction.iloc[:,0], batch_size = BATCH_SIZE_P).flatten()\n",
        "\n",
        "  users = leftOutPredictions.iloc[:,0]    #series of users\n",
        "  # movies = df_movies['movieId']\n",
        "  movies = df_ratings['movieId'].unique()\n",
        "\n",
        "  user_mean_bias , movie_mean_bias =  mean_biases(df_ratings)  \n",
        "  # display(users)\n",
        "  # display(movies)\n",
        "\n",
        "  df_all = pd.DataFrame([(i,s) for i in users for s in movies], columns = ['userId','movieId'])\n",
        "\n",
        "  df_all['user_mean_bias'] = user_mean_bias.loc[df_all['userId'].values].values     \n",
        "  df_all['movie_mean_bias'] = movie_mean_bias.loc[df_all['movieId'].values].values     \n",
        "\n",
        "  # display(df_all)\n",
        "\n",
        "  # display(df_ratings)\n",
        "  # df = pd.DataFrame([(i, s) for i in id for s in st], columns=['id', 'st'])\n",
        "  movies_not_watched = df_all.merge(df_ratings[['userId','movieId']],how = 'outer',on = ['userId','movieId'], indicator = True)\n",
        "  # merged = pd.merge(A,B, how='outer', indicator=True)\n",
        "  # merged[merged['_merge'] == 'left_only']\n",
        "  movies_not_watched = movies_not_watched[movies_not_watched['_merge'] == 'left_only']\n",
        "  movies_not_watched.drop(columns = ['_merge'], inplace = True)\n",
        "  movies_not_watched.reset_index(drop = True, inplace = True)\n",
        "  # movies_not_watched = movies_not_watched[['userId','movieId']]\n",
        "  \n",
        "  # display(movies_not_watched)\n",
        "\n",
        "  \n",
        "  # print(ratings.shape)\n",
        "  # print(movies_not_watched.shape)\n",
        "\n",
        "  #BATCH_SIZE_P should be divisible by \n",
        "  BATCH_SIZE_P = 512 #98    #142 #33\n",
        "  #\n",
        "  # print(np.reshape(np.array(movies_not_watched.values, dtype = np.float),(-1,4)).shape)\n",
        "\n",
        "  ratings = model.predict(np.reshape(np.array(movies_not_watched.values, dtype = np.float),(-1,4)), verbose = 2, batch_size = BATCH_SIZE_P)#.flatten() ##remove .iloc for V1 model\n",
        "  movies_not_watched['ratings'] = ratings\n",
        "\n",
        "  return movies_not_watched"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFFrwAWy7VVy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predictN(movies_not_watched, N):\n",
        " \n",
        "  print(\"movies_not_watched: \",movies_not_watched.shape)\n",
        "  movies_not_watched['new']=movies_not_watched.groupby('userId').cumcount()\n",
        "  df_true=movies_not_watched.set_index(['userId','new']).unstack(fill_value=0).stack(dropna=False).reset_index('userId').reset_index( drop = True)\n",
        "  print(\"df_true:\", df_true.shape)\n",
        "  topN = df_true.groupby('userId')[['movieId','ratings']].apply(lambda x: x.nlargest(N, columns=['ratings']))  #.sort_values(by = 'ratings', ascending = False)[:N] #['movieId']\n",
        "  display(\"topN: \",topN.shape)\n",
        "  \n",
        "  # print(\"topN.values.shape\",topN.values.shape)\n",
        "\n",
        "  # print(topN.values)\n",
        "  # print(\"np.array(topN).shape\",np.array(topN).shape)\n",
        "  # print(\"np.array(topN.values).shape\",np.array(topN.values).shape)\n",
        "  topN_array = np.reshape(np.array(topN),newshape = (6040, N,2))\n",
        "  \n",
        "  # print(topN.values)\n",
        "\n",
        "\n",
        "  return topN_array\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4vR0D4MaP1o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# movies_not_watched = not_watched(model, leftOutPredictions)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LT8KgzY0dNWr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# topNpredicted = predictN(movies_not_watched, 1000)\n",
        "# HitRate(topNpredicted,leftOutPredictions.values)#topNpredicted"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RgmJoyA_OGAg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def evaluate_hitrate(df_ratings, model,minN=10, maxN = 2000):      #, model,df_movies\n",
        "  \n",
        "\n",
        "  df_ratings_train, leftOutPredictions = prepare_data_hitrate(df_ratings)    #leftOutPrediction : (num_user, 5)\n",
        "\n",
        "  # model = getModel()\n",
        "  # model = CFmodel(num_users = num_users, num_movies=num_movies, global_avg = global_avg,hparam =hparam)\n",
        "  # model = train_CFmodel(model, x_train, y_train, x_test, y_test)\n",
        "  \n",
        "  # model = train_model(model, df_ratings_train)\n",
        "  movies_not_watched = not_watched(model, leftOutPredictions,df_ratings)\n",
        "  # topNpredicted = predictN(movies_not_watched, 10)\n",
        "\n",
        "  hit_list = []\n",
        "  for n in range(minN,maxN+1):\n",
        "    topNPredicted = predictN(movies_not_watched,n)\n",
        "    hit_rate = HitRate(topNPredicted, leftOutPredictions.values)\n",
        "    print(\"Hit rate at \",n,\":= \",hit_rate)\n",
        "    hit_list.append(hit_rate)\n",
        "  \n",
        "  return hit_list\n",
        "\n",
        "\n",
        "  # plot(hit_list)\n",
        "\n",
        "  \n",
        "\n",
        "  '''user : the subset of df_ratings for which user = userid'''\n",
        "  # user = df_ratings[df_ratings['userId']==user_id]\n",
        "  # movies_watched_id = user[\"movieId\"]           #df_ratings[ df_ratings['movieId'].isin(user['movieId'].unique())]['movieId']\n",
        "  # print(movies_watched_id)\n",
        "\n",
        "  '''movies watched : contains names of the movies the user has watched'''\n",
        "  # movies_watched = df_movies[df_movies['movieId'].isin(movies_watched_id)]['title']                   #.reset_index() \n",
        "  # movies_loved = df_movies[df_movies['movieId'].isin(user[user['rating'].values>0.6].sort_values(by = 'rating', ascending = False)['movieId'])][:30][['title','genres']]\n",
        "\n",
        "  # print(\"Movies user {} loved! : \".format(user_id))\n",
        "  # print(\"*\"*30)\n",
        "  # display(movies_loved.head(20))\n",
        "  \n",
        "  # print(\"*\"*30) \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GWFODBxSK-AU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# x = evaluate(x_aug,model,20,200)\n",
        "saved = True\n",
        "\n",
        "if saved:\n",
        "  model = saved_model()\n",
        "else:\n",
        "  model = CFmodel(num_users = num_users, num_movies=num_movies, global_avg = global_avg,hparam =hparam)\n",
        "  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6s1Adhxs_XJe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 478
        },
        "outputId": "7fbc52b0-1a8a-47d8-92bb-92c3a21def21"
      },
      "source": [
        "hitrate_result = evaluate_hitrate(df_ratings, model,minN=3930, maxN = 3952)\n",
        "hitrate_result"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "41766/41766 - 43s\n",
            "movies_not_watched:  (21384031, 5)\n",
            "df_true: (22263440, 5)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'topN: '"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "(22263440, 2)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-104-0cc90a0d56cf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhitrate_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate_hitrate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_ratings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mminN\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3930\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxN\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m3952\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mhitrate_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-91-cd7530c55dc5>\u001b[0m in \u001b[0;36mevaluate_hitrate\u001b[0;34m(df_ratings, model, minN, maxN)\u001b[0m\n\u001b[1;32m     16\u001b[0m   \u001b[0mhit_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmaxN\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mtopNPredicted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredictN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmovies_not_watched\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mhit_rate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mHitRate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtopNPredicted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mleftOutPredictions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Hit rate at \"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\":= \"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mhit_rate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-88-5c0991c70a10>\u001b[0m in \u001b[0;36mpredictN\u001b[0;34m(movies_not_watched, N)\u001b[0m\n\u001b[1;32m     13\u001b[0m   \u001b[0;31m# print(\"np.array(topN).shape\",np.array(topN).shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m   \u001b[0;31m# print(\"np.array(topN.values).shape\",np.array(topN.values).shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m   \u001b[0mtopN_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtopN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnewshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m6040\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m   \u001b[0;31m# print(topN.values)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mreshape\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36mreshape\u001b[0;34m(a, newshape, order)\u001b[0m\n\u001b[1;32m    299\u001b[0m            [5, 6]])\n\u001b[1;32m    300\u001b[0m     \"\"\"\n\u001b[0;32m--> 301\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'reshape'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnewshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36m_wrapfunc\u001b[0;34m(obj, method, *args, **kwds)\u001b[0m\n\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mbound\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;31m# A TypeError occurs if the object does have such a method in its\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: cannot reshape array of size 44526880 into shape (6040,3930,2)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8-_ctSIGLsD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CeYvhQC3kEco",
        "colab_type": "text"
      },
      "source": [
        "# Visualizing the embeddings\n",
        "Use df_movies and embedding data of all movies to make an altair plot of the tSNE on embeddings clustered by genre."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yd-ZXEMbKAk3",
        "colab_type": "text"
      },
      "source": [
        "# To do  \n",
        "1. Ensure that all user bias which havent been updated from the training dataset are initialized to mu instead of 0.  \n",
        "  \n",
        "  Prepare data-> prepare_data_leftout->all_user_mean_bias = pd.Series(np.zeros(df_ratings['userId'].unique().max()+1))\n",
        "    \n",
        "    change this to all_user_mean_bias = pd.Series(np.zeroslike(df_ratings['userId'].unique().max()+1), mu)\n",
        "\n",
        "    Retrain and evaluate the accuracy\n",
        "  \n",
        "\n",
        "2. Analyse the hitrate functions and manually check one example to see where is the error.  \n",
        "\n",
        "    Even random guessing should give >0 hitrate when N=3500(total movies ~= 3880)\n",
        "\n",
        "\n",
        "3. Recover 22.6 rmse. This was obtained before the model tuning.\n",
        "\n",
        "4. tSNE on Embeddings, add it to df_movies and cluster according to genre. Plot suing Altair. References: Google developers recommender system link has a plot.\n",
        "\n",
        "5. **Publish it to Github**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QTItQFcRK_XZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 231
        },
        "outputId": "3cdf96a5-76ed-4972-ad6c-f85d7f9cd8de"
      },
      "source": [
        "# df = pd.DataFrame({\"id\": [100, 200, 200, 300, 300, 300], \"val1\": [1.5, 2.5, 4.5, np.nan, 6.5, np.nan], \"val2\": [9.5, 7.5, 8.5, 3.5, np.nan, np.nan]})\n",
        "# df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>val1</th>\n",
              "      <th>val2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>100</td>\n",
              "      <td>1.5</td>\n",
              "      <td>9.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>200</td>\n",
              "      <td>2.5</td>\n",
              "      <td>7.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>200</td>\n",
              "      <td>4.5</td>\n",
              "      <td>8.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>300</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>300</td>\n",
              "      <td>6.5</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>300</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    id  val1  val2\n",
              "0  100   1.5   9.5\n",
              "1  200   2.5   7.5\n",
              "2  200   4.5   8.5\n",
              "3  300   NaN   3.5\n",
              "4  300   6.5   NaN\n",
              "5  300   NaN   NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-Y91876bxwa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "14c4253c-7e8d-4c53-d2a6-79ba22fec324"
      },
      "source": [
        "# df['new']=df.groupby('id').cumcount()\n",
        "# df_true=df.set_index(['id','new']).unstack(fill_value=0).stack(dropna=False).reset_index('id').reset_index( drop = True)\n",
        "# display(df_true)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>val1</th>\n",
              "      <th>val2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>100</td>\n",
              "      <td>1.5</td>\n",
              "      <td>9.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>100</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>100</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>200</td>\n",
              "      <td>2.5</td>\n",
              "      <td>7.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>200</td>\n",
              "      <td>4.5</td>\n",
              "      <td>8.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>200</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>300</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>300</td>\n",
              "      <td>6.5</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>300</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    id  val1  val2\n",
              "0  100   1.5   9.5\n",
              "1  100   0.0   0.0\n",
              "2  100   0.0   0.0\n",
              "3  200   2.5   7.5\n",
              "4  200   4.5   8.5\n",
              "5  200   0.0   0.0\n",
              "6  300   NaN   3.5\n",
              "7  300   6.5   NaN\n",
              "8  300   NaN   NaN"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "870U5aq4b1g9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e556a7fb-95bb-49ef-c4ad-2d8d120946b7"
      },
      "source": [
        "# df_true.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['id', 'val1', 'val2'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fb7FAPsGb5wJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# df_true.index.name"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0hXOB8p_cJPA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "20228407-338c-4c20-c35f-9201fa332f62"
      },
      "source": [
        "# df_true.index"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RangeIndex(start=0, stop=9, step=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x1NdoI1ucLSc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}