01_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Prepration for GoodReads Dataset\n",
    "## (Efficient) Loading, Cleaning, Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "import json\n",
    "import re\n",
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from tqdm.notebook import tqdm\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR = './data'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Data\n",
    "using pandas to read the JSON data - ~using already unzipped JSON file to improve performance~ and to later make the switch to DASK.\n",
    "\n",
    "**only using 30k rows of books and 30k reviews**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_fn_genres = 'goodreads_book_genres_initial.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_fn_books = 'goodreads_books_mystery_thriller_crime.json.gz'\n",
    "in_fn_reviews = 'goodreads_reviews_mystery_thriller_crime.json.gz'\n",
    "in_fn_authors = 'goodreads_book_authors.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "NBOOKS = 50000\n",
    "NREVIEWS = 2*NBOOKS\n",
    "NAUTHORS = 5*NBOOKS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "read chunks of json and save to csv for easier tf.data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUT_DIR = './data/cleaned/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_size = 500\n",
    "\n",
    "book_reader = pd.read_json(os.path.join(DIR, in_fn_books), lines=True, dtype={\n",
    "    \"title\": 'string', \n",
    "    \"description\": 'string', \n",
    "    \"text_reviews_count\": 'uint', \n",
    "    'average_rating': 'float'\n",
    "}, chunksize=chunk_size)\n",
    "genre_reader = pd.read_json(os.path.join(DIR, in_fn_genres), lines=True, dtype={\"book_id\": 'uint32'}, chunksize=chunk_size)\n",
    "authors_reader = pd.read_json(os.path.join(DIR, in_fn_authors), lines=True, chunksize=chunk_size,\n",
    "dtype = {\n",
    "    'author': 'name'\n",
    "})\n",
    "review_reader = pd.read_json(os.path.join(DIR, in_fn_reviews), lines=True, chunksize=chunk_size, \n",
    "dtype = {\n",
    "'book_id': 'uint32',\n",
    "'rating': 'uint8',\n",
    "'review_text': 'string',\n",
    "})\n",
    "\n",
    "# set the datatypes to save memory already when reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write first chunk with header\n",
    "with open(os.path.join(OUT_DIR, \"genres.csv\"), 'a') as f:\n",
    "    chunk = next(genre_reader)\n",
    "    chunk.set_index(\"book_id\", inplace=True)\n",
    "    chunk = pd.concat([chunk.drop(['genres'], axis=1), chunk['genres'].apply(pd.Series)], axis=1)\n",
    "    chunk.rename(columns={\"comics, graphic\": \"comics_graphic\", \n",
    "                     \"history, historical fiction, biography\": \"history_historical_fiction_biography\", \n",
    "                     \"fantasy, paranormal\": \"fantasy_paranormal\", \n",
    "                     \"mystery, thriller, crime\": \"mystery_thriller_crime\"},\n",
    "            inplace=True)\n",
    "    chunk.to_csv(f, header=True, line_terminator='\\n')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(OUT_DIR, \"genres.csv\"), 'a') as f:\n",
    "    no_lines = 2360656\n",
    "    for chunk in tqdm(genre_reader, total=no_lines/chunk_size):\n",
    "        chunk.set_index(\"book_id\", inplace=True)\n",
    "        chunk = pd.concat([chunk.drop(['genres'], axis=1), chunk['genres'].apply(pd.Series)], axis=1)\n",
    "        chunk.rename(columns={\"comics, graphic\": \"comics_graphic\", \n",
    "                    \"history, historical fiction, biography\": \"history_historical_fiction_biography\", \n",
    "                    \"fantasy, paranormal\": \"fantasy_paranormal\", \n",
    "                    \"mystery, thriller, crime\": \"mystery_thriller_crime\"},\n",
    "        inplace=True)\n",
    "\n",
    "        chunk.to_csv(f, header=False, line_terminator='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_book_df(df):\n",
    "    df.set_index(\"book_id\", inplace=True)\n",
    "    df.loc[:,'author_id'] = df['authors'].apply(lambda row: row[0]['author_id']) # just select the first author_id of the list of authors\n",
    "    df.loc[:, 'author_id'] = df['author_id'].astype('int64')\n",
    "    df = df.loc[:, ['title', 'text_reviews_count', 'average_rating', 'description', 'author_id']]\n",
    "    df.loc[:,'title'] = df['title'].str.encode('utf-8')\n",
    "    df.loc[:,'description'] = df['description'].replace(r'\\n',' ', regex=True) \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write first chunk with header\n",
    "with open(os.path.join(OUT_DIR, \"books.csv\"), 'a') as f:\n",
    "    chunk = next(book_reader)\n",
    "    chunk = prepare_book_df(chunk)\n",
    "    chunk.to_csv(f, header=True, line_terminator='\\n')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e25f19225674b8f9461444388427f70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2192.35 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with open(os.path.join(OUT_DIR, \"books.csv\"), 'a') as f:\n",
    "    no_lines = 219235\n",
    "    for chunk in tqdm(book_reader, total=no_lines/chunk_size):\n",
    "        chunk = prepare_book_df(chunk)\n",
    "        chunk.to_csv(f, header=False, line_terminator='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_review_df(df):\n",
    "    df.set_index(\"review_id\", inplace=True)\n",
    "    df.drop(columns=['date_added', 'date_updated', 'read_at', 'started_at', 'n_votes', 'n_comments'], inplace=True)\n",
    "    df.loc[:, 'review_text'] = df['review_text'].replace(r'\\n',' ', regex=True) \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write first chunk with header\n",
    "with open(os.path.join(OUT_DIR, \"reviews.csv\"), 'a') as f:\n",
    "    chunk = next(review_reader)\n",
    "    chunk = prepare_review_df(chunk)\n",
    "    chunk.to_csv(f, header=True, line_terminator='\\n')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(OUT_DIR, \"reviews.csv\"), 'a') as f:\n",
    "    no_lines = 1849236\n",
    "    for chunk in tqdm(review_reader, total=no_lines/chunk_size):\n",
    "        chunk = prepare_review_df(chunk)\n",
    "        chunk.to_csv(f, header=False, line_terminator='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- genres (= books) => 2.2 Mio\n",
    "- authors => 830k\n",
    "- books (mystery,crime,thriller) => 220k\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "read csv and merge dataframes, save to pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_DIR = './models/'\n",
    "DIR = './data/cleaned/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_fn_books = 'books.csv'\n",
    "csv_fn_reviews = 'reviews.csv'\n",
    "csv_fn_genres = 'genres.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk_size = 500\n",
    "\n",
    "book_dtype={\n",
    "    \"title\": 'string', \n",
    "    \"description\": 'string', \n",
    "    \"text_reviews_count\": 'uint', \n",
    "    'average_rating': 'float'\n",
    "}\n",
    "\n",
    "review_dtype = {\n",
    "'book_id': 'uint32',\n",
    "'rating': 'uint8',\n",
    "'review_text': 'string',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text_reviews_count</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>description</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>book_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6066814</th>\n",
       "      <td>Crowner Royal (Crowner John Mystery, #13)</td>\n",
       "      <td>15</td>\n",
       "      <td>3.93</td>\n",
       "      <td>London, 1196. At the command of Richard the Li...</td>\n",
       "      <td>37778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33394837</th>\n",
       "      <td>The House of Memory (Pluto's Snitch #2)</td>\n",
       "      <td>60</td>\n",
       "      <td>4.33</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>242185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29074697</th>\n",
       "      <td>The Slaughtered Virgin of Zenopolis (Inspector...</td>\n",
       "      <td>23</td>\n",
       "      <td>3.49</td>\n",
       "      <td>BATHS, BANKS AND ROMAN INSURRECTION Detective ...</td>\n",
       "      <td>15104629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1902202</th>\n",
       "      <td>Dead in the Morning (Patrick Grant, #1)</td>\n",
       "      <td>8</td>\n",
       "      <td>3.30</td>\n",
       "      <td>Gerald breezily introduced his wife, Helen, to...</td>\n",
       "      <td>190988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9671977</th>\n",
       "      <td>Aristotele e i misteri di Eleusi</td>\n",
       "      <td>3</td>\n",
       "      <td>3.54</td>\n",
       "      <td>\"I misteri di Eleusi\" e il quinto romanzo di A...</td>\n",
       "      <td>337108</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      title  \\\n",
       "book_id                                                       \n",
       "6066814           Crowner Royal (Crowner John Mystery, #13)   \n",
       "33394837            The House of Memory (Pluto's Snitch #2)   \n",
       "29074697  The Slaughtered Virgin of Zenopolis (Inspector...   \n",
       "1902202             Dead in the Morning (Patrick Grant, #1)   \n",
       "9671977                    Aristotele e i misteri di Eleusi   \n",
       "\n",
       "          text_reviews_count  average_rating  \\\n",
       "book_id                                        \n",
       "6066814                   15            3.93   \n",
       "33394837                  60            4.33   \n",
       "29074697                  23            3.49   \n",
       "1902202                    8            3.30   \n",
       "9671977                    3            3.54   \n",
       "\n",
       "                                                description  author_id  \n",
       "book_id                                                                 \n",
       "6066814   London, 1196. At the command of Richard the Li...      37778  \n",
       "33394837                                               <NA>     242185  \n",
       "29074697  BATHS, BANKS AND ROMAN INSURRECTION Detective ...   15104629  \n",
       "1902202   Gerald breezily introduced his wife, Helen, to...     190988  \n",
       "9671977   \"I misteri di Eleusi\" e il quinto romanzo di A...     337108  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_books = pd.read_csv(os.path.join(DIR, csv_fn_books), dtype=book_dtype, low_memory=True)\n",
    "df_books.title = df_books.title.str.strip(\"b\\'\\\"\") \n",
    "df_books.set_index('book_id', inplace=True)\n",
    "df_books.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 219235 entries, 6066814 to 26168430\n",
      "Data columns (total 5 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   title               219235 non-null  string \n",
      " 1   text_reviews_count  219235 non-null  uint32 \n",
      " 2   average_rating      219235 non-null  float64\n",
      " 3   description         198488 non-null  string \n",
      " 4   author_id           219235 non-null  int64  \n",
      "dtypes: float64(1), int64(1), string(2), uint32(1)\n",
      "memory usage: 9.2 MB\n"
     ]
    }
   ],
   "source": [
    "df_books.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5e212a62bced17b4dbe41150e5bb9037</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>6392944</td>\n",
       "      <td>3</td>\n",
       "      <td>I haven't read a fun mystery book in a while a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2ede853b14dc4583f96cf5d120af636f</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>28684704</td>\n",
       "      <td>3</td>\n",
       "      <td>A fun, fast paced science fiction thriller. I ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8e4d61801907e591018bdc3442a9cf2b</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>32283133</td>\n",
       "      <td>0</td>\n",
       "      <td>http://www.telegraph.co.uk/culture/10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>022bb6daffa49adc27f6b20b6ebeb37d</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>17860739</td>\n",
       "      <td>4</td>\n",
       "      <td>An amazing and unique creation: JJ Abrams and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0e317947e1fd341f573192111bb2921d</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>8694005</td>\n",
       "      <td>3</td>\n",
       "      <td>The Name of the Rose is a thrilling Dan Brown-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           user_id   book_id  \\\n",
       "review_id                                                                      \n",
       "5e212a62bced17b4dbe41150e5bb9037  8842281e1d1347389f2ab93d60773d4d   6392944   \n",
       "2ede853b14dc4583f96cf5d120af636f  8842281e1d1347389f2ab93d60773d4d  28684704   \n",
       "8e4d61801907e591018bdc3442a9cf2b  8842281e1d1347389f2ab93d60773d4d  32283133   \n",
       "022bb6daffa49adc27f6b20b6ebeb37d  8842281e1d1347389f2ab93d60773d4d  17860739   \n",
       "0e317947e1fd341f573192111bb2921d  8842281e1d1347389f2ab93d60773d4d   8694005   \n",
       "\n",
       "                                  rating  \\\n",
       "review_id                                  \n",
       "5e212a62bced17b4dbe41150e5bb9037       3   \n",
       "2ede853b14dc4583f96cf5d120af636f       3   \n",
       "8e4d61801907e591018bdc3442a9cf2b       0   \n",
       "022bb6daffa49adc27f6b20b6ebeb37d       4   \n",
       "0e317947e1fd341f573192111bb2921d       3   \n",
       "\n",
       "                                                                        review_text  \n",
       "review_id                                                                            \n",
       "5e212a62bced17b4dbe41150e5bb9037  I haven't read a fun mystery book in a while a...  \n",
       "2ede853b14dc4583f96cf5d120af636f  A fun, fast paced science fiction thriller. I ...  \n",
       "8e4d61801907e591018bdc3442a9cf2b           http://www.telegraph.co.uk/culture/10...  \n",
       "022bb6daffa49adc27f6b20b6ebeb37d  An amazing and unique creation: JJ Abrams and ...  \n",
       "0e317947e1fd341f573192111bb2921d  The Name of the Rose is a thrilling Dan Brown-...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_reviews = pd.read_csv(os.path.join(DIR, csv_fn_reviews), dtype=review_dtype, low_memory=True)\n",
    "df_reviews.set_index('review_id', inplace=True)\n",
    "df_reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 735000 entries, 5e212a62bced17b4dbe41150e5bb9037 to a91f52d910b3988d02d9c72d4731de3c\n",
      "Data columns (total 4 columns):\n",
      " #   Column       Non-Null Count   Dtype \n",
      "---  ------       --------------   ----- \n",
      " 0   user_id      735000 non-null  object\n",
      " 1   book_id      735000 non-null  uint32\n",
      " 2   rating       735000 non-null  uint8 \n",
      " 3   review_text  734832 non-null  string\n",
      "dtypes: object(1), string(1), uint32(1), uint8(1)\n",
      "memory usage: 20.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_reviews.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TODO: Maybe the Data should be indexed/grouped by BookID, not ReviewID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_text</th>\n",
       "      <th>title</th>\n",
       "      <th>text_reviews_count</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>description</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5e212a62bced17b4dbe41150e5bb9037</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>6392944</td>\n",
       "      <td>3</td>\n",
       "      <td>I haven't read a fun mystery book in a while a...</td>\n",
       "      <td>The Murder on the Links (Hercule Poirot, #2)</td>\n",
       "      <td>42</td>\n",
       "      <td>3.8</td>\n",
       "      <td>On a French golf course, a millionaire is foun...</td>\n",
       "      <td>123715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ee9cf5e49795718b9bb086c1cde23116</th>\n",
       "      <td>24d87e891f6f60ae101c1de158cbe672</td>\n",
       "      <td>6392944</td>\n",
       "      <td>4</td>\n",
       "      <td>A fun, fast read! I just adore reading books w...</td>\n",
       "      <td>The Murder on the Links (Hercule Poirot, #2)</td>\n",
       "      <td>42</td>\n",
       "      <td>3.8</td>\n",
       "      <td>On a French golf course, a millionaire is foun...</td>\n",
       "      <td>123715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b5a6af59482018d0814c373723080adb</th>\n",
       "      <td>d083ade0fb4502508b1f3ce59bf1c7db</td>\n",
       "      <td>6392944</td>\n",
       "      <td>4</td>\n",
       "      <td>Full of twists and turns, way better than the ...</td>\n",
       "      <td>The Murder on the Links (Hercule Poirot, #2)</td>\n",
       "      <td>42</td>\n",
       "      <td>3.8</td>\n",
       "      <td>On a French golf course, a millionaire is foun...</td>\n",
       "      <td>123715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2ede853b14dc4583f96cf5d120af636f</th>\n",
       "      <td>8842281e1d1347389f2ab93d60773d4d</td>\n",
       "      <td>28684704</td>\n",
       "      <td>3</td>\n",
       "      <td>A fun, fast paced science fiction thriller. I ...</td>\n",
       "      <td>Dark Matter</td>\n",
       "      <td>1026</td>\n",
       "      <td>4.1</td>\n",
       "      <td>\"Are you happy with your life?\" Those are the ...</td>\n",
       "      <td>442240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31db9238ec11ca671a65a50643f952b7</th>\n",
       "      <td>01ec1a320ffded6b2dd47833f2c8e4fb</td>\n",
       "      <td>28684704</td>\n",
       "      <td>5</td>\n",
       "      <td>Mind-bending and completely unique, take a cha...</td>\n",
       "      <td>Dark Matter</td>\n",
       "      <td>1026</td>\n",
       "      <td>4.1</td>\n",
       "      <td>\"Are you happy with your life?\" Those are the ...</td>\n",
       "      <td>442240</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           user_id   book_id  \\\n",
       "review_id                                                                      \n",
       "5e212a62bced17b4dbe41150e5bb9037  8842281e1d1347389f2ab93d60773d4d   6392944   \n",
       "ee9cf5e49795718b9bb086c1cde23116  24d87e891f6f60ae101c1de158cbe672   6392944   \n",
       "b5a6af59482018d0814c373723080adb  d083ade0fb4502508b1f3ce59bf1c7db   6392944   \n",
       "2ede853b14dc4583f96cf5d120af636f  8842281e1d1347389f2ab93d60773d4d  28684704   \n",
       "31db9238ec11ca671a65a50643f952b7  01ec1a320ffded6b2dd47833f2c8e4fb  28684704   \n",
       "\n",
       "                                  rating  \\\n",
       "review_id                                  \n",
       "5e212a62bced17b4dbe41150e5bb9037       3   \n",
       "ee9cf5e49795718b9bb086c1cde23116       4   \n",
       "b5a6af59482018d0814c373723080adb       4   \n",
       "2ede853b14dc4583f96cf5d120af636f       3   \n",
       "31db9238ec11ca671a65a50643f952b7       5   \n",
       "\n",
       "                                                                        review_text  \\\n",
       "review_id                                                                             \n",
       "5e212a62bced17b4dbe41150e5bb9037  I haven't read a fun mystery book in a while a...   \n",
       "ee9cf5e49795718b9bb086c1cde23116  A fun, fast read! I just adore reading books w...   \n",
       "b5a6af59482018d0814c373723080adb  Full of twists and turns, way better than the ...   \n",
       "2ede853b14dc4583f96cf5d120af636f  A fun, fast paced science fiction thriller. I ...   \n",
       "31db9238ec11ca671a65a50643f952b7  Mind-bending and completely unique, take a cha...   \n",
       "\n",
       "                                                                         title  \\\n",
       "review_id                                                                        \n",
       "5e212a62bced17b4dbe41150e5bb9037  The Murder on the Links (Hercule Poirot, #2)   \n",
       "ee9cf5e49795718b9bb086c1cde23116  The Murder on the Links (Hercule Poirot, #2)   \n",
       "b5a6af59482018d0814c373723080adb  The Murder on the Links (Hercule Poirot, #2)   \n",
       "2ede853b14dc4583f96cf5d120af636f                                   Dark Matter   \n",
       "31db9238ec11ca671a65a50643f952b7                                   Dark Matter   \n",
       "\n",
       "                                  text_reviews_count  average_rating  \\\n",
       "review_id                                                              \n",
       "5e212a62bced17b4dbe41150e5bb9037                  42             3.8   \n",
       "ee9cf5e49795718b9bb086c1cde23116                  42             3.8   \n",
       "b5a6af59482018d0814c373723080adb                  42             3.8   \n",
       "2ede853b14dc4583f96cf5d120af636f                1026             4.1   \n",
       "31db9238ec11ca671a65a50643f952b7                1026             4.1   \n",
       "\n",
       "                                                                        description  \\\n",
       "review_id                                                                             \n",
       "5e212a62bced17b4dbe41150e5bb9037  On a French golf course, a millionaire is foun...   \n",
       "ee9cf5e49795718b9bb086c1cde23116  On a French golf course, a millionaire is foun...   \n",
       "b5a6af59482018d0814c373723080adb  On a French golf course, a millionaire is foun...   \n",
       "2ede853b14dc4583f96cf5d120af636f  \"Are you happy with your life?\" Those are the ...   \n",
       "31db9238ec11ca671a65a50643f952b7  \"Are you happy with your life?\" Those are the ...   \n",
       "\n",
       "                                  author_id  \n",
       "review_id                                    \n",
       "5e212a62bced17b4dbe41150e5bb9037     123715  \n",
       "ee9cf5e49795718b9bb086c1cde23116     123715  \n",
       "b5a6af59482018d0814c373723080adb     123715  \n",
       "2ede853b14dc4583f96cf5d120af636f     442240  \n",
       "31db9238ec11ca671a65a50643f952b7     442240  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_join = pd.merge(df_reviews, df_books, left_on=\"book_id\", right_index=True)\n",
    "df_join.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 735000 entries, 5e212a62bced17b4dbe41150e5bb9037 to a91f52d910b3988d02d9c72d4731de3c\n",
      "Data columns (total 9 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   user_id             735000 non-null  object \n",
      " 1   book_id             735000 non-null  uint32 \n",
      " 2   rating              735000 non-null  uint8  \n",
      " 3   review_text         734832 non-null  string \n",
      " 4   title               735000 non-null  string \n",
      " 5   text_reviews_count  735000 non-null  uint32 \n",
      " 6   average_rating      735000 non-null  float64\n",
      " 7   description         703758 non-null  string \n",
      " 8   author_id           735000 non-null  int64  \n",
      "dtypes: float64(1), int64(1), object(1), string(3), uint32(2), uint8(1)\n",
      "memory usage: 45.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df_join.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_text</th>\n",
       "      <th>title</th>\n",
       "      <th>text_reviews_count</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>description</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>84b45c5862e66c41fde9f1922fd946f7</th>\n",
       "      <td>f10d3a1771caeffebde9fd239547f067</td>\n",
       "      <td>17899948</td>\n",
       "      <td>5</td>\n",
       "      <td>I didn't know anything about the storyline and...</td>\n",
       "      <td>Rebecca</td>\n",
       "      <td>4127</td>\n",
       "      <td>4.21</td>\n",
       "      <td>Last night I dreamt I went to Manderley again ...</td>\n",
       "      <td>2001717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5ba79c8f80f076d432d1eba5ea5b1453</th>\n",
       "      <td>11f5af0f11e1b73863c7a887ef0f2d37</td>\n",
       "      <td>70897</td>\n",
       "      <td>0</td>\n",
       "      <td>This is probably one of those have-to-be-there...</td>\n",
       "      <td>The Secret History</td>\n",
       "      <td>862</td>\n",
       "      <td>4.08</td>\n",
       "      <td>Truly deserving of the accolade Modern Classic...</td>\n",
       "      <td>8719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10e8c869bb59daf5d2f3fae4ab364d9c</th>\n",
       "      <td>c8c04f0d14b8a9c482b8d979a41f963d</td>\n",
       "      <td>37304</td>\n",
       "      <td>4</td>\n",
       "      <td>Davenport meets doctor Weather Karkinnen at a ...</td>\n",
       "      <td>Winter Prey (Lucas Davenport, #5)</td>\n",
       "      <td>424</td>\n",
       "      <td>4.25</td>\n",
       "      <td>It is winter in the remote, dark Wisconsin woo...</td>\n",
       "      <td>4610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c0a1b5b9e12ccac1d0e3ce7844f9dfc2</th>\n",
       "      <td>4a1a45c12198b9ec99c57efa91e957ce</td>\n",
       "      <td>17802724</td>\n",
       "      <td>4</td>\n",
       "      <td>I won this book in a first reads giveaway. I w...</td>\n",
       "      <td>The Husband's Secret</td>\n",
       "      <td>18009</td>\n",
       "      <td>3.92</td>\n",
       "      <td>At the heart of The Husband's Secret is a lett...</td>\n",
       "      <td>322069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49fda46bc527b4b2a0fa0342bf574881</th>\n",
       "      <td>ac74ae567651e7502fb4408fa0da9840</td>\n",
       "      <td>23168811</td>\n",
       "      <td>4</td>\n",
       "      <td>The author is becoming at ease with his main c...</td>\n",
       "      <td>Even the Dead (Quirke #7)</td>\n",
       "      <td>161</td>\n",
       "      <td>3.77</td>\n",
       "      <td>A suspicious death, a pregnant woman suddenly ...</td>\n",
       "      <td>116405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d9b941dfcb7ff9601aa54f1083b0e3d3</th>\n",
       "      <td>65dac923d96ed9e0e3667dd9850d16be</td>\n",
       "      <td>25812109</td>\n",
       "      <td>4</td>\n",
       "      <td>4.5 stars   hell yes. everything but the roman...</td>\n",
       "      <td>The Female of the Species</td>\n",
       "      <td>2026</td>\n",
       "      <td>4.25</td>\n",
       "      <td>A contemporary YA novel that examines rape cul...</td>\n",
       "      <td>5351825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305a063bd1f0ce66b7ab3839971f6573</th>\n",
       "      <td>8f3e35dcd36b81c7161baa9c28d42b1d</td>\n",
       "      <td>13147906</td>\n",
       "      <td>1</td>\n",
       "      <td>Well, it took me forever to listen to this. I ...</td>\n",
       "      <td>Gone Girl</td>\n",
       "      <td>636</td>\n",
       "      <td>4.03</td>\n",
       "      <td>What are you thinking, Amy? The question I've ...</td>\n",
       "      <td>2383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8bf21fcb2c3122f1fb2b14545ffea48a</th>\n",
       "      <td>95e5bfa350481de0f8e073792a170219</td>\n",
       "      <td>215238</td>\n",
       "      <td>3</td>\n",
       "      <td>Certainly not his best book. author wrote a di...</td>\n",
       "      <td>Play Dead</td>\n",
       "      <td>804</td>\n",
       "      <td>3.74</td>\n",
       "      <td>Terrible secrets lead basketball star David Ba...</td>\n",
       "      <td>24689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5c01ae2e360087907e3d0a5a8698fb2d</th>\n",
       "      <td>c99afa5c1c37afb44d1f99aea34f6204</td>\n",
       "      <td>431</td>\n",
       "      <td>3</td>\n",
       "      <td>Ok, just finished the book and I still not sur...</td>\n",
       "      <td>The New York Trilogy</td>\n",
       "      <td>1601</td>\n",
       "      <td>3.92</td>\n",
       "      <td>Paul Auster's signature work, The New York Tri...</td>\n",
       "      <td>296961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5f843238746d321436891c3784e292ea</th>\n",
       "      <td>b0d0e10a1941aeb47ddbd465cbc9c11a</td>\n",
       "      <td>36007655</td>\n",
       "      <td>5</td>\n",
       "      <td>A new author for me and I have marked the firs...</td>\n",
       "      <td>Running out of Time</td>\n",
       "      <td>5</td>\n",
       "      <td>4.23</td>\n",
       "      <td>Restaurant manager Dodie O'Dell's themed food ...</td>\n",
       "      <td>6187623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9156643a50f674e7d0c16e1c31a346ca</th>\n",
       "      <td>4410e6a379ff27e208ee9f4cf45d37dd</td>\n",
       "      <td>608474</td>\n",
       "      <td>5</td>\n",
       "      <td>Read my review at: http://tims-reviews.blogspo...</td>\n",
       "      <td>The Sign of Four</td>\n",
       "      <td>2369</td>\n",
       "      <td>3.95</td>\n",
       "      <td>As a dense yellow fog swirls through the stree...</td>\n",
       "      <td>2448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ff1238f359608dcea6b8c651e0d0572d</th>\n",
       "      <td>c4d8b77e043b6bf78013433315b16df2</td>\n",
       "      <td>26269034</td>\n",
       "      <td>2</td>\n",
       "      <td>If your looking for supernatural this is not i...</td>\n",
       "      <td>Suicide Forest</td>\n",
       "      <td>20</td>\n",
       "      <td>3.65</td>\n",
       "      <td>Just outside of Tokyo lies Aokigahara, a vast ...</td>\n",
       "      <td>5327980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b4c936f216323086bac3d0d4218199d3</th>\n",
       "      <td>962c6a6d3cfeb0ef4f3d23f3ca7285be</td>\n",
       "      <td>17714283</td>\n",
       "      <td>5</td>\n",
       "      <td>Great read! I read this book in one day! I cou...</td>\n",
       "      <td>Critical Pursuit (Critical Pursuit, #1)</td>\n",
       "      <td>115</td>\n",
       "      <td>4.09</td>\n",
       "      <td>Officer Brinna Caruso has built a reputation a...</td>\n",
       "      <td>2998481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a147a6975dbfc84c086f1cb066af59b7</th>\n",
       "      <td>e488cdf79584866ee83ddfb6e4d4bec4</td>\n",
       "      <td>25630384</td>\n",
       "      <td>3</td>\n",
       "      <td>This is a fun, light read. It's very much like...</td>\n",
       "      <td>The Neon Lawyer</td>\n",
       "      <td>14</td>\n",
       "      <td>4.04</td>\n",
       "      <td>4 hours and 26 minutes With money and hope in ...</td>\n",
       "      <td>4711969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8df27e496f9f5c984c6ea0b67d9cee6c</th>\n",
       "      <td>2adf89eb93f2e2d181cced69aa8e312b</td>\n",
       "      <td>1516197</td>\n",
       "      <td>3</td>\n",
       "      <td>There were parts that were really funny but ot...</td>\n",
       "      <td>The Naked Detective (Key West, #8)</td>\n",
       "      <td>26</td>\n",
       "      <td>3.72</td>\n",
       "      <td>The gumshoe bit wasn't Peter Amsterdam's idea....</td>\n",
       "      <td>79588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c0d62f63a0e1e1d10164dac5ce3a368d</th>\n",
       "      <td>14c69742f5a1d4f70627077e193c523f</td>\n",
       "      <td>6587879</td>\n",
       "      <td>3</td>\n",
       "      <td>Fantastic read. Another Joe Hill classic. Love...</td>\n",
       "      <td>Horns</td>\n",
       "      <td>6034</td>\n",
       "      <td>3.92</td>\n",
       "      <td>Ignatius Perrish spent the night drunk and doi...</td>\n",
       "      <td>88506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b32a8bd14e0c1cd918cf33cc3a46258a</th>\n",
       "      <td>467a0068677fd407d92a571442190d86</td>\n",
       "      <td>35299166</td>\n",
       "      <td>4</td>\n",
       "      <td>Best day Ever by Kaira Rouda only took me a da...</td>\n",
       "      <td>Best Day Ever</td>\n",
       "      <td>23</td>\n",
       "      <td>3.81</td>\n",
       "      <td>A loving husband. The perfect killer? 'I wonde...</td>\n",
       "      <td>1105560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d9e7c5d25ba85b7af663caf812ca5cf2</th>\n",
       "      <td>4f2448c5f8c0cc62f91f38bea5eb48a4</td>\n",
       "      <td>16131077</td>\n",
       "      <td>4</td>\n",
       "      <td>I think what kicked this up to 4 stars for me ...</td>\n",
       "      <td>The Shining Girls</td>\n",
       "      <td>3769</td>\n",
       "      <td>3.49</td>\n",
       "      <td>The Girl Who Wouldn't Die Hunts the Killer Who...</td>\n",
       "      <td>426034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c912e2ae74b404a0966fa4f42e520991</th>\n",
       "      <td>7d5ee89439e572cc53643b5775ca742f</td>\n",
       "      <td>2167796</td>\n",
       "      <td>5</td>\n",
       "      <td>It's taken me two years to finally read Deanna...</td>\n",
       "      <td>Silent in the Grave (Lady Julia Grey, #1)</td>\n",
       "      <td>93</td>\n",
       "      <td>3.92</td>\n",
       "      <td>\"Let the wicked be ashamed, and let them be si...</td>\n",
       "      <td>156327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c1f3e8155428b63cd28b4c261b53585f</th>\n",
       "      <td>5b58dd727b90f611a739f6cbc4340b1f</td>\n",
       "      <td>16373187</td>\n",
       "      <td>0</td>\n",
       "      <td>Haiti Noir, Edited by Edwidge Danitcat, Narrat...</td>\n",
       "      <td>Haiti Noir</td>\n",
       "      <td>1</td>\n",
       "      <td>3.76</td>\n",
       "      <td>Haiti has a tragic history and continues to be...</td>\n",
       "      <td>3525</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           user_id   book_id  \\\n",
       "review_id                                                                      \n",
       "84b45c5862e66c41fde9f1922fd946f7  f10d3a1771caeffebde9fd239547f067  17899948   \n",
       "5ba79c8f80f076d432d1eba5ea5b1453  11f5af0f11e1b73863c7a887ef0f2d37     70897   \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c  c8c04f0d14b8a9c482b8d979a41f963d     37304   \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2  4a1a45c12198b9ec99c57efa91e957ce  17802724   \n",
       "49fda46bc527b4b2a0fa0342bf574881  ac74ae567651e7502fb4408fa0da9840  23168811   \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3  65dac923d96ed9e0e3667dd9850d16be  25812109   \n",
       "305a063bd1f0ce66b7ab3839971f6573  8f3e35dcd36b81c7161baa9c28d42b1d  13147906   \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a  95e5bfa350481de0f8e073792a170219    215238   \n",
       "5c01ae2e360087907e3d0a5a8698fb2d  c99afa5c1c37afb44d1f99aea34f6204       431   \n",
       "5f843238746d321436891c3784e292ea  b0d0e10a1941aeb47ddbd465cbc9c11a  36007655   \n",
       "9156643a50f674e7d0c16e1c31a346ca  4410e6a379ff27e208ee9f4cf45d37dd    608474   \n",
       "ff1238f359608dcea6b8c651e0d0572d  c4d8b77e043b6bf78013433315b16df2  26269034   \n",
       "b4c936f216323086bac3d0d4218199d3  962c6a6d3cfeb0ef4f3d23f3ca7285be  17714283   \n",
       "a147a6975dbfc84c086f1cb066af59b7  e488cdf79584866ee83ddfb6e4d4bec4  25630384   \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c  2adf89eb93f2e2d181cced69aa8e312b   1516197   \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d  14c69742f5a1d4f70627077e193c523f   6587879   \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a  467a0068677fd407d92a571442190d86  35299166   \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2  4f2448c5f8c0cc62f91f38bea5eb48a4  16131077   \n",
       "c912e2ae74b404a0966fa4f42e520991  7d5ee89439e572cc53643b5775ca742f   2167796   \n",
       "c1f3e8155428b63cd28b4c261b53585f  5b58dd727b90f611a739f6cbc4340b1f  16373187   \n",
       "\n",
       "                                  rating  \\\n",
       "review_id                                  \n",
       "84b45c5862e66c41fde9f1922fd946f7       5   \n",
       "5ba79c8f80f076d432d1eba5ea5b1453       0   \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c       4   \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2       4   \n",
       "49fda46bc527b4b2a0fa0342bf574881       4   \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3       4   \n",
       "305a063bd1f0ce66b7ab3839971f6573       1   \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a       3   \n",
       "5c01ae2e360087907e3d0a5a8698fb2d       3   \n",
       "5f843238746d321436891c3784e292ea       5   \n",
       "9156643a50f674e7d0c16e1c31a346ca       5   \n",
       "ff1238f359608dcea6b8c651e0d0572d       2   \n",
       "b4c936f216323086bac3d0d4218199d3       5   \n",
       "a147a6975dbfc84c086f1cb066af59b7       3   \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c       3   \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d       3   \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a       4   \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2       4   \n",
       "c912e2ae74b404a0966fa4f42e520991       5   \n",
       "c1f3e8155428b63cd28b4c261b53585f       0   \n",
       "\n",
       "                                                                        review_text  \\\n",
       "review_id                                                                             \n",
       "84b45c5862e66c41fde9f1922fd946f7  I didn't know anything about the storyline and...   \n",
       "5ba79c8f80f076d432d1eba5ea5b1453  This is probably one of those have-to-be-there...   \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c  Davenport meets doctor Weather Karkinnen at a ...   \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2  I won this book in a first reads giveaway. I w...   \n",
       "49fda46bc527b4b2a0fa0342bf574881  The author is becoming at ease with his main c...   \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3  4.5 stars   hell yes. everything but the roman...   \n",
       "305a063bd1f0ce66b7ab3839971f6573  Well, it took me forever to listen to this. I ...   \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a  Certainly not his best book. author wrote a di...   \n",
       "5c01ae2e360087907e3d0a5a8698fb2d  Ok, just finished the book and I still not sur...   \n",
       "5f843238746d321436891c3784e292ea  A new author for me and I have marked the firs...   \n",
       "9156643a50f674e7d0c16e1c31a346ca  Read my review at: http://tims-reviews.blogspo...   \n",
       "ff1238f359608dcea6b8c651e0d0572d  If your looking for supernatural this is not i...   \n",
       "b4c936f216323086bac3d0d4218199d3  Great read! I read this book in one day! I cou...   \n",
       "a147a6975dbfc84c086f1cb066af59b7  This is a fun, light read. It's very much like...   \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c  There were parts that were really funny but ot...   \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d  Fantastic read. Another Joe Hill classic. Love...   \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a  Best day Ever by Kaira Rouda only took me a da...   \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2  I think what kicked this up to 4 stars for me ...   \n",
       "c912e2ae74b404a0966fa4f42e520991  It's taken me two years to finally read Deanna...   \n",
       "c1f3e8155428b63cd28b4c261b53585f  Haiti Noir, Edited by Edwidge Danitcat, Narrat...   \n",
       "\n",
       "                                                                      title  \\\n",
       "review_id                                                                     \n",
       "84b45c5862e66c41fde9f1922fd946f7                                    Rebecca   \n",
       "5ba79c8f80f076d432d1eba5ea5b1453                         The Secret History   \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c          Winter Prey (Lucas Davenport, #5)   \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2                       The Husband's Secret   \n",
       "49fda46bc527b4b2a0fa0342bf574881                  Even the Dead (Quirke #7)   \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3                  The Female of the Species   \n",
       "305a063bd1f0ce66b7ab3839971f6573                                  Gone Girl   \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a                                  Play Dead   \n",
       "5c01ae2e360087907e3d0a5a8698fb2d                       The New York Trilogy   \n",
       "5f843238746d321436891c3784e292ea                        Running out of Time   \n",
       "9156643a50f674e7d0c16e1c31a346ca                           The Sign of Four   \n",
       "ff1238f359608dcea6b8c651e0d0572d                             Suicide Forest   \n",
       "b4c936f216323086bac3d0d4218199d3    Critical Pursuit (Critical Pursuit, #1)   \n",
       "a147a6975dbfc84c086f1cb066af59b7                            The Neon Lawyer   \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c         The Naked Detective (Key West, #8)   \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d                                      Horns   \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a                              Best Day Ever   \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2                          The Shining Girls   \n",
       "c912e2ae74b404a0966fa4f42e520991  Silent in the Grave (Lady Julia Grey, #1)   \n",
       "c1f3e8155428b63cd28b4c261b53585f                                 Haiti Noir   \n",
       "\n",
       "                                  text_reviews_count  average_rating  \\\n",
       "review_id                                                              \n",
       "84b45c5862e66c41fde9f1922fd946f7                4127            4.21   \n",
       "5ba79c8f80f076d432d1eba5ea5b1453                 862            4.08   \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c                 424            4.25   \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2               18009            3.92   \n",
       "49fda46bc527b4b2a0fa0342bf574881                 161            3.77   \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3                2026            4.25   \n",
       "305a063bd1f0ce66b7ab3839971f6573                 636            4.03   \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a                 804            3.74   \n",
       "5c01ae2e360087907e3d0a5a8698fb2d                1601            3.92   \n",
       "5f843238746d321436891c3784e292ea                   5            4.23   \n",
       "9156643a50f674e7d0c16e1c31a346ca                2369            3.95   \n",
       "ff1238f359608dcea6b8c651e0d0572d                  20            3.65   \n",
       "b4c936f216323086bac3d0d4218199d3                 115            4.09   \n",
       "a147a6975dbfc84c086f1cb066af59b7                  14            4.04   \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c                  26            3.72   \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d                6034            3.92   \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a                  23            3.81   \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2                3769            3.49   \n",
       "c912e2ae74b404a0966fa4f42e520991                  93            3.92   \n",
       "c1f3e8155428b63cd28b4c261b53585f                   1            3.76   \n",
       "\n",
       "                                                                        description  \\\n",
       "review_id                                                                             \n",
       "84b45c5862e66c41fde9f1922fd946f7  Last night I dreamt I went to Manderley again ...   \n",
       "5ba79c8f80f076d432d1eba5ea5b1453  Truly deserving of the accolade Modern Classic...   \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c  It is winter in the remote, dark Wisconsin woo...   \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2  At the heart of The Husband's Secret is a lett...   \n",
       "49fda46bc527b4b2a0fa0342bf574881  A suspicious death, a pregnant woman suddenly ...   \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3  A contemporary YA novel that examines rape cul...   \n",
       "305a063bd1f0ce66b7ab3839971f6573  What are you thinking, Amy? The question I've ...   \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a  Terrible secrets lead basketball star David Ba...   \n",
       "5c01ae2e360087907e3d0a5a8698fb2d  Paul Auster's signature work, The New York Tri...   \n",
       "5f843238746d321436891c3784e292ea  Restaurant manager Dodie O'Dell's themed food ...   \n",
       "9156643a50f674e7d0c16e1c31a346ca  As a dense yellow fog swirls through the stree...   \n",
       "ff1238f359608dcea6b8c651e0d0572d  Just outside of Tokyo lies Aokigahara, a vast ...   \n",
       "b4c936f216323086bac3d0d4218199d3  Officer Brinna Caruso has built a reputation a...   \n",
       "a147a6975dbfc84c086f1cb066af59b7  4 hours and 26 minutes With money and hope in ...   \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c  The gumshoe bit wasn't Peter Amsterdam's idea....   \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d  Ignatius Perrish spent the night drunk and doi...   \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a  A loving husband. The perfect killer? 'I wonde...   \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2  The Girl Who Wouldn't Die Hunts the Killer Who...   \n",
       "c912e2ae74b404a0966fa4f42e520991  \"Let the wicked be ashamed, and let them be si...   \n",
       "c1f3e8155428b63cd28b4c261b53585f  Haiti has a tragic history and continues to be...   \n",
       "\n",
       "                                  author_id  \n",
       "review_id                                    \n",
       "84b45c5862e66c41fde9f1922fd946f7    2001717  \n",
       "5ba79c8f80f076d432d1eba5ea5b1453       8719  \n",
       "10e8c869bb59daf5d2f3fae4ab364d9c       4610  \n",
       "c0a1b5b9e12ccac1d0e3ce7844f9dfc2     322069  \n",
       "49fda46bc527b4b2a0fa0342bf574881     116405  \n",
       "d9b941dfcb7ff9601aa54f1083b0e3d3    5351825  \n",
       "305a063bd1f0ce66b7ab3839971f6573       2383  \n",
       "8bf21fcb2c3122f1fb2b14545ffea48a      24689  \n",
       "5c01ae2e360087907e3d0a5a8698fb2d     296961  \n",
       "5f843238746d321436891c3784e292ea    6187623  \n",
       "9156643a50f674e7d0c16e1c31a346ca       2448  \n",
       "ff1238f359608dcea6b8c651e0d0572d    5327980  \n",
       "b4c936f216323086bac3d0d4218199d3    2998481  \n",
       "a147a6975dbfc84c086f1cb066af59b7    4711969  \n",
       "8df27e496f9f5c984c6ea0b67d9cee6c      79588  \n",
       "c0d62f63a0e1e1d10164dac5ce3a368d      88506  \n",
       "b32a8bd14e0c1cd918cf33cc3a46258a    1105560  \n",
       "d9e7c5d25ba85b7af663caf812ca5cf2     426034  \n",
       "c912e2ae74b404a0966fa4f42e520991     156327  \n",
       "c1f3e8155428b63cd28b4c261b53585f       3525  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_join.sample(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "before:\n",
    "\n",
    "Index: 22726 entries, 0e317947e1fd341f573192111bb2921d to 921812c9edc173c6d12e000723b9e667\n",
    "\n",
    "after:\n",
    "\n",
    "Index: 735000 entries, 5e212a62bced17b4dbe41150e5bb9037 to a91f52d910b3988d02d9c72d4731de3c\n",
    "(= number of reviews)\n",
    "\n",
    "after dropNA():\n",
    "703594 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 735000 entries, 5e212a62bced17b4dbe41150e5bb9037 to a91f52d910b3988d02d9c72d4731de3c\n",
      "Data columns (total 9 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   user_id             735000 non-null  object \n",
      " 1   book_id             735000 non-null  uint32 \n",
      " 2   rating              735000 non-null  uint8  \n",
      " 3   review_text         734832 non-null  string \n",
      " 4   title               735000 non-null  string \n",
      " 5   text_reviews_count  735000 non-null  uint32 \n",
      " 6   average_rating      735000 non-null  float64\n",
      " 7   description         703758 non-null  string \n",
      " 8   author_id           735000 non-null  int64  \n",
      "dtypes: float64(1), int64(1), object(1), string(3), uint32(2), uint8(1)\n",
      "memory usage: 45.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df_join.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_join.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>text_reviews_count</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7.035940e+05</td>\n",
       "      <td>703594.000000</td>\n",
       "      <td>703594.000000</td>\n",
       "      <td>703594.000000</td>\n",
       "      <td>7.035940e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.317835e+07</td>\n",
       "      <td>3.672068</td>\n",
       "      <td>2735.980951</td>\n",
       "      <td>3.869114</td>\n",
       "      <td>2.033713e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.054083e+07</td>\n",
       "      <td>1.193176</td>\n",
       "      <td>9487.515139</td>\n",
       "      <td>0.278711</td>\n",
       "      <td>3.437501e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.640000e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.400000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.744612e+06</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>3.720000</td>\n",
       "      <td>1.447300e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.312973e+07</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>216.000000</td>\n",
       "      <td>3.890000</td>\n",
       "      <td>1.563270e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.232339e+07</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>4.050000</td>\n",
       "      <td>3.057577e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.646717e+07</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>78438.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.733059e+07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            book_id         rating  text_reviews_count  average_rating  \\\n",
       "count  7.035940e+05  703594.000000       703594.000000   703594.000000   \n",
       "mean   1.317835e+07       3.672068         2735.980951        3.869114   \n",
       "std    1.054083e+07       1.193176         9487.515139        0.278711   \n",
       "min    1.640000e+02       0.000000            0.000000        0.000000   \n",
       "25%    1.744612e+06       3.000000           38.000000        3.720000   \n",
       "50%    1.312973e+07       4.000000          216.000000        3.890000   \n",
       "75%    2.232339e+07       5.000000         1148.000000        4.050000   \n",
       "max    3.646717e+07       5.000000        78438.000000        5.000000   \n",
       "\n",
       "          author_id  \n",
       "count  7.035940e+05  \n",
       "mean   2.033713e+06  \n",
       "std    3.437501e+06  \n",
       "min    1.400000e+01  \n",
       "25%    1.447300e+04  \n",
       "50%    1.563270e+05  \n",
       "75%    3.057577e+06  \n",
       "max    1.733059e+07  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_join.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = df_join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_text</th>\n",
       "      <th>title</th>\n",
       "      <th>text_reviews_count</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>description</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>544b25ef2e5bca5ed23de6e2ce334633</th>\n",
       "      <td>d991d4797cca0fff1cb9a1d6638eb47a</td>\n",
       "      <td>18770398</td>\n",
       "      <td>4</td>\n",
       "      <td>At one point it seemed I would be reading the ...</td>\n",
       "      <td>Night Film</td>\n",
       "      <td>2497</td>\n",
       "      <td>3.78</td>\n",
       "      <td>Brilliant, haunting, breathtakingly suspensefu...</td>\n",
       "      <td>2362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14f2a43d1c9379f4fe1f0f0ee7ce9beb</th>\n",
       "      <td>34da92c91d6b2a2caddf03b8b714acb9</td>\n",
       "      <td>12875355</td>\n",
       "      <td>2</td>\n",
       "      <td>As a huge Austen fan I have been wanting to re...</td>\n",
       "      <td>Death Comes to Pemberley</td>\n",
       "      <td>6916</td>\n",
       "      <td>3.21</td>\n",
       "      <td>The world is classic Jane Austen. The mystery ...</td>\n",
       "      <td>344522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1ce7b0f009b45367e206bdb7fca2ecc4</th>\n",
       "      <td>1c28c471aa84a1ca3a8882a637d7a76d</td>\n",
       "      <td>34032606</td>\n",
       "      <td>5</td>\n",
       "      <td>This latest entry in the Laurel McKay series h...</td>\n",
       "      <td>Dying for a Diamond (Laurel McKay Mysteries, #6)</td>\n",
       "      <td>14</td>\n",
       "      <td>4.53</td>\n",
       "      <td>Warm breezes, tropical seas, a handsome new hu...</td>\n",
       "      <td>4115315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>07f6064388bcbc76c324f89bc54f896b</th>\n",
       "      <td>c8d8cc9c1058c0caef590bf32001256c</td>\n",
       "      <td>25079805</td>\n",
       "      <td>4</td>\n",
       "      <td>After XO I was a little hesitant to jump into ...</td>\n",
       "      <td>Solitude Creek (Kathryn Dance, #4)</td>\n",
       "      <td>14</td>\n",
       "      <td>3.85</td>\n",
       "      <td>Jeffery Deaver, \"the master of manipulation\" (...</td>\n",
       "      <td>1612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53dc886b45ef2cffe422ca9fe619cfb7</th>\n",
       "      <td>7543b5a00e6dac232dd2a1c0ada849b9</td>\n",
       "      <td>12079574</td>\n",
       "      <td>4</td>\n",
       "      <td>Great start to a new series! I was apprehensiv...</td>\n",
       "      <td>Stay At Home Dead (Stay At Home Dad Mystery #1)</td>\n",
       "      <td>107</td>\n",
       "      <td>3.88</td>\n",
       "      <td>When Deuce Winters, a stay-at-home dad in slee...</td>\n",
       "      <td>6429000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                           user_id   book_id  \\\n",
       "review_id                                                                      \n",
       "544b25ef2e5bca5ed23de6e2ce334633  d991d4797cca0fff1cb9a1d6638eb47a  18770398   \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb  34da92c91d6b2a2caddf03b8b714acb9  12875355   \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4  1c28c471aa84a1ca3a8882a637d7a76d  34032606   \n",
       "07f6064388bcbc76c324f89bc54f896b  c8d8cc9c1058c0caef590bf32001256c  25079805   \n",
       "53dc886b45ef2cffe422ca9fe619cfb7  7543b5a00e6dac232dd2a1c0ada849b9  12079574   \n",
       "\n",
       "                                  rating  \\\n",
       "review_id                                  \n",
       "544b25ef2e5bca5ed23de6e2ce334633       4   \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb       2   \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4       5   \n",
       "07f6064388bcbc76c324f89bc54f896b       4   \n",
       "53dc886b45ef2cffe422ca9fe619cfb7       4   \n",
       "\n",
       "                                                                        review_text  \\\n",
       "review_id                                                                             \n",
       "544b25ef2e5bca5ed23de6e2ce334633  At one point it seemed I would be reading the ...   \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb  As a huge Austen fan I have been wanting to re...   \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4  This latest entry in the Laurel McKay series h...   \n",
       "07f6064388bcbc76c324f89bc54f896b  After XO I was a little hesitant to jump into ...   \n",
       "53dc886b45ef2cffe422ca9fe619cfb7  Great start to a new series! I was apprehensiv...   \n",
       "\n",
       "                                                                             title  \\\n",
       "review_id                                                                            \n",
       "544b25ef2e5bca5ed23de6e2ce334633                                        Night Film   \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb                          Death Comes to Pemberley   \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4  Dying for a Diamond (Laurel McKay Mysteries, #6)   \n",
       "07f6064388bcbc76c324f89bc54f896b                Solitude Creek (Kathryn Dance, #4)   \n",
       "53dc886b45ef2cffe422ca9fe619cfb7   Stay At Home Dead (Stay At Home Dad Mystery #1)   \n",
       "\n",
       "                                  text_reviews_count  average_rating  \\\n",
       "review_id                                                              \n",
       "544b25ef2e5bca5ed23de6e2ce334633                2497            3.78   \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb                6916            3.21   \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4                  14            4.53   \n",
       "07f6064388bcbc76c324f89bc54f896b                  14            3.85   \n",
       "53dc886b45ef2cffe422ca9fe619cfb7                 107            3.88   \n",
       "\n",
       "                                                                        description  \\\n",
       "review_id                                                                             \n",
       "544b25ef2e5bca5ed23de6e2ce334633  Brilliant, haunting, breathtakingly suspensefu...   \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb  The world is classic Jane Austen. The mystery ...   \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4  Warm breezes, tropical seas, a handsome new hu...   \n",
       "07f6064388bcbc76c324f89bc54f896b  Jeffery Deaver, \"the master of manipulation\" (...   \n",
       "53dc886b45ef2cffe422ca9fe619cfb7  When Deuce Winters, a stay-at-home dad in slee...   \n",
       "\n",
       "                                  author_id  \n",
       "review_id                                    \n",
       "544b25ef2e5bca5ed23de6e2ce334633       2362  \n",
       "14f2a43d1c9379f4fe1f0f0ee7ce9beb     344522  \n",
       "1ce7b0f009b45367e206bdb7fca2ecc4    4115315  \n",
       "07f6064388bcbc76c324f89bc54f896b       1612  \n",
       "53dc886b45ef2cffe422ca9fe619cfb7    6429000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 703594 entries, 5e212a62bced17b4dbe41150e5bb9037 to a91f52d910b3988d02d9c72d4731de3c\n",
      "Data columns (total 9 columns):\n",
      " #   Column              Non-Null Count   Dtype  \n",
      "---  ------              --------------   -----  \n",
      " 0   user_id             703594 non-null  object \n",
      " 1   book_id             703594 non-null  uint32 \n",
      " 2   rating              703594 non-null  uint8  \n",
      " 3   review_text         703594 non-null  string \n",
      " 4   title               703594 non-null  string \n",
      " 5   text_reviews_count  703594 non-null  uint32 \n",
      " 6   average_rating      703594 non-null  float64\n",
      " 7   description         703594 non-null  string \n",
      " 8   author_id           703594 non-null  int64  \n",
      "dtypes: float64(1), int64(1), object(1), string(3), uint32(2), uint8(1)\n",
      "memory usage: 43.6+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_pickle(os.path.join(DIR, \"joined_df.pkl\"))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "48bc9e598fcbf8a93b8170aeed7972c61fdd84b24baf2eff9e2683f5bf1d2fdd"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit ('ds-project': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}