01-data-preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation\n",
    "\n",
    "This notebook is a pre-work step that prepares data to be used in the project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ast import literal_eval\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Movie IDs\n",
    "\n",
    "To combine MovieLens with TMBD we need the movie ids for both datasets. We will use the MovieLens ID as the main identifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tmdb_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8844</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15602</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31357</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11862</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id\n",
       "tmdb_id    \n",
       "862       1\n",
       "8844      2\n",
       "15602     3\n",
       "31357     4\n",
       "11862     5"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links = pd.read_csv('data/raw/links.csv', usecols=['movieId','tmdbId'])\n",
    "links.rename(columns={'movieId':'id','tmdbId':'tmdb_id'}, inplace=True)\n",
    "links['tmdb_id'] = links['tmdb_id'].astype('Int64')\n",
    "links.set_index('tmdb_id', inplace=True)\n",
    "links.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Movie Cast & Crew\n",
    "\n",
    "We need to put cast&crew info in a relational format. The dataset presents it in a JSON format.\n",
    "\n",
    "### Cast\n",
    "\n",
    "Extract actor ID for each movie. We will consider only those actors present in at least 10 movies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cast</th>\n",
       "      <th>crew</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[{'cast_id': 3, 'character': 'Taisto Olavi Kas...</td>\n",
       "      <td>[{'credit_id': '52fe420dc3a36847f800001f', 'de...</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[{'cast_id': 5, 'character': 'Nikander', 'cred...</td>\n",
       "      <td>[{'credit_id': '52fe420dc3a36847f8000077', 'de...</td>\n",
       "      <td>61724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[{'cast_id': 42, 'character': 'Ted the Bellhop...</td>\n",
       "      <td>[{'credit_id': '52fe420dc3a36847f800011b', 'de...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[{'cast_id': 7, 'character': 'Frank Wyatt', 'c...</td>\n",
       "      <td>[{'credit_id': '52fe420dc3a36847f800023d', 'de...</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[{'cast_id': 3, 'character': 'Luke Skywalker',...</td>\n",
       "      <td>[{'credit_id': '52fe420dc3a36847f8000437', 'de...</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                cast  \\\n",
       "0  [{'cast_id': 3, 'character': 'Taisto Olavi Kas...   \n",
       "1  [{'cast_id': 5, 'character': 'Nikander', 'cred...   \n",
       "2  [{'cast_id': 42, 'character': 'Ted the Bellhop...   \n",
       "3  [{'cast_id': 7, 'character': 'Frank Wyatt', 'c...   \n",
       "4  [{'cast_id': 3, 'character': 'Luke Skywalker',...   \n",
       "\n",
       "                                                crew     id  \n",
       "0  [{'credit_id': '52fe420dc3a36847f800001f', 'de...   4470  \n",
       "1  [{'credit_id': '52fe420dc3a36847f8000077', 'de...  61724  \n",
       "2  [{'credit_id': '52fe420dc3a36847f800011b', 'de...     18  \n",
       "3  [{'credit_id': '52fe420dc3a36847f800023d', 'de...    479  \n",
       "4  [{'credit_id': '52fe420dc3a36847f8000437', 'de...    260  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "credits = pd.read_csv('data/raw/credits.csv')\n",
    "credits.rename(columns={'id':'tmdb_id'}, inplace=True)\n",
    "credits['tmdb_id'] = credits['tmdb_id'].astype('Int64')\n",
    "credits.set_index('tmdb_id', inplace=True)\n",
    "credits = credits.join(links, how='inner').reset_index(drop=True)\n",
    "credits.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cast_id</th>\n",
       "      <th>character</th>\n",
       "      <th>credit_id</th>\n",
       "      <th>gender</th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>order</th>\n",
       "      <th>profile_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>Taisto Olavi Kasurinen</td>\n",
       "      <td>52fe420dc3a36847f8000029</td>\n",
       "      <td>0</td>\n",
       "      <td>54768</td>\n",
       "      <td>Turo Pajala</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Irmeli Katariina Pihlaja</td>\n",
       "      <td>52fe420dc3a36847f800002d</td>\n",
       "      <td>0</td>\n",
       "      <td>54769</td>\n",
       "      <td>Susanna Haavisto</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>Mikkonen</td>\n",
       "      <td>52fe420dc3a36847f8000031</td>\n",
       "      <td>2</td>\n",
       "      <td>4826</td>\n",
       "      <td>Matti Pellonpää</td>\n",
       "      <td>2</td>\n",
       "      <td>/7WuLvkuWphUAtW6QQwtF3WrwUKE.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Riku</td>\n",
       "      <td>52fe420dc3a36847f8000035</td>\n",
       "      <td>0</td>\n",
       "      <td>54770</td>\n",
       "      <td>Eetu Hilkamo</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cast_id                 character                 credit_id  gender     id  \\\n",
       "0        3    Taisto Olavi Kasurinen  52fe420dc3a36847f8000029       0  54768   \n",
       "1        4  Irmeli Katariina Pihlaja  52fe420dc3a36847f800002d       0  54769   \n",
       "2        5                  Mikkonen  52fe420dc3a36847f8000031       2   4826   \n",
       "3        6                      Riku  52fe420dc3a36847f8000035       0  54770   \n",
       "\n",
       "               name  order                      profile_path  \n",
       "0       Turo Pajala      0                              None  \n",
       "1  Susanna Haavisto      1                              None  \n",
       "2   Matti Pellonpää      2  /7WuLvkuWphUAtW6QQwtF3WrwUKE.jpg  \n",
       "3      Eetu Hilkamo      3                              None  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(literal_eval(credits.cast[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 34s, sys: 328 ms, total: 2min 34s\n",
      "Wall time: 2min 34s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actor_id</th>\n",
       "      <th>actor_name</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>54768</td>\n",
       "      <td>Turo Pajala</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>54769</td>\n",
       "      <td>Susanna Haavisto</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4826</td>\n",
       "      <td>Matti Pellonpää</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54770</td>\n",
       "      <td>Eetu Hilkamo</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4826</td>\n",
       "      <td>Matti Pellonpää</td>\n",
       "      <td>61724</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   actor_id        actor_name     id\n",
       "0     54768       Turo Pajala   4470\n",
       "1     54769  Susanna Haavisto   4470\n",
       "2      4826   Matti Pellonpää   4470\n",
       "3     54770      Eetu Hilkamo   4470\n",
       "4      4826   Matti Pellonpää  61724"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "cast = []\n",
    "for i in credits.id:\n",
    "    df_dict = literal_eval(credits.loc[credits['id'] == i].cast.values[0])\n",
    "    if len(df_dict) > 0:\n",
    "        df = pd.DataFrame.from_dict(df_dict)[['id','name']]\n",
    "        df.columns = ['actor_id','actor_name']\n",
    "        df['id'] = i\n",
    "        cast.append(df)\n",
    "\n",
    "cast = pd.concat(cast, ignore_index=True)\n",
    "cast.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAD4CAYAAADcpoD8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVqklEQVR4nO3df6xk5X3f8ffHQDC2gfBjoXSXdLHZJgGUYFivkUhTO8iAsRKghWatKqwqmk0plmw1lQJOVJCtlaCqTYNS0+Cy4kcdA8Y/2MqmZA1WrEgEuFDCT1M2gZj1rmCTRUBSA1n87R/z3DB7mXt37t07Z5a575c0mjPfOc+5z8Nc7YfnnOeeSVUhSVJX3jXuDkiSlhaDR5LUKYNHktQpg0eS1CmDR5LUqf3H3YF9xZFHHlkrV64cdzck6R3loYce+uuqWjafNgZPs3LlSqampsbdDUl6R0nyV/Nt46k2SVKnDB5JUqcMHklSpwweSVKnDB5JUqcMHklSpwweSVKnDB5JUqcMHklSp7xzwSJZedm3B9afu+oTHfdEkvZtzngkSZ0yeCRJnTJ4JEmdMngkSZ0yeCRJnTJ4JEmdMngkSZ0yeCRJnTJ4JEmdMngkSZ0yeCRJnTJ4JEmdMngkSZ0yeCRJnTJ4JEmdMngkSZ0yeCRJnTJ4JEmdGlnwJDk2yfeSPJXkiSSfbvUrk/woySPtcU5fm8uTbEnydJKz+uqnJnmsvXdtkrT6gUlua/X7k6zsa7MuyTPtsW5U45Qkzc/+Izz2LuC3q+rhJAcDDyXZ3N67pqr+S//OSU4A1gInAv8Y+G6Sf1pVbwLXAeuBPwO+A5wN3AVcDLxUVccnWQtcDfx6ksOBK4DVQLWfvamqXhrheCVJQxjZjKeqtlfVw237VeApYPkcTc4Fbq2q16vqWWALsCbJMcAhVXVfVRVwM3BeX5ub2vYdwBltNnQWsLmqdraw2UwvrCRJY9bJNZ52CuyDwP2t9KkkjybZmOSwVlsOPN/XbGurLW/bM+u7tamqXcDLwBFzHEuSNGYjD54k7wO+Dnymql6hd9rsA8DJwHbgC9O7Dmhec9QX2qa/b+uTTCWZ2rFjx1zDkCQtkpEGT5ID6IXOV6rqGwBV9UJVvVlVPwG+DKxpu28Fju1rvgLY1uorBtR3a5Nkf+BQYOccx9pNVV1fVauravWyZcv2ZqiSpCGNclVbgBuAp6rqi331Y/p2Ox94vG1vAta2lWrHAauAB6pqO/BqktPaMS8C7uxrM71i7QLg3nYd6G7gzCSHtVN5Z7aaJGnMRrmq7XTgN4DHkjzSap8FPpnkZHqnvp4Dfgugqp5IcjvwJL0VcZe2FW0AlwA3AgfRW812V6vfANySZAu9mc7adqydST4PPNj2+1xV7RzJKCVJ8zKy4KmqP2XwtZbvzNFmA7BhQH0KOGlA/TXgwlmOtRHYOGx/JUnd8M4FkqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROGTySpE4ZPJKkThk8kqROjSx4khyb5HtJnkryRJJPt/rhSTYneaY9H9bX5vIkW5I8neSsvvqpSR5r712bJK1+YJLbWv3+JCv72qxrP+OZJOtGNU5J0vyMcsazC/jtqvp54DTg0iQnAJcB91TVKuCe9pr23lrgROBs4EtJ9mvHug5YD6xqj7Nb/WLgpao6HrgGuLod63DgCuDDwBrgiv6AkySNz8iCp6q2V9XDbftV4ClgOXAucFPb7SbgvLZ9LnBrVb1eVc8CW4A1SY4BDqmq+6qqgJtntJk+1h3AGW02dBawuap2VtVLwGbeCitJ0hh1co2nnQL7IHA/cHRVbYdeOAFHtd2WA8/3Ndvaasvb9sz6bm2qahfwMnDEHMea2a/1SaaSTO3YsWMvRihJGtbIgyfJ+4CvA5+pqlfm2nVAreaoL7TNW4Wq66tqdVWtXrZs2RxdkyQtlpEGT5ID6IXOV6rqG638Qjt9Rnt+sdW3Asf2NV8BbGv1FQPqu7VJsj9wKLBzjmNJksZslKvaAtwAPFVVX+x7axMwvcpsHXBnX31tW6l2HL1FBA+003GvJjmtHfOiGW2mj3UBcG+7DnQ3cGaSw9qigjNbTZI0ZvuP8NinA78BPJbkkVb7LHAVcHuSi4EfAhcCVNUTSW4HnqS3Iu7SqnqztbsEuBE4CLirPaAXbLck2UJvprO2HWtnks8DD7b9PldVO0c0TknSPIwseKrqTxl8rQXgjFnabAA2DKhPAScNqL9GC64B720ENg7bX0lSN7xzgSSpUwaPJKlTBo8kqVMGjySpUwaPJKlTBo8kqVMGjySpUwaPJKlTBo8kqVMGjySpU0MFT5K33a5GkqSFGHbG89+TPJDk3yf56VF2SJI02YYKnqr6JeBf0/uOm6kkf5TkYyPtmSRpIg19jaeqngF+D/gd4J8D1yb5QZJ/MarOSZImz7DXeH4hyTXAU8CvAL9aVT/ftq8ZYf8kSRNm2O/j+QPgy8Bnq+rH08Wq2pbk90bSM0nSRBo2eM4Bfjz9jaBJ3gW8u6r+X1XdMrLeSZImzrDXeL5L72unp72n1SRJmpdhg+fdVfW30y/a9ntG0yVJ0iQbNnj+Lskp0y+SnAr8eI79JUkaaNhrPJ8BvpZkW3t9DPDrI+mRJGmiDRU8VfVgkp8DfhYI8IOq+vuR9kySNJGGnfEAfAhY2dp8MAlVdfNIeiVJmlhDBU+SW4APAI8Ab7ZyAQaPJGlehp3xrAZOqKoaZWckSZNv2FVtjwP/aJQdkSQtDcPOeI4EnkzyAPD6dLGqfm0kvZIkTaxhg+fKUXZCkrR0DPt9PH8CPAcc0LYfBB6eq02SjUleTPJ4X+3KJD9K8kh7nNP33uVJtiR5OslZffVTkzzW3rs2SVr9wCS3tfr9SVb2tVmX5Jn2WDfcfwpJUheG/VqE3wTuAP6wlZYD39pDsxuBswfUr6mqk9vjO+34JwBrgRNbmy8l2a/tfx2wHljVHtPHvBh4qaqOp/fVDFe3Yx0OXAF8GFgDXJHksGHGKUkavWEXF1wKnA68Av/wpXBHzdWgqr4P7Bzy+OcCt1bV61X1LLAFWJPkGOCQqrqvrai7GTivr81NbfsO4Iw2GzoL2FxVO6vqJWAzgwNQkjQGwwbP61X1xvSLJPvT+zuehfhUkkfbqbjpmchy4Pm+fba22vK2PbO+W5uq2gW8DBwxx7HeJsn6JFNJpnbs2LHA4UiS5mPY4PmTJJ8FDkryMeBrwP9awM+7jt4fop4MbAe+0OoZsG/NUV9om92LVddX1eqqWr1s2bI5ui1JWizDBs9lwA7gMeC3gO8A8/7m0ap6oarerKqf0PtG0zXtra3AsX27rgC2tfqKAfXd2rQZ2KH0Tu3NdixJ0j5g2FVtP6mqL1fVhVV1Qdue96m2ds1m2vn0/jAVYBOwtq1UO47eIoIHqmo78GqS09r1m4uAO/vaTK9YuwC4t/XpbuDMJIe1U3lntpokaR8w7L3anmXA6aqqev8cbb4KfAQ4MslWeivNPpLk5Has5+jNnqiqJ5LcDjwJ7AIunf6abeASeivkDgLuag+AG4BbkmyhN9NZ2461M8nn6S35BvhcVQ27yEGSNGLzuVfbtHcDFwKHz9Wgqj45oHzDHPtvADYMqE8BJw2ov9b6MehYG4GNc/VPkjQew55q+5u+x4+q6r8CvzLarkmSJtGwp9pO6Xv5LnozoINH0iNJ0kQb9lTbF/q2d9G7PvOvFr03kqSJN+xXX3901B2RJC0Nw55q+w9zvV9VX1yc7kiSJt18VrV9iN7fzgD8KvB9dr81jSRJezSfL4I7papehd7XGwBfq6p/O6qOSZIm07C3zPkZ4I2+128AKxe9N5KkiTfsjOcW4IEk36R314Hz6X1FgSRJ8zLsqrYNSe4C/lkr/Zuq+j+j65YkaVINe6oN4D3AK1X1+8DWdjNPSZLmZdivvr4C+B3g8lY6APifo+qUJGlyDTvjOR/4NeDvAKpqG94yR5K0AMMGzxvtu24KIMl7R9clSdIkGzZ4bk/yh8BPJ/lN4Lv0vkFUkqR52eOqtvbNn7cBPwe8Avws8J+qavOI+yZJmkB7DJ6qqiTfqqpTAcNGkrRXhj3V9mdJPjTSnkiSloRh71zwUeDfJXmO3sq20JsM/cKoOiZJmkxzBk+Sn6mqHwIf76g/kqQJt6cZz7fo3ZX6r5J8var+ZQd9kiRNsD1d40nf9vtH2RFJ0tKwp+CpWbYlSVqQPZ1q+8Ukr9Cb+RzUtuGtxQWHjLR3kqSJM2fwVNV+XXVEkrQ0zOdrESRJ2msGjySpUwaPJKlTIwueJBuTvJjk8b7a4Uk2J3mmPR/W997lSbYkeTrJWX31U5M81t67tt20lCQHJrmt1e9PsrKvzbr2M55Jsm5UY5Qkzd8oZzw3AmfPqF0G3FNVq4B72muSnACsBU5sbb6UZHphw3XAemBVe0wf82Lgpao6HrgGuLod63DgCuDDwBrgiv6AkySN18iCp6q+D+ycUT4XuKlt3wSc11e/taper6pngS3AmiTHAIdU1X3ti+huntFm+lh3AGe02dBZwOaq2llVL9G7o/bMAJQkjUnX13iOrqrtAO35qFZfDjzft9/WVlvetmfWd2tTVbuAl4Ej5jjW2yRZn2QqydSOHTv2YliSpGHtK4sLMqBWc9QX2mb3YtX1VbW6qlYvW7ZsqI5KkvZO18HzQjt9Rnt+sdW3Asf27bcC2NbqKwbUd2uTZH/gUHqn9mY7liRpH9B18GwCpleZrQPu7KuvbSvVjqO3iOCBdjru1SSntes3F81oM32sC4B723Wgu4EzkxzWFhWc2WqSpH3AsF8EN29Jvgp8BDgyyVZ6K82uAm5PcjHwQ+BCgKp6IsntwJPALuDSqnqzHeoSeivkDgLuag+AG4BbkmyhN9NZ2461M8nngQfbfp+rqpmLHCRJYzKy4KmqT87y1hmz7L8B2DCgPgWcNKD+Gi24Bry3Edg4dGclSZ3ZVxYXSJKWCINHktQpg0eS1CmDR5LUKYNHktQpg0eS1CmDR5LUKYNHktSpkf0BqXpWXvbtgfXnrvpExz2RpH2DMx5JUqcMHklSpwweSVKnDB5JUqcMHklSpwweSVKnDB5JUqcMHklSpwweSVKnDB5JUqcMHklSpwweSVKnDB5JUqcMHklSpwweSVKnDB5JUqcMHklSpwweSVKnDB5JUqfGEjxJnkvyWJJHkky12uFJNid5pj0f1rf/5Um2JHk6yVl99VPbcbYkuTZJWv3AJLe1+v1JVnY+SEnSQOOc8Xy0qk6uqtXt9WXAPVW1CrinvSbJCcBa4ETgbOBLSfZrba4D1gOr2uPsVr8YeKmqjgeuAa7uYDySpCHsS6fazgVuats3Aef11W+tqter6llgC7AmyTHAIVV1X1UVcPOMNtPHugM4Y3o2JEkar3EFTwF/nOShJOtb7eiq2g7Qno9q9eXA831tt7ba8rY9s75bm6raBbwMHDGzE0nWJ5lKMrVjx45FGZgkaW77j+nnnl5V25IcBWxO8oM59h00U6k56nO12b1QdT1wPcDq1avf9v4orbzs2wPrz131iS67IUmdG8uMp6q2tecXgW8Ca4AX2ukz2vOLbfetwLF9zVcA21p9xYD6bm2S7A8cCuwcxVgkSfPTefAkeW+Sg6e3gTOBx4FNwLq22zrgzra9CVjbVqodR28RwQPtdNyrSU5r128umtFm+lgXAPe260CSpDEbx6m2o4Fvtmv9+wN/VFX/O8mDwO1JLgZ+CFwIUFVPJLkdeBLYBVxaVW+2Y10C3AgcBNzVHgA3ALck2UJvprO2i4FJkvas8+Cpqr8EfnFA/W+AM2ZpswHYMKA+BZw0oP4aLbgkSfuWfWk5tSRpCTB4JEmdMngkSZ0yeCRJnTJ4JEmdMngkSZ0a1y1zNAtvpSNp0jnjkSR1yuCRJHXK4JEkdcrgkSR1yuCRJHXK4JEkdcrgkSR1yuCRJHXKPyB9h/APSyVNCoPnHc5AkvRO46k2SVKnnPFMKGdCkvZVzngkSZ0yeCRJnTJ4JEmd8hrPEuO1H0njZvAIMJAkdcdTbZKkTjnj0YI5S5K0EAaP5jRbuCykjYEkCTzVJknq2ETPeJKcDfw+sB/wP6rqqjF3aUlzJiQJJjh4kuwH/DfgY8BW4MEkm6rqyfH2TDMt5HTefBhs0r5lYoMHWANsqaq/BEhyK3AuYPAsMYsVbAsJMGd50ttNcvAsB57ve70V+HD/DknWA+vby79N8vQCfs6RwF8vqIeTYcmMP1e/rbTgsQ841jvRkvnsB3Dsb/kn8z3AJAdPBtRqtxdV1wPX79UPSaaqavXeHOOdbCmPfymPHZb2+B373o19kle1bQWO7Xu9Atg2pr5IkppJDp4HgVVJjkvyU8BaYNOY+yRJS97Enmqrql1JPgXcTW859caqemIEP2qvTtVNgKU8/qU8dlja43fseyFVtee9JElaJJN8qk2StA8yeCRJnTJ49kKSs5M8nWRLksvG3Z9RS/JckseSPJJkqtUOT7I5yTPt+bBx93OxJNmY5MUkj/fVZh1vksvb78LTSc4aT68XxyxjvzLJj9rn/0iSc/rem6SxH5vke0meSvJEkk+3+lL57Gcb/+J9/lXlYwEPegsW/gJ4P/BTwJ8DJ4y7XyMe83PAkTNq/xm4rG1fBlw97n4u4nh/GTgFeHxP4wVOaL8DBwLHtd+N/cY9hkUe+5XAfxyw76SN/RjglLZ9MPB/2xiXymc/2/gX7fN3xrNw/3BLnqp6A5i+Jc9Scy5wU9u+CThvfF1ZXFX1fWDnjPJs4z0XuLWqXq+qZ4Et9H5H3pFmGftsJm3s26vq4bb9KvAUvTuhLJXPfrbxz2be4zd4Fm7QLXnm+nAmQQF/nOShdrshgKOrajv0fmGBo8bWu27MNt6l8vvwqSSPtlNx06eaJnbsSVYCHwTuZwl+9jPGD4v0+Rs8C7fHW/JMoNOr6hTg48ClSX553B3ahyyF34frgA8AJwPbgS+0+kSOPcn7gK8Dn6mqV+badUBtEse/aJ+/wbNwS+6WPFW1rT2/CHyT3nT6hSTHALTnF8fXw07MNt6J/32oqheq6s2q+gnwZd46nTJxY09yAL1/dL9SVd9o5SXz2Q8a/2J+/gbPwi2pW/IkeW+Sg6e3gTOBx+mNeV3bbR1w53h62JnZxrsJWJvkwCTHAauAB8bQv5GZ/ke3OZ/e5w8TNvYkAW4AnqqqL/a9tSQ++9nGv6if/7hXULyTH8A59FZ8/AXwu+Puz4jH+n56K1f+HHhierzAEcA9wDPt+fBx93URx/xVeqcU/p7e/9VdPNd4gd9tvwtPAx8fd/9HMPZbgMeAR9s/NsdM6Nh/id6pokeBR9rjnCX02c82/kX7/L1ljiSpU55qkyR1yuCRJHXK4JEkdcrgkSR1yuCRJHXK4JEkdcrgkSR16v8DKDMTpv3/87YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cast['num_movies'] = cast.groupby('actor_id')['actor_id'].transform('count')\n",
    "cast['num_movies'].plot.hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cast.loc[cast.num_movies >= 10].to_csv('data/movie_actor.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Crew\n",
    "\n",
    "Extract ID for movie director, writer and producer. Again, we will consider only those present in at least 10 movies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>credit_id</th>\n",
       "      <th>department</th>\n",
       "      <th>gender</th>\n",
       "      <th>id</th>\n",
       "      <th>job</th>\n",
       "      <th>name</th>\n",
       "      <th>profile_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>52fe420dc3a36847f800001f</td>\n",
       "      <td>Writing</td>\n",
       "      <td>0</td>\n",
       "      <td>16767</td>\n",
       "      <td>Screenplay</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>/8nQcTzKUmRh6MPprd1n6iOauYPf.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>52fe420dc3a36847f8000025</td>\n",
       "      <td>Directing</td>\n",
       "      <td>0</td>\n",
       "      <td>16767</td>\n",
       "      <td>Director</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>/8nQcTzKUmRh6MPprd1n6iOauYPf.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52fe420dc3a36847f800003b</td>\n",
       "      <td>Camera</td>\n",
       "      <td>2</td>\n",
       "      <td>16769</td>\n",
       "      <td>Director of Photography</td>\n",
       "      <td>Timo Salminen</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>52fe420dc3a36847f8000041</td>\n",
       "      <td>Editing</td>\n",
       "      <td>0</td>\n",
       "      <td>54766</td>\n",
       "      <td>Editor</td>\n",
       "      <td>Raija Talvio</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>52fe420dc3a36847f8000047</td>\n",
       "      <td>Art</td>\n",
       "      <td>2</td>\n",
       "      <td>53836</td>\n",
       "      <td>Production Design</td>\n",
       "      <td>Risto Karhula</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>52fe420dc3a36847f800004d</td>\n",
       "      <td>Costume &amp; Make-Up</td>\n",
       "      <td>0</td>\n",
       "      <td>54771</td>\n",
       "      <td>Costume Design</td>\n",
       "      <td>Tuula Hilkamo</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  credit_id         department  gender     id  \\\n",
       "0  52fe420dc3a36847f800001f            Writing       0  16767   \n",
       "1  52fe420dc3a36847f8000025          Directing       0  16767   \n",
       "2  52fe420dc3a36847f800003b             Camera       2  16769   \n",
       "3  52fe420dc3a36847f8000041            Editing       0  54766   \n",
       "4  52fe420dc3a36847f8000047                Art       2  53836   \n",
       "5  52fe420dc3a36847f800004d  Costume & Make-Up       0  54771   \n",
       "\n",
       "                       job            name                      profile_path  \n",
       "0               Screenplay  Aki Kaurismäki  /8nQcTzKUmRh6MPprd1n6iOauYPf.jpg  \n",
       "1                 Director  Aki Kaurismäki  /8nQcTzKUmRh6MPprd1n6iOauYPf.jpg  \n",
       "2  Director of Photography   Timo Salminen                              None  \n",
       "3                   Editor    Raija Talvio                              None  \n",
       "4        Production Design   Risto Karhula                              None  \n",
       "5           Costume Design   Tuula Hilkamo                              None  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(literal_eval(credits.crew[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 32s, sys: 200 ms, total: 2min 33s\n",
      "Wall time: 2min 33s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>person_id</th>\n",
       "      <th>job</th>\n",
       "      <th>person_name</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16767</td>\n",
       "      <td>Screenplay</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16767</td>\n",
       "      <td>Director</td>\n",
       "      <td>Aki Kaurismäki</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16769</td>\n",
       "      <td>Director of Photography</td>\n",
       "      <td>Timo Salminen</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54766</td>\n",
       "      <td>Editor</td>\n",
       "      <td>Raija Talvio</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>53836</td>\n",
       "      <td>Production Design</td>\n",
       "      <td>Risto Karhula</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   person_id                      job     person_name    id\n",
       "0      16767               Screenplay  Aki Kaurismäki  4470\n",
       "1      16767                 Director  Aki Kaurismäki  4470\n",
       "2      16769  Director of Photography   Timo Salminen  4470\n",
       "3      54766                   Editor    Raija Talvio  4470\n",
       "4      53836        Production Design   Risto Karhula  4470"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "crew = []\n",
    "for i in credits.id:\n",
    "    df_dict = literal_eval(credits.loc[credits['id'] == i].crew.values[0])\n",
    "    if len(df_dict) > 0:\n",
    "        df = pd.DataFrame.from_dict(df_dict)[['id','job','name']]\n",
    "        df.columns = ['person_id','job','person_name']\n",
    "        df['id'] = i\n",
    "        crew.append(df)\n",
    "\n",
    "crew = pd.concat(crew, ignore_index=True)\n",
    "crew.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Director                          49108\n",
       "Producer                          43630\n",
       "Writer                            30483\n",
       "Screenplay                        25187\n",
       "Editor                            23854\n",
       "                                  ...  \n",
       "Character Modelling Supervisor        1\n",
       "Dramaturgie                           1\n",
       "CG Engineer                           1\n",
       "Battle Motion Coordinator             1\n",
       "Underwater Gaffer                     1\n",
       "Name: job, Length: 456, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crew['job'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Frequency'>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAD4CAYAAADGmmByAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWb0lEQVR4nO3df7DddX3n8edLUEQryI/AZhNsoGS0wFSFlGXHjruatUTcNtjF3ezsloyTbbYs7ujUnW2wna3+kR3YmUplt7BFcQ1UCylWydbFLYbazs5Q4sWiEJDlKhTSZEkUBHRKMPjeP87ntieXc29O8s25N+fm+Zg5c77f9/l+vvfz8cvk5ef7OT9SVUiSdKheMd8dkCSNN4NEktSJQSJJ6sQgkSR1YpBIkjo5dr47MNdOPfXUWrZs2Xx3Q5LGyn333ffdqlo06LWjLkiWLVvGxMTEfHdDksZKkr+a6TVvbUmSOjFIJEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHVikEiSOjnqPtnexbINXxpYf/zq98xxTyTpyOGMRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUyUiDJMnrk9ye5FtJHk7yD5OcnOSuJI+255P6jr8qyWSSR5Jc3Fe/IMkD7bXrkqTVj0tyW6vfm2TZKMcjSXq5Uc9IPgF8uareBLwZeBjYAGytquXA1rZPknOANcC5wCrg+iTHtPPcAKwHlrfHqlZfBzxTVWcD1wLXjHg8kqRpRhYkSU4A3g7cBFBVL1bV94HVwKZ22Cbg0ra9Gri1qvZW1WPAJHBhksXACVV1T1UVcPO0NlPnuh1YOTVbkSTNjVHOSM4C9gD/I8lfJvlUktcCp1fVLoD2fFo7fgnwZF/7Ha22pG1Pr+/Xpqr2Ac8Cp4xmOJKkQUYZJMcC5wM3VNVbgR/SbmPNYNBMomapz9Zm/xMn65NMJJnYs2fP7L2WJB2UUQbJDmBHVd3b9m+nFyxPtdtVtOfdfcef0dd+KbCz1ZcOqO/XJsmxwInA09M7UlU3VtWKqlqxaNGiwzA0SdKUkQVJVf0/4Mkkb2yllcBDwBZgbautBe5o21uANe2dWGfSW1Tf1m5/PZ/korb+cfm0NlPnugy4u62jSJLmyKh/2OrfA59N8irgO8D76YXX5iTrgCeA9wFU1fYkm+mFzT7gyqp6qZ3nCuAzwPHAne0BvYX8W5JM0puJrBnxeCRJ04w0SKrqfmDFgJdWznD8RmDjgPoEcN6A+gu0IJIkzQ8/2S5J6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjoxSCRJnRgkkqRODBJJUicGiSSpE4NEktTJSIMkyeNJHkhyf5KJVjs5yV1JHm3PJ/Udf1WSySSPJLm4r35BO89kkuuSpNWPS3Jbq9+bZNkoxyNJerm5mJG8o6reUlUr2v4GYGtVLQe2tn2SnAOsAc4FVgHXJzmmtbkBWA8sb49Vrb4OeKaqzgauBa6Zg/FIkvrMx62t1cCmtr0JuLSvfmtV7a2qx4BJ4MIki4ETquqeqirg5mltps51O7ByarYiSZobow6SAv4kyX1J1rfa6VW1C6A9n9bqS4An+9ruaLUlbXt6fb82VbUPeBY4ZXonkqxPMpFkYs+ePYdlYJKknmNHfP63VdXOJKcBdyX51izHDppJ1Cz12drsX6i6EbgRYMWKFS97XZJ06EY6I6mqne15N/AF4ELgqXa7iva8ux2+Azijr/lSYGerLx1Q369NkmOBE4GnRzEWSdJgIwuSJK9N8rqpbeDngQeBLcDadtha4I62vQVY096JdSa9RfVt7fbX80kuausfl09rM3Wuy4C72zqKJGmOjPLW1unAF9ra97HA56rqy0m+BmxOsg54AngfQFVtT7IZeAjYB1xZVS+1c10BfAY4HrizPQBuAm5JMklvJrJmhOORJA0wsiCpqu8Abx5Q/x6wcoY2G4GNA+oTwHkD6i/QgkiSND/8ZLskqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkToYKkiTnHeofSHJMkr9M8sdt/+QkdyV5tD2f1HfsVUkmkzyS5OK++gVJHmivXZckrX5cktta/d4kyw61n5KkQzPsjOS/J9mW5N8lef1B/o0PAg/37W8AtlbVcmBr2yfJOcAa4FxgFXB9kmNamxuA9cDy9ljV6uuAZ6rqbOBa4JqD7JskqaOhgqSqfg74V8AZwESSzyV514HaJVkKvAf4VF95NbCpbW8CLu2r31pVe6vqMWASuDDJYuCEqrqnqgq4eVqbqXPdDqycmq1IkubG0GskVfUo8JvArwP/CLguybeS/NIszX4H+I/Aj/tqp1fVrnbOXcBprb4EeLLvuB2ttqRtT6/v16aq9gHPAqdM70SS9Ukmkkzs2bPnwIOVJA1t2DWSn0lyLb1bVO8EfqGqfrptXztDm38K7K6q+4bsy6CZRM1Sn63N/oWqG6tqRVWtWLRo0ZDdkSQN49ghj/tvwCeBj1TV30wVq2pnkt+coc3bgF9McgnwauCEJL8PPJVkcVXtaretdrfjd9C7dTZlKbCz1ZcOqPe32ZHkWOBE4OkhxyRJOgyGvbV1CfC5qRBJ8ookrwGoqlsGNaiqq6pqaVUto7eIfndV/WtgC7C2HbYWuKNtbwHWtHdinUlvUX1bu/31fJKL2vrH5dPaTJ3rsvY3XjYjkSSNzrBB8hXg+L7917TaobgaeFeSR4F3tX2qajuwGXgI+DJwZVW91NpcQW/BfhL4NnBnq98EnJJkEvg12jvAJElzZ9hbW6+uqh9M7VTVD6ZmJMOoqq8CX23b3wNWznDcRmDjgPoE8LLPslTVC8D7hu2HJOnwG3ZG8sMk50/tJLkA+JtZjpckHSWGnZF8CPjDJFOL3IuBfzGSHkmSxspQQVJVX0vyJuCN9N5y+62q+tFIeyZJGgvDzkgAfhZY1tq8NQlVdfNIeiVJGhtDBUmSW4CfAu4Hpt5JNfV1JZKko9iwM5IVwDl+RkOSNN2w79p6EPh7o+yIJGk8DTsjORV4KMk2YO9Usap+cSS9kiSNjWGD5KOj7IQkaXwN+/bfP0vyk8DyqvpK+1T7MQdqJ0la+Ib9GvlfoffDUb/XSkuAL46oT5KkMTLsYvuV9L4W/jn42x+5Om3WFpKko8KwQbK3ql6c2mm//eFbgSVJQwfJnyX5CHB8+632PwT+5+i6JUkaF8MGyQZgD/AA8G+B/0Xv99slSUe5Yd+19WN6P7X7ydF2R5I0bob9rq3HGLAmUlVnHfYeSZLGysF819aUV9P7VcKTD393JEnjZqg1kqr6Xt/jr6vqd4B3jrZrkqRxMOytrfP7dl9Bb4byupH0SJI0Voa9tfXbfdv7gMeBf37YeyNJGjvDvmvrHaPuiCRpPA17a+vXZnu9qj5+eLojSRo3w34gcQVwBb0va1wC/CpwDr11koFrJUlenWRbkm8k2Z7kY61+cpK7kjzank/qa3NVkskkjyS5uK9+QZIH2mvXJUmrH5fktla/N8myQ/jfQJLUwbBBcipwflV9uKo+DFwALK2qj1XVx2Zosxd4Z1W9GXgLsCrJRfQ+Jb+1qpYDW9s+Sc4B1gDnAquA65NMfVX9DcB6YHl7rGr1dcAzVXU2cC1wzZDjkSQdJsMGyRuAF/v2XwSWzdagen7Qdl/ZHgWsBja1+ibg0ra9Gri1qvZW1WPAJHBhksXACVV1T/vN+JuntZk61+3AyqnZiiRpbgz7rq1bgG1JvkAvDN5L7x/0WbUZxX3A2cDvVtW9SU6vql0AVbUrydTX0S8B/qKv+Y5W+1Hbnl6favNkO9e+JM8CpwDfndaP9fRmNLzhDW8YcsiSpGEM+4HEjcD7gWeA7wPvr6r/PES7l6rqLcBSerOL82Y5fNBMomapz9Zmej9urKoVVbVi0aJFB+i1JOlgDHtrC+A1wHNV9QlgR5Izh21YVd8HvkpvbeOpdruK9ry7HbYDOKOv2VJgZ6svHVDfr037jZQTgacPYkySpI6G/and3wJ+HbiqlV4J/P4B2ixK8vq2fTzwT4BvAVuAte2wtcAdbXsLsKa9E+tMeovq29ptsOeTXNTWPy6f1mbqXJcBd7d1FEnSHBl2jeS9wFuBrwNU1c4kB/qKlMXAprZO8gpgc1X9cZJ7gM1J1gFP0PsCSKpqe5LNwEP0Pj1/ZVW91M51BfAZ4HjgzvYAuAm4JckkvZnImiHHI0k6TIYNkherqpIUQJLXHqhBVX2TXvhMr38PWDlDm43AxgH1CeBl6ytV9QItiCRJ82PYNZLNSX4PeH2SXwG+gj9yJUliiBlJW5e4DXgT8BzwRuA/VdVdI+6bJGkMHDBI2i2tL1bVBYDhIUnaz7C3tv4iyc+OtCeSpLE07GL7O4BfTfI48EN6HwSsqvqZUXVMkjQeZg2SJG+oqieAd89RfyRJY+ZAM5Iv0vvW379K8vmq+mdz0CdJ0hg50BpJ/3dZnTXKjkiSxtOBgqRm2JYkCTjwra03J3mO3szk+LYNf7fYfsJIeydJOuLNGiRVdcxsr0uSdDBfIy9J0ssYJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqZORBUmSM5L8aZKHk2xP8sFWPznJXUkebc8n9bW5KslkkkeSXNxXvyDJA+2165Kk1Y9Lclur35tk2ajGI0kabJQzkn3Ah6vqp4GLgCuTnANsALZW1XJga9unvbYGOBdYBVyfZOrbh28A1gPL22NVq68Dnqmqs4FrgWtGOB5J0gAjC5Kq2lVVX2/bzwMPA0uA1cCmdtgm4NK2vRq4tar2VtVjwCRwYZLFwAlVdU9VFXDztDZT57odWDk1W5EkzY05WSNpt5zeCtwLnF5Vu6AXNsBp7bAlwJN9zXa02pK2Pb2+X5uq2gc8C5wy4O+vTzKRZGLPnj2HaVSSJJiDIEnyE8DngQ9V1XOzHTqgVrPUZ2uzf6HqxqpaUVUrFi1adKAuS5IOwkiDJMkr6YXIZ6vqj1r5qXa7iva8u9V3AGf0NV8K7Gz1pQPq+7VJcixwIvD04R+JJGkmB/rN9kPW1ipuAh6uqo/3vbQFWAtc3Z7v6Kt/LsnHgb9Pb1F9W1W9lOT5JBfRuzV2OfBfp53rHuAy4O62jjKnlm340sD641e/Z457Iklzb2RBArwN+GXggST3t9pH6AXI5iTrgCeA9wFU1fYkm4GH6L3j68qqeqm1uwL4DHA8cGd7QC+obkkySW8msmaE45EkDTCyIKmq/8PgNQyAlTO02QhsHFCfAM4bUH+BFkSSpPnhJ9slSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTgwSSVInBokkqRODRJLUiUEiSerEIJEkdWKQSJI6MUgkSZ0YJJKkTkb5m+1HvWUbvjTja49f/Z457IkkjY4zEklSJwaJJKkTg0SS1IlBIknqZGRBkuTTSXYnebCvdnKSu5I82p5P6nvtqiSTSR5JcnFf/YIkD7TXrkuSVj8uyW2tfm+SZaMaiyRpZqOckXwGWDWttgHYWlXLga1tnyTnAGuAc1ub65Mc09rcAKwHlrfH1DnXAc9U1dnAtcA1IxuJJGlGIwuSqvpz4Olp5dXApra9Cbi0r35rVe2tqseASeDCJIuBE6rqnqoq4OZpbabOdTuwcmq2IkmaO3O9RnJ6Ve0CaM+ntfoS4Mm+43a02pK2Pb2+X5uq2gc8C5wysp5LkgY6UhbbB80kapb6bG1efvJkfZKJJBN79uw5xC5KkgaZ6yB5qt2uoj3vbvUdwBl9xy0Fdrb60gH1/dokORY4kZffSgOgqm6sqhVVtWLRokWHaSiSJJj7INkCrG3ba4E7+upr2juxzqS3qL6t3f56PslFbf3j8mltps51GXB3W0eRJM2hkX3XVpI/AP4xcGqSHcBvAVcDm5OsA54A3gdQVduTbAYeAvYBV1bVS+1UV9B7B9jxwJ3tAXATcEuSSXozkTWjGoskaWYjC5Kq+pczvLRyhuM3AhsH1CeA8wbUX6AFkSRp/hwpi+2SpDFlkEiSOjFIJEmdGCSSpE4MEklSJ/7U7jyZ6Wd4/QleSePGGYkkqRODRJLUiUEiSerEIJEkdeJi+xHGRXhJ48YZiSSpE4NEktSJQSJJ6sQgkSR1YpBIkjrxXVtjwndzSTpSOSORJHXijGTMOVORNN+ckUiSOjFIJEmdeGtrgfKWl6S5YpAcZQwYSYebQSLAgJF06MY+SJKsAj4BHAN8qqqunucuLSgzBcxMDB7p6DPWQZLkGOB3gXcBO4CvJdlSVQ/Nb8+OXocys3E2JI23sQ4S4EJgsqq+A5DkVmA1YJAcYQ52ZnMobQ42eA42wEZ9/GwMWx3JUlXz3YdDluQyYFVV/Zu2/8vAP6iqD0w7bj2wvu2+EXjkEP7cqcB3O3T3SOf4xt9CH6Pjm18/WVWLBr0w7jOSDKi9LBmr6kbgxk5/KJmoqhVdznEkc3zjb6GP0fEducb9A4k7gDP69pcCO+epL5J0VBr3IPkasDzJmUleBawBtsxznyTpqDLWt7aqal+SDwD/m97bfz9dVdtH9Oc63RobA45v/C30MTq+I9RYL7ZLkubfuN/akiTNM4NEktSJQTKEJKuSPJJkMsmG+e7P4ZDk8SQPJLk/yUSrnZzkriSPtueT5rufw0ry6SS7kzzYV5txPEmuatfzkSQXz0+vhzfD+D6a5K/bNbw/ySV9r43b+M5I8qdJHk6yPckHW31BXMNZxrcwrmFV+ZjlQW8R/9vAWcCrgG8A58x3vw7DuB4HTp1W+y/Ahra9Abhmvvt5EON5O3A+8OCBxgOc067jccCZ7foeM99jOITxfRT4DwOOHcfxLQbOb9uvA/5vG8eCuIazjG9BXENnJAf2t1/DUlUvAlNfw7IQrQY2te1NwKXz15WDU1V/Djw9rTzTeFYDt1bV3qp6DJikd52PWDOMbybjOL5dVfX1tv088DCwhAVyDWcZ30zGanwGyYEtAZ7s29/B7P8BjIsC/iTJfe0rZABOr6pd0PsPHzht3np3eMw0noV0TT+Q5Jvt1tfUbZ+xHl+SZcBbgXtZgNdw2vhgAVxDg+TAhvoaljH0tqo6H3g3cGWSt893h+bQQrmmNwA/BbwF2AX8dquP7fiS/ATweeBDVfXcbIcOqB3xYxwwvgVxDQ2SA1uQX8NSVTvb827gC/SmzU8lWQzQnnfPXw8Pi5nGsyCuaVU9VVUvVdWPgU/yd7c+xnJ8SV5J7x/Zz1bVH7XygrmGg8a3UK6hQXJgC+5rWJK8NsnrpraBnwcepDeute2wtcAd89PDw2am8WwB1iQ5LsmZwHJg2zz0r5Opf2Cb99K7hjCG40sS4Cbg4ar6eN9LC+IazjS+BXMN53u1fxwewCX03mXxbeA35rs/h2E8Z9F7R8g3gO1TYwJOAbYCj7bnk+e7rwcxpj+gd2vgR/T+39y62cYD/Ea7no8A757v/h/i+G4BHgC+Se8fnsVjPL6fo3fr5pvA/e1xyUK5hrOMb0FcQ78iRZLUibe2JEmdGCSSpE4MEklSJwaJJKkTg0SS1IlBIknqxCCRJHXy/wFfJvyXOKxMaAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "jobs = ['director','producer','writer']\n",
    "crew['num_movies'] = crew.groupby('person_id')['person_id'].transform('count')\n",
    "crew.loc[crew['job'].str.lower().isin(jobs), 'num_movies'].plot.hist(bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "crew.loc[crew['job'].str.lower().isin(jobs) & (crew['num_movies'] >= 10)].to_csv('data/movie_producer.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Movie Genres\n",
    "\n",
    "Extract genres for each movie from the TMDB data in a relational format. Moreover, select other text fields that will be used as content features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genres</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[{'id': 80, 'name': 'Crime'}, {'id': 35, 'name...</td>\n",
       "      <td>Four Rooms</td>\n",
       "      <td>It's Ted the Bellhop's first night on the job....</td>\n",
       "      <td>Twelve outrageous guests. Four scandalous requ...</td>\n",
       "      <td>Four Rooms</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[{'id': 28, 'name': 'Action'}, {'id': 53, 'nam...</td>\n",
       "      <td>Judgment Night</td>\n",
       "      <td>While racing to a boxing match, Frank, Mike, J...</td>\n",
       "      <td>Don't move. Don't whisper. Don't even breathe.</td>\n",
       "      <td>Judgment Night</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[{'id': 12, 'name': 'Adventure'}, {'id': 28, '...</td>\n",
       "      <td>Star Wars</td>\n",
       "      <td>Princess Leia is captured and held hostage by ...</td>\n",
       "      <td>A long time ago in a galaxy far, far away...</td>\n",
       "      <td>Star Wars</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[{'id': 16, 'name': 'Animation'}, {'id': 10751...</td>\n",
       "      <td>Finding Nemo</td>\n",
       "      <td>Nemo, an adventurous young clownfish, is unexp...</td>\n",
       "      <td>There are 3.7 trillion fish in the ocean. They...</td>\n",
       "      <td>Finding Nemo</td>\n",
       "      <td>6377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[{'id': 35, 'name': 'Comedy'}, {'id': 18, 'nam...</td>\n",
       "      <td>Forrest Gump</td>\n",
       "      <td>A man with a low IQ has accomplished great thi...</td>\n",
       "      <td>The world will never be the same, once you've ...</td>\n",
       "      <td>Forrest Gump</td>\n",
       "      <td>356</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              genres  original_title  \\\n",
       "0  [{'id': 80, 'name': 'Crime'}, {'id': 35, 'name...      Four Rooms   \n",
       "1  [{'id': 28, 'name': 'Action'}, {'id': 53, 'nam...  Judgment Night   \n",
       "2  [{'id': 12, 'name': 'Adventure'}, {'id': 28, '...       Star Wars   \n",
       "3  [{'id': 16, 'name': 'Animation'}, {'id': 10751...    Finding Nemo   \n",
       "4  [{'id': 35, 'name': 'Comedy'}, {'id': 18, 'nam...    Forrest Gump   \n",
       "\n",
       "                                            overview  \\\n",
       "0  It's Ted the Bellhop's first night on the job....   \n",
       "1  While racing to a boxing match, Frank, Mike, J...   \n",
       "2  Princess Leia is captured and held hostage by ...   \n",
       "3  Nemo, an adventurous young clownfish, is unexp...   \n",
       "4  A man with a low IQ has accomplished great thi...   \n",
       "\n",
       "                                             tagline           title    id  \n",
       "0  Twelve outrageous guests. Four scandalous requ...      Four Rooms    18  \n",
       "1     Don't move. Don't whisper. Don't even breathe.  Judgment Night   479  \n",
       "2       A long time ago in a galaxy far, far away...       Star Wars   260  \n",
       "3  There are 3.7 trillion fish in the ocean. They...    Finding Nemo  6377  \n",
       "4  The world will never be the same, once you've ...    Forrest Gump   356  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = pd.read_csv('data/raw/movies_metadata.csv', usecols=['id','original_title','title','overview','tagline','genres'])\n",
    "movies.rename(columns={'id':'tmdb_id'}, inplace=True)\n",
    "movies['tmdb_id'] = pd.to_numeric(movies['tmdb_id'], errors='coerce')\n",
    "movies.dropna(inplace=True)\n",
    "movies.drop_duplicates('tmdb_id', inplace=True)\n",
    "movies.set_index('tmdb_id', inplace=True)\n",
    "movies = movies.join(links, how='inner').reset_index(drop=True)\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies[['id','original_title','title','overview','tagline']].to_csv('data/movie_info.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    name\n",
       "0  80   Crime\n",
       "1  35  Comedy"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(literal_eval(movies.genres[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 53.9 s, sys: 28 ms, total: 53.9 s\n",
      "Wall time: 53.9 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genre_id</th>\n",
       "      <th>genre_name</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80</td>\n",
       "      <td>Crime</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28</td>\n",
       "      <td>Action</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>Thriller</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>80</td>\n",
       "      <td>Crime</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   genre_id genre_name   id\n",
       "0        80      Crime   18\n",
       "1        35     Comedy   18\n",
       "2        28     Action  479\n",
       "3        53   Thriller  479\n",
       "4        80      Crime  479"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "genre = []\n",
    "for i in movies.id:\n",
    "    df_dict = literal_eval(movies.loc[movies['id'] == i].genres.values[0])\n",
    "    if len(df_dict) > 0:\n",
    "        df = pd.DataFrame.from_dict(df_dict)[['id','name']]\n",
    "        df.columns = ['genre_id','genre_name']\n",
    "        df['id'] = i\n",
    "        genre.append(df)\n",
    "\n",
    "genre = pd.concat(genre, ignore_index=True)\n",
    "genre.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre.to_csv('data/movie_genre.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Movie Keywords\n",
    "\n",
    "Extract keywords assigned to each movie in the TMDB dataset. We will merge all keywords in a single text field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>keywords</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[{'id': 240, 'name': 'underdog'}, {'id': 378, ...</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[{'id': 1361, 'name': 'salesclerk'}, {'id': 17...</td>\n",
       "      <td>61724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[{'id': 612, 'name': 'hotel'}, {'id': 613, 'na...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[{'id': 520, 'name': 'chicago'}, {'id': 2231, ...</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[{'id': 803, 'name': 'android'}, {'id': 4270, ...</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            keywords     id\n",
       "0  [{'id': 240, 'name': 'underdog'}, {'id': 378, ...   4470\n",
       "1  [{'id': 1361, 'name': 'salesclerk'}, {'id': 17...  61724\n",
       "2  [{'id': 612, 'name': 'hotel'}, {'id': 613, 'na...     18\n",
       "3  [{'id': 520, 'name': 'chicago'}, {'id': 2231, ...    479\n",
       "4  [{'id': 803, 'name': 'android'}, {'id': 4270, ...    260"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords = pd.read_csv('data/raw/keywords.csv')\n",
    "keywords.rename(columns={'id':'tmdb_id'}, inplace=True)\n",
    "keywords['tmdb_id'] = keywords['tmdb_id'].astype('Int64')\n",
    "keywords.set_index('tmdb_id', inplace=True)\n",
    "keywords = keywords.join(links, how='inner').reset_index(drop=True)\n",
    "keywords.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>240</td>\n",
       "      <td>underdog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>378</td>\n",
       "      <td>prison</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>730</td>\n",
       "      <td>factory worker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1563</td>\n",
       "      <td>prisoner</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1787</td>\n",
       "      <td>helsinki</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10183</td>\n",
       "      <td>independent film</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13072</td>\n",
       "      <td>falling in love</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id              name\n",
       "0    240          underdog\n",
       "1    378            prison\n",
       "2    730    factory worker\n",
       "3   1563          prisoner\n",
       "4   1787          helsinki\n",
       "5  10183  independent film\n",
       "6  13072   falling in love"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(literal_eval(keywords.keywords[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'underdog prison factory worker prisoner helsinki independent film falling in love'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join([i['name'] for i in literal_eval(keywords.keywords[0])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.8 s, sys: 28 ms, total: 58.8 s\n",
      "Wall time: 58.8 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>keywords</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>underdog prison factory worker prisoner helsin...</td>\n",
       "      <td>4470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>salesclerk helsinki garbage independent film</td>\n",
       "      <td>61724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hotel new year's eve witch bet hotel room sper...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chicago drug dealer boxing match escape one night</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>android galaxy hermit death star lightsaber je...</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            keywords     id\n",
       "0  underdog prison factory worker prisoner helsin...   4470\n",
       "1       salesclerk helsinki garbage independent film  61724\n",
       "2  hotel new year's eve witch bet hotel room sper...     18\n",
       "3  chicago drug dealer boxing match escape one night    479\n",
       "4  android galaxy hermit death star lightsaber je...    260"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "keys = []\n",
    "for i in keywords.id:\n",
    "    df_dict = literal_eval(keywords.loc[keywords['id'] == i].keywords.values[0])\n",
    "    if len(df_dict) > 0:\n",
    "        kw = ' '.join([i['name'] for i in df_dict])\n",
    "        df = pd.DataFrame(data={'keywords': [kw]})\n",
    "        df['id'] = i\n",
    "        keys.append(df)\n",
    "\n",
    "keys = pd.concat(keys, ignore_index=True)\n",
    "keys.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys.to_csv('data/movie_keywords.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract User Tags\n",
    "\n",
    "Extract tags assigned to the movies by the users in the MovieLens dataset. They will be combined in a single text field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110</td>\n",
       "      <td>epic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110</td>\n",
       "      <td>Medieval</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>260</td>\n",
       "      <td>sci-fi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>260</td>\n",
       "      <td>space action</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>318</td>\n",
       "      <td>imdb top 250</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId           tag\n",
       "0      110          epic\n",
       "1      110      Medieval\n",
       "2      260        sci-fi\n",
       "3      260  space action\n",
       "4      318  imdb top 250"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags = pd.read_csv('data/raw/tags.csv', usecols=['movieId','tag'])\n",
    "tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 908 ms, sys: 36 ms, total: 944 ms\n",
      "Wall time: 942 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>110</td>\n",
       "      <td>epic Medieval overrated Oscar (Best Picture) e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>260</td>\n",
       "      <td>sci-fi space action classic sci-fi Harrison Fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>318</td>\n",
       "      <td>imdb top 250 justice story line hope inspirati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>480</td>\n",
       "      <td>Dinosaurs dinosaurs action thriller dinosaurs ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>593</td>\n",
       "      <td>psychothriller brilliant exploratory ligne de ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id                                               tags\n",
       "0  110  epic Medieval overrated Oscar (Best Picture) e...\n",
       "1  260  sci-fi space action classic sci-fi Harrison Fo...\n",
       "2  318  imdb top 250 justice story line hope inspirati...\n",
       "3  480  Dinosaurs dinosaurs action thriller dinosaurs ...\n",
       "4  593  psychothriller brilliant exploratory ligne de ..."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tags_groups = tags[['movieId','tag']].dropna().groupby('movieId', sort=False)\n",
    "tags = tags_groups['tag'].agg(tags=lambda x: ' '.join(x)).reset_index(drop=False)\n",
    "tags.columns = ['id', 'tags']\n",
    "tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags.to_csv('data/movie_tags.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter user ratings\n",
    "\n",
    "From the user ratings in the MovieLens dataset, select only those associated with movies with at least 30 votes and users with at least 3 votes. This way we discard irrelevant interactions and reduce data dimension. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>307</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>481</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1091</td>\n",
       "      <td>1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1257</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1449</td>\n",
       "      <td>4.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user    id  rating\n",
       "0     1   307     3.5\n",
       "1     1   481     3.5\n",
       "2     1  1091     1.5\n",
       "3     1  1257     4.5\n",
       "4     1  1449     4.5"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = pd.read_csv('data/raw/ratings.csv', usecols=['userId','movieId','rating'])\n",
    "ratings.columns = ['user','id','rating']\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings['user_votes'] = ratings.groupby('user')['user'].transform('count')\n",
    "ratings['movie_votes'] = ratings.groupby('id')['id'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings.loc[(ratings['movie_votes'] >= 30) & (ratings['user_votes'] >= 3), ['user','id','rating']].to_csv('data/movie_ratings.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}