PreProcessingV2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Getting the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gzip\n",
    "import html5lib \n",
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.DS_Store',\n",
       " 'Dataset_Challenge_Dataset_Agreement.pdf',\n",
       " 'yelp_academic_dataset_business.json',\n",
       " 'yelp_academic_dataset_review.json',\n",
       " 'yelp_academic_dataset_user.json',\n",
       " 'Yelp_Dataset_Challenge_Terms_round_9.pdf']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# Listing all the JSON files in the directory\n",
    "dir_yelp = '../../DataProject/yelp_dataset_challenge_round9/'\n",
    "y_dirs = os.listdir(dir_yelp)\n",
    "\n",
    "#Output dir \n",
    "#if not os.path.exists(out_dir):\n",
    "#    os.makedirs(out_dir)\n",
    "y_dirs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "### This part of the code is based on Prof. Perry's code  \n",
    "### http://ptrckprry.com/course/ssd/data/yelp-academic/01_make_json.py\n",
    "\n",
    "#Setting the structures of the information\n",
    "tables = {}\n",
    "\n",
    "#'elite'\n",
    "\n",
    "tables['user'] = {\n",
    "    'fields': ['user_id', 'name', 'yelping_since', 'average_stars'],\n",
    "    'subfields': {}\n",
    "    }\n",
    "\n",
    "tables['review'] = {\n",
    "    'fields': ['review_id', 'business_id', 'user_id', 'date', 'stars','text'],\n",
    "    'subfields': {}\n",
    "    }\n",
    "\n",
    "tables['business'] = {\n",
    "    'fields': ['business_id','name', 'stars', 'is_open','categories', 'state', 'city', \n",
    "               'longitude','latitude', 'neighborhood'],\n",
    "    'subfields': {}\n",
    "    }\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Formatting the data in Dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../../DataProject/yelp_dataset_challenge_round9/yelp_academic_dataset_business.json\n",
      "../../DataProject/yelp_dataset_challenge_round9/yelp_academic_dataset_review.json\n",
      "../../DataProject/yelp_dataset_challenge_round9/yelp_academic_dataset_user.json\n",
      "ENDED!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def unescape(x):\n",
    "    if isinstance(x, str):\n",
    "        x = html.unescape(x)\n",
    "    return x\n",
    "\n",
    "reviews_ext= []\n",
    "user_ext = []\n",
    "business_ext = []\n",
    "\n",
    "for y_dir in y_dirs:\n",
    "    #The files we're interested in extracting end with \".json\"\n",
    "    r1 = int('user' in  y_dir)\n",
    "    r2 = int('review' in y_dir )\n",
    "    r3 = int('business' in y_dir )\n",
    "    if y_dir.endswith('.json') and (r1==1 or r2==1 or r3==1)  :\n",
    "        print dir_yelp + y_dir\n",
    "        with open(dir_yelp+ y_dir, 'r') as dataset:\n",
    "            #Extracting the lines and formatting accordingly\n",
    "            for line in dataset:\n",
    "                obj = json.loads(line.decode('utf-8'))\n",
    "                tab = tables[obj['type']]\n",
    "                record = {}\n",
    "                #print(line)\n",
    "                for i in tab['fields']:\n",
    "                    #print(i)\n",
    "                    if i in tab['subfields']:\n",
    "                        record[i] = {}\n",
    "                        for j in tab['subfields'][i]:\n",
    "                            record[i][j] = unescape(obj[i][j])\n",
    "                    else:\n",
    "                        record[i] = unescape(obj[i])\n",
    "                if 'review_id' in record: #IS REVIEW\n",
    "                    reviews_ext.append(record)\n",
    "                if 'yelping_since' in record: #IS USER\n",
    "                    user_ext.append(record)\n",
    "                if 'neighborhood' in record:  #IS BUSINESS\n",
    "                    business_ext.append(record)\n",
    "                    \n",
    "print(\"ENDED!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "users = pd.DataFrame(user_ext)\n",
    "businesses = pd.DataFrame(business_ext)\n",
    "reviews = pd.DataFrame(reviews_ext)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def getDfSummary(input_data):\n",
    "    # Get a whole bunch of stats\n",
    "    output_data = input_data.describe().transpose()\n",
    "    # Count NANs\n",
    "    output_data['number_nan'] = input_data.shape[0] - output_data['count']\n",
    "    # Type of data\n",
    "    output_data['types'] = [type(input_data[i][0]).__name__ for i in output_data.index]\n",
    "    # Count unique valuesc\n",
    "    distincts = []\n",
    "    for i in output_data.index:\n",
    "        if isinstance(input_data[i][0], list):\n",
    "            distincts.append(np.nan)\n",
    "        else:\n",
    "            distincts.append(len(input_data[i].unique()))\n",
    "    output_data['number_distinct'] = distincts\n",
    "    # Remove 'count' column since it wasn't asked for\n",
    "    output_data = output_data.drop('count', 1)\n",
    "    return output_data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Users' Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>average_stars</th>\n",
       "      <th>name</th>\n",
       "      <th>user_id</th>\n",
       "      <th>yelping_since</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1029427</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Marcia</td>\n",
       "      <td>mZ4V8gAjQYLWdPiPkhWnPA</td>\n",
       "      <td>2015-01-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029428</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Cathy</td>\n",
       "      <td>z3XH6YdJnz_3kmNMFuB70A</td>\n",
       "      <td>2016-07-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029429</th>\n",
       "      <td>2.0</td>\n",
       "      <td>Hannah</td>\n",
       "      <td>s1l_dY8eGvwMqQNvQXeqNg</td>\n",
       "      <td>2015-11-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029430</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Angel</td>\n",
       "      <td>qNdPduox3xNycsWDJRCFLA</td>\n",
       "      <td>2015-07-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029431</th>\n",
       "      <td>5.0</td>\n",
       "      <td>Al</td>\n",
       "      <td>vp0L6e84U7tBFpmOpsD8RQ</td>\n",
       "      <td>2015-09-25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         average_stars    name                 user_id yelping_since\n",
       "1029427            5.0  Marcia  mZ4V8gAjQYLWdPiPkhWnPA    2015-01-09\n",
       "1029428            5.0   Cathy  z3XH6YdJnz_3kmNMFuB70A    2016-07-19\n",
       "1029429            2.0  Hannah  s1l_dY8eGvwMqQNvQXeqNg    2015-11-11\n",
       "1029430            5.0   Angel  qNdPduox3xNycsWDJRCFLA    2015-07-31\n",
       "1029431            5.0      Al  vp0L6e84U7tBFpmOpsD8RQ    2015-09-25"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "users.tail()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "name_datetime_variable = 'yelping_since'\n",
    "users[name_datetime_variable] = pd.to_datetime(users[name_datetime_variable])\n",
    "users[name_datetime_variable+'_year'] = pd.DatetimeIndex(users[name_datetime_variable]).year\n",
    "users[name_datetime_variable+'_month'] = pd.DatetimeIndex(users[name_datetime_variable]).month\n",
    "users[name_datetime_variable+'_day'] = pd.DatetimeIndex(users[name_datetime_variable]).day\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtgAAAJQCAYAAAC0MBOwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3X+03XV95/vnqwFGCypYzlAmiT3Ym8rKuBSYM8hU69iq\nvQl0NXqn18JqAan3pkxBpctOmzprjXbWujMZtbZyS8mNGoVVKmP9UXNrRqTUVr0VmxPEQEDqKcWS\nTCDHWkGlFQPv+8f+RreHk2Tn7E/O3id5Ptba63y/nx/f/d7ry8558T3fH6kqJEmSJLXxA6MuQJIk\nSTqWGLAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYkSZIaMmBLkiRJ\nDZ0w6gKOxOmnn16Tk5OjLkOSJEnHuB07dny1qiYWMndJBezJyUmmp6dHXYYkSZKOcUm+stC5niIi\nSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKmhJXWbPknSU01u+PhQ8x/YeFGz\n7bXc1mJsT5KOBgP2McBfODoWjPN/x4ZESdKRMGCPgL9cx8fxFJyOp88KbY/C6tg1Tt+LpfYdO56M\n038n821P48eArSXlePpH6Xj6rJKOPv9NkRaPAVtP4REYSZKkhTNgS5KkY4oHdzRqBuwB+EWVJOn7\n+btROjjvgy1JkiQ15BFsSZI0ch4R17HEgC1JkrSE+T8n48dTRCRJkqSGDNiSJElSQ0OdIpJkDfAu\nYBnwnqraOKc/Xf+FwGPAa6vqjiRPAz4N/LOuhg9V1VuGqUWSJEnD80m4w1vwEewky4DrgLXAauCS\nJKvnDFsLrOpe64Hru/ZvAz9VVS8EzgHWJLlgobVIkiRJ42KYU0TOB2aq6v6qehy4GVg3Z8w64Mbq\nuR04NcmZ3fo3uzEndq8aohZJkiRpLAwTsJcDD/at7+7aBhqTZFmSO4F9wK1V9fn53iTJ+iTTSaZn\nZ2eHKFeSJEk6+kZ2kWNVPVFV5wArgPOTPP8g4zZX1VRVTU1MTCxukZIkSdIRGiZg7wFW9q2v6NqO\naExVfR34FLBmiFokSZKksTBMwN4OrEpyVpKTgIuBrXPGbAUuS88FwCNVtTfJRJJTAZI8HXgl8KUh\napEkSZLGwoJv01dV+5NcDdxC7zZ9W6pqV5Iru/5NwDZ6t+iboXebviu66WcCN3R3IvkB4INV9ScL\n/xiSJEnSeBjqPthVtY1eiO5v29S3XMBV88zbCZw7zHtLkiRJ48gnOUqSJEkNGbAlSZKkhgzYkiRJ\nUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJD\nBmxJkiSpIQO2JEmS1JABW5IkSWrIgC1JkiQ1ZMCWJEmSGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZs\nSZIkqSEDtiRJktSQAVuSJElqyIAtSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmS\nJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSp\noaECdpI1Se5LMpNkwzz9SXJt178zyXld+8okn0pyT5JdSd44TB2SJEnSuFhwwE6yDLgOWAusBi5J\nsnrOsLXAqu61Hri+a98PvKmqVgMXAFfNM1eSJElacoY5gn0+MFNV91fV48DNwLo5Y9YBN1bP7cCp\nSc6sqr1VdQdAVX0DuBdYPkQtkiRJ0lgYJmAvBx7sW9/NU0PyYcckmQTOBT4/RC2SJEnSWBjpRY5J\nTgE+DFxTVY8eZMz6JNNJpmdnZxe3QEmSJOkIDROw9wAr+9ZXdG0DjUlyIr1wfVNVfeRgb1JVm6tq\nqqqmJiYmhihXkiRJOvqGCdjbgVVJzkpyEnAxsHXOmK3AZd3dRC4AHqmqvUkCvBe4t6reOUQNkiRJ\n0lg5YaETq2p/kquBW4BlwJaq2pXkyq5/E7ANuBCYAR4Druimvxi4FLgryZ1d25urattC65EkSdJ4\nmdzw8aHmP7DxokaVLK4FB2yALhBvm9O2qW+5gKvmmfdZIMO8tyRJkjSOfJKjJEmS1JABW5IkSWrI\ngC1JkiQ1ZMCWJEmSGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJktSQAVuSJElqyIAt\nSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmS\nJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSpIQO2JEmS1JABW5IkSWrIgC1JkiQ1\nZMCWJEmSGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJktSQAVuSJElqyIAtSZIkNWTA\nliRJkhoyYEuSJEkNDRWwk6xJcl+SmSQb5ulPkmu7/p1Jzuvr25JkX5K7h6lBkiRJGicLDthJlgHX\nAWuB1cAlSVbPGbYWWNW91gPX9/W9H1iz0PeXJEmSxtEwR7DPB2aq6v6qehy4GVg3Z8w64MbquR04\nNcmZAFX1aeBrQ7y/JEmSNHaGCdjLgQf71nd3bUc65pCSrE8ynWR6dnZ2QYVKkiRJi2XsL3Ksqs1V\nNVVVUxMTE6MuR5IkSTqkYQL2HmBl3/qKru1Ix0iSJEnHjGEC9nZgVZKzkpwEXAxsnTNmK3BZdzeR\nC4BHqmrvEO8pSZIkjbUFB+yq2g9cDdwC3At8sKp2JbkyyZXdsG3A/cAM8G7gVw7MT/IB4HPA85Ls\nTvK6hdYiSZIkjYsThplcVdvohej+tk19ywVcdZC5lwzz3pIkSdI4GvuLHCVJkqSlxIAtSZIkNWTA\nliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYk\nSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSpIQO2JEmS1JABW5IkSWrIgC1JkiQ1ZMCWJEmS\nGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJktSQAVuSJElqyIAtSZIkNWTAliRJkhoy\nYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYkSZIaMmBL\nkiRJDRmwJUmSpIaGCthJ1iS5L8lMkg3z9CfJtV3/ziTnDTpXkiRJWooWHLCTLAOuA9YCq4FLkqye\nM2wtsKp7rQeuP4K5kiRJ0pIzzBHs84GZqrq/qh4HbgbWzRmzDrixem4HTk1y5oBzJUmSpCUnVbWw\nicnPAWuq6v/o1i8FXlRVV/eN+RNgY1V9tlu/DfgNYPJwc/u2sZ7e0W+A5wH3Lajgpzod+GqjbWnh\n3A/jw30xPtwX48N9MT7cF+PheNoPP1JVEwuZeELrSlqrqs3A5tbbTTJdVVOtt6sj434YH+6L8eG+\nGB/ui/HhvhgP7ofBDBOw9wAr+9ZXdG2DjDlxgLmSJEnSkjPMOdjbgVVJzkpyEnAxsHXOmK3AZd3d\nRC4AHqmqvQPOlSRJkpacBR/Brqr9Sa4GbgGWAVuqaleSK7v+TcA24EJgBngMuOJQc4f6JEeu+Wkn\nWhD3w/hwX4wP98X4cF+MD/fFeHA/DGDBFzlKkiRJeiqf5ChJkiQ1ZMCWJEmSGjruAraPaB8fSR5I\ncleSO5NMj7qe40mSLUn2Jbm7r+3ZSW5N8uXu52mjrPF4cZB98dYke7rvxp1JLhxljceDJCuTfCrJ\nPUl2JXlj1+73YpEdYl/4vVhkSZ6W5K+SfLHbF7/Vtfu9OIzj6hzs7hHtfw28EthN724ml1TVPSMt\n7DiV5AFgqqqOlxvWj40kLwW+Se9Jq8/v2t4GfK2qNnb/83laVf3GKOs8HhxkX7wV+GZVvWOUtR1P\nuqcMn1lVdyR5BrADeBXwWvxeLKpD7IvX4PdiUSUJcHJVfTPJicBngTcC/xt+Lw7peDuC7SPaJaCq\nPg18bU7zOuCGbvkGer/QdJQdZF9okVXV3qq6o1v+BnAvsBy/F4vuEPtCi6x6vtmtnti9Cr8Xh3W8\nBezlwIN967vxSztKBfxpkh1J1o+6GHFGd596gIeAM0ZZjHh9kp3dKST++XURJZkEzgU+j9+LkZqz\nL8DvxaJLsizJncA+4Naq8nsxgOMtYGu8vKSqzgHWAld1fyrXGKjeuWPHz/lj4+d64LnAOcBe4LdH\nW87xI8kpwIeBa6rq0f4+vxeLa5594fdiBKrqie539Qrg/CTPn9Pv92Iex1vAHuTx7lokVbWn+7kP\n+Ci9U3g0Og935z4eOAdy34jrOW5V1cPdL7UngXfjd2NRdOeYfhi4qao+0jX7vRiB+faF34vRqqqv\nA58C1uD34rCOt4DtI9rHRJKTu4tXSHIy8NPA3YeepaNsK3B5t3w58LER1nJcO/CLq/Nq/G4cdd3F\nXO8F7q2qd/Z1+b1YZAfbF34vFl+SiSSndstPp3eTiC/h9+Kwjqu7iAB0t/X5Xb73iPb/a8QlHZeS\nPJfeUWuAE4A/dF8sniQfAF4GnA48DLwF+GPgg8BzgK8Ar6kqL747yg6yL15G78/gBTwA/HLf+Y46\nCpK8BPgMcBfwZNf8Znrn/vq9WESH2BeX4PdiUSV5Ab2LGJfROyj7war6z0l+CL8Xh3TcBWxJkiTp\naDreThGRJEmSjioDtiRJktSQAVuSJElqyIAtSZIkNWTAliRJkhoyYEvSAiR5IsmdSXYl+WKSNyU5\nqv+mJnl7935vn9P+siQ/3rf+/iQ/dzRrWYgkb+5bnkzifYwlHZNOGHUBkrRE/WP3+GCS/HPgD4Fn\n0ruP9dGyHnh2VT0xp/1lwDeBvzyK793Cm4H/MuoiJOlo8wi2JA2pqvbRC79Xp2cyyWeS3NG9fhwg\nyY1JXnVgXpKbkqzr31Y3/+1J7k5yV5Kf79q3AqcAOw60de2TwJXAr3ZH1H+i63ppkr9Mcn//0ewk\n/yHJ9iQ7k/zWfJ8nyTf7jpb/aZLzk/x5t62f7cY8Lcn7uhq/kOQnu/bXJvlIkk8k+XKSt3XtG4Gn\ndzXe1L3VsiTv7t7nk92T4iRpyfNBM5K0AEm+WVWnzGn7OvA84BvAk1X1T0lWAR+oqqkk/xb41ap6\nVZJnAXcCq6pqf982/h29wLyG3tMdtwMvqqq9871nN+etwDer6h3d+vuBk4GfB84GtlbV/5Lkp4Gf\nA34ZCL3HHb+tqj49Z3sFXFhV/yPJR7ttXQSsBm6oqnOSvAn4l1X1S0nOBj4J/BhwMfCfgHOBbwP3\nAS+pqgf76+/+x2AGmKqqO5N8sKvzD45gN0jSWPIItiS1dyLw7iR3AX9EL5hSVX8BrEoyQe+xzx/u\nD9edl9AL5E9U1cPAXwD/egE1/HFVPVlV9wBndG0/3b2+ANxBL3yvmmfu48AnuuW7gL+oqu90y5N9\ndf5B97m+RO9xyT/W9d1WVY9U1T8B9wA/cpAa/7aq7uyWd/RtW5KWNM/BlqQGkjwXeALYR+887IeB\nF9I7kPFPfUNvBH6R3pHeK45iSd/uL6/v53+tqv/nMHO/U9/78+aTB7ZVVU8mGeT3Rv97P8HBf9fM\nHecpIpKOCR7BlqQhdUekNwG/1wXTZwF7q+pJ4FJgWd/w9wPXAHRHl+f6DPDzSZZ1230p8FeHKeEb\nwDMGKPUW4JeSHDhNY3l3geZCfAb4hW47PwY8h97pIIfynSQnLvD9JGnJMGBL0sIcuGBvF/Cn9M5B\nPnDR4O8Dlyf5Ir3TML51YFJ32se9wPsOst2PAjuBLwJ/Bvx6VT10mFr+X+DVcy5yfIqq+iS9u518\nrjt95UMMFszn8/vAD3Tb+e/Aa6vq24eZsxnY2XeRoyQdk7zIUZIWUZIfpHcu83lV9cio65EktTfU\nEewka5Lcl2QmyYZ5+s9O8rkk307ya0cyV5KONUleQe/o9f9tuJakY9eCj2AnWQb8NfBKYDe9W0ld\n0n9OYXdu348ArwL+oe8WUoedO5/TTz+9JicnF1SvJEmSNKgdO3Z8taomFjJ3mLuInA/MVNX9AElu\nBtbRuyUT8N2HL+xLctGRzp3P5OQk09PTQ5QsSZIkHV6Sryx07jCniCwHHuxb3921He25kiRJ0tga\n+7uIJFmfZDrJ9Ozs7KjLkSRJkg5pmIC9B1jZt76ia2s6t6o2V9VUVU1NTCzoNBhJkiRp0QwTsLfT\ne+TvWUlOovdUsq2LMFeSJEkaWwu+yLGq9ie5mt6TwZYBW6pqV5Iru/5NSX4YmAaeCTyZ5BpgdVU9\nOt/cYT+MJEmSNGpL6kEzU1NT5V1EJEmSdLQl2VFVUwuZO8xt+iRJS9Tkho+PuoTDemDj3Du8StLS\nMPZ3EZEkSZKWEgO2JEmS1JABW5IkSWrIgC1JkiQ15EWOkqQlbSlcsAletCkdTzyCLUmSJDVkwJYk\nSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSpIQO2JEmS1JABW5IkSWrIB81I0gB8mIkkaVAG\n7AH4i1WSJEmD8hQRSZIkqSEDtiRJktSQAVuSJElqyHOwteQthXPkPT9ekqTjhwFb0lHj//xIko5H\nniIiSZIkNWTAliRJkhoaKmAnWZPkviQzSTbM058k13b9O5Oc19f3q0l2Jbk7yQeSPG2YWiRJkqRx\nsOBzsJMsA64DXgnsBrYn2VpV9/QNWwus6l4vAq4HXpRkOfAGYHVV/WOSDwIXA+9faD2SJB0LjrVr\nF461zyMNYpgj2OcDM1V1f1U9DtwMrJszZh1wY/XcDpya5Myu7wTg6UlOAH4Q+J9D1CJJkiSNhWHu\nIrIceLBvfTe9o9SHG7O8qqaTvAP4O+AfgU9W1Sfne5Mk64H1AM95znOGKFcafx7pkSRp6RvJRY5J\nTqN3dPss4F8AJyf5xfnGVtXmqpqqqqmJiYnFLFOSJEk6YsME7D3Ayr71FV3bIGNeAfxtVc1W1XeA\njwA/PkQtkiRJ0lgYJmBvB1YlOSvJSfQuUtw6Z8xW4LLubiIXAI9U1V56p4ZckOQHkwR4OXDvELVI\nkiRJY2HB52BX1f4kVwO3AMuALVW1K8mVXf8mYBtwITADPAZc0fV9PsmHgDuA/cAXgM3DfBBJkiRp\nHAz1qPSq2kYvRPe3bepbLuCqg8x9C/CWYd5fC+OFdJIkSUePT3KUJEmSGjJgS5IkSQ0ZsCVJkqSG\nDNiSJElSQ0Nd5ChJknS8WAo3CYDBbxSwFD7PUr3pgUewJUmSpIYM2JIkSVJDBmxJkiSpIQO2JEmS\n1JABW5IkSWrIgC1JkiQ1ZMCWJEmSGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJktSQ\nAVuSJElqyIAtSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhoYK2EnWJLkvyUySDfP0J8m1Xf/OJOf1\n9Z2a5ENJvpTk3iT/ZphaJEmSpHGw4ICdZBlwHbAWWA1ckmT1nGFrgVXdaz1wfV/fu4BPVNXZwAuB\nexdaiyRJkjQuhjmCfT4wU1X3V9XjwM3Aujlj1gE3Vs/twKlJzkzyLOClwHsBqurxqvr6ELVIkiRJ\nY2GYgL0ceLBvfXfXNsiYs4BZ4H1JvpDkPUlOHqIWSZIkaSyM6iLHE4DzgOur6lzgW8BTzuEGSLI+\nyXSS6dnZ2cWsUZIkSTpiwwTsPcDKvvUVXdsgY3YDu6vq8137h+gF7qeoqs1VNVVVUxMTE0OUK0mS\nJB19wwTs7cCqJGclOQm4GNg6Z8xW4LLubiIXAI9U1d6qegh4MMnzunEvB+4ZohZJkiRpLJyw0IlV\ntT/J1cAtwDJgS1XtSnJl178J2AZcCMwAjwFX9G3i9cBNXTi/f06fJEmStCQtOGADVNU2eiG6v21T\n33IBVx1k7p3A1DDvL0mSJI0bn+QoSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmS\nJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSp\nIQO2JEmS1JABW5IkSWrIgC1JkiQ1ZMCWJEmSGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSED\ntiRJktTQUAE7yZok9yWZSbJhnv4kubbr35nkvDn9y5J8IcmfDFOHJEmSNC4WHLCTLAOuA9YCq4FL\nkqyeM2wtsKp7rQeun9P/RuDehdYgSZIkjZthjmCfD8xU1f1V9ThwM7Buzph1wI3VcztwapIzAZKs\nAC4C3jNEDZIkSdJYGSZgLwce7Fvf3bUNOuZ3gV8HnjzUmyRZn2Q6yfTs7OwQ5UqSJElH30guckzy\nM8C+qtpxuLFVtbmqpqpqamJiYhGqkyRJkhZumIC9B1jZt76iaxtkzIuBn03yAL1TS34qyR8MUYsk\nSZI0FoYJ2NuBVUnOSnIScDGwdc6YrcBl3d1ELgAeqaq9VfWbVbWiqia7eX9WVb84RC2SJEnSWDhh\noROran+Sq4FbgGXAlqraleTKrn8TsA24EJgBHgOuGL5kSZIkaXwtOGADVNU2eiG6v21T33IBVx1m\nG38O/PkwdUiSJEnjwic5SpIkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJktSQAVuSJElqyIAt\nSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmS\nJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSpIQO2JEmS1JABW5IkSWpoqICdZE2S\n+5LMJNkwT3+SXNv170xyXte+MsmnktyTZFeSNw5ThyRJkjQuFhywkywDrgPWAquBS5KsnjNsLbCq\ne60Hru/a9wNvqqrVwAXAVfPMlSRJkpacYY5gnw/MVNX9VfU4cDOwbs6YdcCN1XM7cGqSM6tqb1Xd\nAVBV3wDuBZYPUYskSZI0FoYJ2MuBB/vWd/PUkHzYMUkmgXOBz8/3JknWJ5lOMj07OztEuZIkSdLR\nN9KLHJOcAnwYuKaqHp1vTFVtrqqpqpqamJhY3AIlSZKkIzRMwN4DrOxbX9G1DTQmyYn0wvVNVfWR\nIeqQJEmSxsYwAXs7sCrJWUlOAi4Gts4ZsxW4rLubyAXAI1W1N0mA9wL3VtU7h6hBkiRJGisnLHRi\nVe1PcjVwC7AM2FJVu5Jc2fVvArYBFwIzwGPAFd30FwOXAnclubNre3NVbVtoPZIkSdI4WHDABugC\n8bY5bZv6lgu4ap55nwUyzHtLkiRJ48gnOUqSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLU\nkAFbkiRJasiALUmSJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIYM2JIkSVJDBmxJkiSpIQO2JEmS1JAB\nW5IkSWrIgC1JkiQ1ZMCWJEmSGjJgS5IkSQ0ZsCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJktSQAVuS\nJElqaKiAnWRNkvuSzCTZME9/klzb9e9Mct6gcyVJkqSlaMEBO8ky4DpgLbAauCTJ6jnD1gKrutd6\n4PojmCtJkiQtOcMcwT4fmKmq+6vqceBmYN2cMeuAG6vnduDUJGcOOFeSJElacoYJ2MuBB/vWd3dt\ng4wZZK4kSZK05Jww6gIOJ8l6eqeXAHwzyX2jrGec5b+N9O1PB77aamMj/izNuW/G17G0b2Dkn6cp\n9834ct+MtxF+nmNt3/zIQicOE7D3ACv71ld0bYOMOXGAuQBU1WZg8xB1ahEkma6qqVHXoady34wv\n9834ct+ML/fN+HLffM8wp4hsB1YlOSvJScDFwNY5Y7YCl3V3E7kAeKSq9g44V5IkSVpyFnwEu6r2\nJ7kauAVYBmypql1Jruz6NwHbgAuBGeAx4IpDzR3qk0iSJEljYKhzsKtqG70Q3d+2qW+5gKsGnasl\nzdN4xpf7Zny5b8aX+2Z8uW/Gl/umk14GliRJktSCj0qXJEmSGjJga8GSrEzyqST3JNmV5I2jrknf\nL8myJF9I8iejrkXfL8mpST6U5EtJ7k3yb0Zdk3qS/Gr3b9rdST6Q5Gmjrul4lWRLkn1J7u5re3aS\nW5N8uft52ihrPF4dZN+8vfs3bWeSjyY5dZQ1jpIBW8PYD7ypqlYDFwBX+cj7sfNG4N5RF6F5vQv4\nRFWdDbwQ99NYSLIceAMwVVXPp3ch/sWjreq49n5gzZy2DcBtVbUKuK1b1+J7P0/dN7cCz6+qFwB/\nDfzmYhc1LgzYWrCq2ltVd3TL36AXEHwi55hIsgK4CHjPqGvR90vyLOClwHsBqurxqvr6aKtSnxOA\npyc5AfhB4H+OuJ7jVlV9GvjanOZ1wA3d8g3Aqxa1KAHz75uq+mRV7e9Wb6f3nJPjkgFbTSSZBM4F\nPj/aStTnd4FfB54cdSF6irOAWeB93Sk870ly8qiLElTVHuAdwN8Be+k9v+GTo61Kc5zRPVMD4CHg\njFEWo4P6JeB/jLqIUTFga2hJTgE+DFxTVY+Ouh5Bkp8B9lXVjlHXonmdAJwHXF9V5wLfwj9zj4Xu\nfN519P4n6F8AJyf5xdFWpYPpbgfs7dDGTJL/SO800ptGXcuoGLA1lCQn0gvXN1XVR0Zdj77rxcDP\nJnkAuBn4qSR/MNqS1Gc3sLuqDvzF50P0ArdG7xXA31bVbFV9B/gI8OMjrknf7+EkZwJ0P/eNuB71\nSfJa4GeAX6jj+F7QBmwtWJLQO4f03qp656jr0fdU1W9W1YqqmqR3gdafVZVH4cZEVT0EPJjkeV3T\ny4F7RliSvufvgAuS/GD3b9zL8QLUcbMVuLxbvhz42AhrUZ8ka+idmvizVfXYqOsZJQO2hvFi4FJ6\nR0fv7F4XjrooaYl4PXBTkp3AOcB/GXE9Arq/KnwIuAO4i97vSZ9ONyJJPgB8Dnhekt1JXgdsBF6Z\n5Mv0/uKwcZQ1Hq8Osm9+D3gGcGuXCTYdciPHMJ/kKEmSJDXkEWxJkiSpIQO2JEmS1JABW5IkSWrI\ngC1JkiSFh2GKAAAZRUlEQVQ1ZMCWJEmSGjJgS9IRSlL9D+5JckKS2SR/ssDtnZrkV/rWX7bQbXXz\nz+5ukfWFJD86p+/NfcuTSe5e6PtIkuZnwJakI/ct4PlJnt6tvxLYM8T2TgV+5bCjBvcq4ENVdW5V\n/c2cvjfPN2FcJDlh1DVI0rAM2JK0MNuAi7rlS4APHOhI8uwkf5xkZ5Lbk7yga39rki1J/jzJ/Une\n0E3ZCPxod9T57V3bKUk+lORLSW7qnir4fZKc021/Z5KPJjmte9jTNcC/T/KpOeM3Ak/v3uemrnlZ\nkncn2ZXkkwf+pyHJjyb5RJIdST6T5Ow52/qBJF9OMtG3PpNkont9OMn27vXibsz5ST7XHVn/ywNP\nskzy2iRbk/wZcNuC9oYkjREDtiQtzM3AxUmeBrwA+Hxf328BX6iqF9A7YnxjX9/ZwP8KnA+8JcmJ\nwAbgb6rqnKr6D924c+kF5dXAc+k9OXWuG4Hf6N7nLuAtVbUN2AT8TlX9ZP/gqtoA/GP3Pr/QNa8C\nrquqfwl8Hfh3Xftm4PVV9a+AXwN+f862ngT+ADiwnVcAX6yqWeBd3fv/62577+nGfAn4iao6F/hP\nfP/TK88Dfq6q/u08n1OSlhT/FCdJC1BVO5NM0jt6vW1O90vogmpV/VmSH0ryzK7v41X1beDbSfYB\nZxzkLf6qqnYDJLkTmAQ+e6AzybOAU6vqL7qmG4A/WsBH+duqurNb3gFMJjkF+HHgj/oOnP+zeeZu\nAT4G/C7wS8D7uvZXAKv75j6z2+azgBuSrAIKOLFvW7dW1dcWUL8kjR0DtiQt3FbgHcDLgB8acM63\n+5af4OD/Dg86blhz3+fp9P66+fWqOudQE6vqwSQPJ/kpekfkDxzN/gHggqr6p/7xSX4P+FRVvbr7\nn5M/7+v+1jAfQpLGiaeISNLCbQF+q6rumtP+GbqwmeRlwFer6tFDbOcbwDOO5I2r6hHgH5L8RNd0\nKfAXh5hywHe601IOte1Hgb9N8r8DpOeFBxn+HnqnivxRVT3RtX0SeP2BAUkOBPVn8b2LQV87QK2S\ntCQZsCVpgapqd1VdO0/XW4F/lWQnvQsYLz/Mdv4e+P+S3N13keMgLgfe3r3POcB/HmDOZmBn30WO\nB/MLwOuSfBHYBaw7yLitwCl87/QQgDcAU93Fl/cAV3btbwP+a5Iv4F9QJR3DUlWjrkGStEQlmaJ3\nQeNPHHawJB0nPIIgSVqQJBuAf8/3zr2WJOERbEmSJKmpJXUE+/TTT6/JyclRlyFJkqRj3I4dO75a\nVRMLmbukAvbk5CTT09OjLkOSJEnHuCRfWehc7yIiSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzY\nkiRJUkMGbEmSJKmhJXWbPkmS1Nbkho+PuoTv88DGi0ZdgjQ0j2BLkiRJDRmwJUmSpIYGCthJ1iS5\nL8lMkg3z9J+d5HNJvp3k1+b0PZDkriR3Jpnua392kluTfLn7edrwH0eSJEkarcMG7CTLgOuAtcBq\n4JIkq+cM+xrwBuAdB9nMT1bVOVU11de2AbitqlYBt3XrkiRJ0pI2yBHs84GZqrq/qh4HbgbW9Q+o\nqn1VtR34zhG89zrghm75BuBVRzBXkiRJGkuDBOzlwIN967u7tkEV8KdJdiRZ39d+RlXt7ZYfAs44\ngm1KkiRJY2kxbtP3kqrak+SfA7cm+VJVfbp/QFVVkppvchfK1wM85znPOfrVSpI0BG97J2mQI9h7\ngJV96yu6toFU1Z7u5z7go/ROOQF4OMmZAN3PfQeZv7mqpqpqamJiYtC3lSRJkkZikIC9HViV5Kwk\nJwEXA1sH2XiSk5M848Ay8NPA3V33VuDybvly4GNHUrgkSZI0jg57ikhV7U9yNXALsAzYUlW7klzZ\n9W9K8sPANPBM4Mkk19C748jpwEeTHHivP6yqT3Sb3gh8MMnrgK8Ar2n70SRJkqTFN9A52FW1Ddg2\np21T3/JD9E4dmetR4IUH2ebfAy8fuFJJkiRpCfBJjpIkSVJDBmxJkiSpocW4TZ8kSVIz43QrRG+D\nqPl4BFuSJElqyIAtSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhryLiCRprI3THSPAu0ZIOjyPYEuS\nJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJamig2/QlWQO8C1gGvKeqNs7pPxt4\nH3Ae8B+r6h1d+0rgRuAMoIDNVfWuru+twP8JzHabeXNVbRv2A0mSDm2cbnvnLe8kHYsOG7CTLAOu\nA14J7Aa2J9laVff0Dfsa8AbgVXOm7wfeVFV3JHkGsCPJrX1zf+dAGJckSZKOBYOcInI+MFNV91fV\n48DNwLr+AVW1r6q2A9+Z0763qu7olr8B3Assb1K5JEmSNIYGCdjLgQf71nezgJCcZBI4F/h8X/Pr\nk+xMsiXJaUe6TUmSJGncLMpFjklOAT4MXFNVj3bN1wPPBc4B9gK/fZC565NMJ5menZ2db4gkSZI0\nNgYJ2HuAlX3rK7q2gSQ5kV64vqmqPnKgvaoerqonqupJ4N30TkV5iqraXFVTVTU1MTEx6NtKkiRJ\nIzFIwN4OrEpyVpKTgIuBrYNsPEmA9wL3VtU75/Sd2bf6auDuwUqWJEmSxtdh7yJSVfuTXA3cQu82\nfVuqaleSK7v+TUl+GJgGngk8meQaYDXwAuBS4K4kd3abPHA7vrclOYfe7fseAH657UeTJEmSFt9A\n98HuAvG2OW2b+pYfonfqyFyfBXKQbV46eJmSJEnS0uCTHCVJkqSGDNiSJElSQwZsSZIkqSEDtiRJ\nktSQAVuSJElqyIAtSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLU\nkAFbkiRJasiALUmSJDVkwJYkSZIaMmBLkiRJDQ0UsJOsSXJfkpkkG+bpPzvJ55J8O8mvDTI3ybOT\n3Jrky93P04b/OJIkSdJoHTZgJ1kGXAesBVYDlyRZPWfY14A3AO84grkbgNuqahVwW7cuSZIkLWmD\nHME+H5ipqvur6nHgZmBd/4Cq2ldV24HvHMHcdcAN3fINwKsW+BkkSZKksTFIwF4OPNi3vrtrG8Sh\n5p5RVXu75YeAM+bbQJL1SaaTTM/Ozg74tpIkSdJojMVFjlVVQB2kb3NVTVXV1MTExCJXJkmSJB2Z\nQQL2HmBl3/qKrm0Qh5r7cJIzAbqf+wbcpiRJkjS2BgnY24FVSc5KchJwMbB1wO0fau5W4PJu+XLg\nY4OXLUmSJI2nEw43oKr2J7kauAVYBmypql1Jruz6NyX5YWAaeCbwZJJrgNVV9eh8c7tNbwQ+mOR1\nwFeA17T+cJIkSdJiO2zABqiqbcC2OW2b+pYfonf6x0Bzu/a/B15+JMVKkiRJ424sLnKUJEmSjhUG\nbEmSJKkhA7YkSZLUkAFbkiRJasiALUmSJDU00F1EJEmStDCTGz4+6hK+64GNF426hOOCR7AlSZKk\nhjyCLUlD8uiUJKmfR7AlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmSJDVkwJYk\nSZIaGihgJ1mT5L4kM0k2zNOfJNd2/TuTnNe1Py/JnX2vR5Nc0/W9Ncmevr4L2340SZIkafEd9kEz\nSZYB1wGvBHYD25Nsrap7+oatBVZ1rxcB1wMvqqr7gHP6trMH+GjfvN+pqne0+CCSJEnSOBjkCPb5\nwExV3V9VjwM3A+vmjFkH3Fg9twOnJjlzzpiXA39TVV8ZumpJkiRpTA0SsJcDD/at7+7ajnTMxcAH\n5rS9vjulZEuS0+Z78yTrk0wnmZ6dnR2gXEmSJGl0FuUixyQnAT8L/FFf8/XAc+mdQrIX+O355lbV\n5qqaqqqpiYmJo16rJEmSNIxBAvYeYGXf+oqu7UjGrAXuqKqHDzRU1cNV9URVPQm8m96pKJIkSdKS\nNkjA3g6sSnJWdyT6YmDrnDFbgcu6u4lcADxSVXv7+i9hzukhc87RfjVw9xFXL0mSJI2Zw95FpKr2\nJ7kauAVYBmypql1Jruz6NwHbgAuBGeAx4IoD85OcTO8OJL88Z9NvS3IOUMAD8/RLkiRJS85hAzZA\nVW2jF6L72zb1LRdw1UHmfgv4oXnaLz2iSiVJkqQlwCc5SpIkSQ0ZsCVJkqSGDNiSJElSQwZsSZIk\nqSEDtiRJktSQAVuSJElqyIAtSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkh\nA7YkSZLUkAFbkiRJauiEURcgSXNNbvj4qEv4rgc2XjTqEiRJS8xAR7CTrElyX5KZJBvm6U+Sa7v+\nnUnO6+t7IMldSe5MMt3X/uwktyb5cvfztDYfSZIkSRqdwwbsJMuA64C1wGrgkiSr5wxbC6zqXuuB\n6+f0/2RVnVNVU31tG4DbqmoVcFu3LkmSJC1pgxzBPh+Yqar7q+px4GZg3Zwx64Abq+d24NQkZx5m\nu+uAG7rlG4BXHUHdkiRJ0lgaJGAvBx7sW9/dtQ06poA/TbIjyfq+MWdU1d5u+SHgjPnePMn6JNNJ\npmdnZwcoV5IkSRqdxbiLyEuq6hx6p5FcleSlcwdUVdEL4k9RVZuraqqqpiYmJo5yqZIkSdJwBgnY\ne4CVfesruraBxlTVgZ/7gI/SO+UE4OEDp5F0P/cdafGSJEnSuBkkYG8HViU5K8lJwMXA1jljtgKX\ndXcTuQB4pKr2Jjk5yTMAkpwM/DRwd9+cy7vly4GPDflZJEmSpJE77H2wq2p/kquBW4BlwJaq2pXk\nyq5/E7ANuBCYAR4DruimnwF8NMmB9/rDqvpE17cR+GCS1wFfAV7T7FNJkiRJIzLQg2aqahu9EN3f\ntqlvuYCr5pl3P/DCg2zz74GXH0mxkiRJ0rjzUemSJElSQwZsSZIkqSEDtiRJktSQAVuSJElqyIAt\nSZIkNWTAliRJkhoyYEuSJEkNGbAlSZKkhgzYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiALUmS\nJDVkwJYkSZIaOmHUBUg6+iY3fHzUJXzXAxsvGnUJkiQdVQMdwU6yJsl9SWaSbJinP0mu7fp3Jjmv\na1+Z5FNJ7kmyK8kb++a8NcmeJHd2rwvbfSxJkiRpNA57BDvJMuA64JXAbmB7kq1VdU/fsLXAqu71\nIuD67ud+4E1VdUeSZwA7ktzaN/d3quod7T6OJEmSNFqDHME+H5ipqvur6nHgZmDdnDHrgBur53bg\n1CRnVtXeqroDoKq+AdwLLG9YvyRJkjRWBgnYy4EH+9Z389SQfNgxSSaBc4HP9zW/vjulZEuS0was\nWZIkSRpbi3IXkSSnAB8GrqmqR7vm64HnAucAe4HfPsjc9Ummk0zPzs4uRrmSJEnSgg0SsPcAK/vW\nV3RtA41JciK9cH1TVX3kwICqeriqnqiqJ4F30zsV5SmqanNVTVXV1MTExADlSpIkSaMzSMDeDqxK\nclaSk4CLga1zxmwFLuvuJnIB8EhV7U0S4L3AvVX1zv4JSc7sW301cPeCP4UkSZI0Jg57F5Gq2p/k\nauAWYBmwpap2Jbmy698EbAMuBGaAx4AruukvBi4F7kpyZ9f25qraBrwtyTlAAQ8Av9zsU0mSJEkj\nMtCDZrpAvG1O26a+5QKummfeZ4EcZJuXHlGlkiRJ0hLgo9IlSZKkhgzYkiRJUkMGbEmSJKkhA7Yk\nSZLU0EAXOUr6fpMbPj7qEr7rgY0XjboESZLUxyPYkiRJUkMGbEmSJKkhA7YkSZLUkAFbkiRJasiA\nLUmSJDVkwJYkSZIaMmBLkiRJDRmwJUmSpIZ80IzGgg9ukSRJxwqPYEuSJEkNDXQEO8ka4F3AMuA9\nVbVxTn+6/guBx4DXVtUdh5qb5NnAfwcmgQeA11TVPwz/kSRJkrRQ/lV5eIc9gp1kGXAdsBZYDVyS\nZPWcYWuBVd1rPXD9AHM3ALdV1Sr+//buP9Tuuo7j+PMV036ypluMbf6YxCRuEc7WWtYfhoVbEMsi\nmf84yoqFglJBiSiIQgssSkTlgoKjH/7IoiuJ0WQWq8zGlP1Q5zZTnC1HSs0tcg7f/fH5HDvdzr07\n557P93y/59zXA77s3HO+388+n9cO389n3+/nfj/wcP7ZzMzMzGyodTNFZCWwLyKejYhjwN3A2kn7\nrAU2RfIoME/SohMcuxa4K7++C/hsn20xMzMzM6tdN1NElgAvtP18APhIF/ssOcGxCyPiYH79N2Bh\nl3W2E2jSrR0Y3ts7ZmZmZjPRiKeIRERIik6fSfoqadoJwBFJewZXs+IWAH+vuxKDpu8O7K8qku8A\n61uE863WgOvbd8bOd1r+DlfL+VbL+VZvcsZnzrSgbgbYLwKnt/18Wn6vm31OmubYlyQtioiDeTrJ\noU5/eUSMA+Nd1LPxJG2LiBV112NUOd9qOd/qOeNqOd9qOd9qOd/qlcy4mznYfwaWSTpL0snAOmBi\n0j4TwKVKVgH/zNM/pjt2AlifX68HftlnW8zMzMzManfCK9gRcVzSFcCvSY/auzMidkvakD+/HXiQ\n9Ii+faTH9H1xumNz0RuBeyVdBjwPXFy0ZWZmZmZmNehqDnZEPEgaRLe/d3vb6wAu7/bY/P7LwAW9\nVHYEjMRUlwZzvtVyvtVzxtVyvtVyvtVyvtUrlrHS2NjMzMzMzErwUulmZmZmZgV5gN0HSadL2iLp\nSUm7JV2Z3z9V0m8k7c1/ntJ2zNWS9knaI+nCDmVOSNo1yHY0Vcl8JZ0saVzSM5KelvT5OtrUJIXz\nvUTSTkk7JD0kaUEdbWqaXjOWND/vf0TSLZPK+lDOeJ+kmyWpjjY1Sal8Jb1D0q/yuWG3pI11talJ\nSn5/28p0H5cVPj+4j+ugcMY99XMeYPfnOPCNiBgDVgGXKy0F33EZ+PzZOuD9wGrgVqXl5Mmffw44\nMtgmNFrJfK8BDkXE2cAY8NuBtqSZiuQraQ7wQ+ATEfFBYAdwxcBb00w9ZQz8G7gW+GaHsm4DvgIs\ny9vqius+DErme1NEvA9YDnxM0prKa998JfN1H/f/SubrPq6zIhnPpJ/zALsPEXEwIrbn168CT5FW\nr5xqGfi1wN0R8VpE/IX01JWVAJLeBXwduHFwLWi2kvkCXwK+k8t6IyJm3YI/kxXMV3l7Z76qOhf4\n68Aa0mC9ZhwRRyNiK+kk/yaltQLmRsSj+ZfKN/Hff5dZq1S+EfGviNiSXx8DtpPWbZjVSuUL7uM6\nKZkv7uM6Kphxz/2cB9iFSFpKuvLxJ6ZeBn6qJeUBbgC+R3rMoU3ST76S5uWfb5C0XdJ9khZib+on\n34h4HfgasJN0whkD7qi+1sOly4ynsoSUd0v7ucPoO9/2cuYBnyFd1bKsQL7u46bRT77u47rTT8Yz\n6ec8wC4g/8/8fuCqiDjc/lm+2jTto1oknQO8NyJ+UV0th1e/+ZIeR3ka8IeIOBf4I3BTFXUdRgW+\nvyeRTjzLgcWkW2dXV1Pb4VTgO2zTKJVvvg38U+DmiHi2eEWHlPu4armPq14d/ZwH2H3Kod8P/Dgi\nfp7ffinf0m3d2m0tAz/VkvIfBVZIeg7YCpwt6ZHqa998hfJ9mXTVpHX8fcC5FVd9KBTK9xyAiNif\nT1T3AucNoPpDoceMp/Ii/ztloZX9rFco35ZxYG9E/KB8TYdToXzdx02hUL7u46ZRKOOe+zkPsPuQ\n5+HcATwVEd9v+2iqZeAngHWS3irpLNIvKj0WEbdFxOKIWAp8HHgmIs4fRBuarGC+ATwAnJ/3uwB4\nsuLqN16pfEkDvTFJ78n7fYo0z23Wm0HGHeVbmYclrcplXnqiY2aDUvnmsm4E3g1cVbqew6rg99d9\nXAcF83UfN4WC54je+7mI8DbDjXSiCNKtgify9mlgPmn+3l5gM3Bq2zHXAPuBPcCaDmUuBXbV3bYm\nbCXzBc4EfpfLehg4o+721b0VzndDPtnsIJ3o59fdviZsM8z4OeAV0tMWDgBj+f0VwK6c/y3khcJm\n81YqX9Idgcjf4VY5X667fXVvJb+/bZ+7j6sgX/dxA8m4p37OKzmamZmZmRXkKSJmZmZmZgV5gG1m\nZmZmVpAH2GZmZmZmBXmAbWZmZmZWkAfYZmZmZmYFeYBtZjZklGyVtKbtvS9IeqjOepmZWeLH9JmZ\nDSFJHyCt2LactFTy48DqiNjfR5lzIuJ4oSqamc1avoJtZjaEImIXabGDbwHXAZsiYr+k9ZIek/SE\npFslvQVA0rikbZJ2S7quVY6kA5I2SnocuKiWxpiZjZg5dVfAzMxm7HpgO3AMWJGval8EnBcRxyWN\nA+uAnwDfjohXJM0Btkj6WUS0llM+FBHL62iAmdko8gDbzGxIRcRRSfcARyLiNUmfBD4MbJME8Hbg\nhbz7JZIuI533F5OWCG8NsO8ZbM3NzEabB9hmZsPtjbwBCLgzIq5t30HSMuBKYGVE/EPSj4C3te1y\ndCA1NTObJTwH28xsdGwGLpa0AEDSfElnAHOBV4HDkhYBF9ZYRzOzkecr2GZmIyIidkq6Hticf7nx\ndWADsI00HeRp4Hng9/XV0sxs9PkxfWZmZmZmBXmKiJmZmZlZQR5gm5mZmZkV5AG2mZmZmVlBHmCb\nmZmZmRXkAbaZmZmZWUEeYJuZmZmZFeQBtpmZmZlZQR5gm5mZmZkV9B/+FPej/ecY9wAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x20839f990>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.figure(figsize=(12,10))\n",
    "\n",
    "plt.subplot(311)\n",
    "frequ_values = (users.yelping_since_day.value_counts()/users.yelping_since_day.value_counts().sum()).sort_index()\n",
    "plt.bar(frequ_values.index,frequ_values)\n",
    "plt.xlabel(\"Day of the month\",verticalalignment='top')\n",
    "\n",
    "plt.subplot(312)\n",
    "frequ_values = (users.yelping_since_month.value_counts()/users.yelping_since_month.value_counts().sum()).sort_index()\n",
    "plt.bar(frequ_values.index,frequ_values)\n",
    "plt.xlabel(\"Month of the year\",verticalalignment='top')\n",
    "\n",
    "plt.subplot(313)\n",
    "frequ_values = (users.yelping_since_year.value_counts()/users.yelping_since_year.value_counts().sum()).sort_index()\n",
    "plt.bar(frequ_values.index,frequ_values)\n",
    "plt.xlabel(\"Year\",verticalalignment='top')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>number_nan</th>\n",
       "      <th>types</th>\n",
       "      <th>number_distinct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>average_stars</th>\n",
       "      <td>3.709147</td>\n",
       "      <td>1.097177</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.13</td>\n",
       "      <td>3.88</td>\n",
       "      <td>4.55</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>float64</td>\n",
       "      <td>398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yelping_since_year</th>\n",
       "      <td>2012.569855</td>\n",
       "      <td>2.306572</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2011.00</td>\n",
       "      <td>2013.00</td>\n",
       "      <td>2014.00</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int32</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yelping_since_month</th>\n",
       "      <td>6.420439</td>\n",
       "      <td>3.373801</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.00</td>\n",
       "      <td>6.00</td>\n",
       "      <td>9.00</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int32</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yelping_since_day</th>\n",
       "      <td>15.741980</td>\n",
       "      <td>8.792139</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.00</td>\n",
       "      <td>16.00</td>\n",
       "      <td>23.00</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int32</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            mean       std     min      25%      50%      75%  \\\n",
       "average_stars           3.709147  1.097177     1.0     3.13     3.88     4.55   \n",
       "yelping_since_year   2012.569855  2.306572  2004.0  2011.00  2013.00  2014.00   \n",
       "yelping_since_month     6.420439  3.373801     1.0     4.00     6.00     9.00   \n",
       "yelping_since_day      15.741980  8.792139     1.0     8.00    16.00    23.00   \n",
       "\n",
       "                        max  number_nan    types  number_distinct  \n",
       "average_stars           5.0         0.0  float64              398  \n",
       "yelping_since_year   2017.0         0.0    int32               14  \n",
       "yelping_since_month    12.0         0.0    int32               12  \n",
       "yelping_since_day      31.0         0.0    int32               31  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getDfSummary(users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "user_variables = ['user_id','average_stars','yelping_since_year']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Business' Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>categories</th>\n",
       "      <th>city</th>\n",
       "      <th>is_open</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>name</th>\n",
       "      <th>neighborhood</th>\n",
       "      <th>stars</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>144069</th>\n",
       "      <td>5GslSGCDT3gXei0kKR-P3g</td>\n",
       "      <td>[Food, Ice Cream &amp; Frozen Yogurt]</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>33.626005</td>\n",
       "      <td>-111.977799</td>\n",
       "      <td>Yin Yang Self Serve Frozen Yogurt</td>\n",
       "      <td></td>\n",
       "      <td>4.5</td>\n",
       "      <td>AZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144070</th>\n",
       "      <td>z0GwhlBflcCnVdACU9tQxQ</td>\n",
       "      <td>[Restaurants, Hot Dogs]</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>33.640348</td>\n",
       "      <td>-111.969566</td>\n",
       "      <td>Famous Uncle Al's Hot Dogs and Grille</td>\n",
       "      <td></td>\n",
       "      <td>4.0</td>\n",
       "      <td>AZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144071</th>\n",
       "      <td>a048R60CLFeObJ3mhHdPRg</td>\n",
       "      <td>[Shopping, Mobile Phones]</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>1</td>\n",
       "      <td>33.602173</td>\n",
       "      <td>-111.983636</td>\n",
       "      <td>T-Mobile</td>\n",
       "      <td></td>\n",
       "      <td>3.0</td>\n",
       "      <td>AZ</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   business_id                         categories     city  \\\n",
       "144069  5GslSGCDT3gXei0kKR-P3g  [Food, Ice Cream & Frozen Yogurt]  Phoenix   \n",
       "144070  z0GwhlBflcCnVdACU9tQxQ            [Restaurants, Hot Dogs]  Phoenix   \n",
       "144071  a048R60CLFeObJ3mhHdPRg          [Shopping, Mobile Phones]  Phoenix   \n",
       "\n",
       "        is_open   latitude   longitude                                   name  \\\n",
       "144069        0  33.626005 -111.977799      Yin Yang Self Serve Frozen Yogurt   \n",
       "144070        0  33.640348 -111.969566  Famous Uncle Al's Hot Dogs and Grille   \n",
       "144071        1  33.602173 -111.983636                               T-Mobile   \n",
       "\n",
       "       neighborhood  stars state  \n",
       "144069                 4.5    AZ  \n",
       "144070                 4.0    AZ  \n",
       "144071                 3.0    AZ  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "businesses.tail(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>number_nan</th>\n",
       "      <th>types</th>\n",
       "      <th>number_distinct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>is_open</th>\n",
       "      <td>0.850304</td>\n",
       "      <td>0.356774</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int64</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latitude</th>\n",
       "      <td>38.644758</td>\n",
       "      <td>5.344079</td>\n",
       "      <td>32.998019</td>\n",
       "      <td>33.627346</td>\n",
       "      <td>36.143417</td>\n",
       "      <td>43.611192</td>\n",
       "      <td>57.592285</td>\n",
       "      <td>0.0</td>\n",
       "      <td>float64</td>\n",
       "      <td>115313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>longitude</th>\n",
       "      <td>-92.690002</td>\n",
       "      <td>26.898355</td>\n",
       "      <td>-122.397206</td>\n",
       "      <td>-112.136907</td>\n",
       "      <td>-89.523207</td>\n",
       "      <td>-79.649855</td>\n",
       "      <td>115.086769</td>\n",
       "      <td>0.0</td>\n",
       "      <td>float64</td>\n",
       "      <td>115691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stars</th>\n",
       "      <td>3.637709</td>\n",
       "      <td>0.969585</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>float64</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                mean        std         min         25%        50%        75%  \\\n",
       "is_open     0.850304   0.356774    0.000000    1.000000   1.000000   1.000000   \n",
       "latitude   38.644758   5.344079   32.998019   33.627346  36.143417  43.611192   \n",
       "longitude -92.690002  26.898355 -122.397206 -112.136907 -89.523207 -79.649855   \n",
       "stars       3.637709   0.969585    1.000000    3.000000   3.500000   4.500000   \n",
       "\n",
       "                  max  number_nan    types  number_distinct  \n",
       "is_open      1.000000         0.0    int64                2  \n",
       "latitude    57.592285         0.0  float64           115313  \n",
       "longitude  115.086769         0.0  float64           115691  \n",
       "stars        5.000000         0.0  float64                9  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "getDfSummary(businesses)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>categories</th>\n",
       "      <th>is_open</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>name</th>\n",
       "      <th>stars</th>\n",
       "      <th>state_AZ</th>\n",
       "      <th>state_BW</th>\n",
       "      <th>state_EDH</th>\n",
       "      <th>...</th>\n",
       "      <th>neighborhood_Willowdale</th>\n",
       "      <th>neighborhood_Windemere</th>\n",
       "      <th>neighborhood_Windgap</th>\n",
       "      <th>neighborhood_Woodbridge</th>\n",
       "      <th>neighborhood_Woodland Hills</th>\n",
       "      <th>neighborhood_Worthington Park</th>\n",
       "      <th>neighborhood_Wychwood</th>\n",
       "      <th>neighborhood_Yonge and Eglinton</th>\n",
       "      <th>neighborhood_Yonge and St. Clair</th>\n",
       "      <th>neighborhood_Yorkville</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>144069</th>\n",
       "      <td>5GslSGCDT3gXei0kKR-P3g</td>\n",
       "      <td>[Food, Ice Cream &amp; Frozen Yogurt]</td>\n",
       "      <td>0</td>\n",
       "      <td>33.626005</td>\n",
       "      <td>-111.977799</td>\n",
       "      <td>Yin Yang Self Serve Frozen Yogurt</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144070</th>\n",
       "      <td>z0GwhlBflcCnVdACU9tQxQ</td>\n",
       "      <td>[Restaurants, Hot Dogs]</td>\n",
       "      <td>0</td>\n",
       "      <td>33.640348</td>\n",
       "      <td>-111.969566</td>\n",
       "      <td>Famous Uncle Al's Hot Dogs and Grille</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144071</th>\n",
       "      <td>a048R60CLFeObJ3mhHdPRg</td>\n",
       "      <td>[Shopping, Mobile Phones]</td>\n",
       "      <td>1</td>\n",
       "      <td>33.602173</td>\n",
       "      <td>-111.983636</td>\n",
       "      <td>T-Mobile</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 1292 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   business_id                         categories  is_open  \\\n",
       "144069  5GslSGCDT3gXei0kKR-P3g  [Food, Ice Cream & Frozen Yogurt]        0   \n",
       "144070  z0GwhlBflcCnVdACU9tQxQ            [Restaurants, Hot Dogs]        0   \n",
       "144071  a048R60CLFeObJ3mhHdPRg          [Shopping, Mobile Phones]        1   \n",
       "\n",
       "         latitude   longitude                                   name  stars  \\\n",
       "144069  33.626005 -111.977799      Yin Yang Self Serve Frozen Yogurt    4.5   \n",
       "144070  33.640348 -111.969566  Famous Uncle Al's Hot Dogs and Grille    4.0   \n",
       "144071  33.602173 -111.983636                               T-Mobile    3.0   \n",
       "\n",
       "        state_AZ  state_BW  state_EDH           ...            \\\n",
       "144069         1         0          0           ...             \n",
       "144070         1         0          0           ...             \n",
       "144071         1         0          0           ...             \n",
       "\n",
       "        neighborhood_Willowdale  neighborhood_Windemere  neighborhood_Windgap  \\\n",
       "144069                        0                       0                     0   \n",
       "144070                        0                       0                     0   \n",
       "144071                        0                       0                     0   \n",
       "\n",
       "        neighborhood_Woodbridge  neighborhood_Woodland Hills  \\\n",
       "144069                        0                            0   \n",
       "144070                        0                            0   \n",
       "144071                        0                            0   \n",
       "\n",
       "        neighborhood_Worthington Park  neighborhood_Wychwood  \\\n",
       "144069                              0                      0   \n",
       "144070                              0                      0   \n",
       "144071                              0                      0   \n",
       "\n",
       "        neighborhood_Yonge and Eglinton  neighborhood_Yonge and St. Clair  \\\n",
       "144069                                0                                 0   \n",
       "144070                                0                                 0   \n",
       "144071                                0                                 0   \n",
       "\n",
       "        neighborhood_Yorkville  \n",
       "144069                       0  \n",
       "144070                       0  \n",
       "144071                       0  \n",
       "\n",
       "[3 rows x 1292 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "dummies_cities = pd.get_dummies(businesses['state'], dummy_na=False, prefix='state', prefix_sep='_')\n",
    "businesses_dumm = pd.concat([businesses, dummies_cities], axis=1)\n",
    "\n",
    "dummies_states = pd.get_dummies(businesses['city'], dummy_na=False, prefix='city', prefix_sep='_')\n",
    "businesses_dumm = pd.concat([businesses_dumm, dummies_states], axis=1)\n",
    "\n",
    "dummies_neighborhood = pd.get_dummies(businesses['neighborhood'], dummy_na=False, prefix='neighborhood', prefix_sep='_')\n",
    "businesses_dumm = pd.concat([businesses_dumm, dummies_neighborhood], axis=1)\n",
    "\n",
    "businesses_dumm = businesses_dumm.drop(['city','state','neighborhood'],axis=1)\n",
    "businesses_dumm.tail(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>is_open</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>name</th>\n",
       "      <th>stars</th>\n",
       "      <th>state_AZ</th>\n",
       "      <th>state_BW</th>\n",
       "      <th>state_EDH</th>\n",
       "      <th>state_ELN</th>\n",
       "      <th>...</th>\n",
       "      <th>catBiz_Wine Tours</th>\n",
       "      <th>catBiz_Wineries</th>\n",
       "      <th>catBiz_Wok</th>\n",
       "      <th>catBiz_Women's Clothing</th>\n",
       "      <th>catBiz_Workers Compensation Law</th>\n",
       "      <th>catBiz_Wraps</th>\n",
       "      <th>catBiz_Yelp Events</th>\n",
       "      <th>catBiz_Yoga</th>\n",
       "      <th>catBiz_Ziplining</th>\n",
       "      <th>catBiz_Zoos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>144069</th>\n",
       "      <td>5GslSGCDT3gXei0kKR-P3g</td>\n",
       "      <td>0</td>\n",
       "      <td>33.626005</td>\n",
       "      <td>-111.977799</td>\n",
       "      <td>Yin Yang Self Serve Frozen Yogurt</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144070</th>\n",
       "      <td>z0GwhlBflcCnVdACU9tQxQ</td>\n",
       "      <td>0</td>\n",
       "      <td>33.640348</td>\n",
       "      <td>-111.969566</td>\n",
       "      <td>Famous Uncle Al's Hot Dogs and Grille</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144071</th>\n",
       "      <td>a048R60CLFeObJ3mhHdPRg</td>\n",
       "      <td>1</td>\n",
       "      <td>33.602173</td>\n",
       "      <td>-111.983636</td>\n",
       "      <td>T-Mobile</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 2482 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   business_id  is_open   latitude   longitude  \\\n",
       "144069  5GslSGCDT3gXei0kKR-P3g        0  33.626005 -111.977799   \n",
       "144070  z0GwhlBflcCnVdACU9tQxQ        0  33.640348 -111.969566   \n",
       "144071  a048R60CLFeObJ3mhHdPRg        1  33.602173 -111.983636   \n",
       "\n",
       "                                         name  stars  state_AZ  state_BW  \\\n",
       "144069      Yin Yang Self Serve Frozen Yogurt    4.5         1         0   \n",
       "144070  Famous Uncle Al's Hot Dogs and Grille    4.0         1         0   \n",
       "144071                               T-Mobile    3.0         1         0   \n",
       "\n",
       "        state_EDH  state_ELN     ...       catBiz_Wine Tours  catBiz_Wineries  \\\n",
       "144069          0          0     ...                     0.0              0.0   \n",
       "144070          0          0     ...                     0.0              0.0   \n",
       "144071          0          0     ...                     0.0              0.0   \n",
       "\n",
       "        catBiz_Wok  catBiz_Women's Clothing  catBiz_Workers Compensation Law  \\\n",
       "144069         0.0                      0.0                              0.0   \n",
       "144070         0.0                      0.0                              0.0   \n",
       "144071         0.0                      0.0                              0.0   \n",
       "\n",
       "        catBiz_Wraps  catBiz_Yelp Events  catBiz_Yoga  catBiz_Ziplining  \\\n",
       "144069           0.0                 0.0          0.0               0.0   \n",
       "144070           0.0                 0.0          0.0               0.0   \n",
       "144071           0.0                 0.0          0.0               0.0   \n",
       "\n",
       "        catBiz_Zoos  \n",
       "144069          0.0  \n",
       "144070          0.0  \n",
       "144071          0.0  \n",
       "\n",
       "[3 rows x 2482 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "'''\n",
    ".apply(Series)    converts the series of lists to a dataframe\n",
    ".stack()          puts everything in one column again (creating a multi-level index)\n",
    "pd.get_dummies( ) creating the dummies\n",
    ".sum(level=0)     for remerging the different rows that should be one row \n",
    "                  (by summing up the second level, only keeping the original level (level=0))               \n",
    "'''\n",
    "\n",
    "dummies_categories = pd.get_dummies(businesses['categories'].apply(pd.Series).stack()).sum(level=0)\n",
    "\n",
    "prefix = 'catBiz_'\n",
    "new_columns = []\n",
    "for i in dummies_categories.columns:\n",
    "    new_columns.append( prefix+i )\n",
    "    \n",
    "dummies_categories.columns = new_columns\n",
    "    \n",
    "    \n",
    "businesses_sparse = pd.concat([businesses_dumm, dummies_categories], axis=1)\n",
    "businesses_sparse = businesses_sparse.drop(['categories'],axis=1)\n",
    "businesses_sparse.tail(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "\n",
    "business_variables = []\n",
    "for i in businesses_sparse.columns:\n",
    "    if 'catBiz' in i or 'neighborhood' in i or 'state' in i or 'city' in i or 'name' in i \\\n",
    "       or 'latitude' in i or 'longitude' in i or 'stars' in i or 'is_open' in i or 'id' in i:\n",
    "        business_variables.append(i)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Categories counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     [Tobacco Shops, Nightlife, Vape Shops, Shopping]\n",
       "1    [Caterers, Grocery, Food, Event Planning & Ser...\n",
       "2         [Restaurants, Pizza, Chicken Wings, Italian]\n",
       "3    [Hair Removal, Beauty & Spas, Blow Dry/Out Ser...\n",
       "4    [Hotels & Travel, Event Planning & Services, H...\n",
       "Name: categories, dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "blended_cats = businesses['categories']\n",
    "blended_cats.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "from collections import Counter\n",
    "distinct_cats = Counter()\n",
    "for line in blended_cats: \n",
    "    if line!= None:\n",
    "        for cat in line: \n",
    "            distinct_cats[cat] +=1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1191\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(len(distinct_cats)) #Too many to create a category per each one\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reviews' Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>date</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4153147</th>\n",
       "      <td>KYasaF1nov1bn7phfSgWeg</td>\n",
       "      <td>2015-08-15</td>\n",
       "      <td>GXQnlRo41C4WykC_IP2Lbw</td>\n",
       "      <td>5</td>\n",
       "      <td>Tui was awesome ^^ she made me an amazing jack...</td>\n",
       "      <td>rWL8bGLRPtYAhX97JnRVXg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4153148</th>\n",
       "      <td>KYasaF1nov1bn7phfSgWeg</td>\n",
       "      <td>2016-08-18</td>\n",
       "      <td>tTGXK14g6WKJx78JG3bh_w</td>\n",
       "      <td>3</td>\n",
       "      <td>Overpriced coffee without the free upgrade at ...</td>\n",
       "      <td>nS19cEYuD99dvTCYISf0oA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4153149</th>\n",
       "      <td>KYasaF1nov1bn7phfSgWeg</td>\n",
       "      <td>2014-05-26</td>\n",
       "      <td>EmuqmSacByt96t8G5GK0KQ</td>\n",
       "      <td>4</td>\n",
       "      <td>If ur into Asian desserts this is the place to...</td>\n",
       "      <td>LgpDa5gAoK2q3u4zrjdr-w</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    business_id        date               review_id  stars  \\\n",
       "4153147  KYasaF1nov1bn7phfSgWeg  2015-08-15  GXQnlRo41C4WykC_IP2Lbw      5   \n",
       "4153148  KYasaF1nov1bn7phfSgWeg  2016-08-18  tTGXK14g6WKJx78JG3bh_w      3   \n",
       "4153149  KYasaF1nov1bn7phfSgWeg  2014-05-26  EmuqmSacByt96t8G5GK0KQ      4   \n",
       "\n",
       "                                                      text  \\\n",
       "4153147  Tui was awesome ^^ she made me an amazing jack...   \n",
       "4153148  Overpriced coffee without the free upgrade at ...   \n",
       "4153149  If ur into Asian desserts this is the place to...   \n",
       "\n",
       "                        user_id  \n",
       "4153147  rWL8bGLRPtYAhX97JnRVXg  \n",
       "4153148  nS19cEYuD99dvTCYISf0oA  \n",
       "4153149  LgpDa5gAoK2q3u4zrjdr-w  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "reviews.tail(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "name_datetime_variable = 'date'\n",
    "reviews[name_datetime_variable] = pd.to_datetime(reviews[name_datetime_variable])\n",
    "reviews[name_datetime_variable+'_year'] = pd.DatetimeIndex(reviews[name_datetime_variable]).year\n",
    "reviews[name_datetime_variable+'_month'] = pd.DatetimeIndex(reviews[name_datetime_variable]).month\n",
    "reviews[name_datetime_variable+'_day'] = pd.DatetimeIndex(reviews[name_datetime_variable]).day\n",
    "\n",
    "#reviews = reviews.drop('date',axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtIAAAJQCAYAAACjEoN5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3X+0XWdd7/v3h6QVKD8CNPbUpHVXb4SRw4W25oQKFUGB\nk7Reg0fvsb1IoerIqbYCXj0auWMoHD3HKhwu9lKbE6FAB4VeflVzbKT8EAQuFJKWkjYthW2oNrHQ\nIFIoVUro9/6xZuhidyd75snae62dvF9jrJE5n/k8c33XnHvt9cncc82ZqkKSJEnS4XnEuAuQJEmS\nFiODtCRJktTAIC1JkiQ1MEhLkiRJDQzSkiRJUgODtCRJktTAIC1JkiQ1MEhLkiRJDQzSkiRJUoOl\n4y5gNieeeGJNTU2NuwxJkiQdxW688cavVNXy1vETGaSnpqbYsWPHuMuQJEnSUSzJ3x/JeE/tkCRJ\nkhoYpCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQGE3n5O0k6Gkxtuu6Ixt956bkjquTh\nRl3bpK9PkuaDQVrHhCP5UPYDXpPAnztp8RnlZ48mk0FaE8nQoHHw506SdDgM0vNokv7U6VHVyTVJ\nPyejXt98/5z4c6y+JvnnWNLiZZCWJOkwTNJ/VhdifccSt50Ol1ftkCRJkhp4RFoj4f/i1Yc/J9Li\n42kx0sEZpCVJkiacByImk0F6iD+kkiQtHn5ua9w8R1qSJElqYJCWJEmSGnhqhyRJWhCeijE53Bej\n0euIdJJ1Se5IMp1k0yzLk+SybvnOJGd27Y9M8ukkn02yK8mrR/0CJEmSpHGYM0gnWQJcDqwHVgPn\nJ1k9o9t6YFX32Ahc0bV/C/jJqno6cDqwLslZI6pdkiRJGps+R6TXAtNVtbuqHgCuATbM6LMBuKoG\nbgCWJTm5m7+v63Nc96hRFS9JkiSNS58gvQK4a2h+T9fWq0+SJUluBu4BPlBVn2ovV5IkSZoM837V\njqr6TlWdDqwE1iZ56mz9kmxMsiPJjn379s13WZIkSdIR6ROk9wKnDM2v7NoOq09VfQ34MLButiep\nqi1Vtaaq1ixfvrxHWZIkSdL49AnS24FVSU5LcjxwHrB1Rp+twAXd1TvOAu6tqruTLE+yDCDJo4Dn\nA58bYf2SJEnSWMx5Hemq2p/kEuB6YAlwZVXtSnJRt3wzsA04B5gG7gcu7IafDLy1u/LHI4B3VtVf\njf5lSJIkSQur1w1Zqmobg7A83LZ5aLqAi2cZtxM44whrlCRJkiaOtwiXJEmSGhikJUmSpAYGaUmS\nJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmB\nQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqS\nJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElq\nYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElqYJCW\nJEmSGvQK0knWJbkjyXSSTbMsT5LLuuU7k5zZtZ+S5MNJbkuyK8nLR/0CJEmSpHGYM0gnWQJcDqwH\nVgPnJ1k9o9t6YFX32Ahc0bXvB36zqlYDZwEXzzJWkiRJWnT6HJFeC0xX1e6qegC4Btgwo88G4Koa\nuAFYluTkqrq7qm4CqKpvALcDK0ZYvyRJkjQWfYL0CuCuofk9PDwMz9knyRRwBvCp2Z4kycYkO5Ls\n2LdvX4+yJEmSpPFZuhBPkuQxwHuAV1TV12frU1VbgC0Aa9asqYWoS5IkSUduatN1RzT+zkvPHVEl\nC6vPEem9wClD8yu7tl59khzHIERfXVXvbS9VkiRJmhx9gvR2YFWS05IcD5wHbJ3RZytwQXf1jrOA\ne6vq7iQB3gTcXlWvG2nlkiRJ0hjNeWpHVe1PcglwPbAEuLKqdiW5qFu+GdgGnANMA/cDF3bDnwW8\nGLglyc1d2yurattoX4YkSZK0sHqdI90F320z2jYPTRdw8SzjPg7kCGuUJEmSJo53NpQkSZIaGKQl\nSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKk\nBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZp\nSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIk\nqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFB\nWpIkSWrQK0gnWZfkjiTTSTbNsjxJLuuW70xy5tCyK5Pck+TWURYuSZIkjdOcQTrJEuByYD2wGjg/\nyeoZ3dYDq7rHRuCKoWVvAdaNolhJkiRpUvQ5Ir0WmK6q3VX1AHANsGFGnw3AVTVwA7AsyckAVfVR\n4KujLFqSJEkatz5BegVw19D8nq7tcPtIkiRJR42J+bJhko1JdiTZsW/fvnGXI0mSJB1SnyC9Fzhl\naH5l13a4fQ6pqrZU1ZqqWrN8+fLDGSpJkiQtuD5BejuwKslpSY4HzgO2zuizFbigu3rHWcC9VXX3\niGuVJEmSJsacQbqq9gOXANcDtwPvrKpdSS5KclHXbRuwG5gG/hz4tQPjk7wD+CTw5CR7kvzyiF+D\nJEmStOCW9ulUVdsYhOXhts1D0wVcfJCx5x9JgZIkSdIkmpgvG0qSJEmLiUFakiRJamCQliRJkhoY\npCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJ\nkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQG\nBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQGBmlJ\nkiSpgUFakiRJamCQliRJkhoYpCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhoYpCVJkqQGBmlJkiSp\nQa8gnWRdkjuSTCfZNMvyJLmsW74zyZl9x0qSJEmL0ZxBOskS4HJgPbAaOD/J6hnd1gOrusdG4IrD\nGCtJkiQtOn2OSK8Fpqtqd1U9AFwDbJjRZwNwVQ3cACxLcnLPsZIkSdKik6o6dIfk54F1VfUr3fyL\ngWdU1SVDff4KuLSqPt7Nfwj4HWBqrrFD69jI4Gg2wJOBO47spQFwIvCVEaxHR859MTncF5PDfTEZ\n3A+Tw30xOY6VffGDVbW8dfDSUVZyJKpqC7BllOtMsqOq1oxynWrjvpgc7ovJ4b6YDO6HyeG+mBzu\ni376BOm9wClD8yu7tj59jusxVpIkSVp0+pwjvR1YleS0JMcD5wFbZ/TZClzQXb3jLODeqrq751hJ\nkiRp0ZnziHRV7U9yCXA9sAS4sqp2JbmoW74Z2AacA0wD9wMXHmrsvLyS2Y30VBEdEffF5HBfTA73\nxWRwP0wO98XkcF/0MOeXDSVJkiQ9nHc2lCRJkhoYpCVJkqQGR22Q9tbkkyPJnUluSXJzkh3jrudY\nkuTKJPckuXWo7YlJPpDkC92/TxhnjceCg+yHVyXZ270vbk5yzjhrPFYkOSXJh5PclmRXkpd37b4v\nFtgh9oXvjQWU5JFJPp3ks91+eHXX7nuih6PyHOnu1uSfB54P7GFw9ZDzq+q2sRZ2jEpyJ7Cmqo6F\nC7tPlCTPBu5jcOfRp3ZtfwJ8taou7f6T+YSq+p1x1nm0O8h+eBVwX1W9dpy1HWu6u+6eXFU3JXks\ncCPwQuCl+L5YUIfYF/8R3xsLJkmAE6rqviTHAR8HXg78B3xPzOloPSLtrckloKo+Cnx1RvMG4K3d\n9FsZfHBpHh1kP2gMquruqrqpm/4GcDuwAt8XC+4Q+0ILqAbu62aP6x6F74lejtYgvQK4a2h+D745\nx6mADya5sbsVvMbrpO467wBfAk4aZzHHuF9PsrM79cM/my6wJFPAGcCn8H0xVjP2BfjeWFBJliS5\nGbgH+EBV+Z7o6WgN0posZ1fV6cB64OLuz9yaADU4t+voO79rcbgC+CHgdOBu4L+Pt5xjS5LHAO8B\nXlFVXx9e5vtiYc2yL3xvLLCq+k73Ob0SWJvkqTOW+544iKM1SPe5rbkWSFXt7f69B7iWwak3Gp8v\nd+cmHjhH8Z4x13NMqqovdx9eDwJ/ju+LBdOdB/oe4Oqqem/X7PtiDGbbF743xqeqvgZ8GFiH74le\njtYg7a3JJ0SSE7ovkZDkBOAFwK2HHqV5thV4STf9EuAvx1jLMevAB1TnZ/F9sSC6L1a9Cbi9ql43\ntMj3xQI72L7wvbGwkixPsqybfhSDCzV8Dt8TvRyVV+0A6C6X83oeujX5fx1zScekJD/E4Cg0DG5J\n/3b3xcJJ8g7gOcCJwJeB3wf+AngncCrw98B/rCq/CDePDrIfnsPgT9cF3An8p6HzETVPkpwNfAy4\nBXiwa34lg3NzfV8soEPsi/PxvbFgkjyNwZcJlzA4wPrOqvovSZ6E74k5HbVBWpIkSZpPR+upHZIk\nSdK8MkhLkiRJDQzSkiRJUgODtCRJktTAIC1JkiQ1MEhL0iEk+U6Sm5PsSvLZJL+ZZF5/dyZ5Tfd8\nr5nR/pwkzxyaf0uSn5/PWlokeeXQ9FQSrwMs6ai0dNwFSNKE+5fu1rkk+X7g7cDjGFwLer5sBJ5Y\nVd+Z0f4c4D7gE/P43KPwSuC/jbsISZpvHpGWpJ6629xvBC7JwFSSjyW5qXs8EyDJVUleeGBckquT\nbBheVzf+NUluTXJLkl/o2rcCjwFuPNDWtU8BFwG/0R0h//Fu0bOTfCLJ7uGj00n+c5LtSXYmefVs\nryfJfUNHvz+YZG2Sj3Tr+pmuzyOTvLmr8TNJntu1vzTJe5O8L8kXkvxJ134p8Kiuxqu7p1qS5M+7\n53l/d/c0SVr0vCGLJB1Ckvuq6jEz2r4GPBn4BvBgVf1rklXAO6pqTZKfAH6jql6Y5PHAzcCqqto/\ntI6fYxCM1zG44+F24BlVdfdsz9mNeRVwX1W9tpt/C3AC8AvAU4CtVfW/JHkB8PPAfwLC4Fa/f1JV\nH52xvgLOqaq/TnJtt65zgdXAW6vq9CS/CfzbqvqlJE8B3g/8CHAe8HvAGcC3gDuAs6vqruH6u/8A\nTANrqurmJO/s6nzbYewGSZpIHpGWpHbHAX+e5BbgXQwCKFX1t8CqJMsZ3O74PcMhunM2g+D9nar6\nMvC3wL9rqOEvqurBqroNOKlre0H3+AxwE4OQvWqWsQ8A7+umbwH+tqq+3U1PDdX5tu51fY7BrYJ/\npFv2oaq6t6r+FbgN+MGD1PjFqrq5m75xaN2StKh5jrQkHYYkPwR8B7iHwXnSXwaezuDAxL8Odb0K\n+EUGR24vnMeSvjVc3tC/f1RV/2OOsd+uh/4s+eCBdVXVg0n6fD4MP/d3OPhnysx+ntoh6ajgEWlJ\n6qk7wrwZeEMXQB8P3F1VDwIvBpYMdX8L8AqA7mjxTB8DfiHJkm69zwY+PUcJ3wAe26PU64FfSnLg\n9IoV3RclW3wMeFG3nh8BTmVwGsehfDvJcY3PJ0mLhkFakg7twBfndgEfZHCO8IEv7/0Z8JIkn2Vw\n+sQ3DwzqTte4HXjzQdZ7LbAT+CzwN8BvV9WX5qjlfwI/O+PLhg9TVe9ncHWRT3annbybfgF8Nn8G\nPKJbz/8LvLSqvjXHmC3AzqEvG0rSUckvG0rSPEjyaAbnGp9ZVfeOux5J0uh5RFqSRizJ8xgcjf5/\nDNGSdPSayCPSJ554Yk1NTY27DEmSJB3Fbrzxxq9U1fLW8RN51Y6pqSl27Ngx7jIkSZJ0FEvy90cy\n3lM7JEmSpAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKnBRF7+TpKkmaY2XTfuEuZ0\n56XnjrsESQvIIC1JRzHDpyTNH0/tkCRJkhoYpCVJkqQGBmlJkiSpgUFakiRJamCQliRJkhp41Y4h\nfrtdkiRJfXlEWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZp\nSZIkqYFBWpIkSWrgnQ0laYh3ONVC8WdNWvwM0pIk6Ygshv8UgP8x0Oh5aockSZLUoFeQTrIuyR1J\nppNsmmV5klzWLd+Z5MyhZb+RZFeSW5O8I8kjR/kCJEmSpHGYM0gnWQJcDqwHVgPnJ1k9o9t6YFX3\n2Ahc0Y1dAbwMWFNVTwWWAOeNrHpJkiRpTPqcI70WmK6q3QBJrgE2ALcN9dkAXFVVBdyQZFmSk4ee\n41FJvg08GvjHkVUvaew8N1KSJpe/o+dXnyC9ArhraH4P8IwefVZU1Y4krwX+AfgX4P1V9f7ZniTJ\nRgZHszn11FP7VS9JkjRiiyF8LtbgebSZ16t2JHkCg6PVpwFfA96V5Ber6m0z+1bVFmALwJo1a2o+\n69Li4y81SZI0afp82XAvcMrQ/MqurU+f5wFfrKp9VfVt4L3AM9vLlSRJkiZDnyPS24FVSU5jEI7P\nA/6PGX22Apd0508/A7i3qu5O8g/AWUkezeDUjp8CdoysemmR8gi7JEmL35xBuqr2J7kEuJ7BVTeu\nrKpdSS7qlm8GtgHnANPA/cCF3bJPJXk3cBOwH/gM3ekbkiRJ0mLW6xzpqtrGICwPt20emi7g4oOM\n/X3g94+gRkmSJGnieGdDSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYk\nSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWrQ6xbhWpymNl037hLmdOel5467BEmSpCYekZYkSZIaGKQl\nSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKk\nBgZpSZIkqYFBWpIkSWrQK0gnWZfkjiTTSTbNsjxJLuuW70xy5tCyZUneneRzSW5P8mOjfAGSJEnS\nOMwZpJMsAS4H1gOrgfOTrJ7RbT2wqntsBK4YWvanwPuq6inA04HbR1C3JEmSNFZ9jkivBaarandV\nPQBcA2yY0WcDcFUN3AAsS3JykscDzwbeBFBVD1TV10ZYvyRJkjQWfYL0CuCuofk9XVufPqcB+4A3\nJ/lMkjcmOWG2J0myMcmOJDv27dvX+wVIkiRJ4zDfXzZcCpwJXFFVZwDfBB52jjVAVW2pqjVVtWb5\n8uXzXJYkSZJ0ZPoE6b3AKUPzK7u2Pn32AHuq6lNd+7sZBGtJkiRpUesTpLcDq5KcluR44Dxg64w+\nW4ELuqt3nAXcW1V3V9WXgLuSPLnr91PAbaMqXpIkSRqXpXN1qKr9SS4BrgeWAFdW1a4kF3XLNwPb\ngHOAaeB+4MKhVfw6cHUXwnfPWCZJkiQtSnMGaYCq2sYgLA+3bR6aLuDig4y9GVhzBDVKkiRJE8c7\nG0qSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuS\nJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkN\nDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1KBXkE6yLskdSaaTbJpleZJc\n1i3fmeTMGcuXJPlMkr8aVeGSJEnSOM0ZpJMsAS4H1gOrgfOTrJ7RbT2wqntsBK6YsfzlwO1HXK0k\nSZI0IfockV4LTFfV7qp6ALgG2DCjzwbgqhq4AViW5GSAJCuBc4E3jrBuSZIkaaz6BOkVwF1D83u6\ntr59Xg/8NvBgY42SJEnSxJnXLxsm+Wngnqq6sUffjUl2JNmxb9+++SxLkiRJOmJ9gvRe4JSh+ZVd\nW58+zwJ+JsmdDE4J+ckkb5vtSapqS1Wtqao1y5cv71m+JEmSNB59gvR2YFWS05IcD5wHbJ3RZytw\nQXf1jrOAe6vq7qr63apaWVVT3bi/qapfHOULkCRJksZh6Vwdqmp/kkuA64ElwJVVtSvJRd3yzcA2\n4BxgGrgfuHD+SpYkSZLGb84gDVBV2xiE5eG2zUPTBVw8xzo+AnzksCuUJEmSJpB3NpQkSZIaGKQl\nSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKk\nBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZp\nSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBgZpSZIkqUGvIJ1kXZI7kkwn2TTL8iS5rFu+M8mZXfsp\nST6c5LYku5K8fNQvQJIkSRqHOYN0kiXA5cB6YDVwfpLVM7qtB1Z1j43AFV37fuA3q2o1cBZw8Sxj\nJUmSpEWnzxHptcB0Ve2uqgeAa4ANM/psAK6qgRuAZUlOrqq7q+omgKr6BnA7sGKE9UuSJElj0SdI\nrwDuGprfw8PD8Jx9kkwBZwCfOtwiJUmSpEmzIF82TPIY4D3AK6rq6wfpszHJjiQ79u3btxBlSZIk\nSc36BOm9wClD8yu7tl59khzHIERfXVXvPdiTVNWWqlpTVWuWL1/ep3ZJkiRpbPoE6e3AqiSnJTke\nOA/YOqPPVuCC7uodZwH3VtXdSQK8Cbi9ql430solSZKkMVo6V4eq2p/kEuB6YAlwZVXtSnJRt3wz\nsA04B5gG7gcu7IY/C3gxcEuSm7u2V1bVttG+DEmSJGlhzRmkAbrgu21G2+ah6QIunmXcx4EcYY2S\nJEnSxPHOhpIkSVIDg7QkSZLUwCAtSZIkNTBIS5IkSQ0M0pIkSVIDg7QkSZLUwCAtSZIkNTBIS5Ik\nSQ0M0pIkSVIDg7QkSZLUwCAtSZIkNTBIS5IkSQ0M0pIkSVIDg7QkSZLUwCAtSZIkNTBIS5IkSQ0M\n0pIkSVIDg7QkSZLUwCAtSZIkNTBIS5IkSQ0M0pIkSVIDg7QkSZLUwCAtSZIkNegVpJOsS3JHkukk\nm2ZZniSXdct3Jjmz71hJkiRpMZozSCdZAlwOrAdWA+cnWT2j23pgVffYCFxxGGMlSZKkRafPEem1\nwHRV7a6qB4BrgA0z+mwArqqBG4BlSU7uOVaSJEladPoE6RXAXUPze7q2Pn36jJUkSZIWnaXjLuCA\nJBsZnBYCcF+SO8ZZz6TKH4/16U8EvjLKFY759YyU+2ayHU37x30zUu6bQzia9g2M/fWMlPtmZH7w\nSAb3CdJ7gVOG5ld2bX36HNdjLABVtQXY0qMejUmSHVW1Ztx16OHcN5PN/TO53DeTy30zudw3D+lz\nasd2YFWS05IcD5wHbJ3RZytwQXf1jrOAe6vq7p5jJUmSpEVnziPSVbU/ySXA9cAS4Mqq2pXkom75\nZmAbcA4wDdwPXHiosfPySiRJkqQF1Osc6araxiAsD7dtHpou4OK+Y7VoeerN5HLfTDb3z+Ry30wu\n983kct90MsjAkiRJkg6HtwiXJEmSGhikNackpyT5cJLbkuxK8vJx16TvlWRJks8k+atx16KHJFmW\n5N1JPpfk9iQ/Nu6aNJDkN7rfZ7cmeUeSR467pmNZkiuT3JPk1qG2Jyb5QJIvdP8+YZw1HqsOsm9e\n0/1e25nk2iTLxlnjOBmk1cd+4DerajVwFnCxt3qfOC8Hbh93EXqYPwXeV1VPAZ6O+2giJFkBvAxY\nU1VPZfBl+PPGW9Ux7y3Auhltm4APVdUq4EPdvBbeW3j4vvkA8NSqehrweeB3F7qoSWGQ1pyq6u6q\nuqmb/gaDMOAdKidEkpXAucAbx12LHpLk8cCzgTcBVNUDVfW18ValIUuBRyVZCjwa+Mcx13NMq6qP\nAl+d0bwBeGs3/VbghQtalIDZ901Vvb+q9nezNzC4T8gxySCtw5JkCjgD+NR4K9GQ1wO/DTw47kL0\nPU4D9gFv7k67eWOSE8ZdlKCq9gKvBf4BuJvBvQ/eP96qNIuTuntSAHwJOGmcxeigfgn463EXMS4G\nafWW5DHAe4BXVNXXx12PIMlPA/dU1Y3jrkUPsxQ4E7iiqs4Avol/mp4I3bm2Gxj8Z+cHgBOS/OJ4\nq9KhdJfZ9TJjEybJ/8Xg9M+rx13LuBik1UuS4xiE6Kur6r3jrkff9SzgZ5LcCVwD/GSSt423JHX2\nAHuq6sBfb97NIFhr/J4HfLGq9lXVt4H3As8cc016uC8nORmg+/eeMdejIUleCvw08KI6hq+lbJDW\nnJKEwXmet1fV68Zdjx5SVb9bVSuraorBl6X+pqo8sjYBqupLwF1Jntw1/RRw2xhL0kP+ATgryaO7\n328/hV8EnURbgZd00y8B/nKMtWhIknUMTin8maq6f9z1jJNBWn08C3gxg6OdN3ePc8ZdlLQI/Dpw\ndZKdwOnAfxtzPQK6vxK8G7gJuIXBZ6F3ahujJO8APgk8OcmeJL8MXAo8P8kXGPwV4dJx1nisOsi+\neQPwWOADXSbYfMiVHMW8s6EkSZLUwCPSkiRJUgODtCRJktTAIC1JkiQ1MEhLkiRJDQzSkiRJUgOD\ntCQdRJIavsFNkqVJ9iX5q8b1LUvya0Pzz2ldVzf+Kd2lpz6T5IdnLHvl0PRUkltbn0eSNDuDtCQd\n3DeBpyZ5VDf/fGDvEaxvGfBrc/bq74XAu6vqjKr6uxnLXjnbgEmRZOm4a5CkI2WQlqRD2wac202f\nD7zjwIIkT0zyF0l2JrkhydO69lcluTLJR5LsTvKybsilwA93R5Ff07U9Jsm7k3wuydXdnfa+R5LT\nu/XvTHJtkid0N0V6BfCrST48o/+lwKO657m6a16S5M+T7Ery/gP/OUjyw0nel+TGJB9L8pQZ63pE\nki8kWT5Ctz+GAAAUzUlEQVQ0P51kefd4T5Lt3eNZXZ+1ST7ZHSn/xIG7OyZ5aZKtSf4G+FDT3pCk\nCWKQlqRDuwY4L8kjgacBnxpa9mrgM1X1NAZHgK8aWvYU4N8Da4HfT3IcsAn4u6o6var+c9fvDAaB\neDXwQwzuJDrTVcDvdM9zC/D7VbUN2Az831X13OHOVbUJ+JfueV7UNa8CLq+qfwt8Dfi5rn0L8OtV\n9aPAbwF/NmNdDwJvAw6s53nAZ6tqH/Cn3fP/u259b+z6fA748ao6A/g9vveOjmcCP19VPzHL65Sk\nRcU/rUnSIVTVziRTDI5Gb5ux+Gy6QFpVf5PkSUke1y27rqq+BXwryT3ASQd5ik9X1R6AJDcDU8DH\nDyxM8nhgWVX9bdf0VuBdDS/li1V1czd9IzCV5DHAM4F3DR0I/75Zxl4J/CXweuCXgDd37c8DVg+N\nfVy3zscDb02yCijguKF1faCqvtpQvyRNHIO0JM1tK/Ba4DnAk3qO+dbQ9Hc4+O/bvv2O1MzneRSD\nv0p+rapOP9TAqroryZeT/CSDI+wHjk4/Ajirqv51uH+SNwAfrqqf7f4T8pGhxd88khchSZPEUzsk\naW5XAq+uqltmtH+MLlQmeQ7wlar6+iHW8w3gsYfzxFV1L/DPSX68a3ox8LeHGHLAt7vTSQ617q8D\nX0zyvwNk4OkH6f5GBqd4vKuqvtO1vR/49QMdkhwI5I/noS9lvrRHrZK0KBmkJWkOVbWnqi6bZdGr\ngB9NspPBFwlfMsd6/gn4/5LcOvRlwz5eAryme57Tgf/SY8wWYOfQlw0P5kXALyf5LLAL2HCQfluB\nx/DQaR0ALwPWdF+CvA24qGv/E+CPknwG//Ip6SiWqhp3DZKkCZdkDYMvFv74nJ0l6RjhkQJJ0iEl\n2QT8Kg+dGy1JwiPSkiRJUpOJPCJ94okn1tTU1LjLkCRJ0lHsxhtv/EpVLW8dP5FBempqih07doy7\nDEmSJB3Fkvz9kYz3qh2SJElSA4O0JEmS1MAgLUmSJDXoFaSTrEtyR5Lp7jJIM5e/qLsg/y1JPjF8\nZ6wkd3btNyfxxGdJkiQdFeb8smGSJcDlwPOBPcD2JFur6rahbl8EfqKq/jnJegZ31HrG0PLnVtVX\nRli3JEmSNFZ9jkivBaarandVPQBcw4xbyFbVJ6rqn7vZG4CVoy1TkiRJmix9Ln+3ArhraH4P33u0\neaZfBv56aL6ADyb5DvA/qmrLYVcpSZI0waY2XTfuEr7HnZeeO+4SjgkjvY50kucyCNJnDzWfXVV7\nk3w/8IEkn6uqj84ydiOwEeDUU08dZVmSJEnSyPU5tWMvcMrQ/Mqu7XskeRrwRmBDVf3Tgfaq2tv9\new9wLYNTRR6mqrZU1ZqqWrN8efMNZiRJkqQF0SdIbwdWJTktyfHAecDW4Q5JTgXeC7y4qj4/1H5C\nkscemAZeANw6quIlSZKkcZnz1I6q2p/kEuB6YAlwZVXtSnJRt3wz8HvAk4A/SwKwv6rWACcB13Zt\nS4G3V9X75uWVSJIkSQuo1znSVbUN2DajbfPQ9K8AvzLLuN3A02e2S5IkSYuddzaUJEmSGhikJUmS\npAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYG\naUmSJKmBQVqSJElqYJCWJEmSGhikJUmSpAYGaUmSJKnB0nEXIEmSNNPUpuvGXcL3uPPSc8ddgiaQ\nR6QlSZKkBgZpSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBr2CdJJ1Se5IMp1k0yzLX5RkZ5Jbknwi\nydP7jpUkSZIWozmDdJIlwOXAemA1cH6S1TO6fRH4iar6X4E/ALYcxlhJkiRp0elzRHotMF1Vu6vq\nAeAaYMNwh6r6RFX9czd7A7Cy71hJkiRpMeoTpFcAdw3N7+naDuaXgb9uHCtJkiQtCiO9s2GS5zII\n0mc3jN0IbAQ49dRTR1mWJEmSNHJ9jkjvBU4Zml/ZtX2PJE8D3ghsqKp/OpyxAFW1parWVNWa5cuX\n96ldkiRJGps+QXo7sCrJaUmOB84Dtg53SHIq8F7gxVX1+cMZK0mSJC1Gc57aUVX7k1wCXA8sAa6s\nql1JLuqWbwZ+D3gS8GdJAPZ3R5dnHTtPr0WSJElaML3Oka6qbcC2GW2bh6Z/BfiVvmMlSZKkxc47\nG0qSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuS\nJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkN\nDNKSJElSg6XjLkCSJM2/qU3XjbuE77rz0nPHXYI0Eh6RliRJkhoYpCVJkqQGBmlJkiSpQa8gnWRd\nkjuSTCfZNMvypyT5ZJJvJfmtGcvuTHJLkpuT7BhV4ZIkSdI4zfllwyRLgMuB5wN7gO1JtlbVbUPd\nvgq8DHjhQVbz3Kr6ypEWK0mSJE2KPkek1wLTVbW7qh4ArgE2DHeoqnuqajvw7XmoUZIkSZo4fYL0\nCuCuofk9XVtfBXwwyY1JNh6sU5KNSXYk2bFv377DWL0kSZK08Bbiy4ZnV9XpwHrg4iTPnq1TVW2p\nqjVVtWb58uULUJYkSZLUrk+Q3gucMjS/smvrpar2dv/eA1zL4FQRSZIkaVHrE6S3A6uSnJbkeOA8\nYGuflSc5IcljD0wDLwBubS1WkiRJmhRzXrWjqvYnuQS4HlgCXFlVu5Jc1C3fnOTfADuAxwEPJnkF\nsBo4Ebg2yYHnentVvW9+XookSZK0cOYM0gBVtQ3YNqNt89D0lxic8jHT14GnH0mBkiRJ0iTyzoaS\nJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElS\nA4O0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSA4O0JEmS1MAgLUmSJDUwSEuSJEkNlo67AEmSFqOp\nTdeNu4TvuvPSc8ddgnRM8oi0JEmS1MAgLUmSJDUwSEuSJEkNDNKSJElSg15BOsm6JHckmU6yaZbl\nT0nyySTfSvJbhzNWkiRJWozmDNJJlgCXA+uB1cD5SVbP6PZV4GXAaxvGSpIkSYtOnyPSa4Hpqtpd\nVQ8A1wAbhjtU1T1VtR349uGOlSRJkhajPkF6BXDX0Pyerq2P3mOTbEyyI8mOffv29Vy9JEmSNB4T\n82XDqtpSVWuqas3y5cvHXY4kSZJ0SH2C9F7glKH5lV1bH0cyVpIkSZpYfYL0dmBVktOSHA+cB2zt\nuf4jGStJkiRNrKVzdaiq/UkuAa4HlgBXVtWuJBd1yzcn+TfADuBxwINJXgGsrqqvzzZ2vl6MJEmS\ntFDmDNIAVbUN2DajbfPQ9JcYnLbRa6wkSZK02E3Mlw0lSZKkxcQgLUmSJDUwSEuSJEkNDNKSJElS\nA4O0JEmS1KDXVTskSZpvU5uuG3cJ33XnpeeOuwRJi4BHpCVJkqQGBmlJkiSpgUFakiRJamCQliRJ\nkhoYpCVJkqQGBmlJkiSpgUFakiRJauB1pCXpKOV1mSVpfnlEWpIkSWpgkJYkSZIaGKQlSZKkBgZp\nSZIkqYFBWpIkSWpgkJYkSZIaGKQlSZKkBr2CdJJ1Se5IMp1k0yzLk+SybvnOJGcOLbszyS1Jbk6y\nY5TFS5IkSeMy5w1ZkiwBLgeeD+wBtifZWlW3DXVbD6zqHs8Aruj+PeC5VfWVkVUtSZIkjVmfI9Jr\ngemq2l1VDwDXABtm9NkAXFUDNwDLkpw84lolSZKkidEnSK8A7hqa39O19e1TwAeT3JhkY2uhkiRJ\n0iSZ89SOETi7qvYm+X7gA0k+V1UfndmpC9kbAU499dQFKEuSJElq1+eI9F7glKH5lV1brz5VdeDf\ne4BrGZwq8jBVtaWq1lTVmuXLl/erXpIkSRqTPkF6O7AqyWlJjgfOA7bO6LMVuKC7esdZwL1VdXeS\nE5I8FiDJCcALgFtHWL8kSZI0FnOe2lFV+5NcAlwPLAGurKpdSS7qlm8GtgHnANPA/cCF3fCTgGuT\nHHiut1fV+0b+KiRpAUxtum7cJXzXnZeeO+4SJOmY1+sc6araxiAsD7dtHpou4OJZxu0Gnn6ENUqS\nJEkTxzsbSpIkSQ0M0pIkSVIDg7QkSZLUwCAtSZIkNTBIS5IkSQ0M0pIkSVIDg7QkSZLUwCAtSZIk\nNTBIS5IkSQ0M0pIkSVIDg7QkSZLUYOm4C5B07JradN24S/iuOy89d9wlSJIWGY9IS5IkSQ0M0pIk\nSVIDg7QkSZLUwCAtSZIkNfDLhtJRYpK+uAd+eU+SJpmfGaPhEWlJkiSpgUFakiRJamCQliRJkhoY\npCVJkqQGvb5smGQd8KfAEuCNVXXpjOXplp8D3A+8tKpu6jNWmlR+EUOSJB3KnEekkywBLgfWA6uB\n85OsntFtPbCqe2wErjiMsZIkSdKi0+fUjrXAdFXtrqoHgGuADTP6bACuqoEbgGVJTu45VpIkSVp0\n+pzasQK4a2h+D/CMHn1W9ByrY4SnSkiSpKPJxNyQJclGBqeFANyX5I5x1nOETgS+Mu4ijmIj2b75\n4xFUsoAWsF637/xy+86/I97Gbt9DOua2L/g7Yr6Ncfv+4JGsrE+Q3gucMjS/smvr0+e4HmMBqKot\nwJYe9Uy8JDuqas246zhauX3nl9t3frl955/beH65feeX23d+jXr79jlHejuwKslpSY4HzgO2zuiz\nFbggA2cB91bV3T3HSpIkSYvOnEekq2p/kkuA6xlcwu7KqtqV5KJu+WZgG4NL300zuPzdhYcaOy+v\nRJIkSVpAvc6RrqptDMLycNvmoekCLu479hhwVJyiMsHcvvPL7Tu/3L7zz208v9y+88vtO79Gun0z\nyMCSJEmSDoe3CJckSZIaGKR7SHJKkg8nuS3JriQv79qfmOQDSb7Q/fuEoTG/m2Q6yR1J/v0s69ya\n5NaFfB2TapTbN8nxSbYk+XySzyX5uXG8pkky4u17fpJbkuxM8r4kJ47jNU2Sw92+SZ7U9b8vyRtm\nrOtHu+07neSyJBnHa5o0o9rGSR6d5Lrud8OuJJeO6zVNklH+DA+t08+4zoh/R/gZN8OIt+/hf8ZV\nlY85HsDJwJnd9GOBzzO45fmfAJu69k3AH3fTq4HPAt8HnAb8HbBkaH3/AXg7cOu4X9skPEa5fYFX\nA3/YTT8COHHcr2/cj1FtXwbfqbjnwDbtxr9q3K9v3I+G7XsCcDZwEfCGGev6NHAWEOCvgfXjfn2T\n8BjVNgYeDTy3mz4e+JjbeLQ/w91yP+Pmafv6GTd/27f1M84j0j1U1d1VdVM3/Q3gdgZ3bdwAvLXr\n9lbghd30BuCaqvpWVX2RwdVM1gIkeQzwfwJ/uHCvYLKNcvsCvwT8UbeuB6vqmL8xzgi3b7rHCd2R\n0scB/7hgL2RCHe72rapvVtXHgX8dXk+Sk4HHVdUNNfgtfhUP7ZNj2qi2cVXdX1Uf7qYfAG5icH+D\nY9qoti/4GTebUW5f/Ix7mBFu36bPOIP0YUoyBZwBfAo4qQbXywb4EnBSN32wW6YD/AHw3xlcJlAz\nHMn2TbKsm/+DJDcleVeSk9B3Hcn2rapvA78K3MLgl8tq4E3zX/Xi0XP7HswKBtv6gOHfG+oc4TYe\nXs8y4H8DPjTiEhe1EWxfP+MO4Ui2r59xczuS7dv6GWeQPgzd/7TfA7yiqr4+vKw7gnTIS6AkOR34\n4aq6dv6qXLyOdPsy+LPMSuATVXUm8EngtfNR62I0gp/f4xj8kjkD+AFgJ/C781Pt4jOCn1/NYVTb\nOMlS4B3AZVW1e+SFLlJ+xs0vP+Pm17g+4wzSPXUb+D3A1VX13q75y92fYw/8Wfaerv1gt0z/MWBN\nkjuBjwM/kuQj81/95BvR9v0nBkdBDox/F3DmPJe+KIxo+54OUFV/1/1SeifwzAUof+Id5vY9mL18\n72kGB7a7GNk2PmAL8IWqev3oK12cRrR9/Yw7iBFtXz/jDmJE27fpM84g3UN3rsybgNur6nVDi7YC\nL+mmXwL85VD7eUm+L8lpwCrg01V1RVX9QFVNMTjR/fNV9ZyFeA2TbITbt4D/CTyn6/dTwG3zXP7E\nG9X2ZRDqVidZ3vV7PoNz0Y5pDdt3Vt2fIL+e5KxunRfMNeZYMapt3K3rD4HHA68YdZ2L1Qh/hv2M\nm8UIt6+fcbMY4e+Hts+4muPbiD4KBr8QisFh/pu7xznAkxicX/eF/7+9u2etIgjDMHw/kkIbC7XR\nItrYiI0QLawFtbRQtLLQIp2lghhI5w8QixQWIkLAH2AREUQbCSp+gBYBRSsLEY2FGnwtdgOnEMT1\nJJuc3BdMs2fPMDMsuw/D7A4wB2wb+M9lmq8dvOEPb4UDe/CN5qGPL7AbeNDWdQ8Y77t/fZchj+9k\ne2N5TnND3953//ouHcf3LfAJWKRZC72vPT4BvGzH/hrtplkbvQxrjGlm+au9hpfrOd93//ouw7yG\nB373GbcC4+szbsXH95+fce5sKEmSJHXg0g5JkiSpA4O0JEmS1IFBWpIkSerAIC1JkiR1YJCWJEmS\nOjBIS9IalcbDJMcHjp1McrfPdkmSGn7+TpLWsCT7aXYwO0CzRfBT4FhVLfxHnWNVtTSkJkrShuWM\ntCStYVX1kmZjgIvAFHCzqhaSnE3yOMmzJNeTbAJIMpNkPsmrJFPL9ST5kORqkqfAiV46I0kjZqzv\nBkiS/moaeAL8ACbaWeoTwOGqWkoyA5wGbgOXqupTkjHgfpI7VbW8jfDHqjrQRwckaRQZpCVpjauq\nb0lmgcWq+p7kCHAQmE8CsAV4355+Jsk5mvv7LpqtsZeD9OzqtlySRptBWpLWh19tAQhwo6quDJ6Q\nZC9wAThUVZ+T3AI2D5zybVVaKkkbhGukJWn9mQNOJdkBkGR7knFgK/AV+JJkJ3C0xzZK0shzRlqS\n1pmqepFkGphrXzL8CUwC8zTLOF4D74BH/bVSkkafn7+TJEmSOnBphyRJktSBQVqSJEnqwCAtSZIk\ndWCQliRJkjowSEuSJEkdGKQlSZKkDgzSkiRJUgcGaUmSJKmD31eTY392g8p9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x25fa28150>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.figure(figsize=(12,10))\n",
    "\n",
    "plt.subplot(311)\n",
    "frequ_values = (reviews.date_day.value_counts()/reviews.date_day.value_counts().sum()).sort_index()\n",
    "plt.bar(frequ_values.index,frequ_values)\n",
    "plt.xlabel(\"Day of the month\",verticalalignment='top')\n",
    "\n",
    "plt.subplot(312)\n",
    "frequ_values = (reviews.date_month.value_counts()/reviews.date_month.value_counts().sum()).sort_index()\n",
    "plt.bar(frequ_values.index,frequ_values)\n",
    "plt.xlabel(\"Month of the year\",verticalalignment='top')\n",
    "\n",
    "plt.subplot(313)\n",
    "frequ_values = (reviews.date_year.value_counts()/reviews.date_year.value_counts().sum()).sort_index()\n",
    "plt.bar(frequ_values.index,frequ_values)\n",
    "plt.xlabel(\"Year\",verticalalignment='top')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>number_nan</th>\n",
       "      <th>types</th>\n",
       "      <th>number_distinct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>stars</th>\n",
       "      <td>3.722765</td>\n",
       "      <td>1.405131</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int64</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_year</th>\n",
       "      <td>2013.839790</td>\n",
       "      <td>2.144192</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2014.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int32</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_month</th>\n",
       "      <td>6.540899</td>\n",
       "      <td>3.415820</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int32</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date_day</th>\n",
       "      <td>15.750731</td>\n",
       "      <td>8.798195</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>int32</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   mean       std     min     25%     50%     75%     max  \\\n",
       "stars          3.722765  1.405131     1.0     3.0     4.0     5.0     5.0   \n",
       "date_year   2013.839790  2.144192  2004.0  2013.0  2014.0  2016.0  2017.0   \n",
       "date_month     6.540899  3.415820     1.0     4.0     7.0     9.0    12.0   \n",
       "date_day      15.750731  8.798195     1.0     8.0    16.0    23.0    31.0   \n",
       "\n",
       "            number_nan  types  number_distinct  \n",
       "stars              0.0  int64                5  \n",
       "date_year          0.0  int32               14  \n",
       "date_month         0.0  int32               12  \n",
       "date_day           0.0  int32               31  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getDfSummary( reviews )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reviews_variables = ['business_id','user_id','date_year','stars','text']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Create a sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "np.random.seed(10)\n",
    "shuffle_indices = np.random.permutation(np.arange(len(reviews)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reviews sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2aFiy99vNLklCx3T_tGS9A</td>\n",
       "      <td>KpkOkG6RIf4Ra25Lhhxf1A</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>If you enjoy service by someone who is as comp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2aFiy99vNLklCx3T_tGS9A</td>\n",
       "      <td>bQ7fQq1otn9hKX-gXRsrgA</td>\n",
       "      <td>2010</td>\n",
       "      <td>5</td>\n",
       "      <td>After being on the phone with Verizon Wireless...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2aFiy99vNLklCx3T_tGS9A</td>\n",
       "      <td>r1NUhdNmL6yU9Bn-Yx6FTw</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>Great service! Corey is very service oriented....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id                 user_id  date_year  stars  \\\n",
       "0  2aFiy99vNLklCx3T_tGS9A  KpkOkG6RIf4Ra25Lhhxf1A       2011      5   \n",
       "1  2aFiy99vNLklCx3T_tGS9A  bQ7fQq1otn9hKX-gXRsrgA       2010      5   \n",
       "2  2aFiy99vNLklCx3T_tGS9A  r1NUhdNmL6yU9Bn-Yx6FTw       2011      5   \n",
       "\n",
       "                                                text  \n",
       "0  If you enjoy service by someone who is as comp...  \n",
       "1  After being on the phone with Verizon Wireless...  \n",
       "2  Great service! Corey is very service oriented....  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "reviews_sample = reviews[:10000]\n",
    "reviews_sample = reviews_sample[reviews_variables]\n",
    "\n",
    "reviews_sample.head(3)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Business sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original number of business 144072\n",
      "Comments distribution:\n",
      "Std review_id    92.576557\n",
      "dtype: float64\n",
      "Median 9.0\n",
      "Mean review_id    28.826906\n",
      "dtype: float64\n",
      "\n",
      "After looking at the distribution of reviews per business:\n",
      "\n",
      "We have a mean of  review_id    19.626183\n",
      "dtype: float64 comments per business\n",
      "We want a sample of  10000 then we need  509\n"
     ]
    }
   ],
   "source": [
    "\n",
    "groups = reviews[ ['business_id','review_id'] ].groupby('business_id').count()\n",
    "\n",
    "print \"Original number of business\", len(groups)\n",
    "print \"Comments distribution:\"\n",
    "print \"Std\",np.std(groups)\n",
    "print \"Median\",np.median(groups)\n",
    "print \"Mean\",np.mean(groups)\n",
    "\n",
    "groupfilter  =  groups\n",
    "\n",
    "#pd.DataFrame(groupfilter.sort_values(by='review_id',ascending=False)).to_csv(\"yes.csv\")\n",
    "## percentile 0.98 has 200 reviews\n",
    "## print len(groupfilter[groupfilter.review_id < 200])    #  ->  140922\n",
    "\n",
    "groupfilter  = groupfilter[groupfilter.review_id < 200]\n",
    "\n",
    "sample_size = 10000\n",
    "\n",
    "print \"\\nAfter looking at the distribution of reviews per business:\\n\"\n",
    "print \"We have a mean of \", np.mean(groupfilter), \"comments per business\"\n",
    "sample_business_size = int(float(sample_size/np.mean(groupfilter)))\n",
    "print \"We want a sample of \", sample_size, \"then we need \", sample_business_size\n",
    "\n",
    "sample_biz_id = groupfilter.sample( sample_business_size )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9751</th>\n",
       "      <td>Te-BqwJZt8S0fu9ltlq3Eg</td>\n",
       "      <td>9PmUzfxz3vamMaTzGSesyg</td>\n",
       "      <td>2014</td>\n",
       "      <td>2</td>\n",
       "      <td>the service at this place is awful .. too slow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9752</th>\n",
       "      <td>Te-BqwJZt8S0fu9ltlq3Eg</td>\n",
       "      <td>J9ytHx0WNvYofenZ8BZj0w</td>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>DO NOT waste your time or money going to this ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9753</th>\n",
       "      <td>Te-BqwJZt8S0fu9ltlq3Eg</td>\n",
       "      <td>0m6Cfsn8KAmzFD01wO_BuA</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>Good food, but TERRIBLE service. The hostess r...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 business_id                 user_id  date_year  stars  \\\n",
       "9751  Te-BqwJZt8S0fu9ltlq3Eg  9PmUzfxz3vamMaTzGSesyg       2014      2   \n",
       "9752  Te-BqwJZt8S0fu9ltlq3Eg  J9ytHx0WNvYofenZ8BZj0w       2016      1   \n",
       "9753  Te-BqwJZt8S0fu9ltlq3Eg  0m6Cfsn8KAmzFD01wO_BuA       2015      1   \n",
       "\n",
       "                                                   text  \n",
       "9751  the service at this place is awful .. too slow...  \n",
       "9752  DO NOT waste your time or money going to this ...  \n",
       "9753  Good food, but TERRIBLE service. The hostess r...  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "business_sample = reviews[ reviews['business_id'].isin( sample_biz_id.index ) ].reset_index()\n",
    "business_sample = business_sample[reviews_variables]\n",
    "\n",
    "business_sample.tail(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### User sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original number of people 1029432\n",
      "Comments distribution:\n",
      "Std review_id    13.990401\n",
      "dtype: float64\n",
      "Median 1.0\n",
      "Mean review_id    4.034409\n",
      "dtype: float64\n",
      "1007870\n",
      "\n",
      "After looking at the distribution of reviews per person:\n",
      "\n",
      "We have a mean of  review_id    2.786598\n",
      "dtype: float64 comments per person\n",
      "We want a sample of  10000 then we need  3588\n"
     ]
    }
   ],
   "source": [
    "\n",
    "groups = reviews[ ['user_id','review_id'] ].groupby('user_id').count()\n",
    "\n",
    "print \"Original number of people\", len(groups)\n",
    "print \"Comments distribution:\"\n",
    "print \"Std\",np.std(groups)\n",
    "print \"Median\",np.median(groups)\n",
    "print \"Mean\",np.mean(groups)\n",
    "\n",
    "personfilter =  groups\n",
    "\n",
    "#pd.DataFrame(personfilter.sort_values(by='review_id',ascending=False)).to_csv(\"yes.csv\")\n",
    "## percentile 0.98 has 200 reviews\n",
    "print len(personfilter[personfilter.review_id < 25])    #  ->  1007870\n",
    "\n",
    "personfilter  = personfilter[personfilter.review_id < 25]\n",
    "\n",
    "sample_size = 10000\n",
    "\n",
    "print \"\\nAfter looking at the distribution of reviews per person:\\n\"\n",
    "print \"We have a mean of \", np.mean(personfilter), \"comments per person\"\n",
    "sample_user_size = int(float(sample_size/np.mean(personfilter)))\n",
    "print \"We want a sample of \", sample_size, \"then we need \", sample_user_size\n",
    "\n",
    "sample_usr_id = personfilter.sample( sample_user_size )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10242</th>\n",
       "      <td>v6n4I3fA8eqmO4bnKieOIw</td>\n",
       "      <td>UeQRWJmzsVD-p-viIYIvAg</td>\n",
       "      <td>2014</td>\n",
       "      <td>4</td>\n",
       "      <td>Very nice course. Very well maintained. Beauti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10243</th>\n",
       "      <td>rD32POdIhGN9eR8aUxl-3A</td>\n",
       "      <td>iaQZQj_FRlQ9HJCQtgJb2Q</td>\n",
       "      <td>2014</td>\n",
       "      <td>2</td>\n",
       "      <td>We love Chipotle and this location used to be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10244</th>\n",
       "      <td>oiTq668GwyXo5hMHjdW0kA</td>\n",
       "      <td>GvZiX0sFJpntoAVqRGpIWQ</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>I've been to Stackd twice now and the food was...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  business_id                 user_id  date_year  stars  \\\n",
       "10242  v6n4I3fA8eqmO4bnKieOIw  UeQRWJmzsVD-p-viIYIvAg       2014      4   \n",
       "10243  rD32POdIhGN9eR8aUxl-3A  iaQZQj_FRlQ9HJCQtgJb2Q       2014      2   \n",
       "10244  oiTq668GwyXo5hMHjdW0kA  GvZiX0sFJpntoAVqRGpIWQ       2016      4   \n",
       "\n",
       "                                                    text  \n",
       "10242  Very nice course. Very well maintained. Beauti...  \n",
       "10243  We love Chipotle and this location used to be ...  \n",
       "10244  I've been to Stackd twice now and the food was...  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "users_sample = reviews[ reviews['user_id'].isin( sample_usr_id.index ) ].reset_index()\n",
    "users_sample = users_sample[reviews_variables]\n",
    "\n",
    "users_sample.tail(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Joining metadata to comments -> Reviews sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>b1Qf5tcu5R-kySnNrFPYew</td>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>I have been putting a focus on my health for t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>GS3dVZYhg41LoTNAXWSoUw</td>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>Dr. Trotter is excellent at what he does. I've...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>cD35x9Q9ARsJkD5PQfyJOA</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>Prior to visiting Zentai Wellness, I was havin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 business_id                 user_id  date_year  stars  \\\n",
       "9997  -tLxryf1OpzVP9OSrznprg  b1Qf5tcu5R-kySnNrFPYew       2014      5   \n",
       "9998  -tLxryf1OpzVP9OSrznprg  GS3dVZYhg41LoTNAXWSoUw       2014      5   \n",
       "9999  -tLxryf1OpzVP9OSrznprg  cD35x9Q9ARsJkD5PQfyJOA       2011      5   \n",
       "\n",
       "                                                   text  \n",
       "9997  I have been putting a focus on my health for t...  \n",
       "9998  Dr. Trotter is excellent at what he does. I've...  \n",
       "9999  Prior to visiting Zentai Wellness, I was havin...  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "sample_dataset = reviews_sample\n",
    "sample_dataset.tail(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1. Join: Reviews and user's info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "user_data_selected = users[user_variables]\n",
    "\n",
    "user_data_selected.columns = [ \"usr_\"+col if col != \"user_id\" else col for col in user_data_selected.columns ]\n",
    "rev_user = pd.merge(sample_dataset, user_data_selected, on = \"user_id\", how = \"left\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>usr_average_stars</th>\n",
       "      <th>usr_yelping_since_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>b1Qf5tcu5R-kySnNrFPYew</td>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>I have been putting a focus on my health for t...</td>\n",
       "      <td>4.17</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>GS3dVZYhg41LoTNAXWSoUw</td>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>Dr. Trotter is excellent at what he does. I've...</td>\n",
       "      <td>4.28</td>\n",
       "      <td>2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>cD35x9Q9ARsJkD5PQfyJOA</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>Prior to visiting Zentai Wellness, I was havin...</td>\n",
       "      <td>3.22</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 business_id                 user_id  date_year  stars  \\\n",
       "9997  -tLxryf1OpzVP9OSrznprg  b1Qf5tcu5R-kySnNrFPYew       2014      5   \n",
       "9998  -tLxryf1OpzVP9OSrznprg  GS3dVZYhg41LoTNAXWSoUw       2014      5   \n",
       "9999  -tLxryf1OpzVP9OSrznprg  cD35x9Q9ARsJkD5PQfyJOA       2011      5   \n",
       "\n",
       "                                                   text  usr_average_stars  \\\n",
       "9997  I have been putting a focus on my health for t...               4.17   \n",
       "9998  Dr. Trotter is excellent at what he does. I've...               4.28   \n",
       "9999  Prior to visiting Zentai Wellness, I was havin...               3.22   \n",
       "\n",
       "      usr_yelping_since_year  \n",
       "9997                    2014  \n",
       "9998                    2012  \n",
       "9999                    2011  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "rev_user.tail(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.2. Join: Reviews and business info "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "business_data_selected = businesses_sparse[business_variables]\n",
    "\n",
    "business_data_selected.columns = [ \"biz_\"+col if col != \"business_id\" else col for col in business_data_selected.columns ]\n",
    "rev_user_biz = pd.merge(rev_user, business_data_selected, on = \"business_id\", how = \"left\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 10000)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>usr_average_stars</th>\n",
       "      <th>usr_yelping_since_year</th>\n",
       "      <th>biz_is_open</th>\n",
       "      <th>biz_latitude</th>\n",
       "      <th>biz_longitude</th>\n",
       "      <th>...</th>\n",
       "      <th>biz_catBiz_Wine Tours</th>\n",
       "      <th>biz_catBiz_Wineries</th>\n",
       "      <th>biz_catBiz_Wok</th>\n",
       "      <th>biz_catBiz_Women's Clothing</th>\n",
       "      <th>biz_catBiz_Workers Compensation Law</th>\n",
       "      <th>biz_catBiz_Wraps</th>\n",
       "      <th>biz_catBiz_Yelp Events</th>\n",
       "      <th>biz_catBiz_Yoga</th>\n",
       "      <th>biz_catBiz_Ziplining</th>\n",
       "      <th>biz_catBiz_Zoos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>b1Qf5tcu5R-kySnNrFPYew</td>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>I have been putting a focus on my health for t...</td>\n",
       "      <td>4.17</td>\n",
       "      <td>2014</td>\n",
       "      <td>1</td>\n",
       "      <td>43.662735</td>\n",
       "      <td>-79.376436</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>GS3dVZYhg41LoTNAXWSoUw</td>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>Dr. Trotter is excellent at what he does. I've...</td>\n",
       "      <td>4.28</td>\n",
       "      <td>2012</td>\n",
       "      <td>1</td>\n",
       "      <td>43.662735</td>\n",
       "      <td>-79.376436</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>-tLxryf1OpzVP9OSrznprg</td>\n",
       "      <td>cD35x9Q9ARsJkD5PQfyJOA</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>Prior to visiting Zentai Wellness, I was havin...</td>\n",
       "      <td>3.22</td>\n",
       "      <td>2011</td>\n",
       "      <td>1</td>\n",
       "      <td>43.662735</td>\n",
       "      <td>-79.376436</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 2488 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 business_id                 user_id  date_year  stars  \\\n",
       "9997  -tLxryf1OpzVP9OSrznprg  b1Qf5tcu5R-kySnNrFPYew       2014      5   \n",
       "9998  -tLxryf1OpzVP9OSrznprg  GS3dVZYhg41LoTNAXWSoUw       2014      5   \n",
       "9999  -tLxryf1OpzVP9OSrznprg  cD35x9Q9ARsJkD5PQfyJOA       2011      5   \n",
       "\n",
       "                                                   text  usr_average_stars  \\\n",
       "9997  I have been putting a focus on my health for t...               4.17   \n",
       "9998  Dr. Trotter is excellent at what he does. I've...               4.28   \n",
       "9999  Prior to visiting Zentai Wellness, I was havin...               3.22   \n",
       "\n",
       "      usr_yelping_since_year  biz_is_open  biz_latitude  biz_longitude  \\\n",
       "9997                    2014            1     43.662735     -79.376436   \n",
       "9998                    2012            1     43.662735     -79.376436   \n",
       "9999                    2011            1     43.662735     -79.376436   \n",
       "\n",
       "           ...        biz_catBiz_Wine Tours  biz_catBiz_Wineries  \\\n",
       "9997       ...                          0.0                  0.0   \n",
       "9998       ...                          0.0                  0.0   \n",
       "9999       ...                          0.0                  0.0   \n",
       "\n",
       "      biz_catBiz_Wok  biz_catBiz_Women's Clothing  \\\n",
       "9997             0.0                          0.0   \n",
       "9998             0.0                          0.0   \n",
       "9999             0.0                          0.0   \n",
       "\n",
       "      biz_catBiz_Workers Compensation Law  biz_catBiz_Wraps  \\\n",
       "9997                                  0.0               0.0   \n",
       "9998                                  0.0               0.0   \n",
       "9999                                  0.0               0.0   \n",
       "\n",
       "      biz_catBiz_Yelp Events  biz_catBiz_Yoga  biz_catBiz_Ziplining  \\\n",
       "9997                     0.0              0.0                   0.0   \n",
       "9998                     0.0              0.0                   0.0   \n",
       "9999                     0.0              0.0                   0.0   \n",
       "\n",
       "      biz_catBiz_Zoos  \n",
       "9997              0.0  \n",
       "9998              0.0  \n",
       "9999              0.0  \n",
       "\n",
       "[3 rows x 2488 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "print(len(rev_user_biz), len(reviews_sample))\n",
    "rev_user_biz.tail(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.3. Create new variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "rev_user_biz['usr_year_experience'] = rev_user_biz['date_year']-rev_user_biz['usr_yelping_since_year']\n",
    "\n",
    "import math\n",
    "rev_user_biz['usr_average_stars'] = rev_user_biz['usr_average_stars'].map(lambda x: 0.5 * math.ceil(2.0 * round(x,1)))\n",
    "rev_user_biz['usr_stars_bias']= rev_user_biz['usr_average_stars']-rev_user_biz['biz_stars']\n",
    "\n",
    "rev_user_biz = rev_user_biz.drop('usr_yelping_since_year',axis=1)\n",
    "rev_user_biz = rev_user_biz.drop('biz_stars',axis=1)\n",
    "rev_user_biz = rev_user_biz.drop('usr_average_stars',axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>biz_is_open</th>\n",
       "      <th>biz_latitude</th>\n",
       "      <th>biz_longitude</th>\n",
       "      <th>biz_name</th>\n",
       "      <th>biz_state_AZ</th>\n",
       "      <th>...</th>\n",
       "      <th>biz_catBiz_Wok</th>\n",
       "      <th>biz_catBiz_Women's Clothing</th>\n",
       "      <th>biz_catBiz_Workers Compensation Law</th>\n",
       "      <th>biz_catBiz_Wraps</th>\n",
       "      <th>biz_catBiz_Yelp Events</th>\n",
       "      <th>biz_catBiz_Yoga</th>\n",
       "      <th>biz_catBiz_Ziplining</th>\n",
       "      <th>biz_catBiz_Zoos</th>\n",
       "      <th>usr_year_experience</th>\n",
       "      <th>usr_stars_bias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2aFiy99vNLklCx3T_tGS9A</td>\n",
       "      <td>KpkOkG6RIf4Ra25Lhhxf1A</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>If you enjoy service by someone who is as comp...</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>SharpieTech</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2aFiy99vNLklCx3T_tGS9A</td>\n",
       "      <td>bQ7fQq1otn9hKX-gXRsrgA</td>\n",
       "      <td>2010</td>\n",
       "      <td>5</td>\n",
       "      <td>After being on the phone with Verizon Wireless...</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>SharpieTech</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2aFiy99vNLklCx3T_tGS9A</td>\n",
       "      <td>r1NUhdNmL6yU9Bn-Yx6FTw</td>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>Great service! Corey is very service oriented....</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>SharpieTech</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 2487 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id                 user_id  date_year  stars  \\\n",
       "0  2aFiy99vNLklCx3T_tGS9A  KpkOkG6RIf4Ra25Lhhxf1A       2011      5   \n",
       "1  2aFiy99vNLklCx3T_tGS9A  bQ7fQq1otn9hKX-gXRsrgA       2010      5   \n",
       "2  2aFiy99vNLklCx3T_tGS9A  r1NUhdNmL6yU9Bn-Yx6FTw       2011      5   \n",
       "\n",
       "                                                text  biz_is_open  \\\n",
       "0  If you enjoy service by someone who is as comp...            1   \n",
       "1  After being on the phone with Verizon Wireless...            1   \n",
       "2  Great service! Corey is very service oriented....            1   \n",
       "\n",
       "   biz_latitude  biz_longitude     biz_name  biz_state_AZ       ...        \\\n",
       "0     33.662944    -112.018233  SharpieTech             1       ...         \n",
       "1     33.662944    -112.018233  SharpieTech             1       ...         \n",
       "2     33.662944    -112.018233  SharpieTech             1       ...         \n",
       "\n",
       "   biz_catBiz_Wok  biz_catBiz_Women's Clothing  \\\n",
       "0             0.0                          0.0   \n",
       "1             0.0                          0.0   \n",
       "2             0.0                          0.0   \n",
       "\n",
       "   biz_catBiz_Workers Compensation Law  biz_catBiz_Wraps  \\\n",
       "0                                  0.0               0.0   \n",
       "1                                  0.0               0.0   \n",
       "2                                  0.0               0.0   \n",
       "\n",
       "   biz_catBiz_Yelp Events  biz_catBiz_Yoga  biz_catBiz_Ziplining  \\\n",
       "0                     0.0              0.0                   0.0   \n",
       "1                     0.0              0.0                   0.0   \n",
       "2                     0.0              0.0                   0.0   \n",
       "\n",
       "   biz_catBiz_Zoos  usr_year_experience  usr_stars_bias  \n",
       "0              0.0                    0            -1.0  \n",
       "1              0.0                    0            -0.5  \n",
       "2              0.0                    0            -1.0  \n",
       "\n",
       "[3 rows x 2487 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rev_user_biz.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.4. Clean Text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "from nltk.corpus import stopwords\n",
    "stopw = set(stopwords.words('english'))\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "import re\n",
    "\n",
    "def clean_str(string):\n",
    "    \"\"\"\n",
    "    Tokenization/string cleaning for all datasets except for SST.\n",
    "    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py\n",
    "    \"\"\"\n",
    "    string = re.sub(r\"[^A-Za-z0-9]\", \" \", string)     \n",
    "    string = re.sub(r\"\\'s\", \" \\'s\", string) \n",
    "    string = re.sub(r\"\\'ve\", \" \\'ve\", string) \n",
    "    string = re.sub(r\"n\\'t\", \" n\\'t\", string) \n",
    "    string = re.sub(r\"\\'re\", \" \\'re\", string) \n",
    "    string = re.sub(r\"\\'d\", \" \\'d\", string) \n",
    "    string = re.sub(r\"\\'ll\", \" \\'ll\", string) \n",
    "    string = re.sub(r\",\", \" , \", string) \n",
    "    string = re.sub(r\"!\", \" ! \", string) \n",
    "    string = re.sub(r\"\\(\", \" \\( \", string) \n",
    "    string = re.sub(r\"\\)\", \" \\) \", string) \n",
    "    string = re.sub(r\"\\?\", \" \\? \", string) \n",
    "    string = re.sub(r\"\\s{2,}\", \" \", string) \n",
    "    #string = re.sub(r'\\([^)]*\\)', '', string)\n",
    "\n",
    "    return string.strip().lower()\n",
    "\n",
    "#cleaned_text = []\n",
    "#for review in text_variable:\n",
    "#    cleaned_text.append( [str(word) for word in clean_str(review).split()] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10000x477242 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 1159818 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "text_variable = rev_user_biz['text']\n",
    "\n",
    "cleaned_text = text_variable.map(lambda x: clean_str(x).split())\n",
    "cleaned_text = pd.Series(cleaned_text).map(lambda x: [word for word in x if word not in stopw] )\n",
    "\n",
    "binary_vectorizer1 = CountVectorizer(binary=True,ngram_range=(0,2), stop_words=stopw)\n",
    "clean_review = text_variable.map(lambda x: clean_str(x))\n",
    "binary_vectorizer1.fit( clean_review )\n",
    "\n",
    "rev_text_binary = binary_vectorizer1.transform( clean_review )\n",
    "rev_text_binary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'', 0),\n",
       " (u'served let', 368152),\n",
       " (u'say say', 359040),\n",
       " (u'mattress ready', 256048),\n",
       " (u'dance clubs', 101856),\n",
       " (u'fearful food', 151388),\n",
       " (u'gah', 170547),\n",
       " (u'ppchen tolle', 320292),\n",
       " (u'say saw', 359039),\n",
       " (u'noon missed', 282115)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "vocabulary_list1 = list(zip( binary_vectorizer1.vocabulary_.keys(), binary_vectorizer1.vocabulary_.values()) )\n",
    "vocabulary_list1[0:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10000x4177 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 50628 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "text_variable = rev_user_biz['biz_name']\n",
    "\n",
    "cleaned_name = text_variable.map(lambda x: clean_str(x).split())\n",
    "cleaned_name = pd.Series(cleaned_text).map(lambda x: [word for word in x if word not in stopw] )\n",
    "\n",
    "binary_vectorizer2 = CountVectorizer(binary=True,ngram_range=(0,2), stop_words=stopw)\n",
    "clean_review = text_variable.map(lambda x: clean_str(x))\n",
    "binary_vectorizer2.fit( clean_review )\n",
    "\n",
    "rev_name_binary = binary_vectorizer2.transform( clean_review )\n",
    "rev_name_binary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'', 0),\n",
       " (u'john varvatos', 2047),\n",
       " (u'luckenbooth caf', 2340),\n",
       " (u'sichuan', 3426),\n",
       " (u'asian', 205),\n",
       " (u'francesco', 1564),\n",
       " (u'battleground', 346),\n",
       " (u'canyon park', 671),\n",
       " (u'sweetlix', 3692),\n",
       " (u'chiropractic health', 801)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "vocabulary_list2 = list(zip( binary_vectorizer2.vocabulary_.keys(), binary_vectorizer2.vocabulary_.values()) )\n",
    "vocabulary_list2[0:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10000x481419 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 1210446 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from scipy.sparse import coo_matrix, hstack\n",
    "\n",
    "stack_txt = hstack( [rev_text_binary, rev_name_binary] )\n",
    "stack_txt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.5 Pandas to Sparse (numerical and categorical/dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataFrame\n",
      "   0  1\n",
      "0  1  2\n",
      "1  3  4\n",
      "2  5  6\n",
      "\n",
      "DataFrame to list\n",
      "[[1, 3, 5], [2, 4, 6]]\n",
      "\n",
      "Data in Sparse from DataFrame\n",
      "[[1 2]\n",
      " [3 4]\n",
      " [5 6]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<3x2 sparse matrix of type '<type 'numpy.int64'>'\n",
       "\twith 6 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "##### Example of sparse matrix for the rest of the data\n",
    "\n",
    "row1 = [1, 2]\n",
    "row2 = [3, 4]\n",
    "row3 = [5, 6]\n",
    "rows = pd.DataFrame([row1,row2,row3])\n",
    "\n",
    "print \"DataFrame\\n\", rows\n",
    "print \"\\nDataFrame to list\\n\", rows.transpose().values.tolist()\n",
    "\n",
    "A = coo_matrix( rows )\n",
    "print \"\\nData in Sparse from DataFrame\\n\", A.toarray()\n",
    "A\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>biz_is_open</th>\n",
       "      <th>biz_latitude</th>\n",
       "      <th>biz_longitude</th>\n",
       "      <th>biz_state_AZ</th>\n",
       "      <th>biz_state_BW</th>\n",
       "      <th>biz_state_EDH</th>\n",
       "      <th>biz_state_ELN</th>\n",
       "      <th>biz_state_ESX</th>\n",
       "      <th>...</th>\n",
       "      <th>biz_catBiz_Wok</th>\n",
       "      <th>biz_catBiz_Women's Clothing</th>\n",
       "      <th>biz_catBiz_Workers Compensation Law</th>\n",
       "      <th>biz_catBiz_Wraps</th>\n",
       "      <th>biz_catBiz_Yelp Events</th>\n",
       "      <th>biz_catBiz_Yoga</th>\n",
       "      <th>biz_catBiz_Ziplining</th>\n",
       "      <th>biz_catBiz_Zoos</th>\n",
       "      <th>usr_year_experience</th>\n",
       "      <th>usr_stars_bias</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>33.504054</td>\n",
       "      <td>-112.082813</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>33.504054</td>\n",
       "      <td>-112.082813</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2483 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_year  stars  biz_is_open  biz_latitude  biz_longitude  biz_state_AZ  \\\n",
       "0       2011      5            1     33.662944    -112.018233             1   \n",
       "1       2010      5            1     33.662944    -112.018233             1   \n",
       "2       2011      5            1     33.662944    -112.018233             1   \n",
       "3       2014      5            0     33.504054    -112.082813             1   \n",
       "4       2014      4            0     33.504054    -112.082813             1   \n",
       "\n",
       "   biz_state_BW  biz_state_EDH  biz_state_ELN  biz_state_ESX       ...        \\\n",
       "0             0              0              0              0       ...         \n",
       "1             0              0              0              0       ...         \n",
       "2             0              0              0              0       ...         \n",
       "3             0              0              0              0       ...         \n",
       "4             0              0              0              0       ...         \n",
       "\n",
       "   biz_catBiz_Wok  biz_catBiz_Women's Clothing  \\\n",
       "0             0.0                          0.0   \n",
       "1             0.0                          0.0   \n",
       "2             0.0                          0.0   \n",
       "3             0.0                          0.0   \n",
       "4             0.0                          0.0   \n",
       "\n",
       "   biz_catBiz_Workers Compensation Law  biz_catBiz_Wraps  \\\n",
       "0                                  0.0               0.0   \n",
       "1                                  0.0               0.0   \n",
       "2                                  0.0               0.0   \n",
       "3                                  0.0               0.0   \n",
       "4                                  0.0               0.0   \n",
       "\n",
       "   biz_catBiz_Yelp Events  biz_catBiz_Yoga  biz_catBiz_Ziplining  \\\n",
       "0                     0.0              0.0                   0.0   \n",
       "1                     0.0              0.0                   0.0   \n",
       "2                     0.0              0.0                   0.0   \n",
       "3                     0.0              0.0                   0.0   \n",
       "4                     0.0              0.0                   0.0   \n",
       "\n",
       "   biz_catBiz_Zoos  usr_year_experience  usr_stars_bias  \n",
       "0              0.0                    0            -1.0  \n",
       "1              0.0                    0            -0.5  \n",
       "2              0.0                    0            -1.0  \n",
       "3              0.0                    3             0.5  \n",
       "4              0.0                    3             0.5  \n",
       "\n",
       "[5 rows x 2483 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "names = ['business_id','user_id','review_id','usr_name','biz_name','text']\n",
    "rev_user_biz_clean = rev_user_biz[ [i for i in rev_user_biz.columns  if i not in names ] ]\n",
    "\n",
    "rev_user_biz_clean.head(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10000x2483 sparse matrix of type '<type 'numpy.float64'>'\n",
       "\twith 132913 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "sparse_rev_user_biz = coo_matrix( rev_user_biz_clean )\n",
    "sparse_rev_user_biz\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10000x483902 sparse matrix of type '<type 'numpy.float64'>'\n",
       "\twith 1343359 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "sparse_rev_user_biz = hstack( [stack_txt, sparse_rev_user_biz] )\n",
    "sparse_rev_user_biz\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "import pickle \n",
    "pickle.dump( sparse_rev_user_biz, open('../../DataProject/reviews_yelp_sample_preproc.p', 'wb') )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_year</th>\n",
       "      <th>stars</th>\n",
       "      <th>biz_is_open</th>\n",
       "      <th>biz_latitude</th>\n",
       "      <th>biz_longitude</th>\n",
       "      <th>biz_state_AZ</th>\n",
       "      <th>biz_state_BW</th>\n",
       "      <th>biz_state_EDH</th>\n",
       "      <th>biz_state_ELN</th>\n",
       "      <th>biz_state_ESX</th>\n",
       "      <th>...</th>\n",
       "      <th>biz_catBiz_Workers Compensation Law</th>\n",
       "      <th>biz_catBiz_Wraps</th>\n",
       "      <th>biz_catBiz_Yelp Events</th>\n",
       "      <th>biz_catBiz_Yoga</th>\n",
       "      <th>biz_catBiz_Ziplining</th>\n",
       "      <th>biz_catBiz_Zoos</th>\n",
       "      <th>usr_year_experience</th>\n",
       "      <th>usr_stars_bias</th>\n",
       "      <th>biz_name</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>[enjoy, service, someone, competent, personabl...</td>\n",
       "      <td>[enjoy, service, someone, competent, personabl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>[phone, verizon, wireless, trying, figure, pho...</td>\n",
       "      <td>[phone, verizon, wireless, trying, figure, pho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>33.662944</td>\n",
       "      <td>-112.018233</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>[great, service, corey, service, oriented, wor...</td>\n",
       "      <td>[great, service, corey, service, oriented, wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>33.504054</td>\n",
       "      <td>-112.082813</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>[highly, recommended, went, yesterday, looking...</td>\n",
       "      <td>[highly, recommended, went, yesterday, looking...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>33.504054</td>\n",
       "      <td>-112.082813</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>[walked, looking, specific, piece, furniture, ...</td>\n",
       "      <td>[walked, looking, specific, piece, furniture, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2485 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_year  stars  biz_is_open  biz_latitude  biz_longitude  biz_state_AZ  \\\n",
       "0       2011      5            1     33.662944    -112.018233             1   \n",
       "1       2010      5            1     33.662944    -112.018233             1   \n",
       "2       2011      5            1     33.662944    -112.018233             1   \n",
       "3       2014      5            0     33.504054    -112.082813             1   \n",
       "4       2014      4            0     33.504054    -112.082813             1   \n",
       "\n",
       "   biz_state_BW  biz_state_EDH  biz_state_ELN  biz_state_ESX  \\\n",
       "0             0              0              0              0   \n",
       "1             0              0              0              0   \n",
       "2             0              0              0              0   \n",
       "3             0              0              0              0   \n",
       "4             0              0              0              0   \n",
       "\n",
       "                         ...                          \\\n",
       "0                        ...                           \n",
       "1                        ...                           \n",
       "2                        ...                           \n",
       "3                        ...                           \n",
       "4                        ...                           \n",
       "\n",
       "   biz_catBiz_Workers Compensation Law  biz_catBiz_Wraps  \\\n",
       "0                                  0.0               0.0   \n",
       "1                                  0.0               0.0   \n",
       "2                                  0.0               0.0   \n",
       "3                                  0.0               0.0   \n",
       "4                                  0.0               0.0   \n",
       "\n",
       "   biz_catBiz_Yelp Events  biz_catBiz_Yoga  biz_catBiz_Ziplining  \\\n",
       "0                     0.0              0.0                   0.0   \n",
       "1                     0.0              0.0                   0.0   \n",
       "2                     0.0              0.0                   0.0   \n",
       "3                     0.0              0.0                   0.0   \n",
       "4                     0.0              0.0                   0.0   \n",
       "\n",
       "   biz_catBiz_Zoos  usr_year_experience  usr_stars_bias  \\\n",
       "0              0.0                    0            -1.0   \n",
       "1              0.0                    0            -0.5   \n",
       "2              0.0                    0            -1.0   \n",
       "3              0.0                    3             0.5   \n",
       "4              0.0                    3             0.5   \n",
       "\n",
       "                                            biz_name  \\\n",
       "0  [enjoy, service, someone, competent, personabl...   \n",
       "1  [phone, verizon, wireless, trying, figure, pho...   \n",
       "2  [great, service, corey, service, oriented, wor...   \n",
       "3  [highly, recommended, went, yesterday, looking...   \n",
       "4  [walked, looking, specific, piece, furniture, ...   \n",
       "\n",
       "                                                text  \n",
       "0  [enjoy, service, someone, competent, personabl...  \n",
       "1  [phone, verizon, wireless, trying, figure, pho...  \n",
       "2  [great, service, corey, service, oriented, wor...  \n",
       "3  [highly, recommended, went, yesterday, looking...  \n",
       "4  [walked, looking, specific, piece, furniture, ...  \n",
       "\n",
       "[5 rows x 2485 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "rev_user_biz_clean.loc[:,'biz_name'] = cleaned_name\n",
    "rev_user_biz_clean.loc[:,'text'] = cleaned_text\n",
    "\n",
    "rev_user_biz_clean.head(5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "pickle.dump( rev_user_biz_clean, open('../../DataProject/reviews_yelp_sample.p', 'wb') )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 4. Starting LDA "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sources : \n",
    "    * LDA  and DMR : https://github.com/mpkato/dmr/tree/master/dmr\n",
    "    * sLDA: https://github.com/arongdari/python-topic-model\n",
    "    https://github.com/akullpp/SLDA\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "import vocabulary as v\n",
    "import lda\n",
    "\n",
    "voca = v.Vocabulary()\n",
    "docs = voca.read_corpus( rev_user_biz_clean['text'] )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab 37929\n",
      "reviews 10000\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print \"vocab\",voca.size()\n",
    "print \"reviews\",( len(rev_user_biz_clean['text']) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "reload(lda)\n",
    "\n",
    "lda = lda.LDA( 10, 0.1, 0.01, docs, voca.size() )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1\n",
      "0.01\n",
      "3945.31748205\n"
     ]
    }
   ],
   "source": [
    "print lda.alpha\n",
    "print lda.beta\n",
    "print lda.perplexity()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.23666667,  0.07      ,  0.07      , ...,  0.10333333,\n",
       "         0.17      ,  0.07      ],\n",
       "       [ 0.15      ,  0.15      ,  0.03888889, ...,  0.11296296,\n",
       "         0.15      ,  0.09444444],\n",
       "       [ 0.11666667,  0.17222222,  0.00555556, ...,  0.06111111,\n",
       "         0.17222222,  0.06111111],\n",
       "       ..., \n",
       "       [ 0.10111732,  0.07318436,  0.13463687, ...,  0.10670391,\n",
       "         0.08435754,  0.10111732],\n",
       "       [ 0.1       ,  0.05121951,  0.07560976, ...,  0.05121951,\n",
       "         0.17317073,  0.12439024],\n",
       "       [ 0.12638889,  0.08472222,  0.12638889, ...,  0.14027778,\n",
       "         0.1125    ,  0.02916667]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "lda.topicdist()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}