ExploratoryAnalysis_AddingIncomeTo2018Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sb\n",
    "color = sb.color_palette()\n",
    "import matplotlib as mpl\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn import preprocessing as pp \n",
    "#import the entire linear model\n",
    "from sklearn import linear_model\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score\n",
    "from scipy.stats import pearsonr \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LassoCV\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read the new excel file, this contains the annual pay for indiviual employees\n",
    "#Read Excel data provided. Read the first tab.\n",
    "current_path = os.getcwd()\n",
    "file = '/datasets/ActivatedInsights/ExtraDataForAkira.xlsx'\n",
    "data1 = pd.read_excel(current_path + file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OLD # May 2018 Location Code</th>\n",
       "      <th>New # May 2019 Location Code</th>\n",
       "      <th>Department- Updated</th>\n",
       "      <th>LGBT</th>\n",
       "      <th>Managerial Level- Updated</th>\n",
       "      <th>Meaningful Innovation Opportunities</th>\n",
       "      <th>Pay Type</th>\n",
       "      <th>Responsibility</th>\n",
       "      <th>Role</th>\n",
       "      <th>Tenure</th>\n",
       "      <th>Work Status</th>\n",
       "      <th>Annual pay</th>\n",
       "      <th>zip(home)</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Action/Reason Description</th>\n",
       "      <th>Voluntary/Involuntary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1075.0</td>\n",
       "      <td>1075.0</td>\n",
       "      <td>Admin</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Employee-IC</td>\n",
       "      <td>A lot</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>Neither children nor elders</td>\n",
       "      <td>RECRUITING PROGRAM COORDINATOR</td>\n",
       "      <td>6 months to 9 months</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>52000.0</td>\n",
       "      <td>32806</td>\n",
       "      <td>RECRUITING PROGRAM COORDINATOR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5504.0</td>\n",
       "      <td>5954.0</td>\n",
       "      <td>Activities</td>\n",
       "      <td>No</td>\n",
       "      <td>Employee-IC</td>\n",
       "      <td>A lot</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>Both children and elders</td>\n",
       "      <td>ENRICHMENT COORDINATOR</td>\n",
       "      <td>9 months to 1 year</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>33280.0</td>\n",
       "      <td>80212</td>\n",
       "      <td>ENRICHMENT COORDINATOR</td>\n",
       "      <td>Resignation</td>\n",
       "      <td>Voluntary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5504.0</td>\n",
       "      <td>5954.0</td>\n",
       "      <td>Housekeeping-Maint-Sec</td>\n",
       "      <td>No</td>\n",
       "      <td>Frontline manager</td>\n",
       "      <td>A lot</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>Both children and elders</td>\n",
       "      <td>HEAD HOUSEKEEPER</td>\n",
       "      <td>Over 20 years</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>34632.0</td>\n",
       "      <td>80226</td>\n",
       "      <td>HEAD HOUSEKEEPER</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5504.0</td>\n",
       "      <td>5954.0</td>\n",
       "      <td>Admin</td>\n",
       "      <td>No</td>\n",
       "      <td>Frontline manager</td>\n",
       "      <td>A lot</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>Elders</td>\n",
       "      <td>ASSISTANT GENERAL MANAGER</td>\n",
       "      <td>1 year to 2 years</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>45177.6</td>\n",
       "      <td>80107</td>\n",
       "      <td>ASSISTANT GENERAL MANAGER</td>\n",
       "      <td>Resignation</td>\n",
       "      <td>Voluntary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5504.0</td>\n",
       "      <td>5954.0</td>\n",
       "      <td>Housekeeping-Maint-Sec</td>\n",
       "      <td>No</td>\n",
       "      <td>Employee-IC</td>\n",
       "      <td>A lot</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>Both children and elders</td>\n",
       "      <td>HOUSEKEEPING (CLEANING)</td>\n",
       "      <td>1 year to 2 years</td>\n",
       "      <td>Full-time</td>\n",
       "      <td>29120.0</td>\n",
       "      <td>80219</td>\n",
       "      <td>HOUSEKEEPING (CLEANING)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OLD # May 2018 Location Code  New # May 2019 Location Code  \\\n",
       "0                        1075.0                        1075.0   \n",
       "1                        5504.0                        5954.0   \n",
       "2                        5504.0                        5954.0   \n",
       "3                        5504.0                        5954.0   \n",
       "4                        5504.0                        5954.0   \n",
       "\n",
       "      Department- Updated LGBT Managerial Level- Updated  \\\n",
       "0                   Admin  Yes               Employee-IC   \n",
       "1              Activities   No               Employee-IC   \n",
       "2  Housekeeping-Maint-Sec   No         Frontline manager   \n",
       "3                   Admin   No         Frontline manager   \n",
       "4  Housekeeping-Maint-Sec   No               Employee-IC   \n",
       "\n",
       "  Meaningful Innovation Opportunities Pay Type               Responsibility  \\\n",
       "0                               A lot   Hourly  Neither children nor elders   \n",
       "1                               A lot   Hourly     Both children and elders   \n",
       "2                               A lot   Hourly     Both children and elders   \n",
       "3                               A lot   Hourly                       Elders   \n",
       "4                               A lot   Hourly     Both children and elders   \n",
       "\n",
       "                             Role                Tenure Work Status  \\\n",
       "0  RECRUITING PROGRAM COORDINATOR  6 months to 9 months   Full-time   \n",
       "1          ENRICHMENT COORDINATOR    9 months to 1 year   Full-time   \n",
       "2                HEAD HOUSEKEEPER         Over 20 years   Full-time   \n",
       "3       ASSISTANT GENERAL MANAGER     1 year to 2 years   Full-time   \n",
       "4         HOUSEKEEPING (CLEANING)     1 year to 2 years   Full-time   \n",
       "\n",
       "   Annual pay zip(home)                       Job Title  \\\n",
       "0     52000.0     32806  RECRUITING PROGRAM COORDINATOR   \n",
       "1     33280.0     80212          ENRICHMENT COORDINATOR   \n",
       "2     34632.0     80226                HEAD HOUSEKEEPER   \n",
       "3     45177.6     80107       ASSISTANT GENERAL MANAGER   \n",
       "4     29120.0     80219         HOUSEKEEPING (CLEANING)   \n",
       "\n",
       "  Action/Reason Description Voluntary/Involuntary  \n",
       "0                       NaN                   NaN  \n",
       "1               Resignation             Voluntary  \n",
       "2                       NaN                   NaN  \n",
       "3               Resignation             Voluntary  \n",
       "4                       NaN                   NaN  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Rename columns to remove the space and make it shorter.\n",
    "data1.rename(columns = {'OLD # May 2018 Location Code':'LocationCode2018'}, inplace = True) \n",
    "data1.rename(columns = {'New # May 2019 Location Code':'LocationCode2019'}, inplace = True) \n",
    "data1.rename(columns = {'Annual pay':'AnnualPay'}, inplace = True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1075., 5954., 5573., 5563., 5581., 5945., 5676., 5978., 5643.,\n",
       "       5077., 5963., 5648., 5493., 6501., 5669., 5655., 5678., 5932.,\n",
       "       5671., 5936.,   nan, 5417., 5199., 5944., 5586., 5970., 5126.,\n",
       "       5482., 5484., 5310., 5979., 5938., 5553., 5280., 5024., 5477.,\n",
       "       5683., 5951., 5931., 5964., 5906., 5056., 5269., 5680., 5935.,\n",
       "       5986., 5907., 5331., 5475., 5370., 5649., 5568., 5646., 5929.,\n",
       "       5677., 5473., 5657., 5990., 5476., 5983., 5972., 5930., 5682.,\n",
       "       5495., 5939., 5941., 5200., 5948., 5137., 5937., 5065., 5078.,\n",
       "       5015., 5301., 5974., 5940., 5966., 5681., 5658., 5560., 5478.,\n",
       "       5995., 5640., 5967., 5240., 5981., 5660., 5946., 5437., 5318.,\n",
       "       5565., 5174., 5580., 5999., 5494., 5962., 5481., 5644., 5564.,\n",
       "       5091., 5965., 5359., 5675., 5968., 5953., 5492., 5562., 5663.,\n",
       "       5574., 5567., 5992., 5168., 5109., 5998., 5361., 5020., 5673.,\n",
       "       5674., 5647., 5684., 5372., 5311., 5943., 5665., 5587., 5971.,\n",
       "       5052., 5991., 5653., 5048., 5556., 5933., 5989., 5085., 5486.,\n",
       "       5575., 5265., 5306., 5666., 5409., 5377., 5949., 5490., 5996.,\n",
       "       5220., 5942., 5120., 5208., 5428., 5668., 5585., 5212., 5485.,\n",
       "       5642., 5651., 5480., 5973., 5566., 5993., 5670., 5491., 5552.,\n",
       "       5145., 5378., 5606., 5427., 5021., 5474., 5956., 5032., 5982.,\n",
       "       5479., 5583., 5154., 5124., 5977., 5244., 5955., 5418., 5928.,\n",
       "       5952., 5662., 5650., 5988., 5626., 5961., 5987., 5404., 5969.,\n",
       "       5086., 5679., 5497., 5151., 5960., 5672., 5950., 5255., 5166.,\n",
       "       5664., 5661., 5997., 5975., 5559., 5195., 5483., 5555., 5415.,\n",
       "       5554., 5047., 5167., 5645., 5388., 5984., 5045., 5042., 5652.,\n",
       "       5185., 5558., 5572., 5980., 5569., 5038., 5659., 5571., 5959.,\n",
       "       5957., 5498., 5656., 5667., 5271., 5489., 5488., 5570., 5136.,\n",
       "       5023., 5976., 5602., 5281., 5203., 5487., 5934., 5435., 5584.,\n",
       "       5654., 5947., 5254., 5433., 5641., 5576., 5146., 5320., 5557.,\n",
       "       5985., 5051., 5022., 5496., 5072., 5994., 5561., 5551.])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TestLocationCodes=data1.LocationCode2019.unique()\n",
    "TestLocationCodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load the 2018 May dataset.\n",
    "#Read Excel data provided. Read the first tab.\n",
    "current_path = os.getcwd()\n",
    "file2 = '/datasets/ActivatedInsights/ActivatedInsight.xlsx'\n",
    "data2 = pd.read_excel(current_path + file2)\n",
    "\n",
    "#First two rows are excel column names that only exists for some columns. Remove them for now.\n",
    "#Also some of the last few rows have no data in most columns.\n",
    "data2 = data2.iloc[2:-13]\n",
    "#LocationCode will contain the column for the location code for this study use the May 2019 code\n",
    "LocationCode_for_data = data2.iloc[:,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get median pay for each location. Go through the location code and get the data\n",
    "#Ignore NaN (be default)\n",
    "#DataArrayPCA_answers_df=pd.DataFrame(data=DataArrayPCA_answers)\n",
    "\n",
    "#\n",
    "location_pay = np.zeros((LocationCode_for_data.shape[0],1))\n",
    "\n",
    "#Get the data for each location, calculate the median pay for that location\n",
    "for index in range(LocationCode_for_data.shape[0]):\n",
    "    location_data=data1.loc[data1['LocationCode2019'] == LocationCode_for_data.iloc[index],'AnnualPay']\n",
    "    #calculate the media for that location\n",
    "    location_pay[index] = location_data.median()\n",
    "location_pay = pd.DataFrame(data=location_pay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f98dc37d550>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAEICAYAAACpqsStAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAROklEQVR4nO3dbYxcZ3nG8f/dOG9kSewQurKcqA4lBUJc0nibAqnQOqlEiBHJhyClRcihQZZ4awpGxRSp0A+ogYoSUFGRm1DMS9kEkyoIN22j4C2t1BhsSHCCS2ySNMSkcRGJYSMEGO5+mGMydmZ2Z3ZndvbG/5+02pnnnLNzzePZa8+eM2cdmYkkqaZfG3UASdL8WeKSVJglLkmFWeKSVJglLkmFWeKSVJglLkmFWeI6rkXEmRHxTxHxVET8T0T80agzSf1YNuoA0oh9DPgpMA5cCGyPiHsz8/7RxpJ6E16xqeNVRJwGPAFckJkPNGOfBg5k5uaRhpN65OEUHc9+C/j5kQJv3Au8eER5pL5Z4jqejQGHjhk7BDx7BFmkebHEdTybAU4/Zux04EcjyCLNiyWu49kDwLKIOK9t7CWAJzVVhic2dVyLiCkggTfSenfKPwMv990pqsI9cR3v3gycChwEPge8yQJXJe6JS1Jh7olLUmGWuCQVZolLUmGWuCQVtqh/AOuss87K1atX97XNU089xWmnnTacQENQKW+lrGDeYTPv8Cw06+7du7+fmc/tuDAzF+1j7dq12a8dO3b0vc0oVcpbKWumeYfNvMOz0KzAruzSqx5OkaTCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCFvWye9WwevP2kT32wzesH9ljSxW5Jy5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhfVU4hHx9oi4PyLui4jPRcQpEXFuROyMiH0RcUtEnDTssJKko81Z4hGxCvgTYCIzLwBOAK4BPgB8ODPPA54ArhtmUEnSM/V6OGUZcGpELAOeBTwGXApsa5ZvBa4afDxJ0mwiM+deKeJ64P3Aj4F/A64H7s7M5zfLzwHuaPbUj912I7ARYHx8fO3U1FRfAWdmZhgbG+trm1GqlLdb1j0HDo0gTcuaVWd0XVZpbsG8w1Yp70Kzrlu3bndmTnRatmyujSNiBXAlcC7wJPB54FUdVu340yAztwBbACYmJnJycrK31I3p6Wn63WaUKuXtlvXazdsXP0zj4ddNdl1WaW7BvMNWKe8ws/ZyOOUPgIcy8/8y82fAbcDLgeXN4RWAs4HvDSWhJKmrXkr8EeClEfGsiAjgMuBbwA7g6madDcDtw4koSepmzhLPzJ20TmB+HdjTbLMFeBfwjojYDzwHuHmIOSVJHcx5TBwgM98LvPeY4QeBiweeSJLUM6/YlKTCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCLHFJKswSl6TCeirxiFgeEdsi4r8jYm9EvCwizoyIOyNiX/N5xbDDSpKO1uue+EeAf8nMFwIvAfYCm4G7MvM84K7mviRpEc1Z4hFxOvAK4GaAzPxpZj4JXAlsbVbbClw1rJCSpM4iM2dfIeJCYAvwLVp74buB64EDmbm8bb0nMvMZh1QiYiOwEWB8fHzt1NRUXwFnZmYYGxvra5tRqpS3W9Y9Bw6NIE3LmlVndF1WaW7BvMNWKe9Cs65bt253Zk50WtZLiU8AdwOXZObOiPgI8EPgbb2UeLuJiYnctWtXX+Gnp6eZnJzsa5tRqpS3W9bVm7cvfpjGwzes77qs0tyCeYetUt6FZo2IriXeyzHxR4FHM3Nnc38bcBHweESsbB5gJXBw3gklSfMyZ4ln5v8C342IFzRDl9E6tPJFYEMztgG4fSgJJUldLetxvbcBn42Ik4AHgTfQ+gFwa0RcBzwCvHY4ESVJ3fRU4pl5D9DpeMxlg40jSeqHV2xKUmG9Hk6RFsVs74zZtOYw1w7pnTOzvStGWsrcE5ekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwixxSSrMEpekwnou8Yg4ISK+ERFfau6fGxE7I2JfRNwSEScNL6YkqZN+9sSvB/a23f8A8OHMPA94ArhukMEkSXPrqcQj4mxgPXBTcz+AS4FtzSpbgauGEVCS1F1k5twrRWwD/gp4NvBO4Frg7sx8frP8HOCOzLygw7YbgY0A4+Pja6empvoKODMzw9jYWF/bjFKlvN2y7jlwaARp5jZ+Kjz+4+F87TWrzhj416z0WgDzDtNCs65bt253Zk50WrZsro0j4tXAwczcHRGTR4Y7rNrxp0FmbgG2AExMTOTk5GSn1bqanp6m321GqVLeblmv3bx98cP0YNOaw3xoz5wv2Xl5+HWTA/+alV4LYN5hGmbWXr4jLgFeExFXAKcApwM3AssjYllmHgbOBr43lISSpK7mPCaeme/OzLMzczVwDfDlzHwdsAO4ulltA3D70FJKkjpayPvE3wW8IyL2A88Bbh5MJElSr/o6wJiZ08B0c/tB4OLBR5Ik9corNiWpMEtckgqzxCWpMEtckgqzxCWpsOFc/qaBWD3kKyc3rTm8ZK/OlNQb98QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqTBLXJIKs8QlqbBlow7Qq9Wbt4/kcR++Yf1IHleLaxivr01rDnNtD1/X15gWwj1xSSrMEpekwixxSSrMEpekwixxSSpszhKPiHMiYkdE7I2I+yPi+mb8zIi4MyL2NZ9XDD+uJKldL3vih4FNmfki4KXAWyLifGAzcFdmngfc1dyXJC2iOUs8Mx/LzK83t38E7AVWAVcCW5vVtgJXDSukJKmzyMzeV45YDXwFuAB4JDOXty17IjOfcUglIjYCGwHGx8fXTk1N9RVwZmaGsbEx9hw41Nd2g7Jm1Rl9rX8k7yAM+zmPnwqP/3ioDzFQv6p5+32NDcsgX7uLoVLehWZdt27d7syc6LSs5xKPiDHg34H3Z+ZtEfFkLyXebmJiInft2tVHdJienmZycrLMFZtH8g7CsJ/zpjWH+dCeMhft/srmXSpXbA7ytbsYKuVdaNaI6FriPb07JSJOBL4AfDYzb2uGH4+Ilc3ylcDBeSeUJM1LL+9OCeBmYG9m/k3boi8CG5rbG4DbBx9PkjSbXn43vQR4PbAnIu5pxv4cuAG4NSKuAx4BXjuciJKkbuYs8cz8TyC6LL5ssHEkSf3wik1JKqzOqf4R6fcdIr3+DWlJGgT3xCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgpbNuoA0vFu9ebtI3nch29YP5LH1WC5Jy5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYJS5JhVniklSYl91Lx6ljL/fftOYw1y7SnwDwkv/BcU9ckgpb0J54RFwOfAQ4AbgpM28YSCpJGoJR/bGxT15+2tC+9rz3xCPiBOBjwKuA84E/jIjzBxVMkjS3hRxOuRjYn5kPZuZPgSngysHEkiT1IjJzfhtGXA1cnplvbO6/Hvi9zHzrMettBDY2d18AfLvPhzoL+P68Qo5GpbyVsoJ5h828w7PQrL+Rmc/ttGAhx8Sjw9gzfiJk5hZgy7wfJGJXZk7Md/vFVilvpaxg3mEz7/AMM+tCDqc8CpzTdv9s4HsLiyNJ6sdCSvxrwHkRcW5EnARcA3xxMLEkSb2Y9+GUzDwcEW8F/pXWWww/kZn3DyzZ0+Z9KGZEKuWtlBXMO2zmHZ6hZZ33iU1J0uh5xaYkFWaJS1Jhi1LiEfGJiDgYEfe1jZ0ZEXdGxL7m84pmPCLioxGxPyK+GREXtW2zoVl/X0RsaBtfGxF7mm0+GhGd3v640Lzvi4gDEXFP83FF27J3N4/97Yh4Zdv45c3Y/ojY3DZ+bkTsbJ7HLc2J4flmPScidkTE3oi4PyKub8aX5PzOknepzu8pEfHViLi3yfuXsz1GRJzc3N/fLF893+cx4LyfjIiH2ub3wmZ8KXy/nRAR34iILzX3l+TczpJ3tHObmUP/AF4BXATc1zb2QWBzc3sz8IHm9hXAHbTeh/5SYGczfibwYPN5RXN7RbPsq8DLmm3uAF41hLzvA97ZYd3zgXuBk4Fzge/QOtF7QnP7ecBJzTrnN9vcClzT3P448KYFZF0JXNTcfjbwQJNpSc7vLHmX6vwGMNbcPhHY2cxbx8cA3gx8vLl9DXDLfJ/HgPN+Eri6w/pL4fvtHcA/Al+a7d9v1HM7S96Rzu2i7Iln5leAHxwzfCWwtbm9FbiqbfxT2XI3sDwiVgKvBO7MzB9k5hPAncDlzbLTM/O/sjULn2r7WoPM282VwFRm/iQzHwL20/qTBB3/LEHzk/VSYFuH5z6frI9l5teb2z8C9gKrWKLzO0vebkY9v5mZM83dE5uPnOUx2ud9G3BZk6mv5zGEvN2M9PUQEWcD64Gbmvuz/fuNdG475Z3DosztKI+Jj2fmY9D6xgZ+vRlfBXy3bb1Hm7HZxh/tMD4Mb21+LfpENIcn5pH3OcCTmXl40HmbXy9/h9be15Kf32PywhKd3+bX53uAg7S+4b4zy2P8Mlez/FCTqd/nMbC8mXlkft/fzO+HI+LkY/P2mGvQr4cbgT8DftHcn+3fb+Rz2yHvESOb26V4YrPb5fz9jg/a3wG/CVwIPAZ8qBlfEnkjYgz4AvCnmfnD2VbtM9di5V2y85uZP8/MC2ldlXwx8KJZHmPJ5Y2IC4B3Ay8EfpfWr/HvGnXeiHg1cDAzd7cPz/L1Rzq3XfLCiOd2lCX+ePPrA83ng814t8v5Zxs/u8P4QGXm4803xy+Av6f1zTyfvN+n9WvVsmPG5y0iTqRViJ/NzNua4SU7v53yLuX5PSIznwSmaR3f7PYYv8zVLD+D1qG5fp/HIPNe3hzGysz8CfAPzH9+B/l6uAR4TUQ8TOtQx6W09nSX6tw+I29EfGbkczvXQfNBfQCrOfpE4V9z9Im3Dza313P0yYCv5tMnAx6idSJgRXP7zGbZ15p1j5wMuGIIeVe23X47rWNwAC/m6JMqD9I6obKsuX0uT59UeXGzzec5+sTNmxeQM2gdO7vxmPElOb+z5F2q8/tcYHlz+1TgP4BXd3sM4C0cffLt1vk+jwHnXdk2/zcCNyyF10Nb7kmePlG4JOd2lrwjndsFP5ken/DnaP2K/DNaP22uo3Us6y5gX/P5yJMIWv/ZxHeAPcBE29f5Y1onLfYDb2gbnwDua7b5W5orUQec99NNnm/S+hsx7aXznuaxv03b2WRaZ6cfaJa9p238ebTOQu9vXrAnLyDr79P6leubwD3NxxVLdX5nybtU5/e3gW80ue4D/mK2xwBOae7vb5Y/b77PY8B5v9zM733AZ3j6HSwj/35rvuYkT5fikpzbWfKOdG697F6SCluKJzYlST2yxCWpMEtckgqzxCWpMEtckgqzxCWpMEtckgr7f2NK2Q405CxBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#We may not need log transform for this data.\n",
    "location_pay.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location_pay.isnull().values.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f98d82bcac8>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAO90lEQVR4nO3dYYwc9XnH8e9TG4TjAwwBXZGNekhxaahdaDlREqTqDLxwAgJagUSbIrslstQmqdu4KqZ5EfVFVEcVSZAaqbIgwmoRF0pAoLg0cQlXVKm4tQnEEBNAlBAbajcqOBxFpU6evrh1454P79zu3o0f3/cjWb6Znf/N71bj3/13dmYdmYkkqZ6faTuAJKk3FrgkFWWBS1JRFrgkFWWBS1JRFrgkFWWBS1JRFrgWtIg4OyIeioi3I+L7EfFbbWeSmlrcdgCpZV8G3gWGgUuA7RHxTGY+124sqbvwTkwtVBGxFHgDWJWZL3TW/TWwPzM3txpOasBTKFrIfh748ZHy7ngG+MWW8kizYoFrIRsCDk1bdwg4vYUs0qxZ4FrIJoEzpq07A3irhSzSrFngWsheABZHxMqj1l0M+AamSvBNTC1oETEOJPBxpq5C+Tvgw16FogqcgWuh+31gCXAQuA/4PctbVTgDl6SinIFLUlEWuCQVZYFLUlEWuCQVNa8fZnXOOefkyMjIvO3v7bffZunSpfO2v0GpmhvqZq+aG+pmN3dzu3fv/mFmnjt9/bwW+MjICLt27Zq3/U1MTDA2NjZv+xuUqrmhbvaquaFudnM3FxHfn2m9p1AkqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqah5vRNTNYxs3t7z2E2rD7O+j/GvbLmm57HSQuMMXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqSgLXJKKalTgEfFHEfFcRDwbEfdFxGkRcUFE7IyIFyPiqxFx6lyHlST9VNcCj4jlwB8Ao5m5ClgE3Ax8HvhiZq4E3gBuncugkqT/r+kplMXAkohYDLwPeB24Enig8/g24IbBx5MkvZfIzO4bRWwEPge8A3wT2Ag8mZkf6Dx+PvBoZ4Y+fewGYAPA8PDwpePj44NL38Xk5CRDQ0Pztr9BaTv3nv2Heh47vAQOvNP7vlcvP7P3wX1o+znvR9Xs5m5uzZo1uzNzdPr6xd0GRsRZwPXABcCbwN8CH5lh0xl/E2TmVmArwOjoaI6NjTVP3aeJiQnmc3+D0nbu9Zu39zx20+rD3LGn62H1nl752FjPY/vR9nPej6rZzd2/JqdQrgb+LTP/IzP/B3gQ+DCwrHNKBWAF8NocZZQkzaBJgb8KXB4R74uIAK4Cvgs8DtzY2WYd8PDcRJQkzaRrgWfmTqberHwK2NMZsxW4Dfh0RLwEvB+4ew5zSpKmaXSyMjM/C3x22uqXgcsGnkiS1Ih3YkpSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSURa4JBVlgUtSUY0KPCKWRcQDEfF8ROyNiA9FxNkRsSMiXuz8fdZch5Uk/VTTGfidwN9n5i8AFwN7gc3AY5m5EnissyxJmiddCzwizgB+DbgbIDPfzcw3geuBbZ3NtgE3zFVISdKxIjOPv0HEJcBW4LtMzb53AxuB/Zm57Kjt3sjMY06jRMQGYAPA8PDwpePj44NL38Xk5CRDQ0Pztr9BaTv3nv2Heh47vAQOvNP7vlcvP7P3wX1o+znvR9Xs5m5uzZo1uzNzdPr6JgU+CjwJXJGZOyPiTuBHwKeaFPjRRkdHc9euXT39AL2YmJhgbGxs3vY3KG3nHtm8veexm1Yf5o49i3se/8qWa3oe24+2n/N+VM1u7uYiYsYCb3IOfB+wLzN3dpYfAH4FOBAR53W++XnAwUGFlSR117XAM/PfgR9ExIWdVVcxdTrlEWBdZ9064OE5SShJmlHT17qfAu6NiFOBl4HfYar874+IW4FXgZvmJqIkaSaNCjwznwaOOf/C1GxcktQC78SUpKJ6v1xAmgP9XAHTj3vWLm1lv1I/nIFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlEWuCQVZYFLUlGNCzwiFkXEtyPi653lCyJiZ0S8GBFfjYhT5y6mJGm62czANwJ7j1r+PPDFzFwJvAHcOshgkqTja1TgEbECuAa4q7McwJXAA51NtgE3zEVASdLMIjO7bxTxAPDnwOnAHwPrgScz8wOdx88HHs3MVTOM3QBsABgeHr50fHx8YOG7mZycZGhoaN72Nyht596z/1DPY4eXwIF3Bhhmnlxw5qKSxwq0f7z0ytzNrVmzZndmjk5fv7jbwIi4FjiYmbsjYuzI6hk2nfE3QWZuBbYCjI6O5tjY2EybzYmJiQnmc3+D0nbu9Zu39zx20+rD3LGn62F1wrln7dKSxwq0f7z0ytz9a/Iv7Qrguoj4KHAacAbwJWBZRCzOzMPACuC1uYspSZqu6znwzLw9M1dk5ghwM/CtzPwY8DhwY2ezdcDDc5ZSknSMfq4Dvw34dES8BLwfuHswkSRJTczqZGVmTgATna9fBi4bfCRJUhPeiSlJRVngklSUBS5JRVngklSUBS5JRdW7ZW4BGenjjkhJJz9n4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJUlAUuSUVZ4JJU1OK2AzQ1snn7rMdsWn2Y9T2MO9orW67pa7xq2LP/UN/HSq88xtQrZ+CSVJQFLklFWeCSVJQFLklFWeCSVFTXAo+I8yPi8YjYGxHPRcTGzvqzI2JHRLzY+fusuY8rSTqiyQz8MLApMz8IXA58IiIuAjYDj2XmSuCxzrIkaZ50LfDMfD0zn+p8/RawF1gOXA9s62y2DbhhrkJKko4Vmdl844gR4AlgFfBqZi476rE3MvOY0ygRsQHYADA8PHzp+Ph4T0H37D806zHDS+DAOz3t7v+sXn5mf9+gB5OTkwwNDfX0M7dtEM95G9rM3e8xduR4qcbcza1Zs2Z3Zo5OX9+4wCNiCPhH4HOZ+WBEvNmkwI82Ojqau3btmmX0Kb3eiXnHnv5uNm3jLrmJiQnGxsZ6+pnbNojnvA1t5u73GDtyvFRj7uYiYsYCb3QVSkScAnwNuDczH+ysPhAR53UePw84OKiwkqTumlyFEsDdwN7M/MJRDz0CrOt8vQ54ePDxJEnvpclrxiuAW4A9EfF0Z92fAluA+yPiVuBV4Ka5iShJmknXAs/MfwLiPR6+arBxJElNeSemJBVV73KBedbGlSCD+BxzSSc/Z+CSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFWeCSVJQFLklFLW47gLTQjWze3tf4TasPs76H7/HKlmv62q/a5wxckoqywCWpKAtckoqywCWpKAtckoqywCWpKAtckoqywCWpKAtckoqywCWpKG+llxaofm/h74e38Q+GM3BJKqqvGXhErAXuBBYBd2XmloGkkqQ5MIhXHb18eNhcveLoeQYeEYuALwMfAS4CfjMiLhpUMEnS8fVzCuUy4KXMfDkz3wXGgesHE0uS1E1kZm8DI24E1mbmxzvLtwC/mpmfnLbdBmBDZ/FC4Hu9x521c4AfzuP+BqVqbqibvWpuqJvd3M39XGaeO31lP+fAY4Z1x/w2yMytwNY+9tOziNiVmaNt7LsfVXND3exVc0Pd7ObuXz+nUPYB5x+1vAJ4rb84kqSm+inwfwVWRsQFEXEqcDPwyGBiSZK66fkUSmYejohPAt9g6jLCr2TmcwNLNhitnLoZgKq5oW72qrmhbnZz96nnNzElSe3yTkxJKsoCl6SiTvoCj4i/iIjnI+I7EfFQRCxrO1MTEXFTRDwXET+JiBPikqXjiYi1EfG9iHgpIja3naepiPhKRByMiGfbzjIbEXF+RDweEXs7x8nGtjM1ERGnRcS/RMQzndx/1nam2YqIRRHx7Yj4ettZTvoCB3YAqzLzl4AXgNtbztPUs8BvAE+0HaSb4h+rcA+wtu0QPTgMbMrMDwKXA58o8pz/N3BlZl4MXAKsjYjLW840WxuBvW2HgAVQ4Jn5zcw83Fl8kqnr1U94mbk3M+fzrtV+lP1Yhcx8AvjPtnPMVma+nplPdb5+i6lCWd5uqu5yymRn8ZTOnzJXUkTECuAa4K62s8ACKPBpfhd4tO0QJ6HlwA+OWt5HgTI5WUTECPDLwM52kzTTOQXxNHAQ2JGZJXJ3fAn4E+AnbQeBk+Q/dIiIfwB+doaHPpOZD3e2+QxTLzvvnc9sx9MkdxGNPlZBgxcRQ8DXgD/MzB+1naeJzPwxcEnn/aiHImJVZp7w70FExLXAwczcHRFjbeeBk6TAM/Pq4z0eEeuAa4Gr8gS68L1b7kL8WIUWRMQpTJX3vZn5YNt5Zisz34yICabegzjhCxy4ArguIj4KnAacERF/k5m/3Vagk/4USuc/nbgNuC4z/6vtPCcpP1ZhnkVEAHcDezPzC23naSoizj1yJVhELAGuBp5vN1UzmXl7Zq7IzBGmjvFvtVnesAAKHPhL4HRgR0Q8HRF/1XagJiLi1yNiH/AhYHtEfKPtTO+l8ybxkY9V2AvcfwJ+rMKMIuI+4J+BCyNiX0Tc2namhq4AbgGu7BzXT3dmhie684DHI+I7TP3i35GZrV+OV5W30ktSUQthBi5JJyULXJKKssAlqSgLXJKKssAlqSgLXJKKssAlqaj/BVZ9/CLuq2KZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Scale and standardize\n",
    "scaler_location_pay = pp.StandardScaler().fit(location_pay)\n",
    "location_payS = scaler_location_pay.transform(location_pay)\n",
    "location_payS_df=pd.DataFrame(data=location_payS)\n",
    "location_payS_df.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "location_payS_df.to_pickle(\"./location_pay.pkl\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}