DW08_Imputation_Linear_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "38a3150f",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Boston-house-prices-data\" data-toc-modified-id=\"Boston-house-prices-data-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Boston house prices data</a></span></li><li><span><a href=\"#Splitting-the-data-into-train-and-test-set\" data-toc-modified-id=\"Splitting-the-data-into-train-and-test-set-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Splitting the data into train and test set</a></span></li><li><span><a href=\"#Examine-AGE-column\" data-toc-modified-id=\"Examine-AGE-column-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Examine <code>AGE</code> column</a></span></li><li><span><a href=\"#Examine-the-model-performance-with-different-data\" data-toc-modified-id=\"Examine-the-model-performance-with-different-data-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Examine the model performance with different data</a></span><ul class=\"toc-item\"><li><span><a href=\"#Performance-of-Completed-Data\" data-toc-modified-id=\"Performance-of-Completed-Data-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>Performance of Completed Data</a></span></li><li><span><a href=\"#Performance-of-Dropped-Data\" data-toc-modified-id=\"Performance-of-Dropped-Data-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>Performance of Dropped Data</a></span></li><li><span><a href=\"#Performance-of-Imputed-Data\" data-toc-modified-id=\"Performance-of-Imputed-Data-4.3\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>Performance of Imputed Data</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dab5191",
   "metadata": {},
   "source": [
    "# Missing Value Imputation with Linear Regression\n",
    "\n",
    "`boston_dropna_df` will be a dataset to see what would happen if we just dropped rows with missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "437de15d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:01:22.151238Z",
     "start_time": "2022-05-03T09:01:20.165645Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.datasets import load_boston\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40e275c9",
   "metadata": {},
   "source": [
    "## Boston house prices data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98c428b8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:01:30.456999Z",
     "start_time": "2022-05-03T09:01:30.446016Z"
    }
   },
   "outputs": [],
   "source": [
    "boston = load_boston() # Dataset from the sklearn library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "199c860a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:01:40.104655Z",
     "start_time": "2022-05-03T09:01:40.098585Z"
    }
   },
   "outputs": [],
   "source": [
    "print (boston.DESCR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90a11070",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:02:28.908278Z",
     "start_time": "2022-05-03T09:02:28.896509Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_df = pd.read_pickle('data/boston_df.p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "301da57a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:02:37.069631Z",
     "start_time": "2022-05-03T09:02:37.044699Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dde847f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:03:05.110646Z",
     "start_time": "2022-05-03T09:03:05.082628Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca9311ef",
   "metadata": {},
   "source": [
    "## Splitting the data into train and test set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "debd07d1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:03:57.127428Z",
     "start_time": "2022-05-03T09:03:57.118803Z"
    }
   },
   "source": [
    "#from sklearn.cross_validation import train_test_split\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(boston_df.iloc[:,:-1],boston_df.iloc[:,-1:], random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d2a7a06",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:04:06.234048Z",
     "start_time": "2022-05-03T09:04:06.224993Z"
    }
   },
   "outputs": [],
   "source": [
    "map(pd.np.shape,[X_train, X_test, y_train, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7397a6b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:04:12.314879Z",
     "start_time": "2022-05-03T09:04:12.302369Z"
    }
   },
   "outputs": [],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05fc7393",
   "metadata": {},
   "source": [
    "These testing and training sets form our full dataset, we have prepared some data with missing vaues"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2333aa0f",
   "metadata": {},
   "source": [
    "## Examine `AGE` column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46065c8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:05:57.899933Z",
     "start_time": "2022-05-03T09:05:57.892492Z"
    }
   },
   "outputs": [],
   "source": [
    "#read the data into two dataframes\n",
    "boston_dropna_df = pd.read_pickle('data/boston_dropna_df.p') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "542e1c07",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:06:21.886584Z",
     "start_time": "2022-05-03T09:06:21.876339Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_impute_df = boston_dropna_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2687cc6a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:06:27.542956Z",
     "start_time": "2022-05-03T09:06:27.534575Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_dropna_df['AGE'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04eb4efe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:06:35.958205Z",
     "start_time": "2022-05-03T09:06:35.950487Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_impute_df['AGE'].isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edefbc98",
   "metadata": {},
   "source": [
    "`boston_dropna_df` will be a dataset to see what would happen if we just dropped rows with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf7be3b1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:07:42.097405Z",
     "start_time": "2022-05-03T09:07:42.064125Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_dropna_df.dropna(subset=['AGE'],axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "edd7aec5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:07:47.108180Z",
     "start_time": "2022-05-03T09:07:47.100755Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_dropna_df['AGE'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf299880",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:07:52.564110Z",
     "start_time": "2022-05-03T09:07:52.548984Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_impute_df['AGE'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18380d8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:08:12.419293Z",
     "start_time": "2022-05-03T09:08:12.412103Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_dropna_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8b52166",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:08:18.798482Z",
     "start_time": "2022-05-03T09:08:18.777442Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_dropna_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "174b85ee",
   "metadata": {},
   "source": [
    "## Examine the model performance with different data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7169f985",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:09:42.249815Z",
     "start_time": "2022-05-03T09:09:42.223123Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_fitting_df = boston_dropna_df.drop('y',axis=1)\n",
    "lm_fitting_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5aa78c43",
   "metadata": {},
   "source": [
    "Our target now is the 'AGE' column, we will use the `boston_dropna_df` as the data to fit, we can use data with missing values to train a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93efc88d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:10:06.440385Z",
     "start_time": "2022-05-03T09:10:06.435722Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_for_impute = LinearRegression() #instatiate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cc86dae",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:10:26.723896Z",
     "start_time": "2022-05-03T09:10:26.703848Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_for_impute.fit(lm_fitting_df[[x for x in lm_fitting_df.columns if x != 'AGE']],lm_fitting_df['AGE']) #fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bfd2d9f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:10:35.831506Z",
     "start_time": "2022-05-03T09:10:35.804346Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_impute_df[boston_impute_df['AGE'].isnull()].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b274f7d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:10:41.986233Z",
     "start_time": "2022-05-03T09:10:41.971876Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_for_impute.predict(boston_impute_df.drop(['AGE','y'],axis=1)) \n",
    "#this uses the other features to predict 'AGE' with the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c4a089e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:10:51.615199Z",
     "start_time": "2022-05-03T09:10:51.606073Z"
    }
   },
   "outputs": [],
   "source": [
    "boston_impute_df['AGE'][boston_impute_df['AGE'].isnull()] = lm_for_impute.predict(boston_impute_df.drop(['AGE','y'],axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74aecdab",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:10:59.673223Z",
     "start_time": "2022-05-03T09:10:59.481497Z"
    }
   },
   "outputs": [],
   "source": [
    "boxplot = pd.DataFrame({'imputed': boston_impute_df['AGE'],'full': boston_df['AGE'],'dropped': boston_dropna_df['AGE']})\n",
    "boxplot.plot(kind='box')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be74be22",
   "metadata": {},
   "source": [
    "So these are the imputed values predicted by the `lm` trained on the data that have a value for 'AGE'."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0b987b1",
   "metadata": {},
   "source": [
    "Predicting the price with the full data,\n",
    "\n",
    "\\begin{equation}\n",
    "y_i = \\beta_0 + \\beta_1 X_i + \\epsilon_i\n",
    "\\end{equation}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dd6db1e",
   "metadata": {},
   "source": [
    "### Performance of Completed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c414ed2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:12:28.082537Z",
     "start_time": "2022-05-03T09:12:28.074842Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(boston_df.iloc[:,:-1],\n",
    "                                                    boston_df.iloc[:,-1:],\n",
    "                                                    random_state=111)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72315f1e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:12:32.911522Z",
     "start_time": "2022-05-03T09:12:32.903171Z"
    }
   },
   "outputs": [],
   "source": [
    "map(pd.np.shape,[X_train, X_test, y_train, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d02eb22a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:12:46.622142Z",
     "start_time": "2022-05-03T09:12:46.607572Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_full = LinearRegression()\n",
    "lm_full.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09d94a64",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:17:31.481221Z",
     "start_time": "2022-05-03T09:17:31.473184Z"
    }
   },
   "outputs": [],
   "source": [
    "print ('r-squared for completed model = ',lm_full.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac4b5a50",
   "metadata": {},
   "source": [
    "### Performance of Dropped Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "715e765e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:16:53.925239Z",
     "start_time": "2022-05-03T09:16:53.918386Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(boston_dropna_df.iloc[:,:-1],\n",
    "                                                    boston_dropna_df.iloc[:,-1:],\n",
    "                                                    random_state=111)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6433b1b3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:16:54.216463Z",
     "start_time": "2022-05-03T09:16:54.199518Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_impute = LinearRegression()\n",
    "lm_impute.fit(X_train,y_train)\n",
    "\n",
    "print ('r-squared for this model = ',lm_impute.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60526df7",
   "metadata": {},
   "source": [
    "### Performance of Imputed Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1463ca03",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:17:52.560809Z",
     "start_time": "2022-05-03T09:17:52.552733Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(boston_impute_df.iloc[:,:-1],\n",
    "                                                    boston_impute_df.iloc[:,-1:],\n",
    "                                                    random_state=111)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5e6383c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-05-03T09:17:59.702164Z",
     "start_time": "2022-05-03T09:17:59.691378Z"
    }
   },
   "outputs": [],
   "source": [
    "lm_impute = LinearRegression()\n",
    "lm_impute.fit(X_train,y_train)\n",
    "\n",
    "print ('r-squared for this model = ',lm_impute.score(X_test,y_test))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}