DL4D_06_Laying_Linear_Regression_Foundations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combining Variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Seeing linear regression in action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "from sklearn.preprocessing import scale\n",
    "\n",
    "boston = load_boston()\n",
    "X, y = scale(boston.data), boston.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2 0.741\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "regression = LinearRegression()\n",
    "regression.fit(X, y)\n",
    "\n",
    "print('R2 %0.3f' % regression.score(X, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CRIM:-0.9', 'ZN:1.1', 'INDUS:0.1', 'CHAS:0.7', 'NOX:-2.1', 'RM:2.7', 'AGE:0.0', 'DIS:-3.1', 'RAD:2.7', 'TAX:-2.1', 'PTRATIO:-2.1', 'B:0.9', 'LSTAT:-3.7']\n"
     ]
    }
   ],
   "source": [
    "print([a + ':' + str(round(b, 1)) for a, b in \n",
    "       zip(boston.feature_names, regression.coef_)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mixing Variable Types"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling the responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 1. 0.]\n",
      " [1. 0. 0.]\n",
      " [0. 0. 1.]\n",
      " [1. 0. 0.]\n",
      " [1. 0. 0.]\n",
      " [0. 1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "lbl = LabelEncoder()\n",
    "enc = OneHotEncoder()\n",
    "qualitative = ['red', 'red', 'green', 'blue', \n",
    "               'red', 'blue', 'blue', 'green']\n",
    "labels = lbl.fit_transform(qualitative).reshape(8,1)\n",
    "print(enc.fit_transform(labels).toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dealing with complex relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2: 0.819\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.model_selection  import train_test_split\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "pf = PolynomialFeatures(degree=2)\n",
    "poly_X = pf.fit_transform(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(poly_X,\n",
    "                    y, test_size=0.33, random_state=42)\n",
    "\n",
    "from sklearn.linear_model import Ridge\n",
    "reg_regression = Ridge(alpha=0.1, normalize=True)\n",
    "reg_regression.fit(X_train,y_train)\n",
    "print ('R2: %0.3f' % r2_score(y_test,reg_regression.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Switching to Probabilities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specifying a binary response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[False False False False  True  True  True  True]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([0, 0, 0, 0, 1, 1, 1, 1])\n",
    "b = np.array([1, 2, 3, 4, 5, 6, 7, 8]).reshape(8,1)\n",
    "from sklearn.linear_model import LinearRegression\n",
    "regression = LinearRegression()\n",
    "regression.fit(b,a)\n",
    "print (regression.predict(b)>0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforming numeric estimates into probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In-sample accuracy: 0.973\n",
      "Out-of-sample accuracy: 0.958\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection  import train_test_split\n",
    "\n",
    "binary_y = np.array(y >= 40).astype(int)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "            binary_y, test_size=0.33, random_state=5)\n",
    "logistic = LogisticRegression()\n",
    "logistic.fit(X_train,y_train)\n",
    "from sklearn.metrics import accuracy_score\n",
    "print('In-sample accuracy: %0.3f' % \n",
    "      accuracy_score(y_train, logistic.predict(X_train)))\n",
    "print('Out-of-sample accuracy: %0.3f' % \n",
    "      accuracy_score(y_test, logistic.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   CRIM :  -0.006\n",
      "     ZN :   0.197\n",
      "  INDUS :   0.580\n",
      "   CHAS :  -0.023\n",
      "    NOX :  -0.236\n",
      "     RM :   1.426\n",
      "    AGE :  -0.048\n",
      "    DIS :  -0.365\n",
      "    RAD :   0.645\n",
      "    TAX :  -0.220\n",
      "PTRATIO :  -0.554\n",
      "      B :   0.049\n",
      "  LSTAT :  -0.803\n"
     ]
    }
   ],
   "source": [
    "for var,coef in zip(boston.feature_names,\n",
    "                    logistic.coef_[0]):\n",
    "        print (\"%7s : %7.3f\" %(var, coef)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "classes: [0 1]\n",
      "\n",
      "Probs:\n",
      " [[0.39022779 0.60977221]\n",
      " [0.93856655 0.06143345]\n",
      " [0.98425623 0.01574377]]\n"
     ]
    }
   ],
   "source": [
    "print('\\nclasses:',logistic.classes_)\n",
    "print('\\nProbs:\\n',logistic.predict_proba(X_test)[:3,:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Guessing the Right Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the outcome of incompatible features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random features: 1 -> R2: 0.743\n",
      "Random features: 2 -> R2: 0.743\n",
      "Random features: 4 -> R2: 0.745\n",
      "Random features: 8 -> R2: 0.747\n",
      "Random features: 16 -> R2: 0.752\n",
      "Random features: 32 -> R2: 0.760\n",
      "Random features: 64 -> R2: 0.788\n",
      "Random features: 128 -> R2: 0.842\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                y, test_size=0.33, random_state=42)\n",
    "check = [2**i for i in range(8)]\n",
    "for i in range(2**7+1):\n",
    "    X_train = np.column_stack((X_train,np.random.random(\n",
    "        X_train.shape[0])))\n",
    "    X_test = np.column_stack((X_test,np.random.random(\n",
    "        X_test.shape[0])))\n",
    "    regression.fit(X_train, y_train)\n",
    "    if i in check:\n",
    "        print (\"Random features: %i -> R2: %0.3f\" % \n",
    "               (i, r2_score(y_train,regression.predict(X_train))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2 0.491\n"
     ]
    }
   ],
   "source": [
    "regression.fit(X_train, y_train)\n",
    "print ('R2 %0.3f' \n",
    "   % r2_score(y_test,regression.predict(X_test)))\n",
    "# Please notice that the R2 result may change from run to \n",
    "# run due to the random nature of the experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solving overfitting using selection and regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2: 0.819\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pf = PolynomialFeatures(degree=2)\n",
    "poly_X = pf.fit_transform(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(poly_X,\n",
    "                    y, test_size=0.33, random_state=42)\n",
    "\n",
    "from sklearn.linear_model import Ridge\n",
    "reg_regression = Ridge(alpha=0.1, normalize=True)\n",
    "reg_regression.fit(X_train,y_train)\n",
    "print ('R2: %0.3f' \n",
    "   % r2_score(y_test,reg_regression.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning One Example at a Time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Understanding how SDG is different"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example      1 R2 -6.255 coef: 0.113 -0.071 0.148 -0.040 0.075 -0.021 0.146 -0.113 0.243 0.224 0.118 0.037 0.110\n",
      "Example      2 R2 -6.168 coef: 0.066 -0.139 0.087 -0.078 0.055 -0.114 0.254 -0.054 0.154 0.140 0.282 0.068 0.152\n",
      "Example      4 R2 -6.060 coef: -0.072 -0.195 0.319 -0.171 0.064 -0.206 0.527 0.048 -0.041 0.266 0.075 0.219 0.353\n",
      "Example      8 R2 -5.776 coef: -0.246 -0.504 0.605 -0.343 0.098 0.005 0.807 -0.304 -0.095 0.332 -0.067 0.399 0.024\n",
      "Example     16 R2 -5.145 coef: -0.437 -0.430 0.298 -0.571 -0.002 0.004 0.519 -0.423 -0.279 0.292 -0.544 0.665 -0.065\n",
      "Example     32 R2 -4.495 coef: -0.554 -0.308 0.441 1.224 0.051 0.315 0.387 -0.566 0.055 0.629 -0.367 0.726 -0.513\n",
      "Example     64 R2 -2.947 coef: -0.974 0.419 0.107 1.648 -0.409 1.687 -0.428 -0.201 -0.029 0.447 -1.246 1.166 -1.914\n",
      "Example    128 R2 -1.791 coef: -0.533 0.863 0.118 1.137 -0.585 1.823 -0.290 -0.178 -0.283 0.094 -1.984 1.166 -2.030\n",
      "Example    256 R2 -0.607 coef: -0.783 0.619 -0.177 1.368 -0.771 3.135 -0.305 -0.513 -0.321 -0.204 -2.326 1.239 -2.759\n",
      "Example    512 R2 0.288 coef: -0.659 0.454 0.168 1.303 -0.570 3.071 -0.064 -1.175 0.161 0.222 -2.238 1.079 -2.938\n",
      "Example   1024 R2 0.625 coef: -0.775 0.301 0.179 1.178 -0.759 3.376 -0.176 -1.476 0.305 0.214 -2.191 1.135 -3.286\n",
      "Example   2048 R2 0.698 coef: -0.802 0.315 0.162 1.012 -1.071 3.228 -0.302 -1.886 0.535 0.115 -2.030 1.131 -3.569\n",
      "Example   4096 R2 0.709 coef: -0.866 0.424 0.170 0.973 -1.365 2.983 -0.349 -2.365 0.808 -0.074 -1.933 1.113 -3.744\n",
      "Example   8192 R2 0.715 coef: -0.961 0.638 0.142 0.902 -1.652 2.948 -0.418 -2.690 1.119 -0.403 -1.978 1.097 -3.887\n",
      "Example  16384 R2 0.722 coef: -1.012 0.787 0.233 0.820 -1.805 2.707 -0.393 -2.893 1.479 -0.777 -1.993 1.082 -3.899\n",
      "Example  32768 R2 0.721 coef: -1.077 0.844 0.274 0.872 -1.899 2.773 -0.384 -2.980 1.779 -1.161 -2.006 1.108 -3.955\n",
      "Example  65536 R2 0.726 coef: -1.089 0.889 0.394 0.802 -1.888 2.691 -0.366 -2.983 1.981 -1.314 -2.023 1.091 -3.910\n",
      "Example 131072 R2 0.724 coef: -1.098 0.891 0.374 0.849 -1.905 2.752 -0.371 -3.005 2.026 -1.396 -2.011 1.102 -3.956\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import SGDRegressor\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                    y, test_size=0.33, random_state=42)\n",
    "SGD = SGDRegressor(penalty=None,\n",
    "                   learning_rate='invscaling', \n",
    "                   eta0=0.01, power_t=0.25,\n",
    "                   max_iter=5, tol=None)\n",
    "\n",
    "power = 17\n",
    "check = [2**i for i in range(power+1)]\n",
    "for i in range(400):\n",
    "    for j in range(X_train.shape[0]):\n",
    "        SGD.partial_fit(X_train[j,:].reshape(1,13), \n",
    "                        y_train[j].reshape(1,))\n",
    "        count = (j+1) + X_train.shape[0] * i\n",
    "        if count in check:\n",
    "            R2 = r2_score(y_test,SGD.predict(X_test))\n",
    "            print ('Example %6i R2 %0.3f coef: %s' % \n",
    "            (count, R2, ' '.join(map(lambda x:'%0.3f' %x, SGD.coef_))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}