Ex_05_PySpark_Linear_Regression.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Ex-04-PySpark Linear Regression.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNQvOcLSJC4D"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://downloads.apache.org/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FIH7ZgU6JD6M",
        "outputId": "d855d256-8cee-44a3-fef2-be901152bc45"
      },
      "source": [
        "!pip install pyspark"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 57 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 55.0 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=5d22c8e97cc2c71ae0ab18c7c6f4e90855651e64e004beb3b878a600e8035295\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VQDp31ePJF07"
      },
      "source": [
        "from pyspark import SparkContext, SparkConf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nu0J9qykJZFz"
      },
      "source": [
        "conf = SparkConf().setAppName('Spark Prac').setMaster('local')\n",
        "sc = SparkContext(conf = conf)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "id": "CdunAvM1Jejq",
        "outputId": "1711f183-755a-4ef7-dfd7-cc4e4691e052"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-e86c17f2-67d7-4c57-8635-819a68382d01\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-e86c17f2-67d7-4c57-8635-819a68382d01\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "Saving Admission_Prediction.csv to Admission_Prediction.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTA-EgRcKFSb"
      },
      "source": [
        "import io\n",
        "data1 = io.BytesIO(uploaded['Admission_Prediction.csv'])    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLWtEmhuKoMa"
      },
      "source": [
        "data = sc.textFile('Admission_Prediction.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yFgucSWId9yy"
      },
      "source": [
        "#### Import PySpark ML Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9GiDs0Iib1fR"
      },
      "source": [
        "from __future__ import print_function\n",
        "import pyspark\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import VectorAssembler, VectorIndexer\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.regression import RandomForestRegressor\n",
        "from pyspark.ml.evaluation import RegressionEvaluator"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QVFPPBfkb1iU"
      },
      "source": [
        "spark = SparkSession.builder.appName(\"Spark ML Algo\").getOrCreate()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e1-2JhZ-b1lt"
      },
      "source": [
        "dataframe = spark.read.csv(\"Admission_Prediction.csv\", header=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AztfoAWpb1qW",
        "outputId": "249993bc-6ae4-4823-85d5-ec05804ca26b"
      },
      "source": [
        "type(dataframe)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pyspark.sql.dataframe.DataFrame"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qzabGQgBb1tW",
        "outputId": "dc675700-647e-42b3-ce3c-d630c25b7a1e"
      },
      "source": [
        "dataframe.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+-----------+-----------------+----+----+----+--------+---------------+\n",
            "|GRE Score|TOEFL Score|University Rating| SOP| LOR|CGPA|Research|Chance of Admit|\n",
            "+---------+-----------+-----------------+----+----+----+--------+---------------+\n",
            "|   337.00|     118.00|                4|4.50|4.50|9.65|    1.00|           0.92|\n",
            "|   324.00|     107.00|                4|4.00|4.50|8.87|    1.00|           0.76|\n",
            "|     null|     104.00|                3|3.00|3.50|8.00|    1.00|           0.72|\n",
            "|   322.00|     110.00|                3|3.50|2.50|8.67|    1.00|           0.80|\n",
            "|   314.00|     103.00|                2|2.00|3.00|8.21|    0.00|           0.65|\n",
            "|   330.00|     115.00|                5|4.50|3.00|9.34|    1.00|           0.90|\n",
            "|   321.00|     109.00|             null|3.00|4.00|8.20|    1.00|           0.75|\n",
            "|   308.00|     101.00|                2|3.00|4.00|7.90|    0.00|           0.68|\n",
            "|   302.00|     102.00|                1|2.00|1.50|8.00|    0.00|           0.50|\n",
            "|   323.00|     108.00|                3|3.50|3.00|8.60|    0.00|           0.45|\n",
            "|   325.00|     106.00|                3|3.50|4.00|8.40|    1.00|           0.52|\n",
            "|   327.00|     111.00|                4|4.00|4.50|9.00|    1.00|           0.84|\n",
            "|     null|     112.00|                4|4.00|4.50|9.10|    1.00|           0.78|\n",
            "|   307.00|     109.00|                3|4.00|3.00|8.00|    1.00|           0.62|\n",
            "|   311.00|     104.00|                3|3.50|2.00|8.20|    1.00|           0.61|\n",
            "|   314.00|     105.00|                3|3.50|2.50|8.30|    0.00|           0.54|\n",
            "|   317.00|     107.00|                3|4.00|3.00|8.70|    0.00|           0.66|\n",
            "|   319.00|     106.00|                3|4.00|3.00|8.00|    1.00|           0.65|\n",
            "|   318.00|     110.00|                3|4.00|3.00|8.80|    0.00|           0.63|\n",
            "|   303.00|     102.00|                3|3.50|3.00|8.50|    0.00|           0.62|\n",
            "+---------+-----------+-----------------+----+----+----+--------+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6h0a6XO1b1wF",
        "outputId": "1d966c2e-d570-4478-8a01-990bde74d8d0"
      },
      "source": [
        "dataframe.printSchema()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- GRE Score: string (nullable = true)\n",
            " |-- TOEFL Score: string (nullable = true)\n",
            " |-- University Rating: string (nullable = true)\n",
            " |-- SOP: string (nullable = true)\n",
            " |-- LOR: string (nullable = true)\n",
            " |-- CGPA: string (nullable = true)\n",
            " |-- Research: string (nullable = true)\n",
            " |-- Chance of Admit: string (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KqiZAPIMb1yi"
      },
      "source": [
        "from pyspark.sql.functions import col\n",
        "new_dataframe = dataframe.select(*(col(c).cast(\"float\").alias(c) for c in dataframe.columns))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jcTuJC-ikNSm",
        "outputId": "1c4e0d24-5b9a-4da6-d0da-f1ca09a8583c"
      },
      "source": [
        "new_dataframe.printSchema()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- GRE Score: float (nullable = true)\n",
            " |-- TOEFL Score: float (nullable = true)\n",
            " |-- University Rating: float (nullable = true)\n",
            " |-- SOP: float (nullable = true)\n",
            " |-- LOR: float (nullable = true)\n",
            " |-- CGPA: float (nullable = true)\n",
            " |-- Research: float (nullable = true)\n",
            " |-- Chance of Admit: float (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FiEierfbkNWF"
      },
      "source": [
        "from pyspark.sql.functions import col, count, isnan, when"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YVqXbKK3kNZN",
        "outputId": "bbd10721-54a9-4dea-916d-d45b9da1f557"
      },
      "source": [
        "for c in new_dataframe.columns:\n",
        "  print(c)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GRE Score\n",
            "TOEFL Score\n",
            "University Rating\n",
            "SOP\n",
            "LOR\n",
            "CGPA\n",
            "Research\n",
            "Chance of Admit\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oeJrSXQ5b14c",
        "outputId": "c739b133-15d2-483d-e262-69bd4344d29d"
      },
      "source": [
        "### Checking for null ir nan type values in out columns\n",
        "new_dataframe.select([count(when(col(c).isNull(), c)).alias(c) for c in new_dataframe.columns]).show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|GRE Score|TOEFL Score|University Rating|SOP|LOR|CGPA|Research|Chance of Admit|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|       15|         10|               15|  0|  0|   0|       0|              0|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I3_MsHiupjuN"
      },
      "source": [
        "from pyspark.ml.feature import Imputer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edtWrqmmsMnE"
      },
      "source": [
        "imputer = Imputer(inputCols=[\"GRE Score\", \"TOEFL Score\",\"University Rating\"], \n",
        "                  outputCols=[\"GRE Score\", \"TOEFL Score\",\"University Rating\"])\n",
        "model = imputer.fit(new_dataframe)\n",
        "\n",
        "imputed_data = model.transform(new_dataframe)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "phzcbc8PsXbG",
        "outputId": "7d128f8f-81a0-4aa5-96e2-1151fc5918ca"
      },
      "source": [
        "imputed_data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[GRE Score: float, TOEFL Score: float, University Rating: float, SOP: float, LOR: float, CGPA: float, Research: float, Chance of Admit: float]"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "__U8GAMqsi2t",
        "outputId": "0ecf7a34-df0b-4c43-fa6c-a566b3e2925d"
      },
      "source": [
        "#checking for null ir nan type values in our columns\n",
        "imputed_data.select([count(when(col(c).isNull(), c)).alias(c) for c in imputed_data.columns]).show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|GRE Score|TOEFL Score|University Rating|SOP|LOR|CGPA|Research|Chance of Admit|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|        0|          0|                0|  0|  0|   0|       0|              0|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N7zkak-fsltU"
      },
      "source": [
        "features = imputed_data.drop('Chance of Admit')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I925bBLQsnq1",
        "outputId": "859ed41f-c0bb-4fc5-fa2c-eea453154535"
      },
      "source": [
        "features"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[GRE Score: float, TOEFL Score: float, University Rating: float, SOP: float, LOR: float, CGPA: float, Research: float]"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CutmTZhTso9c"
      },
      "source": [
        "#let's assemble our features together using vectorAssembler\n",
        "assembler = VectorAssembler( inputCols=features.columns,outputCol=\"features\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-G5yD1Fsq0s"
      },
      "source": [
        "output = assembler.transform(imputed_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j3ku1CRazpIg"
      },
      "source": [
        "#### Linear Regressor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtSW13HxzuOs"
      },
      "source": [
        "output= output.select(\"features\", \"Chance of Admit\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_bAHs6LzyPm"
      },
      "source": [
        "output = assembler.transform(imputed_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YDlX0iyHz0E1"
      },
      "source": [
        "output= output.select(\"features\", \"Chance of Admit\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X2_MwCrn0Wce"
      },
      "source": [
        "train_df,test_df = output.randomSplit([0.7, 0.3])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FpIJaQi40Yq-",
        "outputId": "141e9f37-edbc-4004-8681-51bc4821db61"
      },
      "source": [
        "train_df.show()\n",
        "test_df.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------------+---------------+\n",
            "|            features|Chance of Admit|\n",
            "+--------------------+---------------+\n",
            "|[290.0,100.0,1.0,...|           0.47|\n",
            "|[290.0,104.0,4.0,...|           0.45|\n",
            "|[293.0,97.0,2.0,2...|           0.64|\n",
            "|[294.0,95.0,1.0,1...|           0.49|\n",
            "|[295.0,96.0,2.0,1...|           0.47|\n",
            "|[295.0,99.0,2.0,2...|           0.57|\n",
            "|[295.0,101.0,2.0,...|           0.69|\n",
            "|[296.0,97.0,2.0,1...|           0.49|\n",
            "|[296.0,99.0,2.0,2...|           0.61|\n",
            "|[296.0,101.0,1.0,...|            0.6|\n",
            "|[297.0,96.0,2.0,2...|           0.43|\n",
            "|[297.0,96.0,2.0,2...|           0.34|\n",
            "|[297.0,98.0,2.0,2...|           0.59|\n",
            "|[297.0,99.0,4.0,3...|           0.54|\n",
            "|[297.0,100.0,1.0,...|           0.52|\n",
            "|[297.0,101.0,3.0,...|           0.57|\n",
            "|[298.0,97.0,3.121...|           0.45|\n",
            "|[298.0,98.0,2.0,1...|           0.44|\n",
            "|[298.0,98.0,2.0,4...|           0.34|\n",
            "|[298.0,99.0,1.0,1...|           0.53|\n",
            "+--------------------+---------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+--------------------+---------------+\n",
            "|            features|Chance of Admit|\n",
            "+--------------------+---------------+\n",
            "|[294.0,93.0,1.0,1...|           0.46|\n",
            "|[295.0,93.0,1.0,2...|           0.46|\n",
            "|[295.0,99.0,1.0,2...|           0.37|\n",
            "|[296.0,95.0,2.0,3...|           0.44|\n",
            "|[296.0,99.0,2.0,3...|           0.47|\n",
            "|[298.0,92.0,1.0,2...|           0.51|\n",
            "|[298.0,105.0,3.0,...|           0.69|\n",
            "|[299.0,97.0,3.0,5...|           0.38|\n",
            "|[299.0,102.0,3.0,...|           0.56|\n",
            "|[300.0,95.0,2.0,3...|           0.62|\n",
            "|[300.0,100.0,3.0,...|           0.64|\n",
            "|[300.0,102.0,2.0,...|           0.56|\n",
            "|[300.0,102.0,3.0,...|           0.63|\n",
            "|[300.0,104.0,3.0,...|           0.71|\n",
            "|[301.0,98.0,1.0,2...|           0.67|\n",
            "|[301.0,99.0,3.0,2...|           0.68|\n",
            "|[301.0,102.0,3.0,...|           0.68|\n",
            "|[302.0,101.0,2.0,...|           0.46|\n",
            "|[303.0,98.0,1.0,2...|           0.56|\n",
            "|[303.0,99.0,3.0,2...|           0.36|\n",
            "+--------------------+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9w_Rj1O40a8H"
      },
      "source": [
        "lin_reg = LinearRegression(featuresCol = 'features', labelCol='Chance of Admit')\n",
        "linear_model = lin_reg.fit(train_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K2eaD4xt0fNf",
        "outputId": "fa945ef9-e718-44ea-f2d0-39866f388b66"
      },
      "source": [
        "print(\"Coefficients: \" + str(linear_model.coefficients))\n",
        "print(\"Intercept: \" + str(linear_model.intercept))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Coefficients: [0.0015330494946342618,0.0023519563543600987,0.0076305691461127645,-0.000640853503839564,0.018731808306761612,0.12635412086451406,0.025797081114095]\n",
            "Intercept: -1.2012592480061368\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "up7nM4LE0g0f",
        "outputId": "bbeaa0f8-9809-4d65-87ff-2ed3ff96f041"
      },
      "source": [
        "trainSummary = linear_model.summary\n",
        "print(\"RMSE: %f\" % trainSummary.rootMeanSquaredError)\n",
        "print(\"r2: %f\" % trainSummary.r2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "RMSE: 0.059358\n",
            "r2: 0.823672\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U1ZBTuVd0jBm",
        "outputId": "18ad16cf-df3e-4fc0-aae3-ade8ce44e8ba"
      },
      "source": [
        "# prediction\n",
        "\n",
        "predictions = linear_model.transform(test_df)\n",
        "predictions.select(\"prediction\",\"Chance of Admit\",\"features\").show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------------------+---------------+--------------------+\n",
            "|         prediction|Chance of Admit|            features|\n",
            "+-------------------+---------------+--------------------+\n",
            "|0.44228849630862066|           0.46|[294.0,93.0,1.0,1...|\n",
            "|0.42328441874278155|           0.46|[295.0,93.0,1.0,2...|\n",
            "| 0.4747813232256899|           0.37|[295.0,99.0,1.0,2...|\n",
            "| 0.5052685980765481|           0.44|[296.0,95.0,2.0,3...|\n",
            "| 0.4841250147454388|           0.47|[296.0,99.0,2.0,3...|\n",
            "| 0.5114524516204115|           0.51|[298.0,92.0,1.0,2...|\n",
            "| 0.6771850593675526|           0.69|[298.0,105.0,3.0,...|\n",
            "| 0.5383836327973122|           0.38|[299.0,97.0,3.0,5...|\n",
            "|  0.672084228922913|           0.56|[299.0,102.0,3.0,...|\n",
            "| 0.5879557326497913|           0.62|[300.0,95.0,2.0,3...|\n",
            "| 0.6916804096917735|           0.64|[300.0,100.0,3.0,...|\n",
            "|  0.544725539922505|           0.56|[300.0,102.0,2.0,...|\n",
            "| 0.5983465665738099|           0.63|[300.0,102.0,3.0,...|\n",
            "| 0.6111528133071027|           0.71|[300.0,104.0,3.0,...|\n",
            "| 0.5936452975807367|           0.67|[301.0,98.0,1.0,2...|\n",
            "| 0.6452748975717912|           0.68|[301.0,99.0,3.0,2...|\n",
            "| 0.6118974865184446|           0.68|[301.0,102.0,3.0,...|\n",
            "| 0.5842684316716316|           0.46|[302.0,101.0,2.0,...|\n",
            "| 0.5135338911642724|           0.56|[303.0,98.0,1.0,2...|\n",
            "| 0.5324104956893265|           0.36|[303.0,99.0,3.0,2...|\n",
            "+-------------------+---------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aIk5xQ4M0lm4",
        "outputId": "dda4396b-5982-4ec3-efad-1971c8b3a359"
      },
      "source": [
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "pred_evaluator = RegressionEvaluator(predictionCol=\"prediction\", \\\n",
        "                 labelCol=\"Chance of Admit\",metricName=\"r2\")\n",
        "print(\"R Squared (R2) on test data =\", pred_evaluator.evaluate(predictions))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "R Squared (R2) on test data = 0.8110789276193163\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "77zUE5qX0vu9"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}