Tutorial 6-Example Of Pyspark ML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "16da6c54",
   "metadata": {},
   "source": [
    "### Examples Of Pyspark ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "0b9da3ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark=SparkSession.builder.appName('Missing').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "735525da",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read The dataset\n",
    "training = spark.read.csv('test1.csv',header=True,inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "d6e038c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "6b3dd5ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Name: string (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- Experience: integer (nullable = true)\n",
      " |-- Salary: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "training.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "5d3227e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Name', 'age', 'Experience', 'Salary']"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cffef5b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "[Age,Experience]----> new feature--->independent feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "e6273555",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "featureassembler=VectorAssembler(inputCols=[\"age\",\"Experience\"],outputCol=\"Independent Features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "0b69744c",
   "metadata": {},
   "outputs": [],
   "source": [
    "output=featureassembler.transform(training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "60961194",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+--------------------+\n",
      "|     Name|age|Experience|Salary|Independent Features|\n",
      "+---------+---+----------+------+--------------------+\n",
      "|    Krish| 31|        10| 30000|         [31.0,10.0]|\n",
      "|Sudhanshu| 30|         8| 25000|          [30.0,8.0]|\n",
      "|    Sunny| 29|         4| 20000|          [29.0,4.0]|\n",
      "|     Paul| 24|         3| 20000|          [24.0,3.0]|\n",
      "|   Harsha| 21|         1| 15000|          [21.0,1.0]|\n",
      "|  Shubham| 23|         2| 18000|          [23.0,2.0]|\n",
      "+---------+---+----------+------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "output.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "2c27434a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Name', 'age', 'Experience', 'Salary', 'Independent Features']"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "54a0ccab",
   "metadata": {},
   "outputs": [],
   "source": [
    "finalized_data=output.select(\"Independent Features\",\"Salary\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "f7a73845",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+\n",
      "|Independent Features|Salary|\n",
      "+--------------------+------+\n",
      "|         [31.0,10.0]| 30000|\n",
      "|          [30.0,8.0]| 25000|\n",
      "|          [29.0,4.0]| 20000|\n",
      "|          [24.0,3.0]| 20000|\n",
      "|          [21.0,1.0]| 15000|\n",
      "|          [23.0,2.0]| 18000|\n",
      "+--------------------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "finalized_data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "0b11192b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import LinearRegression\n",
    "##train test split\n",
    "train_data,test_data=finalized_data.randomSplit([0.75,0.25])\n",
    "regressor=LinearRegression(featuresCol='Independent Features', labelCol='Salary')\n",
    "regressor=regressor.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "fa4ec997",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([-5000.0, 7000.0])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Coefficients\n",
    "regressor.coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "eba911b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "118999.99999893687"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Intercepts\n",
    "regressor.intercept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "2ba2bc70",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Prediction\n",
    "pred_results=regressor.evaluate(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "489d6392",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+-----------------+\n",
      "|Independent Features|Salary|       prediction|\n",
      "+--------------------+------+-----------------+\n",
      "|          [21.0,1.0]| 15000|20999.99999996154|\n",
      "|          [29.0,4.0]| 20000|2000.000000192551|\n",
      "|         [31.0,10.0]| 30000|33999.99999993094|\n",
      "+--------------------+------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_results.predictions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "0534e854",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9333.333333233308, 125333333.3306847)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_results.meanAbsoluteError,pred_results.meanSquaredError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70de559b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}