5_2_decision_tree_regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "167e9486-465f-4936-acc2-37200e8992c2",
   "metadata": {},
   "source": [
    "**Lesson exercise notebook for Spark-for-Machine-Learning-AI Course**\n",
    "* Created by Kevin Chao (kevinchao@gmail.com)\n",
    "* Latest updated on Feb 14, 2024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "00c9001d-b2f9-4d9a-9e59-0b165c6dd718",
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName(\"YourAppName\").getOrCreate()\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from pyspark.ml.regression import DecisionTreeRegressor\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "76ae2c62-e07b-4052-b743-deee54de023e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel('Exercise_Files/CCPP/Folds5x2_pp.xlsx', sheet_name='Sheet1')\n",
    "pp_df = spark.createDataFrame(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dd46181b-77f7-40d9-a946-7378ee868438",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(AT=14.96, V=41.76, AP=1024.07, RH=73.17, PE=463.26)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pp_df.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c981f414-277d-4bec-8cf4-5410618c3da9",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorAssembler = VectorAssembler(inputCols=[\"AT\", \"V\", \"AP\", \"RH\"], outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "da2588b3-91a7-4649-b440-5df31fc30c96",
   "metadata": {},
   "outputs": [],
   "source": [
    "vpp_df = vectorAssembler.transform(pp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "506f5c7f-1cab-4f77-9883-3f27228c3553",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(AT=14.96, V=41.76, AP=1024.07, RH=73.17, PE=463.26, features=DenseVector([14.96, 41.76, 1024.07, 73.17]))]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vpp_df.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1f72bd26-833d-435e-924c-623fdbc809f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = vpp_df.randomSplit([0.7, 0.3])\n",
    "train_df = splits[0]\n",
    "test_df = splits[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "863c6f1f-e4da-4ee1-bd00-0c3feba58973",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6686"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5cb17716-f2f9-4050-82b7-71aaa0bdc183",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2882"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fb2fe11a-b157-4d3b-9fcd-dff0ad9e6be4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9568"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vpp_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d86a74cf-19df-40a7-91ee-19c3a2eb4e86",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = DecisionTreeRegressor(featuresCol=\"features\", labelCol=\"PE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ff646c3d-f9ff-4cd8-96ad-fcfa21d52c27",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_model = dt.fit(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ee9b51cc-962e-4939-a395-2dd558d1e85a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_predictions = dt_model.transform(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "99ab2765-39ab-4b36-9623-e79a86133706",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_evaluator = RegressionEvaluator(labelCol=\"PE\", predictionCol=\"prediction\", metricName=\"rmse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e39a86ab-184e-4243-ab92-30eb1e3aae6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "rmse = dt_evaluator.evaluate(dt_predictions)  # root-mean-squared-error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6e9a845e-81ca-4b0a-bf7a-88c9d1e39dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.496111531306097"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}