Tutorial 3- Pyspark Dataframe- Handling Missing Values.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ded0783b",
   "metadata": {},
   "source": [
    "### Pyspark Handling Missing Values\n",
    "- Dropping Columns\n",
    "- Dropping Rows\n",
    "- Various Parameter In Dropping functionalities\n",
    "- Handling Missing values by Mean, MEdian And Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "805e7382",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark=SparkSession.builder.appName('Practise').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e48ebc07",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pyspark=spark.read.csv('test2.csv',header=True,inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "53ab7bbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Name: string (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- Experience: integer (nullable = true)\n",
      " |-- Salary: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ed677b30",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|null|      null| 40000|\n",
      "|     null|  34|        10| 38000|\n",
      "|     null|  36|      null|  null|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "523d3c4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+------+\n",
      "| age|Experience|Salary|\n",
      "+----+----------+------+\n",
      "|  31|        10| 30000|\n",
      "|  30|         8| 25000|\n",
      "|  29|         4| 20000|\n",
      "|  24|         3| 20000|\n",
      "|  21|         1| 15000|\n",
      "|  23|         2| 18000|\n",
      "|null|      null| 40000|\n",
      "|  34|        10| 38000|\n",
      "|  36|      null|  null|\n",
      "+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "##drop the columns\n",
    "df_pyspark.drop('Name').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9c041e07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|null|      null| 40000|\n",
      "|     null|  34|        10| 38000|\n",
      "|     null|  36|      null|  null|\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "36845e38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.na.drop().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "156e41cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "|         |   |          |      |\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### any==how\n",
    "df_pyspark.na.drop(how=\"any\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e9af0da0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "|     null| 34|        10| 38000|\n",
      "|         |   |          |      |\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "##threshold\n",
    "df_pyspark.na.drop(how=\"any\",thresh=3).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "787fc949",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+---+----------+------+\n",
      "|     Name|age|Experience|Salary|\n",
      "+---------+---+----------+------+\n",
      "|    Krish| 31|        10| 30000|\n",
      "|Sudhanshu| 30|         8| 25000|\n",
      "|    Sunny| 29|         4| 20000|\n",
      "|     Paul| 24|         3| 20000|\n",
      "|   Harsha| 21|         1| 15000|\n",
      "|  Shubham| 23|         2| 18000|\n",
      "|     null| 34|        10| 38000|\n",
      "|     null| 36|      null|  null|\n",
      "|         |   |          |      |\n",
      "+---------+---+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "##Subset\n",
    "df_pyspark.na.drop(how=\"any\",subset=['Age']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "72bad9ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+--------------+--------------+------+\n",
      "|     Name|           age|    Experience|Salary|\n",
      "+---------+--------------+--------------+------+\n",
      "|    Krish|            31|            10| 30000|\n",
      "|Sudhanshu|            30|             8| 25000|\n",
      "|    Sunny|            29|             4| 20000|\n",
      "|     Paul|            24|             3| 20000|\n",
      "|   Harsha|            21|             1| 15000|\n",
      "|  Shubham|            23|             2| 18000|\n",
      "|   Mahesh|Missing Values|Missing Values| 40000|\n",
      "|     null|            34|            10| 38000|\n",
      "|     null|            36|Missing Values|  null|\n",
      "|         |              |              |      |\n",
      "+---------+--------------+--------------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Filling the Missing Value\n",
    "df_pyspark.na.fill('Missing Values',['Experience','age']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "64e01bb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+\n",
      "|     Name| age|Experience|Salary|\n",
      "+---------+----+----------+------+\n",
      "|    Krish|  31|        10| 30000|\n",
      "|Sudhanshu|  30|         8| 25000|\n",
      "|    Sunny|  29|         4| 20000|\n",
      "|     Paul|  24|         3| 20000|\n",
      "|   Harsha|  21|         1| 15000|\n",
      "|  Shubham|  23|         2| 18000|\n",
      "|   Mahesh|null|      null| 40000|\n",
      "|     null|  34|        10| 38000|\n",
      "|     null|  36|      null|  null|\n",
      "|         |    |          |      |\n",
      "+---------+----+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b66832fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Name: string (nullable = true)\n",
      " |-- age: string (nullable = true)\n",
      " |-- Experience: string (nullable = true)\n",
      " |-- Salary: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_pyspark.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e31190f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import Imputer\n",
    "\n",
    "imputer = Imputer(\n",
    "    inputCols=['age', 'Experience', 'Salary'], \n",
    "    outputCols=[\"{}_imputed\".format(c) for c in ['age', 'Experience', 'Salary']]\n",
    "    ).setStrategy(\"median\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d84c4a3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "|     Name| age|Experience|Salary|age_imputed|Experience_imputed|Salary_imputed|\n",
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "|    Krish|  31|        10| 30000|         31|                10|         30000|\n",
      "|Sudhanshu|  30|         8| 25000|         30|                 8|         25000|\n",
      "|    Sunny|  29|         4| 20000|         29|                 4|         20000|\n",
      "|     Paul|  24|         3| 20000|         24|                 3|         20000|\n",
      "|   Harsha|  21|         1| 15000|         21|                 1|         15000|\n",
      "|  Shubham|  23|         2| 18000|         23|                 2|         18000|\n",
      "|   Mahesh|null|      null| 40000|         29|                 4|         40000|\n",
      "|     null|  34|        10| 38000|         34|                10|         38000|\n",
      "|     null|  36|      null|  null|         36|                 4|         20000|\n",
      "+---------+----+----------+------+-----------+------------------+--------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Add imputation cols to df\n",
    "imputer.fit(df_pyspark).transform(df_pyspark).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cd38651",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b63a65ae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2992b6a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d995bc12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c17fd913",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdf76805",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "835812ae",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}