DataPrep_Demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# DataPrep"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "DataPrep lets you prepare your data using a single library with a few lines of code.\n",
        "\n",
        "Currently, you can use DataPrep to:\n",
        "\n",
        "- Collect data from common data sources (through dataprep.connector)\n",
        "- Do your exploratory data analysis (through dataprep.eda)\n",
        "- Clean and standardize data (through dataprep.clean)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RQor3GuQYioW"
      },
      "source": [
        "# Install the latest develop version of DataPrep"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zfm1HnEfYvuw",
        "outputId": "ea49db83-5aa8-4ba0-f05d-ad41093c6aaa"
      },
      "outputs": [],
      "source": [
        "!pip install -U git+https://github.com/sfu-db/dataprep.git@develop --quiet"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AcYKGvqxY-kK"
      },
      "source": [
        "# Load Modules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xWNmIBbyqD9z",
        "outputId": "595576bd-fb63-4bed-adac-db12bace7ad6"
      },
      "outputs": [],
      "source": [
        "from dataprep.eda import *\n",
        "from dataprep.datasets import load_dataset\n",
        "from dataprep.eda import plot, plot_correlation, plot_missing, plot_diff, create_report"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I9TeCMXAZFcI"
      },
      "source": [
        "# Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "Z2kJTELiZKh3"
      },
      "outputs": [],
      "source": [
        "df = load_dataset(\"titanic\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xIqWyKoWZX7A"
      },
      "source": [
        "# Create Overview"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Wj8pVgmoqrof",
        "outputId": "6214693a-22b6-4e7f-804f-f8a3c252b936"
      },
      "outputs": [
        {
          "ename": "AttributeError",
          "evalue": "scipy.stats.stats is deprecated and has no attribute _power_div_lambda_names. Try looking in scipy.stats instead.",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[1;32m/Users/shamsuddeenmuhammad/Documents/VScode/python4dataanalysis/DataPrep_Demo.ipynb Cell 10'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/shamsuddeenmuhammad/Documents/VScode/python4dataanalysis/DataPrep_Demo.ipynb#ch0000007?line=0'>1</a>\u001b[0m plot(df)\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/__init__.py:101\u001b[0m, in \u001b[0;36mplot\u001b[0;34m(df, col1, col2, col3, config, display, dtype, progress)\u001b[0m\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/__init__.py?line=97'>98</a>\u001b[0m cfg \u001b[39m=\u001b[39m Config\u001b[39m.\u001b[39mfrom_dict(display, config)\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/__init__.py?line=99'>100</a>\u001b[0m \u001b[39mwith\u001b[39;00m ProgressBar(minimum\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m, disable\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m progress):\n\u001b[0;32m--> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/__init__.py?line=100'>101</a>\u001b[0m     itmdt \u001b[39m=\u001b[39m compute(df, col1, col2, col3, cfg\u001b[39m=\u001b[39;49mcfg, dtype\u001b[39m=\u001b[39;49mdtype)\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/__init__.py?line=102'>103</a>\u001b[0m to_render \u001b[39m=\u001b[39m render(itmdt, cfg)\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/__init__.py?line=104'>105</a>\u001b[0m \u001b[39mreturn\u001b[39;00m Container(to_render, itmdt\u001b[39m.\u001b[39mvisual_type, cfg)\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/__init__.py:72\u001b[0m, in \u001b[0;36mcompute\u001b[0;34m(df, col1, col2, col3, cfg, display, dtype)\u001b[0m\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/__init__.py?line=68'>69</a>\u001b[0m x, y, z \u001b[39m=\u001b[39m col1, col2, col3\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/__init__.py?line=70'>71</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39many\u001b[39m([x, y, z]):\n\u001b[0;32m---> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/__init__.py?line=71'>72</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m compute_overview(df, cfg, dtype)\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/__init__.py?line=73'>74</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39msum\u001b[39m(v \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mfor\u001b[39;00m v \u001b[39min\u001b[39;00m (x, y, z)) \u001b[39m==\u001b[39m \u001b[39m2\u001b[39m:\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/__init__.py?line=74'>75</a>\u001b[0m     x \u001b[39m=\u001b[39m x \u001b[39mor\u001b[39;00m y \u001b[39mor\u001b[39;00m z\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py:57\u001b[0m, in \u001b[0;36mcompute_overview\u001b[0;34m(df, cfg, dtype)\u001b[0m\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=54'>55</a>\u001b[0m col_dtype \u001b[39m=\u001b[39m frame\u001b[39m.\u001b[39mget_eda_dtype(col)\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=55'>56</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(col_dtype, Continuous) \u001b[39mand\u001b[39;00m (cfg\u001b[39m.\u001b[39mhist\u001b[39m.\u001b[39menable \u001b[39mor\u001b[39;00m cfg\u001b[39m.\u001b[39minsight\u001b[39m.\u001b[39menable):\n\u001b[0;32m---> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=56'>57</a>\u001b[0m     data\u001b[39m.\u001b[39mappend((col, col_dtype, _cont_calcs(frame\u001b[39m.\u001b[39;49mframe[col]\u001b[39m.\u001b[39;49mdropna(), cfg)))\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=57'>58</a>\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(col_dtype, (Nominal, GeoGraphy, GeoPoint, SmallCardNum)) \u001b[39mand\u001b[39;00m (\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=58'>59</a>\u001b[0m     cfg\u001b[39m.\u001b[39mbar\u001b[39m.\u001b[39menable \u001b[39mor\u001b[39;00m cfg\u001b[39m.\u001b[39minsight\u001b[39m.\u001b[39menable\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=59'>60</a>\u001b[0m ):\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=60'>61</a>\u001b[0m     srs \u001b[39m=\u001b[39m frame\u001b[39m.\u001b[39mget_col_as_str(col)\u001b[39m.\u001b[39mdropna()\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py:125\u001b[0m, in \u001b[0;36m_cont_calcs\u001b[0;34m(srs, cfg)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=121'>122</a>\u001b[0m data[\u001b[39m\"\u001b[39m\u001b[39mhist\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m da\u001b[39m.\u001b[39mhistogram(srs, bins\u001b[39m=\u001b[39mcfg\u001b[39m.\u001b[39mhist\u001b[39m.\u001b[39mbins, \u001b[39mrange\u001b[39m\u001b[39m=\u001b[39m(srs\u001b[39m.\u001b[39mmin(), srs\u001b[39m.\u001b[39mmax()))\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=123'>124</a>\u001b[0m \u001b[39mif\u001b[39;00m cfg\u001b[39m.\u001b[39minsight\u001b[39m.\u001b[39menable:\n\u001b[0;32m--> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=124'>125</a>\u001b[0m     data[\u001b[39m\"\u001b[39m\u001b[39mchisq\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m chisquare(data[\u001b[39m\"\u001b[39;49m\u001b[39mhist\u001b[39;49m\u001b[39m\"\u001b[39;49m][\u001b[39m0\u001b[39;49m])\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=125'>126</a>\u001b[0m     data[\u001b[39m\"\u001b[39m\u001b[39mnorm\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m normaltest(data[\u001b[39m\"\u001b[39m\u001b[39mhist\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m0\u001b[39m])\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dataprep/eda/distribution/compute/overview.py?line=126'>127</a>\u001b[0m     data[\u001b[39m\"\u001b[39m\u001b[39mskew\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m skewtest(data[\u001b[39m\"\u001b[39m\u001b[39mhist\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39m0\u001b[39m])\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py:136\u001b[0m, in \u001b[0;36mchisquare\u001b[0;34m(f_obs, f_exp, ddof, axis)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=133'>134</a>\u001b[0m \u001b[39m@derived_from\u001b[39m(scipy\u001b[39m.\u001b[39mstats)\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=134'>135</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mchisquare\u001b[39m(f_obs, f_exp\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, ddof\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, axis\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m):\n\u001b[0;32m--> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=135'>136</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m power_divergence(f_obs, f_exp\u001b[39m=\u001b[39;49mf_exp, ddof\u001b[39m=\u001b[39;49mddof, axis\u001b[39m=\u001b[39;49maxis, lambda_\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mpearson\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py:144\u001b[0m, in \u001b[0;36mpower_divergence\u001b[0;34m(f_obs, f_exp, ddof, axis, lambda_)\u001b[0m\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=138'>139</a>\u001b[0m \u001b[39m@derived_from\u001b[39m(scipy\u001b[39m.\u001b[39mstats)\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=139'>140</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mpower_divergence\u001b[39m(f_obs, f_exp\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, ddof\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, axis\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, lambda_\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=141'>142</a>\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(lambda_, \u001b[39mstr\u001b[39m):\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=142'>143</a>\u001b[0m         \u001b[39m# TODO: public api\u001b[39;00m\n\u001b[0;32m--> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=143'>144</a>\u001b[0m         \u001b[39mif\u001b[39;00m lambda_ \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m scipy\u001b[39m.\u001b[39;49mstats\u001b[39m.\u001b[39;49mstats\u001b[39m.\u001b[39;49m_power_div_lambda_names:\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=144'>145</a>\u001b[0m             names \u001b[39m=\u001b[39m \u001b[39mrepr\u001b[39m(\u001b[39mlist\u001b[39m(scipy\u001b[39m.\u001b[39mstats\u001b[39m.\u001b[39mstats\u001b[39m.\u001b[39m_power_div_lambda_names\u001b[39m.\u001b[39mkeys()))[\u001b[39m1\u001b[39m:\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=145'>146</a>\u001b[0m             \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=146'>147</a>\u001b[0m                 \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39minvalid string for lambda_: \u001b[39m\u001b[39m{\u001b[39;00mlambda_\u001b[39m!r}\u001b[39;00m\u001b[39m. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=147'>148</a>\u001b[0m                 \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mValid strings are \u001b[39m\u001b[39m{\u001b[39;00mnames\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m    <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/dask/array/stats.py?line=148'>149</a>\u001b[0m             )\n",
            "File \u001b[0;32m~/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py:54\u001b[0m, in \u001b[0;36m__getattr__\u001b[0;34m(name)\u001b[0m\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=51'>52</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__getattr__\u001b[39m(name):\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=52'>53</a>\u001b[0m     \u001b[39mif\u001b[39;00m name \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m __all__:\n\u001b[0;32m---> <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=53'>54</a>\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mAttributeError\u001b[39;00m(\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=54'>55</a>\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39mscipy.stats.stats is deprecated and has no attribute \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=55'>56</a>\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mname\u001b[39m}\u001b[39;00m\u001b[39m. Try looking in scipy.stats instead.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=57'>58</a>\u001b[0m     warnings\u001b[39m.\u001b[39mwarn(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mPlease use `\u001b[39m\u001b[39m{\u001b[39;00mname\u001b[39m}\u001b[39;00m\u001b[39m` from the `scipy.stats` namespace, \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=58'>59</a>\u001b[0m                   \u001b[39m\"\u001b[39m\u001b[39mthe `scipy.stats.stats` namespace is deprecated.\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=59'>60</a>\u001b[0m                   category\u001b[39m=\u001b[39m\u001b[39mDeprecationWarning\u001b[39;00m, stacklevel\u001b[39m=\u001b[39m\u001b[39m2\u001b[39m)\n\u001b[1;32m     <a href='file:///Users/shamsuddeenmuhammad/anaconda3/envs/datascience/lib/python3.9/site-packages/scipy/stats/stats.py?line=61'>62</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mgetattr\u001b[39m(_stats_py, name)\n",
            "\u001b[0;31mAttributeError\u001b[0m: scipy.stats.stats is deprecated and has no attribute _power_div_lambda_names. Try looking in scipy.stats instead."
          ]
        }
      ],
      "source": [
        "plot(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ltGYqdCsZg15"
      },
      "source": [
        "# Understand Missing values"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PfImxa2AaFjC"
      },
      "source": [
        "## 1. Missing value overview"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 543
        },
        "id": "g_0eQMZNaJrz",
        "outputId": "9e975596-5b20-43f3-b95b-2af7e92a37fd"
      },
      "outputs": [],
      "source": [
        "plot_missing(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wzl968IKaM2r"
      },
      "source": [
        "## 2. Impact of missing values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Had4sXbTaQIU",
        "outputId": "cdc576f1-4312-4eef-cc12-fe6b1de18e92"
      },
      "outputs": [],
      "source": [
        "plot_missing(df, \"Age\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k7HTLr8DaS7q"
      },
      "source": [
        "# Understand correlation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4YhbfQqgaWwZ"
      },
      "source": [
        "## 1. Correlation overview"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "id": "6Jy1HlTGaZkS",
        "outputId": "c420f6bf-0053-4627-aedb-55782ea55207"
      },
      "outputs": [],
      "source": [
        "plot_correlation(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4SGbAg4pabxj"
      },
      "source": [
        "## 2. Understand how other columns correlated to the given column"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "id": "mQQqYjZoajl7",
        "outputId": "85e31362-e1b4-45c4-d05f-820f92112998"
      },
      "outputs": [],
      "source": [
        "plot_correlation(df, \"Age\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3KeG_hSlapdb"
      },
      "source": [
        "# Understand single column"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ba73KV_Xasr0"
      },
      "source": [
        "## 1. Numerical column"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 688
        },
        "id": "TVBEf2Joav7V",
        "outputId": "42d103c5-1960-45b0-95fc-89e370748501"
      },
      "outputs": [],
      "source": [
        "plot(df, \"Age\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "El3g7704ax4j"
      },
      "source": [
        "## 2. Categorical column"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 580
        },
        "id": "xxtUntCHa3we",
        "outputId": "8cef6003-45d4-4491-acad-8b3521b2a266"
      },
      "outputs": [],
      "source": [
        "plot(df, \"Sex\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7UwrFo7ja6bd"
      },
      "source": [
        "# Understand column relationship"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9J1i7VnBa8Jk"
      },
      "source": [
        "## 1. Numerical vs Numerical"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "id": "lnNQepOPbBs-",
        "outputId": "85a5f077-6d07-40ad-bfe7-6fbeb7b7afe3"
      },
      "outputs": [],
      "source": [
        "plot(df, \"Age\", \"Fare\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nNj1ZMQnbDYE"
      },
      "source": [
        "## 2. Numerical vs Categorical"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "id": "-WouiN_3bGHF",
        "outputId": "4b1378fd-140f-4b42-ea9f-95626df4b2c5"
      },
      "outputs": [],
      "source": [
        "plot(df, \"Age\", \"Sex\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dP2B7z6_bINU"
      },
      "source": [
        "## 3. Categorical vs Categorical"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "id": "lfdZeenXbLNf",
        "outputId": "c9a187d6-1ba3-45f4-8a00-d28193ef0646"
      },
      "outputs": [],
      "source": [
        "plot(df, \"Sex\", \"Survived\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vYL46mD5bN1u"
      },
      "source": [
        "# Customize Visualization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I5gzSXf5bQwO"
      },
      "source": [
        "## 1. Show only interested visualizations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "id": "Qt6YN0tFbTzP",
        "outputId": "8be179c4-639a-43bf-9bec-b8a8cbd584fb"
      },
      "outputs": [],
      "source": [
        " plot(df, \"Age\", display = [\"Histogram\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YqzRQs8PbV9V"
      },
      "source": [
        "## 2. Set config for visualizations."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 543
        },
        "id": "tklNKCXmbYn-",
        "outputId": "cb7724fa-ddfb-45bd-8dba-c4c103cc5cda"
      },
      "outputs": [],
      "source": [
        "plot(df, \"Age\", display = [\"Histogram\"], config = {\"hist.bins\": 1000, \"height\":500})"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SZPBDjcUbohu"
      },
      "source": [
        "# Compare two dataframes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "JjldI_TMbtyn",
        "outputId": "cf4dbde8-029d-46f7-e11d-bdf94fc75ea5"
      },
      "outputs": [],
      "source": [
        "df1 = df[df[\"Survived\"] == 0]\n",
        "df2 = df[df[\"Survived\"] == 1]\n",
        "plot_diff([df1, df2])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FgdZfyQnbud2"
      },
      "source": [
        "# Create an html report"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "G-dOE2QTbyjB",
        "outputId": "475a2352-02d4-49bb-acf1-c1b6e9205d11"
      },
      "outputs": [],
      "source": [
        "create_report(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qVS7xm-lbzKP"
      },
      "source": [
        "# Analyze geographical data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 580
        },
        "id": "F20SeWXtb2EP",
        "outputId": "1ed15509-419f-4d9f-9410-0c1420c101be"
      },
      "outputs": [],
      "source": [
        "country = load_dataset(\"countries\")\n",
        "plot(country, \"Country\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "DataPrep Demo.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "895aa688263384567493505d09875376e7685297a6922210da729f70c5caa3cf"
    },
    "kernelspec": {
      "display_name": "Python 3.9.12",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}