Data_Preparation_and_Feature_Selection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "K2dUN5mRyb4h"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Jatansahu/GOLDDIGGER_GROUP12_PROJECT_01/blob/main/Data_Preparation_and_Feature_Selection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#1) Importing Libraries"
      ],
      "metadata": {
        "id": "l4RgjiumxZEi"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wYXdupC598K8"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "# import polars as p\n",
        "import numpy as np\n",
        "import warnings\n",
        "warnings.filterwarnings(action = \"ignore\")\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.preprocessing import LabelEncoder,StandardScaler"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#2) Loading Dataset"
      ],
      "metadata": {
        "id": "vm3FBzstxgVT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"https://raw.githubusercontent.com/Jatansahu/GOLDDIGGER_GROUP12_PROJECT_01/main/dataset.csv\")"
      ],
      "metadata": {
        "id": "6caeZMwz-yUY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Previewing data\n",
        "data.head()"
      ],
      "metadata": {
        "id": "_6Goi0WyxjzK",
        "outputId": "2362bb5f-a4a4-42ad-c022-ff531bda7fca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 690
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   YearStart  YearEnd LocationAbbr LocationDesc  \\\n",
              "0       2011     2011           AL      Alabama   \n",
              "1       2011     2011           AL      Alabama   \n",
              "2       2011     2011           AL      Alabama   \n",
              "3       2011     2011           AL      Alabama   \n",
              "4       2011     2011           AL      Alabama   \n",
              "\n",
              "                                   Datasource                    Class  \\\n",
              "0  Behavioral Risk Factor Surveillance System  Obesity / Weight Status   \n",
              "1  Behavioral Risk Factor Surveillance System  Obesity / Weight Status   \n",
              "2  Behavioral Risk Factor Surveillance System  Obesity / Weight Status   \n",
              "3  Behavioral Risk Factor Surveillance System  Obesity / Weight Status   \n",
              "4  Behavioral Risk Factor Surveillance System  Obesity / Weight Status   \n",
              "\n",
              "                     Topic                                           Question  \\\n",
              "0  Obesity / Weight Status  Percent of adults aged 18 years and older who ...   \n",
              "1  Obesity / Weight Status  Percent of adults aged 18 years and older who ...   \n",
              "2  Obesity / Weight Status  Percent of adults aged 18 years and older who ...   \n",
              "3  Obesity / Weight Status  Percent of adults aged 18 years and older who ...   \n",
              "4  Obesity / Weight Status  Percent of adults aged 18 years and older who ...   \n",
              "\n",
              "   Data_Value_Unit Data_Value_Type  ...  \\\n",
              "0              NaN           Value  ...   \n",
              "1              NaN           Value  ...   \n",
              "2              NaN           Value  ...   \n",
              "3              NaN           Value  ...   \n",
              "4              NaN           Value  ...   \n",
              "\n",
              "                               GeoLocation  ClassID TopicID QuestionID  \\\n",
              "0  (32.84057112200048, -86.63186076199969)      OWS    OWS1       Q036   \n",
              "1  (32.84057112200048, -86.63186076199969)      OWS    OWS1       Q036   \n",
              "2  (32.84057112200048, -86.63186076199969)      OWS    OWS1       Q036   \n",
              "3  (32.84057112200048, -86.63186076199969)      OWS    OWS1       Q036   \n",
              "4  (32.84057112200048, -86.63186076199969)      OWS    OWS1       Q036   \n",
              "\n",
              "   DataValueTypeID  LocationID  StratificationCategory1  \\\n",
              "0            VALUE           1                    Total   \n",
              "1            VALUE           1                   Gender   \n",
              "2            VALUE           1                   Gender   \n",
              "3            VALUE           1                Education   \n",
              "4            VALUE           1                Education   \n",
              "\n",
              "         Stratification1 StratificationCategoryId1 StratificationID1  \n",
              "0                  Total                       OVR           OVERALL  \n",
              "1                   Male                       GEN              MALE  \n",
              "2                 Female                       GEN            FEMALE  \n",
              "3  Less than high school                       EDU             EDUHS  \n",
              "4   High school graduate                       EDU         EDUHSGRAD  \n",
              "\n",
              "[5 rows x 33 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4fbd4df5-f484-4395-aac9-ef481f098601\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>YearStart</th>\n",
              "      <th>YearEnd</th>\n",
              "      <th>LocationAbbr</th>\n",
              "      <th>LocationDesc</th>\n",
              "      <th>Datasource</th>\n",
              "      <th>Class</th>\n",
              "      <th>Topic</th>\n",
              "      <th>Question</th>\n",
              "      <th>Data_Value_Unit</th>\n",
              "      <th>Data_Value_Type</th>\n",
              "      <th>...</th>\n",
              "      <th>GeoLocation</th>\n",
              "      <th>ClassID</th>\n",
              "      <th>TopicID</th>\n",
              "      <th>QuestionID</th>\n",
              "      <th>DataValueTypeID</th>\n",
              "      <th>LocationID</th>\n",
              "      <th>StratificationCategory1</th>\n",
              "      <th>Stratification1</th>\n",
              "      <th>StratificationCategoryId1</th>\n",
              "      <th>StratificationID1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2011</td>\n",
              "      <td>2011</td>\n",
              "      <td>AL</td>\n",
              "      <td>Alabama</td>\n",
              "      <td>Behavioral Risk Factor Surveillance System</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Percent of adults aged 18 years and older who ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Value</td>\n",
              "      <td>...</td>\n",
              "      <td>(32.84057112200048, -86.63186076199969)</td>\n",
              "      <td>OWS</td>\n",
              "      <td>OWS1</td>\n",
              "      <td>Q036</td>\n",
              "      <td>VALUE</td>\n",
              "      <td>1</td>\n",
              "      <td>Total</td>\n",
              "      <td>Total</td>\n",
              "      <td>OVR</td>\n",
              "      <td>OVERALL</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2011</td>\n",
              "      <td>2011</td>\n",
              "      <td>AL</td>\n",
              "      <td>Alabama</td>\n",
              "      <td>Behavioral Risk Factor Surveillance System</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Percent of adults aged 18 years and older who ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Value</td>\n",
              "      <td>...</td>\n",
              "      <td>(32.84057112200048, -86.63186076199969)</td>\n",
              "      <td>OWS</td>\n",
              "      <td>OWS1</td>\n",
              "      <td>Q036</td>\n",
              "      <td>VALUE</td>\n",
              "      <td>1</td>\n",
              "      <td>Gender</td>\n",
              "      <td>Male</td>\n",
              "      <td>GEN</td>\n",
              "      <td>MALE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2011</td>\n",
              "      <td>2011</td>\n",
              "      <td>AL</td>\n",
              "      <td>Alabama</td>\n",
              "      <td>Behavioral Risk Factor Surveillance System</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Percent of adults aged 18 years and older who ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Value</td>\n",
              "      <td>...</td>\n",
              "      <td>(32.84057112200048, -86.63186076199969)</td>\n",
              "      <td>OWS</td>\n",
              "      <td>OWS1</td>\n",
              "      <td>Q036</td>\n",
              "      <td>VALUE</td>\n",
              "      <td>1</td>\n",
              "      <td>Gender</td>\n",
              "      <td>Female</td>\n",
              "      <td>GEN</td>\n",
              "      <td>FEMALE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2011</td>\n",
              "      <td>2011</td>\n",
              "      <td>AL</td>\n",
              "      <td>Alabama</td>\n",
              "      <td>Behavioral Risk Factor Surveillance System</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Percent of adults aged 18 years and older who ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Value</td>\n",
              "      <td>...</td>\n",
              "      <td>(32.84057112200048, -86.63186076199969)</td>\n",
              "      <td>OWS</td>\n",
              "      <td>OWS1</td>\n",
              "      <td>Q036</td>\n",
              "      <td>VALUE</td>\n",
              "      <td>1</td>\n",
              "      <td>Education</td>\n",
              "      <td>Less than high school</td>\n",
              "      <td>EDU</td>\n",
              "      <td>EDUHS</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2011</td>\n",
              "      <td>2011</td>\n",
              "      <td>AL</td>\n",
              "      <td>Alabama</td>\n",
              "      <td>Behavioral Risk Factor Surveillance System</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Obesity / Weight Status</td>\n",
              "      <td>Percent of adults aged 18 years and older who ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Value</td>\n",
              "      <td>...</td>\n",
              "      <td>(32.84057112200048, -86.63186076199969)</td>\n",
              "      <td>OWS</td>\n",
              "      <td>OWS1</td>\n",
              "      <td>Q036</td>\n",
              "      <td>VALUE</td>\n",
              "      <td>1</td>\n",
              "      <td>Education</td>\n",
              "      <td>High school graduate</td>\n",
              "      <td>EDU</td>\n",
              "      <td>EDUHSGRAD</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 33 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4fbd4df5-f484-4395-aac9-ef481f098601')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-4fbd4df5-f484-4395-aac9-ef481f098601 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-4fbd4df5-f484-4395-aac9-ef481f098601');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-69a7f838-66b2-4e65-8092-08b301b6d386\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-69a7f838-66b2-4e65-8092-08b301b6d386')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-69a7f838-66b2-4e65-8092-08b301b6d386 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Looking at the above dataset our target variable is the column \"data_value\".**"
      ],
      "metadata": {
        "id": "zJ4IXDnUxqLO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#3) Data Description"
      ],
      "metadata": {
        "id": "kzx6RPeZyj3r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data.describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "1TpN8GYtJAel",
        "outputId": "8eeef91d-fa56-4b0c-afdc-9ec76da03f21"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          YearStart       YearEnd  Data_Value_Unit    Data_Value  \\\n",
              "count  53392.000000  53392.000000              0.0  48346.000000   \n",
              "mean    2013.281465   2013.281465              NaN     31.156681   \n",
              "std        1.693300      1.693300              NaN     10.247033   \n",
              "min     2011.000000   2011.000000              NaN      0.900000   \n",
              "25%     2012.000000   2012.000000              NaN     24.100000   \n",
              "50%     2013.000000   2013.000000              NaN     30.700000   \n",
              "75%     2015.000000   2015.000000              NaN     37.000000   \n",
              "max     2016.000000   2016.000000              NaN     77.600000   \n",
              "\n",
              "       Data_Value_Alt  Low_Confidence_Limit  High_Confidence_Limit   \\\n",
              "count    48346.000000          48346.000000            48346.000000   \n",
              "mean        31.156681             26.892227               35.989997   \n",
              "std         10.247033             10.038584               11.205813   \n",
              "min          0.900000              0.300000                3.000000   \n",
              "25%         24.100000             20.000000               28.200000   \n",
              "50%         30.700000             26.450000               35.600000   \n",
              "75%         37.000000             32.900000               42.200000   \n",
              "max         77.600000             69.500000               87.700000   \n",
              "\n",
              "        Sample_Size    LocationID  \n",
              "count   48346.00000  53392.000000  \n",
              "mean     3889.19286     30.282215  \n",
              "std     19829.42129     16.821318  \n",
              "min        50.00000      1.000000  \n",
              "25%       566.00000     17.000000  \n",
              "50%      1209.00000     30.000000  \n",
              "75%      2519.00000     44.000000  \n",
              "max    476876.00000     78.000000  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1f201409-f133-493f-817e-45c495ff2676\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>YearStart</th>\n",
              "      <th>YearEnd</th>\n",
              "      <th>Data_Value_Unit</th>\n",
              "      <th>Data_Value</th>\n",
              "      <th>Data_Value_Alt</th>\n",
              "      <th>Low_Confidence_Limit</th>\n",
              "      <th>High_Confidence_Limit</th>\n",
              "      <th>Sample_Size</th>\n",
              "      <th>LocationID</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>53392.000000</td>\n",
              "      <td>53392.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>48346.000000</td>\n",
              "      <td>48346.000000</td>\n",
              "      <td>48346.000000</td>\n",
              "      <td>48346.000000</td>\n",
              "      <td>48346.00000</td>\n",
              "      <td>53392.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>2013.281465</td>\n",
              "      <td>2013.281465</td>\n",
              "      <td>NaN</td>\n",
              "      <td>31.156681</td>\n",
              "      <td>31.156681</td>\n",
              "      <td>26.892227</td>\n",
              "      <td>35.989997</td>\n",
              "      <td>3889.19286</td>\n",
              "      <td>30.282215</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.693300</td>\n",
              "      <td>1.693300</td>\n",
              "      <td>NaN</td>\n",
              "      <td>10.247033</td>\n",
              "      <td>10.247033</td>\n",
              "      <td>10.038584</td>\n",
              "      <td>11.205813</td>\n",
              "      <td>19829.42129</td>\n",
              "      <td>16.821318</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>2011.000000</td>\n",
              "      <td>2011.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>0.300000</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>50.00000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>2012.000000</td>\n",
              "      <td>2012.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>24.100000</td>\n",
              "      <td>24.100000</td>\n",
              "      <td>20.000000</td>\n",
              "      <td>28.200000</td>\n",
              "      <td>566.00000</td>\n",
              "      <td>17.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>2013.000000</td>\n",
              "      <td>2013.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>30.700000</td>\n",
              "      <td>30.700000</td>\n",
              "      <td>26.450000</td>\n",
              "      <td>35.600000</td>\n",
              "      <td>1209.00000</td>\n",
              "      <td>30.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>2015.000000</td>\n",
              "      <td>2015.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>37.000000</td>\n",
              "      <td>37.000000</td>\n",
              "      <td>32.900000</td>\n",
              "      <td>42.200000</td>\n",
              "      <td>2519.00000</td>\n",
              "      <td>44.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2016.000000</td>\n",
              "      <td>2016.000000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>77.600000</td>\n",
              "      <td>77.600000</td>\n",
              "      <td>69.500000</td>\n",
              "      <td>87.700000</td>\n",
              "      <td>476876.00000</td>\n",
              "      <td>78.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1f201409-f133-493f-817e-45c495ff2676')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-1f201409-f133-493f-817e-45c495ff2676 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-1f201409-f133-493f-817e-45c495ff2676');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1c5cc4de-51ad-4afc-9211-18be42253e1f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1c5cc4de-51ad-4afc-9211-18be42253e1f')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1c5cc4de-51ad-4afc-9211-18be42253e1f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FZTRKziCJGwe",
        "outputId": "5931fd5c-d5ed-4af2-dfef-474fb45da348"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 53392 entries, 0 to 53391\n",
            "Data columns (total 33 columns):\n",
            " #   Column                      Non-Null Count  Dtype  \n",
            "---  ------                      --------------  -----  \n",
            " 0   YearStart                   53392 non-null  int64  \n",
            " 1   YearEnd                     53392 non-null  int64  \n",
            " 2   LocationAbbr                53392 non-null  object \n",
            " 3   LocationDesc                53392 non-null  object \n",
            " 4   Datasource                  53392 non-null  object \n",
            " 5   Class                       53392 non-null  object \n",
            " 6   Topic                       53392 non-null  object \n",
            " 7   Question                    53392 non-null  object \n",
            " 8   Data_Value_Unit             0 non-null      float64\n",
            " 9   Data_Value_Type             53392 non-null  object \n",
            " 10  Data_Value                  48346 non-null  float64\n",
            " 11  Data_Value_Alt              48346 non-null  float64\n",
            " 12  Data_Value_Footnote_Symbol  5046 non-null   object \n",
            " 13  Data_Value_Footnote         5046 non-null   object \n",
            " 14  Low_Confidence_Limit        48346 non-null  float64\n",
            " 15  High_Confidence_Limit       48346 non-null  float64\n",
            " 16  Sample_Size                 48346 non-null  float64\n",
            " 17  Total                       1907 non-null   object \n",
            " 18  Age(years)                  11438 non-null  object \n",
            " 19  Education                   7628 non-null   object \n",
            " 20  Gender                      3814 non-null   object \n",
            " 21  Income                      13349 non-null  object \n",
            " 22  Race/Ethnicity              15256 non-null  object \n",
            " 23  GeoLocation                 52384 non-null  object \n",
            " 24  ClassID                     53392 non-null  object \n",
            " 25  TopicID                     53392 non-null  object \n",
            " 26  QuestionID                  53392 non-null  object \n",
            " 27  DataValueTypeID             53392 non-null  object \n",
            " 28  LocationID                  53392 non-null  int64  \n",
            " 29  StratificationCategory1     53392 non-null  object \n",
            " 30  Stratification1             53392 non-null  object \n",
            " 31  StratificationCategoryId1   53392 non-null  object \n",
            " 32  StratificationID1           53392 non-null  object \n",
            "dtypes: float64(6), int64(3), object(24)\n",
            "memory usage: 13.4+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "data.shape"
      ],
      "metadata": {
        "id": "CeglxH0ixvB4",
        "outputId": "b1bed081-6493-4351-f40e-db9b937570ac",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(53392, 33)"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "col = data.columns\n",
        "col"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amMCBZMvEg5X",
        "outputId": "76d3d38c-5ff5-4e82-e186-c6781009ad63"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['YearStart', 'YearEnd', 'LocationAbbr', 'LocationDesc', 'Datasource',\n",
              "       'Class', 'Topic', 'Question', 'Data_Value_Unit', 'Data_Value_Type',\n",
              "       'Data_Value', 'Data_Value_Alt', 'Data_Value_Footnote_Symbol',\n",
              "       'Data_Value_Footnote', 'Low_Confidence_Limit', 'High_Confidence_Limit ',\n",
              "       'Sample_Size', 'Total', 'Age(years)', 'Education', 'Gender', 'Income',\n",
              "       'Race/Ethnicity', 'GeoLocation', 'ClassID', 'TopicID', 'QuestionID',\n",
              "       'DataValueTypeID', 'LocationID', 'StratificationCategory1',\n",
              "       'Stratification1', 'StratificationCategoryId1', 'StratificationID1'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#4) Selecting necessary columns"
      ],
      "metadata": {
        "id": "K2dUN5mRyb4h"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 4.1) Selection of columns"
      ],
      "metadata": {
        "id": "9qslLlso6Dv-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Selecting features which will help us to classify different stratification categories.**\n",
        "\n",
        "Most of the predictive values are correlated to each other so we dropped those columns which have less important(Refer EDA file)."
      ],
      "metadata": {
        "id": "2PBuhfq2ywmK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = data[['YearEnd','LocationID','Sample_Size','Age(years)','Education','Gender','Income','Race/Ethnicity','ClassID','QuestionID','StratificationCategory1','Stratification1','StratificationCategoryId1','StratificationID1','Total','Data_Value']]"
      ],
      "metadata": {
        "id": "utDFeBxwy_SG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#dropping columns\n",
        "for i in df.columns:\n",
        "  data = data.drop([i],1)"
      ],
      "metadata": {
        "id": "GnBryor66M0j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "id": "zldl6WTdzDyN",
        "outputId": "90cdcaa2-78b3-47d3-f529-77ab9012bc1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       YearEnd  LocationID  Sample_Size Age(years)              Education  \\\n",
              "0         2011           1       7304.0        NaN                    NaN   \n",
              "1         2011           1       2581.0        NaN                    NaN   \n",
              "2         2011           1       4723.0        NaN                    NaN   \n",
              "3         2011           1       1153.0        NaN  Less than high school   \n",
              "4         2011           1       2402.0        NaN   High school graduate   \n",
              "...        ...         ...          ...        ...                    ...   \n",
              "53387     2016          78          NaN        NaN                    NaN   \n",
              "53388     2016          78          NaN        NaN                    NaN   \n",
              "53389     2016          78          NaN        NaN                    NaN   \n",
              "53390     2016          78          NaN        NaN                    NaN   \n",
              "53391     2016          78          NaN        NaN                    NaN   \n",
              "\n",
              "       Gender Income                 Race/Ethnicity ClassID QuestionID  \\\n",
              "0         NaN    NaN                            NaN     OWS       Q036   \n",
              "1        Male    NaN                            NaN     OWS       Q036   \n",
              "2      Female    NaN                            NaN     OWS       Q036   \n",
              "3         NaN    NaN                            NaN     OWS       Q036   \n",
              "4         NaN    NaN                            NaN     OWS       Q036   \n",
              "...       ...    ...                            ...     ...        ...   \n",
              "53387     NaN    NaN                          Asian      PA       Q047   \n",
              "53388     NaN    NaN      Hawaiian/Pacific Islander      PA       Q047   \n",
              "53389     NaN    NaN  American Indian/Alaska Native      PA       Q047   \n",
              "53390     NaN    NaN                2 or more races      PA       Q047   \n",
              "53391     NaN    NaN                          Other      PA       Q047   \n",
              "\n",
              "      StratificationCategory1                Stratification1  \\\n",
              "0                       Total                          Total   \n",
              "1                      Gender                           Male   \n",
              "2                      Gender                         Female   \n",
              "3                   Education          Less than high school   \n",
              "4                   Education           High school graduate   \n",
              "...                       ...                            ...   \n",
              "53387          Race/Ethnicity                          Asian   \n",
              "53388          Race/Ethnicity      Hawaiian/Pacific Islander   \n",
              "53389          Race/Ethnicity  American Indian/Alaska Native   \n",
              "53390          Race/Ethnicity                2 or more races   \n",
              "53391          Race/Ethnicity                          Other   \n",
              "\n",
              "      StratificationCategoryId1 StratificationID1  Total  Data_Value  \n",
              "0                           OVR           OVERALL  Total        32.0  \n",
              "1                           GEN              MALE    NaN        32.3  \n",
              "2                           GEN            FEMALE    NaN        31.8  \n",
              "3                           EDU             EDUHS    NaN        33.6  \n",
              "4                           EDU         EDUHSGRAD    NaN        32.8  \n",
              "...                         ...               ...    ...         ...  \n",
              "53387                      RACE           RACEASN    NaN         NaN  \n",
              "53388                      RACE           RACEHPI    NaN         NaN  \n",
              "53389                      RACE           RACENAA    NaN         NaN  \n",
              "53390                      RACE         RACE2PLUS    NaN         NaN  \n",
              "53391                      RACE           RACEOTH    NaN         NaN  \n",
              "\n",
              "[53392 rows x 16 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2d9e6a45-2e76-4e56-b186-41711dfebc50\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>YearEnd</th>\n",
              "      <th>LocationID</th>\n",
              "      <th>Sample_Size</th>\n",
              "      <th>Age(years)</th>\n",
              "      <th>Education</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Income</th>\n",
              "      <th>Race/Ethnicity</th>\n",
              "      <th>ClassID</th>\n",
              "      <th>QuestionID</th>\n",
              "      <th>StratificationCategory1</th>\n",
              "      <th>Stratification1</th>\n",
              "      <th>StratificationCategoryId1</th>\n",
              "      <th>StratificationID1</th>\n",
              "      <th>Total</th>\n",
              "      <th>Data_Value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>7304.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Total</td>\n",
              "      <td>Total</td>\n",
              "      <td>OVR</td>\n",
              "      <td>OVERALL</td>\n",
              "      <td>Total</td>\n",
              "      <td>32.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>2581.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Male</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Gender</td>\n",
              "      <td>Male</td>\n",
              "      <td>GEN</td>\n",
              "      <td>MALE</td>\n",
              "      <td>NaN</td>\n",
              "      <td>32.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>4723.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Female</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Gender</td>\n",
              "      <td>Female</td>\n",
              "      <td>GEN</td>\n",
              "      <td>FEMALE</td>\n",
              "      <td>NaN</td>\n",
              "      <td>31.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1153.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Less than high school</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Education</td>\n",
              "      <td>Less than high school</td>\n",
              "      <td>EDU</td>\n",
              "      <td>EDUHS</td>\n",
              "      <td>NaN</td>\n",
              "      <td>33.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>2402.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>High school graduate</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Education</td>\n",
              "      <td>High school graduate</td>\n",
              "      <td>EDU</td>\n",
              "      <td>EDUHSGRAD</td>\n",
              "      <td>NaN</td>\n",
              "      <td>32.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53387</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Asian</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Asian</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEASN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53388</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Hawaiian/Pacific Islander</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Hawaiian/Pacific Islander</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEHPI</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53389</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>American Indian/Alaska Native</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>American Indian/Alaska Native</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACENAA</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53390</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2 or more races</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>2 or more races</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACE2PLUS</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53391</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Other</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Other</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEOTH</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>53392 rows × 16 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2d9e6a45-2e76-4e56-b186-41711dfebc50')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2d9e6a45-2e76-4e56-b186-41711dfebc50 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2d9e6a45-2e76-4e56-b186-41711dfebc50');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-da9dd95c-e717-40b8-82b2-b18067369de1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-da9dd95c-e717-40b8-82b2-b18067369de1')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-da9dd95c-e717-40b8-82b2-b18067369de1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 4.2) Reason for removing columns"
      ],
      "metadata": {
        "id": "qDbsjceU58N3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "*   'Low_Confidence_Limit' => It is the lower limit of our target variable. Closer to Data_value.\n",
        "*   'High_Confidence_Limit' => It is the higher limit of our target variable. Closer to Data_value.\n",
        "\n",
        "*   'Data_Value_Alt' => Closer to Data_value.\n",
        "*   'YearEnd' => We have taken starting year. We will take either 'Yearstart' or 'Yearend. Otherwise predator will correlated.\n",
        "\n",
        "Most of the variables are not relevant to the target variables."
      ],
      "metadata": {
        "id": "XeU7Ct6G56E7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#5) Data Standardization"
      ],
      "metadata": {
        "id": "wu36Gk7JzfoW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##5.1) Standardizing column names"
      ],
      "metadata": {
        "id": "HZCDuWQEzyVe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.columns = df.columns.str.lower()\n",
        "df.head()"
      ],
      "metadata": {
        "id": "Gvd1pUobzn1W",
        "outputId": "224e1b00-ece1-4815-d8a1-da0379841071",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   yearend  locationid  sample_size age(years)              education  gender  \\\n",
              "0     2011           1       7304.0        NaN                    NaN     NaN   \n",
              "1     2011           1       2581.0        NaN                    NaN    Male   \n",
              "2     2011           1       4723.0        NaN                    NaN  Female   \n",
              "3     2011           1       1153.0        NaN  Less than high school     NaN   \n",
              "4     2011           1       2402.0        NaN   High school graduate     NaN   \n",
              "\n",
              "  income race/ethnicity classid questionid stratificationcategory1  \\\n",
              "0    NaN            NaN     OWS       Q036                   Total   \n",
              "1    NaN            NaN     OWS       Q036                  Gender   \n",
              "2    NaN            NaN     OWS       Q036                  Gender   \n",
              "3    NaN            NaN     OWS       Q036               Education   \n",
              "4    NaN            NaN     OWS       Q036               Education   \n",
              "\n",
              "         stratification1 stratificationcategoryid1 stratificationid1  total  \\\n",
              "0                  Total                       OVR           OVERALL  Total   \n",
              "1                   Male                       GEN              MALE    NaN   \n",
              "2                 Female                       GEN            FEMALE    NaN   \n",
              "3  Less than high school                       EDU             EDUHS    NaN   \n",
              "4   High school graduate                       EDU         EDUHSGRAD    NaN   \n",
              "\n",
              "   data_value  \n",
              "0        32.0  \n",
              "1        32.3  \n",
              "2        31.8  \n",
              "3        33.6  \n",
              "4        32.8  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-848cefc5-7e03-40a8-8459-18b3fa1e56b7\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>yearend</th>\n",
              "      <th>locationid</th>\n",
              "      <th>sample_size</th>\n",
              "      <th>age(years)</th>\n",
              "      <th>education</th>\n",
              "      <th>gender</th>\n",
              "      <th>income</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>classid</th>\n",
              "      <th>questionid</th>\n",
              "      <th>stratificationcategory1</th>\n",
              "      <th>stratification1</th>\n",
              "      <th>stratificationcategoryid1</th>\n",
              "      <th>stratificationid1</th>\n",
              "      <th>total</th>\n",
              "      <th>data_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>7304.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Total</td>\n",
              "      <td>Total</td>\n",
              "      <td>OVR</td>\n",
              "      <td>OVERALL</td>\n",
              "      <td>Total</td>\n",
              "      <td>32.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>2581.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Male</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Gender</td>\n",
              "      <td>Male</td>\n",
              "      <td>GEN</td>\n",
              "      <td>MALE</td>\n",
              "      <td>NaN</td>\n",
              "      <td>32.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>4723.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Female</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Gender</td>\n",
              "      <td>Female</td>\n",
              "      <td>GEN</td>\n",
              "      <td>FEMALE</td>\n",
              "      <td>NaN</td>\n",
              "      <td>31.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1153.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Less than high school</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Education</td>\n",
              "      <td>Less than high school</td>\n",
              "      <td>EDU</td>\n",
              "      <td>EDUHS</td>\n",
              "      <td>NaN</td>\n",
              "      <td>33.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>2402.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>High school graduate</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Education</td>\n",
              "      <td>High school graduate</td>\n",
              "      <td>EDU</td>\n",
              "      <td>EDUHSGRAD</td>\n",
              "      <td>NaN</td>\n",
              "      <td>32.8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-848cefc5-7e03-40a8-8459-18b3fa1e56b7')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-848cefc5-7e03-40a8-8459-18b3fa1e56b7 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-848cefc5-7e03-40a8-8459-18b3fa1e56b7');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6b9b044c-09ca-4390-8460-ed894ae9c797\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6b9b044c-09ca-4390-8460-ed894ae9c797')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6b9b044c-09ca-4390-8460-ed894ae9c797 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##5.2) Optimizing data (Data partitioning)\n"
      ],
      "metadata": {
        "id": "X6DEjjhhz411"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "distinct_values = df['stratificationcategoryid1'].unique()\n",
        "distinct_values"
      ],
      "metadata": {
        "id": "FipluSvGzqr2",
        "outputId": "4b20e01d-c95e-4d56-9cbf-0d3c07b7f743",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['OVR', 'GEN', 'EDU', 'AGEYR', 'INC', 'RACE'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Made 6 different dataframes for each category.**"
      ],
      "metadata": {
        "id": "fQTjUlzS0Ghl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "total = df[df['stratificationcategoryid1'] == 'OVR'] # total\n",
        "gen = df[df['stratificationcategoryid1'] == 'GEN'] #gender\n",
        "edu  = df[df['stratificationcategoryid1'] == 'EDU'] #Education\n",
        "age = df[df['stratificationcategoryid1'] == 'AGEYR'] #Age Range of sample\n",
        "inc  = df[df['stratificationcategoryid1'] == 'INC'] #income range\n",
        "race = df[df['stratificationcategoryid1'] == 'RACE'] # race"
      ],
      "metadata": {
        "id": "994m_DUWztAA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##5.3) Size of each category dataframe"
      ],
      "metadata": {
        "id": "T945dMlK0ahO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Shape of category Race/ethinicity is {race.shape}')\n",
        "print(f'Shape of category Income range is {inc.shape}')\n",
        "print(f'Shape of category Age range of sample is {age.shape}')\n",
        "print(f'Shape of category Education is {edu.shape}')\n",
        "print(f'Shape of category Gender is {gen.shape}')\n",
        "print(f'Shape of category total is {total.shape}')"
      ],
      "metadata": {
        "id": "VliD0h3qzxBH",
        "outputId": "2508028e-0bef-4128-e698-ae573c880c36",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of category Race/ethinicity is (15256, 16)\n",
            "Shape of category Income range is (13349, 16)\n",
            "Shape of category Age range of sample is (11438, 16)\n",
            "Shape of category Education is (7628, 16)\n",
            "Shape of category Gender is (3814, 16)\n",
            "Shape of category total is (1907, 16)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#6) Looking for Null values"
      ],
      "metadata": {
        "id": "4Yk5hJDzx2dW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##6.1) For total dataframe\n",
        "\n"
      ],
      "metadata": {
        "id": "txwZ6Pat0hJJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(data.isnull().sum())"
      ],
      "metadata": {
        "id": "ZL9S3ExFx9oH",
        "outputId": "417b412d-9038-43c6-b00b-9b42c5d0750a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "YearStart                         0\n",
            "LocationAbbr                      0\n",
            "LocationDesc                      0\n",
            "Datasource                        0\n",
            "Class                             0\n",
            "Topic                             0\n",
            "Question                          0\n",
            "Data_Value_Unit               53392\n",
            "Data_Value_Type                   0\n",
            "Data_Value_Alt                 5046\n",
            "Data_Value_Footnote_Symbol    48346\n",
            "Data_Value_Footnote           48346\n",
            "Low_Confidence_Limit           5046\n",
            "High_Confidence_Limit          5046\n",
            "GeoLocation                    1008\n",
            "TopicID                           0\n",
            "DataValueTypeID                   0\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##6.2) Missing values (Category wise)"
      ],
      "metadata": {
        "id": "T9gvVhgpj4XX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###6.2.1) Race"
      ],
      "metadata": {
        "id": "lHMcPh6Xm1vc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "race.isnull().sum()"
      ],
      "metadata": {
        "id": "lIbHw1NwkZPc",
        "outputId": "9b210efc-b821-4756-fb91-3ae45c2e3587",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                          0\n",
              "locationid                       0\n",
              "sample_size                   5046\n",
              "age(years)                   15256\n",
              "education                    15256\n",
              "gender                       15256\n",
              "income                       15256\n",
              "race/ethnicity                   0\n",
              "classid                          0\n",
              "questionid                       0\n",
              "stratificationcategory1          0\n",
              "stratification1                  0\n",
              "stratificationcategoryid1        0\n",
              "stratificationid1                0\n",
              "total                        15256\n",
              "data_value                    5046\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "  **For race, sample_size and data_value is also null so we will remove these.**"
      ],
      "metadata": {
        "id": "GLVAoGbgtlM4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "race= race.dropna(subset=['data_value'])"
      ],
      "metadata": {
        "id": "GKoM-v6WuL7W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "race.isnull().sum()"
      ],
      "metadata": {
        "id": "AAz2XPjAukH0",
        "outputId": "490fe0b1-caa2-40d9-a1fb-a253b81f7046",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                          0\n",
              "locationid                       0\n",
              "sample_size                      0\n",
              "age(years)                   10210\n",
              "education                    10210\n",
              "gender                       10210\n",
              "income                       10210\n",
              "race/ethnicity                   0\n",
              "classid                          0\n",
              "questionid                       0\n",
              "stratificationcategory1          0\n",
              "stratification1                  0\n",
              "stratificationcategoryid1        0\n",
              "stratificationid1                0\n",
              "total                        10210\n",
              "data_value                       0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###6.2.2) Income"
      ],
      "metadata": {
        "id": "diwNRhiNm8mz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "inc.isnull().sum()"
      ],
      "metadata": {
        "id": "SHyj0SQfksOd",
        "outputId": "ca291f02-1d05-461b-b7df-5c6f7afdad3d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                          0\n",
              "locationid                       0\n",
              "sample_size                      0\n",
              "age(years)                   13349\n",
              "education                    13349\n",
              "gender                       13349\n",
              "income                           0\n",
              "race/ethnicity               13349\n",
              "classid                          0\n",
              "questionid                       0\n",
              "stratificationcategory1          0\n",
              "stratification1                  0\n",
              "stratificationcategoryid1        0\n",
              "stratificationid1                0\n",
              "total                        13349\n",
              "data_value                       0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###6.2.3) Age"
      ],
      "metadata": {
        "id": "5t38VhFMnBsP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "age.isnull().sum()"
      ],
      "metadata": {
        "id": "HwkcvlAdm0w6",
        "outputId": "5eb2b7e4-97ff-41b4-a857-2380e36f52cc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                          0\n",
              "locationid                       0\n",
              "sample_size                      0\n",
              "age(years)                       0\n",
              "education                    11438\n",
              "gender                       11438\n",
              "income                       11438\n",
              "race/ethnicity               11438\n",
              "classid                          0\n",
              "questionid                       0\n",
              "stratificationcategory1          0\n",
              "stratification1                  0\n",
              "stratificationcategoryid1        0\n",
              "stratificationid1                0\n",
              "total                        11438\n",
              "data_value                       0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###6.2.4) Education"
      ],
      "metadata": {
        "id": "x_hJPnm7ndmb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "edu.isnull().sum()"
      ],
      "metadata": {
        "id": "Q22Qet6hnY59",
        "outputId": "ce378751-6e79-4197-e95c-8dbe9eec67e7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                         0\n",
              "locationid                      0\n",
              "sample_size                     0\n",
              "age(years)                   7628\n",
              "education                       0\n",
              "gender                       7628\n",
              "income                       7628\n",
              "race/ethnicity               7628\n",
              "classid                         0\n",
              "questionid                      0\n",
              "stratificationcategory1         0\n",
              "stratification1                 0\n",
              "stratificationcategoryid1       0\n",
              "stratificationid1               0\n",
              "total                        7628\n",
              "data_value                      0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 6.2.5) Gender"
      ],
      "metadata": {
        "id": "lEtJ0WjynlPB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gen.isnull().sum()"
      ],
      "metadata": {
        "id": "m3JUwC03nj9_",
        "outputId": "b836114c-2f56-4f00-d6b1-1255f589dddd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                         0\n",
              "locationid                      0\n",
              "sample_size                     0\n",
              "age(years)                   3814\n",
              "education                    3814\n",
              "gender                          0\n",
              "income                       3814\n",
              "race/ethnicity               3814\n",
              "classid                         0\n",
              "questionid                      0\n",
              "stratificationcategory1         0\n",
              "stratification1                 0\n",
              "stratificationcategoryid1       0\n",
              "stratificationid1               0\n",
              "total                        3814\n",
              "data_value                      0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###6.2.6) Total"
      ],
      "metadata": {
        "id": "t8HqGFZFntqV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "total.isnull().sum()"
      ],
      "metadata": {
        "id": "EjFZ0vOcnrIO",
        "outputId": "40dd6ac0-3870-4fef-bb00-c653ab640888",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "yearend                         0\n",
              "locationid                      0\n",
              "sample_size                     0\n",
              "age(years)                   1907\n",
              "education                    1907\n",
              "gender                       1907\n",
              "income                       1907\n",
              "race/ethnicity               1907\n",
              "classid                         0\n",
              "questionid                      0\n",
              "stratificationcategory1         0\n",
              "stratification1                 0\n",
              "stratificationcategoryid1       0\n",
              "stratificationid1               0\n",
              "total                           0\n",
              "data_value                      0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##6.3) Observation\n",
        "\n",
        "1.From the above null values we know that for each category other cateregeries values(column) are null so we will remove those feature for particular dataframe\n",
        "\n",
        "Eg. for race total category other categories(age, education, gender, income, race) are null.\n",
        "\n",
        "2 . All the category values are also given in stratification column so we can also drop this column."
      ],
      "metadata": {
        "id": "bDqu4tVMn50u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "race.head()"
      ],
      "metadata": {
        "id": "LqX3X0AX4TwO",
        "outputId": "3216f4c9-7f37-4203-b8c1-abeb5f155563",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    yearend  locationid  sample_size age(years) education gender income  \\\n",
              "20     2011           1       5042.0        NaN       NaN    NaN    NaN   \n",
              "21     2011           1       1878.0        NaN       NaN    NaN    NaN   \n",
              "22     2011           1        115.0        NaN       NaN    NaN    NaN   \n",
              "25     2011           1         61.0        NaN       NaN    NaN    NaN   \n",
              "26     2011           1         58.0        NaN       NaN    NaN    NaN   \n",
              "\n",
              "                   race/ethnicity classid questionid stratificationcategory1  \\\n",
              "20             Non-Hispanic White     OWS       Q036          Race/Ethnicity   \n",
              "21             Non-Hispanic Black     OWS       Q036          Race/Ethnicity   \n",
              "22                       Hispanic     OWS       Q036          Race/Ethnicity   \n",
              "25  American Indian/Alaska Native     OWS       Q036          Race/Ethnicity   \n",
              "26                2 or more races     OWS       Q036          Race/Ethnicity   \n",
              "\n",
              "                  stratification1 stratificationcategoryid1 stratificationid1  \\\n",
              "20             Non-Hispanic White                      RACE           RACEWHT   \n",
              "21             Non-Hispanic Black                      RACE           RACEBLK   \n",
              "22                       Hispanic                      RACE           RACEHIS   \n",
              "25  American Indian/Alaska Native                      RACE           RACENAA   \n",
              "26                2 or more races                      RACE         RACE2PLUS   \n",
              "\n",
              "   total  data_value  \n",
              "20   NaN        29.8  \n",
              "21   NaN        40.1  \n",
              "22   NaN        28.6  \n",
              "25   NaN        32.9  \n",
              "26   NaN        27.8  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-df9e7623-9ede-4035-8854-50fb523c05fc\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>yearend</th>\n",
              "      <th>locationid</th>\n",
              "      <th>sample_size</th>\n",
              "      <th>age(years)</th>\n",
              "      <th>education</th>\n",
              "      <th>gender</th>\n",
              "      <th>income</th>\n",
              "      <th>race/ethnicity</th>\n",
              "      <th>classid</th>\n",
              "      <th>questionid</th>\n",
              "      <th>stratificationcategory1</th>\n",
              "      <th>stratification1</th>\n",
              "      <th>stratificationcategoryid1</th>\n",
              "      <th>stratificationid1</th>\n",
              "      <th>total</th>\n",
              "      <th>data_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>5042.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Non-Hispanic White</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Non-Hispanic White</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEWHT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>29.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Non-Hispanic Black</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Non-Hispanic Black</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEBLK</td>\n",
              "      <td>NaN</td>\n",
              "      <td>40.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>115.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Hispanic</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Hispanic</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEHIS</td>\n",
              "      <td>NaN</td>\n",
              "      <td>28.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>61.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>American Indian/Alaska Native</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>American Indian/Alaska Native</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACENAA</td>\n",
              "      <td>NaN</td>\n",
              "      <td>32.9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>58.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2 or more races</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>2 or more races</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACE2PLUS</td>\n",
              "      <td>NaN</td>\n",
              "      <td>27.8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df9e7623-9ede-4035-8854-50fb523c05fc')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-df9e7623-9ede-4035-8854-50fb523c05fc button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-df9e7623-9ede-4035-8854-50fb523c05fc');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c7fc1369-7f51-4f6a-8a41-4d4760c8838c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c7fc1369-7f51-4f6a-8a41-4d4760c8838c')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c7fc1369-7f51-4f6a-8a41-4d4760c8838c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##6.4) Dropping Missing values\n",
        "(According to the above observations)"
      ],
      "metadata": {
        "id": "TXfuA5Sot4mI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "columns_to_drop = ['race/ethnicity','income','age(years)','gender','education','total']\n",
        "race.drop(columns=columns_to_drop, inplace=True)\n",
        "columns_to_drop = ['income','race/ethnicity','age(years)','gender','education','total']\n",
        "inc.drop(columns=columns_to_drop, inplace=True)\n",
        "columns_to_drop = ['race/ethnicity','income','age(years)','gender','education','total']\n",
        "age.drop(columns=columns_to_drop, inplace=True)\n",
        "columns_to_drop = ['race/ethnicity','income','age(years)','gender','education','total']\n",
        "edu.drop(columns=columns_to_drop, inplace=True)\n",
        "columns_to_drop = ['race/ethnicity','income','age(years)','gender','education','total']\n",
        "gen.drop(columns=columns_to_drop, inplace=True)\n",
        "columns_to_drop = ['race/ethnicity','income','age(years)','gender','education','total']\n",
        "total.drop(columns=columns_to_drop, inplace=True)"
      ],
      "metadata": {
        "id": "t5K5hmuDr76L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataframes = [race,inc,age,edu,gen,total]"
      ],
      "metadata": {
        "id": "LbicKtJKvs9g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i, df in enumerate(dataframes, start=1):\n",
        "    null_count = df.isnull().sum().sum()\n",
        "    print(f\"Null values in df{i}: {null_count}\")"
      ],
      "metadata": {
        "id": "S1dDo_NWtSHV",
        "outputId": "2bc9d0e2-a27a-41c5-880f-f37d4d1f87f2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Null values in df1: 0\n",
            "Null values in df2: 0\n",
            "Null values in df3: 0\n",
            "Null values in df4: 0\n",
            "Null values in df5: 0\n",
            "Null values in df6: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Now we don't have any null values in our datasets.**"
      ],
      "metadata": {
        "id": "mXeShpLs5mOa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#7) Preprocessing"
      ],
      "metadata": {
        "id": "8AI2CjGW2YQZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 7.1) Removing outliers for each dataframes"
      ],
      "metadata": {
        "id": "-8E9afv54y6-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#creating function to remove outliers by using interquartile function\n",
        "def remove_outliers(df):\n",
        "    for col in df.select_dtypes(include=np.number).columns:\n",
        "        q1 = df[col].quantile(0.25)\n",
        "        q3 = df[col].quantile(0.75)\n",
        "        iqr = q3 - q1\n",
        "        lower_bound = q1 - (1.5 * iqr)\n",
        "        upper_bound = q3 + (1.5 * iqr)\n",
        "        df.drop(df[(df[col] < lower_bound) | (df[col] > upper_bound)].index, inplace=True)"
      ],
      "metadata": {
        "id": "32fqQLw24yI2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in dataframes:\n",
        "  remove_outliers(i)"
      ],
      "metadata": {
        "id": "hc6zIAMv5nhB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 7.2) Size of each category after removing outliers"
      ],
      "metadata": {
        "id": "uA6x6UPA6_b_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Shape of category Race/ethinicity is {race.shape}')\n",
        "print(f'Shape of category Income range is {inc.shape}')\n",
        "print(f'Shape of category Age range of sample is {age.shape}')\n",
        "print(f'Shape of category Education is {edu.shape}')\n",
        "print(f'Shape of category Gender is {gen.shape}')\n",
        "print(f'Shape of category total is {total.shape}')"
      ],
      "metadata": {
        "id": "kiQH-_Rz7i9C",
        "outputId": "8ebf9e92-bf9f-4d9b-fc23-f651bb05d255",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of category Race/ethinicity is (8253, 10)\n",
            "Shape of category Income range is (12102, 10)\n",
            "Shape of category Age range of sample is (10701, 10)\n",
            "Shape of category Education is (7008, 10)\n",
            "Shape of category Gender is (3525, 10)\n",
            "Shape of category total is (1732, 10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 8) Feature Selection"
      ],
      "metadata": {
        "id": "ujUJdemv8O9K"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 8.1) Dropping correlated feature"
      ],
      "metadata": {
        "id": "ZFzoyIo_8_1H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "race.head(3)"
      ],
      "metadata": {
        "id": "h7gCYjtF5jwb",
        "outputId": "c504cfb8-b8f6-4821-de2d-b4692bfad5e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    yearend  locationid  sample_size classid questionid  \\\n",
              "21     2011           1       1878.0     OWS       Q036   \n",
              "22     2011           1        115.0     OWS       Q036   \n",
              "25     2011           1         61.0     OWS       Q036   \n",
              "\n",
              "   stratificationcategory1                stratification1  \\\n",
              "21          Race/Ethnicity             Non-Hispanic Black   \n",
              "22          Race/Ethnicity                       Hispanic   \n",
              "25          Race/Ethnicity  American Indian/Alaska Native   \n",
              "\n",
              "   stratificationcategoryid1 stratificationid1  data_value  \n",
              "21                      RACE           RACEBLK        40.1  \n",
              "22                      RACE           RACEHIS        28.6  \n",
              "25                      RACE           RACENAA        32.9  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a771b526-f6b4-4f22-b8f3-04698bc9e29a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>yearend</th>\n",
              "      <th>locationid</th>\n",
              "      <th>sample_size</th>\n",
              "      <th>classid</th>\n",
              "      <th>questionid</th>\n",
              "      <th>stratificationcategory1</th>\n",
              "      <th>stratification1</th>\n",
              "      <th>stratificationcategoryid1</th>\n",
              "      <th>stratificationid1</th>\n",
              "      <th>data_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Non-Hispanic Black</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEBLK</td>\n",
              "      <td>40.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>115.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>Hispanic</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACEHIS</td>\n",
              "      <td>28.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>61.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>Race/Ethnicity</td>\n",
              "      <td>American Indian/Alaska Native</td>\n",
              "      <td>RACE</td>\n",
              "      <td>RACENAA</td>\n",
              "      <td>32.9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a771b526-f6b4-4f22-b8f3-04698bc9e29a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a771b526-f6b4-4f22-b8f3-04698bc9e29a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a771b526-f6b4-4f22-b8f3-04698bc9e29a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7568afa2-2f97-4712-af73-29b9cd42279b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7568afa2-2f97-4712-af73-29b9cd42279b')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7568afa2-2f97-4712-af73-29b9cd42279b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "columns_to_drop = ['stratificationcategory1','stratification1','stratificationcategoryid1']\n",
        "race = race.drop(columns=columns_to_drop)\n",
        "inc = inc.drop(columns=columns_to_drop)\n",
        "age = age.drop(columns=columns_to_drop)\n",
        "edu = edu.drop(columns=columns_to_drop)\n",
        "gen = gen.drop(columns=columns_to_drop)\n",
        "total = total.drop(columns=columns_to_drop)"
      ],
      "metadata": {
        "id": "YQEVoR0r8M8Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "race"
      ],
      "metadata": {
        "id": "JgkRLRcH96VX",
        "outputId": "d68ce7ea-8a6f-424d-b3ac-2a2e87462ea8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       yearend  locationid  sample_size classid questionid stratificationid1  \\\n",
              "21        2011           1       1878.0     OWS       Q036           RACEBLK   \n",
              "22        2011           1        115.0     OWS       Q036           RACEHIS   \n",
              "25        2011           1         61.0     OWS       Q036           RACENAA   \n",
              "26        2011           1         58.0     OWS       Q036         RACE2PLUS   \n",
              "49        2011           1       1878.0     OWS       Q037           RACEBLK   \n",
              "...        ...         ...          ...     ...        ...               ...   \n",
              "53334     2016          66        511.0      PA       Q047           RACEHPI   \n",
              "53336     2016          66        203.0      PA       Q047         RACE2PLUS   \n",
              "53384     2016          78        154.0      PA       Q047           RACEWHT   \n",
              "53385     2016          78        820.0      PA       Q047           RACEBLK   \n",
              "53386     2016          78        178.0      PA       Q047           RACEHIS   \n",
              "\n",
              "       data_value  \n",
              "21           40.1  \n",
              "22           28.6  \n",
              "25           32.9  \n",
              "26           27.8  \n",
              "49           30.1  \n",
              "...           ...  \n",
              "53334        34.6  \n",
              "53336        21.5  \n",
              "53384        18.3  \n",
              "53385        24.1  \n",
              "53386        30.3  \n",
              "\n",
              "[8253 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6ff079be-859f-40ae-992e-aacac7d466aa\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>yearend</th>\n",
              "      <th>locationid</th>\n",
              "      <th>sample_size</th>\n",
              "      <th>classid</th>\n",
              "      <th>questionid</th>\n",
              "      <th>stratificationid1</th>\n",
              "      <th>data_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>RACEBLK</td>\n",
              "      <td>40.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>115.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>RACEHIS</td>\n",
              "      <td>28.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>61.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>RACENAA</td>\n",
              "      <td>32.9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>58.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>RACE2PLUS</td>\n",
              "      <td>27.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q037</td>\n",
              "      <td>RACEBLK</td>\n",
              "      <td>30.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53334</th>\n",
              "      <td>2016</td>\n",
              "      <td>66</td>\n",
              "      <td>511.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>RACEHPI</td>\n",
              "      <td>34.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53336</th>\n",
              "      <td>2016</td>\n",
              "      <td>66</td>\n",
              "      <td>203.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>RACE2PLUS</td>\n",
              "      <td>21.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53384</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>154.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>RACEWHT</td>\n",
              "      <td>18.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53385</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>820.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>RACEBLK</td>\n",
              "      <td>24.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53386</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>178.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>RACEHIS</td>\n",
              "      <td>30.3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8253 rows × 7 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6ff079be-859f-40ae-992e-aacac7d466aa')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-6ff079be-859f-40ae-992e-aacac7d466aa button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-6ff079be-859f-40ae-992e-aacac7d466aa');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7ae92bdb-781a-4903-9ffb-093428860322\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7ae92bdb-781a-4903-9ffb-093428860322')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7ae92bdb-781a-4903-9ffb-093428860322 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "1.stratificationcategory1, stratificationcategoryid1--> all the data is for income so we removed.\n",
        "\n",
        "2.stratification1 ---> It is a income range and we already have stratificationid1 as abbrevation for this."
      ],
      "metadata": {
        "id": "_ltZ__kY9c6_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 8.2) Encoding"
      ],
      "metadata": {
        "id": "VeXwoJKZ-6SS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(race.dtypes)"
      ],
      "metadata": {
        "id": "82THjZuR_B3R",
        "outputId": "0f6519c1-2d17-415a-f84d-04fb68a521b4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "yearend                int64\n",
            "locationid             int64\n",
            "sample_size          float64\n",
            "classid               object\n",
            "questionid            object\n",
            "stratificationid1     object\n",
            "data_value           float64\n",
            "dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###8.2.1) Label Encoding (for ordinal categorical data):\n",
        " Using label encoding when the categorical data have an inherent order or hierarchy."
      ],
      "metadata": {
        "id": "uRVcFkIC_Ng5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import LabelEncoder,StandardScaler"
      ],
      "metadata": {
        "id": "lbw4kCZTCRK0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "le = LabelEncoder()\n",
        "# race['stratificationid1'] = le.fit_transform(race[\"stratificationid1\"])\n",
        "inc['stratificationid1'] = le.fit_transform(inc[\"stratificationid1\"])\n",
        "age['stratificationid1'] = le.fit_transform(age[\"stratificationid1\"])\n",
        "gen['stratificationid1'] = le.fit_transform(gen[\"stratificationid1\"])\n",
        "edu['stratificationid1'] = le.fit_transform(edu[\"stratificationid1\"])\n",
        "total['stratificationid1'] = le.fit_transform(total[\"classid\"])"
      ],
      "metadata": {
        "id": "GNYL_wemFFCv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataframe = [race,inc,age,gen,edu,total]"
      ],
      "metadata": {
        "id": "ZtsSdbQyGt1A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "race"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "sI3M2CXaHNgU",
        "outputId": "8b2b47e0-9fa5-44d9-b787-69dcdc1f6801"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       yearend  locationid  sample_size classid questionid  stratificationid1  \\\n",
              "21        2011           1       1878.0     OWS       Q036                  2   \n",
              "22        2011           1        115.0     OWS       Q036                  3   \n",
              "25        2011           1         61.0     OWS       Q036                  5   \n",
              "26        2011           1         58.0     OWS       Q036                  0   \n",
              "49        2011           1       1878.0     OWS       Q037                  2   \n",
              "...        ...         ...          ...     ...        ...                ...   \n",
              "53334     2016          66        511.0      PA       Q047                  4   \n",
              "53336     2016          66        203.0      PA       Q047                  0   \n",
              "53384     2016          78        154.0      PA       Q047                  7   \n",
              "53385     2016          78        820.0      PA       Q047                  2   \n",
              "53386     2016          78        178.0      PA       Q047                  3   \n",
              "\n",
              "       data_value  \n",
              "21           40.1  \n",
              "22           28.6  \n",
              "25           32.9  \n",
              "26           27.8  \n",
              "49           30.1  \n",
              "...           ...  \n",
              "53334        34.6  \n",
              "53336        21.5  \n",
              "53384        18.3  \n",
              "53385        24.1  \n",
              "53386        30.3  \n",
              "\n",
              "[8253 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-517652d0-b629-460a-bcd3-34baa8411251\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>yearend</th>\n",
              "      <th>locationid</th>\n",
              "      <th>sample_size</th>\n",
              "      <th>classid</th>\n",
              "      <th>questionid</th>\n",
              "      <th>stratificationid1</th>\n",
              "      <th>data_value</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>2</td>\n",
              "      <td>40.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>115.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>3</td>\n",
              "      <td>28.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>61.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>5</td>\n",
              "      <td>32.9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>58.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q036</td>\n",
              "      <td>0</td>\n",
              "      <td>27.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>OWS</td>\n",
              "      <td>Q037</td>\n",
              "      <td>2</td>\n",
              "      <td>30.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53334</th>\n",
              "      <td>2016</td>\n",
              "      <td>66</td>\n",
              "      <td>511.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>4</td>\n",
              "      <td>34.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53336</th>\n",
              "      <td>2016</td>\n",
              "      <td>66</td>\n",
              "      <td>203.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>0</td>\n",
              "      <td>21.5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53384</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>154.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>7</td>\n",
              "      <td>18.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53385</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>820.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>2</td>\n",
              "      <td>24.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53386</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>178.0</td>\n",
              "      <td>PA</td>\n",
              "      <td>Q047</td>\n",
              "      <td>3</td>\n",
              "      <td>30.3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8253 rows × 7 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-517652d0-b629-460a-bcd3-34baa8411251')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-517652d0-b629-460a-bcd3-34baa8411251 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-517652d0-b629-460a-bcd3-34baa8411251');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-413c7bf3-b7e9-4c24-ac9c-453dee3472c0\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-413c7bf3-b7e9-4c24-ac9c-453dee3472c0')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-413c7bf3-b7e9-4c24-ac9c-453dee3472c0 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 8.2.2) One-Hot Encoding (for nominal categorical data):\n",
        "Using one-hot encoding when there is no specific order or hierarchy among the categories."
      ],
      "metadata": {
        "id": "mhrYMT2kHrIw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "race = pd.get_dummies(race , columns=[\"classid\",\"questionid\",'stratificationid1'])\n",
        "inc = pd.get_dummies(inc, columns=[\"classid\",\"questionid\"])\n",
        "age = pd.get_dummies(age , columns=[\"classid\",\"questionid\"])\n",
        "gen = pd.get_dummies(gen , columns=[\"classid\",\"questionid\"])\n",
        "edu = pd.get_dummies(edu , columns=[\"classid\",\"questionid\"])\n",
        "total = pd.get_dummies(total , columns=[\"classid\",\"questionid\"])\n"
      ],
      "metadata": {
        "id": "csOmnH36W2Q9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 8.3) Saving files into csv\n",
        "\n",
        "Saving each category dataframes into csv file for further process."
      ],
      "metadata": {
        "id": "Yhu2SZxfxt5u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataframes = [race, inc, age, edu, gen, total]\n",
        "categories = ['race', 'income', 'age', 'education', 'gender', 'total']"
      ],
      "metadata": {
        "id": "1TosSfEzvOud"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i, df in enumerate(dataframes):\n",
        "    filename = f'{categories[i]}.csv'\n",
        "    df.to_csv(filename, index=False)"
      ],
      "metadata": {
        "id": "Pc38G1c6xonD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataframes[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444
        },
        "id": "s1-A8J6aQM-e",
        "outputId": "d65d65dd-92c2-469e-c8b8-1098d9e0b7a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       yearend  locationid  sample_size  data_value  classid_FV  classid_OWS  \\\n",
              "21        2011           1       1878.0        40.1           0            1   \n",
              "22        2011           1        115.0        28.6           0            1   \n",
              "25        2011           1         61.0        32.9           0            1   \n",
              "26        2011           1         58.0        27.8           0            1   \n",
              "49        2011           1       1878.0        30.1           0            1   \n",
              "...        ...         ...          ...         ...         ...          ...   \n",
              "53334     2016          66        511.0        34.6           0            0   \n",
              "53336     2016          66        203.0        21.5           0            0   \n",
              "53384     2016          78        154.0        18.3           0            0   \n",
              "53385     2016          78        820.0        24.1           0            0   \n",
              "53386     2016          78        178.0        30.3           0            0   \n",
              "\n",
              "       classid_PA  questionid_Q018  questionid_Q019  questionid_Q036  ...  \\\n",
              "21              0                0                0                1  ...   \n",
              "22              0                0                0                1  ...   \n",
              "25              0                0                0                1  ...   \n",
              "26              0                0                0                1  ...   \n",
              "49              0                0                0                0  ...   \n",
              "...           ...              ...              ...              ...  ...   \n",
              "53334           1                0                0                0  ...   \n",
              "53336           1                0                0                0  ...   \n",
              "53384           1                0                0                0  ...   \n",
              "53385           1                0                0                0  ...   \n",
              "53386           1                0                0                0  ...   \n",
              "\n",
              "       questionid_Q046  questionid_Q047  stratificationid1_0  \\\n",
              "21                   0                0                    0   \n",
              "22                   0                0                    0   \n",
              "25                   0                0                    0   \n",
              "26                   0                0                    1   \n",
              "49                   0                0                    0   \n",
              "...                ...              ...                  ...   \n",
              "53334                0                1                    0   \n",
              "53336                0                1                    1   \n",
              "53384                0                1                    0   \n",
              "53385                0                1                    0   \n",
              "53386                0                1                    0   \n",
              "\n",
              "       stratificationid1_1  stratificationid1_2  stratificationid1_3  \\\n",
              "21                       0                    1                    0   \n",
              "22                       0                    0                    1   \n",
              "25                       0                    0                    0   \n",
              "26                       0                    0                    0   \n",
              "49                       0                    1                    0   \n",
              "...                    ...                  ...                  ...   \n",
              "53334                    0                    0                    0   \n",
              "53336                    0                    0                    0   \n",
              "53384                    0                    0                    0   \n",
              "53385                    0                    1                    0   \n",
              "53386                    0                    0                    1   \n",
              "\n",
              "       stratificationid1_4  stratificationid1_5  stratificationid1_6  \\\n",
              "21                       0                    0                    0   \n",
              "22                       0                    0                    0   \n",
              "25                       0                    1                    0   \n",
              "26                       0                    0                    0   \n",
              "49                       0                    0                    0   \n",
              "...                    ...                  ...                  ...   \n",
              "53334                    1                    0                    0   \n",
              "53336                    0                    0                    0   \n",
              "53384                    0                    0                    0   \n",
              "53385                    0                    0                    0   \n",
              "53386                    0                    0                    0   \n",
              "\n",
              "       stratificationid1_7  \n",
              "21                       0  \n",
              "22                       0  \n",
              "25                       0  \n",
              "26                       0  \n",
              "49                       0  \n",
              "...                    ...  \n",
              "53334                    0  \n",
              "53336                    0  \n",
              "53384                    1  \n",
              "53385                    0  \n",
              "53386                    0  \n",
              "\n",
              "[8253 rows x 24 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-31b67ac4-091e-4a8b-8638-44b5284b7a3d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>yearend</th>\n",
              "      <th>locationid</th>\n",
              "      <th>sample_size</th>\n",
              "      <th>data_value</th>\n",
              "      <th>classid_FV</th>\n",
              "      <th>classid_OWS</th>\n",
              "      <th>classid_PA</th>\n",
              "      <th>questionid_Q018</th>\n",
              "      <th>questionid_Q019</th>\n",
              "      <th>questionid_Q036</th>\n",
              "      <th>...</th>\n",
              "      <th>questionid_Q046</th>\n",
              "      <th>questionid_Q047</th>\n",
              "      <th>stratificationid1_0</th>\n",
              "      <th>stratificationid1_1</th>\n",
              "      <th>stratificationid1_2</th>\n",
              "      <th>stratificationid1_3</th>\n",
              "      <th>stratificationid1_4</th>\n",
              "      <th>stratificationid1_5</th>\n",
              "      <th>stratificationid1_6</th>\n",
              "      <th>stratificationid1_7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>40.1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>115.0</td>\n",
              "      <td>28.6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>61.0</td>\n",
              "      <td>32.9</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>58.0</td>\n",
              "      <td>27.8</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>2011</td>\n",
              "      <td>1</td>\n",
              "      <td>1878.0</td>\n",
              "      <td>30.1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53334</th>\n",
              "      <td>2016</td>\n",
              "      <td>66</td>\n",
              "      <td>511.0</td>\n",
              "      <td>34.6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53336</th>\n",
              "      <td>2016</td>\n",
              "      <td>66</td>\n",
              "      <td>203.0</td>\n",
              "      <td>21.5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53384</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>154.0</td>\n",
              "      <td>18.3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53385</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>820.0</td>\n",
              "      <td>24.1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53386</th>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>178.0</td>\n",
              "      <td>30.3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8253 rows × 24 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-31b67ac4-091e-4a8b-8638-44b5284b7a3d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-31b67ac4-091e-4a8b-8638-44b5284b7a3d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-31b67ac4-091e-4a8b-8638-44b5284b7a3d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-84ab050f-e027-4ec8-891d-c32f2b19a374\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-84ab050f-e027-4ec8-891d-c32f2b19a374')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-84ab050f-e027-4ec8-891d-c32f2b19a374 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gpVnsRytQPuy"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}