00_purple_air_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code to read purple air sensor network data, and export it as a csv file.\n",
    "The latitude and longitude are bounded to approximately contain the United States, but may include some locations in Canada / Mexico."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Child 566 lists parent 565, but parent does not exist!\n",
      "Child 2861 lists parent 2860, but parent does not exist!\n",
      "Initialized 22,385 sensors!\n",
      "15898\n"
     ]
    }
   ],
   "source": [
    "# Code from https://github.com/ReagentX/purple_air_api#listing-all-useful-sensors\n",
    "from purpleair.network import SensorList\n",
    "p = SensorList()  # Initialized 23,145 sensors!\n",
    "useful = [s for s in p.all_sensors if s.is_useful()]  # List of sensors with no defects\n",
    "print(len(useful))  # 17,426"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = p.to_dataframe('useful', channel = 'parent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['parent', 'lat', 'lon', 'name', 'location_type', 'pm_2.5', 'temp_f',\n",
       "       'temp_c', 'humidity', 'pressure', 'p_0_3_um', 'p_0_5_um', 'p_1_0_um',\n",
       "       'p_2_5_um', 'p_5_0_um', 'p_10_0_um', 'pm1_0_cf_1', 'pm2_5_cf_1',\n",
       "       'pm10_0_cf_1', 'pm1_0_atm', 'pm2_5_atm', 'pm10_0_atm', 'last_seen',\n",
       "       'model', 'adc', 'rssi', 'hidden', 'flagged', 'downgraded', 'age',\n",
       "       'brightness', 'hardware', 'version', 'last_update_check', 'created',\n",
       "       'uptime', 'is_owner', '10min_avg', '30min_avg', '1hour_avg',\n",
       "       '6hour_avg', '1day_avg', '1week_avg'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>parent</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>name</th>\n",
       "      <th>location_type</th>\n",
       "      <th>pm_2.5</th>\n",
       "      <th>temp_f</th>\n",
       "      <th>temp_c</th>\n",
       "      <th>humidity</th>\n",
       "      <th>pressure</th>\n",
       "      <th>...</th>\n",
       "      <th>last_update_check</th>\n",
       "      <th>created</th>\n",
       "      <th>uptime</th>\n",
       "      <th>is_owner</th>\n",
       "      <th>10min_avg</th>\n",
       "      <th>30min_avg</th>\n",
       "      <th>1hour_avg</th>\n",
       "      <th>6hour_avg</th>\n",
       "      <th>1day_avg</th>\n",
       "      <th>1week_avg</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25999</th>\n",
       "      <td>None</td>\n",
       "      <td>30.053808</td>\n",
       "      <td>-95.494643</td>\n",
       "      <td>Villages of Bridgestone AQI</td>\n",
       "      <td>outside</td>\n",
       "      <td>1.00</td>\n",
       "      <td>50.0</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>62.0</td>\n",
       "      <td>1022.37</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>2.47</td>\n",
       "      <td>4.19</td>\n",
       "      <td>8.10</td>\n",
       "      <td>19.66</td>\n",
       "      <td>17.15</td>\n",
       "      <td>15.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49409</th>\n",
       "      <td>None</td>\n",
       "      <td>18.759182</td>\n",
       "      <td>99.017172</td>\n",
       "      <td>\"First's Place\"</td>\n",
       "      <td>outside</td>\n",
       "      <td>42.26</td>\n",
       "      <td>73.0</td>\n",
       "      <td>22.777778</td>\n",
       "      <td>43.0</td>\n",
       "      <td>974.24</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>42.61</td>\n",
       "      <td>43.03</td>\n",
       "      <td>42.74</td>\n",
       "      <td>39.90</td>\n",
       "      <td>35.09</td>\n",
       "      <td>31.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42073</th>\n",
       "      <td>None</td>\n",
       "      <td>47.185173</td>\n",
       "      <td>-122.176855</td>\n",
       "      <td>#1</td>\n",
       "      <td>outside</td>\n",
       "      <td>7.46</td>\n",
       "      <td>57.0</td>\n",
       "      <td>13.888889</td>\n",
       "      <td>48.0</td>\n",
       "      <td>1003.78</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>8.28</td>\n",
       "      <td>8.76</td>\n",
       "      <td>9.58</td>\n",
       "      <td>13.30</td>\n",
       "      <td>13.25</td>\n",
       "      <td>11.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53069</th>\n",
       "      <td>None</td>\n",
       "      <td>47.190197</td>\n",
       "      <td>-122.177992</td>\n",
       "      <td>#2</td>\n",
       "      <td>outside</td>\n",
       "      <td>7.68</td>\n",
       "      <td>71.0</td>\n",
       "      <td>21.666667</td>\n",
       "      <td>33.0</td>\n",
       "      <td>1003.66</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>8.07</td>\n",
       "      <td>8.43</td>\n",
       "      <td>9.45</td>\n",
       "      <td>14.39</td>\n",
       "      <td>14.58</td>\n",
       "      <td>13.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94891</th>\n",
       "      <td>None</td>\n",
       "      <td>40.160043</td>\n",
       "      <td>-79.272304</td>\n",
       "      <td>#9</td>\n",
       "      <td>outside</td>\n",
       "      <td>9.28</td>\n",
       "      <td>42.0</td>\n",
       "      <td>5.555556</td>\n",
       "      <td>50.0</td>\n",
       "      <td>971.35</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>9.07</td>\n",
       "      <td>7.58</td>\n",
       "      <td>6.71</td>\n",
       "      <td>6.30</td>\n",
       "      <td>7.76</td>\n",
       "      <td>12.12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      parent        lat         lon                          name  \\\n",
       "id                                                                  \n",
       "25999   None  30.053808  -95.494643   Villages of Bridgestone AQI   \n",
       "49409   None  18.759182   99.017172               \"First's Place\"   \n",
       "42073   None  47.185173 -122.176855                            #1   \n",
       "53069   None  47.190197 -122.177992                            #2   \n",
       "94891   None  40.160043  -79.272304                            #9   \n",
       "\n",
       "      location_type  pm_2.5  temp_f     temp_c  humidity  pressure  ...  \\\n",
       "id                                                                  ...   \n",
       "25999       outside    1.00    50.0  10.000000      62.0   1022.37  ...   \n",
       "49409       outside   42.26    73.0  22.777778      43.0    974.24  ...   \n",
       "42073       outside    7.46    57.0  13.888889      48.0   1003.78  ...   \n",
       "53069       outside    7.68    71.0  21.666667      33.0   1003.66  ...   \n",
       "94891       outside    9.28    42.0   5.555556      50.0    971.35  ...   \n",
       "\n",
       "      last_update_check created uptime is_owner 10min_avg 30min_avg 1hour_avg  \\\n",
       "id                                                                              \n",
       "25999              None    None   None    False      2.47      4.19      8.10   \n",
       "49409              None    None   None    False     42.61     43.03     42.74   \n",
       "42073              None    None   None    False      8.28      8.76      9.58   \n",
       "53069              None    None   None    False      8.07      8.43      9.45   \n",
       "94891              None    None   None    False      9.07      7.58      6.71   \n",
       "\n",
       "      6hour_avg 1day_avg 1week_avg  \n",
       "id                                  \n",
       "25999     19.66    17.15     15.22  \n",
       "49409     39.90    35.09     31.60  \n",
       "42073     13.30    13.25     11.98  \n",
       "53069     14.39    14.58     13.23  \n",
       "94891      6.30     7.76     12.12  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 15898 entries, 25999 to 108592\n",
      "Data columns (total 43 columns):\n",
      " #   Column             Non-Null Count  Dtype         \n",
      "---  ------             --------------  -----         \n",
      " 0   parent             0 non-null      object        \n",
      " 1   lat                15898 non-null  float64       \n",
      " 2   lon                15898 non-null  float64       \n",
      " 3   name               15898 non-null  object        \n",
      " 4   location_type      15862 non-null  object        \n",
      " 5   pm_2.5             15898 non-null  float64       \n",
      " 6   temp_f             15898 non-null  float64       \n",
      " 7   temp_c             15898 non-null  float64       \n",
      " 8   humidity           15898 non-null  float64       \n",
      " 9   pressure           15898 non-null  float64       \n",
      " 10  p_0_3_um           0 non-null      object        \n",
      " 11  p_0_5_um           0 non-null      object        \n",
      " 12  p_1_0_um           0 non-null      object        \n",
      " 13  p_2_5_um           0 non-null      object        \n",
      " 14  p_5_0_um           0 non-null      object        \n",
      " 15  p_10_0_um          0 non-null      object        \n",
      " 16  pm1_0_cf_1         0 non-null      object        \n",
      " 17  pm2_5_cf_1         0 non-null      object        \n",
      " 18  pm10_0_cf_1        0 non-null      object        \n",
      " 19  pm1_0_atm          0 non-null      object        \n",
      " 20  pm2_5_atm          0 non-null      object        \n",
      " 21  pm10_0_atm         0 non-null      object        \n",
      " 22  last_seen          15898 non-null  datetime64[ns]\n",
      " 23  model              15898 non-null  object        \n",
      " 24  adc                0 non-null      object        \n",
      " 25  rssi               0 non-null      object        \n",
      " 26  hidden             15898 non-null  bool          \n",
      " 27  flagged            15898 non-null  bool          \n",
      " 28  downgraded         15898 non-null  bool          \n",
      " 29  age                15898 non-null  int64         \n",
      " 30  brightness         0 non-null      object        \n",
      " 31  hardware           0 non-null      object        \n",
      " 32  version            0 non-null      object        \n",
      " 33  last_update_check  0 non-null      object        \n",
      " 34  created            0 non-null      object        \n",
      " 35  uptime             0 non-null      object        \n",
      " 36  is_owner           15898 non-null  bool          \n",
      " 37  10min_avg          15898 non-null  float64       \n",
      " 38  30min_avg          15898 non-null  float64       \n",
      " 39  1hour_avg          15898 non-null  float64       \n",
      " 40  6hour_avg          15898 non-null  float64       \n",
      " 41  1day_avg           15898 non-null  float64       \n",
      " 42  1week_avg          15898 non-null  float64       \n",
      "dtypes: bool(4), datetime64[ns](1), float64(13), int64(1), object(24)\n",
      "memory usage: 4.9+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(axis = 1, thresh = 5000, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_us = df.query('(lat > 20) and (lat < 50) and (lon > -125) and (lon < -65)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14163, 22)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_us.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lat               0\n",
       "lon               0\n",
       "name              0\n",
       "location_type    28\n",
       "pm_2.5            0\n",
       "temp_f            0\n",
       "temp_c            0\n",
       "humidity          0\n",
       "pressure          0\n",
       "last_seen         0\n",
       "model             0\n",
       "hidden            0\n",
       "flagged           0\n",
       "downgraded        0\n",
       "age               0\n",
       "is_owner          0\n",
       "10min_avg         0\n",
       "30min_avg         0\n",
       "1hour_avg         0\n",
       "6hour_avg         0\n",
       "1day_avg          0\n",
       "1week_avg         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_us.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4918082"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_us.memory_usage(deep=True).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_us.to_csv('data/original_data/purpleair_original.csv')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d271bf565492fcdfddddd56664cf342af32c3e3c5a688596f852c6a7c40d32c4"
  },
  "kernelspec": {
   "display_name": "Python 3.10.0 ('project4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}