examples/notebook.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.6 64-bit"
  },
  "interpreter": {
   "hash": "4cd7ab41f5fca4b9b44701077e38c5ffd31fe66a6cab21e0214b68d958d0e462"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Examples\n",
    "\n",
    "1. Simple HTML table\n",
    "2. Custom table selector\n",
    "3. Apache server directory\n",
    "\n",
    "## Read a simple HTML table\n",
    "\n",
    "Read data from `<table>` element(s), from a local file path or web URL:"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "from intake_html_table import HtmlTableSource"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "src = HtmlTableSource(\"table.html\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "source": [
    "df = src.read()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "df.head()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "        Column1       Column2       Column3\n",
       "0  Row1 Column1  Row1 Column2  Row1 Column3\n",
       "1  Row2 Column1  Row2 Column2  Row2 Column3\n",
       "2  Row3 Column1  Row3 Column2  Row3 Column3"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column1</th>\n",
       "      <th>Column2</th>\n",
       "      <th>Column3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Row1 Column1</td>\n",
       "      <td>Row1 Column2</td>\n",
       "      <td>Row1 Column3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Row2 Column1</td>\n",
       "      <td>Row2 Column2</td>\n",
       "      <td>Row2 Column3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Row3 Column1</td>\n",
       "      <td>Row3 Column2</td>\n",
       "      <td>Row3 Column3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Use `pandas.read_html` kwargs\n",
    "\n",
    "Optionally pass through kwargs for more control over reading the source table(s):"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "source": [
    "src = HtmlTableSource(\"document.html\", attrs={\"id\": \"data\"})"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "df = src.read()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "source": [
    "df.head()"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   Column1 Column2  Column3           Column4\n",
       "0       11    R1C2    False  2021-04-03 12:23\n",
       "1       22    R2C2     True  2021-03-30 08:39\n",
       "2       33    R3C2    False  2021-04-02 18:17\n",
       "3       44    R4C2     True  2021-07-09 01:23"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Column1</th>\n",
       "      <th>Column2</th>\n",
       "      <th>Column3</th>\n",
       "      <th>Column4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11</td>\n",
       "      <td>R1C2</td>\n",
       "      <td>False</td>\n",
       "      <td>2021-04-03 12:23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22</td>\n",
       "      <td>R2C2</td>\n",
       "      <td>True</td>\n",
       "      <td>2021-03-30 08:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>R3C2</td>\n",
       "      <td>False</td>\n",
       "      <td>2021-04-02 18:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44</td>\n",
       "      <td>R4C2</td>\n",
       "      <td>True</td>\n",
       "      <td>2021-07-09 01:23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Read an Apache Server directory\n",
    "\n",
    "For example, the National Centers for Environmental Information (NCEI) makes data available in an Apache directory structure\n",
    "at <https://www.ncei.noaa.gov/data>.\n",
    "\n",
    "The *Global Summary of the Day* is organized by year, with a CSV file for each station for each year at\n",
    "<https://www.ncei.noaa.gov/data/global-summary-of-the-day/access>.\n",
    "\n",
    "Here we build a [catalog](https://intake.readthedocs.io/en/latest/catalog.html) from the root of the directory and list the subdirectories.\n",
    "\n"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "from intake_html_table import ApacheDirectoryCatalog"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "cat = ApacheDirectoryCatalog(\"https://www.ncei.noaa.gov/data/global-summary-of-the-day/access/\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "# [:10] to print only the first 10 items\n",
    "list(cat)[:10]\n"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['parent',\n",
       " '1929',\n",
       " '1930',\n",
       " '1931',\n",
       " '1932',\n",
       " '1933',\n",
       " '1934',\n",
       " '1935',\n",
       " '1936',\n",
       " '1937']"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "We can list an individual subdirectory as well "
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "list(cat['2021'])[:10]"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "['parent',\n",
       " '01001099999.csv',\n",
       " '01001499999.csv',\n",
       " '01002099999.csv',\n",
       " '01003099999.csv',\n",
       " '01006099999.csv',\n",
       " '01007099999.csv',\n",
       " '01008099999.csv',\n",
       " '01009099999.csv',\n",
       " '01010099999.csv']"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "source": [
    "This directory contains CSV files. We can read them via the catalog:"
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "source": [
    "df = cat['2021']['01001099999.csv'].read()"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "source": [
    "df.info()"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 197 entries, 0 to 196\n",
      "Data columns (total 28 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   STATION           197 non-null    int64  \n",
      " 1   DATE              197 non-null    object \n",
      " 2   LATITUDE          197 non-null    float64\n",
      " 3   LONGITUDE         197 non-null    float64\n",
      " 4   ELEVATION         197 non-null    float64\n",
      " 5   NAME              197 non-null    object \n",
      " 6   TEMP              197 non-null    float64\n",
      " 7   TEMP_ATTRIBUTES   197 non-null    int64  \n",
      " 8   DEWP              197 non-null    float64\n",
      " 9   DEWP_ATTRIBUTES   197 non-null    int64  \n",
      " 10  SLP               197 non-null    float64\n",
      " 11  SLP_ATTRIBUTES    197 non-null    int64  \n",
      " 12  STP               197 non-null    float64\n",
      " 13  STP_ATTRIBUTES    197 non-null    int64  \n",
      " 14  VISIB             197 non-null    float64\n",
      " 15  VISIB_ATTRIBUTES  197 non-null    int64  \n",
      " 16  WDSP              197 non-null    float64\n",
      " 17  WDSP_ATTRIBUTES   197 non-null    int64  \n",
      " 18  MXSPD             197 non-null    float64\n",
      " 19  GUST              197 non-null    float64\n",
      " 20  MAX               197 non-null    float64\n",
      " 21  MAX_ATTRIBUTES    197 non-null    object \n",
      " 22  MIN               197 non-null    float64\n",
      " 23  MIN_ATTRIBUTES    197 non-null    object \n",
      " 24  PRCP              197 non-null    float64\n",
      " 25  PRCP_ATTRIBUTES   197 non-null    object \n",
      " 26  SNDP              197 non-null    float64\n",
      " 27  FRSHTT            197 non-null    int64  \n",
      "dtypes: float64(15), int64(8), object(5)\n",
      "memory usage: 43.2+ KB\n"
     ]
    }
   ],
   "metadata": {}
  }
 ]
}