Data_Extracting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Citibike Data Visualization (03-2020 to 04-2020)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The very first step is to extract data from the website. The simplest way is downloading specific files from the website. Here is another way that I found to be very useful if we want to extract multiple files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import necessary dependencies\n",
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import zipfile\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#url to be queried\n",
    "url = 'https://s3.amazonaws.com/tripdata/'\n",
    "\n",
    "#get url content into soup\n",
    "r = requests.get(url)\n",
    "soup = bs(r.text, \"xml\")\n",
    "\n",
    "#check for the tag\n",
    "#print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#extract data from soup\n",
    "data = soup.find_all('Key')\n",
    "\n",
    "#store text of the data as a list\n",
    "clean_data = []\n",
    "for i in range(0,len(data)-1):\n",
    "    clean_data.append(data[i].text)\n",
    "\n",
    "#clean_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#unzip files from the clean_data list\n",
    "for cd in clean_data:\n",
    "    cd_url = url + cd\n",
    "    \n",
    "    #download each data file using the new url\n",
    "    with open(cd, \"wb\") as f:\n",
    "        response = requests.get(cd_url)\n",
    "        f.write(response.content)\n",
    "    \n",
    "    #unzip data files\n",
    "    with zipfile.ZipFile(cd, \"r\") as zip_ref:\n",
    "        zip_ref.extractall(\"tripdata\")\n",
    "        \n",
    "    #remove the old zipped file after unziping\n",
    "    os.remove(cd)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the above script I was able to go over the data at different time period and work on the ones that I found to be interesting. \n",
    "\n",
    "After going over all the files, I decided to work on March and April of 2020. For this, I used glob module to retrive the files of 2020 only and then to the specific months. Then the files merged and exported as a single file. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set the working directory\n",
    "path = \"tripdata\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#match csv files by name/pattern\n",
    "all_files = glob.glob(os.path.join(path, \"20*.csv\"))\n",
    "#all_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#combine all files in the list\n",
    "CHUNK_SIZE=5000\n",
    "output_file = \"Resources/202003_and_04_tripdata.csv\"\n",
    "\n",
    "first_one = True\n",
    "for month in ['03','04']:\n",
    "    if not first_one:\n",
    "        skip_row=[0]\n",
    "    else:\n",
    "        skip_row=[]\n",
    "    \n",
    "    chunk_container = pd.read_csv(\"tripdata/2020\"+month+\"-citibike-tripdata.csv\", chunksize=CHUNK_SIZE)\n",
    "    for chunk in chunk_container:\n",
    "        chunk.to_csv(output_file,mode=\"a\", index=False)\n",
    "    first_one = False"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}