W3_Assignment_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "W3_Assignment_2.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "5DPDtB7DTcRk",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Segmenting and Clustering Neighborhoods in Toronto"
      ]
    },
    {
      "metadata": {
        "id": "Ufrej3VFUmnO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Libraries"
      ]
    },
    {
      "metadata": {
        "id": "rXoCiBbBUtba",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Let's import all needed libraries."
      ]
    },
    {
      "metadata": {
        "id": "iHrFl9BA5FTX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import requests\n",
        "from bs4 import BeautifulSoup"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "H-lWzdWKU-wk",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 1. Step 1: Scraping"
      ]
    },
    {
      "metadata": {
        "id": "h1pDiyw26iKp",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Let's repeat what we did in the first notebook."
      ]
    },
    {
      "metadata": {
        "id": "gP9j2Qdc64a8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "wikipedia_page = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\n",
        "source = requests.get(wikipedia_page).text\n",
        "soup = BeautifulSoup(source, 'lxml')\n",
        "table = soup.find('table')\n",
        "data = []\n",
        "for i, row in enumerate(table.find_all('tr')):\n",
        "  entry = [item.text for item in row.find_all('td')]\n",
        "  if entry and entry[1] != 'Not assigned':\n",
        "    if entry[0] not in [item['Postcode'] for item in data]:\n",
        "      data.append({'Postcode': entry[0],\n",
        "                   'Borough': entry[1],\n",
        "                   'Neighbourhood': entry[2].rstrip() if entry[2].rstrip() != 'Not assigned' else entry[1]})\n",
        "    else:\n",
        "      index = [i for i, item in enumerate(data) if item['Postcode'] == entry[0]][0]\n",
        "      data[index]['Neighbourhood'] += ', '+entry[2].rstrip()\n",
        "df = pd.DataFrame(data=data, columns=['Postcode', 'Borough', 'Neighbourhood'])\n",
        "df.columns = ['PostalCode', 'Borough', 'Neighborhood']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-sMUYdSiDfSd",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Step 2: Adding coordinates"
      ]
    },
    {
      "metadata": {
        "id": "SxatgXIqDmen",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Now we need to add the neighborhoods' coordinates to our DataFrame. We'll get our data from the csv file provided, because of issues with the Geocoder package"
      ]
    },
    {
      "metadata": {
        "id": "fecB9JZj9ZoJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "47d7ea93-555a-435d-b758-e39796424637"
      },
      "cell_type": "code",
      "source": [
        "coord_df = pd.read_csv('http://cocl.us/Geospatial_data')\n",
        "coord_df.head(5)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Postal Code</th>\n",
              "      <th>Latitude</th>\n",
              "      <th>Longitude</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>M1B</td>\n",
              "      <td>43.806686</td>\n",
              "      <td>-79.194353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>M1C</td>\n",
              "      <td>43.784535</td>\n",
              "      <td>-79.160497</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>M1E</td>\n",
              "      <td>43.763573</td>\n",
              "      <td>-79.188711</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>M1G</td>\n",
              "      <td>43.770992</td>\n",
              "      <td>-79.216917</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>M1H</td>\n",
              "      <td>43.773136</td>\n",
              "      <td>-79.239476</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Postal Code   Latitude  Longitude\n",
              "0         M1B  43.806686 -79.194353\n",
              "1         M1C  43.784535 -79.160497\n",
              "2         M1E  43.763573 -79.188711\n",
              "3         M1G  43.770992 -79.216917\n",
              "4         M1H  43.773136 -79.239476"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "metadata": {
        "id": "gQbXB_eXEKCi",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Now that we have the coordinates in a DataFrame, we need to merge it with our neighborhoods' DataFrame."
      ]
    },
    {
      "metadata": {
        "id": "zTJg511F_TBT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        },
        "outputId": "f8ea56bb-a3f5-4a47-8b38-a209fe9cab75"
      },
      "cell_type": "code",
      "source": [
        "df_with_coords = pd.merge(df, coord_df, left_on='PostalCode', right_on='Postal Code', suffixes=('', '')).drop('Postal Code', 1)\n",
        "df_with_coords.head(10)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>PostalCode</th>\n",
              "      <th>Borough</th>\n",
              "      <th>Neighborhood</th>\n",
              "      <th>Latitude</th>\n",
              "      <th>Longitude</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>M3A</td>\n",
              "      <td>North York</td>\n",
              "      <td>Parkwoods</td>\n",
              "      <td>43.753259</td>\n",
              "      <td>-79.329656</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>M4A</td>\n",
              "      <td>North York</td>\n",
              "      <td>Victoria Village</td>\n",
              "      <td>43.725882</td>\n",
              "      <td>-79.315572</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>M5A</td>\n",
              "      <td>Downtown Toronto</td>\n",
              "      <td>Harbourfront, Regent Park</td>\n",
              "      <td>43.654260</td>\n",
              "      <td>-79.360636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>M6A</td>\n",
              "      <td>North York</td>\n",
              "      <td>Lawrence Heights, Lawrence Manor</td>\n",
              "      <td>43.718518</td>\n",
              "      <td>-79.464763</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>M7A</td>\n",
              "      <td>Queen's Park</td>\n",
              "      <td>Queen's Park</td>\n",
              "      <td>43.662301</td>\n",
              "      <td>-79.389494</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>M9A</td>\n",
              "      <td>Etobicoke</td>\n",
              "      <td>Islington Avenue</td>\n",
              "      <td>43.667856</td>\n",
              "      <td>-79.532242</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>M1B</td>\n",
              "      <td>Scarborough</td>\n",
              "      <td>Rouge, Malvern</td>\n",
              "      <td>43.806686</td>\n",
              "      <td>-79.194353</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>M3B</td>\n",
              "      <td>North York</td>\n",
              "      <td>Don Mills North</td>\n",
              "      <td>43.745906</td>\n",
              "      <td>-79.352188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>M4B</td>\n",
              "      <td>East York</td>\n",
              "      <td>Woodbine Gardens, Parkview Hill</td>\n",
              "      <td>43.706397</td>\n",
              "      <td>-79.309937</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>M5B</td>\n",
              "      <td>Downtown Toronto</td>\n",
              "      <td>Ryerson, Garden District</td>\n",
              "      <td>43.657162</td>\n",
              "      <td>-79.378937</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  PostalCode           Borough                      Neighborhood   Latitude  \\\n",
              "0        M3A        North York                         Parkwoods  43.753259   \n",
              "1        M4A        North York                  Victoria Village  43.725882   \n",
              "2        M5A  Downtown Toronto         Harbourfront, Regent Park  43.654260   \n",
              "3        M6A        North York  Lawrence Heights, Lawrence Manor  43.718518   \n",
              "4        M7A      Queen's Park                      Queen's Park  43.662301   \n",
              "5        M9A         Etobicoke                  Islington Avenue  43.667856   \n",
              "6        M1B       Scarborough                    Rouge, Malvern  43.806686   \n",
              "7        M3B        North York                   Don Mills North  43.745906   \n",
              "8        M4B         East York   Woodbine Gardens, Parkview Hill  43.706397   \n",
              "9        M5B  Downtown Toronto          Ryerson, Garden District  43.657162   \n",
              "\n",
              "   Longitude  \n",
              "0 -79.329656  \n",
              "1 -79.315572  \n",
              "2 -79.360636  \n",
              "3 -79.464763  \n",
              "4 -79.389494  \n",
              "5 -79.532242  \n",
              "6 -79.194353  \n",
              "7 -79.352188  \n",
              "8 -79.309937  \n",
              "9 -79.378937  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "metadata": {
        "id": "0uoY2alOEiEP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Let's check the shape of our DataFrame."
      ]
    },
    {
      "metadata": {
        "id": "_nc8eTSHCUPz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7318c3fb-43da-4919-e801-4990b6be7349"
      },
      "cell_type": "code",
      "source": [
        "print('There are {} rows in our DataFrame.'.format(df.shape[0]))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 103 rows in our DataFrame.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}