ContraCostaWebScraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "## Set Up"
      ],
      "metadata": {
        "id": "IKcO4ZJ5zktN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cglmw3tBzXOI",
        "outputId": "5e83feb1-1ccf-493b-c598-75ec28c71f38"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
            "Executing: /tmp/apt-key-gpghome.DUxCqMEvES/gpg.1.sh --keyserver keyserver.ubuntu.com --recv-keys DCC9EFBF77E11517\n",
            "gpg: key DCC9EFBF77E11517: public key \"Debian Stable Release Key (10/buster) <debian-release@lists.debian.org>\" imported\n",
            "gpg: Total number processed: 1\n",
            "gpg:               imported: 1\n",
            "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
            "Executing: /tmp/apt-key-gpghome.8couxsGoKR/gpg.1.sh --keyserver keyserver.ubuntu.com --recv-keys 648ACFD622F3D138\n",
            "gpg: key DC30D7C23CBBABEE: public key \"Debian Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>\" imported\n",
            "gpg: Total number processed: 1\n",
            "gpg:               imported: 1\n",
            "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
            "Executing: /tmp/apt-key-gpghome.GpNftFE4HT/gpg.1.sh --keyserver keyserver.ubuntu.com --recv-keys 112695A0E562B32A\n",
            "gpg: key 4DFAB270CAA96DFA: public key \"Debian Security Archive Automatic Signing Key (10/buster) <ftpmaster@debian.org>\" imported\n",
            "gpg: Total number processed: 1\n",
            "gpg:               imported: 1\n",
            "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
            "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
            "Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)).\n",
            "Get:1 http://deb.debian.org/debian buster InRelease [122 kB]\n",
            "Get:2 http://deb.debian.org/debian buster-updates InRelease [56.6 kB]\n",
            "Get:3 http://deb.debian.org/debian-security buster/updates InRelease [34.8 kB]\n",
            "Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Get:5 http://deb.debian.org/debian buster/main amd64 Packages [10.7 MB]\n",
            "Get:6 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n",
            "Get:7 http://deb.debian.org/debian buster-updates/main amd64 Packages [9,745 B]\n",
            "Get:8 http://deb.debian.org/debian-security buster/updates/main amd64 Packages [796 kB]\n",
            "Hit:9 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:10 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Ign:11 https://r2u.stat.illinois.edu/ubuntu jammy InRelease\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n",
            "Get:13 https://r2u.stat.illinois.edu/ubuntu jammy Release [5,713 B]\n",
            "Hit:14 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Get:15 https://r2u.stat.illinois.edu/ubuntu jammy Release.gpg [793 B]\n",
            "Hit:16 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Hit:17 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Hit:18 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
            "Get:19 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2,050 kB]\n",
            "Get:20 https://r2u.stat.illinois.edu/ubuntu jammy/main all Packages [8,192 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,418 kB]\n",
            "Get:22 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,128 kB]\n",
            "Get:23 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,682 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,315 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,756 kB]\n",
            "Get:26 https://r2u.stat.illinois.edu/ubuntu jammy/main amd64 Packages [2,547 kB]\n",
            "Fetched 35.1 MB in 6s (5,460 kB/s)\n",
            "Reading package lists... Done\n",
            "W: Skipping acquire of configured file 'main/source/Sources' as repository 'https://r2u.stat.illinois.edu/ubuntu jammy InRelease' does not seem to provide it (sources.list entry misspelt?)\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  chromium-common chromium-sandbox libevent-2.1-6 libfontenc1 libgudev-1.0-0 libicu63\n",
            "  libimobiledevice6 libjpeg62-turbo libjsoncpp1 libplist3 libre2-5 libu2f-udev libudev1\n",
            "  libupower-glib3 libusbmuxd6 libvpx5 libwebp6 libxkbfile1 libxtst6 libxxf86dga1\n",
            "  notification-daemon systemd-hwe-hwdb udev upower usbmuxd x11-utils\n",
            "Suggested packages:\n",
            "  chromium-l10n chromium-shell libusbmuxd-tools mesa-utils\n",
            "The following NEW packages will be installed:\n",
            "  chromium chromium-common chromium-driver chromium-sandbox libevent-2.1-6 libfontenc1\n",
            "  libgudev-1.0-0 libicu63 libimobiledevice6 libjpeg62-turbo libjsoncpp1 libplist3 libre2-5\n",
            "  libu2f-udev libupower-glib3 libusbmuxd6 libvpx5 libwebp6 libxkbfile1 libxtst6 libxxf86dga1\n",
            "  notification-daemon systemd-hwe-hwdb udev upower usbmuxd x11-utils\n",
            "The following packages will be upgraded:\n",
            "  libudev1\n",
            "1 upgraded, 27 newly installed, 0 to remove and 55 not upgraded.\n",
            "Need to get 76.8 MB of archives.\n",
            "After this operation, 267 MB of additional disk space will be used.\n",
            "Get:1 http://deb.debian.org/debian buster/main amd64 libevent-2.1-6 amd64 2.1.8-stable-4 [177 kB]\n",
            "Get:2 http://deb.debian.org/debian buster/main amd64 libicu63 amd64 63.1-6+deb10u3 [8,293 kB]\n",
            "Get:3 http://deb.debian.org/debian buster/main amd64 libjpeg62-turbo amd64 1:1.5.2-2+deb10u1 [133 kB]\n",
            "Get:4 http://deb.debian.org/debian buster/main amd64 libjsoncpp1 amd64 1.7.4-3 [75.6 kB]\n",
            "Get:5 http://deb.debian.org/debian buster/main amd64 libre2-5 amd64 20190101+dfsg-2 [161 kB]\n",
            "Get:6 http://deb.debian.org/debian-security buster/updates/main amd64 libvpx5 amd64 1.7.0-3+deb10u3 [800 kB]\n",
            "Get:7 http://deb.debian.org/debian-security buster/updates/main amd64 libwebp6 amd64 0.6.1-2+deb10u3 [263 kB]\n",
            "Get:8 http://deb.debian.org/debian buster/main amd64 chromium-common amd64 90.0.4430.212-1~deb10u1 [1,423 kB]\n",
            "Get:9 http://deb.debian.org/debian buster/main amd64 chromium amd64 90.0.4430.212-1~deb10u1 [58.3 MB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libudev1 amd64 249.11-0ubuntu3.12 [78.2 kB]\n",
            "Get:11 http://deb.debian.org/debian buster/main amd64 chromium-driver amd64 90.0.4430.212-1~deb10u1 [4,703 kB]\n",
            "Get:12 http://deb.debian.org/debian buster/main amd64 chromium-sandbox amd64 90.0.4430.212-1~deb10u1 [146 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 udev amd64 249.11-0ubuntu3.12 [1,557 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfontenc1 amd64 1:1.1.4-1build3 [14.7 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxkbfile1 amd64 1:1.1.0-1build3 [71.8 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxtst6 amd64 2:1.2.3-1build4 [13.4 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 libxxf86dga1 amd64 2:1.1.5-0ubuntu3 [12.6 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 x11-utils amd64 7.7+5build2 [206 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 libgudev-1.0-0 amd64 1:237-2build1 [16.3 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 libplist3 amd64 2.2.0-6build2 [32.1 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 libusbmuxd6 amd64 2.0.2-3build2 [20.4 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 libimobiledevice6 amd64 1.3.0-6build3 [71.1 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu jammy/main amd64 libu2f-udev all 1.1.10-3build2 [4,190 B]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 libupower-glib3 amd64 0.99.17-1 [46.7 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 notification-daemon amd64 3.20.0-4build1 [57.6 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 systemd-hwe-hwdb all 249.11.5 [3,228 B]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 upower amd64 0.99.17-1 [86.7 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 usbmuxd amd64 1.1.1-2build2 [42.8 kB]\n",
            "Fetched 76.8 MB in 3s (28.8 MB/s)\n",
            "(Reading database ... 123586 files and directories currently installed.)\n",
            "Preparing to unpack .../libudev1_249.11-0ubuntu3.12_amd64.deb ...\n",
            "Unpacking libudev1:amd64 (249.11-0ubuntu3.12) over (249.11-0ubuntu3.10) ...\n",
            "Setting up libudev1:amd64 (249.11-0ubuntu3.12) ...\n",
            "Selecting previously unselected package udev.\n",
            "(Reading database ... 123586 files and directories currently installed.)\n",
            "Preparing to unpack .../00-udev_249.11-0ubuntu3.12_amd64.deb ...\n",
            "Unpacking udev (249.11-0ubuntu3.12) ...\n",
            "Selecting previously unselected package libevent-2.1-6:amd64.\n",
            "Preparing to unpack .../01-libevent-2.1-6_2.1.8-stable-4_amd64.deb ...\n",
            "Unpacking libevent-2.1-6:amd64 (2.1.8-stable-4) ...\n",
            "Selecting previously unselected package libicu63:amd64.\n",
            "Preparing to unpack .../02-libicu63_63.1-6+deb10u3_amd64.deb ...\n",
            "Unpacking libicu63:amd64 (63.1-6+deb10u3) ...\n",
            "Selecting previously unselected package libjpeg62-turbo:amd64.\n",
            "Preparing to unpack .../03-libjpeg62-turbo_1%3a1.5.2-2+deb10u1_amd64.deb ...\n",
            "Unpacking libjpeg62-turbo:amd64 (1:1.5.2-2+deb10u1) ...\n",
            "Selecting previously unselected package libjsoncpp1:amd64.\n",
            "Preparing to unpack .../04-libjsoncpp1_1.7.4-3_amd64.deb ...\n",
            "Unpacking libjsoncpp1:amd64 (1.7.4-3) ...\n",
            "Selecting previously unselected package libre2-5:amd64.\n",
            "Preparing to unpack .../05-libre2-5_20190101+dfsg-2_amd64.deb ...\n",
            "Unpacking libre2-5:amd64 (20190101+dfsg-2) ...\n",
            "Selecting previously unselected package libvpx5:amd64.\n",
            "Preparing to unpack .../06-libvpx5_1.7.0-3+deb10u3_amd64.deb ...\n",
            "Unpacking libvpx5:amd64 (1.7.0-3+deb10u3) ...\n",
            "Selecting previously unselected package libwebp6:amd64.\n",
            "Preparing to unpack .../07-libwebp6_0.6.1-2+deb10u3_amd64.deb ...\n",
            "Unpacking libwebp6:amd64 (0.6.1-2+deb10u3) ...\n",
            "Selecting previously unselected package libfontenc1:amd64.\n",
            "Preparing to unpack .../08-libfontenc1_1%3a1.1.4-1build3_amd64.deb ...\n",
            "Unpacking libfontenc1:amd64 (1:1.1.4-1build3) ...\n",
            "Selecting previously unselected package libxkbfile1:amd64.\n",
            "Preparing to unpack .../09-libxkbfile1_1%3a1.1.0-1build3_amd64.deb ...\n",
            "Unpacking libxkbfile1:amd64 (1:1.1.0-1build3) ...\n",
            "Selecting previously unselected package libxtst6:amd64.\n",
            "Preparing to unpack .../10-libxtst6_2%3a1.2.3-1build4_amd64.deb ...\n",
            "Unpacking libxtst6:amd64 (2:1.2.3-1build4) ...\n",
            "Selecting previously unselected package libxxf86dga1:amd64.\n",
            "Preparing to unpack .../11-libxxf86dga1_2%3a1.1.5-0ubuntu3_amd64.deb ...\n",
            "Unpacking libxxf86dga1:amd64 (2:1.1.5-0ubuntu3) ...\n",
            "Selecting previously unselected package x11-utils.\n",
            "Preparing to unpack .../12-x11-utils_7.7+5build2_amd64.deb ...\n",
            "Unpacking x11-utils (7.7+5build2) ...\n",
            "Selecting previously unselected package chromium-common.\n",
            "Preparing to unpack .../13-chromium-common_90.0.4430.212-1~deb10u1_amd64.deb ...\n",
            "Unpacking chromium-common (90.0.4430.212-1~deb10u1) ...\n",
            "Selecting previously unselected package chromium.\n",
            "Preparing to unpack .../14-chromium_90.0.4430.212-1~deb10u1_amd64.deb ...\n",
            "Unpacking chromium (90.0.4430.212-1~deb10u1) ...\n",
            "Selecting previously unselected package chromium-driver.\n",
            "Preparing to unpack .../15-chromium-driver_90.0.4430.212-1~deb10u1_amd64.deb ...\n",
            "Unpacking chromium-driver (90.0.4430.212-1~deb10u1) ...\n",
            "Selecting previously unselected package chromium-sandbox.\n",
            "Preparing to unpack .../16-chromium-sandbox_90.0.4430.212-1~deb10u1_amd64.deb ...\n",
            "Unpacking chromium-sandbox (90.0.4430.212-1~deb10u1) ...\n",
            "Selecting previously unselected package libgudev-1.0-0:amd64.\n",
            "Preparing to unpack .../17-libgudev-1.0-0_1%3a237-2build1_amd64.deb ...\n",
            "Unpacking libgudev-1.0-0:amd64 (1:237-2build1) ...\n",
            "Selecting previously unselected package libplist3:amd64.\n",
            "Preparing to unpack .../18-libplist3_2.2.0-6build2_amd64.deb ...\n",
            "Unpacking libplist3:amd64 (2.2.0-6build2) ...\n",
            "Selecting previously unselected package libusbmuxd6:amd64.\n",
            "Preparing to unpack .../19-libusbmuxd6_2.0.2-3build2_amd64.deb ...\n",
            "Unpacking libusbmuxd6:amd64 (2.0.2-3build2) ...\n",
            "Selecting previously unselected package libimobiledevice6:amd64.\n",
            "Preparing to unpack .../20-libimobiledevice6_1.3.0-6build3_amd64.deb ...\n",
            "Unpacking libimobiledevice6:amd64 (1.3.0-6build3) ...\n",
            "Selecting previously unselected package libu2f-udev.\n",
            "Preparing to unpack .../21-libu2f-udev_1.1.10-3build2_all.deb ...\n",
            "Unpacking libu2f-udev (1.1.10-3build2) ...\n",
            "Selecting previously unselected package libupower-glib3:amd64.\n",
            "Preparing to unpack .../22-libupower-glib3_0.99.17-1_amd64.deb ...\n",
            "Unpacking libupower-glib3:amd64 (0.99.17-1) ...\n",
            "Selecting previously unselected package notification-daemon.\n",
            "Preparing to unpack .../23-notification-daemon_3.20.0-4build1_amd64.deb ...\n",
            "Unpacking notification-daemon (3.20.0-4build1) ...\n",
            "Selecting previously unselected package systemd-hwe-hwdb.\n",
            "Preparing to unpack .../24-systemd-hwe-hwdb_249.11.5_all.deb ...\n",
            "Unpacking systemd-hwe-hwdb (249.11.5) ...\n",
            "Selecting previously unselected package upower.\n",
            "Preparing to unpack .../25-upower_0.99.17-1_amd64.deb ...\n",
            "Unpacking upower (0.99.17-1) ...\n",
            "Selecting previously unselected package usbmuxd.\n",
            "Preparing to unpack .../26-usbmuxd_1.1.1-2build2_amd64.deb ...\n",
            "Unpacking usbmuxd (1.1.1-2build2) ...\n",
            "Setting up libplist3:amd64 (2.2.0-6build2) ...\n",
            "Setting up libxtst6:amd64 (2:1.2.3-1build4) ...\n",
            "Setting up libxxf86dga1:amd64 (2:1.1.5-0ubuntu3) ...\n",
            "Setting up chromium-sandbox (90.0.4430.212-1~deb10u1) ...\n",
            "Setting up libicu63:amd64 (63.1-6+deb10u3) ...\n",
            "Setting up notification-daemon (3.20.0-4build1) ...\n",
            "Setting up libfontenc1:amd64 (1:1.1.4-1build3) ...\n",
            "Setting up libjpeg62-turbo:amd64 (1:1.5.2-2+deb10u1) ...\n",
            "Setting up udev (249.11-0ubuntu3.12) ...\n",
            "invoke-rc.d: could not determine current runlevel\n",
            "invoke-rc.d: policy-rc.d denied execution of start.\n",
            "Setting up libwebp6:amd64 (0.6.1-2+deb10u3) ...\n",
            "Setting up libevent-2.1-6:amd64 (2.1.8-stable-4) ...\n",
            "Setting up systemd-hwe-hwdb (249.11.5) ...\n",
            "Setting up libusbmuxd6:amd64 (2.0.2-3build2) ...\n",
            "Setting up libupower-glib3:amd64 (0.99.17-1) ...\n",
            "Setting up libre2-5:amd64 (20190101+dfsg-2) ...\n",
            "Setting up libxkbfile1:amd64 (1:1.1.0-1build3) ...\n",
            "Setting up libimobiledevice6:amd64 (1.3.0-6build3) ...\n",
            "Setting up libgudev-1.0-0:amd64 (1:237-2build1) ...\n",
            "Setting up libvpx5:amd64 (1.7.0-3+deb10u3) ...\n",
            "Setting up libjsoncpp1:amd64 (1.7.4-3) ...\n",
            "Setting up libu2f-udev (1.1.10-3build2) ...\n",
            "Setting up upower (0.99.17-1) ...\n",
            "Setting up usbmuxd (1.1.1-2build2) ...\n",
            "Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory\n",
            "Adding system user `usbmux' (UID 104) ...\n",
            "Adding new user `usbmux' (UID 104) with group `plugdev' ...\n",
            "Not creating home directory `/var/lib/usbmux'.\n",
            "Setting up x11-utils (7.7+5build2) ...\n",
            "Setting up chromium-common (90.0.4430.212-1~deb10u1) ...\n",
            "Setting up chromium (90.0.4430.212-1~deb10u1) ...\n",
            "update-alternatives: using /usr/bin/chromium to provide /usr/bin/x-www-browser (x-www-browser) in auto mode\n",
            "update-alternatives: using /usr/bin/chromium to provide /usr/bin/gnome-www-browser (gnome-www-browser) in auto mode\n",
            "Setting up chromium-driver (90.0.4430.212-1~deb10u1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for dbus (1.12.20-2ubuntu4.1) ...\n",
            "Processing triggers for hicolor-icon-theme (0.17-2) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "Collecting selenium\n",
            "  Downloading selenium-4.22.0-py3-none-any.whl (9.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.4/9.4 MB\u001b[0m \u001b[31m38.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: urllib3[socks]<3,>=1.26 in /usr/local/lib/python3.10/dist-packages (from selenium) (2.0.7)\n",
            "Collecting trio~=0.17 (from selenium)\n",
            "  Downloading trio-0.26.0-py3-none-any.whl (475 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m475.7/475.7 kB\u001b[0m \u001b[31m33.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trio-websocket~=0.9 (from selenium)\n",
            "  Downloading trio_websocket-0.11.1-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: certifi>=2021.10.8 in /usr/local/lib/python3.10/dist-packages (from selenium) (2024.7.4)\n",
            "Requirement already satisfied: typing_extensions>=4.9.0 in /usr/local/lib/python3.10/dist-packages (from selenium) (4.12.2)\n",
            "Requirement already satisfied: websocket-client>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from selenium) (1.8.0)\n",
            "Requirement already satisfied: attrs>=23.2.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (23.2.0)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (2.4.0)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (3.7)\n",
            "Collecting outcome (from trio~=0.17->selenium)\n",
            "  Downloading outcome-1.3.0.post0-py2.py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: sniffio>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from trio~=0.17->selenium) (1.2.2)\n",
            "Collecting wsproto>=0.14 (from trio-websocket~=0.9->selenium)\n",
            "  Downloading wsproto-1.2.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
            "Collecting h11<1,>=0.9.0 (from wsproto>=0.14->trio-websocket~=0.9->selenium)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: outcome, h11, wsproto, trio, trio-websocket, selenium\n",
            "Successfully installed h11-0.14.0 outcome-1.3.0.post0 selenium-4.22.0 trio-0.26.0 trio-websocket-0.11.1 wsproto-1.2.0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "%%shell\n",
        "# Ubuntu no longer distributes chromium-browser outside of snap\n",
        "#\n",
        "# Proposed solution: https://askubuntu.com/questions/1204571/how-to-install-chromium-without-snap\n",
        "\n",
        "# Add debian buster\n",
        "cat > /etc/apt/sources.list.d/debian.list <<'EOF'\n",
        "deb [arch=amd64 signed-by=/usr/share/keyrings/debian-buster.gpg] http://deb.debian.org/debian buster main\n",
        "deb [arch=amd64 signed-by=/usr/share/keyrings/debian-buster-updates.gpg] http://deb.debian.org/debian buster-updates main\n",
        "deb [arch=amd64 signed-by=/usr/share/keyrings/debian-security-buster.gpg] http://deb.debian.org/debian-security buster/updates main\n",
        "EOF\n",
        "\n",
        "# Add keys\n",
        "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DCC9EFBF77E11517\n",
        "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 648ACFD622F3D138\n",
        "apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 112695A0E562B32A\n",
        "\n",
        "apt-key export 77E11517 | gpg --dearmour -o /usr/share/keyrings/debian-buster.gpg\n",
        "apt-key export 22F3D138 | gpg --dearmour -o /usr/share/keyrings/debian-buster-updates.gpg\n",
        "apt-key export E562B32A | gpg --dearmour -o /usr/share/keyrings/debian-security-buster.gpg\n",
        "\n",
        "# Prefer debian repo for chromium* packages only\n",
        "# Note the double-blank lines between entries\n",
        "cat > /etc/apt/preferences.d/chromium.pref << 'EOF'\n",
        "Package: *\n",
        "Pin: release a=eoan\n",
        "Pin-Priority: 500\n",
        "\n",
        "\n",
        "Package: *\n",
        "Pin: origin \"deb.debian.org\"\n",
        "Pin-Priority: 300\n",
        "\n",
        "\n",
        "Package: chromium*\n",
        "Pin: origin \"deb.debian.org\"\n",
        "Pin-Priority: 700\n",
        "EOF\n",
        "\n",
        "# Install chromium and chromium-driver\n",
        "apt-get update\n",
        "apt-get install chromium chromium-driver\n",
        "\n",
        "# Install selenium\n",
        "pip install selenium"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Driver"
      ],
      "metadata": {
        "id": "KSVyopcl6yjE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from selenium import webdriver\n",
        "from selenium.webdriver.chrome.service import Service\n",
        "\n",
        "service = Service(executable_path=r'/usr/bin/chromedriver')\n",
        "options = webdriver.ChromeOptions()\n",
        "options.add_argument('--headless')\n",
        "options.add_argument('--no-sandbox')\n",
        "options.add_argument('--disable-dev-shm-usage')\n",
        "\n",
        "url = \"https://crsecurepayment.com/RW/?ln=en\"\n",
        "\n",
        "driver = webdriver.Chrome(service=service, options=options)\n",
        "driver.get(url)"
      ],
      "metadata": {
        "id": "4Jcetfcw0dDI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from selenium.webdriver.common.by import By\n",
        "documents_tab = driver.find_element(By.XPATH, '/html/body/form/div[3]/div/div[7]/article/ul/li[4]/a')\n",
        "documents_tab.click()"
      ],
      "metadata": {
        "id": "4Tn8puho1xHp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "search_start = driver.find_element(By.ID, 'MainContent_MainMenu1_SearchByDocType1_FromDate')\n",
        "search_end = driver.find_element(By.ID, 'MainContent_MainMenu1_SearchByDocType1_ToDate')"
      ],
      "metadata": {
        "id": "f6u8octJzoDu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_deed_types():\n",
        "  type_xpath_str = \"/html/body/form/div[3]/div/div[7]/article/div[6]/div/table/tbody/tr/td[1]/table/tbody/tr[2]/td/div/table/tbody/tr[2]/td[1]/table/tbody/tr[2]/td/div/table/tbody/tr[{}]/td[2]\"\n",
        "  idx = 1\n",
        "  deed_types = []\n",
        "  while True:\n",
        "    try:\n",
        "      document_type = driver.find_element(By.XPATH, type_xpath_str.format(idx)).text\n",
        "      idx += 1\n",
        "      if (\"DEED\" in document_type.upper()):\n",
        "        deed_types.append((document_type, idx))\n",
        "    except:\n",
        "      break\n",
        "  return deed_types"
      ],
      "metadata": {
        "id": "TkFMjc1A4qIC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deed_types = get_deed_types()"
      ],
      "metadata": {
        "id": "mL44dulT58EE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deed_types[0:1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pezvNIMj7yDk",
        "outputId": "dabbd4d7-836c-464e-9503-e4e6bd7d7da4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('AMEND RESTATED DEED OF TRUST', 22)]"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def make_search(start_date, end_date, deed_types):\n",
        "  search_start.send_keys(start_date)\n",
        "  search_end.send_keys(end_date)\n",
        "\n",
        "  check_box_xpath = \"/html/body/form/div[3]/div/div[7]/article/div[6]/div/table/tbody/tr/td[1]/table/tbody/tr[2]/td/div/table/tbody/tr[2]/td[1]/table/tbody/tr[2]/td/div/table/tbody/tr[{}]/td[1]/label/span\"\n",
        "  for _ ,idx in deed_types:\n",
        "    check_box = driver.find_element(By.XPATH, check_box_xpath.format(idx))\n",
        "    check_box.click()\n",
        "\n",
        "  search_button = driver.find_element(By.ID, 'MainContent_MainMenu1_SearchByDocType1_btnSearch')\n",
        "  search_button.click()"
      ],
      "metadata": {
        "id": "iy2MhSHG2U5g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "make_search(\"12/10/2013\", \"12/10/2023\", deed_types[0:1])"
      ],
      "metadata": {
        "id": "CkAiUBWa5AnC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "driver.find_element(By.ID, \"EntityTitleDocNum_docNumber\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 758
        },
        "id": "v7tWrhxe4QOj",
        "outputId": "d38b2ecd-6221-43ae-a8f5-2546aea53aa8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NoSuchElementException",
          "evalue": "Message: no such element: Unable to locate element: {\"method\":\"css selector\",\"selector\":\"[id=\"EntityTitleDocNum_docNumber\"]\"}\n  (Session info: headless chrome=90.0.4430.212); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\nStacktrace:\n#0 0x5723d846a7f9 <unknown>\n#1 0x5723d840a3b3 <unknown>\n#2 0x5723d8152016 <unknown>\n#3 0x5723d818681e <unknown>\n#4 0x5723d81bc8fb <unknown>\n#5 0x5723d81a9ded <unknown>\n#6 0x5723d81ba9e1 <unknown>\n#7 0x5723d81a9c93 <unknown>\n#8 0x5723d817bce4 <unknown>\n#9 0x5723d817d4d2 <unknown>\n#10 0x5723d8436542 <unknown>\n#11 0x5723d8445ce7 <unknown>\n#12 0x5723d84459e4 <unknown>\n#13 0x5723d844a13a <unknown>\n#14 0x5723d84465b9 <unknown>\n#15 0x5723d842be00 <unknown>\n#16 0x5723d845d5d2 <unknown>\n#17 0x5723d845d778 <unknown>\n#18 0x5723d8475a1f <unknown>\n#19 0x7de711e3fac3 <unknown>\n#20 0x7de711ed1850 <unknown>\n",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNoSuchElementException\u001b[0m                    Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-8163cbf59308>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_element\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mID\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"EntityTitleDocNum_docNumber\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py\u001b[0m in \u001b[0;36mfind_element\u001b[0;34m(self, by, value)\u001b[0m\n\u001b[1;32m    746\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf'[name=\"{value}\"]'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    747\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 748\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFIND_ELEMENT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"using\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mby\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"value\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"value\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    749\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    750\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfind_elements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mby\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mID\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mWebElement\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, driver_command, params)\u001b[0m\n\u001b[1;32m    352\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    353\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 354\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    355\u001b[0m             \u001b[0mresponse\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"value\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"value\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/selenium/webdriver/remote/errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    227\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"alert\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"text\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    228\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 229\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNoSuchElementException\u001b[0m: Message: no such element: Unable to locate element: {\"method\":\"css selector\",\"selector\":\"[id=\"EntityTitleDocNum_docNumber\"]\"}\n  (Session info: headless chrome=90.0.4430.212); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#no-such-element-exception\nStacktrace:\n#0 0x5723d846a7f9 <unknown>\n#1 0x5723d840a3b3 <unknown>\n#2 0x5723d8152016 <unknown>\n#3 0x5723d818681e <unknown>\n#4 0x5723d81bc8fb <unknown>\n#5 0x5723d81a9ded <unknown>\n#6 0x5723d81ba9e1 <unknown>\n#7 0x5723d81a9c93 <unknown>\n#8 0x5723d817bce4 <unknown>\n#9 0x5723d817d4d2 <unknown>\n#10 0x5723d8436542 <unknown>\n#11 0x5723d8445ce7 <unknown>\n#12 0x5723d84459e4 <unknown>\n#13 0x5723d844a13a <unknown>\n#14 0x5723d84465b9 <unknown>\n#15 0x5723d842be00 <unknown>\n#16 0x5723d845d5d2 <unknown>\n#17 0x5723d845d778 <unknown>\n#18 0x5723d8475a1f <unknown>\n#19 0x7de711e3fac3 <unknown>\n#20 0x7de711ed1850 <unknown>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "OmCN-mzm8y-6"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}