0_DataScraper.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "<div style=\"display:flex; flex-direction:row; justify-content:space-between\">\n",
        "    <a href=\"./README.md\" target=\"_blank\">Project Overview</a>\n",
        "    <a href=\"./1_DataAnalysis.ipynb\" target=\"_blank\">Data Analysis</a>\n",
        "</div>\n",
        "\n",
        "---\n",
        "\n",
        "# Data Collection\n",
        "\n",
        "<p>Run on <a href=\"https://githubtocolab.com/crystalcheong/crypto-genie/blob/main/0_DataScraper.ipynb\" ><img alt=\"Google Colab\" src=\"https://img.shields.io/badge/Colab-F9AB00?style=for-the-badge&logo=googlecolab&color=525252\" height=\"25\" style=\"vertical-align:middle\" /></a></p>\n",
        "\n",
        "---\n",
        "\n",
        "📌 In this notebook, it scrapes/retrieves historical and current datasets to be utilized in the project.\n",
        "\n",
        "📑 Data Sources:\n",
        "- Financial Data - <a href=\"https://sg.finance.yahoo.com/cryptocurrencies/\" target=\"_blank\">Yahoo Finance</a> / <a href=\"https://pypi.org/project/yfinance/\" target=\"_blank\">yfinance</a>\n",
        "- Search Trends Data - <a href=\"https://trends.google.com/trends/?geo=SG\" target=\"_blank\">Google Trends</a> / <a href=\"https://pypi.org/project/pytrends/\" target=\"_blank\">pytrends</a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "cellView": "form",
        "id": "a12mZaGBjpmo"
      },
      "outputs": [],
      "source": [
        "#@title Basic Libaries { run: \"auto\" }\n",
        "#@markdown Getting started with the essential libraries\n",
        "\n",
        "import os\n",
        "import sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sb\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline\n",
        "pd.options.display.float_format = '{:,.2f}'.format\n",
        "sb.set(style=\"darkgrid\", context=\"talk\")\n",
        "plt.style.use(\"dark_background\")\n",
        "plt.rcParams.update({\"grid.linewidth\":0.5, \"grid.alpha\":0.5})\n",
        "\n",
        "# Add-on preferences\n",
        "from IPython.core.interactiveshell import InteractiveShell\n",
        "InteractiveShell.ast_node_interactivity = \"all\"\n",
        "\n",
        "import warnings as wn\n",
        "wn.simplefilter(action=\"ignore\", category=FutureWarning)\n",
        "\n",
        "from copy import deepcopy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lA7kdAYdifBQ",
        "outputId": "36484a0d-3833-47b1-9a20-a5c450940ca4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Using Google Colab: False\n",
            "Using Google Drive: False\n"
          ]
        }
      ],
      "source": [
        "#@title Global Attributes { run: \"auto\" }\n",
        "#@markdown Define environment states\n",
        "\n",
        "onDrive = False #@param {type:\"boolean\"}\n",
        "\n",
        "try:\n",
        "  import google.colab\n",
        "  onColab = True\n",
        "\n",
        "  from google.colab import files\n",
        "  \n",
        "  if onDrive:\n",
        "    from google.colab import drive\n",
        "    drive.mount('drive')\n",
        "except ImportError or ModuleNotFoundError:\n",
        "  onColab = False\n",
        "  onDrive = False\n",
        "\n",
        "print(f\"Using Google Colab: {onColab}\");\n",
        "print(f\"Using Google Drive: {onDrive}\");"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hV7rQ5FjUWZg"
      },
      "source": [
        "## Financial Data\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bgG3XIncUrBu",
        "outputId": "5c13b82a-1c3b-4a47-9297-b7d862fbea3f"
      },
      "outputs": [],
      "source": [
        "#@title Add-on Libaries { run: \"auto\" }\n",
        "\n",
        "if onColab:\n",
        "  !{sys.executable} -m pip install -q yfinance\n",
        "\n",
        "import yfinance as yf\n",
        "import math\n",
        "from datetime import datetime\n",
        "from typing import List"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Data Retrieval\n",
        "*Fetch ticker historical data from YahooFinance*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LijOcNk0UOje",
        "outputId": "149cc62d-fa2a-4332-f8a2-c3b37d02c34c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[*********************100%***********************]  1 of 1 completed\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Close</th>\n",
              "      <th>Volume</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2770</th>\n",
              "      <td>2022-04-18</td>\n",
              "      <td>39,721.20</td>\n",
              "      <td>40,986.32</td>\n",
              "      <td>38,696.19</td>\n",
              "      <td>40,826.21</td>\n",
              "      <td>33705182072</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2771</th>\n",
              "      <td>2022-04-19</td>\n",
              "      <td>40,828.18</td>\n",
              "      <td>41,672.96</td>\n",
              "      <td>40,618.63</td>\n",
              "      <td>41,502.75</td>\n",
              "      <td>25303206547</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2772</th>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>41,501.75</td>\n",
              "      <td>42,126.30</td>\n",
              "      <td>40,961.10</td>\n",
              "      <td>41,374.38</td>\n",
              "      <td>27819532341</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2773</th>\n",
              "      <td>2022-04-21</td>\n",
              "      <td>41,371.52</td>\n",
              "      <td>42,893.58</td>\n",
              "      <td>40,063.83</td>\n",
              "      <td>40,527.36</td>\n",
              "      <td>35372786395</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2774</th>\n",
              "      <td>2022-04-22</td>\n",
              "      <td>40,404.45</td>\n",
              "      <td>40,777.76</td>\n",
              "      <td>40,391.52</td>\n",
              "      <td>40,555.98</td>\n",
              "      <td>37344608256</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           Date      Open      High       Low     Close       Volume\n",
              "2770 2022-04-18 39,721.20 40,986.32 38,696.19 40,826.21  33705182072\n",
              "2771 2022-04-19 40,828.18 41,672.96 40,618.63 41,502.75  25303206547\n",
              "2772 2022-04-20 41,501.75 42,126.30 40,961.10 41,374.38  27819532341\n",
              "2773 2022-04-21 41,371.52 42,893.58 40,063.83 40,527.36  35372786395\n",
              "2774 2022-04-22 40,404.45 40,777.76 40,391.52 40,555.98  37344608256"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "coinSymbols = 'BTC' #@param [\"BTC\", \"ETH\"]{allow-input: true}\n",
        "\n",
        "# List crypto tickers\n",
        "coins = [coinSymbols]\n",
        "coinTickers = [f\"{c}-USD\" for c in coins]\n",
        "\n",
        "# Fetch all crypto tickers\n",
        "coinTickerData = yf.download(tickers = \" \".join(coinTickers), period=\"max\", group_by = 'ticker', threads = True, auto_adjust = True, prepost = True).reset_index()\n",
        "coinTickerData.tail()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Catalogue and Format Dataset\n",
        "*Stores all ticker dataset into single repo*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z_WuBMZnWsSP",
        "outputId": "91b2fe55-0564-4610-e239-d2dae516f0d8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>info</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>BTC-USD</th>\n",
              "      <td>{'name': 'Bitcoin', 'shortName': 'BTC'}</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            info\n",
              "BTC-USD  {'name': 'Bitcoin', 'shortName': 'BTC'}"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tickerData = deepcopy(coinTickerData)\n",
        "allTickers = {}\n",
        "\n",
        "for idx, ticker in enumerate(coinTickers):\n",
        "  # Retrieve main ticker information\n",
        "  tickerInfo = yf.Ticker(ticker).info\n",
        "  tName = tickerInfo[\"name\"]\n",
        "  tNameShort = tickerInfo[\"fromCurrency\"]\n",
        "\n",
        "  # Extract focal data\n",
        "  if(len(coinTickers) > 1):\n",
        "    data = tickerData[[\"Date\" , ticker]]\n",
        "\n",
        "    # Flatten the dataframe headers\n",
        "    if(len(data) > 1):\n",
        "      data.columns = data.columns.get_level_values(1)\n",
        "      data.columns.values[0] = \"Date\"\n",
        "  else:\n",
        "    data = tickerData\n",
        "\n",
        "  # Format headers\n",
        "  data.columns = data.columns.str.upper()\n",
        "\n",
        "  # Store into data repo\n",
        "  allTickers[ticker] = {\n",
        "    \"info\": {\n",
        "        \"name\": tName,\n",
        "        \"shortName\": tNameShort\n",
        "    },\n",
        "    \"data\": data\n",
        "  }\n",
        "\n",
        "# Display brief info about tickers\n",
        "pd.DataFrame.from_dict(allTickers, orient='index').iloc[:, :-1]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Extract Specific Tickers Columns\n",
        "*Stores all ticker dataset into single repo*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dec7d-UfXm_k",
        "outputId": "947c093d-71da-4751-9f90-5b6a6abb0847"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>OPEN</th>\n",
              "      <th>HIGH</th>\n",
              "      <th>LOW</th>\n",
              "      <th>CLOSE</th>\n",
              "      <th>VOLUME</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>DATE</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2022-04-16</th>\n",
              "      <td>40,552.32</td>\n",
              "      <td>40,633.68</td>\n",
              "      <td>40,078.43</td>\n",
              "      <td>40,424.48</td>\n",
              "      <td>16833150693</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-04-17</th>\n",
              "      <td>40,417.78</td>\n",
              "      <td>40,570.73</td>\n",
              "      <td>39,620.89</td>\n",
              "      <td>39,716.95</td>\n",
              "      <td>19087633042</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-04-18</th>\n",
              "      <td>39,721.20</td>\n",
              "      <td>40,986.32</td>\n",
              "      <td>38,696.19</td>\n",
              "      <td>40,826.21</td>\n",
              "      <td>33705182072</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-04-19</th>\n",
              "      <td>40,828.18</td>\n",
              "      <td>41,672.96</td>\n",
              "      <td>40,618.63</td>\n",
              "      <td>41,502.75</td>\n",
              "      <td>25303206547</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2022-04-20</th>\n",
              "      <td>41,453.36</td>\n",
              "      <td>42,126.30</td>\n",
              "      <td>41,282.08</td>\n",
              "      <td>41,842.68</td>\n",
              "      <td>27917529088</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                OPEN      HIGH       LOW     CLOSE       VOLUME\n",
              "DATE                                                           \n",
              "2022-04-16 40,552.32 40,633.68 40,078.43 40,424.48  16833150693\n",
              "2022-04-17 40,417.78 40,570.73 39,620.89 39,716.95  19087633042\n",
              "2022-04-18 39,721.20 40,986.32 38,696.19 40,826.21  33705182072\n",
              "2022-04-19 40,828.18 41,672.96 40,618.63 41,502.75  25303206547\n",
              "2022-04-20 41,453.36 42,126.30 41,282.08 41,842.68  27917529088"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "def getTickerSeries(src:pd.DataFrame, ticker:str, cols:List[str] = []) -> pd.DataFrame:\n",
        "  tickerData = deepcopy(src[ticker][\"data\"])\n",
        "  if(len(cols) > 0) and set(cols).issubset(set(tickerData.columns.to_list())):\n",
        "    tickerData = tickerData[cols]\n",
        "  return tickerData\n",
        "\n",
        "# targetCols=[\"DATE\", \"CLOSE\"]\n",
        "targetCols = '' #@param {type:\"string\"}\n",
        "cols = list(map(str.strip, targetCols.split(',')))\n",
        "\n",
        "# Initialise with first ticker\n",
        "tickerClose = getTickerSeries(src=allTickers, ticker=coinTickers[0], cols=cols)\n",
        "\n",
        "# Append if there are more tickers\n",
        "if len(coinTickers) > 1:  \n",
        "  for ticker in coinTickers[1:]:\n",
        "    tickerClose = pd.merge(tickerClose, getTickerSeries(src=allTickers, ticker=ticker, cols=cols), on=\"DATE\", suffixes=tuple(f\"_{coin}\" for coin in coins))\n",
        "\n",
        "# Display data preview\n",
        "tickerClose = tickerClose.set_index('DATE')\n",
        "tickerClose.tail()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l4fooKrfctXB"
      },
      "source": [
        "## Search Trends Data\n",
        "\n",
        "----\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a7OjyCvwdBzO",
        "outputId": "38b65887-1da4-4802-c565-23613c15d709"
      },
      "outputs": [],
      "source": [
        "#@title Add-on Libaries { run: \"auto\" }\n",
        "\n",
        "if onColab:\n",
        "  !{sys.executable} -m pip install -q pytrends iteration_utilities\n",
        "\n",
        "from pytrends.request import TrendReq\n",
        "from datetime import date\n",
        "from typing import List\n",
        "from iteration_utilities import grouper\n",
        "import glob"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Trend Data Scrape \n",
        "*Archived search terms, with specified timeframe*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "NB2KRIzLIJs3",
        "outputId": "bf6ec173-5b6e-4a21-f23b-b02695b4ab60"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>dtime</th>\n",
              "      <th>cryptocurrency</th>\n",
              "      <th>isPartial</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2021-01-01 00:00:00</td>\n",
              "      <td>24</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2021-01-01 01:00:00</td>\n",
              "      <td>28</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2021-01-01 02:00:00</td>\n",
              "      <td>42</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2021-01-01 03:00:00</td>\n",
              "      <td>29</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2021-01-01 04:00:00</td>\n",
              "      <td>43</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8784</th>\n",
              "      <td>2021-12-30 21:00:00</td>\n",
              "      <td>46</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8785</th>\n",
              "      <td>2021-12-30 22:00:00</td>\n",
              "      <td>52</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8786</th>\n",
              "      <td>2021-12-30 23:00:00</td>\n",
              "      <td>57</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8787</th>\n",
              "      <td>2021-12-31 00:00:00</td>\n",
              "      <td>74</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8788</th>\n",
              "      <td>2021-12-31 00:00:00</td>\n",
              "      <td>40</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>8789 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                   dtime  cryptocurrency  isPartial\n",
              "0    2021-01-01 00:00:00              24      False\n",
              "1    2021-01-01 01:00:00              28      False\n",
              "2    2021-01-01 02:00:00              42      False\n",
              "3    2021-01-01 03:00:00              29      False\n",
              "4    2021-01-01 04:00:00              43      False\n",
              "...                  ...             ...        ...\n",
              "8784 2021-12-30 21:00:00              46      False\n",
              "8785 2021-12-30 22:00:00              52      False\n",
              "8786 2021-12-30 23:00:00              57      False\n",
              "8787 2021-12-31 00:00:00              74      False\n",
              "8788 2021-12-31 00:00:00              40      False\n",
              "\n",
              "[8789 rows x 3 columns]"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SUCCESSFULL EXPORTED to [/home/ccyq/crypto-genie/data/searchTrends/cryptocurrency/trend-2021.csv]\n"
          ]
        }
      ],
      "source": [
        "terms = 'cryptocurrency' #@param {type:\"string\"}\n",
        "if len(terms) == 0:\n",
        "  terms = 'cryptocurrency'\n",
        "searchTerms = [t for t in list(map(str.strip, terms.split(','))) if len(t) > 0]\n",
        "\n",
        "\n",
        "minYear =  2021 #@param {type:\"number\"}\n",
        "minMth, minDay = 1,1\n",
        "maxYear = date.today().year\n",
        "maxMth, maxDay = 12,31\n",
        "\n",
        "if minYear < maxYear - 5:\n",
        "  print(f\"{minYear} is too far back, min. set to start from {maxYear - 5}\")\n",
        "  minYear = maxYear - 5\n",
        "singleYear = True #@param {type:\"boolean\"}\n",
        "\n",
        "# Retrive archive data for a specific year\n",
        "def annualArchive(searchTerms:List[str], year:int):\n",
        "  googleTrend = TrendReq()\n",
        "  googleTrend.build_payload(kw_list=searchTerms, timeframe=\"all\")\n",
        "  searchTrendData = googleTrend.get_historical_interest(searchTerms, year_start=year, month_start=minMth, day_start=minDay, hour_start=0, year_end=year, month_end=maxMth, day_end=maxDay, hour_end=0, cat=0, geo='', gprop='', sleep=0)\n",
        "  searchTrendData.reset_index(inplace=True)\n",
        "  searchTrendData.columns.values[0] = \"dtime\"\n",
        "  # searchTrendData.set_index('dtime', inplace=True)\n",
        "  return searchTrendData\n",
        "\n",
        "# Centralised archive repo\n",
        "trendArchive = {}\n",
        "\n",
        "if singleYear:\n",
        "  # Append to archive repo\n",
        "  trendArchive[minYear] = annualArchive(searchTerms, minYear)\n",
        "else:\n",
        "  for year in range(minYear, maxYear):\n",
        "    # Append to archive repo\n",
        "    trendArchive[minYear] = annualArchive(searchTerms, year)\n",
        "\n",
        "# Merge all into single archive dataset\n",
        "trendArchiveData = pd.concat(list(trendArchive.values()))\n",
        "trendArchiveData\n",
        "\n",
        "#@markdown ---\n",
        "#@markdown Export Dataset<br/>\n",
        "\n",
        "exportCSV = True #@param {type:\"boolean\"}\n",
        "\n",
        "try:\n",
        "  if len(trendArchiveData) > 1 and exportCSV:\n",
        "\n",
        "    # Export configurations\n",
        "    fileName = f\"trend-{minYear}.csv\" if singleYear else f\"trend-{minYear}_{maxYear-1}.csv\"\n",
        "    exportFolderPath = \"data/searchTrends/cryptocurrency\" #@param {type:\"string\"}\n",
        "    if len(exportFolderPath) > 0 and not exportFolderPath.endswith('/'):\n",
        "      exportFolderPath += '/'\n",
        "    filePath = f'{exportFolderPath if len(exportFolderPath) > 0 else \"\"}{fileName}'\n",
        "\n",
        "    # Export destinations\n",
        "    if onColab:\n",
        "      if onDrive:\n",
        "        filePath = f'/content/drive/My Drive/{filePath}'\n",
        "        trendArchiveData.to_csv(filePath, encoding='utf-8', index=False)\n",
        "        print(f\"SUCCESSFULL EXPORTED to [{filePath}]\")\n",
        "      else:\n",
        "        trendArchiveData.to_csv(fileName) \n",
        "        files.download(fileName)\n",
        "        print(f\"SUCCESSFULL EXPORTED to [{fileName}]\")\n",
        "    else:\n",
        "      BASE_DIR = os.getcwd()\n",
        "      filePath = os.path.join(BASE_DIR, filePath)\n",
        "      trendArchiveData.to_csv(filePath, encoding='utf-8', index=False)\n",
        "      print(f\"SUCCESSFULL EXPORTED to [{filePath}]\")\n",
        "\n",
        "except NameError:\n",
        "  print(\"NO DATA TO EXPORT\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xiz6nkp1L2iY"
      },
      "source": [
        "### Trends Archive Compilation\n",
        "\n",
        "----\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "2le2FkZSLzlq",
        "outputId": "362cd019-fb81-4b9a-9e43-a931683241d6"
      },
      "outputs": [],
      "source": [
        "combineCSV = True #@param {type:\"boolean\"}\n",
        "\n",
        "if combineCSV:\n",
        "  # Import source\n",
        "  importFolderPath = \"data/searchTrends\" #@param {type:\"string\"}\n",
        "  if len(importFolderPath) > 0 and not importFolderPath.endswith('/'):\n",
        "    importFolderPath += '/'\n",
        "  if onColab:\n",
        "    if onDrive:\n",
        "      importFolderPath = f'/content/drive/My Drive/{importFolderPath if len(importFolderPath) > 0 else \"\"}'\n",
        "    else:\n",
        "      importFolderPath = f\"/content/{importFolderPath if len(importFolderPath) > 0 else ''}\"\n",
        "  filesPath = f'{importFolderPath}*.csv'\n",
        "\n",
        "  # Import process\n",
        "  allTrends = []\n",
        "  try:\n",
        "    if len(glob.glob(filesPath)):\n",
        "      for f in glob.glob(filesPath):\n",
        "        trendData = pd.read_csv(f, header=0).dropna()\n",
        "        trendData\n",
        "        allTrends.append(trendData)\n",
        "\n",
        "      if len(allTrends) == len(glob.glob(filesPath)):\n",
        "        print(f\"SUCCESSFULLY IMPORTED from [{filesPath}]\")\n",
        "        allTrends = pd.concat(allTrends, axis=1)\n",
        "        _, i = np.unique(allTrends.columns, return_index=True)\n",
        "        allTrends = allTrends.iloc[:, i]\n",
        "        allTrends['dtime'] = pd.to_datetime(allTrends['dtime'], errors='coerce')\n",
        "        # allTrends.set_index('dtime', inplace=True)\n",
        "        allTrends\n",
        "        \n",
        "        # @markdown ---\n",
        "        # @markdown Export Dataset<br/>\n",
        "\n",
        "        exportCSV = True #@param {type:\"boolean\"}\n",
        "\n",
        "        if exportCSV:\n",
        "          #  Retrieve Historical Time Range from Data\n",
        "          earliestDate = allTrends.head(1)['dtime'].values[0].astype('datetime64[D]').item()\n",
        "          latestDate = allTrends.tail(1)['dtime'].values[0].astype('datetime64[D]').item()\n",
        "          print(f\"Historical year range: {earliestDate.year} to {latestDate.year}\")\n",
        "\n",
        "          # Export configurations\n",
        "          exportFolderPath = \"\" #@param {type:\"string\"} \n",
        "          fileName = f\"trend-{earliestDate.year}.csv\" if earliestDate.year == latestDate.year else f\"trend-{earliestDate.year}_{latestDate.year}.csv\"\n",
        "          if len(exportFolderPath) > 0 and not exportFolderPath.endswith('/'):\n",
        "            exportFolderPath += '/'\n",
        "          filePath = f'{exportFolderPath if len(exportFolderPath) > 0 else \"\"}{fileName}'\n",
        "\n",
        "          # Export destinations\n",
        "          if onColab:\n",
        "            if onDrive:\n",
        "              filePath = f'/content/drive/My Drive/{filePath}'\n",
        "              allTrends.to_csv(filePath, encoding='utf-8', index=False)\n",
        "              print(f\"SUCCESSFULL EXPORTED to [{filePath}]\")\n",
        "            else:\n",
        "              allTrends.to_csv(fileName, encoding='utf-8', index=False)\n",
        "              files.download(fileName)\n",
        "              print(f\"SUCCESSFULL EXPORTED to [{fileName}]\")\n",
        "          else:\n",
        "            BASE_DIR = os.getcwd()\n",
        "            filePath = os.path.join(BASE_DIR, filePath)\n",
        "            allTrends.to_csv(filePath, encoding='utf-8', index=False)\n",
        "            print(f\"SUCCESSFULL EXPORTED to [{filePath}]\")\n",
        "\n",
        "  except NameError:\n",
        "    print(\"NO DATA TO IMPORT\")\n",
        "  except FileNotFoundError:\n",
        "      print(\"ERROR - FILE NOT FOUND. [Run 'Trends Scraper' to export the file.]\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4utNf5WL030i"
      },
      "source": [
        "## Dataset Merger (Financial + Search Trends)\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Partial Trends Scraper\n",
        "*Supplement [archive search trends dataset](./searchTrends/trend-2017_2021.csv) with real-time updates to match [latest financial data](#financial-data)*\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 458
        },
        "id": "B7IlqS-K6mNR",
        "outputId": "b76fb710-0b9c-4941-8f55-11324def43a8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2014 is too far back, min. set to start from 2017\n",
            "Year Range: 2017 to 2021\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bitcoin</th>\n",
              "      <th>cryptocurrency</th>\n",
              "      <th>dtime</th>\n",
              "      <th>isPartial</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>28</td>\n",
              "      <td>30</td>\n",
              "      <td>2017-01-01 00:00:00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>32</td>\n",
              "      <td>16</td>\n",
              "      <td>2017-01-01 01:00:00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>33</td>\n",
              "      <td>17</td>\n",
              "      <td>2017-01-01 02:00:00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>41</td>\n",
              "      <td>54</td>\n",
              "      <td>2017-01-01 03:00:00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>33</td>\n",
              "      <td>18</td>\n",
              "      <td>2017-01-01 04:00:00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46609</th>\n",
              "      <td>68</td>\n",
              "      <td>44</td>\n",
              "      <td>2022-04-20 09:00:00</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46610</th>\n",
              "      <td>71</td>\n",
              "      <td>46</td>\n",
              "      <td>2022-04-20 10:00:00</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46611</th>\n",
              "      <td>72</td>\n",
              "      <td>44</td>\n",
              "      <td>2022-04-20 11:00:00</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46612</th>\n",
              "      <td>82</td>\n",
              "      <td>43</td>\n",
              "      <td>2022-04-20 12:00:00</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46613</th>\n",
              "      <td>77</td>\n",
              "      <td>43</td>\n",
              "      <td>2022-04-20 13:00:00</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>46614 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       bitcoin  cryptocurrency               dtime  isPartial\n",
              "0           28              30 2017-01-01 00:00:00       True\n",
              "1           32              16 2017-01-01 01:00:00       True\n",
              "2           33              17 2017-01-01 02:00:00       True\n",
              "3           41              54 2017-01-01 03:00:00       True\n",
              "4           33              18 2017-01-01 04:00:00       True\n",
              "...        ...             ...                 ...        ...\n",
              "46609       68              44 2022-04-20 09:00:00      False\n",
              "46610       71              46 2022-04-20 10:00:00      False\n",
              "46611       72              44 2022-04-20 11:00:00      False\n",
              "46612       82              43 2022-04-20 12:00:00      False\n",
              "46613       77              43 2022-04-20 13:00:00       True\n",
              "\n",
              "[46614 rows x 4 columns]"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Retrieve Time Range from TickerData\n",
        "minYear = (coinTickerData.head(1)['Date'].values[0].astype('datetime64[D]').item()).year\n",
        "latestDate = (coinTickerData.tail(1)['Date'].values[0].astype('datetime64[D]').item())\n",
        "maxYear = latestDate.year\n",
        "if minYear < maxYear - 5:\n",
        "  print(f\"{minYear} is too far back, min. set to start from {maxYear - 5}\")\n",
        "  minYear = maxYear - 5\n",
        "print(f\"Year Range: {minYear} to {maxYear -1}\")\n",
        "\n",
        "fileName = f\"trend-{minYear}.csv\" if minYear == maxYear-1 else f\"trend-{minYear}_{maxYear-1}.csv\"\n",
        "importFolderPath = \"data/searchTrends\" #@param {type:\"string\"}\n",
        "if len(importFolderPath) > 0 and not importFolderPath.endswith('/'):\n",
        "  importFolderPath += '/'\n",
        "\n",
        "if onColab:\n",
        "  if onDrive:\n",
        "    importFolderPath = f'/content/drive/My Drive/{importFolderPath if len(importFolderPath) > 0 else \"\"}'\n",
        "  else:\n",
        "    importFolderPath = f\"/content/{importFolderPath if len(importFolderPath) > 0 else ''}\"\n",
        "else:\n",
        "  BASE_DIR = os.getcwd()\n",
        "  importFolderPath = os.path.join(BASE_DIR, f\"{importFolderPath if len(importFolderPath) > 0 else ''}\")\n",
        "filePath = f'{importFolderPath}{fileName}'\n",
        "\n",
        "# Import CSV \n",
        "try:\n",
        "  allTrendData = pd.read_csv(filePath, header=0)\n",
        "  allTrendData.dropna(inplace=True)\n",
        "\n",
        "  # Retrive search terms from dataset\n",
        "  searchTerms = [term for term in list(set(allTrendData.columns) - set(['dtime', 'isPartial']))]\n",
        "\n",
        "  # Reassign dtypes\n",
        "  allTrendData['dtime'] = pd.to_datetime(allTrendData['dtime'], errors='coerce')\n",
        "  for idx, val in enumerate(searchTerms):\n",
        "    allTrendData[searchTerms[idx]] = pd.to_numeric(allTrendData[searchTerms[idx]], errors='coerce')\n",
        "  allTrendData['isPartial'] = allTrendData['isPartial'].astype('bool')\n",
        "\n",
        "  # Retrieve current trend data\n",
        "  recentTrendData = []\n",
        "  if len(searchTerms) > 0:\n",
        "    for term in searchTerms:\n",
        "      googleTrend = TrendReq()\n",
        "      googleTrend.build_payload(kw_list=[term], timeframe=\"all\")\n",
        "      searchTrendData = googleTrend.get_historical_interest([term], year_start=latestDate.year, month_start=1, day_start=1, hour_start=0, year_end=latestDate.year, month_end=latestDate.month, day_end=latestDate.day, hour_end=int(datetime.now().strftime(\"%H\")), cat=0, geo='', gprop='', sleep=0)\n",
        "      searchTrendData.reset_index(inplace=True)\n",
        "      searchTrendData.columns.values[0] = \"dtime\"\n",
        "\n",
        "      recentTrendData.append(searchTrendData)\n",
        "      \n",
        "  recentTrendData = pd.concat(recentTrendData, axis=1)\n",
        "  _, i = np.unique(recentTrendData.columns, return_index=True)\n",
        "  recentTrendData = recentTrendData.iloc[:, i]\n",
        "  recentTrendData['dtime'] = pd.to_datetime(recentTrendData['dtime'], errors='coerce')\n",
        "\n",
        "  # Update search trend dataset with recent\n",
        "  allTrendData = allTrendData.merge(recentTrendData, how=\"outer\")\n",
        "\n",
        "  allTrendData\n",
        "except FileNotFoundError:\n",
        "    print(\"ERROR - FILE NOT FOUND. [Run 'Trends Scraper' to export the file.]\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ltDcHjy6o_fN"
      },
      "source": [
        "### Feature Engineering\n",
        "*Timeframe specific extraction/transformation of relevant variables from [search trends dataset](#partial-trends-scraper)*\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Hourly Search Trends\n",
        "*View the hourly trend*\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "rPNoV4IY___4",
        "outputId": "2cb80560-7983-4aca-8489-0e163d06eaab"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>BITCOIN</th>\n",
              "      <th>CRYPTOCURRENCY</th>\n",
              "      <th>DATE</th>\n",
              "      <th>TIME</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>28</td>\n",
              "      <td>30</td>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>32</td>\n",
              "      <td>16</td>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>01:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>33</td>\n",
              "      <td>17</td>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>02:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>41</td>\n",
              "      <td>54</td>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>03:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>33</td>\n",
              "      <td>18</td>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>04:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46609</th>\n",
              "      <td>68</td>\n",
              "      <td>44</td>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>09:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46610</th>\n",
              "      <td>71</td>\n",
              "      <td>46</td>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>10:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46611</th>\n",
              "      <td>72</td>\n",
              "      <td>44</td>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>11:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46612</th>\n",
              "      <td>82</td>\n",
              "      <td>43</td>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>12:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46613</th>\n",
              "      <td>77</td>\n",
              "      <td>43</td>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>13:00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>46614 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       BITCOIN  CRYPTOCURRENCY        DATE      TIME\n",
              "0           28              30  2017-01-01  00:00:00\n",
              "1           32              16  2017-01-01  01:00:00\n",
              "2           33              17  2017-01-01  02:00:00\n",
              "3           41              54  2017-01-01  03:00:00\n",
              "4           33              18  2017-01-01  04:00:00\n",
              "...        ...             ...         ...       ...\n",
              "46609       68              44  2022-04-20  09:00:00\n",
              "46610       71              46  2022-04-20  10:00:00\n",
              "46611       72              44  2022-04-20  11:00:00\n",
              "46612       82              43  2022-04-20  12:00:00\n",
              "46613       77              43  2022-04-20  13:00:00\n",
              "\n",
              "[46614 rows x 4 columns]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "searchTrend = deepcopy(allTrendData)\n",
        "\n",
        "# Dataframe formatting\n",
        "if \"dtime\" in searchTrend.columns:\n",
        "  searchTrend['date'] = pd.to_datetime(searchTrend['dtime']).dt.date\n",
        "  searchTrend['time'] = pd.to_datetime(searchTrend['dtime']).dt.time\n",
        "  searchTrend.drop(['dtime', 'isPartial'], axis=1, inplace=True)\n",
        "searchTrend.columns = searchTrend.columns.str.upper()\n",
        "\n",
        "# Preview data\n",
        "searchTrend"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QehJBA6yAJEc"
      },
      "source": [
        "#### Daily Average Search Percentile\n",
        "*The mean interest per day*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "0u9KRUcaAHaP",
        "outputId": "8895e47b-c8ec-4089-bc6d-15999d4b013e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DATE</th>\n",
              "      <th>BITCOIN</th>\n",
              "      <th>CRYPTOCURRENCY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>37.00</td>\n",
              "      <td>18.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2017-01-02</td>\n",
              "      <td>61.00</td>\n",
              "      <td>27.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2017-01-03</td>\n",
              "      <td>59.00</td>\n",
              "      <td>34.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2017-01-04</td>\n",
              "      <td>66.00</td>\n",
              "      <td>32.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2017-01-05</td>\n",
              "      <td>82.00</td>\n",
              "      <td>34.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1931</th>\n",
              "      <td>2022-04-16</td>\n",
              "      <td>73.00</td>\n",
              "      <td>54.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1932</th>\n",
              "      <td>2022-04-17</td>\n",
              "      <td>73.00</td>\n",
              "      <td>54.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1933</th>\n",
              "      <td>2022-04-18</td>\n",
              "      <td>89.00</td>\n",
              "      <td>65.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1934</th>\n",
              "      <td>2022-04-19</td>\n",
              "      <td>81.00</td>\n",
              "      <td>53.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1935</th>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>69.00</td>\n",
              "      <td>48.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1936 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           DATE  BITCOIN  CRYPTOCURRENCY\n",
              "0    2017-01-01    37.00           18.00\n",
              "1    2017-01-02    61.00           27.00\n",
              "2    2017-01-03    59.00           34.00\n",
              "3    2017-01-04    66.00           32.00\n",
              "4    2017-01-05    82.00           34.00\n",
              "...         ...      ...             ...\n",
              "1931 2022-04-16    73.00           54.00\n",
              "1932 2022-04-17    73.00           54.00\n",
              "1933 2022-04-18    89.00           65.00\n",
              "1934 2022-04-19    81.00           53.00\n",
              "1935 2022-04-20    69.00           48.00\n",
              "\n",
              "[1936 rows x 3 columns]"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "avgDailySearchTrend = deepcopy(searchTrend)\n",
        "\n",
        "# Dataframe formatting\n",
        "avgDailySearchTrend = avgDailySearchTrend.groupby(['Date'.upper()]).mean().round(0)\n",
        "avgDailySearchTrend.reset_index(inplace=True)\n",
        "avgDailySearchTrend['Date'.upper()] = avgDailySearchTrend['Date'.upper()].astype('datetime64[ns]')\n",
        "\n",
        "# Preview data\n",
        "avgDailySearchTrend"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Monthly Search Percentile\n",
        "*Rescrape monthly search trend*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "lX0uiKPZCQNi",
        "outputId": "50b49493-9477-4779-d936-696215907c1e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DATE</th>\n",
              "      <th>BITCOIN</th>\n",
              "      <th>CRYPTOCURRENCY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2004-01-01</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2004-02-01</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2004-03-01</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2004-04-01</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2004-05-01</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>215</th>\n",
              "      <td>2021-12-01</td>\n",
              "      <td>35</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>216</th>\n",
              "      <td>2022-01-01</td>\n",
              "      <td>38</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>217</th>\n",
              "      <td>2022-02-01</td>\n",
              "      <td>33</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>2022-03-01</td>\n",
              "      <td>29</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>219</th>\n",
              "      <td>2022-04-01</td>\n",
              "      <td>24</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>220 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          DATE  BITCOIN  CRYPTOCURRENCY\n",
              "0   2004-01-01        0               0\n",
              "1   2004-02-01        0               0\n",
              "2   2004-03-01        0               0\n",
              "3   2004-04-01        0               0\n",
              "4   2004-05-01        0               0\n",
              "..         ...      ...             ...\n",
              "215 2021-12-01       35               4\n",
              "216 2022-01-01       38               4\n",
              "217 2022-02-01       33               3\n",
              "218 2022-03-01       29               2\n",
              "219 2022-04-01       24               2\n",
              "\n",
              "[220 rows x 3 columns]"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Scrape from Google Trends \n",
        "googleTrend.build_payload(kw_list=searchTerms, timeframe=\"all\")\n",
        "monthlySearchTrend = googleTrend.interest_over_time()\n",
        "monthlySearchTrend.reset_index(inplace=True)\n",
        "\n",
        "# Dataframe formatting\n",
        "monthlySearchTrend.drop(['isPartial'], axis=1, inplace=True)\n",
        "monthlySearchTrend.columns = monthlySearchTrend.columns.str.upper()\n",
        "\n",
        "# Preview data\n",
        "monthlySearchTrend"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Crypto - Trends Combined Dataset\n",
        "*Merge / Export the above datasets*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "CjvfQCVj7alU",
        "outputId": "f90d63ba-308d-45fb-b95f-e2d6e00e929a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DATE</th>\n",
              "      <th>OPEN</th>\n",
              "      <th>HIGH</th>\n",
              "      <th>LOW</th>\n",
              "      <th>CLOSE</th>\n",
              "      <th>VOLUME</th>\n",
              "      <th>BITCOIN</th>\n",
              "      <th>CRYPTOCURRENCY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2017-01-01</td>\n",
              "      <td>963.66</td>\n",
              "      <td>1,003.08</td>\n",
              "      <td>958.70</td>\n",
              "      <td>998.33</td>\n",
              "      <td>147775008</td>\n",
              "      <td>37.00</td>\n",
              "      <td>18.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2017-01-02</td>\n",
              "      <td>998.62</td>\n",
              "      <td>1,031.39</td>\n",
              "      <td>996.70</td>\n",
              "      <td>1,021.75</td>\n",
              "      <td>222184992</td>\n",
              "      <td>61.00</td>\n",
              "      <td>27.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2017-01-03</td>\n",
              "      <td>1,021.60</td>\n",
              "      <td>1,044.08</td>\n",
              "      <td>1,021.60</td>\n",
              "      <td>1,043.84</td>\n",
              "      <td>185168000</td>\n",
              "      <td>59.00</td>\n",
              "      <td>34.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2017-01-04</td>\n",
              "      <td>1,044.40</td>\n",
              "      <td>1,159.42</td>\n",
              "      <td>1,044.40</td>\n",
              "      <td>1,154.73</td>\n",
              "      <td>344945984</td>\n",
              "      <td>66.00</td>\n",
              "      <td>32.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2017-01-05</td>\n",
              "      <td>1,156.73</td>\n",
              "      <td>1,191.10</td>\n",
              "      <td>910.42</td>\n",
              "      <td>1,013.38</td>\n",
              "      <td>510199008</td>\n",
              "      <td>82.00</td>\n",
              "      <td>34.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1931</th>\n",
              "      <td>2022-04-16</td>\n",
              "      <td>40,552.32</td>\n",
              "      <td>40,633.68</td>\n",
              "      <td>40,078.43</td>\n",
              "      <td>40,424.48</td>\n",
              "      <td>16833150693</td>\n",
              "      <td>73.00</td>\n",
              "      <td>54.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1932</th>\n",
              "      <td>2022-04-17</td>\n",
              "      <td>40,417.78</td>\n",
              "      <td>40,570.73</td>\n",
              "      <td>39,620.89</td>\n",
              "      <td>39,716.95</td>\n",
              "      <td>19087633042</td>\n",
              "      <td>73.00</td>\n",
              "      <td>54.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1933</th>\n",
              "      <td>2022-04-18</td>\n",
              "      <td>39,721.20</td>\n",
              "      <td>40,986.32</td>\n",
              "      <td>38,696.19</td>\n",
              "      <td>40,826.21</td>\n",
              "      <td>33705182072</td>\n",
              "      <td>89.00</td>\n",
              "      <td>65.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1934</th>\n",
              "      <td>2022-04-19</td>\n",
              "      <td>40,828.18</td>\n",
              "      <td>41,672.96</td>\n",
              "      <td>40,618.63</td>\n",
              "      <td>41,502.75</td>\n",
              "      <td>25303206547</td>\n",
              "      <td>81.00</td>\n",
              "      <td>53.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1935</th>\n",
              "      <td>2022-04-20</td>\n",
              "      <td>41,453.36</td>\n",
              "      <td>42,126.30</td>\n",
              "      <td>41,282.08</td>\n",
              "      <td>41,842.68</td>\n",
              "      <td>27917529088</td>\n",
              "      <td>69.00</td>\n",
              "      <td>48.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1936 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           DATE      OPEN      HIGH       LOW     CLOSE       VOLUME  BITCOIN  \\\n",
              "0    2017-01-01    963.66  1,003.08    958.70    998.33    147775008    37.00   \n",
              "1    2017-01-02    998.62  1,031.39    996.70  1,021.75    222184992    61.00   \n",
              "2    2017-01-03  1,021.60  1,044.08  1,021.60  1,043.84    185168000    59.00   \n",
              "3    2017-01-04  1,044.40  1,159.42  1,044.40  1,154.73    344945984    66.00   \n",
              "4    2017-01-05  1,156.73  1,191.10    910.42  1,013.38    510199008    82.00   \n",
              "...         ...       ...       ...       ...       ...          ...      ...   \n",
              "1931 2022-04-16 40,552.32 40,633.68 40,078.43 40,424.48  16833150693    73.00   \n",
              "1932 2022-04-17 40,417.78 40,570.73 39,620.89 39,716.95  19087633042    73.00   \n",
              "1933 2022-04-18 39,721.20 40,986.32 38,696.19 40,826.21  33705182072    89.00   \n",
              "1934 2022-04-19 40,828.18 41,672.96 40,618.63 41,502.75  25303206547    81.00   \n",
              "1935 2022-04-20 41,453.36 42,126.30 41,282.08 41,842.68  27917529088    69.00   \n",
              "\n",
              "      CRYPTOCURRENCY  \n",
              "0              18.00  \n",
              "1              27.00  \n",
              "2              34.00  \n",
              "3              32.00  \n",
              "4              34.00  \n",
              "...              ...  \n",
              "1931           54.00  \n",
              "1932           54.00  \n",
              "1933           65.00  \n",
              "1934           53.00  \n",
              "1935           48.00  \n",
              "\n",
              "[1936 rows x 8 columns]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "(1936, 8)"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SUCCESSFULL EXPORTED to [/home/ccyq/crypto-genie/data/BTC-SearchTrend.csv]\n"
          ]
        }
      ],
      "source": [
        "exportCSV = True #@param {type:\"boolean\"}\n",
        "\n",
        "# Prep Crypto dataset\n",
        "closes = deepcopy(tickerClose)\n",
        "# Prep Trends dataset\n",
        "trend = deepcopy(avgDailySearchTrend)\n",
        "\n",
        "# Merge with corresponding trend data\n",
        "closeTrend = pd.merge(closes, trend, on=\"Date\".upper())\n",
        "closeTrend.dropna(inplace=True)\n",
        "# Data formatting\n",
        "closeTrend.columns = closeTrend.columns.str.upper()\n",
        "# Display dataset preview\n",
        "closeTrend\n",
        "closeTrend.shape\n",
        "\n",
        "# Export to CSV\n",
        "if exportCSV:\n",
        "  fileName = f\"{coins[0]}-SearchTrend.csv\"\n",
        "  exportFolderPath = \"data\" #@param {type:\"string\"} \n",
        "  if len(exportFolderPath) > 0 and not exportFolderPath.endswith('/'):\n",
        "    exportFolderPath += '/'\n",
        "  filePath = f'{exportFolderPath if len(exportFolderPath) > 0 else \"\"}{fileName}'\n",
        "\n",
        "  if onColab:\n",
        "    if onDrive:\n",
        "      filePath = f'/content/drive/My Drive/{filePath}'\n",
        "      closeTrend.to_csv(filePath, encoding='utf-8', index=False)\n",
        "      print(f\"SUCCESSFULL EXPORTED to [{filePath}]\")\n",
        "    else:\n",
        "      closeTrend.to_csv(fileName, encoding='utf-8', index=False)\n",
        "      files.download(fileName)\n",
        "      print(f\"SUCCESSFULL EXPORTED to [{fileName}]\")\n",
        "  else:\n",
        "    BASE_DIR = os.getcwd()\n",
        "    filePath = os.path.join(BASE_DIR, filePath)\n",
        "    closeTrend.to_csv(filePath, encoding='utf-8', index=False)\n",
        "    print(f\"SUCCESSFULL EXPORTED to [{filePath}]\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ifBqYOGGXjzT"
      },
      "source": [
        "---\n",
        "\n",
        "<p align=\"center\" style=\"font-style:italic;\">END OF NOTEBOOK</p>\n",
        "\n",
        "<div style=\"display:flex; flex-direction:row; justify-content:space-between\">\n",
        "    <a href=\"./README.md\" target=\"_blank\">Project Overview</a>\n",
        "    <a href=\"./1_DataAnalysis.ipynb\" target=\"_blank\">Data Analysis</a>\n",
        "</div>"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "0-DataScraper",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}