Rough File -  Web Scraping of GitHub Topics Repositories.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f37df2e3",
   "metadata": {},
   "source": [
    "# Top Repositories for GitHub Projects"
   ]
  },

  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "21d7d631",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup \n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dc765796",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics_url = 'https://github.com/topics'\n",
    "response = requests.get(topics_url)\n",
    "response.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d9c148d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "170789"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fc5e8c70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n\\n<!DOCTYPE html>\\n<html\\n  lang=\"en\"\\n  \\n  data-color-mode=\"auto\" data-light-theme=\"light\" data-dark-theme=\"dark\"\\n  data-a11y-animated-images=\"system\" data-a11y-link-underlines=\"true\"\\n  >\\n\\n\\n\\n\\n  <head>\\n    <meta charset=\"utf-8\">\\n  <link rel=\"dns-prefetch\" href=\"https://github.githubassets.com\">\\n  <link rel=\"dns-prefetch\" href=\"https://avatars.githubusercontent.com\">\\n  <link rel=\"dns-prefetch\" href=\"https://github-cloud.s3.amazonaws.com\">\\n  <link rel=\"dns-prefetch\" href=\"https://user-images.githubusercontent.com/\">\\n  <link rel=\"preconnect\" href=\"https://github.githubassets.com\" crossorigin>\\n  <link rel=\"preconnect\" href=\"https://avatars.githubusercontent.com\">\\n\\n  \\n\\n  <link crossorigin=\"anonymous\" media=\"all\" rel=\"stylesheet\" href=\"https://github.githubassets.com/assets/light-38f1bf52eeeb.css\" /><link crossorigin=\"anonymous\" media=\"all\" rel=\"stylesheet\" href=\"https://github.githubassets.com/assets/dark-56010aa53a8f.css\" /><link data-color-theme=\"dark_dimmed\" crossorigin=\"anonymous\" media=\"a'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's see how the content of page look like\n",
    "page_content = response.text\n",
    "page_content[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c982697a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's save the html content to see it \n",
    "with open('githubtopics.html', 'w', encoding='utf-8') as f:\n",
    "    f.write(page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5e5340bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = BeautifulSoup(page_content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f6598955",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "658e964f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selection_class = 'f3 lh-condensed mb-0 mt-1 Link--primary'\n",
    "topic_title_tags = doc.find_all('p', {'class': selection_class})\n",
    "len(topic_title_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d0fe5204",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"f3 lh-condensed mb-0 mt-1 Link--primary\">3D</p>,\n",
       " <p class=\"f3 lh-condensed mb-0 mt-1 Link--primary\">Ajax</p>,\n",
       " <p class=\"f3 lh-condensed mb-0 mt-1 Link--primary\">Algorithm</p>,\n",
       " <p class=\"f3 lh-condensed mb-0 mt-1 Link--primary\">Amp</p>,\n",
       " <p class=\"f3 lh-condensed mb-0 mt-1 Link--primary\">Android</p>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_title_tags[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a5c8d7b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc_selector = 'f5 color-fg-muted mb-0 mt-1'\n",
    "topic_desc_tags = doc.find_all('p', {'class': desc_selector})\n",
    "len(topic_desc_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5cc82968",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"f5 color-fg-muted mb-0 mt-1\">\n",
       "           3D refers to the use of three-dimensional graphics, modeling, and animation in various industries.\n",
       "         </p>,\n",
       " <p class=\"f5 color-fg-muted mb-0 mt-1\">\n",
       "           Ajax is a technique for creating interactive web applications.\n",
       "         </p>,\n",
       " <p class=\"f5 color-fg-muted mb-0 mt-1\">\n",
       "           Algorithms are self-contained sequences that carry out a variety of tasks.\n",
       "         </p>,\n",
       " <p class=\"f5 color-fg-muted mb-0 mt-1\">\n",
       "           Amp is a non-blocking concurrency library for PHP.\n",
       "         </p>,\n",
       " <p class=\"f5 color-fg-muted mb-0 mt-1\">\n",
       "           Android is an operating system built by Google designed for mobile devices.\n",
       "         </p>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_desc_tags[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "561b65dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_title_tag0 = topic_title_tags[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5e1c2cf0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"no-underline flex-1 d-flex flex-column\" href=\"/topics/3d\">\n",
       "<p class=\"f3 lh-condensed mb-0 mt-1 Link--primary\">3D</p>\n",
       "<p class=\"f5 color-fg-muted mb-0 mt-1\">\n",
       "          3D refers to the use of three-dimensional graphics, modeling, and animation in various industries.\n",
       "        </p>\n",
       "</a>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_title_tag0.parent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "88ad965e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_link_tags = doc.find_all('a', {'class': 'no-underline flex-grow-0'})\n",
    "len(topic_link_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f4da6fdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://github.com/topics/3d\n"
     ]
    }
   ],
   "source": [
    "#Let's check that we are getting correct result \n",
    "\n",
    "topic0_url = \"https://github.com\" + topic_link_tags[0]['href']\n",
    "print(topic0_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ebfa9bd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['3D', 'Ajax', 'Algorithm', 'Amp', 'Android', 'Angular', 'Ansible', 'API', 'Arduino', 'ASP.NET', 'Atom', 'Awesome Lists', 'Amazon Web Services', 'Azure', 'Babel', 'Bash', 'Bitcoin', 'Bootstrap', 'Bot', 'C', 'Chrome', 'Chrome extension', 'Command line interface', 'Clojure', 'Code quality', 'Code review', 'Compiler', 'Continuous integration', 'COVID-19', 'C++']\n"
     ]
    }
   ],
   "source": [
    "topic_titles = []\n",
    "\n",
    "for tag in topic_title_tags:\n",
    "    topic_titles.append(tag.text)\n",
    "    \n",
    "print(topic_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "497d09ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['3D refers to the use of three-dimensional graphics, modeling, and animation in various industries.',\n",
       " 'Ajax is a technique for creating interactive web applications.',\n",
       " 'Algorithms are self-contained sequences that carry out a variety of tasks.',\n",
       " 'Amp is a non-blocking concurrency library for PHP.',\n",
       " 'Android is an operating system built by Google designed for mobile devices.']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_descriptions = []\n",
    "\n",
    "for desc in topic_desc_tags:\n",
    "    topic_descriptions.append(desc.text.strip())\n",
    "    \n",
    "topic_descriptions[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6bb01488",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://github.com/topics/3d',\n",
       " 'https://github.com/topics/ajax',\n",
       " 'https://github.com/topics/algorithm',\n",
       " 'https://github.com/topics/amphp',\n",
       " 'https://github.com/topics/android']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_urls = []\n",
    "base_url = \"https://github.com\"\n",
    "\n",
    "for url in topic_link_tags:\n",
    "    topic_urls.append(base_url + url['href'])\n",
    "\n",
    "topic_urls[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4a8e1e44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3D</td>\n",
       "      <td>3D refers to the use of three-dimensional grap...</td>\n",
       "      <td>https://github.com/topics/3d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ajax</td>\n",
       "      <td>Ajax is a technique for creating interactive w...</td>\n",
       "      <td>https://github.com/topics/ajax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Algorithm</td>\n",
       "      <td>Algorithms are self-contained sequences that c...</td>\n",
       "      <td>https://github.com/topics/algorithm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Amp</td>\n",
       "      <td>Amp is a non-blocking concurrency library for ...</td>\n",
       "      <td>https://github.com/topics/amphp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Android</td>\n",
       "      <td>Android is an operating system built by Google...</td>\n",
       "      <td>https://github.com/topics/android</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Angular</td>\n",
       "      <td>Angular is an open source web application plat...</td>\n",
       "      <td>https://github.com/topics/angular</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ansible</td>\n",
       "      <td>Ansible is a simple and powerful automation en...</td>\n",
       "      <td>https://github.com/topics/ansible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>API</td>\n",
       "      <td>An API (Application Programming Interface) is ...</td>\n",
       "      <td>https://github.com/topics/api</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Arduino</td>\n",
       "      <td>Arduino is an open source platform for buildin...</td>\n",
       "      <td>https://github.com/topics/arduino</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ASP.NET</td>\n",
       "      <td>ASP.NET is a web framework for building modern...</td>\n",
       "      <td>https://github.com/topics/aspnet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Atom</td>\n",
       "      <td>Atom is a open source text editor built with w...</td>\n",
       "      <td>https://github.com/topics/atom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Awesome Lists</td>\n",
       "      <td>An awesome list is a list of awesome things cu...</td>\n",
       "      <td>https://github.com/topics/awesome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Amazon Web Services</td>\n",
       "      <td>Amazon Web Services provides on-demand cloud c...</td>\n",
       "      <td>https://github.com/topics/aws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Azure</td>\n",
       "      <td>Azure is a cloud computing service created by ...</td>\n",
       "      <td>https://github.com/topics/azure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Babel</td>\n",
       "      <td>Babel is a compiler for writing next generatio...</td>\n",
       "      <td>https://github.com/topics/babel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Bash</td>\n",
       "      <td>Bash is a shell and command language interpret...</td>\n",
       "      <td>https://github.com/topics/bash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Bitcoin</td>\n",
       "      <td>Bitcoin is a cryptocurrency developed by Satos...</td>\n",
       "      <td>https://github.com/topics/bitcoin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Bootstrap</td>\n",
       "      <td>Bootstrap is an HTML, CSS, and JavaScript fram...</td>\n",
       "      <td>https://github.com/topics/bootstrap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Bot</td>\n",
       "      <td>A bot is an application that runs automated ta...</td>\n",
       "      <td>https://github.com/topics/bot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C</td>\n",
       "      <td>C is a general purpose programming language th...</td>\n",
       "      <td>https://github.com/topics/c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chrome</td>\n",
       "      <td>Chrome is a web browser from the tech company ...</td>\n",
       "      <td>https://github.com/topics/chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chrome extension</td>\n",
       "      <td>Chrome extensions enable users to customize th...</td>\n",
       "      <td>https://github.com/topics/chrome-extension</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Command line interface</td>\n",
       "      <td>A CLI, or command-line interface, is a console...</td>\n",
       "      <td>https://github.com/topics/cli</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Clojure</td>\n",
       "      <td>Clojure is a dynamic, general-purpose programm...</td>\n",
       "      <td>https://github.com/topics/clojure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Code quality</td>\n",
       "      <td>Automate your code review with style, quality,...</td>\n",
       "      <td>https://github.com/topics/code-quality</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Code review</td>\n",
       "      <td>Ensure your code meets quality standards and s...</td>\n",
       "      <td>https://github.com/topics/code-review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Compiler</td>\n",
       "      <td>Compilers are software that translate higher-l...</td>\n",
       "      <td>https://github.com/topics/compiler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Continuous integration</td>\n",
       "      <td>Automatically build and test your code as you ...</td>\n",
       "      <td>https://github.com/topics/continuous-integration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>The coronavirus disease 2019 (COVID-19) is an ...</td>\n",
       "      <td>https://github.com/topics/covid-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C++</td>\n",
       "      <td>C++ is a general purpose and object-oriented p...</td>\n",
       "      <td>https://github.com/topics/cpp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               topic_title                                        description  \\\n",
       "0                       3D  3D refers to the use of three-dimensional grap...   \n",
       "1                     Ajax  Ajax is a technique for creating interactive w...   \n",
       "2                Algorithm  Algorithms are self-contained sequences that c...   \n",
       "3                      Amp  Amp is a non-blocking concurrency library for ...   \n",
       "4                  Android  Android is an operating system built by Google...   \n",
       "5                  Angular  Angular is an open source web application plat...   \n",
       "6                  Ansible  Ansible is a simple and powerful automation en...   \n",
       "7                      API  An API (Application Programming Interface) is ...   \n",
       "8                  Arduino  Arduino is an open source platform for buildin...   \n",
       "9                  ASP.NET  ASP.NET is a web framework for building modern...   \n",
       "10                    Atom  Atom is a open source text editor built with w...   \n",
       "11           Awesome Lists  An awesome list is a list of awesome things cu...   \n",
       "12     Amazon Web Services  Amazon Web Services provides on-demand cloud c...   \n",
       "13                   Azure  Azure is a cloud computing service created by ...   \n",
       "14                   Babel  Babel is a compiler for writing next generatio...   \n",
       "15                    Bash  Bash is a shell and command language interpret...   \n",
       "16                 Bitcoin  Bitcoin is a cryptocurrency developed by Satos...   \n",
       "17               Bootstrap  Bootstrap is an HTML, CSS, and JavaScript fram...   \n",
       "18                     Bot  A bot is an application that runs automated ta...   \n",
       "19                       C  C is a general purpose programming language th...   \n",
       "20                  Chrome  Chrome is a web browser from the tech company ...   \n",
       "21        Chrome extension  Chrome extensions enable users to customize th...   \n",
       "22  Command line interface  A CLI, or command-line interface, is a console...   \n",
       "23                 Clojure  Clojure is a dynamic, general-purpose programm...   \n",
       "24            Code quality  Automate your code review with style, quality,...   \n",
       "25             Code review  Ensure your code meets quality standards and s...   \n",
       "26                Compiler  Compilers are software that translate higher-l...   \n",
       "27  Continuous integration  Automatically build and test your code as you ...   \n",
       "28                COVID-19  The coronavirus disease 2019 (COVID-19) is an ...   \n",
       "29                     C++  C++ is a general purpose and object-oriented p...   \n",
       "\n",
       "                                                 url  \n",
       "0                       https://github.com/topics/3d  \n",
       "1                     https://github.com/topics/ajax  \n",
       "2                https://github.com/topics/algorithm  \n",
       "3                    https://github.com/topics/amphp  \n",
       "4                  https://github.com/topics/android  \n",
       "5                  https://github.com/topics/angular  \n",
       "6                  https://github.com/topics/ansible  \n",
       "7                      https://github.com/topics/api  \n",
       "8                  https://github.com/topics/arduino  \n",
       "9                   https://github.com/topics/aspnet  \n",
       "10                    https://github.com/topics/atom  \n",
       "11                 https://github.com/topics/awesome  \n",
       "12                     https://github.com/topics/aws  \n",
       "13                   https://github.com/topics/azure  \n",
       "14                   https://github.com/topics/babel  \n",
       "15                    https://github.com/topics/bash  \n",
       "16                 https://github.com/topics/bitcoin  \n",
       "17               https://github.com/topics/bootstrap  \n",
       "18                     https://github.com/topics/bot  \n",
       "19                       https://github.com/topics/c  \n",
       "20                  https://github.com/topics/chrome  \n",
       "21        https://github.com/topics/chrome-extension  \n",
       "22                     https://github.com/topics/cli  \n",
       "23                 https://github.com/topics/clojure  \n",
       "24            https://github.com/topics/code-quality  \n",
       "25             https://github.com/topics/code-review  \n",
       "26                https://github.com/topics/compiler  \n",
       "27  https://github.com/topics/continuous-integration  \n",
       "28                https://github.com/topics/covid-19  \n",
       "29                     https://github.com/topics/cpp  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's create \n",
    "\n",
    "topics_dict = {'topic_title' : topic_titles, 'description' : topic_descriptions, 'url' : topic_urls}\n",
    "\n",
    "topics_df = pd.DataFrame(topics_dict)\n",
    "topics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c9352820",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's create  a csv file \n",
    "\n",
    "topics_df.to_csv('github_topics.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "127c3e6d",
   "metadata": {},
   "source": [
    "## Getting information out of a topic page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6de8f8bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://github.com/topics/3d'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_page_url = topic_urls[0]\n",
    "topic_page_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e94c60b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response = requests.get(topic_page_url)\n",
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d60182b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "488728"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d8c41db2",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_doc = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "274a29b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":97088,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"c72fbd5c69a8ee7c9c53a4e65de2b93c8fc7552dd793945819639bc165c0f0ba\" data-turbo=\"false\" data-view-component=\"true\" href=\"/mrdoob\">\n",
       "             mrdoob\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":576201,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"4a2667db3d63a1739c412e059e5da95afe419df83f70949b5d59dc3478f5c79a\" data-turbo=\"false\" data-view-component=\"true\" href=\"/mrdoob/three.js\">\n",
       "             three.js\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":45790596,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"3631fd8d52f4cabfc79d79ff64c69f4b01193561c15fdbebe2e938fb5c659ca6\" data-turbo=\"false\" data-view-component=\"true\" href=\"/pmndrs\">\n",
       "             pmndrs\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":172521926,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"642553dc7d4ffd4a71455d086a76b927fd338477f8a4affe2a70abc205373db7\" data-turbo=\"false\" data-view-component=\"true\" href=\"/pmndrs/react-three-fiber\">\n",
       "             react-three-fiber\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":509841,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"f5fbcbcae567cb54999ca77df523ec3e63bf16aaf66cfe150df67964bd49444d\" data-turbo=\"false\" data-view-component=\"true\" href=\"/libgdx\">\n",
       "             libgdx\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":5373551,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"93e1fbfd676bec2f3c89b7de544fddc92181afdf52544c2bedf8c11ec0d75a2c\" data-turbo=\"false\" data-view-component=\"true\" href=\"/libgdx/libgdx\">\n",
       "             libgdx\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":4855800,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"a1d0cdc67cb2b8c6bde702eaaf3e49c102d6c79761dd9388495e03cb75473502\" data-turbo=\"false\" data-view-component=\"true\" href=\"/BabylonJS\">\n",
       "             BabylonJS\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":11007313,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"4543a4bac703ddf1e723be30e195f4ee9093c06ebc961fff86939beb0bf98e33\" data-turbo=\"false\" data-view-component=\"true\" href=\"/BabylonJS/Babylon.js\">\n",
       "             Babylon.js\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":1029298,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"72cab1d85a871dc19468acc3d9c4778e100acbea03d49accfa78f7c92042609b\" data-turbo=\"false\" data-view-component=\"true\" href=\"/ssloy\">\n",
       "             ssloy\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":29397594,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"53d48dcce88c7bf8263707b062ba365c4bfc1b4dc8c257009a1f2517731c25f9\" data-turbo=\"false\" data-view-component=\"true\" href=\"/ssloy/tinyrenderer\">\n",
       "             tinyrenderer\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":1661343,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"bc2848886fc92527623051680d76178173cc38fd4b5a64ba8e0da37cbf73d243\" data-turbo=\"false\" data-view-component=\"true\" href=\"/lettier\">\n",
       "             lettier\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":186309296,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"bca9f8a5c565ae6f1fbba1832ccebf18d51fe998975ae5cb55251f655b8b9cd5\" data-turbo=\"false\" data-view-component=\"true\" href=\"/lettier/3d-game-shaders-for-beginners\">\n",
       "             3d-game-shaders-for-beginners\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":2306237,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"7a7b36f5724d013a0286371472e4e3d68c50e8fa3c26d065843baa8542fc202b\" data-turbo=\"false\" data-view-component=\"true\" href=\"/FreeCAD\">\n",
       "             FreeCAD\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":5736080,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"4dd7decd3a6df1295ac657dfaa6061a37f49e87224a240f13e848793fceaf92f\" data-turbo=\"false\" data-view-component=\"true\" href=\"/FreeCAD/FreeCAD\">\n",
       "             FreeCAD\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":16024092,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"3f5fe74b0179d71ddc0b548f01d4723c23e1dd08b5397840b2427db1fd064a5b\" data-turbo=\"false\" data-view-component=\"true\" href=\"/aframevr\">\n",
       "             aframevr\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":42488903,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"2e04f568e88deddd90b028a35a3de9f18f43bc1af3ae7b41641bfca3ddadd1fd\" data-turbo=\"false\" data-view-component=\"true\" href=\"/aframevr/aframe\">\n",
       "             aframe\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":54716382,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"5f36cea1e023375dab3e603179e74e61c57f2d2f9aa7597c494c8b2983152ed3\" data-turbo=\"false\" data-view-component=\"true\" href=\"/CesiumGS\">\n",
       "             CesiumGS\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":3606738,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"fd32d7f4708e0174472636c37def63ec56e1a9bbe0842a507797edb03a0934cd\" data-turbo=\"false\" data-view-component=\"true\" href=\"/CesiumGS/cesium\">\n",
       "             cesium\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":52924476,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"f7abd38a6e5fdfcda241dd548b1aee405d2d37941bc45bac414badafd5bdf221\" data-turbo=\"false\" data-view-component=\"true\" href=\"/blender\">\n",
       "             blender\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":213916896,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"d153eaddc10bbb1fcfe8530b05aa7584e86a76129fe06af21fc8faa74a81cc5f\" data-turbo=\"false\" data-view-component=\"true\" href=\"/blender/blender\">\n",
       "             blender\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":4772066,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"043f9b13b3cf093673da140a9fc3c23e7fda2f83fbb943fbb9b4e435843aefb5\" data-turbo=\"false\" data-view-component=\"true\" href=\"/MonoGame\">\n",
       "             MonoGame\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":1579990,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"9c0d3935765856d2dbd1457130702b19557da40f63966925a69beae585642f92\" data-turbo=\"false\" data-view-component=\"true\" href=\"/MonoGame/MonoGame\">\n",
       "             MonoGame\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":573910,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"2475f866571e8bd178375300685d85cf3c27a8ea6724d129ccecbe209a9f67d9\" data-turbo=\"false\" data-view-component=\"true\" href=\"/metafizzy\">\n",
       "             metafizzy\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":174164624,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"2d034d00e2dd43d427d347a28a28848919641085f0f0640422b0d045d6be60a7\" data-turbo=\"false\" data-view-component=\"true\" href=\"/metafizzy/zdog\">\n",
       "             zdog\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":23507030,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"ab13c1ae4c8573c529241a1e71101548a03d08680ea3cec8d9a6b9917c6d1404\" data-turbo=\"false\" data-view-component=\"true\" href=\"/isl-org\">\n",
       "             isl-org\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":75413130,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"869962979c5e689586dda7a0806a1593693995526d5363f18aced7340315d4b6\" data-turbo=\"false\" data-view-component=\"true\" href=\"/isl-org/Open3D\">\n",
       "             Open3D\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":16217333,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"2e5adca28b20f67de2c3817307944835dbdd6e5b4c82003a26b647685a290cdd\" data-turbo=\"false\" data-view-component=\"true\" href=\"/timzhang642\">\n",
       "             timzhang642\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":100120455,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"ead47df2876d7e14a4450598c983a4c8b9cadfabf26a54c3dcd472e4d47e57f9\" data-turbo=\"false\" data-view-component=\"true\" href=\"/timzhang642/3D-Machine-Learning\">\n",
       "             3D-Machine-Learning\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":6295625,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"e59ea0472751fec51e44a07520b8b915decd84396a38b66c744a43e410148ca1\" data-turbo=\"false\" data-view-component=\"true\" href=\"/a1studmuffin\">\n",
       "             a1studmuffin\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":61088333,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"15c751fd0e7e90619826d22f6e080d446b372a091acf7df673793094deb64467\" data-turbo=\"false\" data-view-component=\"true\" href=\"/a1studmuffin/SpaceshipGenerator\">\n",
       "             SpaceshipGenerator\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":106621677,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"70d640c7ed3589913f92e0a864dff091edd0a7724d2ce3a627710ec582edf663\" data-turbo=\"false\" data-view-component=\"true\" href=\"/nerfstudio-project\">\n",
       "             nerfstudio-project\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":498449014,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"9111508a625c34effb89a73132658f3fd4804221d0d89efc82e467081099278b\" data-turbo=\"false\" data-view-component=\"true\" href=\"/nerfstudio-project/nerfstudio\">\n",
       "             nerfstudio\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":5639024,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"c4bd43ca998833f4e79bc1825f1edab46bba9c459adaa89d6ea8e90b9b863030\" data-turbo=\"false\" data-view-component=\"true\" href=\"/domlysz\">\n",
       "             domlysz\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":19577136,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"eedffc7c9a3f608ddfe6390f4a31c3ad533fb1ecff23689cf0d6950373aa7fec\" data-turbo=\"false\" data-view-component=\"true\" href=\"/domlysz/BlenderGIS\">\n",
       "             BlenderGIS\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":83981865,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"256ebe9e59dd043f6ef2aba9e06019dbee0c7cf36c8db177c5bcb4f44c6099be\" data-turbo=\"false\" data-view-component=\"true\" href=\"/FyroxEngine\">\n",
       "             FyroxEngine\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":178597005,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"82d32b017eff488a891ece89faf194e70807430a29391c08a7c425977eb76300\" data-turbo=\"false\" data-view-component=\"true\" href=\"/FyroxEngine/Fyrox\">\n",
       "             Fyrox\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":1342004,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"c08389c6a7ca3286828b46d5463ecf9ba942b3dc3a6bd4c98cabaec65c1f964b\" data-turbo=\"false\" data-view-component=\"true\" href=\"/google\">\n",
       "             google\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":153651476,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"2955c612ccc9834bd3dc287e8932b66a75d0ca64f2b5d9c965f99b3b36651951\" data-turbo=\"false\" data-view-component=\"true\" href=\"/google/model-viewer\">\n",
       "             model-viewer\n",
       " </a> </h3>,\n",
       " <h3 class=\"f3 color-fg-muted text-normal lh-condensed\">\n",
       " <a class=\"Link\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"OWNER\",\"click_visual_representation\":\"REPOSITORY_OWNER_HEADING\",\"actor_id\":null,\"record_id\":466370,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"69c012f59bd036fe442c2ae1b31112e6adbe4826c6ea6b358cda033006b6e00f\" data-turbo=\"false\" data-view-component=\"true\" href=\"/openscad\">\n",
       "             openscad\n",
       " </a>          /\n",
       "           <a class=\"Link text-bold wb-break-word\" data-hydro-click='{\"event_type\":\"explore.click\",\"payload\":{\"click_context\":\"REPOSITORY_CARD\",\"click_target\":\"REPOSITORY\",\"click_visual_representation\":\"REPOSITORY_NAME_HEADING\",\"actor_id\":null,\"record_id\":1049088,\"originating_url\":\"https://github.com/topics/3d\",\"user_id\":null}}' data-hydro-click-hmac=\"01ce3838d3fcac4c3cf6ca9352a172be410abbbe74acbf287b39f77060261e9f\" data-turbo=\"false\" data-view-component=\"true\" href=\"/openscad/openscad\">\n",
       "             openscad\n",
       " </a> </h3>]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h3_class = 'f3 color-fg-muted text-normal lh-condensed'\n",
    "repo_tags = topic_doc.find_all('h3', {'class': h3_class})\n",
    "repo_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4a53bcd9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mrdoob\n",
      "three.js\n"
     ]
    }
   ],
   "source": [
    "a_tags = repo_tags[0].find_all('a')\n",
    "print(a_tags[0].text.strip())              #Giving us user name\n",
    "print(a_tags[1].text.strip())              #Giving us repository name for that user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "81383300",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://github.com/mrdoob/three.js'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo_urls = base_url + a_tags[1]['href']        #we also want repository url so \n",
    "repo_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "28d2e527",
   "metadata": {},
   "outputs": [],
   "source": [
    "star_tags = topic_doc.find_all('span', {'class': 'Counter js-social-count'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6202e024",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'96.6k'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "star_tags[0].text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fdf74ee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's conver it to number\n",
    "def parse_star_count(stars_str):\n",
    "    stars_str = stars_str.strip()\n",
    "    if stars_str[-1] == 'k': \n",
    "        return int(float(stars_str[:-1]) * 1000)\n",
    "    return int(stars_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "be205a7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96600"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parse_star_count(star_tags[0].text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "04c8e88d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_repo_info(h3_tag, star_tag):\n",
    "    #return all the required info about a repository\n",
    "    a_tags = h3_tag.find_all('a')\n",
    "    username = a_tags[0].text.strip()\n",
    "    repo_name = a_tags[1].text.strip()\n",
    "    repo_url = base_url + a_tags[1]['href']\n",
    "    stars = parse_star_count(star_tag.text.strip())\n",
    "    return username, repo_name, stars, repo_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9bba6af6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('mrdoob', 'three.js', 96600, 'https://github.com/mrdoob/three.js')\n",
      "('pmndrs', 'react-three-fiber', 24800, 'https://github.com/pmndrs/react-three-fiber')\n"
     ]
    }
   ],
   "source": [
    "print(get_repo_info(repo_tags[0], star_tags[0]))\n",
    "print(get_repo_info(repo_tags[1], star_tags[1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "af778fce",
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_repos_dict = {\n",
    "    'username': [], \n",
    "    'repo_name': [], \n",
    "    'stars': [], \n",
    "    'repo_url': []\n",
    "}\n",
    "\n",
    "\n",
    "for i in range(len(repo_tags)):\n",
    "    repo_info = get_repo_info(repo_tags[i], star_tags[i])\n",
    "    topic_repos_dict['username'].append(repo_info[0])\n",
    "    topic_repos_dict['repo_name'].append(repo_info[1])\n",
    "    topic_repos_dict['stars'].append(repo_info[2])\n",
    "    topic_repos_dict['repo_url'].append(repo_info[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e2c149ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>stars</th>\n",
       "      <th>repo_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mrdoob</td>\n",
       "      <td>three.js</td>\n",
       "      <td>96600</td>\n",
       "      <td>https://github.com/mrdoob/three.js</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pmndrs</td>\n",
       "      <td>react-three-fiber</td>\n",
       "      <td>24800</td>\n",
       "      <td>https://github.com/pmndrs/react-three-fiber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>libgdx</td>\n",
       "      <td>libgdx</td>\n",
       "      <td>22300</td>\n",
       "      <td>https://github.com/libgdx/libgdx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BabylonJS</td>\n",
       "      <td>Babylon.js</td>\n",
       "      <td>21800</td>\n",
       "      <td>https://github.com/BabylonJS/Babylon.js</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ssloy</td>\n",
       "      <td>tinyrenderer</td>\n",
       "      <td>18600</td>\n",
       "      <td>https://github.com/ssloy/tinyrenderer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    username          repo_name  stars  \\\n",
       "0     mrdoob           three.js  96600   \n",
       "1     pmndrs  react-three-fiber  24800   \n",
       "2     libgdx             libgdx  22300   \n",
       "3  BabylonJS         Babylon.js  21800   \n",
       "4      ssloy       tinyrenderer  18600   \n",
       "\n",
       "                                      repo_url  \n",
       "0           https://github.com/mrdoob/three.js  \n",
       "1  https://github.com/pmndrs/react-three-fiber  \n",
       "2             https://github.com/libgdx/libgdx  \n",
       "3      https://github.com/BabylonJS/Babylon.js  \n",
       "4        https://github.com/ssloy/tinyrenderer  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_repos_df = pd.DataFrame(topic_repos_dict)\n",
    "topic_repos_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e48c843f",
   "metadata": {},
   "source": [
    "## Final code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "4ed8dec3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now Let's do it for all topics (till now we are just working for 3D which is 1st topic) and combine everything in one cell\n",
    "import os\n",
    "\n",
    "def get_topic_page(topic_url):\n",
    "    #Download the page\n",
    "    response = requests.get(topic_url)\n",
    "    #Check sucessful response\n",
    "    if response.status_code != 200:\n",
    "        raise Exception('Failed to load page {}'.format(topic_url))\n",
    "    #Parse using beautifulSoup\n",
    "    topic_doc = BeautifulSoup(response.text, 'html.parser')\n",
    "    return topic_doc\n",
    "\n",
    "\n",
    "def get_repo_info(h3_tag, star_tag):\n",
    "    #return all the required info about a repository\n",
    "    a_tags = h3_tag.find_all('a')\n",
    "    username = a_tags[0].text.strip()\n",
    "    repo_name = a_tags[1].text.strip()\n",
    "    repo_url = base_url + a_tags[1]['href']\n",
    "    stars = parse_star_count(star_tag.text.strip())\n",
    "    return username, repo_name, stars, repo_url\n",
    "\n",
    "\n",
    "def get_topic_repos(topic_doc):\n",
    "    #get the h3 tags containg repo title, repo url and username\n",
    "    h3_class = 'f3 color-fg-muted text-normal lh-condensed'\n",
    "    repo_tags = topic_doc.find_all('h3', {'class': h3_class})\n",
    "    #get star tages\n",
    "    star_tags = topic_doc.find_all('span', {'class': 'Counter js-social-count'})\n",
    "    #get repo info\n",
    "    \n",
    "    topic_repos_dict = {\n",
    "        'username': [], \n",
    "        'repo_name': [], \n",
    "        'stars': [], \n",
    "        'repo_url': []\n",
    "    }\n",
    "    \n",
    "    for i in range(len(repo_tags)):\n",
    "        repo_info = get_repo_info(repo_tags[i], star_tags[i])\n",
    "        topic_repos_dict['username'].append(repo_info[0])\n",
    "        topic_repos_dict['repo_name'].append(repo_info[1])\n",
    "        topic_repos_dict['stars'].append(repo_info[2])\n",
    "        topic_repos_dict['repo_url'].append(repo_info[3])\n",
    "        \n",
    "    return pd.DataFrame(topic_repos_dict)\n",
    " \n",
    "def scrape_topic(topic_url, path):\n",
    "    \n",
    "    if os.path.exists(path):\n",
    "        print(\"The file {} already exists. Skipping...\".format(path))\n",
    "        return\n",
    "    topic_df = get_topic_repos(get_topic_page(topic_url))\n",
    "    topic_df.to_csv(path + '.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3400ebd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>repo_name</th>\n",
       "      <th>stars</th>\n",
       "      <th>repo_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>public-apis</td>\n",
       "      <td>public-apis</td>\n",
       "      <td>273000</td>\n",
       "      <td>https://github.com/public-apis/public-apis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neovim</td>\n",
       "      <td>neovim</td>\n",
       "      <td>72200</td>\n",
       "      <td>https://github.com/neovim/neovim</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tiangolo</td>\n",
       "      <td>fastapi</td>\n",
       "      <td>66500</td>\n",
       "      <td>https://github.com/tiangolo/fastapi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>strapi</td>\n",
       "      <td>strapi</td>\n",
       "      <td>58100</td>\n",
       "      <td>https://github.com/strapi/strapi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hoppscotch</td>\n",
       "      <td>hoppscotch</td>\n",
       "      <td>57700</td>\n",
       "      <td>https://github.com/hoppscotch/hoppscotch</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ocornut</td>\n",
       "      <td>imgui</td>\n",
       "      <td>52800</td>\n",
       "      <td>https://github.com/ocornut/imgui</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>slatedocs</td>\n",
       "      <td>slate</td>\n",
       "      <td>35600</td>\n",
       "      <td>https://github.com/slatedocs/slate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Kong</td>\n",
       "      <td>insomnia</td>\n",
       "      <td>31600</td>\n",
       "      <td>https://github.com/Kong/insomnia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>trpc</td>\n",
       "      <td>trpc</td>\n",
       "      <td>30600</td>\n",
       "      <td>https://github.com/trpc/trpc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>httpie</td>\n",
       "      <td>cli</td>\n",
       "      <td>30600</td>\n",
       "      <td>https://github.com/httpie/cli</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>hasura</td>\n",
       "      <td>graphql-engine</td>\n",
       "      <td>30600</td>\n",
       "      <td>https://github.com/hasura/graphql-engine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Binaryify</td>\n",
       "      <td>NeteaseCloudMusicApi</td>\n",
       "      <td>30000</td>\n",
       "      <td>https://github.com/Binaryify/NeteaseCloudMusicApi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ccxt</td>\n",
       "      <td>ccxt</td>\n",
       "      <td>30000</td>\n",
       "      <td>https://github.com/ccxt/ccxt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>encode</td>\n",
       "      <td>django-rest-framework</td>\n",
       "      <td>26900</td>\n",
       "      <td>https://github.com/encode/django-rest-framework</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>YMFE</td>\n",
       "      <td>yapi</td>\n",
       "      <td>26800</td>\n",
       "      <td>https://github.com/YMFE/yapi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>StanGirard</td>\n",
       "      <td>quivr</td>\n",
       "      <td>26700</td>\n",
       "      <td>https://github.com/StanGirard/quivr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>littlecodersh</td>\n",
       "      <td>ItChat</td>\n",
       "      <td>24600</td>\n",
       "      <td>https://github.com/littlecodersh/ItChat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>directus</td>\n",
       "      <td>directus</td>\n",
       "      <td>24200</td>\n",
       "      <td>https://github.com/directus/directus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>microsoft</td>\n",
       "      <td>api-guidelines</td>\n",
       "      <td>22200</td>\n",
       "      <td>https://github.com/microsoft/api-guidelines</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>shieldfy</td>\n",
       "      <td>API-Security-Checklist</td>\n",
       "      <td>21700</td>\n",
       "      <td>https://github.com/shieldfy/API-Security-Check...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         username               repo_name   stars  \\\n",
       "0     public-apis             public-apis  273000   \n",
       "1          neovim                  neovim   72200   \n",
       "2        tiangolo                 fastapi   66500   \n",
       "3          strapi                  strapi   58100   \n",
       "4      hoppscotch              hoppscotch   57700   \n",
       "5         ocornut                   imgui   52800   \n",
       "6       slatedocs                   slate   35600   \n",
       "7            Kong                insomnia   31600   \n",
       "8            trpc                    trpc   30600   \n",
       "9          httpie                     cli   30600   \n",
       "10         hasura          graphql-engine   30600   \n",
       "11      Binaryify    NeteaseCloudMusicApi   30000   \n",
       "12           ccxt                    ccxt   30000   \n",
       "13         encode   django-rest-framework   26900   \n",
       "14           YMFE                    yapi   26800   \n",
       "15     StanGirard                   quivr   26700   \n",
       "16  littlecodersh                  ItChat   24600   \n",
       "17       directus                directus   24200   \n",
       "18      microsoft          api-guidelines   22200   \n",
       "19       shieldfy  API-Security-Checklist   21700   \n",
       "\n",
       "                                             repo_url  \n",
       "0          https://github.com/public-apis/public-apis  \n",
       "1                    https://github.com/neovim/neovim  \n",
       "2                 https://github.com/tiangolo/fastapi  \n",
       "3                    https://github.com/strapi/strapi  \n",
       "4            https://github.com/hoppscotch/hoppscotch  \n",
       "5                    https://github.com/ocornut/imgui  \n",
       "6                  https://github.com/slatedocs/slate  \n",
       "7                    https://github.com/Kong/insomnia  \n",
       "8                        https://github.com/trpc/trpc  \n",
       "9                       https://github.com/httpie/cli  \n",
       "10           https://github.com/hasura/graphql-engine  \n",
       "11  https://github.com/Binaryify/NeteaseCloudMusicApi  \n",
       "12                       https://github.com/ccxt/ccxt  \n",
       "13    https://github.com/encode/django-rest-framework  \n",
       "14                       https://github.com/YMFE/yapi  \n",
       "15                https://github.com/StanGirard/quivr  \n",
       "16            https://github.com/littlecodersh/ItChat  \n",
       "17               https://github.com/directus/directus  \n",
       "18        https://github.com/microsoft/api-guidelines  \n",
       "19  https://github.com/shieldfy/API-Security-Check...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's try one more example using created function \n",
    "\n",
    "get_topic_repos(get_topic_page(topic_urls[7]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f4b1398e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://github.com/topics/ansible'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let's try another example\n",
    "topic_urls[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "f7a6a1e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's see the output and save it to the csv file \n",
    "get_topic_repos(get_topic_page(topic_urls[6])).to_csv('githubansible.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f38e454f",
   "metadata": {},
   "source": [
    "Write a single function to:\n",
    "    \n",
    "    1. Get the list of topics from the topics page\n",
    "    2. Get the list of top repos from the individual topic page\n",
    "    3. for each topic, create a csv of the top repos for the topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "14d2bbfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_topic_titles(doc):\n",
    "    selection_class = 'f3 lh-condensed mb-0 mt-1 Link--primary'\n",
    "    topic_title_tags = doc.find_all('p', {'class': selection_class})\n",
    "    topic_titles = []\n",
    "    for tag in topic_title_tags:\n",
    "        topic_titles.append(tag.text)\n",
    "    return topic_titles\n",
    "  \n",
    "def get_topic_descs(doc):\n",
    "    desc_selector = 'f5 color-fg-muted mb-0 mt-1'\n",
    "    topic_desc_tags = doc.find_all('p', {'class': desc_selector})\n",
    "    topic_descriptions = []\n",
    "    for desc in topic_desc_tags:\n",
    "        topic_descriptions.append(desc.text.strip())\n",
    "    return topic_descriptions\n",
    "\n",
    "def get_topic_urls(doc):\n",
    "    topic_link_tags = doc.find_all('a', {'class': 'no-underline flex-grow-0'})\n",
    "    topic_urls = []\n",
    "    base_url = \"https://github.com\"\n",
    "    for url in topic_link_tags:\n",
    "        topic_urls.append(base_url + url['href'])\n",
    "    return topic_urls\n",
    "    \n",
    "def scrape_topics():\n",
    "    topics_url = 'https://github.com/topics'\n",
    "    response = requests.get(topics_url)\n",
    "    #Check sucessful response\n",
    "    if response.status_code != 200:\n",
    "        raise Exception('Failed to load page {}'.format(topic_url))\n",
    "    doc = BeautifulSoup(response.text, 'html.parser')\n",
    "    \n",
    "    topics_dict = {\n",
    "        'topic_title': get_topic_titles(doc),\n",
    "        'description': get_topic_descs(doc),\n",
    "        'url': get_topic_urls(doc)\n",
    "    }\n",
    "    return pd.DataFrame(topics_dict)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "09fb41db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3D</td>\n",
       "      <td>3D refers to the use of three-dimensional grap...</td>\n",
       "      <td>https://github.com/topics/3d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ajax</td>\n",
       "      <td>Ajax is a technique for creating interactive w...</td>\n",
       "      <td>https://github.com/topics/ajax</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Algorithm</td>\n",
       "      <td>Algorithms are self-contained sequences that c...</td>\n",
       "      <td>https://github.com/topics/algorithm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Amp</td>\n",
       "      <td>Amp is a non-blocking concurrency library for ...</td>\n",
       "      <td>https://github.com/topics/amphp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Android</td>\n",
       "      <td>Android is an operating system built by Google...</td>\n",
       "      <td>https://github.com/topics/android</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Angular</td>\n",
       "      <td>Angular is an open source web application plat...</td>\n",
       "      <td>https://github.com/topics/angular</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Ansible</td>\n",
       "      <td>Ansible is a simple and powerful automation en...</td>\n",
       "      <td>https://github.com/topics/ansible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>API</td>\n",
       "      <td>An API (Application Programming Interface) is ...</td>\n",
       "      <td>https://github.com/topics/api</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Arduino</td>\n",
       "      <td>Arduino is an open source platform for buildin...</td>\n",
       "      <td>https://github.com/topics/arduino</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ASP.NET</td>\n",
       "      <td>ASP.NET is a web framework for building modern...</td>\n",
       "      <td>https://github.com/topics/aspnet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Atom</td>\n",
       "      <td>Atom is a open source text editor built with w...</td>\n",
       "      <td>https://github.com/topics/atom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Awesome Lists</td>\n",
       "      <td>An awesome list is a list of awesome things cu...</td>\n",
       "      <td>https://github.com/topics/awesome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Amazon Web Services</td>\n",
       "      <td>Amazon Web Services provides on-demand cloud c...</td>\n",
       "      <td>https://github.com/topics/aws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Azure</td>\n",
       "      <td>Azure is a cloud computing service created by ...</td>\n",
       "      <td>https://github.com/topics/azure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Babel</td>\n",
       "      <td>Babel is a compiler for writing next generatio...</td>\n",
       "      <td>https://github.com/topics/babel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Bash</td>\n",
       "      <td>Bash is a shell and command language interpret...</td>\n",
       "      <td>https://github.com/topics/bash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Bitcoin</td>\n",
       "      <td>Bitcoin is a cryptocurrency developed by Satos...</td>\n",
       "      <td>https://github.com/topics/bitcoin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Bootstrap</td>\n",
       "      <td>Bootstrap is an HTML, CSS, and JavaScript fram...</td>\n",
       "      <td>https://github.com/topics/bootstrap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Bot</td>\n",
       "      <td>A bot is an application that runs automated ta...</td>\n",
       "      <td>https://github.com/topics/bot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C</td>\n",
       "      <td>C is a general purpose programming language th...</td>\n",
       "      <td>https://github.com/topics/c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Chrome</td>\n",
       "      <td>Chrome is a web browser from the tech company ...</td>\n",
       "      <td>https://github.com/topics/chrome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Chrome extension</td>\n",
       "      <td>Chrome extensions enable users to customize th...</td>\n",
       "      <td>https://github.com/topics/chrome-extension</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Command line interface</td>\n",
       "      <td>A CLI, or command-line interface, is a console...</td>\n",
       "      <td>https://github.com/topics/cli</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Clojure</td>\n",
       "      <td>Clojure is a dynamic, general-purpose programm...</td>\n",
       "      <td>https://github.com/topics/clojure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Code quality</td>\n",
       "      <td>Automate your code review with style, quality,...</td>\n",
       "      <td>https://github.com/topics/code-quality</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Code review</td>\n",
       "      <td>Ensure your code meets quality standards and s...</td>\n",
       "      <td>https://github.com/topics/code-review</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Compiler</td>\n",
       "      <td>Compilers are software that translate higher-l...</td>\n",
       "      <td>https://github.com/topics/compiler</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Continuous integration</td>\n",
       "      <td>Automatically build and test your code as you ...</td>\n",
       "      <td>https://github.com/topics/continuous-integration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>COVID-19</td>\n",
       "      <td>The coronavirus disease 2019 (COVID-19) is an ...</td>\n",
       "      <td>https://github.com/topics/covid-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C++</td>\n",
       "      <td>C++ is a general purpose and object-oriented p...</td>\n",
       "      <td>https://github.com/topics/cpp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               topic_title                                        description  \\\n",
       "0                       3D  3D refers to the use of three-dimensional grap...   \n",
       "1                     Ajax  Ajax is a technique for creating interactive w...   \n",
       "2                Algorithm  Algorithms are self-contained sequences that c...   \n",
       "3                      Amp  Amp is a non-blocking concurrency library for ...   \n",
       "4                  Android  Android is an operating system built by Google...   \n",
       "5                  Angular  Angular is an open source web application plat...   \n",
       "6                  Ansible  Ansible is a simple and powerful automation en...   \n",
       "7                      API  An API (Application Programming Interface) is ...   \n",
       "8                  Arduino  Arduino is an open source platform for buildin...   \n",
       "9                  ASP.NET  ASP.NET is a web framework for building modern...   \n",
       "10                    Atom  Atom is a open source text editor built with w...   \n",
       "11           Awesome Lists  An awesome list is a list of awesome things cu...   \n",
       "12     Amazon Web Services  Amazon Web Services provides on-demand cloud c...   \n",
       "13                   Azure  Azure is a cloud computing service created by ...   \n",
       "14                   Babel  Babel is a compiler for writing next generatio...   \n",
       "15                    Bash  Bash is a shell and command language interpret...   \n",
       "16                 Bitcoin  Bitcoin is a cryptocurrency developed by Satos...   \n",
       "17               Bootstrap  Bootstrap is an HTML, CSS, and JavaScript fram...   \n",
       "18                     Bot  A bot is an application that runs automated ta...   \n",
       "19                       C  C is a general purpose programming language th...   \n",
       "20                  Chrome  Chrome is a web browser from the tech company ...   \n",
       "21        Chrome extension  Chrome extensions enable users to customize th...   \n",
       "22  Command line interface  A CLI, or command-line interface, is a console...   \n",
       "23                 Clojure  Clojure is a dynamic, general-purpose programm...   \n",
       "24            Code quality  Automate your code review with style, quality,...   \n",
       "25             Code review  Ensure your code meets quality standards and s...   \n",
       "26                Compiler  Compilers are software that translate higher-l...   \n",
       "27  Continuous integration  Automatically build and test your code as you ...   \n",
       "28                COVID-19  The coronavirus disease 2019 (COVID-19) is an ...   \n",
       "29                     C++  C++ is a general purpose and object-oriented p...   \n",
       "\n",
       "                                                 url  \n",
       "0                       https://github.com/topics/3d  \n",
       "1                     https://github.com/topics/ajax  \n",
       "2                https://github.com/topics/algorithm  \n",
       "3                    https://github.com/topics/amphp  \n",
       "4                  https://github.com/topics/android  \n",
       "5                  https://github.com/topics/angular  \n",
       "6                  https://github.com/topics/ansible  \n",
       "7                      https://github.com/topics/api  \n",
       "8                  https://github.com/topics/arduino  \n",
       "9                   https://github.com/topics/aspnet  \n",
       "10                    https://github.com/topics/atom  \n",
       "11                 https://github.com/topics/awesome  \n",
       "12                     https://github.com/topics/aws  \n",
       "13                   https://github.com/topics/azure  \n",
       "14                   https://github.com/topics/babel  \n",
       "15                    https://github.com/topics/bash  \n",
       "16                 https://github.com/topics/bitcoin  \n",
       "17               https://github.com/topics/bootstrap  \n",
       "18                     https://github.com/topics/bot  \n",
       "19                       https://github.com/topics/c  \n",
       "20                  https://github.com/topics/chrome  \n",
       "21        https://github.com/topics/chrome-extension  \n",
       "22                     https://github.com/topics/cli  \n",
       "23                 https://github.com/topics/clojure  \n",
       "24            https://github.com/topics/code-quality  \n",
       "25             https://github.com/topics/code-review  \n",
       "26                https://github.com/topics/compiler  \n",
       "27  https://github.com/topics/continuous-integration  \n",
       "28                https://github.com/topics/covid-19  \n",
       "29                     https://github.com/topics/cpp  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scrape_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "b1bbfd2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_topics_repos():\n",
    "    print('Scraping list of topics')\n",
    "    topics_df = scrape_topics()\n",
    "    \n",
    "    os.makedirs('Github_topics', exist_ok=True)    \n",
    "    for index, row in topics_df.iterrows():\n",
    "        print('Scraping top repositories for \"{}\"'. format(row['topic_title']))\n",
    "        scrape_topic(row['url'], 'Github_topics/{}.csv'.format(row['topic_title']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "ee74eb47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping list of topics\n",
      "Scraping top repositories for \"3D\"\n",
      "Scraping top repositories for \"Ajax\"\n",
      "Scraping top repositories for \"Algorithm\"\n",
      "Scraping top repositories for \"Amp\"\n",
      "Scraping top repositories for \"Android\"\n",
      "Scraping top repositories for \"Angular\"\n",
      "Scraping top repositories for \"Ansible\"\n",
      "Scraping top repositories for \"API\"\n",
      "Scraping top repositories for \"Arduino\"\n",
      "Scraping top repositories for \"ASP.NET\"\n",
      "Scraping top repositories for \"Atom\"\n",
      "Scraping top repositories for \"Awesome Lists\"\n",
      "Scraping top repositories for \"Amazon Web Services\"\n",
      "Scraping top repositories for \"Azure\"\n",
      "Scraping top repositories for \"Babel\"\n",
      "Scraping top repositories for \"Bash\"\n",
      "Scraping top repositories for \"Bitcoin\"\n",
      "Scraping top repositories for \"Bootstrap\"\n",
      "Scraping top repositories for \"Bot\"\n",
      "Scraping top repositories for \"C\"\n",
      "Scraping top repositories for \"Chrome\"\n",
      "Scraping top repositories for \"Chrome extension\"\n",
      "Scraping top repositories for \"Command line interface\"\n",
      "Scraping top repositories for \"Clojure\"\n",
      "Scraping top repositories for \"Code quality\"\n",
      "Scraping top repositories for \"Code review\"\n",
      "Scraping top repositories for \"Compiler\"\n",
      "Scraping top repositories for \"Continuous integration\"\n",
      "Scraping top repositories for \"COVID-19\"\n",
      "Scraping top repositories for \"C++\"\n"
     ]
    }
   ],
   "source": [
    "scrape_topics_repos()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "51e88ade",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping list of topics\n",
      "Scraping top repositories for \"3D\"\n",
      "The file 3D.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Ajax\"\n",
      "The file Ajax.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Algorithm\"\n",
      "The file Algorithm.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Amp\"\n",
      "The file Amp.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Android\"\n",
      "The file Android.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Angular\"\n",
      "The file Angular.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Ansible\"\n",
      "The file Ansible.csv already exists. Skipping...\n",
      "Scraping top repositories for \"API\"\n",
      "The file API.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Arduino\"\n",
      "The file Arduino.csv already exists. Skipping...\n",
      "Scraping top repositories for \"ASP.NET\"\n",
      "The file ASP.NET.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Atom\"\n",
      "The file Atom.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Awesome Lists\"\n",
      "The file Awesome Lists.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Amazon Web Services\"\n",
      "The file Amazon Web Services.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Azure\"\n",
      "The file Azure.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Babel\"\n",
      "The file Babel.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Bash\"\n",
      "The file Bash.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Bitcoin\"\n",
      "The file Bitcoin.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Bootstrap\"\n",
      "The file Bootstrap.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Bot\"\n",
      "The file Bot.csv already exists. Skipping...\n",
      "Scraping top repositories for \"C\"\n",
      "The file C.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Chrome\"\n",
      "The file Chrome.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Chrome extension\"\n",
      "The file Chrome extension.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Command line interface\"\n",
      "The file Command line interface.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Clojure\"\n",
      "The file Clojure.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Code quality\"\n",
      "The file Code quality.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Code review\"\n",
      "The file Code review.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Compiler\"\n",
      "The file Compiler.csv already exists. Skipping...\n",
      "Scraping top repositories for \"Continuous integration\"\n",
      "The file Continuous integration.csv already exists. Skipping...\n",
      "Scraping top repositories for \"COVID-19\"\n",
      "The file COVID-19.csv already exists. Skipping...\n",
      "Scraping top repositories for \"C++\"\n",
      "The file C++.csv already exists. Skipping...\n"
     ]
    }
   ],
   "source": [
    "scrape_topics_repos()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c8d5a03",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}