Iyanuloluwa_scrapping_starter.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python [conda env:.conda-10x]",
      "language": "python",
      "name": "conda-env-.conda-10x-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "scrapping_starter.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dmarinere/10academysolution/blob/master/Iyanuloluwa_scrapping_starter.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CjpspbVS1FiJ",
        "colab_type": "text"
      },
      "source": [
        "## Web scrapping using python\n",
        "\n",
        "#### References\n",
        "1. [Practical Introduction to Web Scraping in Python](https://realpython.com/python-web-scraping-practical-introduction/)\n",
        "2. [Web Scraping using Python](https://www.datacamp.com/community/tutorials/web-scraping-using-python)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-7n21WJ1FiL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# $ python3 -m venv venv\n",
        "# $ . ./venv/bin/activate"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFJnUxburn6Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FIMF1I4y1FiR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        },
        "outputId": "e53d8b37-00de-47fc-f220-456699d3460f"
      },
      "source": [
        "#Better\n",
        "!pip install requests BeautifulSoup4 fire"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (2.23.0)\n",
            "Requirement already satisfied: BeautifulSoup4 in /usr/local/lib/python3.6/dist-packages (4.6.3)\n",
            "Collecting fire\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/34/a7/0e22e70778aca01a52b9c899d9c145c6396d7b613719cd63db97ffa13f2f/fire-0.3.1.tar.gz (81kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 3.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests) (2020.6.20)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from fire) (1.12.0)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from fire) (1.1.0)\n",
            "Building wheels for collected packages: fire\n",
            "  Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fire: filename=fire-0.3.1-py2.py3-none-any.whl size=111005 sha256=b31e5f68cfa2d52db579f5254bb42599799895ae8a0b99ff3a7d36bfb0bbf422\n",
            "  Stored in directory: /root/.cache/pip/wheels/c1/61/df/768b03527bf006b546dce284eb4249b185669e65afc5fbb2ac\n",
            "Successfully built fire\n",
            "Installing collected packages: fire\n",
            "Successfully installed fire-0.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ONq8zoln1FiW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from requests import get\n",
        "from requests.exceptions import RequestException\n",
        "from contextlib import closing\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import os, sys\n",
        "import re \n",
        "import fire"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CqrQK3xE1Fib",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#%%writefile ../pyscrap_url.py\n",
        "\n",
        "def simple_get(url):\n",
        "    \"\"\"\n",
        "    Attempts to get the content at `url` by making an HTTP GET request.\n",
        "    If the content-type of response is some kind of HTML/XML, return the\n",
        "    text content, otherwise return None.\n",
        "    \"\"\"\n",
        "    try:\n",
        "        with closing(get(url, stream=True)) as resp:\n",
        "            if is_good_response(resp):\n",
        "                return resp.content  #.encode(BeautifulSoup.original_encoding)\n",
        "            else:\n",
        "                return None\n",
        "\n",
        "    except RequestException as e:\n",
        "        log_error('Error during requests to {0} : {1}'.format(url, str(e)))\n",
        "        return None\n",
        "\n",
        "\n",
        "def is_good_response(resp):\n",
        "    \"\"\"\n",
        "    Returns True if the response seems to be HTML, False otherwise.\n",
        "    \"\"\"\n",
        "    content_type = resp.headers['Content-Type'].lower()\n",
        "    return (resp.status_code == 200 \n",
        "            and content_type is not None \n",
        "            and content_type.find('html') > -1)\n",
        "\n",
        "\n",
        "def log_error(e):\n",
        "    \"\"\"\n",
        "    It is always a good idea to log errors. \n",
        "    This function just prints them, but you can\n",
        "    make it do anything.\n",
        "    \"\"\"\n",
        "    print(e)\n",
        "    \n",
        "def get_elements(url, tag='',search={}, fname=None):\n",
        "    \"\"\"\n",
        "    Downloads a page specified by the url parameter\n",
        "    and returns a list of strings, one per tag element\n",
        "    \"\"\"\n",
        "    \n",
        "    if isinstance(url,str):\n",
        "        response = simple_get(url)\n",
        "    else:\n",
        "        #if already it is a loaded html page\n",
        "        response = url\n",
        "\n",
        "    if response is not None:\n",
        "        html = BeautifulSoup(response, 'html.parser')\n",
        "        \n",
        "        res = []\n",
        "        if tag:    \n",
        "            for li in html.select(tag):\n",
        "                for name in li.text.split('\\n'):\n",
        "                    if len(name) > 0:\n",
        "                        res.append(name.strip())\n",
        "                       \n",
        "                \n",
        "        if search:\n",
        "            soup = html            \n",
        "            \n",
        "            \n",
        "            r = ''\n",
        "            if 'find' in search.keys():\n",
        "                print('findaing',search['find'])\n",
        "                soup = soup.find(**search['find'])\n",
        "                r = soup\n",
        "\n",
        "                \n",
        "            if 'find_all' in search.keys():\n",
        "                print('findaing all of',search['find_all'])\n",
        "                r = soup.find_all(**search['find_all'])\n",
        "   \n",
        "            if r:\n",
        "                for x in list(r):\n",
        "                    if len(x) > 0:\n",
        "                        res.extend(x)\n",
        "            \n",
        "        return res\n",
        "\n",
        "    # Raise an exception if we failed to get any data from the url\n",
        "    raise Exception('Error retrieving contents at {}'.format(url))    \n",
        "    \n",
        "    \n",
        "if get_ipython().__class__.__name__ == '__main__':\n",
        "    fire(get_tag_elements)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WPB1CpTy1Fif",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "fe30ca08-821d-488b-e96e-1ef421a94958"
      },
      "source": [
        "res = get_elements('https://africafreak.com/100-most-influential-twitter-users-in-africa',tag='h2')\n",
        "res[:3]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['100. Jeffrey Gettleman (@gettleman)',\n",
              " '99. Africa24 Media (@a24media)',\n",
              " '98. Scapegoat (@andiMakinana)']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VU-dOlskYlT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "numbers = []\n",
        "for item in range(100):\n",
        "  number = res[item].split('.')\n",
        "  numbers.append(number)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2I5jX7A9uBKb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "2da4e17c-36bf-4f93-b11c-9a8ea3ea1479"
      },
      "source": [
        "numbers[:5]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['100', ' Jeffrey Gettleman (@gettleman)'],\n",
              " ['99', ' Africa24 Media (@a24media)'],\n",
              " ['98', ' Scapegoat (@andiMakinana)'],\n",
              " ['97', ' Africa Check (@AfricaCheck)'],\n",
              " ['96', ' James Copnall (@JamesCopnall)']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8X7dYy0D1uGq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "index = []\n",
        "for char in numbers:\n",
        "  index.append(char[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zE-N76L619EC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "141f2762-07e9-4210-f353-7112448544e7"
      },
      "source": [
        "index[:3]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['100', '99', '98']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UA-HhrOHkubl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "names =[]\n",
        "for name in numbers:\n",
        "  names.append(name[-1].split(\"(\"))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yrnKt4T7uFrt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7801727c-0081-40a5-d379-a692fe3aa961"
      },
      "source": [
        "names[:2]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[' Jeffrey Gettleman ', '@gettleman)'], [' Africa24 Media ', '@a24media)']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fYCceI7b2rt0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fullname = []\n",
        "for name in names:\n",
        "  fullname.append(name[0].strip())\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yhVHcdMI2_Sf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1a791163-77c2-4e2c-af10-ad6c8555bfe8"
      },
      "source": [
        "fullname[:3]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Jeffrey Gettleman', 'Africa24 Media', 'Scapegoat']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xXFHWvKt_9r7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "41e31283-beea-49a1-bd60-bfc7b588c063"
      },
      "source": [
        "names[:4]"
      ],
      "execution_count": 169,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[' Jeffrey Gettleman ', '@gettleman)'],\n",
              " [' Africa24 Media ', '@a24media)'],\n",
              " [' Scapegoat ', '@andiMakinana)'],\n",
              " [' Africa Check ', '@AfricaCheck)']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 169
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NvVxcdd6sNK0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "handle =[]\n",
        "for name in names:\n",
        "  handle.append(name[-1][:-1])"
      ],
      "execution_count": 150,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jkaVO19G3PlX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "86fdc171-dd0f-4f0c-95f8-2a70b7ce4c2a"
      },
      "source": [
        "handle[:3]\n"
      ],
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['@gettleman', '@a24media', '@andiMakinana']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LLqHmA-TV-YR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.DataFrame(list(zip(index,fullname, handle)), columns=['Number','Name', 'Handle'])"
      ],
      "execution_count": 115,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MkkZQ9n-FQhg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"Number\"]= df.Number.astype(int)"
      ],
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bFGmaeRYySjl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.sort_values('Number', inplace=True)"
      ],
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wq5akpOjWRpS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.set_index('Number', inplace=True)"
      ],
      "execution_count": 118,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RbCfX3RGF689",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "84c77ae8-3af8-4023-8e59-8ac5159abad0"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Handle</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Number</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Trevor Noah</td>\n",
              "      <td>@Trevornoah</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Gareth Cliff</td>\n",
              "      <td>@GarethCliff</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Zuma</td>\n",
              "      <td>@SAPresident</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>News24</td>\n",
              "      <td>@News24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Julius Sello Malema</td>\n",
              "      <td>@Julius_S_Malema</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                       Name            Handle\n",
              "Number                                       \n",
              "1               Trevor Noah       @Trevornoah\n",
              "2              Gareth Cliff      @GarethCliff\n",
              "3                      Zuma      @SAPresident\n",
              "4                    News24           @News24\n",
              "5       Julius Sello Malema  @Julius_S_Malema"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KDdxGZQW1Fim",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "url= 'https://www.atlanticcouncil.org/blogs/africasource/african-leaders-respond-to-coronavirus-on-twitter/#east-africa'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CpghvHNVPrcf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "9b6f9ec8-9fea-4068-846c-8ad6984c5650"
      },
      "source": [
        "  response = simple_get(url)\n",
        "  res = get_elements(response, search={'find_all':{'class_':'wp-block-embed__wrapper'}})\n",
        "  pattern = \"@[a-zA-Z]+\"\n",
        "  twitter_handle = re.findall(pattern, str(res))\n",
        "  print(\"The total number of African leaders extracted\", len(twitter_handle))\n",
        "  twitter_handle[:5]"
      ],
      "execution_count": 176,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "findaing all of {'class_': 'wp-block-embed__wrapper'}\n",
            "The total number of African leaders extracted 52\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['@EswatiniGovern',\n",
              " '@MalawiGovt',\n",
              " '@hagegeingob',\n",
              " '@FinanceSC',\n",
              " '@PresidencyZA']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 176
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l09Y2OXD1Fi1",
        "colab_type": "text"
      },
      "source": [
        "## Web scrapping using bash script\n",
        "If the web site has a quite simple HTML, you can easily use curl to perform the request and then extract the needed values using bash commands grep, cut , sed, ..\n",
        "\n",
        "This tutorial is adapted from [this](https://medium.com/@LiliSousa/web-scraping-with-bash-690e4ee7f98d) medium article"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y6dX6t2wFggl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%bash\n",
        "curl https://www.atlanticcouncil.org/blogs/africasource/african-leaders-respond-to-coronavirus-on-twitter/#east-africa > tmp_file\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QTEa_IJG1Fi2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%bash \n",
        "#curl the page and save content to tmp_file\n",
        "url = https://www.atlanticcouncil.org/blogs/africasource/african-leaders-respond-to-coronavirus-on-twitter/#east-africa\n",
        "curl -X GET $url -o tmp_file\n",
        "\n",
        "\n",
        "#!/bin/bash\n",
        "\n",
        "# write headers to CSV file\n",
        "echo \"Name, twitter_id\" >> extractData.csv\n",
        "n=\"1\"\n",
        "while [ $n -lt 2 ]\n",
        "do\n",
        "  \n",
        "  #get title\n",
        "  title=$(cat tmp_file | grep \"class=\\\"css-901oao css-16my406 r-1qd0xha r-ad9z0x r-bcqeeo r-qvutc0\\\" | cut -d ';' -f1 )\n",
        "  \n",
        "  #get auth\n",
        "  twitter_id=$(cat tmp_file |grep -A1 \"class=\\\"css-1dbjc4n r-18u37iz r-1wbh5a2\"\" | tail -1)\n",
        "\n",
        "  echo \"$title, $twitter_id\" >> extractData.csv\n",
        "\n",
        "  n=$[$n+1]\n",
        "\n",
        "done"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qzwJg9urxGiw",
        "colab_type": "text"
      },
      "source": [
        "### Twitter Scraping to find the follower counts and most used Hastags"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Rp8bRK44Tg9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tweepy\n",
        "from tweepy.streaming import StreamListener\n",
        "from tweepy import OAuthHandler\n",
        "from tweepy import Stream\n"
      ],
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vrr15Npbx3A7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        },
        "outputId": "26830138-35f7-4d5e-a2dc-1545364e0810"
      },
      "source": [
        "!pip install tweepy\n",
        "!pip install locale\n",
        "!pip install threading"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tweepy in /usr/local/lib/python3.6/dist-packages (3.6.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tweepy) (1.12.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tweepy) (1.3.0)\n",
            "Requirement already satisfied: PySocks>=1.5.7 in /usr/local/lib/python3.6/dist-packages (from tweepy) (1.7.1)\n",
            "Requirement already satisfied: requests>=2.11.1 in /usr/local/lib/python3.6/dist-packages (from tweepy) (2.23.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->tweepy) (3.1.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.11.1->tweepy) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.11.1->tweepy) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.11.1->tweepy) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.11.1->tweepy) (2.10)\n",
            "\u001b[31mERROR: Could not find a version that satisfies the requirement locale (from versions: none)\u001b[0m\n",
            "\u001b[31mERROR: No matching distribution found for locale\u001b[0m\n",
            "\u001b[31mERROR: Could not find a version that satisfies the requirement threading (from versions: none)\u001b[0m\n",
            "\u001b[31mERROR: No matching distribution found for threading\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u-Cz90_l8Mo8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import logging\n",
        "import os\n",
        "\n",
        "logger = logging.getLogger()"
      ],
      "execution_count": 84,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "psUTngLq7hw4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_api():\n",
        "  consumer_key = os.environ.get('consumer_key')\n",
        "  consumer_secret = os.environ.get('consumer_secret')\n",
        "  access_token = os.environ.get('access_token')\n",
        "  access_token_secret = os.environ.get('access_token_secret')\n",
        "  auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
        "  auth.set_access_token(access_token, access_token_secret)\n",
        "  api = tweepy.API(auth, wait_on_rate_limit=True,\n",
        "                     wait_on_rate_limit_notify=True)\n",
        "  try:\n",
        "    api.verify_credentials()\n",
        "  except Exception as e:\n",
        "    logger.error(\"Error creating API\", exc_info=True)\n",
        "    raise e\n",
        "    logger.info(\"API created\")\n",
        "    return api"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "drIB42K412oK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "followers_count = {}\n",
        "for count in range(len(handle)):\n",
        "  try:\n",
        "    user = api.get_user(handle[count])\n",
        "    followers_count[handle[count]]= user.followers_count\n",
        "  except Exception as e:\n",
        "    pass"
      ],
      "execution_count": 151,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fo9eTD5MOKNw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "448b6d3c-a224-4521-8285-1f7f3e424fe1"
      },
      "source": [
        "len(followers_count)"
      ],
      "execution_count": 157,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "93"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 157
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6HCaHUvzTPJS",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PnOANzZ3yPOQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['Follower_count']= df['Handle'].map(followers_count)"
      ],
      "execution_count": 212,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BnydDM5tNTPY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.sort_values('Follower_count', inplace=True, ascending=False)"
      ],
      "execution_count": 213,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sA_FXrY2bSpV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "7c71519e-7f82-4d20-a841-c830d96bc946"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 214,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Handle</th>\n",
              "      <th>Follower_count</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Number</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Trevor Noah</td>\n",
              "      <td>@Trevornoah</td>\n",
              "      <td>10798919.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>News24</td>\n",
              "      <td>@News24</td>\n",
              "      <td>3573060.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Julius Sello Malema</td>\n",
              "      <td>@Julius_S_Malema</td>\n",
              "      <td>3123283.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Gareth Cliff</td>\n",
              "      <td>@GarethCliff</td>\n",
              "      <td>1974337.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Euphonik™♛</td>\n",
              "      <td>@euphonik</td>\n",
              "      <td>1753274.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                       Name            Handle  Follower_count\n",
              "Number                                                       \n",
              "1               Trevor Noah       @Trevornoah      10798919.0\n",
              "4                    News24           @News24       3573060.0\n",
              "5       Julius Sello Malema  @Julius_S_Malema       3123283.0\n",
              "2              Gareth Cliff      @GarethCliff       1974337.0\n",
              "15               Euphonik™♛         @euphonik       1753274.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 214
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDjUJ4pUQZBv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "first_ten = df.head(10)"
      ],
      "execution_count": 215,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zuEKRuY-WI6G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " first_ten.to_csv (\"/content/drive/My Drive/Colab Notebooks/African_influencer.csv\", index = False, header=True)"
      ],
      "execution_count": 188,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cEKWcuT6Nvte",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "african_follower = {}\n",
        "new_one ={}\n",
        "fullname = []\n",
        "for count in range(len(twitter_handle)):\n",
        "  try:\n",
        "    user = api.get_user(twitter_handle[count])\n",
        "    african_follower[twitter_handle[count]]= user.followers_count\n",
        "    new_one[twitter_handle[count]]= user.name\n",
        "\n",
        "    \n",
        "  except Exception as e:\n",
        "    pass"
      ],
      "execution_count": 216,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "048oUFcqVU3T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2 = pd.DataFrame(twitter_handle, columns=['Twitter Handle'])"
      ],
      "execution_count": 197,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFadqJekZMKu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2['Follower_count']= df2['Twitter Handle'].map(african_follower)"
      ],
      "execution_count": 209,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oRotfJvjbvAI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2[\"Name\"] = df2['Twitter Handle'].map(new_one)"
      ],
      "execution_count": 217,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3uWjQaFZNJJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2.sort_values('Follower_count', inplace=True, ascending=False)"
      ],
      "execution_count": 219,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fw5VDPLdFR2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df2.drop_duplicates(inplace=True) "
      ],
      "execution_count": 225,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ha59P0vjgpbn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "outputId": "60d014d5-3c19-4ea1-c394-02a5eba51f9d"
      },
      "source": [
        "df2.head(10)"
      ],
      "execution_count": 230,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Twitter Handle</th>\n",
              "      <th>Follower_count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>@WHO</td>\n",
              "      <td>8090689.0</td>\n",
              "      <td>World Health Organization (WHO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>@MBuhari</td>\n",
              "      <td>3266027.0</td>\n",
              "      <td>Muhammadu Buhari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>@BorisJohnson</td>\n",
              "      <td>2905525.0</td>\n",
              "      <td>Boris Johnson #StayAlert</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>@PaulKagame</td>\n",
              "      <td>1979494.0</td>\n",
              "      <td>Paul Kagame</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>@KagutaMuseveni</td>\n",
              "      <td>1807478.0</td>\n",
              "      <td>Yoweri K Museveni</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>@PresidencyZA</td>\n",
              "      <td>1596127.0</td>\n",
              "      <td>Presidency | South Africa 🇿🇦</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>@NAkufoAddo</td>\n",
              "      <td>1502376.0</td>\n",
              "      <td>Nana Akufo-Addo</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>@CyrilRamaphosa</td>\n",
              "      <td>1481545.0</td>\n",
              "      <td>Cyril Ramaphosa 🇿🇦 #StaySafe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>@StateHouseKenya</td>\n",
              "      <td>1101220.0</td>\n",
              "      <td>State House Kenya</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>@MagufuliJP</td>\n",
              "      <td>865395.0</td>\n",
              "      <td>Dr John Magufuli</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Twitter Handle  Follower_count                             Name\n",
              "16              @WHO       8090689.0  World Health Organization (WHO)\n",
              "47          @MBuhari       3266027.0                 Muhammadu Buhari\n",
              "22     @BorisJohnson       2905525.0         Boris Johnson #StayAlert\n",
              "48       @PaulKagame       1979494.0                      Paul Kagame\n",
              "29   @KagutaMuseveni       1807478.0                Yoweri K Museveni\n",
              "4      @PresidencyZA       1596127.0     Presidency | South Africa 🇿🇦\n",
              "41       @NAkufoAddo       1502376.0                  Nana Akufo-Addo\n",
              "21   @CyrilRamaphosa       1481545.0     Cyril Ramaphosa 🇿🇦 #StaySafe\n",
              "17  @StateHouseKenya       1101220.0                State House Kenya\n",
              "19       @MagufuliJP        865395.0                 Dr John Magufuli"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 230
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xjNyYMZGgkH5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "topten_african_leader = df.head(10)"
      ],
      "execution_count": 232,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U9TC-BKscYs3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "topten_african_leader.to_csv(\"/content/drive/My Drive/Colab Notebooks/African_leaders.csv\", index = False, header=True)"
      ],
      "execution_count": 233,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEyu5v1bZaNH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "40410fb8-1a0b-483e-c248-a5ff2ce35e97"
      },
      "source": [
        "df2.head()"
      ],
      "execution_count": 234,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Twitter Handle</th>\n",
              "      <th>Follower_count</th>\n",
              "      <th>Name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>@WHO</td>\n",
              "      <td>8090689.0</td>\n",
              "      <td>World Health Organization (WHO)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>@MBuhari</td>\n",
              "      <td>3266027.0</td>\n",
              "      <td>Muhammadu Buhari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>@BorisJohnson</td>\n",
              "      <td>2905525.0</td>\n",
              "      <td>Boris Johnson #StayAlert</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>@PaulKagame</td>\n",
              "      <td>1979494.0</td>\n",
              "      <td>Paul Kagame</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>@KagutaMuseveni</td>\n",
              "      <td>1807478.0</td>\n",
              "      <td>Yoweri K Museveni</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Twitter Handle  Follower_count                             Name\n",
              "16             @WHO       8090689.0  World Health Organization (WHO)\n",
              "47         @MBuhari       3266027.0                 Muhammadu Buhari\n",
              "22    @BorisJohnson       2905525.0         Boris Johnson #StayAlert\n",
              "48      @PaulKagame       1979494.0                      Paul Kagame\n",
              "29  @KagutaMuseveni       1807478.0                Yoweri K Museveni"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 234
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v4_eBDJlcBnA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}