1_crawling_data_g20.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install dependencies\n",
    "# %pip install --user --upgrade git+https://github.com/twintproject/twint.git@origin/master#egg=twint\n",
    "# %pip install aiohttp==3.7.0\n",
    "# %pip install nest_asyncio\n",
    "# %pip install pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Libraries\n",
    "import twint\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "import time\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "timestr = time.strftime(\"%Y%m%d\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Konfigurasi Crawling dengan Twint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#configuration\n",
    "c = twint.Config()   \n",
    "c.Limit = 1000 # How much data needed       \n",
    "c.Lang = \"en\" # Language\n",
    "c.Since = \"2021-01-01\"\n",
    "c.Until = \"2022-10-07\"\n",
    "c.Hide_output = True\n",
    "c.Store_csv = True\n",
    "c.Search = \"ktt g20 bali\"  # key words to look for.\n",
    "# c.Output = timestr + \"_id_g20.csv\"\n",
    "c.Pandas = True\n",
    "c.Debug = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start proses crawling dari twitter dengan mengambil data 'Date', 'Username' dan 'Tweet'-nya itu sendiri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start crawling\n",
      "[!] No more data! Scraping will stop now.\n",
      "found 0 deleted tweets in this search.\n",
      "[!] No more data! Scraping will stop now.\n",
      "found 0 deleted tweets in this search.\n",
      "Finished\n",
      "-------\n",
      "CPU times: total: 562 ms\n",
      "Wall time: 22.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "try:\n",
    "  print(\"Start crawling\")\n",
    "\n",
    "  # Start crawling\n",
    "  twint.run.Search(c)\n",
    "  # convert output to dataframe format\n",
    "  data = twint.output.panda.Tweets_df[['date', 'username', 'tweet']]\n",
    "except Exception as e:\n",
    "  print(e)\n",
    "  \n",
    "print(\"Finished\")\n",
    "print(\"-------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preview hasil crawling data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>username</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2022-10-05 12:21:14</td>\n",
       "      <td>jeslintuse2</td>\n",
       "      <td>Nusa Tenggara Timur Pemerintah Pastikan Semua ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-10-05 12:17:55</td>\n",
       "      <td>usharanimuni</td>\n",
       "      <td>Pemerintah pastikan hotel di Bali siap sambut ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-10-05 12:13:03</td>\n",
       "      <td>diaharmayanti1</td>\n",
       "      <td>Pemerintah pastikan hotel di bali siap sambut ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2022-10-05 12:12:31</td>\n",
       "      <td>SeptianiMestika</td>\n",
       "      <td>Bali Compendium akan dibawa Indonesia ke KTT G...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2022-10-05 12:11:59</td>\n",
       "      <td>diaharmayanti1</td>\n",
       "      <td>Kesiapan jaringan telekomunikasi indonesia men...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1061</th>\n",
       "      <td>2022-10-02 11:21:39</td>\n",
       "      <td>RasyaMahardika7</td>\n",
       "      <td>Dukung dan sukseskan KTT G20 di Bali.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1062</th>\n",
       "      <td>2022-10-02 09:22:10</td>\n",
       "      <td>kalla_jengking</td>\n",
       "      <td>Persiapan KTT G20 on the Right Track  Menlu Re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>2022-10-02 09:11:02</td>\n",
       "      <td>WWishfm107</td>\n",
       "      <td>Dukung KTT G20 Bali Indonesia siap ditengah de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1064</th>\n",
       "      <td>2022-10-02 09:09:15</td>\n",
       "      <td>garuda080</td>\n",
       "      <td>Menjelang pertemuan KTT G20, Menlu Retno menga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1065</th>\n",
       "      <td>2022-10-02 09:08:18</td>\n",
       "      <td>Anggita_lung</td>\n",
       "      <td>Menlu Retno Marsudi mengatakan persiapan Indon...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1066 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     date         username  \\\n",
       "0     2022-10-05 12:21:14      jeslintuse2   \n",
       "1     2022-10-05 12:17:55     usharanimuni   \n",
       "2     2022-10-05 12:13:03   diaharmayanti1   \n",
       "3     2022-10-05 12:12:31  SeptianiMestika   \n",
       "4     2022-10-05 12:11:59   diaharmayanti1   \n",
       "...                   ...              ...   \n",
       "1061  2022-10-02 11:21:39  RasyaMahardika7   \n",
       "1062  2022-10-02 09:22:10   kalla_jengking   \n",
       "1063  2022-10-02 09:11:02       WWishfm107   \n",
       "1064  2022-10-02 09:09:15        garuda080   \n",
       "1065  2022-10-02 09:08:18     Anggita_lung   \n",
       "\n",
       "                                                  tweet  \n",
       "0     Nusa Tenggara Timur Pemerintah Pastikan Semua ...  \n",
       "1     Pemerintah pastikan hotel di Bali siap sambut ...  \n",
       "2     Pemerintah pastikan hotel di bali siap sambut ...  \n",
       "3     Bali Compendium akan dibawa Indonesia ke KTT G...  \n",
       "4     Kesiapan jaringan telekomunikasi indonesia men...  \n",
       "...                                                 ...  \n",
       "1061              Dukung dan sukseskan KTT G20 di Bali.  \n",
       "1062  Persiapan KTT G20 on the Right Track  Menlu Re...  \n",
       "1063  Dukung KTT G20 Bali Indonesia siap ditengah de...  \n",
       "1064  Menjelang pertemuan KTT G20, Menlu Retno menga...  \n",
       "1065  Menlu Retno Marsudi mengatakan persiapan Indon...  \n",
       "\n",
       "[1066 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# menampilkan dataframe\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Export dataframe to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert to CSV & Excel\n",
    "data.to_csv('data_raw_g20.csv', index = False)\n",
    "# data.to_excel(r'data_raw_g20.xlsx', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "81794d4967e6c3204c66dcd87b604927b115b27c00565d3d43f05ba2f3a2cb0d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}