Roadmap_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Web Scrapping \n",
    "### 'Roadmap, Feedback Period: Artificial Intelligence - ethical and legal requirements' (*N* = 133) \n",
    "This code: \n",
    "- crawles data from the EC website (i.e., pdf-documents)\n",
    "- extracts text data from the pdfs\n",
    "- translates non-English text into English\n",
    "- merges extracted text-from-pdf with meta-data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install pdfplumber"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install deep-translator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver             # importing selenium webdriver\n",
    "from csv import writer                     # import writer for inserting data into csv later on\n",
    "from bs4 import BeautifulSoup              # used to format or edit scraped data\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import time                                # wait upto 3 seconds when data is being scrapped\n",
    "\n",
    "options = Options()\n",
    "options.headless = True                    #get the data from EC site without opening goolge chrome window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-25-26a654b260ec>:1: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome('C:/Users/batzdova/Desktop/chromedriver', options=options)       #location where the installed file for selenium web driver is.\n"
     ]
    }
   ],
   "source": [
    "driver = webdriver.Chrome('C:/Users/batzdova/Desktop/chromedriver', options=options)       #location where the installed file for selenium web driver is.    \n",
    "soup = list()\n",
    "for i in range(0,14):                                                                     #with 31 p. to scrape containg 304 unique docs, will scrape each site for getting the links of all 304 pages. \n",
    "    if(i==0):\n",
    "        driver.get(\"https://ec.europa.eu/info/law/better-regulation/have-your-say/initiatives/12527-Kunstliche-Intelligenz-ethische-und-rechtliche-Anforderungen/feedback_de?p_id=8242911\")\n",
    "    else:\n",
    "        driver.get(\"https://ec.europa.eu/info/law/better-regulation/have-your-say/initiatives/12527-Kunstliche-Intelligenz-ethische-und-rechtliche-Anforderungen/feedback_de?p_id=8242911&page=\"+str(i))\n",
    "    time.sleep(3)                                                                         #Used to add wait of 3 secs so that scrapping even on slow connection. Incase of >304 document than you must add wait of 4/ 5 secs.\n",
    "    soup.append(BeautifulSoup(driver.page_source, 'html'))                                #Get the data into readable format and append it into soup list.\n",
    "driver.quit() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linkData = list()                              # variable for <a> tag reference.\n",
    "# linkData = soup.find_all('a',{'class':'ecl-link'},href=True)\n",
    "for i in soup:\n",
    "    linkData.append(i.find_all('a',href=True)) # Get all the <a> tags from soup list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get only the required references from links which start with F5\n",
    "# for example a unique site has address https://ec.europa.eu/info/law/better-regulation/have-your-say/initiatives/12527-Artificial-intelligence-ethical-and-legal-requirements/F243432'\n",
    "# And we need only the last string of this address for download and scraping document.\n",
    "tagid = list()\n",
    "for j in linkData:\n",
    "    for i in j:\n",
    "        test = i['href'].split('/')\n",
    "        if test:\n",
    "            if test[-1].startswith(\"F5\"):\n",
    "                tagid.append(test[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "133"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tagid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data retrieval\n",
    "##### + download each file from the EC site and rename it \n",
    "##### + save it into directory\n",
    "###### (Run this code only when you haven't downloaded the files, yet.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-19-315688472213>:11: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome('C:/Users/batzdova/Desktop/chromedriver')\n"
     ]
    }
   ],
   "source": [
    "from selenium.webdriver.common.by import By\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "import os\n",
    "import glob\n",
    "from pathlib import Path\n",
    "\n",
    "for idd in tagid: #call references for 304 page\n",
    "    driver = webdriver.Chrome('C:/Users/batzdova/Desktop/chromedriver')\n",
    "    driver.get('https://ec.europa.eu/info/law/better-regulation/have-your-say/initiatives/12527-Kunstliche-Intelligenz-ethische-und-rechtliche-Anforderungen/'+str(idd))\n",
    "    try: # Download the specific file if found on a page with HTML CLASS NAME ecl-file__download\n",
    "        download_file = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, \"ecl-file__download\"))) # check if class(downloading document) exist.\n",
    "        driver.execute_script(\"arguments[0].click();\", download_file) #click on class ecl-file__download\n",
    "        time.sleep(3)\n",
    "    except:\n",
    "        continue\n",
    "    PDF_DIR = Path(\"C:/Users/batzdova/Desktop/\") #Path to save downloaded file.\n",
    "    PDF_PATTERN = r'*.pdf'\n",
    "    latest_file = max(PDF_DIR.glob(PDF_PATTERN), key=lambda f: f.stat().st_ctime,default=0)\n",
    "    PDF_new_path = Path(\"C:/Users/batzdova/Desktop/\" + str(idd + \".pdf\"))\n",
    "    os.rename(latest_file , PDF_new_path) # rename file and give a reference name to it for example if file name is 34knjo34n2nj34.pdf than that renamed it to F213422_en.pdf(reference name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Save Data into CSV\n",
    "\n",
    "##### + save data it into a csv-file. \n",
    "##### + merge text from the downloaded pdf filewith meta-data in csv-file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pdfplumber\n",
    "from deep_translator import GoogleTranslator\n",
    "data1 = list()\n",
    "with open(\"scrap3.csv\",\"w\",newline=\"\",encoding=\"utf-8\") as f:                                                    # Ceate a new file if does not exist, or if exist rewrite data\n",
    "    newWriter = writer(f) # Create new writeer\n",
    "    header = [\"Feedback reference\",\"Submitted on\",\"Submitted by\",\"User type\",\"Organisation\",\"Organisation size\",\n",
    "              \"Transparency register number\",\"Country of origin\",\"Initiative\",\"Paragraph\",\"pdf\"]                 # Column Values\n",
    "    newWriter.writerow(header) # Create Columns\n",
    "    \n",
    "    driver = webdriver.Chrome('C:/Users/batzdova/Desktop/chromedriver')                                          # driver location\n",
    "    for idd in tagid: # call each unique reference for 304 pages\n",
    "        driver.get('https://ec.europa.eu/info/law/better-regulation/have-your-say/initiatives/12527-Kunstliche-Intelligenz-ethische-und-rechtliche-Anforderungen/'+str(idd))\n",
    "        time.sleep(3) \n",
    "        soupnew = BeautifulSoup(driver.page_source, 'html.parser')                                               # get all page content\n",
    "        linkDatanew = soupnew.find_all(\"div\", {\"class\": \"ecl-u-mb-m\"})                                           # get a div with class name ecl-u-mb-m\n",
    "        linkpara = soupnew.find(\"p\",{\"class\" : \"ecl-u-type-paragraph-m\"})                                        # get a div with class name ecl-u-type-paragraph-m for paragraph csv data\n",
    "        linkcsv = str(soupnew.find_all(\"div\", {\"class\":\"ng-tns-c188-0\"}))                                        # get a div with class name ng-tns-c188-0 for other columns e-g submitted-by\n",
    "        # Finding the specific data in linkcsv variable, if not found than wrote - into that row\n",
    "        infoList = list()\n",
    "        for i in linkDatanew:\n",
    "            infoList.append(str(i.get_text().strip()))\n",
    "        infoList.append(linkpara.get_text() if linkpara else \"-\")\n",
    "        if \"Eingereicht am\" not in linkcsv:\n",
    "            infoList.insert(1,\"-\")\n",
    "        if \"Eingereicht von\" not in linkcsv:\n",
    "            infoList.insert(2,\"-\")\n",
    "        if \"Nutzertyp\" not in linkcsv:\n",
    "            infoList.insert(3,\"-\")\n",
    "        if \"Organisation\" not in linkcsv:\n",
    "            infoList.insert(4,\"-\")\n",
    "        if \"Organisationsgröße\" not in linkcsv:\n",
    "            infoList.insert(5,\"-\")\n",
    "        if \"Transparenzregisternummer\" not in linkcsv:\n",
    "            infoList.insert(6,\"-\")\n",
    "        if \"Herkunftsland\" not in linkcsv:\n",
    "            infoList.insert(7,\"-\")\n",
    "        if \"Initiative\" not in linkcsv:\n",
    "            infoList.insert(8,\"-\")\n",
    "        \n",
    "        try: # try method will extract text from pdf documents and save it into csv file\n",
    "            with pdfplumber.open('C:/Users/batzdova/Desktop/Roadmap/files/'+str(idd)+'.pdf') as temp:\n",
    "                data1 = list()\n",
    "                translated = list()\n",
    "                num = temp.pages\n",
    "                for i,pg in enumerate(num):\n",
    "                    page = temp.pages[i]\n",
    "                    page1 = page.extract_text()\n",
    "                    data1.append(page1)\n",
    "                for i in data1:\n",
    "                    translated.append(GoogleTranslator(source='auto', target='en').translate(i))\n",
    "                print(temp)\n",
    "                infoList.insert(10, translated) # save the document text into infoList\n",
    "        except:\n",
    "            continue\n",
    "        finally:\n",
    "            newWriter.writerow(infoList) # write the infoList into csv file."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}