UNscrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlopen\n",
    "import re\n",
    "import wget\n",
    "import requests\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.common.exceptions import WebDriverException\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import Select\n",
    "import json\n",
    "import os\n",
    "import time\n",
    "import csv\n",
    "import sys\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getLinks(url):\n",
    "    html_page = urlopen(url)\n",
    "    soup = BeautifulSoup(html_page)\n",
    "    links = []\n",
    " \n",
    "    for link in soup.findAll('a', attrs={'href': re.compile(\"^http://daccess-ods.un.org/\")}):\n",
    "        fullurl = link.get('href')\n",
    "        if fullurl.endswith('Lang=E'):\n",
    "            links.append(link.get('href'))\n",
    " \n",
    "    return links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0'}\n",
    "link_list = []\n",
    "\n",
    "for i in range(1, 19): #number of pages\n",
    "    url = \"http://unbisnet.un.org:8080/ipac20/ipac.jsp?session=1TO268794774E.1154423&profile=speech&page=%s\" % i + \"&group=0&term=%22United+Kingdom%22&index=.SC&uindex=&aspect=power&menu=search&ri=2&limitbox_1=SI01+%3D+si_s&ultype=PD01&uloper=%3E&ullimit=1993&ultype=PD01&uloper=%3C&ullimit=2011&source=~!horizon&1532687982385\"\n",
    "    l = getLinks(url)\n",
    "    link_list.append(l)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flat_list = [item for sublist in link_list for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"UK/1994-2010.txt\", 'w') as file_handler:\n",
    "    for item in flat_list:\n",
    "        file_handler.write(\"{}\\n\".format(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#in order to download stuff without having to handle the popup, the following options need to be set in the browser profile:\n",
    "profile = webdriver.FirefoxProfile()\n",
    "profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'application/pdf,application/x-pdf')\n",
    "#tells the browser not to use the default download folder\n",
    "profile.set_preference('browser.download.folderList', 2)\n",
    "#don't show progress bar\n",
    "profile.set_preference('browser.download.manager.showWhenStarting', False)\n",
    "#save to a specific folder\n",
    "profile.set_preference('browser.download.dir', '/Users/zhanna.terechshenko/Dropbox/Dissertation/UK/data')\n",
    "profile.set_preference(\"browser.download.useDownloadDir\", True)\n",
    "profile.set_preference(\"pdfjs.disabled\", True)\n",
    "\n",
    "\n",
    "browser = webdriver.Firefox(firefox_profile=profile)\n",
    "browser.set_page_load_timeout(25)\n",
    "\n",
    "for k in flat_list[786:]:\n",
    "    browser.get(k)\n",
    "    time.sleep(3)\n",
    "    elem = browser.find_element_by_id('download')\n",
    "    elem.click()\n",
    "    time.sleep(3)\n",
    "#go to the website\n",
    "#url = \"http://daccess-ods.un.org/access.nsf/Get?Open&DS=S/PV.8146&Lang=E\"\n",
    "#browser.get(url)\n",
    "\n",
    "#elem = browser.find_element_by_id('download')\n",
    "#elem.click()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "elem = browser.find_element_by_id('download')\n",
    "elem.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}