Scraping Museums.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "\n",
    "# making a list of all museums by regions as Tripadvisor doesn't have a single place with all the museums of a country.\n",
    "regions_list = [\n",
    "    \"https://www.tripadvisor.fr/Attractions-g11038913-Activities-c49-Auvergne_Rhone_Alpes.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g11038854-Activities-c49-Bourgogne_Franche_Comte.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187094-Activities-c49-Brittany.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187115-Activities-c49-Centre_Val_de_Loire.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187139-Activities-c49-Corsica.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g11038876-Activities-c49-Grand_Est.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g11038886-Activities-c49-Hauts_de_France.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187144-Activities-c49-Ile_de_France.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187179-Activities-c49-Normandy.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g11038861-Activities-c49-Nouvelle_Aquitaine.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g11038881-Activities-c49-Occitanie.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187192-Activities-c49-Pays_de_la_Loire.html\",\n",
    "    \"https://www.tripadvisor.fr/Attractions-g187208-Activities-c49-Provence_Alpes_Cote_d_Azur.html\"\n",
    "]\n",
    "\n",
    "#function to get the html from a page\n",
    "def get_soup(url):\n",
    "    response = requests.get(url)\n",
    "    r = response.content\n",
    "    soup = BeautifulSoup(r, \"html.parser\")\n",
    "    return soup\n",
    "        \n",
    "#function to check if there is another page after the current one\n",
    "def next_page():\n",
    "    if soup.find(class_=\"ui_button nav next primary\"):\n",
    "        url = \"https://www.tripadvisor.fr\" + soup.find(class_=\"ui_button nav next primary\")[\"href\"]\n",
    "        return url\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "# coordinates are only available through javascript code\n",
    "def get_coordinates():    \n",
    "    scripts = soup.find_all('script')\n",
    "    for script in scripts:\n",
    "        if 'window.__WEB_CONTEXT__=' in script.text:\n",
    "            jsonStr = script.text\n",
    "            jsonStr = jsonStr.split('window.__WEB_CONTEXT__={pageManifest:')[-1]\n",
    "    \n",
    "    try:\n",
    "        lat = re.findall('\"latitude\":\"(.*?)\"', jsonStr)\n",
    "        lon = re.findall('\"longitude\":\"(.*?)\"', jsonStr)\n",
    "\n",
    "        if len(lat)>0:\n",
    "            return (lat[0], lon[0])\n",
    "        else:\n",
    "            return \"Nan\"\n",
    "    except:\n",
    "        return \"Nan\"\n",
    "\n",
    "# main function where I gather all the informations I want \n",
    "def scrape_page():\n",
    "    #scraping name of the city\n",
    "    if soup.find(class_=\"eQSJNhO6\") != None:\n",
    "        city = soup.find(class_=\"eQSJNhO6\").find(\"a\").text\n",
    "    else:\n",
    "        city=\"NaN\"\n",
    "    city_list.append(city)\n",
    "    \n",
    "    if soup.find(class_=\"ui_header h1\") !=None:\n",
    "        title = soup.find(class_=\"ui_header h1\").text\n",
    "    else:\n",
    "        title = \"NaN\"\n",
    "    title_list.append(title)\n",
    "    #scrape ratings\n",
    "    if soup.find(class_=\"_1NKYRldB\") !=None : \n",
    "        rating_temp = soup.find(class_=\"_1NKYRldB\").find_all(\"span\")[0][\"class\"][1]\n",
    "    else:\n",
    "        rating_temp= \"NaN\"\n",
    "    rating_list.append(rating_temp)\n",
    "    # avis\n",
    "    if soup.find(class_=\"_1NKYRldB\") !=None : \n",
    "        avis = soup.find(class_=\"_1NKYRldB\").find_all(\"span\")[1].text\n",
    "    else : avis = \"NaN\"\n",
    "    avis_list.append(avis)\n",
    "    #labels\n",
    "    if soup.find(class_=\"_3RTCF0T0\") !=None : \n",
    "        labels = soup.find(class_=\"_3RTCF0T0\").text\n",
    "    else:\n",
    "        labels = \"NaN\"\n",
    "    labels_list.append(labels)\n",
    "    if soup.find(class_=\"LjCWTZdN\") !=None : \n",
    "        adress = soup.find(class_=\"LjCWTZdN\").text\n",
    "    else:\n",
    "        adress = \"NaN\"\n",
    "    adress_list.append(adress)\n",
    "    \n",
    "    coordinates = get_coordinates()\n",
    "    coo_list.append(coordinates)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First we get a full list of the url of museums in France"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_list = []\n",
    "name_list = []\n",
    "count = 0\n",
    "for region in regions_list:\n",
    "    soup = get_soup(region)\n",
    "    list_attractions = [\"https://www.tripadvisor.fr\" + i[\"href\"] for i in soup.find_all(class_=\"_1QKQOve4\")]\n",
    "    l_names  = [i.find(\"h2\").text for i in soup.find_all(class_=\"_1QKQOve4\")]\n",
    "    url_list += list_attractions\n",
    "    name_list += l_names\n",
    "    count +=1\n",
    "    time.sleep(1)\n",
    "    next = next_page()\n",
    "    while next:\n",
    "        soup = get_soup(next)\n",
    "        list_attractions = [\"https://www.tripadvisor.fr\" + i[\"href\"] for i in soup.find_all(class_=\"_1QKQOve4\")]\n",
    "        l_names  = [i.find(\"h2\").text for i in soup.find_all(class_=\"_1QKQOve4\")]\n",
    "        url_list += list_attractions\n",
    "        name_list += l_names\n",
    "        time.sleep(1)\n",
    "        next = next_page()\n",
    "        count +=1\n",
    "        print(count)\n",
    "print(\"len url_list : \", len(url_list))\n",
    "df_url = pd.DataFrame({\"name\" : name_list, \"url\" : url_list})\n",
    "df_url.to_csv(\"url musées tripadvisor.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Then we proceed to scraping each page individually and create a dataframe from it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#initialize data storage\n",
    "city_list = []\n",
    "rating_list = []\n",
    "avis_list = []\n",
    "labels_list = []\n",
    "adress_list = []\n",
    "coordinates_list = []\n",
    "title_list = []\n",
    "coo_list= []\n",
    "\n",
    "\n",
    "nb = 0\n",
    "for url in url_list:\n",
    "    soup = get_soup(url)\n",
    "    scrape_page()\n",
    "    nb +=1\n",
    "    time.sleep(1)\n",
    "    if nb%50 == 0:\n",
    "        print(nb)\n",
    "    \n",
    "df_infos = pd.DataFrame({\"city\" : city_list, \"title\" : title_list, \"rating\" : rating_list, \"avis\" : avis_list, \"labels\" : labels_list, \"adress\" : adress_list, \"coordinates\" : coo_list})\n",
    "df_infos.to_csv(\"liste musées 3.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# cleaning data\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}