mission_to_mars.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import requests\n",
    "\n",
    "#import selenium\n",
    "from selenium import webdriver"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 1 - Scraping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NASA Mars News"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import webpage data\n",
    "url1 = \"https://mars.nasa.gov/news/\"\n",
    "\n",
    "#response = requests.get(url)\n",
    "driver.get(url1)\n",
    "html = driver.page_source\n",
    "\n",
    "#soup = bs(response.text,'lxml')\n",
    "soup = bs(html,'html.parser')\n",
    "#print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect the latest News Title and Paragraph\n",
    "# Empty lists returned with ('ul', class_=\"item_list\"), ('li', class_=\"slide\")\n",
    "article_list = soup.body.find_all('div', class_=\"image_and_description_container\")\n",
    "article_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not returning all items\n",
    "soup.body.find_all('div', class_=\"content_title\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_headline = (soup.body.find_all('div', class_=\"content_title\"))[0]\n",
    "headline = first_headline.a.text.strip()\n",
    "headline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### TO DO : Retrieve first article teaser \n",
    "\n",
    "# Empty list: ('div', class_=\"article_teaser_body\"), ('div', class_=\"list_text\")\n",
    "# Empty list: soup.select(\"[class~=article_teaser]\")\n",
    "#first_teaser = soup.body.div.find_all(\"div\",class_=\"article_teaser_body\")\n",
    "#first_teaser\n",
    "\n",
    "first_teaser = (soup.body.find_all('div', class_=\"article_teaser_body\"))[0]\n",
    "teaser = first_teaser.text\n",
    "print(teaser)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## JPL Mars Space Images - Featured Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from splinter import Browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "executable_path = {'executable_path':'chromedriver.exe'}\n",
    "browser = Browser('chrome',**executable_path, headless=False)\n",
    "\n",
    "url2 = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'\n",
    "browser.visit(url2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(url2)\n",
    "html2 = driver.page_source\n",
    "soup2 = bs(html2,'html.parser')\n",
    "print(soup2.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Index to make a single object not a list, even though there is only one result\n",
    "retrieval2 = (soup2.body.find_all(\"div\", class_=\"carousel_items\"))[0]\n",
    "retrieval2b = (soup2.body.find_all(\"div\", class_=\"carousel_items\"))\n",
    "retrieval2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retrieval2.article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split URL out\n",
    "for div in retrieval2b:\n",
    "    test=div.find('article')['style']\n",
    "test = test.split('(\\'')[1].split('\\')')[0]\n",
    "print(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Current url. Perhaps split to retrieve index URL\n",
    "driver.current_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "featured_image_url = \"https://www.jpl.nasa.gov\" + test\n",
    "print(featured_image_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mars Weather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "url3 = \"https://space-facts.com/mars/\"\n",
    "tables = pd.read_html(url3)\n",
    "df = tables[0]\n",
    "df.to_html('mars_weather.html')\n",
    "html_table = df.to_html()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<table border=\"1\" class=\"dataframe\">  <thead>    <tr style=\"text-align: right;\">      <th></th>      <th>0</th>      <th>1</th>    </tr>  </thead>  <tbody>    <tr>      <th>0</th>      <td>Equatorial Diameter:</td>      <td>6,792 km</td>    </tr>    <tr>      <th>1</th>      <td>Polar Diameter:</td>      <td>6,752 km</td>    </tr>    <tr>      <th>2</th>      <td>Mass:</td>      <td>6.42 x 10^23 kg (10.7% Earth)</td>    </tr>    <tr>      <th>3</th>      <td>Moons:</td>      <td>2 (Phobos &amp; Deimos)</td>    </tr>    <tr>      <th>4</th>      <td>Orbit Distance:</td>      <td>227,943,824 km (1.52 AU)</td>    </tr>    <tr>      <th>5</th>      <td>Orbit Period:</td>      <td>687 days (1.9 years)</td>    </tr>    <tr>      <th>6</th>      <td>Surface Temperature:</td>      <td>-153 to 20 °C</td>    </tr>    <tr>      <th>7</th>      <td>First Record:</td>      <td>2nd millennium BC</td>    </tr>    <tr>      <th>8</th>      <td>Recorded By:</td>      <td>Egyptian astronomers</td>    </tr>  </tbody></table>'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read html file back into jupyter notebook?\n",
    "html_table = html_table.replace('\\n','')\n",
    "html_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mars Hemispheres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from splinter import Browser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "executable_path = {'executable_path':'chromedriver.exe'}\n",
    "browser = Browser('chrome',**executable_path, headless=False)\n",
    "\n",
    "url4 = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'\n",
    "browser.visit(url4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html lang=\"en\" xmlns=\"http://www.w3.org/1999/xhtml\">\n",
      " <head>\n",
      "  <link href=\"//ajax.googleapis.com/ajax/libs/jqueryui/1.11.3/themes/smoothness/jquery-ui.css\" rel=\"stylesheet\" type=\"text/css\"/>\n",
      "  <script async=\"\" src=\"https://ssl.google-analytics.com/ga.js\" type=\"text/javascript\">\n",
      "  </script>\n",
      "  <script src=\"https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js\" type=\"text/javascript\">\n",
      "  </script>\n",
      "  <title>\n",
      "   Astropedia Search Results | USGS Astrogeology Science Center\n",
      "  </title>\n",
      "  <meta content=\"USGS Astrogeology Science Center Astropedia search results.\" name=\"description\"/>\n",
      "  <meta content=\"USGS,Astrogeology Science Center,Cartography,Geology,Space,Geological Survey,Mapping\" name=\"keywords\"/>\n",
      "  <meta content=\"IE=edge\" http-equiv=\"X-UA-Compatible\"/>\n",
      "  <meta content=\"text/html; charset=utf-8\" http-equiv=\"Content-Type\"/>\n",
      "  <meta content=\"width=device-width, initial-scale=1, maximum-scale=1\" name=\"viewport\"/>\n",
      "  <meta content=\"x61hXXVj7wtfBSNOPnTftajMsZ5yB2W-qRoyr7GtOKM\" name=\"google-site-verification\"/>\n",
      "  <!--<link rel=\"stylesheet\" href=\"http://fonts.googleapis.com/css?family=Open+Sans:400italic,400,bold\"/>-->\n",
      "  <link href=\"/css/main.css\" media=\"screen\" rel=\"stylesheet\"/>\n",
      "  <link href=\"/css/print.css\" media=\"print\" rel=\"stylesheet\"/>\n",
      "  <!--[if lt IE 9]>\n",
      "\t\t\t<script src=\"http://html5shiv.googlecode.com/svn/trunk/html5.js\"></script>\n",
      "\t\t\t<script src=\"/js/respond.min.js\"></script>\n",
      "\t\t\t<link rel=\"stylesheet\" type=\"text/css\" href=\"/css/ie.css\"/>\n",
      "                        <script>\n",
      "                          document.createElement('header');\n",
      "                          document.createElement('nav');\n",
      "                          document.createElement('section');\n",
      "                          document.createElement('article');\n",
      "                          document.createElement('aside');\n",
      "                          document.createElement('footer');\n",
      "                          document.createElement('hgroup');\n",
      "                        </script>\n",
      "                  <![endif]-->\n",
      "  <link href=\"/favicon.ico\" rel=\"icon\" type=\"image/x-ico\"/>\n",
      " </head>\n",
      " <body id=\"results\">\n",
      "  <header>\n",
      "   <h1>\n",
      "    Astrogeology Science Center\n",
      "   </h1>\n",
      "   <a href=\"http://www.usgs.gov\">\n",
      "    <img alt=\"USGS: Science for a Changing World\" class=\"logo\" height=\"70\" src=\"/images/usgs_logo_main_2x.png\" width=\"180\"/>\n",
      "   </a>\n",
      "  </header>\n",
      "  <div class=\"wrapper\">\n",
      "   <nav>\n",
      "    <a href=\"#\" id=\"nav-toggle\" title=\"Navigation Menu\">\n",
      "     Menu\n",
      "    </a>\n",
      "    <ul class=\"dropdown dropdown-horizontal\" id=\"yw0\">\n",
      "     <li>\n",
      "      <a href=\"/\">\n",
      "       Home\n",
      "      </a>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"/about\">\n",
      "       About\n",
      "      </a>\n",
      "      <ul>\n",
      "       <li>\n",
      "        <a href=\"/about/careers\">\n",
      "         Careers\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/contact\">\n",
      "         Contact\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/about/events\">\n",
      "         Events\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/site/glossary\">\n",
      "         Glossary\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/about/mission\">\n",
      "         Mission\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/news\">\n",
      "         News\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/people\">\n",
      "         People\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/about/using-our-images\">\n",
      "         Using Our Images\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/about/visitors\">\n",
      "         Visitors\n",
      "        </a>\n",
      "       </li>\n",
      "      </ul>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"/facilities\">\n",
      "       Labs / Facilities\n",
      "      </a>\n",
      "      <ul>\n",
      "       <li>\n",
      "        <a href=\"/facilities/flynn-creek-crater-sample-collection\">\n",
      "         Flynn Creek Crater Sample Collection\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://www.moon-cal.org\">\n",
      "         Lunar Calibration Project\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/facilities/meteor-crater-sample-collection\">\n",
      "         Meteor Crater Sample Collection\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/facilities/mrctr\">\n",
      "         MRCTR GIS Lab\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/facilities/cartography-and-imaging-sciences-node-of-nasa-planetary-data-system\">\n",
      "         PDS Cartography and Imaging Sciences Node\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/pds/annex\">\n",
      "         PDS IMG Annex\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/facilities/photogrammetry-guest-facility\">\n",
      "         Photogrammetry Guest Facility\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/rpif\">\n",
      "         Regional Planetary Information Facility (RPIF)\n",
      "        </a>\n",
      "       </li>\n",
      "      </ul>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"/maps\">\n",
      "       Maps / Products\n",
      "      </a>\n",
      "      <ul>\n",
      "       <li>\n",
      "        <a href=\"/search\">\n",
      "         Product Search\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://planetarynames.wr.usgs.gov\">\n",
      "         Gazetteer of Planetary Nomenclature\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://planetarymapping.wr.usgs.gov\">\n",
      "         Geologic Mapping Program\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://pilot.wr.usgs.gov\">\n",
      "         Planetary Image Locator Tool (PILOT)\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/search/planetary-index\">\n",
      "         Planetary Map Index\n",
      "        </a>\n",
      "       </li>\n",
      "      </ul>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"/geology\">\n",
      "       Missions / Research\n",
      "      </a>\n",
      "      <ul>\n",
      "       <li>\n",
      "        <a href=\"/geology/mars-dunes\">\n",
      "         Mars Dunes\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/geology/mars-ice\">\n",
      "         Mars Ice\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/missions\">\n",
      "         Mission Support\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/solar-system\">\n",
      "         Solar System\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"/groups\">\n",
      "         Working Groups\n",
      "        </a>\n",
      "       </li>\n",
      "      </ul>\n",
      "     </li>\n",
      "     <li>\n",
      "      <a href=\"/tools\">\n",
      "       Tools\n",
      "      </a>\n",
      "      <ul>\n",
      "       <li>\n",
      "        <a href=\"http://planetarynames.wr.usgs.gov\">\n",
      "         Gazetteer of Planetary Nomenclature\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://isis.astrogeology.usgs.gov\">\n",
      "         Integrated Software for Imagers and Spectrometers (ISIS)\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://astrogeology.usgs.gov/tools/map-a-planet-2\">\n",
      "         Map a Planet 2\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://pilot.wr.usgs.gov\">\n",
      "         Planetary Image Locator Tool (PILOT)\n",
      "        </a>\n",
      "       </li>\n",
      "       <li>\n",
      "        <a href=\"http://astrocloud.wr.usgs.gov/\">\n",
      "         Projection on the Web (POW)\n",
      "        </a>\n",
      "       </li>\n",
      "      </ul>\n",
      "     </li>\n",
      "    </ul>\n",
      "    <form action=\"/search/results\" class=\"search\" id=\"search\" method=\"get\">\n",
      "     <input title=\"Search Astropedia\" type=\"submit\" value=\"\"/>\n",
      "     <input name=\"q\" placeholder=\"Search\" type=\"text\"/>\n",
      "     <input name=\"__ncforminfo\" type=\"hidden\" value=\"G4S_6RtkLwocGun64kUqZxF1gA9hVCVmRNiZmrcL0tFDfamJyrnTFHR_YscHkAMWI0ARqICt7eBJWUiNyCMhraJ0g8GYfOHGUuoqgcQOpJI=\"/>\n",
      "    </form>\n",
      "   </nav>\n",
      "   <div class=\"container\">\n",
      "    <form action=\"/search/results\" class=\"bar widget block\" id=\"search-bar\">\n",
      "     <input name=\"q\" type=\"hidden\" value=\"hemisphere-enhanced\"/>\n",
      "     <input name=\"target\" type=\"hidden\" value=\"Mars\"/>\n",
      "     <input name=\"__ncforminfo\" type=\"hidden\" value=\"G4S_6RtkLwocGun64kUqZxF1gA9hVCVmRNiZmrcL0tEO_ItU1eF36vSdsxVsWe1U7HPonYUEyL53ZKrotfMAPh87V-VdhyluMQIlAJmCOv4YvnOwvz9uCQ==\"/>\n",
      "    </form>\n",
      "    <div class=\"full-content\">\n",
      "     <section class=\"block\" id=\"results-accordian\">\n",
      "      <div class=\"result-list\" data-section=\"product\" id=\"product-section\">\n",
      "       <div class=\"accordian\">\n",
      "        <h2>\n",
      "         Products\n",
      "        </h2>\n",
      "        <span class=\"count\">\n",
      "         4 Results\n",
      "        </span>\n",
      "        <span class=\"collapse\">\n",
      "         Collapse\n",
      "        </span>\n",
      "       </div>\n",
      "       <div class=\"collapsible results\">\n",
      "        <div class=\"item\">\n",
      "         <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/cerberus_enhanced\">\n",
      "          <img alt=\"Cerberus Hemisphere Enhanced thumbnail\" class=\"thumb\" src=\"/cache/images/dfaf3849e74bf973b59eb50dab52b583_cerberus_enhanced.tif_thumb.png\"/>\n",
      "         </a>\n",
      "         <div class=\"description\">\n",
      "          <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/cerberus_enhanced\">\n",
      "           <h3>\n",
      "            Cerberus Hemisphere Enhanced\n",
      "           </h3>\n",
      "          </a>\n",
      "          <span class=\"subtitle\" style=\"float:left\">\n",
      "           image/tiff 21 MB\n",
      "          </span>\n",
      "          <span class=\"pubDate\" style=\"float:right\">\n",
      "          </span>\n",
      "          <br/>\n",
      "          <p>\n",
      "           Mosaic of the Cerberus hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. This mosaic is composed of 104 Viking Orbiter images acquired…\n",
      "          </p>\n",
      "         </div>\n",
      "         <!-- end description -->\n",
      "        </div>\n",
      "        <div class=\"item\">\n",
      "         <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/schiaparelli_enhanced\">\n",
      "          <img alt=\"Schiaparelli Hemisphere Enhanced thumbnail\" class=\"thumb\" src=\"/cache/images/7677c0a006b83871b5a2f66985ab5857_schiaparelli_enhanced.tif_thumb.png\"/>\n",
      "         </a>\n",
      "         <div class=\"description\">\n",
      "          <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/schiaparelli_enhanced\">\n",
      "           <h3>\n",
      "            Schiaparelli Hemisphere Enhanced\n",
      "           </h3>\n",
      "          </a>\n",
      "          <span class=\"subtitle\" style=\"float:left\">\n",
      "           image/tiff 35 MB\n",
      "          </span>\n",
      "          <span class=\"pubDate\" style=\"float:right\">\n",
      "          </span>\n",
      "          <br/>\n",
      "          <p>\n",
      "           Mosaic of the Schiaparelli hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. The images were acquired in 1980 during early northern…\n",
      "          </p>\n",
      "         </div>\n",
      "         <!-- end description -->\n",
      "        </div>\n",
      "        <div class=\"item\">\n",
      "         <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/syrtis_major_enhanced\">\n",
      "          <img alt=\"Syrtis Major Hemisphere Enhanced thumbnail\" class=\"thumb\" src=\"/cache/images/aae41197e40d6d4f3ea557f8cfe51d15_syrtis_major_enhanced.tif_thumb.png\"/>\n",
      "         </a>\n",
      "         <div class=\"description\">\n",
      "          <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/syrtis_major_enhanced\">\n",
      "           <h3>\n",
      "            Syrtis Major Hemisphere Enhanced\n",
      "           </h3>\n",
      "          </a>\n",
      "          <span class=\"subtitle\" style=\"float:left\">\n",
      "           image/tiff 25 MB\n",
      "          </span>\n",
      "          <span class=\"pubDate\" style=\"float:right\">\n",
      "          </span>\n",
      "          <br/>\n",
      "          <p>\n",
      "           Mosaic of the Syrtis Major hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. This mosaic is composed of about 100 red and violet…\n",
      "          </p>\n",
      "         </div>\n",
      "         <!-- end description -->\n",
      "        </div>\n",
      "        <div class=\"item\">\n",
      "         <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/valles_marineris_enhanced\">\n",
      "          <img alt=\"Valles Marineris Hemisphere Enhanced thumbnail\" class=\"thumb\" src=\"/cache/images/04085d99ec3713883a9a57f42be9c725_valles_marineris_enhanced.tif_thumb.png\"/>\n",
      "         </a>\n",
      "         <div class=\"description\">\n",
      "          <a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/valles_marineris_enhanced\">\n",
      "           <h3>\n",
      "            Valles Marineris Hemisphere Enhanced\n",
      "           </h3>\n",
      "          </a>\n",
      "          <span class=\"subtitle\" style=\"float:left\">\n",
      "           image/tiff 27 MB\n",
      "          </span>\n",
      "          <span class=\"pubDate\" style=\"float:right\">\n",
      "          </span>\n",
      "          <br/>\n",
      "          <p>\n",
      "           Mosaic of the Valles Marineris hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. The distance is 2500 kilometers from the surface of…\n",
      "          </p>\n",
      "         </div>\n",
      "         <!-- end description -->\n",
      "        </div>\n",
      "        <script>\n",
      "         addBases=[];;if(typeof resetLayerSwitcher===\"function\"){resetLayerSwitcher(false)};var productTotal = 4;\n",
      "        </script>\n",
      "       </div>\n",
      "       <!-- end this-section -->\n",
      "      </div>\n",
      "     </section>\n",
      "    </div>\n",
      "   </div>\n",
      "   <div class=\"icons projects black scroll-wrapper\">\n",
      "    <div class=\"scroll\">\n",
      "     <a class=\"icon\" href=\"http://isis.astrogeology.usgs.gov\" title=\"Integrated Software for Imagers and Spectrometers\">\n",
      "      <img alt=\"ISIS Logo\" height=\"112\" src=\"/images/logos/isis_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       ISIS\n",
      "      </span>\n",
      "     </a>\n",
      "     <a class=\"icon\" href=\"http://planetarynames.wr.usgs.gov\" title=\"Gazetteer of Planetary Nomenclature\">\n",
      "      <img alt=\"Nomenclature Logo\" height=\"112\" src=\"/images/logos/nomenclature_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       Planetary Nomenclature\n",
      "      </span>\n",
      "     </a>\n",
      "     <a class=\"icon\" href=\"http://astrogeology.usgs.gov/tools/map\" title=\"Map a Planet 2\">\n",
      "      <img alt=\"Map-a-Planet Logo\" height=\"112\" src=\"/images/logos/map_a_planet_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       Map a Planet 2\n",
      "      </span>\n",
      "     </a>\n",
      "     <a class=\"icon\" href=\"/facilities/imaging-node-of-nasa-planetary-data-system-pds\" title=\"PDS Imaging Node\">\n",
      "      <img alt=\"PDS Logo\" height=\"112\" src=\"/images/pds_logo-black-web.png\"/>\n",
      "      <span class=\"label\">\n",
      "       PDS Imaging Node\n",
      "      </span>\n",
      "     </a>\n",
      "     <!--\n",
      "\t\t\t\t\t\t<a title=\"Astropedia Search\" href=\"/search\" class=\"icon\">\n",
      "\t\t\t\t\t\t\t<img alt=\"Astropedia Logo\" height=\"112\" width=\"112\" src=\"/images/logos/astropedia_2x.jpg\"/>\n",
      "\t\t\t\t\t\t\t<span class=\"label\">Astropedia</span>\n",
      "\t\t\t\t\t\t</a>\n",
      "-->\n",
      "     <a class=\"icon\" href=\"/rpif\" title=\"Regional Planetary Image Facility\">\n",
      "      <img alt=\"RPIF Logo\" height=\"112\" src=\"/images/logos/rpif_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       RPIF\n",
      "      </span>\n",
      "     </a>\n",
      "     <a class=\"icon\" href=\"/facilities/photogrammetry-guest-facility\" title=\"Photogrammetry Guest Facility\">\n",
      "      <img alt=\"Photogrammetry Guest Faciltiy Logo\" height=\"112\" src=\"/images/logos/photogrammetry_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       Photogrammetry Guest Facility\n",
      "      </span>\n",
      "     </a>\n",
      "     <a class=\"icon\" href=\"http://pilot.wr.usgs.gov\" title=\"Planetary Image Locator Tool\">\n",
      "      <img alt=\"Pilot Logo\" height=\"112\" src=\"/images/logos/pilot_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       PILOT\n",
      "      </span>\n",
      "     </a>\n",
      "     <a class=\"icon\" href=\"/facilities/mrctr\" title=\"Mapping, Remote-sensing, Cartography, Technology and Research GIS Lab\">\n",
      "      <img alt=\"MRCTR GIS Lab Logo\" height=\"112\" src=\"/images/logos/mrctr_2x.jpg\" width=\"112\"/>\n",
      "      <span class=\"label\">\n",
      "       MRCTR GIS Lab\n",
      "      </span>\n",
      "     </a>\n",
      "    </div>\n",
      "   </div>\n",
      "   <footer>\n",
      "    <div class=\"left\">\n",
      "     <a href=\"http://astrogeology.usgs.gov\">\n",
      "      Home\n",
      "     </a>\n",
      "     |\n",
      "     <a href=\"http://astrogeology.usgs.gov/contact\">\n",
      "      Contact\n",
      "     </a>\n",
      "     |\n",
      "     <a href=\"http://astrogeology.usgs.gov/about/events\">\n",
      "      Events\n",
      "     </a>\n",
      "     |\n",
      "     <a href=\"http://astrogeology.usgs.gov/news\">\n",
      "      News\n",
      "     </a>\n",
      "    </div>\n",
      "    <div class=\"right\">\n",
      "     <a href=\"http://www.doi.gov\">\n",
      "      U.S. Department of Interior\n",
      "     </a>\n",
      "     |\n",
      "     <a href=\"http://www.usgs.gov\">\n",
      "      U.S. Geological Survey\n",
      "     </a>\n",
      "     |\n",
      "     <a href=\"http://www.usa.gov\">\n",
      "      USA.gov\n",
      "     </a>\n",
      "    </div>\n",
      "   </footer>\n",
      "  </div>\n",
      "  <!--\n",
      "\t\t<div class=\"credit\">\n",
      "\t\t\t<small>Background Credits: NASA/USGS</small>\n",
      "\t\t</div>\n",
      "-->\n",
      "  <div class=\"page-background\" style=\"\n",
      "\t\t\tbackground:url('/images/backgrounds/mars.jpg');\n",
      "\t\t\tfilter:progid:DXImageTransform.Microsoft.AlphaImageLoader(\n",
      "\t\t\t\tsrc='/images/backgrounds/mars.jpg', sizingMethod='scale');\n",
      "\t\t\">\n",
      "  </div>\n",
      "  <script type=\"text/javascript\">\n",
      "   var baseUrl = \"\";\n",
      "\n",
      "\n",
      "var _gaq = _gaq || [];_gaq.push(['_setAccount', 'UA-27613186-1']);_gaq.push(['_trackPageview']);(function() { var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);})();\n",
      "  </script>\n",
      "  <script src=\"//ajax.googleapis.com/ajax/libs/jqueryui/1.11.3/jquery-ui.min.js\" type=\"text/javascript\">\n",
      "  </script>\n",
      "  <script src=\"/js/general.js\" type=\"text/javascript\">\n",
      "  </script>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "driver.get(url4)\n",
    "html4 = driver.page_source\n",
    "soup4 = bs(html4,'html.parser')\n",
    "print(soup4.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<div class=\"description\"><a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/cerberus_enhanced\"><h3>Cerberus Hemisphere Enhanced</h3></a><span class=\"subtitle\" style=\"float:left\">image/tiff 21 MB</span><span class=\"pubDate\" style=\"float:right\"></span><br/><p>Mosaic of the Cerberus hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. This mosaic is composed of 104 Viking Orbiter images acquired…</p></div>,\n",
       " <div class=\"description\"><a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/schiaparelli_enhanced\"><h3>Schiaparelli Hemisphere Enhanced</h3></a><span class=\"subtitle\" style=\"float:left\">image/tiff 35 MB</span><span class=\"pubDate\" style=\"float:right\"></span><br/><p>Mosaic of the Schiaparelli hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. The images were acquired in 1980 during early northern…</p></div>,\n",
       " <div class=\"description\"><a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/syrtis_major_enhanced\"><h3>Syrtis Major Hemisphere Enhanced</h3></a><span class=\"subtitle\" style=\"float:left\">image/tiff 25 MB</span><span class=\"pubDate\" style=\"float:right\"></span><br/><p>Mosaic of the Syrtis Major hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. This mosaic is composed of about 100 red and violet…</p></div>,\n",
       " <div class=\"description\"><a class=\"itemLink product-item\" href=\"/search/map/Mars/Viking/valles_marineris_enhanced\"><h3>Valles Marineris Hemisphere Enhanced</h3></a><span class=\"subtitle\" style=\"float:left\">image/tiff 27 MB</span><span class=\"pubDate\" style=\"float:right\"></span><br/><p>Mosaic of the Valles Marineris hemisphere of Mars projected into point perspective, a view similar to that which one would see from a spacecraft. The distance is 2500 kilometers from the surface of…</p></div>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retrieval4 = soup4.body.find_all('div', class_=\"description\")\n",
    "retrieval4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find hyperlinks to click through\n",
    "\n",
    "# Testing results of loop\n",
    "link_loop =[]\n",
    "# Testing results of links found\n",
    "link_list = []\n",
    "# Testing results of titles found\n",
    "title_list = []\n",
    "# List of dictionaries required by assignment\n",
    "hemisphere_dicts = []\n",
    "\n",
    "for each in retrieval4:\n",
    "    test2 = each.find('a')['href']\n",
    "    link_loop.append(test2)\n",
    "    \n",
    "    url5 = 'https://astrogeology.usgs.gov'+test2\n",
    "    browser.visit(url5)\n",
    "    driver.get(url5)\n",
    "    html5 = driver.page_source\n",
    "    soup5 = bs(html5,'html.parser')\n",
    "    test3 = (soup5.body.find('img', class_=\"wide-image\"))['src']\n",
    "    img_url = 'https://astrogeology.usgs.gov'+test3\n",
    "    link_list.append(img_url)\n",
    "    \n",
    "    hemisphere = soup5.body.find('h2').text.strip()\n",
    "    hemisphere = hemisphere.split(' Enhanced')[0]\n",
    "    title_list.append(hemisphere)\n",
    "    hemisphere_dict = {\"title\":hemisphere, \"img_url\":img_url}\n",
    "    hemisphere_dicts.append(hemisphere_dict)\n",
    "    \n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/search/map/Mars/Viking/cerberus_enhanced',\n",
       " '/search/map/Mars/Viking/schiaparelli_enhanced',\n",
       " '/search/map/Mars/Viking/syrtis_major_enhanced',\n",
       " '/search/map/Mars/Viking/valles_marineris_enhanced']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "link_loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://astrogeology.usgs.gov/cache/images/cfa62af2557222a02478f1fcd781d445_cerberus_enhanced.tif_full.jpg',\n",
       " 'https://astrogeology.usgs.gov/cache/images/3cdd1cbf5e0813bba925c9030d13b62e_schiaparelli_enhanced.tif_full.jpg',\n",
       " 'https://astrogeology.usgs.gov/cache/images/ae209b4e408bb6c3e67b6af38168cf28_syrtis_major_enhanced.tif_full.jpg',\n",
       " 'https://astrogeology.usgs.gov/cache/images/7cf2da4bf549ed01c17f206327be4db7_valles_marineris_enhanced.tif_full.jpg']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "link_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Cerberus Hemisphere',\n",
       " 'Schiaparelli Hemisphere',\n",
       " 'Syrtis Major Hemisphere',\n",
       " 'Valles Marineris Hemisphere']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'img_url': 'https://astrogeology.usgs.gov/cache/images/cfa62af2557222a02478f1fcd781d445_cerberus_enhanced.tif_full.jpg',\n",
       "  'title': 'Cerberus Hemisphere'},\n",
       " {'img_url': 'https://astrogeology.usgs.gov/cache/images/3cdd1cbf5e0813bba925c9030d13b62e_schiaparelli_enhanced.tif_full.jpg',\n",
       "  'title': 'Schiaparelli Hemisphere'},\n",
       " {'img_url': 'https://astrogeology.usgs.gov/cache/images/ae209b4e408bb6c3e67b6af38168cf28_syrtis_major_enhanced.tif_full.jpg',\n",
       "  'title': 'Syrtis Major Hemisphere'},\n",
       " {'img_url': 'https://astrogeology.usgs.gov/cache/images/7cf2da4bf549ed01c17f206327be4db7_valles_marineris_enhanced.tif_full.jpg',\n",
       "  'title': 'Valles Marineris Hemisphere'}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hemisphere_dicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}