Web Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Web scraping with BS:\n",
    "\n",
    "A tutorial for website scrapping using the Requests and BeautifullSoup libraries. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests # To be able to access websites we wish to scrap.\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import pandas as pd\n",
    "from IPython.display import Image, HTML # To be able to display images within a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='http://ensias.um5.ac.ma/page/ing%C3%A9nieurs' # We will extract names of Allumni from my own school ENSIAS for the 2015 promo."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "req=requests.get(url) \n",
    "status=req.status_code # Status of the connection attempt if everything is good this should have a 200 value.\n",
    "encoding=req.encoding\n",
    "text=BeautifulSoup(req.text,'html.parser')\n",
    "print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tr height=\"20\"><td align=\"right\" height=\"20\" style=\"height:20px;\">183</td>\n",
       "<td align=\"left\">ZRAIBI</td>\n",
       "<td align=\"left\">SALMA</td>\n",
       "</tr>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=text.findAll('tr') # Finding all tags with 'tr' in them\n",
    "for item in data:\n",
    "    pass\n",
    "item # We only catch the last one to see how it looks like."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<td align=\"right\" height=\"20\" style=\"height:20px;\">183</td>,\n",
       " '\\n',\n",
       " <td align=\"left\">ZRAIBI</td>,\n",
       " '\\n',\n",
       " <td align=\"left\">SALMA</td>,\n",
       " '\\n']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a=item.contents # Turns the response obtained into a list\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of students promo 2015: \n",
      "\n",
      "First name          Last name      \n",
      "\n",
      "ABI                 YASSIR         \n",
      "ABIDALLAH           YOUSSEF        \n",
      "ACHAOUD             HANANE         \n",
      "ADDAD               ABDELHALIM     \n",
      "AFIF                MOHAMMED       \n",
      "AIT EL HAJ          KHADIJA        \n",
      "AIT EL HARRAJ       AMINE          \n",
      "AIT MANSOUR         YOUSSEF        \n",
      "AITOUNA             MOHAMED        \n",
      "AL MAACH            WAHIBA         \n",
      "ALAOUI MRANI        SOUKAINA       \n",
      "ALLAOU              MARIAME        \n",
      "ALMAMOUN            ZAKARIAE       \n",
      "AMARA               HALA           \n",
      "AMCHICH             IMANE          \n",
      "AMENCHAR            NABIL          \n",
      "AMMARI              KHALID         \n",
      "AMNAS               ASMAE          \n",
      "AOULADLAHCEN        MOHAMMED       \n",
      "AOUNI               HAMZA          \n",
      "AOURAGH             YOUNESSE       \n",
      "AOUTIL              AHMED          \n",
      "AQQA                MILOUD         \n",
      "BAHADOU             HIND           \n",
      "BARBARE             MOHAMMED AMINE \n",
      "BELAKHAL            HAMZA          \n",
      "BELGHITI            MEHDI          \n",
      "BELJEBBAR           ALAEDDINE      \n",
      "BELKANDOUCI         BAHIYA         \n",
      "BELLAHCENE          TAHA           \n",
      "BELRHALMIA          WALID          \n",
      "BEN SALEH           SAIDA          \n",
      "BENABDALMALEK       ZAKARIYA       \n",
      "BENHALIMA           FADOUA         \n",
      "BENKACEM            ILIAS          \n",
      "BENKIRANE           MEHDI          \n",
      "BENMOUHOU           SOUAD          \n",
      "BETACHNI            TARIQ          \n",
      "BEZRHOUD            IBRAHIM        \n",
      "BEZRHOUDI           MOHAMMED REDA  \n",
      "BOUALOU             OUMNIA         \n",
      "BOUCHNAFA           ANASS          \n",
      "BOUDHAIM            BACHIR         \n",
      "BOUDOUH             HAMZA          \n",
      "BOUFALA             ABDELALI       \n",
      "BOUHMID             MUSTAPHA       \n",
      "BOUKHARTA           MEHDI          \n",
      "BOULAICH            MOHAMED        \n",
      "BOUQOUROU           KELTOUM        \n",
      "BOURHIM             SOFIA          \n",
      "BOURZIZ             NABIL          \n",
      "BOUSADIK            HOUSSAM        \n",
      "BOUTJIM             ABDELMAJID     \n",
      "CHABLI              SAAD           \n",
      "CHARIF TRIBAK       SARA           \n",
      "CHARMOUKI           HICHAM         \n",
      "CHERBI              MEHDI          \n",
      "DAHA                FATIMA ZOHRA   \n",
      "DAHMANE             YOUSSEF        \n",
      "DJEKMANI            AZZ EDDINE     \n",
      "DJOHAR              REDA           \n",
      "DRIOUI              HAFSA          \n",
      "EL ALAMI            SALIM          \n",
      "EL ALLAM            HASSNA         \n",
      "EL AMRANI           MOHAMMED       \n",
      "EL ASSAL            SOUFIANE       \n",
      "EL BOUATMANI        IMAD           \n",
      "EL HMAIDI           MADIHA         \n",
      "EL HOUR             ABDELHAFID     \n",
      "EL IBRAHIMI         NOUSSAIBA      \n",
      "EL JAOUSSI          YOUSSEF        \n",
      "EL MADANI           SOUMIA         \n",
      "EL MESSOUSSI        MOHAMMED YASSIR\n",
      "EL MOUDDEN          WAIIL          \n",
      "EL MZOURI           HAYAT          \n",
      "EL OUAKILI          HAMZA          \n",
      "EL YOUSSOUFI        KHADIJA        \n",
      "ELAISSAOUI          ISMAIL         \n",
      "ERRARHBI            AYOUB          \n",
      "ESSAADANI           SOUFIANE       \n",
      "ESSEIDI             ANAS           \n",
      "EZZAKRI             NOURDINE       \n",
      "FAHIM               ABDELOUAHID    \n",
      "FATEMI              SOUMAYA        \n",
      "FHAMNI              YOUNES         \n",
      "GAIZI               ZINEB          \n",
      "GEBRATI             SARA           \n",
      "GRINI               ACHRAF         \n",
      "HABIB               YOUNESS JALAL  \n",
      "HADI                HALIMA         \n",
      "HAJJAR              MOHAMED AYOUB  \n",
      "HAJOULI             SOUKAINA       \n",
      "HAMDANE             ABD ALMAJIDE   \n",
      "HAMIDA              AYOUB          \n",
      "HAMZA               MARIAMA        \n",
      "HIDAYAT ALLAH       YOUNES         \n",
      "IBEN BRAHIM         SOUMAYA        \n",
      "IDRISSI             OSSAMA         \n",
      "IMAM                ANAS           \n",
      "IZDOUZEN            SAID           \n",
      "JABAL               GHIZLANE       \n",
      "JABRI               IMANE          \n",
      "KADIMI SKALLI       DOUNIA         \n",
      "KAHLI               ILIASS         \n",
      "KANDOUSSI           EL MEHDI       \n",
      "KHAMLICHI           AMINE          \n",
      "KHANNAT             AICHA          \n",
      "KINANI              YOUNESS        \n",
      "KORACHI             ZINEB          \n",
      "KORAMI              YASSIR         \n",
      "LAGMANI             SALMA          \n",
      "LAKSSIOUER          AHMED          \n",
      "LANOUNI             ACHRAF         \n",
      "LAOUZI              ROUKIA         \n",
      "LBOUCHOUARI         MUSTAPHA       \n",
      "LEMBARKI            MOHAMED AMINE  \n",
      "LOGDALI             SOUFIANE       \n",
      "LOTFI EL IDRISSI    OTHMANE        \n",
      "LOUGHZAFI           YACIRE         \n",
      "MACHKOUR            MONCEF         \n",
      "MAGHRAOUI           ABDELHAKIM     \n",
      "MAIOUAK             MARIEM         \n",
      "MAKBOUB             HAJAR          \n",
      "MAMMAR              GHITA          \n",
      "MASBAH              REDOUAN        \n",
      "MASK                MERYEM         \n",
      "MASSOU              SANA           \n",
      "MEDRANI             SOUFIA         \n",
      "MERIZAK             YOUSSEF        \n",
      "MERZAQ              SELIM          \n",
      "MESBAHI             SOUMIA         \n",
      "MESSOUDI            SOUNDOUSS      \n",
      "METWALLI            AYOUB          \n",
      "MIFDAL              ABDELALI       \n",
      "MIHI                KAOUTAR        \n",
      "MORCHID             ZAKARIA        \n",
      "MOUDEN              KENZA          \n",
      "MOUHARRIR           SARA           \n",
      "MOULOU              ZAKARIA        \n",
      "NABIL               SOUFIANE       \n",
      "NAJIH               YAAKOUB        \n",
      "NEJJARI             LOUBNA         \n",
      "OMARI               SOUFIANE       \n",
      "OUAGAGUE            MOSTAFA        \n",
      "OUKHEYI             BRAHIM         \n",
      "OULHAJ              JAMILA         \n",
      "RACHID              AMINE          \n",
      "RAOUDI              YASSINE        \n",
      "RAYADH              CHAIMAE        \n",
      "REZZOUQI            HASNAE         \n",
      "REZZOUQI            HAJAR          \n",
      "ROUANE              KAWTAR         \n",
      "SAADI               MASSYLE        \n",
      "SABRE               NABIL          \n",
      "SADEQ               ALI            \n",
      "SAIDI               MOHAMMED AMINE \n",
      "SAIED               YASSINE        \n",
      "SAILOUHI            LAMIAA         \n",
      "SAKKAH              HASNAE         \n",
      "SAOUALAH            JIHAD          \n",
      "SARA                HAMZA          \n",
      "SEFIANE             HAMZA          \n",
      "SIDIKOU ISSA HAROUNAAMINATOU       \n",
      "SOALI               YASSAMINE      \n",
      "TAHIR               KAOUTAR        \n",
      "TAHIRI AMINE        MOHAMMED       \n",
      "TANTAOUI            MOHAMED        \n",
      "TAOUDI EDRISSI      HASNAA         \n",
      "TAZLAFT             AMINE          \n",
      "TERHZAZ             SIHAM          \n",
      "TERIKI              FATIMA EZZAHRA \n",
      "TORRES              DINA           \n",
      "YACHAOUI            AYMAN          \n",
      "YOUSFI              AOLA           \n",
      "ZAHIR               OMAR           \n",
      "ZAIDI               OUSSAMA        \n",
      "ZAKHNINI            ABDELLAH       \n",
      "ZEKRAOUI            IMANE          \n",
      "ZEROUAL             YASSINE        \n",
      "ZEROUAL             ISSAM          \n",
      "ZNAIDI              YASSER         \n",
      "ZOUAGUI             ZINEB          \n",
      "ZRAIBI              SALMA          \n"
     ]
    }
   ],
   "source": [
    "print(\"List of students promo 2015: \\n\")\n",
    "print(f\"{'First name':<20}{'Last name':<15}\\n\")\n",
    "for item in data:\n",
    "    a=item.contents\n",
    "    print(f\"{a[2].text:<20}{a[4].text:<15}\") # .text method returns the contents of the tag as a string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now for another task, we will now try to retrieve images too. We will be using for this task Jumia."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://www.jumia.com.ng/laptops/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "# Same story here for establishing connection and getting the response.\n",
    "req=requests.get(url)\n",
    "status=req.status_code\n",
    "encoding=req.encoding\n",
    "text=BeautifulSoup(req.text,'html.parser')\n",
    "print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"link\" href=\"https://www.jumia.com.ng/dell-alienware-corei7.3-16gb-ram-2tb-hdd256gb-ssd-8gb-nvidia-geforce-gtx-windows10-home-bag-32gb-flash-wireless-mouse-36472029.html\"> <div class=\"top\"> </div> <div class=\"image-wrapper default-state\"><img alt=\"Alienware Corei7.3-(16GB RAM, 2TB HDD+256GB SSD) 8GB NVIDIA GeForce GTX ,Windows10 Home+ BAG, 32GB FLASH, WIRELESS MOUSE\" class=\"lazy image\" data-image-vertical=\"1\" data-placeholder=\"placeholder_m_1.jpg\" data-sku=\"DE168EL1ISGUFNAFAMZ\" data-src=\"https://ng.jumia.is/Ql8rdB0tB68gwOLSpn9a1LZ3Xn0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/92/027463/1.jpg?8665\" height=\"220\" src=\"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7\" width=\"220\"/><noscript><img class=\"image\" height=\"220\" src=\"https://ng.jumia.is/Ql8rdB0tB68gwOLSpn9a1LZ3Xn0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/92/027463/1.jpg?8665\" width=\"220\"/></noscript></div> <h2 class=\"title\"><span class=\"brand\">DELL </span> <span class=\"name\" dir=\"ltr\">Alienware Corei7.3-(16GB RAM, 2TB HDD+256GB SSD) 8GB NVIDIA GeForce GTX ,Windows10 Home+ BAG, 32GB FLASH, WIRELESS MOUSE</span></h2><div class=\"price-container clearfix\"> <span class=\"price-box ri\"> <span class=\"price\"><span data-currency-iso=\"NGN\">₦</span> <span data-price=\"430000\" dir=\"ltr\">430,000</span> </span> <span class=\"price -old -no-special\"> </span> </span></div> <div class=\"list -sizes\" data-selected-sku=\"\"> </div> <button class=\"osh-btn -primary -add-to-cart js-link js-add_cart_tracking\" data-add-to-cart-type=\"catalogPage\" data-emit-event=\"cart-item.add\" data-js-uri=\"https://www.jumia.com.ng/ajax/cart/add/?return=json&amp;keepMessages=false&amp;sku=DE168EL1ISGUFNAFAMZ-49273261&amp;crossSellSku=\" data-sku=\"DE168EL1ISGUFNAFAMZ\" data-sku-simple=\"DE168EL1ISGUFNAFAMZ-49273261\"> <span class=\"label\">Buy now</span> </button> <div class=\"osh-popup -template -checkout link hidden js-back-in-stock-reminder-popup-DE168EL1ISGUFNAFAMZ\" data-close-callback=\"\" data-href=\"https://www.jumia.com.ng/ajax/reminder/getreminderform/sku/DE168EL1ISGUFNAFAMZ/\" data-open-callback=\"popup.BackInStockReminder.open\" data-style=\"back-in-stock-reminder\" data-title=\"Please notify me when this product is available again\" data-type=\"ajax\"></div> </a>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# By inspecting the web html code, we can know the name of tags for which we are interested for the scraping.\n",
    "# In this case the <a> tag that has a 'link' class is the one containing the product data, image, name, brand, etc..\n",
    "# So that's what we will look up using findAll. We will work on the last one as an exhibit to know its structure.\n",
    "data=text.findAll('a',{'class':'link'})\n",
    "for item in data:\n",
    "    pass\n",
    "item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mark:  DELL \n",
      "Nom:  Alienware Corei7.3-(16GB RAM, 2TB HDD+256GB SSD) 8GB NVIDIA GeForce GTX ,Windows10 Home+ BAG, 32GB FLASH, WIRELESS MOUSE \n",
      "Prix:  ₦ 430,000\n"
     ]
    }
   ],
   "source": [
    "brand=item.find_all('span',{'class':'brand'})[0].text.strip() # Strip to avoid any formating trouble\n",
    "name=item.find_all('span',{'class':'name'})[0].text.strip()\n",
    "price=item.find_all('span',{'class':'price'})[0].text.strip()\n",
    "print('Mark: ',brand,'\\nNom: ',name,'\\nPrix: ',price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://ng.jumia.is/AJQFFwlEvzNiA0hhxrmAPn4hNU0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/43/662352/1.jpg?9611'"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img=item.find_all('img',{'class':'lazy image'})\n",
    "img\n",
    "img_src=img[0].get('data-src')\n",
    "img_src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_img(url_img,id):\n",
    "    r=requests.get(url_img)\n",
    "    if r.status_code==200:\n",
    "        #img = Image.open(BytesIO(r.content))\n",
    "        img_container=\"E:/git/Web-Scraping/\"+str(id)+'.jpg'\n",
    "        with open(img_container ,'wb') as f:\n",
    "            f.write(r.content) \n",
    "            print('Image saved!')\n",
    "    else:\n",
    "        print('Connection trouble!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image saved!\n"
     ]
    }
   ],
   "source": [
    "get_img(img_src,2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](2.jpg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "def path_to_image_html(path):\n",
    "    '''\n",
    "     This function essentially convert the image url to \n",
    "     '<img src=\"'+ path + '\"/>' format. And one can put any\n",
    "     formatting adjustments to control the height, aspect ratio, size etc.\n",
    "     within as in the below example. \n",
    "    '''\n",
    "    return (\"<img src=\"+\"'\"+path+\"'\"+\"/>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Brand</th>\n",
       "      <th>Specs</th>\n",
       "      <th>Price</th>\n",
       "      <th>Images</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hp</td>\n",
       "      <td>HP Pavilion 15 8th Gen Intel Core I5  1TB HDD 12GB RAM Win10</td>\n",
       "      <td>₦ 185,900</td>\n",
       "      <td><img src='https://ng.jumia.is/AJQFFwlEvzNiA0hhxrmAPn4hNU0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/43/662352/1.jpg?9611'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hp</td>\n",
       "      <td>250 G7, Intel Core I3, 1TB Hdd, 4GB Ram, Bluetooth, Webcam, Win 10, Plus Bag.</td>\n",
       "      <td>₦ 131,500</td>\n",
       "      <td><img src='https://ng.jumia.is/hI4RG1cn9dVfdaB220y5MB8Av3c=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/29/778723/1.jpg?9796'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Stream 11-Intel Celeron (4GB/32GB)Windows10+32Gb Flash Drive</td>\n",
       "      <td>₦ 59,800</td>\n",
       "      <td><img src='https://ng.jumia.is/GyO1-_IwxO9oFSA6wE4vgRhQJks=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/85/685982/1.jpg?8923'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Pavilion 15 8th Gen Intel Core I5 1TB HDD 12GB RAM Win10</td>\n",
       "      <td>₦ 186,000</td>\n",
       "      <td><img src='https://ng.jumia.is/2dRr5adVG4Oa21cK3jERZNA_RnI=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/78/832203/2.jpg?9719'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Lenovo</td>\n",
       "      <td>Ideapad Intel Celeron N4000 8thGen  4GB RAM 500GB HDD Wins10 + 32GB Flash Drive</td>\n",
       "      <td>₦ 78,800</td>\n",
       "      <td><img src='https://ng.jumia.is/S5ytgflbFMktiYthdGKlOvBF-hY=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/13/557092/1.jpg?4788'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Stream 11 Intel Celeron Mini Laptop(32GB HDD/2GB Ram- 32GB Flash - USB LIght)Wins 10 White</td>\n",
       "      <td>₦ 57,000</td>\n",
       "      <td><img src='https://ng.jumia.is/uHRmPDg3YOGVkNNSOIHx2jnBFWU=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/56/437933/1.jpg?8964'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Hp</td>\n",
       "      <td>15 AMD Dual Core E2 9000e 500GB HDD, 4GB RAM, WLAN, Webcam, Windows 10</td>\n",
       "      <td>₦ 77,400</td>\n",
       "      <td><img src='https://ng.jumia.is/uti-pq_zPPx7ej16Zu-ZIEEASk4=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/86/033762/1.jpg?1007'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Hp</td>\n",
       "      <td>15, AMD DUAL CORE, 500gb Hdd, 4gb RaM 15.6\" Win10 +16GB And Wireless Mouse</td>\n",
       "      <td>₦ 77,500</td>\n",
       "      <td><img src='https://ng.jumia.is/7XiaTAylWW_odPooIvEzUeNQez0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/31/593214/1.jpg?1306'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Hp</td>\n",
       "      <td>15 Intel Celeron (4GB RAM, 500GB HDD+ Free 32GB Flash- USB Light) Windows 10</td>\n",
       "      <td>₦ 83,500</td>\n",
       "      <td><img src='https://ng.jumia.is/3UZuclpjBQNMYA0oRIOgyieurp0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/97/690772/1.jpg?9024'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Lenovo</td>\n",
       "      <td>Ideapad 4GBRAM/500 HDD Intel Celeron Dual Core Windows 10 +free Mouse</td>\n",
       "      <td>₦ 78,990</td>\n",
       "      <td><img src='https://ng.jumia.is/z9-uTQuJTzNtRKdsQ67kz-H1cIo=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/28/981753/1.jpg?3618'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Generic</td>\n",
       "      <td>New Cream Apple Notebook Protective Shell Multi-model Specifications Apple 11 Inch 12 Inch 13inch 15 Pink</td>\n",
       "      <td>₦ 12,042</td>\n",
       "      <td><img src='https://ng.jumia.is/s0yTBX8tSJNSi6F6r34mSTVmpgk=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/18/846822/1.jpg?3134'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>DELL</td>\n",
       "      <td>Alienware 15 R5 Intel Core I7 1TB 256 16GB 8GB Nvidia 1080 Wins 10</td>\n",
       "      <td>₦ 1,900,000</td>\n",
       "      <td><img src='https://ng.jumia.is/55uyArYml9EnUF06OQzzlEKfRAo=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/87/347102/1.jpg?9039'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Microsoft</td>\n",
       "      <td>Surface Book 2 13\" Intel Core I7 512GB 16GB 2GB Nvidia 1050 Win 10 Pro</td>\n",
       "      <td>₦ 1,650,000</td>\n",
       "      <td><img src='https://ng.jumia.is/1ll8UatD71SVFkIyV6r04cajF7I=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/10/822652/1.jpg?5755'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Apple</td>\n",
       "      <td>Macbook Pro Intel Core I9,2.9GHz,1tb/16gb,Radeon Pro 560,15 Inch,Backlit-GREY</td>\n",
       "      <td>₦ 1,150,000</td>\n",
       "      <td><img src='https://ng.jumia.is/FqgXCN5tbKMmt8U_xRKkpjABibI=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/10/525093/1.jpg?0186'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Apple</td>\n",
       "      <td>Macbook PRO Intel Corei7, (16GB RAM 256GB )15 Inch MAC OS -2017 Touchbar</td>\n",
       "      <td>₦ 645,000</td>\n",
       "      <td><img src='https://ng.jumia.is/MHsoJ-izHmu2EOlBBfljKmYHStM=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/68/872652/1.jpg?5828'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Hp</td>\n",
       "      <td>OMEN X- 7thgeneration, Intel Core I7- (1TB+512 SSD,32GB), 8GB Nvidia GTX 1080,  17.3-Inch , Windows 10 Home</td>\n",
       "      <td>₦ 1,200,000</td>\n",
       "      <td><img src='https://ng.jumia.is/K4kHZ1I4uo3BoTLBqlSsPuPc2cI=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/63/560053/1.jpg?4135'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Microsoft</td>\n",
       "      <td>SURFACE BOOK 2 - 15\" Touch-Screen PixelSense™ Display - Intel Core I7 / 1TB/ 16GB NVIDIA GEFORCE GTX 1060 6GB GO GPU</td>\n",
       "      <td>₦ 920,000</td>\n",
       "      <td><img src='https://ng.jumia.is/PYThAKDW1GtFlB_TDvv11D2jqTU=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/83/505004/1.jpg?9437'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MacBook Pro® - 13.3\" - Intel Core I7 - 16GB Memory - 512GB - Space Gray/2017</td>\n",
       "      <td>₦ 630,000</td>\n",
       "      <td><img src='https://ng.jumia.is/MH0wJBF_dljuYCSg2VEhpkr5_Vs=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/53/455491/2.jpg?6665'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MACBOOK AIR,Intel Corei5,1.8GHz,256GB SSD,8GB RAM,2017 Edition</td>\n",
       "      <td>₦ 460,000</td>\n",
       "      <td><img src='https://ng.jumia.is/rVoG-U3VrwVet7usbn2Bt3wybMM=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/45/297891/1.jpg?4915'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Apple</td>\n",
       "      <td>Mac-book Pro 2017 16gb 256ssd Core I7 Quality Brand</td>\n",
       "      <td>₦ 730,000</td>\n",
       "      <td><img src='https://ng.jumia.is/vs4jfNDpq74O8lECXe8wpSk6ZrY=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/50/562152/1.jpg?6110'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Apple</td>\n",
       "      <td>Macbook Pro With Touchbar 8th Gen Intel Core I5 2.3GHz (8GB, 512GB SSD) 13.3-Inch Laptop (2018 Edition) - Grey</td>\n",
       "      <td>₦ 620,000</td>\n",
       "      <td><img src='https://ng.jumia.is/616leE79fWimpO2AqGltIweD5NE=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/54/071912/1.jpg?1290'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MacBook Air 13.3'' 1.8ghz Core I5 8GB, 128GB SSD 2017 Edition</td>\n",
       "      <td>₦ 339,999</td>\n",
       "      <td><img src='https://ng.jumia.is/rjR12Pmy8NbWfgvbm0iTssfoI8A=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/30/996853/1.jpg?1178'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MacBook Air 13inch Intel Core I5 1.6GHZ 8GB 128GB Mac OS GOLD</td>\n",
       "      <td>₦ 450,000</td>\n",
       "      <td><img src='https://ng.jumia.is/dAsGNpxBLZ_zChs7TkVFxiX5lxU=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/92/213382/1.jpg?5815'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Mini Laptop Intel Celeron 4GB,32GB Wins10s 11.6+32gb Flash</td>\n",
       "      <td>₦ 59,400</td>\n",
       "      <td><img src='https://ng.jumia.is/jgiiFMOiSowbJ4ehBjSBaag88j8=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/85/597304/1.jpg?4900'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Pavilion 15-Intel Core I7 (12GB RAM + 1TB HDD) Windows10 15\" Turbo Boost</td>\n",
       "      <td>₦ 290,000</td>\n",
       "      <td><img src='https://ng.jumia.is/ii5WXin6HnbRET211bDjIVAWKAg=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/88/493214/1.jpg?1306'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Spectre X360 13t Intel Core I7-8550U,1.8GHz,512ssd/16gb,Win10</td>\n",
       "      <td>₦ 545,000</td>\n",
       "      <td><img src='https://ng.jumia.is/6i7_bD8rXPROE-zQjhsQ5StMkE4=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/89/078603/1.jpg?5317'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MacBook Pro 13\" 2.3GHz Intel Core I5 8GB 128GB MPXR2LL/A (Newest Version) Silver</td>\n",
       "      <td>₦ 450,000</td>\n",
       "      <td><img src='https://ng.jumia.is/kljGaJURFKNATj1fk5D-1Zk4hkY=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/94/532281/1.jpg?4152'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Pavilion 15 15.6-Inch Intel Core I5-8250U, 8th Gen. (1TB HDD,12GB RAM) Backlit Keyboard Windows 10 Laptop + 32GB Flash</td>\n",
       "      <td>₦ 185,950</td>\n",
       "      <td><img src='https://ng.jumia.is/l6mZPuE321i5t6jmwewE2sv4oTI=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/01/854763/1.jpg?5418'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Samsung</td>\n",
       "      <td>8550P5C,3RD GEN, Intel I5,4gb,500gb,windows 8</td>\n",
       "      <td>₦ 220,000</td>\n",
       "      <td><img src='https://ng.jumia.is/OcEe-Xh6XnhNBOEfI4WBrNBLin0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/21/033842/1.jpg?2331'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Hp</td>\n",
       "      <td>ELITEBOOK 1040 G4 - INTEL CORE I7 (2.7 GHZ)- 14\" TOUCH SCREEN - 16 GB RAM - 512 GB SSD - WINDOWS 10 PRO</td>\n",
       "      <td>₦ 495,000</td>\n",
       "      <td><img src='https://ng.jumia.is/SblGhqwmnGl8yxZyR3AcfQ-928w=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/57/537043/1.jpg?9477'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Hp</td>\n",
       "      <td>HP OMEN 15-,8Th Gen Intel Corei7 2.2GHz,1TB HDD+128GB SSD 12GB RAM ,Windows 10 Home</td>\n",
       "      <td>₦ 480,000</td>\n",
       "      <td><img src='https://ng.jumia.is/U8R_RepA_eXUYBhMivCIX3t37rc=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/43/956232/1.jpg?2829'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>DELL</td>\n",
       "      <td>XPS 13 9365(2-IN-1)7Th Gen Intel Corei7 Touchscreen Slim 13\" Free Microsoft 2019</td>\n",
       "      <td>₦ 495,000</td>\n",
       "      <td><img src='https://ng.jumia.is/tpiNf6o5dBmqJz-MGunjmksupnM=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/33/735733/1.jpg?1064'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MACBOOK PRO RETINA, 13\" Core I5 2.3GHz 8GB Ram- 128GB Ssd Mid 2017</td>\n",
       "      <td>₦ 415,000</td>\n",
       "      <td><img src='https://ng.jumia.is/ZfbxqSTI426WUFMLzmpncg-qpPk=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/50/151383/1.jpg?0657'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Hp</td>\n",
       "      <td>255 AMD Quad Core (500GB HDD,4GB RAM,+ Free Bag) Freedos</td>\n",
       "      <td>₦ 80,990</td>\n",
       "      <td><img src='https://ng.jumia.is/tqf5MHFsTK-Io7pZ5jfQKqfM3mw=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/73/943031/1.jpg?2105'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Apple</td>\n",
       "      <td>MacBook Pro - 13\" Touch Bar - Intel Core I5 - 8GB Memory -128GB SSD( 2019 Model) SILVER</td>\n",
       "      <td>₦ 430,000</td>\n",
       "      <td><img src='https://ng.jumia.is/5t1VdaGdC092fbcacv2t6sTv4Nk=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/59/507204/1.jpg?0062'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Spectre X360 15-df0013dx - 15.6'' 4K UHD Touch - I7-8565U - 16gb RAM - 512gb SSD - Nvidia GeForce MX150</td>\n",
       "      <td>₦ 390,000</td>\n",
       "      <td><img src='https://ng.jumia.is/ap5nyxwyIyExeBwpiAqwvnsHPSA=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/65/620614/1.jpg?4760'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Spectre X360 13-AE015 Core I7-8850U 16GB RAM/360GB SSD 13.3 FHD Touch Convertible</td>\n",
       "      <td>₦ 390,000</td>\n",
       "      <td><img src='https://ng.jumia.is/AlUvns6Fzigou-fQ0K_otkZCDpo=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/65/911863/1.jpg?5956'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>DELL</td>\n",
       "      <td>Alienware Corei7.3- (16GB RAM, 1TB HDD + 256GB SSD) 8GB NVIDIA GeForce GTX , Windows 10 Home+ 32gb Flashdrive</td>\n",
       "      <td>₦ 410,000</td>\n",
       "      <td><img src='https://ng.jumia.is/lzJzTLbse9yA1tUmtV66nZyapOY=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/41/027463/1.jpg?8665'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Hp</td>\n",
       "      <td>Pavillion Touchsmart 15 Intel Core I7 16GB 2TB HDD,Graphics: 4GB Nvidia</td>\n",
       "      <td>₦ 340,000</td>\n",
       "      <td><img src='https://ng.jumia.is/FYMVyeybZAbmNXdvi98sp39Dc7c=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/29/940123/1.jpg?8188'/></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>DELL</td>\n",
       "      <td>Alienware Corei7.3-(16GB RAM, 2TB HDD+256GB SSD) 8GB NVIDIA GeForce GTX ,Windows10 Home+ BAG, 32GB FLASH, WIRELESS MOUSE</td>\n",
       "      <td>₦ 430,000</td>\n",
       "      <td><img src='https://ng.jumia.is/Ql8rdB0tB68gwOLSpn9a1LZ3Xn0=/fit-in/220x220/filters:fill(white):sharpen(1,0,false):quality(100)/product/92/027463/1.jpg?8665'/></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's create a dataframe that goes through the computer offers page of Jumia and returns all computers with their\n",
    "#respective name, brand and image.\n",
    "\n",
    "Jdf=pd.DataFrame(columns=['Brand','Specs','Price'])\n",
    "i=0\n",
    "images=[]\n",
    "for item in data:\n",
    "    brand=item.find_all('span',{'class':'brand'})[0].text.strip() \n",
    "    name=item.find_all('span',{'class':'name'})[0].text.strip()\n",
    "    price=item.find_all('span',{'class':'price'})[0].text.strip()\n",
    "    img=item.find_all('img',{'class':'lazy image'})\n",
    "    if (len(img) > 1):\n",
    "        img_src=img[1].get('data-src')\n",
    "    else:\n",
    "        img_src=img[0].get('data-src')\n",
    "    Jdf.loc[i]=[brand,name,price]\n",
    "    images.append(img_src)\n",
    "    i+=1\n",
    "Jdf['Images']=images\n",
    "print(len(Jdf))\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "HTML(Jdf.to_html(escape=False ,formatters=dict(Images=path_to_image_html)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now one final practice, from the main page of Wikipedia, we want to extract the languages available and the number of articles for each language.\n",
    "![](Wiki.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataFrame=pd.DataFrame(columns=['Lang', 'Num Artic'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://www.wikipedia.org/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "req=requests.get(url)\n",
    "status=req.status_code\n",
    "encoding=req.encoding\n",
    "text=BeautifulSoup(req.text,'html.parser')\n",
    "print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<div class=\"central-featured\" data-el-section=\"primary links\">\n",
       "<!-- Rankings from http://stats.wikimedia.org/EN/Sitemap.htm -->\n",
       "<!-- Article counts from http://meta.wikimedia.org/wiki/List_of_Wikipedias/Table -->\n",
       "<!-- #1. en.wikipedia.org - 1 750 870 000 views/day -->\n",
       "<div class=\"central-featured-lang lang1\" dir=\"ltr\" lang=\"en\">\n",
       "<a class=\"link-box\" data-slogan=\"The Free Encyclopedia\" href=\"//en.wikipedia.org/\" id=\"js-link-box-en\" title=\"English â Wikipedia â The Free Encyclopedia\">\n",
       "<strong>English</strong>\n",
       "<small><bdi dir=\"ltr\">5 935 000+</bdi> <span>articles</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #2. es.wikipedia.org - 258 167 000 views/day -->\n",
       "<div class=\"central-featured-lang lang2\" dir=\"ltr\" lang=\"es\">\n",
       "<a class=\"link-box\" data-slogan=\"La enciclopedia libre\" href=\"//es.wikipedia.org/\" id=\"js-link-box-es\" title=\"EspaÃ±ol â Wikipedia â La enciclopedia libre\">\n",
       "<strong>EspaÃ±ol</strong>\n",
       "<small><bdi dir=\"ltr\">1 546 000+</bdi> <span>artÃ­culos</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #3. ja.wikipedia.org - 241 813 000 views/day -->\n",
       "<div class=\"central-featured-lang lang3\" dir=\"ltr\" lang=\"ja\">\n",
       "<a class=\"link-box\" data-slogan=\"ããªã¼ç¾ç§äºå\n",
       "¸\" href=\"//ja.wikipedia.org/\" id=\"js-link-box-ja\" title=\"Nihongo â ã¦ã£ã­ããã£ã¢ â ããªã¼ç¾ç§äºå\n",
       "¸\">\n",
       "<strong>æ¥æ¬èª</strong>\n",
       "<small><bdi dir=\"ltr\">1 169 000+</bdi> <span>è¨äº</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #4. de.wikipedia.org - 201 562 000 views/day -->\n",
       "<div class=\"central-featured-lang lang4\" dir=\"ltr\" lang=\"de\">\n",
       "<a class=\"link-box\" data-slogan=\"Die freie EnzyklopÃ¤die\" href=\"//de.wikipedia.org/\" id=\"js-link-box-de\" title=\"Deutsch â Wikipedia â Die freie EnzyklopÃ¤die\">\n",
       "<strong>Deutsch</strong>\n",
       "<small><bdi dir=\"ltr\">2 345 000+</bdi> <span>Artikel</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #5. ru.wikipedia.org - 182 638 000 views/day -->\n",
       "<div class=\"central-featured-lang lang5\" dir=\"ltr\" lang=\"ru\">\n",
       "<a class=\"link-box\" data-slogan=\"Ð¡Ð²Ð¾Ð±Ð¾Ð´Ð½Ð°Ñ ÑÐ½ÑÐ¸ÐºÐ»Ð¾Ð¿ÐµÐ´Ð¸Ñ\" href=\"//ru.wikipedia.org/\" id=\"js-link-box-ru\" title=\"Russkiy â ÐÐ¸ÐºÐ¸Ð¿ÐµÐ´Ð¸Ñ â Ð¡Ð²Ð¾Ð±Ð¾Ð´Ð½Ð°Ñ ÑÐ½ÑÐ¸ÐºÐ»Ð¾Ð¿ÐµÐ´Ð¸Ñ\">\n",
       "<strong>Ð ÑÑÑÐºÐ¸Ð¹</strong>\n",
       "<small><bdi dir=\"ltr\">1 569 000+</bdi> <span>ÑÑÐ°ÑÐµÐ¹</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #6. fr.wikipedia.org - 160 506 000 views/day -->\n",
       "<div class=\"central-featured-lang lang6\" dir=\"ltr\" lang=\"fr\">\n",
       "<a class=\"link-box\" data-slogan=\"LâencyclopÃ©die libre\" href=\"//fr.wikipedia.org/\" id=\"js-link-box-fr\" title=\"FranÃ§ais â WikipÃ©dia â LâencyclopÃ©die libre\">\n",
       "<strong>FranÃ§ais</strong>\n",
       "<small><bdi dir=\"ltr\">2 141 000+</bdi> <span>articles</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #7. it.wikipedia.org - 122 982 000 views/day -->\n",
       "<div class=\"central-featured-lang lang7\" dir=\"ltr\" lang=\"it\">\n",
       "<a class=\"link-box\" data-slogan=\"L'enciclopedia libera\" href=\"//it.wikipedia.org/\" id=\"js-link-box-it\" title=\"Italiano â Wikipedia â L'enciclopedia libera\">\n",
       "<strong>Italiano</strong>\n",
       "<small><bdi dir=\"ltr\">1 554 000+</bdi> <span>voci</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #8. zh.wikipedia.org - 89 577 000 views/day -->\n",
       "<div class=\"central-featured-lang lang8\" dir=\"ltr\" lang=\"zh\">\n",
       "<a class=\"link-box\" data-converttitle-hans=\"ZhÅngwÃ©n â ç»´åºç¾ç§ â èªç±çç¾ç§å\n",
       "¨ä¹¦\" data-slogan=\"èªç±çç¾ç§å\n",
       "¨æ¸\" href=\"//zh.wikipedia.org/\" id=\"js-link-box-zh\" title=\"ZhÅngwÃ©n â ç¶­åºç¾ç§ â èªç±çç¾ç§å\n",
       "¨æ¸\">\n",
       "<strong>ä¸­æ</strong>\n",
       "<small><bdi dir=\"ltr\">1 074 000+</bdi> <span data-convert-hans=\"æ¡ç®\" id=\"zh_art\">æ¢ç®</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #9. pt.wikipedia.org - 82 353 000 views/day -->\n",
       "<div class=\"central-featured-lang lang9\" dir=\"ltr\" lang=\"pt\">\n",
       "<a class=\"link-box\" data-slogan=\"A enciclopÃ©dia livre\" href=\"//pt.wikipedia.org/\" id=\"js-link-box-pt\" title=\"PortuguÃªs â WikipÃ©dia â A enciclopÃ©dia livre\">\n",
       "<strong>PortuguÃªs</strong>\n",
       "<small><bdi dir=\"ltr\">1 014 000+</bdi> <span>artigos</span></small>\n",
       "</a>\n",
       "</div>\n",
       "<!-- #10. pl.wikipedia.org - 55 278 000 views/day -->\n",
       "<div class=\"central-featured-lang lang10\" dir=\"ltr\" lang=\"pl\">\n",
       "<a class=\"link-box\" data-slogan=\"Wolna encyklopedia\" href=\"//pl.wikipedia.org/\" id=\"js-link-box-pl\" title=\"Polski â Wikipedia â Wolna encyklopedia\">\n",
       "<strong>Polski</strong>\n",
       "<small><bdi dir=\"ltr\">1 360 000+</bdi> <span>haseÅ</span></small>\n",
       "</a>\n",
       "</div>\n",
       "</div>"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=text.findAll('div',{'class':'central-featured'})\n",
    "for item in data:\n",
    "    pass\n",
    "item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=7\n",
    "j=0\n",
    "while i<44:\n",
    "    l=[item.contents[i].findAll('strong')[0].text.strip(),item.contents[i].findAll('bdi')[0].text.strip()]\n",
    "    DataFrame.loc[j]=l\n",
    "    i+=4\n",
    "    j+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lang</th>\n",
       "      <th>Num Artic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>English</td>\n",
       "      <td>5 935 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EspaÃ±ol</td>\n",
       "      <td>1 546 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>æ¥æ¬èª</td>\n",
       "      <td>1 169 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deutsch</td>\n",
       "      <td>2 345 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ð ÑÑÑÐºÐ¸Ð¹</td>\n",
       "      <td>1 569 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>FranÃ§ais</td>\n",
       "      <td>2 141 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Italiano</td>\n",
       "      <td>1 554 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ä¸­æ</td>\n",
       "      <td>1 074 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>PortuguÃªs</td>\n",
       "      <td>1 014 000+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Polski</td>\n",
       "      <td>1 360 000+</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Lang   Num Artic\n",
       "0  English         5 935 000+\n",
       "1  EspaÃ±ol        1 546 000+\n",
       "2  æ¥æ¬èª       1 169 000+\n",
       "3  Deutsch         2 345 000+\n",
       "4  Ð ÑÑÑÐºÐ¸Ð¹  1 569 000+\n",
       "5  FranÃ§ais       2 141 000+\n",
       "6  Italiano        1 554 000+\n",
       "7  ä¸­æ          1 074 000+\n",
       "8  PortuguÃªs      1 014 000+\n",
       "9  Polski          1 360 000+"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see we have some problems w/ formating languages names, this due to encoding, but we can verify results are correct. Otherwise, dealing with the encoding or systematically modify the languages manually are easy tasks to do."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}