Pokémon scraping (Gotta Catch 'Em All).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will try to get the information of all Pokémons (Name,id,attribute,etc...) from: https://pokemondb.net/pokedex/national\n",
    "using web scraping techniques."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests # To be able to access websites we wish to scrap.\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import pandas as pd\n",
    "import time\n",
    "from IPython.display import Image, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://pokemondb.net/pokedex/national'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "req=requests.get(url) \n",
    "status=req.status_code # Status of the connection attempt if everything is good this should have a 200 value.\n",
    "encoding=req.encoding\n",
    "text=BeautifulSoup(req.text,'html.parser')\n",
    "print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<span class=\"infocard-lg-data text-muted\"><small>#809</small><br/><a class=\"ent-name\" href=\"/pokedex/melmetal\">Melmetal</a><br/><small><a class=\"itype steel\" href=\"/type/steel\">Steel</a></small></span>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=text.findAll('span',{'class':'infocard-lg-data'})\n",
    "for item in data:\n",
    "    pass\n",
    "item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<span class=\"infocard-lg-img\"><a href=\"/pokedex/melmetal\"><span class=\"img-fixed img-sprite\" data-alt=\"Melmetal icon\" data-src=\"https://img.pokemondb.net/sprites/ultra-sun-ultra-moon/small/melmetal.jpg\"></span></a></span>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgs=text.findAll('span',{'class':'infocard-lg-img'})\n",
    "for img in imgs:\n",
    "    pass\n",
    "img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://img.pokemondb.net/sprites/ultra-sun-ultra-moon/small/melmetal.jpg'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PokId=item.find('small').text\n",
    "PokName=item.find('a',{'class':'ent-name'}).text\n",
    "PokAttrib=item.find('a',{'class':'itype'}).text\n",
    "PokImg=img.find('a').find('span',{'class':'img-fixed'}).get('data-src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_img(url_img,id):\n",
    "    r=requests.get(url_img)\n",
    "    if r.status_code==200:\n",
    "        #img = Image.open(BytesIO(r.content))\n",
    "        img_container=\"C:/Users/YsfEss/Desktop/Pokemon/\"+str(id)+'.jpg'\n",
    "        with open(img_container ,'wb') as f:\n",
    "            f.write(r.content) \n",
    "            print('Image saved!')\n",
    "    else:\n",
    "        print('Connection trouble!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Name</th>\n",
       "      <th>Attribute</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#001</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#002</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>#003</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>#004</td>\n",
       "      <td>Charmander</td>\n",
       "      <td>Fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>#005</td>\n",
       "      <td>Charmeleon</td>\n",
       "      <td>Fire</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Id        Name Attribute\n",
       "0  #001   Bulbasaur     Grass\n",
       "1  #002     Ivysaur     Grass\n",
       "2  #003    Venusaur     Grass\n",
       "3  #004  Charmander      Fire\n",
       "4  #005  Charmeleon      Fire"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating database of Pokemons\n",
    "data=text.findAll('span',{'class':'infocard-lg-data'})\n",
    "PokeDB=pd.DataFrame(columns=['Id','Name','Attribute'])\n",
    "i=0\n",
    "for item in data:\n",
    "    PokId=item.find('small').text\n",
    "    PokName=item.find('a',{'class':'ent-name'}).text\n",
    "    PokAttrib=item.find('a',{'class':'itype'}).text\n",
    "    temp=[PokId,PokName,PokAttrib]\n",
    "    PokeDB.loc[i]=temp  \n",
    "    i+=1\n",
    "pd.DataFrame.to_csv(PokeDB,'PokeDB.csv')\n",
    "PokeDB.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving images:\n",
    "i=1\n",
    "for img in imgs:\n",
    "    PokImg=img.find('a').find('span',{'class':'img-fixed'}).get('data-src')\n",
    "    get_img(PokImg,i)\n",
    "    i+=1\n",
    "    if i%10==0:\n",
    "        time.sleep(2) # To not be agressive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we will extract info about Pokédex data, Training & Breeding into 3 tables. \n",
    "The choosen pokemon will be: Gengar https://pokemondb.net/pokedex/gengar\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://pokemondb.net/pokedex/gengar'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "req=requests.get(url) \n",
    "status=req.status_code # Status of the connection attempt if everything is good this should have a 200 value.\n",
    "encoding=req.encoding\n",
    "text=BeautifulSoup(req.text,'html.parser')\n",
    "print(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=text.find_all('table',{'class':'vitals-table'})\n",
    "tables=[]\n",
    "i=0\n",
    "for item in data:\n",
    "    if i<3:\n",
    "        tables.append(item)\n",
    "    i+=1\n",
    "Pokedex=tables[0]\n",
    "Training=tables[1]\n",
    "Breeding=tables[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tr>\n",
       "<th>Local №</th>\n",
       "<td>094 <small class=\"text-muted\">(Red/Blue/Yellow)</small><br/>060 <small class=\"text-muted\">(Gold/Silver/Crystal)</small><br/>094 <small class=\"text-muted\">(FireRed/LeafGreen)</small><br/>071 <small class=\"text-muted\">(Diamond/Pearl)</small><br/>071 <small class=\"text-muted\">(Platinum)</small><br/>060 <small class=\"text-muted\">(HeartGold/SoulSilver)</small><br/>032 <small class=\"text-muted\">(X/Y — Mountain Kalos)</small><br/>063 <small class=\"text-muted\">(Sun/Moon — Alola dex)</small><br/>075 <small class=\"text-muted\">(U.Sun/U.Moon — Alola dex)</small><br/>094 <small class=\"text-muted\">(Let's Go Pikachu/Let's Go Eevee)</small><br/></td>\n",
       "</tr>"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Pokedex.find_all('tr')[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pokédex data:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>National N</th>\n",
       "      <th>Type</th>\n",
       "      <th>Power</th>\n",
       "      <th>Species</th>\n",
       "      <th>Height</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Abilities</th>\n",
       "      <th>Local N</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>094</td>\n",
       "      <td>Ghost</td>\n",
       "      <td>Poison</td>\n",
       "      <td>Shadow Pokémon</td>\n",
       "      <td>1.5 m (4′11″)</td>\n",
       "      <td>40.5 kg (89.3 lbs)</td>\n",
       "      <td>Cursed Body</td>\n",
       "      <td>094 :(Red/Blue/Yellow)<br>060 :(Gold/Silver/Crystal)<br>094 :(FireRed/LeafGreen)<br>071 :(Diamond/Pearl)<br>071 :(Platinum)<br>060 :(HeartGold/SoulSilver)<br>032 :(X/Y — Mountain Kalos)<br>063 :(Sun/Moon — Alola dex)<br>075 :(U.Sun/U.Moon — Alola dex)<br>094 :(Let's Go Pikachu/Let's Go Eevee)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pokedex:\n",
    "TPokedex=pd.DataFrame(columns=['National N','Type','Power','Species','Height','Weight','Abilities','Local N'])\n",
    "K=Pokedex.find_all('tr')\n",
    "NatNum=K[0].find('strong').text\n",
    "Type=K[1].find_all('a')[0].text\n",
    "Pow=K[1].find_all('a')[1].text\n",
    "Spec=K[2].find('td').text\n",
    "Height=K[3].find('td').text\n",
    "Weight=K[4].find('td').text\n",
    "Abilities=K[5].find('a').text\n",
    "\n",
    "# Cleaning the list form useless tags\n",
    "cleanL=[]\n",
    "d=Pokedex.find_all('tr')[6].find_all('td')[0].contents\n",
    "for i in range (len(d)):\n",
    "    if str(d[i])!='<br/>':\n",
    "        cleanL.append(d[i])\n",
    "\n",
    "LocalNum=[]\n",
    "for i in range (0,len(cleanL)-1,2):\n",
    "    LocalNum.append([cleanL[i],cleanL[i+1].text])\n",
    "LocalNum \n",
    "\n",
    "\n",
    "TPokedex.loc[0]=[NatNum,Type,Pow,Spec,Height,Weight,Abilities,\"\\n\".join([':'.join(item) for item in LocalNum])]\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "print('Pokédex data:')\n",
    "HTML(TPokedex.to_html(escape=False).replace(\"\\\\n\",\"<br>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Table:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EV yield</th>\n",
       "      <th>Catch rate</th>\n",
       "      <th>Base Friendship</th>\n",
       "      <th>Base Exp.</th>\n",
       "      <th>Growth Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3 Special Attack</td>\n",
       "      <td>45 (5.9% with PokéBall, full HP)</td>\n",
       "      <td>70 (normal)</td>\n",
       "      <td>225</td>\n",
       "      <td>Medium Slow</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training:\n",
    "\n",
    "TTrain=pd.DataFrame(columns=['EV yield','Catch rate','Base Friendship','Base Exp.','Growth Rate'])\n",
    "K=Training.find_all('tr')\n",
    "EVyield=K[0].find('td').text.strip()\n",
    "Catchr=K[1].find_all('td')[0].text\n",
    "BFri=K[2].find_all('td')[0].text\n",
    "BExp=K[3].find('td').text\n",
    "GrRate=K[4].find('td').text\n",
    "\n",
    "TTrain.loc[0]=[EVyield,Catchr,BFri,BExp,GrRate]\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "print('Training Table:')\n",
    "HTML(TTrain.to_html(escape=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Breading Table:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Egg Groups</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Egg cycles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Amorphous</td>\n",
       "      <td>50% male, 50% female</td>\n",
       "      <td>20 (4,884–5,140 steps)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Breading: \n",
    "\n",
    "TBread=pd.DataFrame(columns=['Egg Groups','Gender','Egg cycles'])\n",
    "K=Breeding.find_all('tr')\n",
    "EGr=K[0].find('a').text.strip()\n",
    "Gen=K[1].find_all('td')[0].text\n",
    "EgCy=K[2].find_all('td')[0].text.strip()\n",
    "\n",
    "\n",
    "TBread.loc[0]=[EGr,Gen,EgCy]\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "print('Breading Table:')\n",
    "HTML(TBread.to_html(escape=False))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}