BeautifulSoup- - 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html = '<!DOCTYPE html>\\\n",
    "<html>\\\n",
    "<head>\\\n",
    "<title> Testing Web Page </title>\\\n",
    "</head>\\\n",
    "<body>\\\n",
    "<h1> Web Scraping </h1>\\\n",
    "<p id = \"first_para\">\\\n",
    "Let\\'s start learning \\\n",
    "<b>\\\n",
    "Web Scraping\\\n",
    "</b>\\\n",
    "</p>\\\n",
    "<p class = \"abc\" id = \"second_para\">\\\n",
    "You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a>\\\n",
    "</p>\\\n",
    "<p class = \"abc\">\\\n",
    "<a href = \"https://codingninjas.in/\"> Coding Ninjas </a>\\\n",
    "</p>\\\n",
    "</body>\\\n",
    "</html>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<!DOCTYPE html>\n",
       "<html><head><title> Testing Web Page </title></head><body><h1> Web Scraping </h1><p id=\"first_para\">Let's start learning <b>Web Scraping</b></p><p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p><p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p></body></html>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = BeautifulSoup(html, 'html.parser')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   Testing Web Page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <h1>\n",
      "   Web Scraping\n",
      "  </h1>\n",
      "  <p id=\"first_para\">\n",
      "   Let's start learning\n",
      "   <b>\n",
      "    Web Scraping\n",
      "   </b>\n",
      "  </p>\n",
      "  <p class=\"abc\" id=\"second_para\">\n",
      "   You can read more about BeautifulSoup from\n",
      "   <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\">\n",
      "    here\n",
      "   </a>\n",
      "  </p>\n",
      "  <p class=\"abc\">\n",
      "   <a href=\"https://codingninjas.in/\">\n",
      "    Coding Ninjas\n",
      "   </a>\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "print(data.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<title> Testing Web Page </title>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<head><title> Testing Web Page </title></head>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h1> Web Scraping </h1>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.h1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.h2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<title> Testing Web Page </title>\n",
      "title\n",
      " Testing Web Page \n"
     ]
    }
   ],
   "source": [
    "print(data.title)\n",
    "print(data.title.name)\n",
    "print(data.title.string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n"
     ]
    }
   ],
   "source": [
    "print(data.title.attrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'first_para'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.p.attrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'first_para'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.p['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'first_para'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.p.get('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# data.p['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\" Testing Web Page  Web Scraping Let's start learning Web ScrapingYou can read more about BeautifulSoup from  here  Coding Ninjas \""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.find('p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.find('pr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h1> Web Scraping </h1>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.find('h1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>\n",
      "<p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p>\n",
      "<p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p>\n"
     ]
    }
   ],
   "source": [
    "li = data.find_all('p')\n",
    "\n",
    "for i in li : \n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>,\n",
       " <p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p>,\n",
       " <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a>,\n",
       " <p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p>,\n",
       " <a href=\"https://codingninjas.in/\"> Coding Ninjas </a>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.find_all(['p', 'a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<html><head><title> Testing Web Page </title></head><body><h1> Web Scraping </h1><p id=\"first_para\">Let's start learning <b>Web Scraping</b></p><p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p><p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p></body></html>,\n",
       " <head><title> Testing Web Page </title></head>,\n",
       " <title> Testing Web Page </title>,\n",
       " <body><h1> Web Scraping </h1><p id=\"first_para\">Let's start learning <b>Web Scraping</b></p><p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p><p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p></body>,\n",
       " <h1> Web Scraping </h1>,\n",
       " <p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>,\n",
       " <b>Web Scraping</b>,\n",
       " <p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p>,\n",
       " <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a>,\n",
       " <p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p>,\n",
       " <a href=\"https://codingninjas.in/\"> Coding Ninjas </a>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.find_all(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.find_all(id = 'first_para')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p>,\n",
       " <p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p>]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.find_all(class_ = 'abc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   Testing Web Page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <h1>\n",
      "   Web Scraping\n",
      "  </h1>\n",
      "  <p id=\"first_para\">\n",
      "   Let's start learning\n",
      "   <b>\n",
      "    Web Scraping\n",
      "   </b>\n",
      "  </p>\n",
      "  <p class=\"abc\" id=\"second_para\">\n",
      "   You can read more about BeautifulSoup from\n",
      "   <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\">\n",
      "    here\n",
      "   </a>\n",
      "  </p>\n",
      "  <p class=\"abc\">\n",
      "   <a href=\"https://codingninjas.in/\">\n",
      "    Coding Ninjas\n",
      "   </a>\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "print(data.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<head><title> Testing Web Page </title></head>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<title> Testing Web Page </title>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<title> Testing Web Page </title>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Testing Web Page '"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.title.string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>\n",
      "<p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p>\n",
      "<p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p>\n"
     ]
    }
   ],
   "source": [
    "li = data.find_all('p')\n",
    "\n",
    "for i in li : \n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "None\n",
      " Coding Ninjas \n"
     ]
    }
   ],
   "source": [
    "for i in li : \n",
    "    print(i.string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"Let's start learning \", 'Web Scraping']\n",
      "['You can read more about BeautifulSoup from ', ' here ']\n",
      "[' Coding Ninjas ']\n"
     ]
    }
   ],
   "source": [
    "for i in li : \n",
    "    print(list(i.strings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"Let's start learning\", 'Web Scraping']\n",
      "['You can read more about BeautifulSoup from', 'here']\n",
      "['Coding Ninjas']\n"
     ]
    }
   ],
   "source": [
    "for i in li : \n",
    "    print(list(i.stripped_strings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "[<head><title> Testing Web Page </title></head>, <body><h1> Web Scraping </h1><p id=\"first_para\">Let's start learning <b>Web Scraping</b></p><p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p><p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p></body>]\n"
     ]
    }
   ],
   "source": [
    "li = data.html.contents\n",
    "print(len(li))\n",
    "print(li)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<head><title> Testing Web Page </title></head>\n",
      "<body><h1> Web Scraping </h1><p id=\"first_para\">Let's start learning <b>Web Scraping</b></p><p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p><p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p></body>\n"
     ]
    }
   ],
   "source": [
    "li_22 = data.html.children\n",
    "for i in li_22 : \n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    }
   ],
   "source": [
    "desc = list(data.html.descendants)\n",
    "\n",
    "print(len(desc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<head><title> Testing Web Page </title></head>,\n",
       " <title> Testing Web Page </title>,\n",
       " ' Testing Web Page ',\n",
       " <body><h1> Web Scraping </h1><p id=\"first_para\">Let's start learning <b>Web Scraping</b></p><p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p><p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p></body>,\n",
       " <h1> Web Scraping </h1>,\n",
       " ' Web Scraping ',\n",
       " <p id=\"first_para\">Let's start learning <b>Web Scraping</b></p>,\n",
       " \"Let's start learning \",\n",
       " <b>Web Scraping</b>,\n",
       " 'Web Scraping',\n",
       " <p class=\"abc\" id=\"second_para\">You can read more about BeautifulSoup from <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a></p>,\n",
       " 'You can read more about BeautifulSoup from ',\n",
       " <a href=\"https://www.crummy.com/software/BeautifulSoup/bs4/doc/\"> here </a>,\n",
       " ' here ',\n",
       " <p class=\"abc\"><a href=\"https://codingninjas.in/\"> Coding Ninjas </a></p>,\n",
       " <a href=\"https://codingninjas.in/\"> Coding Ninjas </a>,\n",
       " ' Coding Ninjas ']"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}