21. Text-Summarizer NLTK.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extraction-based summarization\n",
    "**In extraction-based summarization, a subset of words that represent the most important points is pulled from a piece of text and combined to make a summary.** "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code Blue-print\n",
    "\n",
    "### Creating a dictionary for the word frequency table\n",
    "frequency_table = _create_dictionary_table(article)\n",
    "\n",
    "### Tokenizing the sentences\n",
    "sentences = sent_tokenize(article)\n",
    "\n",
    "### Algorithm for scoring a sentence by its words\n",
    "sentence_scores = _calculate_sentence_scores(sentences, frequency_table)\n",
    "\n",
    "### Getting the threshold\n",
    "threshold = _calculate_average_score(sentence_scores)\n",
    "\n",
    "### Producing the summary\n",
    "article_summary = _get_article_summary(sentences, sentence_scores, 1.5 * threshold)\n",
    "\n",
    "print(article_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bs4 as BeautifulSoup\n",
    "import urllib.request  \n",
    "\n",
    "# Fetching the content from the URL\n",
    "fetched_data = urllib.request.urlopen('https://en.wikipedia.org/wiki/20th_century')\n",
    "\n",
    "article_read = fetched_data.read()\n",
    "\n",
    "# Parsing the URL content and storing in a variable\n",
    "article_parsed = BeautifulSoup.BeautifulSoup(article_read,'html.parser')\n",
    "\n",
    "# Returning <p> tags\n",
    "paragraphs = article_parsed.find_all('p')\n",
    "\n",
    "article_content = ''\n",
    "\n",
    "# Looping through the paragraphs and adding them to the variable\n",
    "for p in paragraphs:  \n",
    "    article_content += p.text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BeautifulSoup converts the incoming text to Unicode characters and the outgoing text to UTF-8 characters, saving you the hassle of managing different charset encodings while scraping text from the web."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21224"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(article_content)   #len of char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One time run\n",
    "# import nltk\n",
    "# nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "def _create_dictionary_table(text_string):\n",
    "    stemmer=PorterStemmer()\n",
    "    words = word_tokenize(text_string)   #tokenizing string\n",
    "    frequency_table=dict()      #creating a dict object\n",
    "    stop_words=stopwords.words(\"english\")\n",
    "    \n",
    "    for wd in  words:\n",
    "        wd=stemmer.stem(wd)\n",
    "        if wd in stop_words:\n",
    "            continue\n",
    "        if wd in frequency_table.keys():\n",
    "            frequency_table[wd]+=1\n",
    "        else:\n",
    "            frequency_table[wd]=1\n",
    "            \n",
    "    return frequency_table\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'20th': 20,\n",
       " '(': 10,\n",
       " 'twentieth': 1,\n",
       " ')': 10,\n",
       " 'centuri': 46,\n",
       " 'began': 4,\n",
       " 'januari': 1,\n",
       " '1': 3,\n",
       " ',': 299,\n",
       " '1901': 2,\n",
       " '[': 39,\n",
       " ']': 39,\n",
       " 'end': 14,\n",
       " 'decemb': 1,\n",
       " '31': 1,\n",
       " '2000': 2,\n",
       " '.': 125,\n",
       " '2': 2,\n",
       " 'It': 6,\n",
       " 'wa': 36,\n",
       " 'tenth': 1,\n",
       " 'final': 2,\n",
       " '2nd': 1,\n",
       " 'millennium': 1,\n",
       " 'unlik': 1,\n",
       " 'year': 12,\n",
       " 'leap': 2,\n",
       " 'first': 19,\n",
       " 'gregorian': 1,\n",
       " 'calendar': 1,\n",
       " 'sinc': 9,\n",
       " '1600': 1,\n",
       " 'domin': 3,\n",
       " 'chain': 2,\n",
       " 'event': 1,\n",
       " 'herald': 2,\n",
       " 'signific': 4,\n",
       " 'chang': 7,\n",
       " 'world': 49,\n",
       " 'histori': 7,\n",
       " 'redefin': 1,\n",
       " 'era': 1,\n",
       " ':': 4,\n",
       " 'spanish': 1,\n",
       " 'flu': 1,\n",
       " 'pandem': 1,\n",
       " 'war': 39,\n",
       " 'I': 4,\n",
       " 'II': 8,\n",
       " 'nuclear': 10,\n",
       " 'power': 12,\n",
       " 'space': 4,\n",
       " 'explor': 3,\n",
       " 'nation': 19,\n",
       " 'decolon': 3,\n",
       " 'cold': 5,\n",
       " 'post-cold': 1,\n",
       " 'conflict': 8,\n",
       " ';': 12,\n",
       " 'intergovernment': 1,\n",
       " 'organ': 3,\n",
       " 'cultur': 7,\n",
       " 'homogen': 2,\n",
       " 'develop': 17,\n",
       " 'emerg': 3,\n",
       " 'transport': 4,\n",
       " 'commun': 10,\n",
       " 'technolog': 20,\n",
       " 'poverti': 2,\n",
       " 'reduct': 1,\n",
       " 'popul': 11,\n",
       " 'growth': 2,\n",
       " 'awar': 2,\n",
       " 'environment': 5,\n",
       " 'degrad': 1,\n",
       " 'ecolog': 2,\n",
       " 'extinct': 2,\n",
       " '3': 1,\n",
       " '4': 1,\n",
       " 'birth': 1,\n",
       " 'digit': 2,\n",
       " 'revolut': 9,\n",
       " 'enabl': 1,\n",
       " 'wide': 2,\n",
       " 'adopt': 1,\n",
       " 'mo': 1,\n",
       " 'transistor': 1,\n",
       " 'integr': 2,\n",
       " 'circuit': 1,\n",
       " 'saw': 3,\n",
       " 'great': 3,\n",
       " 'advanc': 7,\n",
       " 'gener': 3,\n",
       " 'medic': 4,\n",
       " 'late': 4,\n",
       " '1980': 3,\n",
       " 'allow': 3,\n",
       " 'near-instantan': 1,\n",
       " 'worldwid': 1,\n",
       " 'comput': 6,\n",
       " 'genet': 1,\n",
       " 'modif': 1,\n",
       " 'life': 6,\n",
       " 'largest': 2,\n",
       " 'transform': 2,\n",
       " 'order': 2,\n",
       " 'fall': 1,\n",
       " 'rome': 1,\n",
       " 'global': 19,\n",
       " 'total': 3,\n",
       " 'fertil': 1,\n",
       " 'rate': 2,\n",
       " 'sea': 1,\n",
       " 'level': 2,\n",
       " 'rise': 3,\n",
       " 'collaps': 3,\n",
       " 'increas': 5,\n",
       " 'result': 10,\n",
       " 'competit': 3,\n",
       " 'land': 1,\n",
       " 'dwindl': 1,\n",
       " 'resourc': 3,\n",
       " 'acceler': 2,\n",
       " 'deforest': 1,\n",
       " 'water': 1,\n",
       " 'deplet': 1,\n",
       " 'mass': 3,\n",
       " 'mani': 15,\n",
       " \"'s\": 22,\n",
       " 'speci': 1,\n",
       " 'declin': 2,\n",
       " 'consequ': 1,\n",
       " 'dealt': 1,\n",
       " 'man-mad': 1,\n",
       " 'warm': 4,\n",
       " 'extrem': 1,\n",
       " 'weather': 1,\n",
       " 'condit': 1,\n",
       " 'averag': 2,\n",
       " 'temperatur': 1,\n",
       " 'earth': 3,\n",
       " 'ha': 3,\n",
       " 'littl': 2,\n",
       " '1°': 1,\n",
       " 'celsiu': 1,\n",
       " '2°': 1,\n",
       " 'fahrenheit': 1,\n",
       " '1880': 1,\n",
       " 'two-third': 1,\n",
       " 'occur': 3,\n",
       " '1975': 1,\n",
       " 'roughli': 2,\n",
       " '0.15–0.20': 1,\n",
       " '°C': 1,\n",
       " 'per': 1,\n",
       " 'decad': 4,\n",
       " '5': 1,\n",
       " 'repercuss': 1,\n",
       " 'craft': 1,\n",
       " 'peopl': 14,\n",
       " 'unit': 13,\n",
       " 'ani': 1,\n",
       " 'previou': 1,\n",
       " 'time': 9,\n",
       " 'human': 8,\n",
       " 'exemplifi': 1,\n",
       " 'establish': 8,\n",
       " 'intern': 8,\n",
       " 'law': 2,\n",
       " 'aid': 4,\n",
       " 'marshal': 2,\n",
       " 'plan—which': 1,\n",
       " 'spent': 1,\n",
       " '$': 2,\n",
       " '13': 2,\n",
       " 'billion': 8,\n",
       " '100': 2,\n",
       " '2019': 1,\n",
       " 'US': 6,\n",
       " 'dollar': 1,\n",
       " '6': 2,\n",
       " 'rebuild': 2,\n",
       " 'economi': 4,\n",
       " 'post-war': 4,\n",
       " 'nations—launch': 1,\n",
       " '``': 5,\n",
       " 'pax': 1,\n",
       " 'americana': 1,\n",
       " \"''\": 3,\n",
       " 'throughout': 1,\n",
       " 'latter': 2,\n",
       " 'half': 4,\n",
       " 'rivalri': 1,\n",
       " 'state': 11,\n",
       " 'soviet': 9,\n",
       " 'union': 8,\n",
       " 'creat': 3,\n",
       " 'enorm': 1,\n",
       " 'tension': 1,\n",
       " 'around': 4,\n",
       " 'manifest': 1,\n",
       " 'variou': 3,\n",
       " 'arm': 4,\n",
       " 'omnipres': 1,\n",
       " 'danger': 1,\n",
       " 'prolifer': 2,\n",
       " 'dissolut': 1,\n",
       " '1991': 2,\n",
       " 'european': 4,\n",
       " 'allianc': 2,\n",
       " 'west': 3,\n",
       " 'though': 1,\n",
       " 'one': 7,\n",
       " 'six': 1,\n",
       " 'live': 4,\n",
       " 'communist': 4,\n",
       " 'rule': 2,\n",
       " 'mostli': 1,\n",
       " 'china': 5,\n",
       " 'rapidli': 3,\n",
       " 'econom': 6,\n",
       " 'geopolit': 1,\n",
       " 'took': 1,\n",
       " 'two-hundr': 1,\n",
       " 'thousand': 2,\n",
       " '1804': 1,\n",
       " 'reach': 5,\n",
       " '7': 1,\n",
       " 'estim': 4,\n",
       " '1927': 2,\n",
       " '1999': 1,\n",
       " 'concentr': 1,\n",
       " 'east': 4,\n",
       " 'south': 2,\n",
       " 'southeast': 1,\n",
       " 'asia': 4,\n",
       " '8': 1,\n",
       " '9': 1,\n",
       " '10': 1,\n",
       " 'literaci': 1,\n",
       " '80': 2,\n",
       " '%': 1,\n",
       " 'penicillin': 1,\n",
       " 'breakthrough': 1,\n",
       " 'combin': 3,\n",
       " 'campaign': 1,\n",
       " 'erad': 1,\n",
       " 'smallpox': 2,\n",
       " 'diseas': 5,\n",
       " 'respons': 1,\n",
       " 'death': 4,\n",
       " 'natur': 2,\n",
       " 'disast': 1,\n",
       " 'yield': 1,\n",
       " 'unpreced': 3,\n",
       " 'onli': 4,\n",
       " 'exist': 1,\n",
       " 'lab': 1,\n",
       " '11': 1,\n",
       " 'machin': 1,\n",
       " 'util': 1,\n",
       " 'area': 1,\n",
       " 'product': 2,\n",
       " 'feed': 1,\n",
       " 'increasingli': 1,\n",
       " 'intric': 1,\n",
       " 'suppli': 1,\n",
       " 'mean': 2,\n",
       " 'mankind': 1,\n",
       " 'longer': 2,\n",
       " 'constrain': 1,\n",
       " 'much': 1,\n",
       " 'could': 2,\n",
       " 'produc': 4,\n",
       " 'rather': 1,\n",
       " \"'\": 1,\n",
       " 'willing': 1,\n",
       " 'consum': 1,\n",
       " 'trade': 3,\n",
       " 'improv': 2,\n",
       " 'revers': 1,\n",
       " 'limit': 2,\n",
       " 'set': 3,\n",
       " 'food-produc': 1,\n",
       " 'techniqu': 1,\n",
       " 'use': 2,\n",
       " 'neolith': 1,\n",
       " 'period': 4,\n",
       " 'greatli': 1,\n",
       " 'enhanc': 1,\n",
       " 'divers': 1,\n",
       " 'food': 3,\n",
       " 'avail': 2,\n",
       " 'upturn': 1,\n",
       " 'qualiti': 2,\n",
       " 'nutrit': 1,\n",
       " 'earli': 1,\n",
       " '19th': 4,\n",
       " 'expect': 2,\n",
       " 'thirti': 1,\n",
       " 'lifespan-averag': 1,\n",
       " 'exceed': 1,\n",
       " '40+': 1,\n",
       " 'achiev': 1,\n",
       " '70+': 1,\n",
       " 'three': 1,\n",
       " 'earlier': 1,\n",
       " '12': 1,\n",
       " 'global-scal': 1,\n",
       " 'across': 2,\n",
       " 'contin': 1,\n",
       " 'ocean': 2,\n",
       " 'becam': 7,\n",
       " 'major': 7,\n",
       " 'polit': 6,\n",
       " 'issu': 3,\n",
       " 'acknowledg': 1,\n",
       " 'along': 2,\n",
       " 'right': 3,\n",
       " 'self-determin': 1,\n",
       " 'offici': 1,\n",
       " 'mid-centuri': 1,\n",
       " 'relat': 1,\n",
       " 'region': 2,\n",
       " 'shift': 1,\n",
       " 'way': 3,\n",
       " 'ideolog': 3,\n",
       " 'societi': 4,\n",
       " 'scienc': 3,\n",
       " 'medicin': 2,\n",
       " 'may': 2,\n",
       " 'seen': 3,\n",
       " 'scientif': 5,\n",
       " 'progress': 1,\n",
       " 'dawn': 1,\n",
       " 'civil': 2,\n",
       " 'term': 2,\n",
       " 'like': 2,\n",
       " 'genocid': 2,\n",
       " 'enter': 1,\n",
       " 'common': 1,\n",
       " 'usag': 1,\n",
       " 'discoveri': 1,\n",
       " 'theori': 4,\n",
       " 'rel': 1,\n",
       " 'quantum': 1,\n",
       " 'physic': 2,\n",
       " 'profoundli': 1,\n",
       " 'foundat': 1,\n",
       " 'model': 1,\n",
       " 'forc': 2,\n",
       " 'scientist': 3,\n",
       " 'realiz': 1,\n",
       " 'univers': 1,\n",
       " 'complex': 1,\n",
       " 'previous': 1,\n",
       " 'believ': 1,\n",
       " 'dash': 1,\n",
       " 'hope': 1,\n",
       " 'fear': 1,\n",
       " 'last': 3,\n",
       " 'detail': 1,\n",
       " 'knowledg': 2,\n",
       " 'fill': 1,\n",
       " 'start': 2,\n",
       " 'hors': 2,\n",
       " 'simpl': 1,\n",
       " 'automobil': 3,\n",
       " 'freighter': 1,\n",
       " 'high-spe': 1,\n",
       " 'rail': 1,\n",
       " 'cruis': 1,\n",
       " 'ship': 1,\n",
       " 'commerci': 1,\n",
       " 'air': 1,\n",
       " 'travel': 2,\n",
       " 'shuttl': 1,\n",
       " 'pack': 1,\n",
       " 'anim': 3,\n",
       " 'everi': 1,\n",
       " 'basic': 1,\n",
       " 'form': 7,\n",
       " 'person': 1,\n",
       " 'replac': 2,\n",
       " 'buse': 1,\n",
       " 'within': 1,\n",
       " 'made': 4,\n",
       " 'possibl': 1,\n",
       " 'exploit': 1,\n",
       " 'fossil': 2,\n",
       " 'fuel': 3,\n",
       " 'offer': 1,\n",
       " 'energi': 1,\n",
       " 'easili': 1,\n",
       " 'portabl': 1,\n",
       " 'also': 8,\n",
       " 'caus': 4,\n",
       " 'concern': 2,\n",
       " 'pollut': 1,\n",
       " 'long-term': 1,\n",
       " 'impact': 2,\n",
       " 'environ': 2,\n",
       " 'take': 1,\n",
       " 'footstep': 1,\n",
       " 'moon': 1,\n",
       " 'media': 1,\n",
       " 'telecommun': 1,\n",
       " 'inform': 4,\n",
       " 'especi': 2,\n",
       " 'paperback': 1,\n",
       " 'book': 1,\n",
       " 'public': 1,\n",
       " 'educ': 1,\n",
       " 'internet': 3,\n",
       " 'health': 1,\n",
       " '35': 2,\n",
       " '65': 1,\n",
       " 'rapid': 1,\n",
       " 'howev': 2,\n",
       " 'warfar': 4,\n",
       " 'destruct': 4,\n",
       " 'alon': 2,\n",
       " 'kill': 5,\n",
       " '60': 2,\n",
       " 'million': 8,\n",
       " 'weapon': 6,\n",
       " 'gave': 1,\n",
       " 'humankind': 2,\n",
       " 'annihil': 1,\n",
       " 'short': 1,\n",
       " 'imperi': 1,\n",
       " 'system': 1,\n",
       " 'empir': 4,\n",
       " 'expans': 2,\n",
       " 'colon': 1,\n",
       " 'ceas': 1,\n",
       " 'factor': 1,\n",
       " 'affair': 2,\n",
       " 'far': 2,\n",
       " 'cooper': 1,\n",
       " 'clash': 1,\n",
       " 'openli': 1,\n",
       " '1945': 3,\n",
       " 'violenc': 2,\n",
       " 'ever': 2,\n",
       " 'popular': 3,\n",
       " 'music': 8,\n",
       " 'influenc': 4,\n",
       " 'western': 4,\n",
       " 'corpor': 1,\n",
       " 'arguabl': 1,\n",
       " 'truli': 1,\n",
       " 'dure': 7,\n",
       " 'fought': 1,\n",
       " 'new': 8,\n",
       " 'invent': 3,\n",
       " 'tank': 1,\n",
       " 'chemic': 1,\n",
       " 'aircraft': 1,\n",
       " 'modifi': 1,\n",
       " 'tactic': 1,\n",
       " 'strategi': 1,\n",
       " 'four': 2,\n",
       " 'trench': 1,\n",
       " 'europ': 9,\n",
       " '20': 2,\n",
       " 'dead': 2,\n",
       " 'tripl': 2,\n",
       " 'entent': 2,\n",
       " 'franc': 3,\n",
       " 'britain': 3,\n",
       " 'russia': 1,\n",
       " 'later': 3,\n",
       " 'join': 1,\n",
       " 'itali': 3,\n",
       " 'romania': 1,\n",
       " 'victori': 3,\n",
       " 'central': 1,\n",
       " 'germani': 8,\n",
       " 'austria-hungari': 1,\n",
       " 'ottoman': 2,\n",
       " 'bulgaria': 1,\n",
       " 'In': 6,\n",
       " 'addit': 1,\n",
       " 'annex': 1,\n",
       " 'coloni': 2,\n",
       " 'possess': 1,\n",
       " 'vanquish': 1,\n",
       " 'exact': 1,\n",
       " 'punit': 1,\n",
       " 'restitut': 1,\n",
       " 'payment': 1,\n",
       " 'plung': 1,\n",
       " 'particular': 1,\n",
       " 'depress': 2,\n",
       " 'austro-hungarian': 1,\n",
       " 'dismantl': 1,\n",
       " 'conclus': 1,\n",
       " 'russian': 2,\n",
       " 'overthrow': 1,\n",
       " 'tsarist': 1,\n",
       " 'regim': 1,\n",
       " 'nichola': 1,\n",
       " 'onset': 1,\n",
       " 'bolshevik': 1,\n",
       " 'At': 2,\n",
       " 'begin': 2,\n",
       " 'british': 2,\n",
       " '15': 2,\n",
       " 'act': 2,\n",
       " 'policeman': 1,\n",
       " 'past': 1,\n",
       " 'fascism': 1,\n",
       " 'movement': 2,\n",
       " 'grew': 1,\n",
       " 'angst': 1,\n",
       " '1930': 2,\n",
       " 'gain': 2,\n",
       " 'momentum': 1,\n",
       " 'spain': 1,\n",
       " '1920': 3,\n",
       " 'culmin': 2,\n",
       " 'spark': 1,\n",
       " 'nazi': 1,\n",
       " 'aggress': 1,\n",
       " 'expens': 1,\n",
       " 'neighbor': 1,\n",
       " 'meanwhil': 2,\n",
       " 'japan': 3,\n",
       " 'industri': 4,\n",
       " 'axi': 2,\n",
       " 'militari': 5,\n",
       " 'expansion': 1,\n",
       " 'pacif': 1,\n",
       " 'brought': 1,\n",
       " 'surpris': 1,\n",
       " 'attack': 1,\n",
       " 'drew': 1,\n",
       " 'dramat': 2,\n",
       " 'success': 1,\n",
       " 'defeat': 2,\n",
       " 'invad': 1,\n",
       " 'poland': 1,\n",
       " 'kingdom': 1,\n",
       " 'canada': 1,\n",
       " 'alli': 3,\n",
       " 'invas': 2,\n",
       " 'manchuria': 1,\n",
       " 'drop': 1,\n",
       " 'two': 3,\n",
       " 'atom': 1,\n",
       " 'bomb': 1,\n",
       " 'left': 3,\n",
       " 'occupi': 1,\n",
       " 'divid': 2,\n",
       " 'rest': 1,\n",
       " 'eastern': 1,\n",
       " 'puppet': 1,\n",
       " 'rebuilt': 1,\n",
       " 'american': 5,\n",
       " 'plan': 1,\n",
       " 'boom': 1,\n",
       " 'affect': 3,\n",
       " 'close': 1,\n",
       " 'stand': 1,\n",
       " 'superpow': 1,\n",
       " 'soon': 1,\n",
       " 'hostil': 1,\n",
       " 'anoth': 3,\n",
       " 'compet': 2,\n",
       " 'democrat': 1,\n",
       " 'capit': 1,\n",
       " 'iron': 1,\n",
       " 'curtain': 1,\n",
       " 'berlin': 1,\n",
       " 'wall': 1,\n",
       " 'nato': 1,\n",
       " 'warsaw': 1,\n",
       " 'pact': 1,\n",
       " 'engag': 2,\n",
       " 'decades-long': 1,\n",
       " 'standoff': 1,\n",
       " 'known': 1,\n",
       " 'mark': 1,\n",
       " 'race': 2,\n",
       " 'ussr': 2,\n",
       " 'second': 3,\n",
       " 'side': 3,\n",
       " 'suffici': 1,\n",
       " 'number': 3,\n",
       " 'planet': 2,\n",
       " 'large-scal': 1,\n",
       " 'exchang': 2,\n",
       " 'mutual': 1,\n",
       " 'assur': 1,\n",
       " 'credit': 1,\n",
       " 'historian': 1,\n",
       " 'prevent': 1,\n",
       " 'unabl': 2,\n",
       " 'strike': 2,\n",
       " 'without': 1,\n",
       " 'ensur': 1,\n",
       " 'equal': 2,\n",
       " 'devast': 1,\n",
       " 'retaliatori': 1,\n",
       " 'directli': 2,\n",
       " 'play': 1,\n",
       " 'seri': 1,\n",
       " 'proxi': 1,\n",
       " 'world—particularli': 1,\n",
       " 'korea': 3,\n",
       " 'cuba': 2,\n",
       " 'vietnam': 2,\n",
       " 'afghanistan—a': 1,\n",
       " 'sought': 1,\n",
       " 'export': 1,\n",
       " 'attempt': 1,\n",
       " 'contain': 1,\n",
       " 'led': 5,\n",
       " 'substanti': 1,\n",
       " 'invest': 1,\n",
       " 'research': 4,\n",
       " 'innov': 1,\n",
       " 'beyond': 1,\n",
       " 'battlefield': 1,\n",
       " 'european-colon': 1,\n",
       " 'africa': 4,\n",
       " 'independ': 2,\n",
       " 'process': 2,\n",
       " 'open': 2,\n",
       " 'door': 1,\n",
       " 'sever': 1,\n",
       " 'exert': 1,\n",
       " 'strong': 2,\n",
       " 'presenc': 1,\n",
       " 'spread': 3,\n",
       " 'advent': 1,\n",
       " 'hollywood': 2,\n",
       " 'motion': 1,\n",
       " 'pictur': 1,\n",
       " 'broadway': 1,\n",
       " 'rock': 2,\n",
       " 'roll': 2,\n",
       " 'pop': 1,\n",
       " 'fast': 1,\n",
       " 'big-box': 1,\n",
       " 'store': 1,\n",
       " 'hip-hop': 1,\n",
       " 'lifestyl': 1,\n",
       " 'continu': 5,\n",
       " 'includ': 4,\n",
       " 'lead': 2,\n",
       " 'band': 1,\n",
       " 'countri': 4,\n",
       " 'swedish': 1,\n",
       " 'abba': 2,\n",
       " 'sing': 1,\n",
       " 'english': 1,\n",
       " 'pressur': 1,\n",
       " 'govern': 3,\n",
       " 'support': 1,\n",
       " 'dismantled—with': 1,\n",
       " 'notabl': 1,\n",
       " 'except': 1,\n",
       " 'north': 3,\n",
       " 'laos—follow': 1,\n",
       " 'awkward': 1,\n",
       " 'transit': 1,\n",
       " 'market': 1,\n",
       " 'follow': 2,\n",
       " 'successor': 1,\n",
       " 'leagu': 1,\n",
       " 'forum': 1,\n",
       " 'discuss': 1,\n",
       " 'diplomat': 1,\n",
       " 'enact': 1,\n",
       " 'resolut': 1,\n",
       " 'topic': 1,\n",
       " 'conduct': 1,\n",
       " 'protect': 1,\n",
       " 'sovereignti': 1,\n",
       " 'peacekeep': 1,\n",
       " 'consist': 2,\n",
       " 'troop': 1,\n",
       " 'provid': 1,\n",
       " 'agenc': 1,\n",
       " 'help': 1,\n",
       " 'reliev': 1,\n",
       " 'famin': 2,\n",
       " 'suppress': 1,\n",
       " 'local': 1,\n",
       " 'slowli': 1,\n",
       " 'third': 2,\n",
       " 'channel': 1,\n",
       " 'green': 1,\n",
       " 'parti': 1,\n",
       " 'commenc': 1,\n",
       " 'social': 2,\n",
       " 'debate.du': 1,\n",
       " 'expand': 2,\n",
       " 'indirectli': 1,\n",
       " 'light': 1,\n",
       " 'bulb': 1,\n",
       " 'telephon': 2,\n",
       " 'supertank': 1,\n",
       " 'airlin': 1,\n",
       " 'motorway': 1,\n",
       " 'radio': 2,\n",
       " 'televis': 1,\n",
       " 'antibiot': 1,\n",
       " 'frozen': 1,\n",
       " 'microcomput': 1,\n",
       " 'mobil': 1,\n",
       " 'engin': 2,\n",
       " 'profession': 1,\n",
       " 'development—much': 1,\n",
       " 'motiv': 1,\n",
       " 'race—drov': 1,\n",
       " 'everyday': 1,\n",
       " 'discrimin': 1,\n",
       " 'base': 2,\n",
       " 'sex': 1,\n",
       " 'although': 1,\n",
       " 'atlant': 1,\n",
       " 'slave': 1,\n",
       " 'fight': 1,\n",
       " 'non-whit': 1,\n",
       " 'white-domin': 1,\n",
       " 'america': 1,\n",
       " 'taboo': 1,\n",
       " 'sexism': 1,\n",
       " 'fell': 1,\n",
       " 'By': 1,\n",
       " 'women': 1,\n",
       " 'legal': 1,\n",
       " 'men': 1,\n",
       " 'part': 4,\n",
       " 'racism': 1,\n",
       " 'come': 1,\n",
       " 'abhorr': 1,\n",
       " '16': 1,\n",
       " 'attitud': 1,\n",
       " 'toward': 2,\n",
       " 'homosexu': 1,\n",
       " 'radic': 1,\n",
       " 'alter': 1,\n",
       " 'daili': 1,\n",
       " 'appear': 1,\n",
       " 'sustain': 1,\n",
       " 'peac': 1,\n",
       " 'record': 3,\n",
       " 'indian': 1,\n",
       " 'subcontin': 1,\n",
       " 'sixth': 1,\n",
       " 'attain': 1,\n",
       " 'indigen': 1,\n",
       " 'ancient': 1,\n",
       " 'compris': 1,\n",
       " 'fifth': 1,\n",
       " 'near-complet': 1,\n",
       " 'old': 1,\n",
       " 'nearli': 1,\n",
       " 'foreign': 1,\n",
       " 'undergo': 1,\n",
       " '18th': 1,\n",
       " 'termin': 1,\n",
       " 'posit': 1,\n",
       " 'india': 1,\n",
       " 'terror': 1,\n",
       " 'dictatorship': 1,\n",
       " 'press': 1,\n",
       " 'still': 1,\n",
       " 'blight': 1,\n",
       " 'small-scal': 1,\n",
       " 'violent': 1,\n",
       " 'ethnic': 1,\n",
       " 'despot': 1,\n",
       " 'kim': 1,\n",
       " 'jong-il': 1,\n",
       " 'threaten': 2,\n",
       " 'destabil': 1,\n",
       " 'virus': 1,\n",
       " 'nile': 1,\n",
       " 'viru': 3,\n",
       " 'malaria': 1,\n",
       " 'larg': 1,\n",
       " 'infect': 1,\n",
       " 'hiv': 1,\n",
       " 'becom': 1,\n",
       " 'epidem': 1,\n",
       " 'southern': 1,\n",
       " 'done': 1,\n",
       " 'climat': 1,\n",
       " 'consid': 1,\n",
       " 'long': 1,\n",
       " 'problem': 1,\n",
       " 'habit': 1,\n",
       " '17': 1,\n",
       " 'argument': 1,\n",
       " 'due': 1,\n",
       " 'human-caus': 1,\n",
       " 'emiss': 2,\n",
       " 'greenhous': 1,\n",
       " 'gase': 1,\n",
       " 'particularli': 1,\n",
       " 'carbon': 2,\n",
       " 'dioxid': 2,\n",
       " 'burn': 1,\n",
       " '18': 1,\n",
       " 'thi': 2,\n",
       " 'prompt': 1,\n",
       " 'negoti': 1,\n",
       " 'sign': 1,\n",
       " 'kyoto': 1,\n",
       " 'treati': 1,\n",
       " 'mandatori': 1,\n",
       " '1.6': 1,\n",
       " '6.1': 1,\n",
       " '19': 1,\n",
       " 'action': 2,\n",
       " 'hundr': 1,\n",
       " 'politicid': 1,\n",
       " 'murder': 1,\n",
       " '50': 1,\n",
       " 'citat': 1,\n",
       " 'need': 1,\n",
       " 'rudolph': 1,\n",
       " 'rummel': 1,\n",
       " '262,000,000': 1,\n",
       " 'democid': 1,\n",
       " 'exclud': 1,\n",
       " 'battl': 1,\n",
       " 'civilian': 2,\n",
       " 'unintent': 1,\n",
       " 'riot': 1,\n",
       " 'mob': 1,\n",
       " '21': 1,\n",
       " 'accord': 1,\n",
       " 'charl': 1,\n",
       " 'tilli': 1,\n",
       " 'altogeth': 1,\n",
       " 'die': 3,\n",
       " 'direct': 1,\n",
       " 'back': 1,\n",
       " 'cours': 1,\n",
       " 'compar': 1,\n",
       " 'war-induc': 1,\n",
       " 'indirect': 1,\n",
       " 'effect': 1,\n",
       " '22': 1,\n",
       " 'approxim': 1,\n",
       " '70': 1,\n",
       " '1914': 1,\n",
       " '23': 1,\n",
       " 'phonograph': 1,\n",
       " 'dissemin': 1,\n",
       " 'broadcast': 1,\n",
       " 'massiv': 1,\n",
       " 'audienc': 1,\n",
       " 'prior': 1,\n",
       " 'experienc': 1,\n",
       " 'perform': 1,\n",
       " 'genr': 1,\n",
       " 'famou': 1,\n",
       " 'rever': 1,\n",
       " 'artist': 2,\n",
       " 'loui': 1,\n",
       " 'armstrong': 1,\n",
       " 'richard': 2,\n",
       " 'igor': 1,\n",
       " 'stravinski': 1,\n",
       " 'gustav': 1,\n",
       " 'mahler': 1,\n",
       " 'georg': 1,\n",
       " 'gershwin': 1,\n",
       " 'sergei': 2,\n",
       " 'prokofiev': 1,\n",
       " 'benjamin': 1,\n",
       " 'britten': 1,\n",
       " 'mauric': 1,\n",
       " 'ravel': 1,\n",
       " 'arnold': 1,\n",
       " 'schoenberg': 1,\n",
       " 'dmitri': 1,\n",
       " 'shostakovich': 1,\n",
       " 'aaron': 1,\n",
       " 'copland': 1,\n",
       " 'béla': 1,\n",
       " 'bartók': 1,\n",
       " 'ernesto': 1,\n",
       " 'lecuona': 1,\n",
       " 'rachmaninoff': 1,\n",
       " 'strauss': 1,\n",
       " 'theloni': 1,\n",
       " 'monk': 1,\n",
       " 'ella': 1,\n",
       " 'fitzgerald': 1,\n",
       " 'duke': 1,\n",
       " 'ellington': 1,\n",
       " 'bing': 1,\n",
       " 'crosbi': 1,\n",
       " 'beach': 1,\n",
       " 'boy': 1,\n",
       " 'beatl': 1,\n",
       " 'harri': 1,\n",
       " 'belafont': 1,\n",
       " 'chuck': 1,\n",
       " 'berri': 1,\n",
       " 'jame': 1,\n",
       " 'brown': 1,\n",
       " 'mile': 1,\n",
       " 'davi': 1,\n",
       " 'bob': 2,\n",
       " 'dylan': 1,\n",
       " 'jimi': 1,\n",
       " 'hendrix': 1,\n",
       " 'eagl': 1,\n",
       " 'michael': 1,\n",
       " 'jackson': 1,\n",
       " 'elton': 1,\n",
       " 'john': 1,\n",
       " 'bee': 1,\n",
       " 'gee': 1,\n",
       " 'barbra': 1,\n",
       " 'streisand': 1,\n",
       " 'cher': 1,\n",
       " 'nat': 1,\n",
       " 'king': 1,\n",
       " 'cole': 1,\n",
       " 'robert': 1,\n",
       " 'johnson': 1,\n",
       " 'zeppelin': 1,\n",
       " 'leonard': 1,\n",
       " 'cohen': 1,\n",
       " 'queen': 1,\n",
       " 'madonna': 1,\n",
       " 'marley': 1,\n",
       " 'metallica': 1,\n",
       " 'charli': 1,\n",
       " 'parker': 1,\n",
       " 'pink': 1,\n",
       " 'floyd': 1,\n",
       " 'david': 1,\n",
       " 'bowi': 1,\n",
       " 'elvi': 1,\n",
       " 'presley': 1,\n",
       " 'stone': 1,\n",
       " 'frank': 1,\n",
       " 'sinatra': 1,\n",
       " 'stevi': 1,\n",
       " 'wonder': 1,\n",
       " 'radiohead': 1,\n",
       " 'aretha': 1,\n",
       " 'franklin': 1,\n",
       " 'tupac': 1,\n",
       " 'shakur': 1,\n",
       " 'nirvana': 1,\n",
       " 'notori': 1,\n",
       " 'b.i.g.': 1,\n",
       " 'amr': 1,\n",
       " 'diab': 1,\n",
       " 'fairuz': 1,\n",
       " 'umm': 1,\n",
       " 'kulthum': 1,\n",
       " 'abdel': 1,\n",
       " 'halim': 1,\n",
       " 'hafez': 1,\n",
       " 'randi': 1,\n",
       " 'newman': 1,\n",
       " 'film': 9,\n",
       " 'medium': 1,\n",
       " 'modern': 1,\n",
       " 'movi': 1,\n",
       " 'theatr': 1,\n",
       " 'pittsburgh': 1,\n",
       " '1905': 1,\n",
       " '29': 1,\n",
       " 'center': 1,\n",
       " 'black': 1,\n",
       " 'white': 2,\n",
       " 'technicolor': 1,\n",
       " 'color': 2,\n",
       " 'sound': 1,\n",
       " 'full-length': 3,\n",
       " 'featur': 2,\n",
       " 'jazz': 1,\n",
       " 'singer': 1,\n",
       " 'releas': 3,\n",
       " 'academi': 1,\n",
       " 'award': 1,\n",
       " '1929': 1,\n",
       " 'cel': 1,\n",
       " 'snow': 1,\n",
       " 'seven': 1,\n",
       " 'dwarf': 1,\n",
       " '1937': 1,\n",
       " 'computer-gener': 1,\n",
       " 'imageri': 1,\n",
       " 'cgi-anim': 1,\n",
       " 'toy': 1,\n",
       " ...}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_create_dictionary_table(article_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Tokenizing the article into sentences**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "\n",
    "sentences = sent_tokenize(article_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The 20t'"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[0][:7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Finding the weighted frequencies of the sentences\n",
    "\n",
    "Importantly, to ensure long sentences do not have unnecessarily high scores over short sentences, we divided each score of a sentence by the number of words found in that sentence.\n",
    "\n",
    "Also, to optimize the dictionary’s memory, we arbitrarily added sentence[:10], which refers to the **first 10 characters** in each sentence. **Entire sentence need not be stored only first 10 characters** (saves dict() memory)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _calculate_sentence_scores(sentences, frequency_table):\n",
    "    sentence_weight=dict()\n",
    "    \n",
    "    for sent in sentences:\n",
    "        sentence_wordcount_without_stop_words = 0\n",
    "\n",
    "        for word_weight in frequency_table.keys():\n",
    "            \n",
    "            if word_weight in sent.lower():\n",
    "                sentence_wordcount_without_stop_words+=1\n",
    "                \n",
    "                if sent[:10] in sentence_weight:\n",
    "                    sentence_weight[sent[:10]]+=frequency_table[word_weight]\n",
    "                else:\n",
    "                    sentence_weight[sent[:10]]= frequency_table[word_weight]\n",
    "                    \n",
    "        sentence_weight[sent[:10]] = sentence_weight[sent[:10]] / sentence_wordcount_without_stop_words\n",
    "        \n",
    "    return sentence_weight\n",
    "\n",
    "\n",
    "# def _calculate_sentence_scores(sentences, frequency_table):\n",
    "#     sentence_weight=dict()\n",
    "    \n",
    "#     for sent in sentences:\n",
    "#         sentence_wordcount_without_stop_words = 0   # initialize to 0 before every sentence count\n",
    "        \n",
    "#         for word_weight in sent:\n",
    "            \n",
    "#             if word_weight in frequency_table:\n",
    "#                 sentence_wordcount_without_stop_words+=1\n",
    "                \n",
    "#                 if sent[:10] in sentence_weight:\n",
    "#                     sentence_weight[sent[:10]]+=frequency_table[word_weight]\n",
    "#                 else:\n",
    "#                     sentence_weight[sent[:10]]= frequency_table[word_weight]\n",
    "                    \n",
    "#         sentence_weight[sent[:10]] = sentence_weight[sent[:10]] / sentence_wordcount_without_stop_words\n",
    "        \n",
    "#     return sentence_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'The 20th (': 28.85,\n",
       " '[2] It was': 27.333333333333332,\n",
       " 'Unlike mos': 25.571428571428573,\n",
       " 'The 20th c': 15.266586354821651,\n",
       " 'It saw gre': 19.233333333333334,\n",
       " 'Man-made g': 12.226415094339623,\n",
       " '[5]\\nThe re': 24.966666666666665,\n",
       " 'The Marsha': 10.27027027027027,\n",
       " 'Throughout': 19.125,\n",
       " 'The dissol': 16.205128205128204,\n",
       " 'It took ov': 16.41860465116279,\n",
       " '[8][9][10]': 18.153846153846153,\n",
       " 'Penicillin': 9.96551724137931,\n",
       " '[11] Machi': 18.314285714285713,\n",
       " 'Trade impr': 16.6,\n",
       " 'Until the ': 15.35,\n",
       " '[12]\\nThe c': 21.944444444444443,\n",
       " 'Nationalis': 17.96969696969697,\n",
       " 'The centur': 27.85,\n",
       " 'Terms like': 28.227272727272727,\n",
       " 'Scientific': 22.282651072124757,\n",
       " 'It was a c': 21.75,\n",
       " 'Horses and': 19.307692307692307,\n",
       " 'These deve': 18.48148148148148,\n",
       " 'Humans exp': 39.333333333333336,\n",
       " 'Mass media': 18.90625,\n",
       " 'Advancemen': 11.85,\n",
       " 'Rapid tech': 29.68421052631579,\n",
       " 'World War ': 26.583333333333332,\n",
       " 'However, t': 47.27272727272727,\n",
       " 'For the fi': 23.714285714285715,\n",
       " 'The last t': 23.954545454545453,\n",
       " '[13]\\nThe w': 20.075,\n",
       " 'Technologi': 23.0,\n",
       " 'After more': 15.928571428571429,\n",
       " 'In additio': 18.037037037037038,\n",
       " 'The Austro': 21.545454545454547,\n",
       " 'The Russia': 16.571428571428573,\n",
       " 'The victor': 40.642857142857146,\n",
       " 'At the beg': 35.125,\n",
       " 'Fascism, a': 15.21951219512195,\n",
       " 'Meanwhile,': 29.479224376731306,\n",
       " \"Japan's mi\": 26.25,\n",
       " 'After some': 18.93103448275862,\n",
       " 'After the ': 17.174057649667404,\n",
       " 'In total, ': 48.5,\n",
       " 'East Germa': 15.333333333333334,\n",
       " 'Western Eu': 22.214285714285715,\n",
       " 'With the A': 29.842105263157894,\n",
       " 'Allies dur': 21.037037037037038,\n",
       " 'They forme': 10.48,\n",
       " 'The period': 17.852941176470587,\n",
       " 'Mutually a': 22.045454545454547,\n",
       " 'Unable to ': 19.93548387096774,\n",
       " 'The techno': 18.75,\n",
       " 'In the lat': 27.94736842105263,\n",
       " \"The US's g\": 19.64516129032258,\n",
       " 'Britain al': 21.46153846153846,\n",
       " 'Following ': 28.818181818181817,\n",
       " 'It enacted': 26.666666666666668,\n",
       " 'Peacekeepi': 18.071428571428573,\n",
       " 'Europe slo': 25.954545454545453,\n",
       " 'In the las': 21.869565217391305,\n",
       " 'In many co': 29.470588235294116,\n",
       " 'Increasing': 15.390243902439025,\n",
       " 'Inventions': 15.136363636363637,\n",
       " 'Although t': 17.620689655172413,\n",
       " 'During the': 61.42857142857143,\n",
       " 'By the end': 31.176470588235293,\n",
       " '[16] Attit': 16.15,\n",
       " 'Communicat': 35.0,\n",
       " 'Europe app': 21.0,\n",
       " 'The people': 27.130434782608695,\n",
       " 'China, an ': 25.695652173913043,\n",
       " 'With the e': 32.0,\n",
       " 'The world ': 35.61125319693094,\n",
       " 'Since the ': 41.083333333333336,\n",
       " 'The influe': 30.31578947368421,\n",
       " 'Terrorism,': 39.166666666666664,\n",
       " 'Despots su': 15.166666666666666,\n",
       " 'Disease th': 26.428571428571427,\n",
       " 'New viruse': 15.2,\n",
       " 'Malaria an': 18.5,\n",
       " 'Millions w': 44.8,\n",
       " 'The virus ': 19.333333333333332,\n",
       " 'Based on r': 17.633333333333333,\n",
       " '[17] One a': 19.0,\n",
       " '[18] This ': 28.63157894736842,\n",
       " 'World popu': 15.352941176470589,\n",
       " '[19][20]\\nT': 15.368421052631579,\n",
       " 'This inclu': 37.714285714285715,\n",
       " 'The deaths': 16.941176470588236,\n",
       " '[citation ': 20.35483870967742,\n",
       " '[21] Accor': 20.344827586206897,\n",
       " 'Most likel': 14.0,\n",
       " '\"[22] It i': 22.535714285714285,\n",
       " '[23]\\nThe i': 26.136363636363637,\n",
       " 'Prior to t': 34.06666666666667,\n",
       " 'Many new g': 21.75,\n",
       " \"The world'\": 5.364963503649635,\n",
       " 'Film as an': 22.22222222222222,\n",
       " 'The first ': 14.285714285714286,\n",
       " '[29] Holly': 16.6,\n",
       " 'While the ': 30.941176470588236,\n",
       " 'Sound film': 25.894736842105264,\n",
       " 'The Academ': 18.333333333333332,\n",
       " 'Animation ': 20.928571428571427,\n",
       " 'Computer-g': 20.22222222222222,\n",
       " 'Video game': 9.696969696969697,\n",
       " 'Multiple n': 19.88888888888889,\n",
       " 'In the fir': 24.61904761904762,\n",
       " 'The develo': 14.384615384615385,\n",
       " 'Later in t': 41.0,\n",
       " '[33] Other': 10.73913043478261,\n",
       " '[35]\\nOne o': 18.75,\n",
       " 'Organized ': 25.136363636363637,\n",
       " 'A technolo': 14.511627906976743,\n",
       " '[49]': 16.2}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_calculate_sentence_scores(sentences,_create_dictionary_table(article_content))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating the threshold of the sentences\n",
    "\n",
    "To further tweak the kind of sentences eligible for summarization, we’ll create the average score for the sentences. With this threshold, we can avoid selecting the sentences with a lower score than the average score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _calculate_average_score(sentence_weight):\n",
    "    \n",
    "    # Calculating the average score for the sentences\n",
    "    sum_values = 0\n",
    "    for entry in sentence_weight.keys():\n",
    "        sum_values+=sentence_weight[entry]\n",
    "    \n",
    "    average_score= sum_values/len(sentence_weight)\n",
    "    return average_score  #threshold value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.878991308947914"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "_calculate_average_score(_calculate_sentence_scores(sentences,_create_dictionary_table(article_content)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting the summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_article_summary(sentences, sentence_weight, threshold):\n",
    "    sentence_counter = 0\n",
    "    article_summary = ''\n",
    "\n",
    "    for sent in sentences:\n",
    "        if sent[:10] in sentence_weight and sentence_weight[sent[:10]] >= (threshold):\n",
    "            article_summary += \" \" + sent    #entire sentence is conacatnated not just sent[:10]\n",
    "            sentence_counter += 1\n",
    "\n",
    "    return article_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _run_article_summary(article):\n",
    "    \n",
    "    #creating a dictionary for the word frequency table\n",
    "    frequency_table = _create_dictionary_table(article)\n",
    "\n",
    "    #tokenizing the sentences\n",
    "    sentences = sent_tokenize(article)\n",
    "\n",
    "    #algorithm for scoring a sentence by its words\n",
    "    sentence_scores = _calculate_sentence_scores(sentences, frequency_table)\n",
    "\n",
    "    #getting the threshold\n",
    "    threshold = _calculate_average_score(sentence_scores)\n",
    "\n",
    "    #producing the summary ## tweak around with threshold value to adjust length of the summary\n",
    "    article_summary = _get_article_summary(sentences, sentence_scores, 1.5 * threshold)\n",
    "\n",
    "    return article_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Humans explored space for the first time, taking their first footsteps on the Moon. However, these same wars resulted in the destruction of the imperial system. The victorious Bolsheviks then established the Soviet Union, the world's first communist state. At the beginning of the period, the British Empire was the world's most powerful nation,[15] having acted as the world's policeman for the past century. In total, World War II left some 60 million people dead. At the beginning of the century, strong discrimination based on race and sex was significant in general society. During the century, the social taboo of sexism fell. Communications and information technology, transportation technology, and medical advances had radically altered daily lives. Since the US was in a dominant position, a major part of the process was Americanization. Terrorism, dictatorship, and the spread of nuclear weapons were pressing global issues. Millions were infected with HIV, the virus which causes AIDS. This includes deaths caused by wars, genocide, politicide and mass murders. Prior to the 20th century, music was generally only experienced in live performances. Later in the 20th century, the development of computers led to the establishment of a theory of computation.\n"
     ]
    }
   ],
   "source": [
    "(print(_run_article_summary(article_content)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Here is the entire code for the simple extractive text summarizer in machine learning:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Humans explored space for the first time, taking their first footsteps on the Moon. However, these same wars resulted in the destruction of the imperial system. The victorious Bolsheviks then established the Soviet Union, the world's first communist state. At the beginning of the period, the British Empire was the world's most powerful nation,[15] having acted as the world's policeman for the past century. In total, World War II left some 60 million people dead. At the beginning of the century, strong discrimination based on race and sex was significant in general society. During the century, the social taboo of sexism fell. Communications and information technology, transportation technology, and medical advances had radically altered daily lives. Since the US was in a dominant position, a major part of the process was Americanization. Terrorism, dictatorship, and the spread of nuclear weapons were pressing global issues. Millions were infected with HIV, the virus which causes AIDS. This includes deaths caused by wars, genocide, politicide and mass murders. Prior to the 20th century, music was generally only experienced in live performances. Later in the 20th century, the development of computers led to the establishment of a theory of computation.\n"
     ]
    }
   ],
   "source": [
    "#importing libraries\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "import bs4 as BeautifulSoup\n",
    "import urllib.request  \n",
    "\n",
    "#fetching the content from the URL\n",
    "fetched_data = urllib.request.urlopen('https://en.wikipedia.org/wiki/20th_century')\n",
    "\n",
    "article_read = fetched_data.read()\n",
    "\n",
    "#parsing the URL content and storing in a variable\n",
    "article_parsed = BeautifulSoup.BeautifulSoup(article_read,'html.parser')\n",
    "\n",
    "#returning <p> tags\n",
    "paragraphs = article_parsed.find_all('p')\n",
    "\n",
    "article_content = ''\n",
    "\n",
    "#looping through the paragraphs and adding them to the variable\n",
    "for p in paragraphs:  \n",
    "    article_content += p.text\n",
    "\n",
    "\n",
    "def _create_dictionary_table(text_string) -> dict:\n",
    "   \n",
    "    #removing stop words\n",
    "    stop_words = set(stopwords.words(\"english\"))\n",
    "    \n",
    "    words = word_tokenize(text_string)\n",
    "    \n",
    "    #reducing words to their root form\n",
    "    stem = PorterStemmer()\n",
    "    \n",
    "    #creating dictionary for the word frequency table\n",
    "    frequency_table = dict()\n",
    "    for wd in words:\n",
    "        wd = stem.stem(wd)\n",
    "        if wd in stop_words:\n",
    "            continue\n",
    "        if wd in frequency_table:\n",
    "            frequency_table[wd] += 1\n",
    "        else:\n",
    "            frequency_table[wd] = 1\n",
    "\n",
    "    return frequency_table\n",
    "\n",
    "\n",
    "def _calculate_sentence_scores(sentences, frequency_table) -> dict:   \n",
    "\n",
    "    #algorithm for scoring a sentence by its words\n",
    "    sentence_weight = dict()\n",
    "\n",
    "    for sentence in sentences:\n",
    "        sentence_wordcount = (len(word_tokenize(sentence)))\n",
    "        sentence_wordcount_without_stop_words = 0\n",
    "        for word_weight in frequency_table:\n",
    "            if word_weight in sentence.lower():\n",
    "                sentence_wordcount_without_stop_words += 1\n",
    "                if sentence[:7] in sentence_weight:\n",
    "                    sentence_weight[sentence[:7]] += frequency_table[word_weight]\n",
    "                else:\n",
    "                    sentence_weight[sentence[:7]] = frequency_table[word_weight]\n",
    "\n",
    "        sentence_weight[sentence[:7]] = sentence_weight[sentence[:7]] / sentence_wordcount_without_stop_words\n",
    "\n",
    "       \n",
    "\n",
    "    return sentence_weight\n",
    "\n",
    "def _calculate_average_score(sentence_weight) -> int:\n",
    "   \n",
    "    #calculating the average score for the sentences\n",
    "    sum_values = 0\n",
    "    for entry in sentence_weight:\n",
    "        sum_values += sentence_weight[entry]\n",
    "\n",
    "    #getting sentence average value from source text\n",
    "    average_score = (sum_values / len(sentence_weight))\n",
    "\n",
    "    return average_score\n",
    "\n",
    "def _get_article_summary(sentences, sentence_weight, threshold):\n",
    "    sentence_counter = 0\n",
    "    article_summary = ''\n",
    "\n",
    "    for sentence in sentences:\n",
    "        if sentence[:7] in sentence_weight and sentence_weight[sentence[:7]] >= (threshold):\n",
    "            article_summary += \" \" + sentence\n",
    "            sentence_counter += 1\n",
    "\n",
    "    return article_summary\n",
    "\n",
    "\n",
    "#Our main function\n",
    "def _run_article_summary(article):\n",
    "    \n",
    "    #creating a dictionary for the word frequency table\n",
    "    frequency_table = _create_dictionary_table(article)\n",
    "\n",
    "    #tokenizing the sentences\n",
    "    sentences = sent_tokenize(article)\n",
    "\n",
    "    #algorithm for scoring a sentence by its words\n",
    "    sentence_scores = _calculate_sentence_scores(sentences, frequency_table)\n",
    "\n",
    "    #getting the threshold\n",
    "    threshold = _calculate_average_score(sentence_scores)\n",
    "\n",
    "    #producing the summary\n",
    "    article_summary = _get_article_summary(sentences, sentence_scores, 1.5 * threshold) \n",
    "\n",
    "    return article_summary\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    summary_results = _run_article_summary(article_content)\n",
    "    print(summary_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}