Day 85.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Natural Language Processing Core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "showing info https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.Tokenizing Words and Sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = \"\"\"Tell General Howard I know his heart. What he told me before, I have it in my heart. I am tired of fighting.\n",
    "Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead. The old men are all dead. It is the young men who\n",
    "say yes or no. He who led on the young men is dead. It is cold, and we have no blankets; the little children are freezing \n",
    "to death. My people, some of them, have run away to the hills, and have no blankets, no food. No one knows where they are \n",
    "– perhaps freezing to death. I want to have time to look for my children, and see how many of them I can find. Maybe I \n",
    "shall find them among the dead. Hear me, my Chiefs! I am tired; my heart is sick and sad. From where the sun now stands \n",
    "I will fight no more forever.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tell General Howard I know his heart.',\n",
       " 'What he told me before, I have it in my heart.',\n",
       " 'I am tired of fighting.',\n",
       " 'Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead.',\n",
       " 'The old men are all dead.',\n",
       " 'It is the young men who\\nsay yes or no.',\n",
       " 'He who led on the young men is dead.',\n",
       " 'It is cold, and we have no blankets; the little children are freezing \\nto death.',\n",
       " 'My people, some of them, have run away to the hills, and have no blankets, no food.',\n",
       " 'No one knows where they are \\n– perhaps freezing to death.',\n",
       " 'I want to have time to look for my children, and see how many of them I can find.',\n",
       " 'Maybe I \\nshall find them among the dead.',\n",
       " 'Hear me, my Chiefs!',\n",
       " 'I am tired; my heart is sick and sad.',\n",
       " 'From where the sun now stands \\nI will fight no more forever.']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = nltk.sent_tokenize(paragraph)   # seperate each sentences of our paragraph\n",
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = nltk.word_tokenize(paragraph)       # seperate each words of sentences of our paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tell',\n",
       " 'General',\n",
       " 'Howard',\n",
       " 'I',\n",
       " 'know',\n",
       " 'his',\n",
       " 'heart',\n",
       " '.',\n",
       " 'What',\n",
       " 'he',\n",
       " 'told',\n",
       " 'me',\n",
       " 'before',\n",
       " ',',\n",
       " 'I',\n",
       " 'have',\n",
       " 'it',\n",
       " 'in',\n",
       " 'my',\n",
       " 'heart',\n",
       " '.',\n",
       " 'I',\n",
       " 'am',\n",
       " 'tired',\n",
       " 'of',\n",
       " 'fighting',\n",
       " '.',\n",
       " 'Our',\n",
       " 'Chiefs',\n",
       " 'are',\n",
       " 'killed',\n",
       " ';',\n",
       " 'Looking',\n",
       " 'Glass',\n",
       " 'is',\n",
       " 'dead',\n",
       " ',',\n",
       " 'Ta',\n",
       " 'Hool',\n",
       " 'Hool',\n",
       " 'Shute',\n",
       " 'is',\n",
       " 'dead',\n",
       " '.',\n",
       " 'The',\n",
       " 'old',\n",
       " 'men',\n",
       " 'are',\n",
       " 'all',\n",
       " 'dead',\n",
       " '.',\n",
       " 'It',\n",
       " 'is',\n",
       " 'the',\n",
       " 'young',\n",
       " 'men',\n",
       " 'who',\n",
       " 'say',\n",
       " 'yes',\n",
       " 'or',\n",
       " 'no',\n",
       " '.',\n",
       " 'He',\n",
       " 'who',\n",
       " 'led',\n",
       " 'on',\n",
       " 'the',\n",
       " 'young',\n",
       " 'men',\n",
       " 'is',\n",
       " 'dead',\n",
       " '.',\n",
       " 'It',\n",
       " 'is',\n",
       " 'cold',\n",
       " ',',\n",
       " 'and',\n",
       " 'we',\n",
       " 'have',\n",
       " 'no',\n",
       " 'blankets',\n",
       " ';',\n",
       " 'the',\n",
       " 'little',\n",
       " 'children',\n",
       " 'are',\n",
       " 'freezing',\n",
       " 'to',\n",
       " 'death',\n",
       " '.',\n",
       " 'My',\n",
       " 'people',\n",
       " ',',\n",
       " 'some',\n",
       " 'of',\n",
       " 'them',\n",
       " ',',\n",
       " 'have',\n",
       " 'run',\n",
       " 'away',\n",
       " 'to',\n",
       " 'the',\n",
       " 'hills',\n",
       " ',',\n",
       " 'and',\n",
       " 'have',\n",
       " 'no',\n",
       " 'blankets',\n",
       " ',',\n",
       " 'no',\n",
       " 'food',\n",
       " '.',\n",
       " 'No',\n",
       " 'one',\n",
       " 'knows',\n",
       " 'where',\n",
       " 'they',\n",
       " 'are',\n",
       " '–',\n",
       " 'perhaps',\n",
       " 'freezing',\n",
       " 'to',\n",
       " 'death',\n",
       " '.',\n",
       " 'I',\n",
       " 'want',\n",
       " 'to',\n",
       " 'have',\n",
       " 'time',\n",
       " 'to',\n",
       " 'look',\n",
       " 'for',\n",
       " 'my',\n",
       " 'children',\n",
       " ',',\n",
       " 'and',\n",
       " 'see',\n",
       " 'how',\n",
       " 'many',\n",
       " 'of',\n",
       " 'them',\n",
       " 'I',\n",
       " 'can',\n",
       " 'find',\n",
       " '.',\n",
       " 'Maybe',\n",
       " 'I',\n",
       " 'shall',\n",
       " 'find',\n",
       " 'them',\n",
       " 'among',\n",
       " 'the',\n",
       " 'dead',\n",
       " '.',\n",
       " 'Hear',\n",
       " 'me',\n",
       " ',',\n",
       " 'my',\n",
       " 'Chiefs',\n",
       " '!',\n",
       " 'I',\n",
       " 'am',\n",
       " 'tired',\n",
       " ';',\n",
       " 'my',\n",
       " 'heart',\n",
       " 'is',\n",
       " 'sick',\n",
       " 'and',\n",
       " 'sad',\n",
       " '.',\n",
       " 'From',\n",
       " 'where',\n",
       " 'the',\n",
       " 'sun',\n",
       " 'now',\n",
       " 'stands',\n",
       " 'I',\n",
       " 'will',\n",
       " 'fight',\n",
       " 'no',\n",
       " 'more',\n",
       " 'forever',\n",
       " '.']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.Stemming\n",
    "\"Stemming is process of reducing infected or derived words to their word stem, base or root form\"\n",
    "###### Words representation may not have any meaning.\n",
    "##### Takes less time\n",
    "##### Use stemming when meaning of words are not important for analysis. example spam detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import stemming from nltk\n",
    "from nltk.stem import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init the stemming\n",
    "stemmer = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tell gener howard I know hi heart .',\n",
       " 'what he told me befor , I have it in my heart .',\n",
       " 'I am tire of fight .',\n",
       " 'our chief are kill ; look glass is dead , Ta hool hool shute is dead .',\n",
       " 'the old men are all dead .',\n",
       " 'It is the young men who say ye or no .',\n",
       " 'He who led on the young men is dead .',\n",
       " 'It is cold , and we have no blanket ; the littl children are freez to death .',\n",
       " 'My peopl , some of them , have run away to the hill , and have no blanket , no food .',\n",
       " 'No one know where they are – perhap freez to death .',\n",
       " 'I want to have time to look for my children , and see how mani of them I can find .',\n",
       " 'mayb I shall find them among the dead .',\n",
       " 'hear me , my chief !',\n",
       " 'I am tire ; my heart is sick and sad .',\n",
       " 'from where the sun now stand I will fight no more forev .']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#stemming\n",
    "for i in range(len(sentences)):\n",
    "    words = nltk.word_tokenize(sentences[i])\n",
    "    newwords = [stemmer.stem(word) for word in words]\n",
    "    sentences[i]= ' '.join(newwords)\n",
    "    \n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.Lemmatization\n",
    "\"Same as Stemming but intermediate representation/root form has a meaning\"\n",
    "###### Words representation  have  meaning.\n",
    "##### Takes more time than Stemming\n",
    "##### Use Lemmatization when meaning of words are  important for analysis. example Question answer application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import Lemmatization from NLTK\n",
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = nltk.sent_tokenize(paragraph)   # seperate each sentences of our paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init the Lemmatization model\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['TellGeneralHowardIknowhisheart.',\n",
       " 'Whathetoldmebefore,Ihaveitinmyheart.',\n",
       " 'Iamtiredoffighting.',\n",
       " 'OurChiefsarekilled;LookingGlassisdead,TaHoolHoolShuteisdead.',\n",
       " 'Theoldmenarealldead.',\n",
       " 'Itistheyoungmenwhosayyesorno.',\n",
       " 'Hewholedontheyoungmenisdead.',\n",
       " 'Itiscold,andwehavenoblanket;thelittlechildarefreezingtodeath.',\n",
       " 'Mypeople,someofthem,haverunawaytothehill,andhavenoblanket,nofood.',\n",
       " 'Nooneknowwheretheyare–perhapsfreezingtodeath.',\n",
       " 'Iwanttohavetimetolookformychild,andseehowmanyofthemIcanfind.',\n",
       " 'MaybeIshallfindthemamongthedead.',\n",
       " 'Hearme,myChiefs!',\n",
       " 'Iamtired;myheartissickandsad.',\n",
       " 'FromwherethesunnowstandIwillfightnomoreforever.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lemmatization\n",
    "for i in range(len(sentences)):\n",
    "    words = nltk.word_tokenize(sentences[i])\n",
    "    newwords = [lemmatizer.lemmatize(word) for word in words]\n",
    "    sentences[i] = ''.join(newwords)\n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.Stop Word Removal using nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = nltk.sent_tokenize(paragraph)   # seperate each sentences of our paragraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Tell General Howard I know heart .',\n",
       " 'What told , I heart .',\n",
       " 'I tired fighting .',\n",
       " 'Our Chiefs killed ; Looking Glass dead , Ta Hool Hool Shute dead .',\n",
       " 'The old men dead .',\n",
       " 'It young men say yes .',\n",
       " 'He led young men dead .',\n",
       " 'It cold , blankets ; little children freezing death .',\n",
       " 'My people , , run away hills , blankets , food .',\n",
       " 'No one knows – perhaps freezing death .',\n",
       " 'I want time look children , see many I find .',\n",
       " 'Maybe I shall find among dead .',\n",
       " 'Hear , Chiefs !',\n",
       " 'I tired ; heart sick sad .',\n",
       " 'From sun stands I fight forever .']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stop word removal\n",
    "for i in range(len(sentences)):\n",
    "    words = nltk.word_tokenize(sentences[i])\n",
    "    newwords = [word for word in words if word not in stopwords.words('english')]\n",
    "    sentences[i] = ' '.join(newwords)\n",
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.Parts of Speech Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = nltk.word_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagged_words =nltk.pos_tag(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Tell_NNP General_NNP Howard_NNP I_PRP know_VBP his_PRP$ heart_NN ._. What_WP he_PRP told_VBD me_PRP before_IN ,_, I_PRP have_VBP it_PRP in_IN my_PRP$ heart_NN ._. I_PRP am_VBP tired_VBN of_IN fighting_VBG ._. Our_PRP$ Chiefs_NNS are_VBP killed_VBN ;_: Looking_VBG Glass_NNP is_VBZ dead_JJ ,_, Ta_NNP Hool_NNP Hool_NNP Shute_NNP is_VBZ dead_JJ ._. The_DT old_JJ men_NNS are_VBP all_DT dead_JJ ._. It_PRP is_VBZ the_DT young_JJ men_NNS who_WP say_VBP yes_UH or_CC no_DT ._. He_PRP who_WP led_VBD on_IN the_DT young_JJ men_NNS is_VBZ dead_JJ ._. It_PRP is_VBZ cold_JJ ,_, and_CC we_PRP have_VBP no_DT blankets_NNS ;_: the_DT little_JJ children_NNS are_VBP freezing_VBG to_TO death_NN ._. My_PRP$ people_NNS ,_, some_DT of_IN them_PRP ,_, have_VBP run_VBN away_RB to_TO the_DT hills_NNS ,_, and_CC have_VBP no_DT blankets_NNS ,_, no_DT food_NN ._. No_DT one_NN knows_VBZ where_WRB they_PRP are_VBP –_JJ perhaps_RB freezing_VBG to_TO death_NN ._. I_PRP want_VBP to_TO have_VB time_NN to_TO look_VB for_IN my_PRP$ children_NNS ,_, and_CC see_VB how_WRB many_JJ of_IN them_PRP I_PRP can_MD find_VB ._. Maybe_RB I_PRP shall_MD find_VB them_PRP among_IN the_DT dead_JJ ._. Hear_VB me_PRP ,_, my_PRP$ Chiefs_JJ !_. I_PRP am_VBP tired_VBN ;_: my_PRP$ heart_NN is_VBZ sick_JJ and_CC sad_JJ ._. From_IN where_WRB the_DT sun_NN now_RB stands_VBZ I_PRP will_MD fight_VB no_DT more_JJR forever_RB ._.'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tags = []\n",
    "for tw in tagged_words:\n",
    "    word_tags.append(tw[0]+\"_\"+tw[1])\n",
    "    \n",
    "tagged_paragraph = ' '.join(word_tags)\n",
    "tagged_paragraph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.Named Entity Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph1 = \"Fouder of Wavy AI Research Foundation is from Pakistan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Fouder',\n",
       " 'of',\n",
       " 'Wavy',\n",
       " 'AI',\n",
       " 'Research',\n",
       " 'Foundation',\n",
       " 'is',\n",
       " 'from',\n",
       " 'Pakistan']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = nltk.word_tokenize(paragraph1)\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Fouder', 'NN'),\n",
       " ('of', 'IN'),\n",
       " ('Wavy', 'NNP'),\n",
       " ('AI', 'NNP'),\n",
       " ('Research', 'NNP'),\n",
       " ('Foundation', 'NNP'),\n",
       " ('is', 'VBZ'),\n",
       " ('from', 'IN'),\n",
       " ('Pakistan', 'NNP')]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_words = nltk.pos_tag(words)\n",
    "tagged_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "namedEnt = nltk.ne_chunk(tagged_words)\n",
    "namedEnt.draw()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7.Building a Bags of Word Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import nltk\n",
    "import re\n",
    "import heapq\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = \"\"\"Tell General Howard I know his heart. What he told me before, I have it in my heart. I am tired of fighting.\n",
    "Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead. The old men are all dead. It is the young men who\n",
    "say yes or no. He who led on the young men is dead. It is cold, and we have no blankets; the little children are freezing \n",
    "to death. My people, some of them, have run away to the hills, and have no blankets, no food. No one knows where they are \n",
    "– perhaps freezing to death. I want to have time to look for my children, and see how many of them I can find. Maybe I \n",
    "shall find them among the dead. Hear me, my Chiefs! I am tired; my heart is sick and sad. From where the sun now stands \n",
    "I will fight no more forever.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset= nltk.sent_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Clean the text\n",
    "for i in range(len(dataset)):\n",
    "    dataset[i] = dataset[i].lower()\n",
    "    dataset[i] = re.sub(r'\\W',' ', dataset[i])\n",
    "    dataset[i] = re.sub(r'\\s+',' ', dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the histogram\n",
    "word2count = {}\n",
    "for data in dataset:\n",
    "    words = nltk.word_tokenize(data)\n",
    "    for word in words:\n",
    "        if word not in word2count.keys():\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a 100 most frequent words from above dictionaries\n",
    "freq_words = heapq.nlargest(100, word2count, key= word2count.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# finally building our BOW model\n",
    "X = []\n",
    "for data in dataset:\n",
    "    vector = []\n",
    "    for word in freq_words:\n",
    "        if word in nltk.word_tokenize(data):\n",
    "            vector.append(1)\n",
    "        else:\n",
    "            vector.append(0)\n",
    "    X.append(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [1, 0, 1, ..., 0, 0, 0],\n",
       "       [1, 1, 0, ..., 1, 1, 1]])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Bag_of_Word_Model = np.asarray(X)\n",
    "Bag_of_Word_Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8.Building the TF-IDF Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import nltk\n",
    "import re\n",
    "import heapq\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph = \"\"\"Tell General Howard I know his heart. What he told me before, I have it in my heart. I am tired of fighting.\n",
    "Our Chiefs are killed; Looking Glass is dead, Ta Hool Hool Shute is dead. The old men are all dead. It is the young men who\n",
    "say yes or no. He who led on the young men is dead. It is cold, and we have no blankets; the little children are freezing \n",
    "to death. My people, some of them, have run away to the hills, and have no blankets, no food. No one knows where they are \n",
    "– perhaps freezing to death. I want to have time to look for my children, and see how many of them I can find. Maybe I \n",
    "shall find them among the dead. Hear me, my Chiefs! I am tired; my heart is sick and sad. From where the sun now stands \n",
    "I will fight no more forever.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset= nltk.sent_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  Clean the text\n",
    "for i in range(len(dataset)):\n",
    "    dataset[i] = dataset[i].lower()\n",
    "    dataset[i] = re.sub(r'\\W',' ', dataset[i])\n",
    "    dataset[i] = re.sub(r'\\s+',' ', dataset[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating the histogram\n",
    "word2count = {}\n",
    "for data in dataset:\n",
    "    words = nltk.word_tokenize(data)\n",
    "    for word in words:\n",
    "        if word not in word2count.keys():\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take a 100 most frequent words from above dictionaries\n",
    "freq_words = heapq.nlargest(100, word2count, key= word2count.get)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IDF Matrix\n",
    "word_idfs = {}\n",
    "for word in  freq_words:\n",
    "    doc_count = 0\n",
    "    for data in dataset:\n",
    "        if word in nltk.word_tokenize(data):\n",
    "            doc_count += 1\n",
    "    word_idfs[word] = np.log((len(dataset)/doc_count)+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF Matrix\n",
    "tf_matrix = {}\n",
    "for word in freq_words:\n",
    "    doc_tf = []\n",
    "    for data in dataset:\n",
    "        frequency = 0\n",
    "        for w in nltk.word_tokenize(data):\n",
    "            if w== word:\n",
    "                frequency += 1\n",
    "        tf_word = frequency/len(nltk.word_tokenize(data))\n",
    "        doc_tf.append(tf_word)\n",
    "    tf_matrix[word] = doc_tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF-IDF Calculation\n",
    "tfidf_matrix = []\n",
    "for word in tf_matrix.keys():\n",
    "    tfidf = []\n",
    "    for value in tf_matrix[word]:\n",
    "        score = value * word_idfs[word]\n",
    "        tfidf.append(score)\n",
    "    tfidf_matrix.append(tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.asarray(tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.16359033, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.10410294, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.22902646, 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.        , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.12723692, 0.        , 0.15403271, ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.09542769, 0.09542769, 0.        , ..., 0.23104906, 0.23104906,\n",
       "        0.23104906]])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TF_IDF_Model = np.transpose(X)\n",
    "TF_IDF_Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.Building the N-Gram Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. N-Gram Modeling - Character Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data\n",
    "text= \"\"\"There are many different types of essay writing and, depending on what you are writing for, the format and approach \n",
    "can change. We’ve designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in \n",
    "the right direction – no matter if your essay is designed to persuade the college admissions team that you’re the right \n",
    "candidate, if you are making your case to win a scholarship, or simply if you are looking for help with your homework There are\n",
    "many different types of essay writing and, depending on what you are writing for, the format and approach can change. We’ve \n",
    "designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in the right direction \n",
    "– no matter if your essay is designed to persuade the college admissions team that you’re the right candidate, if you are \n",
    "making your case to win a scholarship, or simply if you are looking for help with your homework\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating the n-gram\n",
    "n = 3\n",
    "ngrams = {}\n",
    "for i in range(len(text)-n):\n",
    "    gram = text[i:i+n]  #text[0:3] = The\n",
    "    if gram not in ngrams.keys():\n",
    "        ngrams[gram] = []\n",
    "    ngrams[gram].append(text[i+n]) #text[0+3] = text[3] = r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There looking for, the right \n",
      "can change. We’ve designed in a scholarship, or simply is dependidate, if\n"
     ]
    }
   ],
   "source": [
    "# Testing our n-gram model\n",
    "currentGram = text[0:n]\n",
    "result = currentGram\n",
    "for i in range(100):\n",
    "    if currentGram not in ngrams.keys():\n",
    "        break\n",
    "    possibilities = ngrams[currentGram]\n",
    "    nextItem = possibilities[random.randrange(len(possibilities))]\n",
    "    result += nextItem\n",
    "    currentGram = result[len(result)-n: len(result)]\n",
    "    \n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2. N-Gram Modeling - Word Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import librararies\n",
    "import random\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample data\n",
    "text= \"\"\"There are many different types of essay writing and, depending on what you are writing for, the format and approach \n",
    "can change. We’ve designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in \n",
    "the right direction – no matter if your essay is designed to persuade the college admissions team that you’re the right \n",
    "candidate, if you are making your case to win a scholarship, or simply if you are looking for help with your homework There are\n",
    "many different types of essay writing and, depending on what you are writing for, the format and approach can change. We’ve \n",
    "designed our Essay Writing Center to provide you with key tips and pointers so that you can get started in the right direction \n",
    "– no matter if your essay is designed to persuade the college admissions team that you’re the right candidate, if you are \n",
    "making your case to win a scholarship, or simply if you are looking for help with your homework\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating the n-gram\n",
    "n = 3\n",
    "ngrams = {}\n",
    "words = nltk.word_tokenize(text)\n",
    "for i in range(len(words)-n):\n",
    "    gram = ' '.join(words[i:i+n])\n",
    "    if gram not in ngrams.keys():\n",
    "        ngrams[gram] = []\n",
    "    ngrams[gram].append(words[i+n]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'rwords' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-25-ca4e84d04d6d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0mnextItem\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpossibilities\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpossibilities\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[0mresult\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;34m' '\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mnextItem\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m     \u001b[0mcurrentGram\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m' '\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrwords\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrwords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrwords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'rwords' is not defined"
     ]
    }
   ],
   "source": [
    "# Testing our n-gram model\n",
    "currentGram = ' '.join(words[0:n])\n",
    "result = currentGram\n",
    "for i in range(30):\n",
    "    if currentGram not in ngrams.keys():\n",
    "        break\n",
    "    possibilities = ngrams[currentGram]\n",
    "    nextItem = possibilities[random.randrange(len(possibilities))]\n",
    "    result += ' '+nextItem\n",
    "    rword = nltk.word_\n",
    "    currentGram = ' '.join(rwords[len(rwords)-n:len(rwords)])\n",
    "    \n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}