TF-IDF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "documentA='in sooth I know not why I am so sad it wearies me you say it wearies you but how I caught it found it or came by it what stuff tis made of whereof it is born I am to learn and such a want wit sadness makes of me that I have much ado to know myself'\n",
    "documentB='your mind is tossing on the ocean there where your argosies with portly sail like signors and rich burghers on the flood or as it were the pageants of the sea do overpeer the petty traffickers that curtsy to them do them reverence as they fly by them with their woven wings'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bagOfWordsA=documentA.split(' ')\n",
    "bagOfWordsB=documentB.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['in',\n",
       " 'sooth',\n",
       " 'I',\n",
       " 'know',\n",
       " 'not',\n",
       " 'why',\n",
       " 'I',\n",
       " 'am',\n",
       " 'so',\n",
       " 'sad',\n",
       " 'it',\n",
       " 'wearies',\n",
       " 'me',\n",
       " 'you',\n",
       " 'say',\n",
       " 'it',\n",
       " 'wearies',\n",
       " 'you',\n",
       " 'but',\n",
       " 'how',\n",
       " 'I',\n",
       " 'caught',\n",
       " 'it',\n",
       " 'found',\n",
       " 'it',\n",
       " 'or',\n",
       " 'came',\n",
       " 'by',\n",
       " 'it',\n",
       " 'what',\n",
       " 'stuff',\n",
       " 'tis',\n",
       " 'made',\n",
       " 'of',\n",
       " 'whereof',\n",
       " 'it',\n",
       " 'is',\n",
       " 'born',\n",
       " 'I',\n",
       " 'am',\n",
       " 'to',\n",
       " 'learn',\n",
       " 'and',\n",
       " 'such',\n",
       " 'a',\n",
       " 'want',\n",
       " 'wit',\n",
       " 'sadness',\n",
       " 'makes',\n",
       " 'of',\n",
       " 'me',\n",
       " 'that',\n",
       " 'I',\n",
       " 'have',\n",
       " 'much',\n",
       " 'ado',\n",
       " 'to',\n",
       " 'know',\n",
       " 'myself']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagOfWordsA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['your',\n",
       " 'mind',\n",
       " 'is',\n",
       " 'tossing',\n",
       " 'on',\n",
       " 'the',\n",
       " 'ocean',\n",
       " 'there',\n",
       " 'where',\n",
       " 'your',\n",
       " 'argosies',\n",
       " 'with',\n",
       " 'portly',\n",
       " 'sail',\n",
       " 'like',\n",
       " 'signors',\n",
       " 'and',\n",
       " 'rich',\n",
       " 'burghers',\n",
       " 'on',\n",
       " 'the',\n",
       " 'flood',\n",
       " 'or',\n",
       " 'as',\n",
       " 'it',\n",
       " 'were',\n",
       " 'the',\n",
       " 'pageants',\n",
       " 'of',\n",
       " 'the',\n",
       " 'sea',\n",
       " 'do',\n",
       " 'overpeer',\n",
       " 'the',\n",
       " 'petty',\n",
       " 'traffickers',\n",
       " 'that',\n",
       " 'curtsy',\n",
       " 'to',\n",
       " 'them',\n",
       " 'do',\n",
       " 'them',\n",
       " 'reverence',\n",
       " 'as',\n",
       " 'they',\n",
       " 'fly',\n",
       " 'by',\n",
       " 'them',\n",
       " 'with',\n",
       " 'their',\n",
       " 'woven',\n",
       " 'wings']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagOfWordsB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'I',\n",
       " 'a',\n",
       " 'ado',\n",
       " 'am',\n",
       " 'and',\n",
       " 'argosies',\n",
       " 'as',\n",
       " 'born',\n",
       " 'burghers',\n",
       " 'but',\n",
       " 'by',\n",
       " 'came',\n",
       " 'caught',\n",
       " 'curtsy',\n",
       " 'do',\n",
       " 'flood',\n",
       " 'fly',\n",
       " 'found',\n",
       " 'have',\n",
       " 'how',\n",
       " 'in',\n",
       " 'is',\n",
       " 'it',\n",
       " 'know',\n",
       " 'learn',\n",
       " 'like',\n",
       " 'made',\n",
       " 'makes',\n",
       " 'me',\n",
       " 'mind',\n",
       " 'much',\n",
       " 'myself',\n",
       " 'not',\n",
       " 'ocean',\n",
       " 'of',\n",
       " 'on',\n",
       " 'or',\n",
       " 'overpeer',\n",
       " 'pageants',\n",
       " 'petty',\n",
       " 'portly',\n",
       " 'reverence',\n",
       " 'rich',\n",
       " 'sad',\n",
       " 'sadness',\n",
       " 'sail',\n",
       " 'say',\n",
       " 'sea',\n",
       " 'signors',\n",
       " 'so',\n",
       " 'sooth',\n",
       " 'stuff',\n",
       " 'such',\n",
       " 'that',\n",
       " 'the',\n",
       " 'their',\n",
       " 'them',\n",
       " 'there',\n",
       " 'they',\n",
       " 'tis',\n",
       " 'to',\n",
       " 'tossing',\n",
       " 'traffickers',\n",
       " 'want',\n",
       " 'wearies',\n",
       " 'were',\n",
       " 'what',\n",
       " 'where',\n",
       " 'whereof',\n",
       " 'why',\n",
       " 'wings',\n",
       " 'wit',\n",
       " 'with',\n",
       " 'woven',\n",
       " 'you',\n",
       " 'your'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniqueWords=set(bagOfWordsA).union(set(bagOfWordsB))\n",
    "uniqueWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from nltk.corpus import stopwords\n",
    "#stop_words=set(stopwords.words('english'))\n",
    "#stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filtered=[]\n",
    "#for w in uniqueWords:\n",
    "#    if w not in stop_words:\n",
    "#        filtered.append(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "numOfWordsA=dict.fromkeys(uniqueWords,0)\n",
    "for word in bagOfWordsA:\n",
    "    numOfWordsA[word]+=1\n",
    "\n",
    "numOfWordsB=dict.fromkeys(uniqueWords,0)\n",
    "for word in bagOfWordsB:\n",
    "    numOfWordsB[word]+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'made': 1,\n",
       " 'wearies': 2,\n",
       " 'curtsy': 0,\n",
       " 'petty': 0,\n",
       " 'want': 1,\n",
       " 'burghers': 0,\n",
       " 'flood': 0,\n",
       " 'you': 2,\n",
       " 'ado': 1,\n",
       " 'is': 1,\n",
       " 'in': 1,\n",
       " 'have': 1,\n",
       " 'found': 1,\n",
       " 'of': 2,\n",
       " 'born': 1,\n",
       " 'pageants': 0,\n",
       " 'them': 0,\n",
       " 'were': 0,\n",
       " 'that': 1,\n",
       " 'fly': 0,\n",
       " 'not': 1,\n",
       " 'ocean': 0,\n",
       " 'such': 1,\n",
       " 'portly': 0,\n",
       " 'their': 0,\n",
       " 'what': 1,\n",
       " 'or': 1,\n",
       " 'with': 0,\n",
       " 'mind': 0,\n",
       " 'by': 1,\n",
       " 'like': 0,\n",
       " 'caught': 1,\n",
       " 'tis': 1,\n",
       " 'so': 1,\n",
       " 'tossing': 0,\n",
       " 'overpeer': 0,\n",
       " 'came': 1,\n",
       " 'on': 0,\n",
       " 'they': 0,\n",
       " 'stuff': 1,\n",
       " 'how': 1,\n",
       " 'am': 2,\n",
       " 'know': 2,\n",
       " 'why': 1,\n",
       " 'reverence': 0,\n",
       " 'signors': 0,\n",
       " 'wings': 0,\n",
       " 'a': 1,\n",
       " 'me': 2,\n",
       " 'sadness': 1,\n",
       " 'where': 0,\n",
       " 'and': 1,\n",
       " 'your': 0,\n",
       " 'whereof': 1,\n",
       " 'the': 0,\n",
       " 'as': 0,\n",
       " 'sad': 1,\n",
       " 'but': 1,\n",
       " 'much': 1,\n",
       " 'sail': 0,\n",
       " 'say': 1,\n",
       " 'rich': 0,\n",
       " 'myself': 1,\n",
       " 'there': 0,\n",
       " 'it': 6,\n",
       " 'traffickers': 0,\n",
       " 'to': 2,\n",
       " 'learn': 1,\n",
       " 'sea': 0,\n",
       " 'woven': 0,\n",
       " 'argosies': 0,\n",
       " 'do': 0,\n",
       " 'wit': 1,\n",
       " 'makes': 1,\n",
       " 'I': 5,\n",
       " 'sooth': 1}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numOfWordsA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'made': 0,\n",
       " 'wearies': 0,\n",
       " 'curtsy': 1,\n",
       " 'petty': 1,\n",
       " 'want': 0,\n",
       " 'burghers': 1,\n",
       " 'flood': 1,\n",
       " 'you': 0,\n",
       " 'ado': 0,\n",
       " 'is': 1,\n",
       " 'in': 0,\n",
       " 'have': 0,\n",
       " 'found': 0,\n",
       " 'of': 1,\n",
       " 'born': 0,\n",
       " 'pageants': 1,\n",
       " 'them': 3,\n",
       " 'were': 1,\n",
       " 'that': 1,\n",
       " 'fly': 1,\n",
       " 'not': 0,\n",
       " 'ocean': 1,\n",
       " 'such': 0,\n",
       " 'portly': 1,\n",
       " 'their': 1,\n",
       " 'what': 0,\n",
       " 'or': 1,\n",
       " 'with': 2,\n",
       " 'mind': 1,\n",
       " 'by': 1,\n",
       " 'like': 1,\n",
       " 'caught': 0,\n",
       " 'tis': 0,\n",
       " 'so': 0,\n",
       " 'tossing': 1,\n",
       " 'overpeer': 1,\n",
       " 'came': 0,\n",
       " 'on': 2,\n",
       " 'they': 1,\n",
       " 'stuff': 0,\n",
       " 'how': 0,\n",
       " 'am': 0,\n",
       " 'know': 0,\n",
       " 'why': 0,\n",
       " 'reverence': 1,\n",
       " 'signors': 1,\n",
       " 'wings': 1,\n",
       " 'a': 0,\n",
       " 'me': 0,\n",
       " 'sadness': 0,\n",
       " 'where': 1,\n",
       " 'and': 1,\n",
       " 'your': 2,\n",
       " 'whereof': 0,\n",
       " 'the': 5,\n",
       " 'as': 2,\n",
       " 'sad': 0,\n",
       " 'but': 0,\n",
       " 'much': 0,\n",
       " 'sail': 1,\n",
       " 'say': 0,\n",
       " 'rich': 1,\n",
       " 'myself': 0,\n",
       " 'there': 1,\n",
       " 'it': 1,\n",
       " 'traffickers': 1,\n",
       " 'to': 1,\n",
       " 'learn': 0,\n",
       " 'sea': 1,\n",
       " 'woven': 1,\n",
       " 'argosies': 1,\n",
       " 'do': 2,\n",
       " 'wit': 0,\n",
       " 'makes': 0,\n",
       " 'I': 0,\n",
       " 'sooth': 0}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numOfWordsB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeTF(wordDict, bagOfWords):\n",
    "    tfDict = {} #dictionary\n",
    "    bagOfWordsCount = len(bagOfWords)\n",
    "    for word, count in wordDict.items():\n",
    "        tfDict[word] = count / float(bagOfWordsCount)\n",
    "    return tfDict\n",
    "#funtion to compute tf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfA=computeTF(numOfWordsA,bagOfWordsA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'made': 0.01694915254237288,\n",
       " 'wearies': 0.03389830508474576,\n",
       " 'curtsy': 0.0,\n",
       " 'petty': 0.0,\n",
       " 'want': 0.01694915254237288,\n",
       " 'burghers': 0.0,\n",
       " 'flood': 0.0,\n",
       " 'you': 0.03389830508474576,\n",
       " 'ado': 0.01694915254237288,\n",
       " 'is': 0.01694915254237288,\n",
       " 'in': 0.01694915254237288,\n",
       " 'have': 0.01694915254237288,\n",
       " 'found': 0.01694915254237288,\n",
       " 'of': 0.03389830508474576,\n",
       " 'born': 0.01694915254237288,\n",
       " 'pageants': 0.0,\n",
       " 'them': 0.0,\n",
       " 'were': 0.0,\n",
       " 'that': 0.01694915254237288,\n",
       " 'fly': 0.0,\n",
       " 'not': 0.01694915254237288,\n",
       " 'ocean': 0.0,\n",
       " 'such': 0.01694915254237288,\n",
       " 'portly': 0.0,\n",
       " 'their': 0.0,\n",
       " 'what': 0.01694915254237288,\n",
       " 'or': 0.01694915254237288,\n",
       " 'with': 0.0,\n",
       " 'mind': 0.0,\n",
       " 'by': 0.01694915254237288,\n",
       " 'like': 0.0,\n",
       " 'caught': 0.01694915254237288,\n",
       " 'tis': 0.01694915254237288,\n",
       " 'so': 0.01694915254237288,\n",
       " 'tossing': 0.0,\n",
       " 'overpeer': 0.0,\n",
       " 'came': 0.01694915254237288,\n",
       " 'on': 0.0,\n",
       " 'they': 0.0,\n",
       " 'stuff': 0.01694915254237288,\n",
       " 'how': 0.01694915254237288,\n",
       " 'am': 0.03389830508474576,\n",
       " 'know': 0.03389830508474576,\n",
       " 'why': 0.01694915254237288,\n",
       " 'reverence': 0.0,\n",
       " 'signors': 0.0,\n",
       " 'wings': 0.0,\n",
       " 'a': 0.01694915254237288,\n",
       " 'me': 0.03389830508474576,\n",
       " 'sadness': 0.01694915254237288,\n",
       " 'where': 0.0,\n",
       " 'and': 0.01694915254237288,\n",
       " 'your': 0.0,\n",
       " 'whereof': 0.01694915254237288,\n",
       " 'the': 0.0,\n",
       " 'as': 0.0,\n",
       " 'sad': 0.01694915254237288,\n",
       " 'but': 0.01694915254237288,\n",
       " 'much': 0.01694915254237288,\n",
       " 'sail': 0.0,\n",
       " 'say': 0.01694915254237288,\n",
       " 'rich': 0.0,\n",
       " 'myself': 0.01694915254237288,\n",
       " 'there': 0.0,\n",
       " 'it': 0.1016949152542373,\n",
       " 'traffickers': 0.0,\n",
       " 'to': 0.03389830508474576,\n",
       " 'learn': 0.01694915254237288,\n",
       " 'sea': 0.0,\n",
       " 'woven': 0.0,\n",
       " 'argosies': 0.0,\n",
       " 'do': 0.0,\n",
       " 'wit': 0.01694915254237288,\n",
       " 'makes': 0.01694915254237288,\n",
       " 'I': 0.0847457627118644,\n",
       " 'sooth': 0.01694915254237288}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'made': 0.0,\n",
       " 'wearies': 0.0,\n",
       " 'curtsy': 0.019230769230769232,\n",
       " 'petty': 0.019230769230769232,\n",
       " 'want': 0.0,\n",
       " 'burghers': 0.019230769230769232,\n",
       " 'flood': 0.019230769230769232,\n",
       " 'you': 0.0,\n",
       " 'ado': 0.0,\n",
       " 'is': 0.019230769230769232,\n",
       " 'in': 0.0,\n",
       " 'have': 0.0,\n",
       " 'found': 0.0,\n",
       " 'of': 0.019230769230769232,\n",
       " 'born': 0.0,\n",
       " 'pageants': 0.019230769230769232,\n",
       " 'them': 0.057692307692307696,\n",
       " 'were': 0.019230769230769232,\n",
       " 'that': 0.019230769230769232,\n",
       " 'fly': 0.019230769230769232,\n",
       " 'not': 0.0,\n",
       " 'ocean': 0.019230769230769232,\n",
       " 'such': 0.0,\n",
       " 'portly': 0.019230769230769232,\n",
       " 'their': 0.019230769230769232,\n",
       " 'what': 0.0,\n",
       " 'or': 0.019230769230769232,\n",
       " 'with': 0.038461538461538464,\n",
       " 'mind': 0.019230769230769232,\n",
       " 'by': 0.019230769230769232,\n",
       " 'like': 0.019230769230769232,\n",
       " 'caught': 0.0,\n",
       " 'tis': 0.0,\n",
       " 'so': 0.0,\n",
       " 'tossing': 0.019230769230769232,\n",
       " 'overpeer': 0.019230769230769232,\n",
       " 'came': 0.0,\n",
       " 'on': 0.038461538461538464,\n",
       " 'they': 0.019230769230769232,\n",
       " 'stuff': 0.0,\n",
       " 'how': 0.0,\n",
       " 'am': 0.0,\n",
       " 'know': 0.0,\n",
       " 'why': 0.0,\n",
       " 'reverence': 0.019230769230769232,\n",
       " 'signors': 0.019230769230769232,\n",
       " 'wings': 0.019230769230769232,\n",
       " 'a': 0.0,\n",
       " 'me': 0.0,\n",
       " 'sadness': 0.0,\n",
       " 'where': 0.019230769230769232,\n",
       " 'and': 0.019230769230769232,\n",
       " 'your': 0.038461538461538464,\n",
       " 'whereof': 0.0,\n",
       " 'the': 0.09615384615384616,\n",
       " 'as': 0.038461538461538464,\n",
       " 'sad': 0.0,\n",
       " 'but': 0.0,\n",
       " 'much': 0.0,\n",
       " 'sail': 0.019230769230769232,\n",
       " 'say': 0.0,\n",
       " 'rich': 0.019230769230769232,\n",
       " 'myself': 0.0,\n",
       " 'there': 0.019230769230769232,\n",
       " 'it': 0.019230769230769232,\n",
       " 'traffickers': 0.019230769230769232,\n",
       " 'to': 0.019230769230769232,\n",
       " 'learn': 0.0,\n",
       " 'sea': 0.019230769230769232,\n",
       " 'woven': 0.019230769230769232,\n",
       " 'argosies': 0.019230769230769232,\n",
       " 'do': 0.038461538461538464,\n",
       " 'wit': 0.0,\n",
       " 'makes': 0.0,\n",
       " 'I': 0.0,\n",
       " 'sooth': 0.0}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfB=computeTF(numOfWordsB,bagOfWordsB)\n",
    "tfB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeIDF(documents):\n",
    "    import math\n",
    "    N = len(documents)\n",
    "    \n",
    "    idfDict = dict.fromkeys(documents[0].keys(), 0)\n",
    "    for document in documents:\n",
    "        for word, val in document.items():\n",
    "            if val > 0:\n",
    "                idfDict[word] += 1\n",
    "    \n",
    "    for word, val in idfDict.items():\n",
    "        idfDict[word] = math.log(N / float(val))\n",
    "    return idfDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "idfs = computeIDF([numOfWordsA, numOfWordsB])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'made': 0.6931471805599453,\n",
       " 'wearies': 0.6931471805599453,\n",
       " 'curtsy': 0.6931471805599453,\n",
       " 'petty': 0.6931471805599453,\n",
       " 'want': 0.6931471805599453,\n",
       " 'burghers': 0.6931471805599453,\n",
       " 'flood': 0.6931471805599453,\n",
       " 'you': 0.6931471805599453,\n",
       " 'ado': 0.6931471805599453,\n",
       " 'is': 0.0,\n",
       " 'in': 0.6931471805599453,\n",
       " 'have': 0.6931471805599453,\n",
       " 'found': 0.6931471805599453,\n",
       " 'of': 0.0,\n",
       " 'born': 0.6931471805599453,\n",
       " 'pageants': 0.6931471805599453,\n",
       " 'them': 0.6931471805599453,\n",
       " 'were': 0.6931471805599453,\n",
       " 'that': 0.0,\n",
       " 'fly': 0.6931471805599453,\n",
       " 'not': 0.6931471805599453,\n",
       " 'ocean': 0.6931471805599453,\n",
       " 'such': 0.6931471805599453,\n",
       " 'portly': 0.6931471805599453,\n",
       " 'their': 0.6931471805599453,\n",
       " 'what': 0.6931471805599453,\n",
       " 'or': 0.0,\n",
       " 'with': 0.6931471805599453,\n",
       " 'mind': 0.6931471805599453,\n",
       " 'by': 0.0,\n",
       " 'like': 0.6931471805599453,\n",
       " 'caught': 0.6931471805599453,\n",
       " 'tis': 0.6931471805599453,\n",
       " 'so': 0.6931471805599453,\n",
       " 'tossing': 0.6931471805599453,\n",
       " 'overpeer': 0.6931471805599453,\n",
       " 'came': 0.6931471805599453,\n",
       " 'on': 0.6931471805599453,\n",
       " 'they': 0.6931471805599453,\n",
       " 'stuff': 0.6931471805599453,\n",
       " 'how': 0.6931471805599453,\n",
       " 'am': 0.6931471805599453,\n",
       " 'know': 0.6931471805599453,\n",
       " 'why': 0.6931471805599453,\n",
       " 'reverence': 0.6931471805599453,\n",
       " 'signors': 0.6931471805599453,\n",
       " 'wings': 0.6931471805599453,\n",
       " 'a': 0.6931471805599453,\n",
       " 'me': 0.6931471805599453,\n",
       " 'sadness': 0.6931471805599453,\n",
       " 'where': 0.6931471805599453,\n",
       " 'and': 0.0,\n",
       " 'your': 0.6931471805599453,\n",
       " 'whereof': 0.6931471805599453,\n",
       " 'the': 0.6931471805599453,\n",
       " 'as': 0.6931471805599453,\n",
       " 'sad': 0.6931471805599453,\n",
       " 'but': 0.6931471805599453,\n",
       " 'much': 0.6931471805599453,\n",
       " 'sail': 0.6931471805599453,\n",
       " 'say': 0.6931471805599453,\n",
       " 'rich': 0.6931471805599453,\n",
       " 'myself': 0.6931471805599453,\n",
       " 'there': 0.6931471805599453,\n",
       " 'it': 0.0,\n",
       " 'traffickers': 0.6931471805599453,\n",
       " 'to': 0.0,\n",
       " 'learn': 0.6931471805599453,\n",
       " 'sea': 0.6931471805599453,\n",
       " 'woven': 0.6931471805599453,\n",
       " 'argosies': 0.6931471805599453,\n",
       " 'do': 0.6931471805599453,\n",
       " 'wit': 0.6931471805599453,\n",
       " 'makes': 0.6931471805599453,\n",
       " 'I': 0.6931471805599453,\n",
       " 'sooth': 0.6931471805599453}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def computeTFIDF(tfBagOfWords, idfs):\n",
    "    tfidf = {}\n",
    "    for word, val in tfBagOfWords.items():\n",
    "        tfidf[word] = val * idfs[word]\n",
    "    return tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidfA = computeTFIDF(tfA, idfs)\n",
    "tfidfB = computeTFIDF(tfB, idfs)\n",
    "df = pd.DataFrame([tfidfA, tfidfB])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>made</th>\n",
       "      <th>wearies</th>\n",
       "      <th>curtsy</th>\n",
       "      <th>petty</th>\n",
       "      <th>want</th>\n",
       "      <th>burghers</th>\n",
       "      <th>flood</th>\n",
       "      <th>you</th>\n",
       "      <th>ado</th>\n",
       "      <th>is</th>\n",
       "      <th>...</th>\n",
       "      <th>to</th>\n",
       "      <th>learn</th>\n",
       "      <th>sea</th>\n",
       "      <th>woven</th>\n",
       "      <th>argosies</th>\n",
       "      <th>do</th>\n",
       "      <th>wit</th>\n",
       "      <th>makes</th>\n",
       "      <th>I</th>\n",
       "      <th>sooth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.011748</td>\n",
       "      <td>0.023497</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.011748</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.023497</td>\n",
       "      <td>0.011748</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011748</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.011748</td>\n",
       "      <td>0.011748</td>\n",
       "      <td>0.058741</td>\n",
       "      <td>0.011748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.01333</td>\n",
       "      <td>0.02666</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 76 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       made   wearies   curtsy    petty      want  burghers    flood  \\\n",
       "0  0.011748  0.023497  0.00000  0.00000  0.011748   0.00000  0.00000   \n",
       "1  0.000000  0.000000  0.01333  0.01333  0.000000   0.01333  0.01333   \n",
       "\n",
       "        you       ado   is  ...   to     learn      sea    woven  argosies  \\\n",
       "0  0.023497  0.011748  0.0  ...  0.0  0.011748  0.00000  0.00000   0.00000   \n",
       "1  0.000000  0.000000  0.0  ...  0.0  0.000000  0.01333  0.01333   0.01333   \n",
       "\n",
       "        do       wit     makes         I     sooth  \n",
       "0  0.00000  0.011748  0.011748  0.058741  0.011748  \n",
       "1  0.02666  0.000000  0.000000  0.000000  0.000000  \n",
       "\n",
       "[2 rows x 76 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://towardsdatascience.com/natural-language-processing-feature-engineering-using-tf-idf-e8b9d00e7e76\n",
    "#https://towardsdatascience.com/tf-term-frequency-idf-inverse-document-frequency-from-scratch-in-python-6c2b61b78558"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}