Results5k topics -databases.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'analyze_results' from 'analyze_results.py'>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "### WE SHOULD HAVE 3 DIFF SAMPLES\n",
    "path_data_file = \"../Data/reviews_clean.p\"\n",
    "\n",
    "name_model_pickle_lda_r = \"../models/lda5k_lda.p\"\n",
    "name_model_pickle_lda_b = \"../models/b_lda5k_lda.p\"\n",
    "name_model_pickle_lda_u = \"../models/u_lda5k_lda.p\"\n",
    "\n",
    "num_words_to_compare = 10\n",
    "distances = ['euclidean','cosine','correlation']\n",
    "\n",
    "import pandas as pd\n",
    "import lda\n",
    "from scipy.spatial import distance\n",
    "from scipy.spatial.distance import pdist, squareform\n",
    "\n",
    "## You should have these scripts\n",
    "import vocabulary as v\n",
    "import numpy as np\n",
    "import analyze_results as ar\n",
    "reload(ar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_pickle(path_data_file)\n",
    "\n",
    "result_lda_r = pd.read_pickle(name_model_pickle_lda_r)\n",
    "result_lda_u = pd.read_pickle(name_model_pickle_lda_u)\n",
    "result_lda_b = pd.read_pickle(name_model_pickle_lda_b)\n",
    "\n",
    "voca = v.Vocabulary()\n",
    "docs = voca.read_corpus( data['text'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "_, _, top_lda_r = ar.prepare_data_distances(result_lda_r, num_words_to_compare, voca)\n",
    "_, _, top_lda_b = ar.prepare_data_distances(result_lda_b, num_words_to_compare, voca)\n",
    "_, _, top_lda_u = ar.prepare_data_distances(result_lda_u, num_words_to_compare, voca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>recommended</td>\n",
       "      <td>great</td>\n",
       "      <td>chair</td>\n",
       "      <td>couches</td>\n",
       "      <td>def</td>\n",
       "      <td>months</td>\n",
       "      <td>ignored</td>\n",
       "      <td>needed</td>\n",
       "      <td>nice</td>\n",
       "      <td>text</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>somewhat</td>\n",
       "      <td>mouth</td>\n",
       "      <td>25</td>\n",
       "      <td>alfresco</td>\n",
       "      <td>boy</td>\n",
       "      <td>japanese</td>\n",
       "      <td>vowed</td>\n",
       "      <td>rabe</td>\n",
       "      <td>indicating</td>\n",
       "      <td>takes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>time</td>\n",
       "      <td>would</td>\n",
       "      <td>months</td>\n",
       "      <td>specific</td>\n",
       "      <td>def</td>\n",
       "      <td>service</td>\n",
       "      <td>noticed</td>\n",
       "      <td>chair</td>\n",
       "      <td>metal</td>\n",
       "      <td>even</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>batter</td>\n",
       "      <td>couches</td>\n",
       "      <td>recommended</td>\n",
       "      <td>chair</td>\n",
       "      <td>service</td>\n",
       "      <td>great</td>\n",
       "      <td>def</td>\n",
       "      <td>ignored</td>\n",
       "      <td>mercedes</td>\n",
       "      <td>brecksville</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>gab</td>\n",
       "      <td>shops</td>\n",
       "      <td>massieve</td>\n",
       "      <td>breakfast</td>\n",
       "      <td>realize</td>\n",
       "      <td>juicy</td>\n",
       "      <td>provided</td>\n",
       "      <td>humble</td>\n",
       "      <td>shawarma</td>\n",
       "      <td>dinning</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda      W_lda_0  W_lda_1      W_lda_2    W_lda_3  W_lda_4  \\\n",
       "0           0  recommended    great        chair    couches      def   \n",
       "1           1     somewhat    mouth           25   alfresco      boy   \n",
       "2           2         time    would       months   specific      def   \n",
       "3           3       batter  couches  recommended      chair  service   \n",
       "4           4          gab    shops     massieve  breakfast  realize   \n",
       "\n",
       "    W_lda_5   W_lda_6  W_lda_7     W_lda_8      W_lda_9  \n",
       "0    months   ignored   needed        nice         text  \n",
       "1  japanese     vowed     rabe  indicating        takes  \n",
       "2   service   noticed    chair       metal         even  \n",
       "3     great       def  ignored    mercedes  brecksville  \n",
       "4     juicy  provided   humble    shawarma      dinning  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_lda_r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>favorite</td>\n",
       "      <td>small</td>\n",
       "      <td>fixed</td>\n",
       "      <td>spent</td>\n",
       "      <td>green</td>\n",
       "      <td>consistency</td>\n",
       "      <td>seriously</td>\n",
       "      <td>obvious</td>\n",
       "      <td>walks</td>\n",
       "      <td>less</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>deposit</td>\n",
       "      <td>disappeared</td>\n",
       "      <td>shouts</td>\n",
       "      <td>serve</td>\n",
       "      <td>apologetic</td>\n",
       "      <td>selections</td>\n",
       "      <td>saving</td>\n",
       "      <td>coating</td>\n",
       "      <td>coconut</td>\n",
       "      <td>instantly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>spent</td>\n",
       "      <td>man</td>\n",
       "      <td>name</td>\n",
       "      <td>fixed</td>\n",
       "      <td>order</td>\n",
       "      <td>crinoline</td>\n",
       "      <td>preoccupied</td>\n",
       "      <td>1</td>\n",
       "      <td>coffee</td>\n",
       "      <td>personally</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>season</td>\n",
       "      <td>live</td>\n",
       "      <td>man</td>\n",
       "      <td>horror</td>\n",
       "      <td>sort</td>\n",
       "      <td>consistency</td>\n",
       "      <td>lately</td>\n",
       "      <td>taking</td>\n",
       "      <td>replace</td>\n",
       "      <td>saturday</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>fixed</td>\n",
       "      <td>spent</td>\n",
       "      <td>small</td>\n",
       "      <td>green</td>\n",
       "      <td>live</td>\n",
       "      <td>tables</td>\n",
       "      <td>eye</td>\n",
       "      <td>sort</td>\n",
       "      <td>appx</td>\n",
       "      <td>prices</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda   W_lda_0      W_lda_1 W_lda_2 W_lda_3     W_lda_4      W_lda_5  \\\n",
       "0           0  favorite        small   fixed   spent       green  consistency   \n",
       "1           1   deposit  disappeared  shouts   serve  apologetic   selections   \n",
       "2           2     spent          man    name   fixed       order    crinoline   \n",
       "3           3    season         live     man  horror        sort  consistency   \n",
       "4           4     fixed        spent   small   green        live       tables   \n",
       "\n",
       "       W_lda_6  W_lda_7  W_lda_8     W_lda_9  \n",
       "0    seriously  obvious    walks        less  \n",
       "1       saving  coating  coconut   instantly  \n",
       "2  preoccupied        1   coffee  personally  \n",
       "3       lately   taking  replace    saturday  \n",
       "4          eye     sort     appx      prices  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_lda_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>recommended</td>\n",
       "      <td>great</td>\n",
       "      <td>chair</td>\n",
       "      <td>couches</td>\n",
       "      <td>def</td>\n",
       "      <td>months</td>\n",
       "      <td>ignored</td>\n",
       "      <td>needed</td>\n",
       "      <td>nice</td>\n",
       "      <td>text</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>somewhat</td>\n",
       "      <td>mouth</td>\n",
       "      <td>25</td>\n",
       "      <td>alfresco</td>\n",
       "      <td>boy</td>\n",
       "      <td>japanese</td>\n",
       "      <td>vowed</td>\n",
       "      <td>rabe</td>\n",
       "      <td>indicating</td>\n",
       "      <td>takes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>time</td>\n",
       "      <td>would</td>\n",
       "      <td>months</td>\n",
       "      <td>specific</td>\n",
       "      <td>def</td>\n",
       "      <td>service</td>\n",
       "      <td>noticed</td>\n",
       "      <td>chair</td>\n",
       "      <td>metal</td>\n",
       "      <td>even</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>batter</td>\n",
       "      <td>couches</td>\n",
       "      <td>recommended</td>\n",
       "      <td>chair</td>\n",
       "      <td>service</td>\n",
       "      <td>great</td>\n",
       "      <td>def</td>\n",
       "      <td>ignored</td>\n",
       "      <td>mercedes</td>\n",
       "      <td>brecksville</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>gab</td>\n",
       "      <td>shops</td>\n",
       "      <td>massieve</td>\n",
       "      <td>breakfast</td>\n",
       "      <td>realize</td>\n",
       "      <td>juicy</td>\n",
       "      <td>provided</td>\n",
       "      <td>humble</td>\n",
       "      <td>shawarma</td>\n",
       "      <td>dinning</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda      W_lda_0  W_lda_1      W_lda_2    W_lda_3  W_lda_4  \\\n",
       "0           0  recommended    great        chair    couches      def   \n",
       "1           1     somewhat    mouth           25   alfresco      boy   \n",
       "2           2         time    would       months   specific      def   \n",
       "3           3       batter  couches  recommended      chair  service   \n",
       "4           4          gab    shops     massieve  breakfast  realize   \n",
       "\n",
       "    W_lda_5   W_lda_6  W_lda_7     W_lda_8      W_lda_9  \n",
       "0    months   ignored   needed        nice         text  \n",
       "1  japanese     vowed     rabe  indicating        takes  \n",
       "2   service   noticed    chair       metal         even  \n",
       "3     great       def  ignored    mercedes  brecksville  \n",
       "4     juicy  provided   humble    shawarma      dinning  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_lda_u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Comparing model 1, topic', 0)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0            (0, 1.0)  (0, 0.976347927246)  (0, 0.303048795305)\n",
      "1  (1, 2.00024500728)   (1, 1.99697298928)   (1, 1.99950223727)\n",
      "2  (2, 3.00000324982)   (4, 1.99932035104)   (4, 1.99992902993)\n",
      "3  (3, 4.00000625541)   (2, 1.99943546423)   (3, 1.99993180527)\n",
      "4  (4, 5.00008591165)   (3, 1.99944424034)   (2, 1.99995340643)\n",
      "('Comparing model 1, topic', 1)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00048992456)  (0, 0.946335630253)  (0, 0.292124784402)\n",
      "1            (1, 2.0)   (1, 1.99409076607)   (1, 1.99884991606)\n",
      "2  (2, 3.00020734854)   (4, 1.99772309611)    (4, 1.9996083392)\n",
      "3  (3, 4.00007916877)   (2, 1.99794285332)   (3, 1.99960929121)\n",
      "4  (4, 5.00000650961)   (3, 1.99794299472)   (2, 1.99966614417)\n",
      "('Comparing model 1, topic', 2)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00000974943)  (0, 0.980143139828)  (0, 0.305192003854)\n",
      "1  (1, 2.00031100938)   (1, 1.99726458443)   (1, 1.99958538459)\n",
      "2            (2, 3.0)   (4, 1.99945119726)   (4, 1.99995920526)\n",
      "3  (3, 4.00001479892)   (2, 1.99955065487)   (3, 1.99996031508)\n",
      "4  (4, 5.00011056848)   (3, 1.99956234665)   (2, 1.99997572327)\n",
      "('Comparing model 1, topic', 3)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00002502135)  (0, 0.971061544035)  (0, 0.299371278541)\n",
      "1  (1, 2.00015833283)   (1, 1.99650035564)   (1, 1.99930574463)\n",
      "2  (2, 3.00001973187)   (4, 1.99909929059)   (4, 1.99984934022)\n",
      "3            (3, 4.0)   (2, 1.99923507158)   (3, 1.99985006332)\n",
      "4  (4, 5.00005908676)   (3, 1.99923582444)   (2, 1.99988431335)\n",
      "('Comparing model 1, topic', 4)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00042946973)  (0, 0.947229736898)  (0, 0.295743546517)\n",
      "1  (1, 2.00001627398)    (1, 1.9942858108)   (1, 1.99912929329)\n",
      "2  (2, 3.00018427718)   (4, 1.99781673514)   (4, 1.99975016469)\n",
      "3   (3, 4.0000738582)    (2, 1.9980304895)   (3, 1.99975717888)\n",
      "4            (4, 5.0)   (3, 1.99804451105)    (2, 1.9997995415)\n",
      "('Comparing model 2, topic', 0)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0            (0, 1.0)  (0, 0.976347927246)  (0, 0.303048795305)\n",
      "1  (1, 2.00024500728)   (1, 1.99697298928)   (1, 1.99950223727)\n",
      "2  (2, 3.00000324982)   (4, 1.99932035104)   (4, 1.99992902993)\n",
      "3  (3, 4.00000625541)   (2, 1.99943546423)   (3, 1.99993180527)\n",
      "4  (4, 5.00008591165)   (3, 1.99944424034)   (2, 1.99995340643)\n",
      "('Comparing model 2, topic', 1)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00048992456)  (0, 0.946335630253)  (0, 0.292124784402)\n",
      "1            (1, 2.0)   (1, 1.99409076607)   (1, 1.99884991606)\n",
      "2  (2, 3.00020734854)   (4, 1.99772309611)    (4, 1.9996083392)\n",
      "3  (3, 4.00007916877)   (2, 1.99794285332)   (3, 1.99960929121)\n",
      "4  (4, 5.00000650961)   (3, 1.99794299472)   (2, 1.99966614417)\n",
      "('Comparing model 2, topic', 2)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00000974943)  (0, 0.980143139828)  (0, 0.305192003854)\n",
      "1  (1, 2.00031100938)   (1, 1.99726458443)   (1, 1.99958538459)\n",
      "2            (2, 3.0)   (4, 1.99945119726)   (4, 1.99995920526)\n",
      "3  (3, 4.00001479892)   (2, 1.99955065487)   (3, 1.99996031508)\n",
      "4  (4, 5.00011056848)   (3, 1.99956234665)   (2, 1.99997572327)\n",
      "('Comparing model 2, topic', 3)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00002502135)  (0, 0.971061544035)  (0, 0.299371278541)\n",
      "1  (1, 2.00015833283)   (1, 1.99650035564)   (1, 1.99930574463)\n",
      "2  (2, 3.00001973187)   (4, 1.99909929059)   (4, 1.99984934022)\n",
      "3            (3, 4.0)   (2, 1.99923507158)   (3, 1.99985006332)\n",
      "4  (4, 5.00005908676)   (3, 1.99923582444)   (2, 1.99988431335)\n",
      "('Comparing model 2, topic', 4)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00042946973)  (0, 0.947229736898)  (0, 0.295743546517)\n",
      "1  (1, 2.00001627398)    (1, 1.9942858108)   (1, 1.99912929329)\n",
      "2  (2, 3.00018427718)   (4, 1.99781673514)   (4, 1.99975016469)\n",
      "3   (3, 4.0000738582)    (2, 1.9980304895)   (3, 1.99975717888)\n",
      "4            (4, 5.0)   (3, 1.99804451105)    (2, 1.9997995415)\n"
     ]
    }
   ],
   "source": [
    "ar.get_matrix_distances(result_lda_r,result_lda_u, num_words_to_compare, voca, distances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "get_matrix_distances(result_lda_r,result_lda_b )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "name_model_pickle_dmr_b = \"../models/dmr5k_dmr_biz.p\"\n",
    "name_model_pickle_dmr_y = \"../models/dmr5k_dmr_y.p\"\n",
    "\n",
    "result_dmr_y = pd.read_pickle(name_model_pickle_dmr_y)\n",
    "result_dmr_b = pd.read_pickle(name_model_pickle_dmr_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "_, _, top_dmr_b = ar.prepare_data_distances(result_dmr_b, num_words_to_compare, voca)\n",
    "_, _, top_dmr_y = ar.prepare_data_distances(result_dmr_y, num_words_to_compare, voca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>recommended</td>\n",
       "      <td>def</td>\n",
       "      <td>months</td>\n",
       "      <td>couches</td>\n",
       "      <td>would</td>\n",
       "      <td>time</td>\n",
       "      <td>great</td>\n",
       "      <td>ignored</td>\n",
       "      <td>needed</td>\n",
       "      <td>far</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>recommended</td>\n",
       "      <td>couches</td>\n",
       "      <td>chair</td>\n",
       "      <td>batter</td>\n",
       "      <td>great</td>\n",
       "      <td>months</td>\n",
       "      <td>def</td>\n",
       "      <td>service</td>\n",
       "      <td>time</td>\n",
       "      <td>needed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>chair</td>\n",
       "      <td>specific</td>\n",
       "      <td>batter</td>\n",
       "      <td>wanted</td>\n",
       "      <td>going</td>\n",
       "      <td>service</td>\n",
       "      <td>signs</td>\n",
       "      <td>anniversary</td>\n",
       "      <td>cliche</td>\n",
       "      <td>sold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>tranche</td>\n",
       "      <td>bekanntenkreis</td>\n",
       "      <td>batch</td>\n",
       "      <td>reviewing</td>\n",
       "      <td>fuit</td>\n",
       "      <td>disinfect</td>\n",
       "      <td>budweiser</td>\n",
       "      <td>hochpreisig</td>\n",
       "      <td>centaine</td>\n",
       "      <td>worried</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>25</td>\n",
       "      <td>somewhat</td>\n",
       "      <td>mouth</td>\n",
       "      <td>boy</td>\n",
       "      <td>japanese</td>\n",
       "      <td>boast</td>\n",
       "      <td>television</td>\n",
       "      <td>disfigured</td>\n",
       "      <td>indicating</td>\n",
       "      <td>hamburger</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda      W_lda_0         W_lda_1 W_lda_2    W_lda_3   W_lda_4  \\\n",
       "0           0  recommended             def  months    couches     would   \n",
       "1           1  recommended         couches   chair     batter     great   \n",
       "2           2        chair        specific  batter     wanted     going   \n",
       "3           3      tranche  bekanntenkreis   batch  reviewing      fuit   \n",
       "4           4           25        somewhat   mouth        boy  japanese   \n",
       "\n",
       "     W_lda_5     W_lda_6      W_lda_7     W_lda_8    W_lda_9  \n",
       "0       time       great      ignored      needed        far  \n",
       "1     months         def      service        time     needed  \n",
       "2    service       signs  anniversary      cliche       sold  \n",
       "3  disinfect   budweiser  hochpreisig    centaine    worried  \n",
       "4      boast  television   disfigured  indicating  hamburger  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_dmr_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cluster_lda</th>\n",
       "      <th>W_lda_0</th>\n",
       "      <th>W_lda_1</th>\n",
       "      <th>W_lda_2</th>\n",
       "      <th>W_lda_3</th>\n",
       "      <th>W_lda_4</th>\n",
       "      <th>W_lda_5</th>\n",
       "      <th>W_lda_6</th>\n",
       "      <th>W_lda_7</th>\n",
       "      <th>W_lda_8</th>\n",
       "      <th>W_lda_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>would</td>\n",
       "      <td>def</td>\n",
       "      <td>specific</td>\n",
       "      <td>time</td>\n",
       "      <td>months</td>\n",
       "      <td>noticed</td>\n",
       "      <td>service</td>\n",
       "      <td>selection</td>\n",
       "      <td>another</td>\n",
       "      <td>metal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>recommended</td>\n",
       "      <td>chair</td>\n",
       "      <td>couches</td>\n",
       "      <td>great</td>\n",
       "      <td>def</td>\n",
       "      <td>months</td>\n",
       "      <td>ignored</td>\n",
       "      <td>however</td>\n",
       "      <td>needed</td>\n",
       "      <td>text</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>great</td>\n",
       "      <td>time</td>\n",
       "      <td>back</td>\n",
       "      <td>texting</td>\n",
       "      <td>months</td>\n",
       "      <td>mom</td>\n",
       "      <td>something</td>\n",
       "      <td>recommend</td>\n",
       "      <td>needed</td>\n",
       "      <td>best</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>batter</td>\n",
       "      <td>couches</td>\n",
       "      <td>recommended</td>\n",
       "      <td>chair</td>\n",
       "      <td>ignored</td>\n",
       "      <td>great</td>\n",
       "      <td>service</td>\n",
       "      <td>mercedes</td>\n",
       "      <td>steamed</td>\n",
       "      <td>brecksville</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>somewhat</td>\n",
       "      <td>mouth</td>\n",
       "      <td>25</td>\n",
       "      <td>alfresco</td>\n",
       "      <td>boy</td>\n",
       "      <td>japanese</td>\n",
       "      <td>gab</td>\n",
       "      <td>rabe</td>\n",
       "      <td>vowed</td>\n",
       "      <td>gone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Cluster_lda      W_lda_0  W_lda_1      W_lda_2   W_lda_3  W_lda_4   W_lda_5  \\\n",
       "0           0        would      def     specific      time   months   noticed   \n",
       "1           1  recommended    chair      couches     great      def    months   \n",
       "2           2        great     time         back   texting   months       mom   \n",
       "3           3       batter  couches  recommended     chair  ignored     great   \n",
       "4           4     somewhat    mouth           25  alfresco      boy  japanese   \n",
       "\n",
       "     W_lda_6    W_lda_7  W_lda_8      W_lda_9  \n",
       "0    service  selection  another        metal  \n",
       "1    ignored    however   needed         text  \n",
       "2  something  recommend   needed         best  \n",
       "3    service   mercedes  steamed  brecksville  \n",
       "4        gab       rabe    vowed         gone  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_dmr_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Comparing model 1, topic', 0)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00000206462)  (0, 0.975652082674)  (0, 0.185506228883)\n",
      "1   (1, 2.0000015176)   (1, 1.99890112852)   (1, 1.99996362448)\n",
      "2  (2, 3.00001281801)   (4, 1.99915921099)   (3, 1.99996629557)\n",
      "3  (3, 4.00001896922)   (2, 1.99944656355)   (4, 1.99996840763)\n",
      "4  (4, 5.00018102043)   (3, 1.99950143209)   (2, 1.99997167689)\n",
      "('Comparing model 1, topic', 1)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00001264177)  (0, 0.975124691589)  (0, 0.183314141121)\n",
      "1  (1, 2.00000680426)   (1, 1.99886624585)    (3, 1.9999076219)\n",
      "2  (2, 3.00000910515)     (4, 1.999121855)   (1, 1.99991299951)\n",
      "3  (3, 4.00000701191)   (2, 1.99940690055)   (4, 1.99991672582)\n",
      "4  (4, 5.00017385493)   (3, 1.99945518802)   (2, 1.99991967168)\n",
      "('Comparing model 1, topic', 2)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00000920543)  (0, 0.975316160862)  (0, 0.184505801666)\n",
      "1  (1, 2.00000467325)   (1, 1.99887767984)    (3, 1.9999298901)\n",
      "2  (2, 3.00000957896)   (4, 1.99913377164)   (1, 1.99993717281)\n",
      "3  (3, 4.00000919333)   (2, 1.99941938395)   (4, 1.99993990666)\n",
      "4  (4, 5.00017554316)   (3, 1.99946856007)   (2, 1.99994145493)\n",
      "('Comparing model 1, topic', 3)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00009270593)  (0, 0.968261277602)  (0, 0.180146159988)\n",
      "1  (1, 2.00004693437)   (1, 1.99847012983)   (3, 1.99979035789)\n",
      "2  (2, 3.00003885011)   (4, 1.99876404741)   (1, 1.99979419514)\n",
      "3  (3, 4.00001755439)   (2, 1.99909418205)   (4, 1.99980226973)\n",
      "4   (4, 5.0001256926)   (3, 1.99915164188)   (2, 1.99980449408)\n",
      "('Comparing model 1, topic', 4)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine         correlation\n",
      "0  (0, 1.00023841915)  (0, 0.958067853687)  (0, 0.17862478648)\n",
      "1  (1, 2.00012261681)   (1, 1.99786057496)  (3, 1.99974043968)\n",
      "2  (2, 3.00010544375)   (4, 1.99821414118)  (1, 1.99974127186)\n",
      "3  (3, 4.00005808344)   (2, 1.99862333262)  (4, 1.99975129113)\n",
      "4  (4, 5.00006594414)   (3, 1.99869843431)   (2, 1.9997556904)\n",
      "('Comparing model 2, topic', 0)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00000206462)  (0, 0.977001474458)  (0, 0.113483015797)\n",
      "1  (1, 2.00000632091)   (1, 1.99884083474)   (1, 1.99990685219)\n",
      "2  (2, 3.00000306849)   (2, 1.99937177719)   (3, 1.99994987087)\n",
      "3  (3, 4.00002317749)   (3, 1.99942838595)   (4, 1.99995572665)\n",
      "4  (4, 5.00004768929)   (4, 1.99943386824)    (2, 1.9999662302)\n",
      "('Comparing model 2, topic', 1)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00000303519)  (0, 0.977542206851)  (0, 0.113786395704)\n",
      "1  (1, 2.00000680426)   (1, 1.99886624585)   (1, 1.99991205589)\n",
      "2   (2, 3.0000031155)   (2, 1.99939018602)   (3, 1.99995233259)\n",
      "3  (3, 4.00002346739)   (3, 1.99944507276)   (4, 1.99995851149)\n",
      "4  (4, 5.00004904799)   (4, 1.99945084721)   (2, 1.99996953479)\n",
      "('Comparing model 2, topic', 2)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00003845338)  (0, 0.982482388261)  (0, 0.112882070758)\n",
      "1   (1, 2.0000136577)   (1, 1.99905020203)   (1, 1.99986207165)\n",
      "2  (2, 3.00000957896)   (2, 1.99952760393)   (3, 1.99991459898)\n",
      "3  (3, 4.00002913767)   (3, 1.99956616296)    (4, 1.9999242294)\n",
      "4  (4, 5.00006326696)   (4, 1.99957504971)   (2, 1.99993731042)\n",
      "('Comparing model 2, topic', 3)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00007587418)  (0, 0.979204744161)  (0, 0.111054170469)\n",
      "1  (1, 2.00001402378)    (1, 1.9988162028)   (1, 1.99973012879)\n",
      "2  (2, 3.00001225777)   (2, 1.99935552733)   (3, 1.99980972685)\n",
      "3  (3, 4.00001755439)   (3, 1.99939419939)   (4, 1.99982397912)\n",
      "4  (4, 5.00004646687)   (4, 1.99940731815)   (2, 1.99983860029)\n",
      "('Comparing model 2, topic', 4)\n",
      "5 ' Clusters that are closer to topic = -1\n",
      "Format: (cluster number, similarity measure)\n",
      "            euclidean               cosine          correlation\n",
      "0  (0, 1.00090470928)  (0, 0.933877333872)  (0, 0.109064292732)\n",
      "1  (1, 2.00043459766)   (1, 1.99579144217)   (1, 1.99968871444)\n",
      "2  (2, 3.00029256281)   (2, 1.99687688593)   (3, 1.99977314956)\n",
      "3  (3, 4.00015711464)   (3, 1.99700402735)    (4, 1.9997868129)\n",
      "4  (4, 5.00006594414)   (4, 1.99701695692)   (2, 1.99981041726)\n"
     ]
    }
   ],
   "source": [
    "ar.get_matrix_distances(result_dmr_y,result_dmr_b,num_words_to_compare, voca, distances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dist = ar.get_spearman_matrix(result_dmr_b, result_dmr_y,voca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.024462</td>\n",
       "      <td>0.078902</td>\n",
       "      <td>0.04829</td>\n",
       "      <td>0.046117</td>\n",
       "      <td>0.053551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1        2         3         4\n",
       "0  0.024462  0.078902  0.04829  0.046117  0.053551\n",
       "1  0.000000  0.000000  0.00000  0.000000  0.000000\n",
       "2  0.000000  0.000000  0.00000  0.000000  0.000000\n",
       "3  0.000000  0.000000  0.00000  0.000000  0.000000\n",
       "4  0.000000  0.000000  0.00000  0.000000  0.000000"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist = pd.DataFrame(dist)\n",
    "dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004338</td>\n",
       "      <td>0.037401</td>\n",
       "      <td>0.009698</td>\n",
       "      <td>-0.001321</td>\n",
       "      <td>0.053551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4\n",
       "0  0.004338  0.037401  0.009698 -0.001321  0.053551\n",
       "1  0.000000  0.000000  0.000000  0.000000  0.000000\n",
       "2  0.000000  0.000000  0.000000  0.000000  0.000000\n",
       "3  0.000000  0.000000  0.000000  0.000000  0.000000\n",
       "4  0.000000  0.000000  0.000000  0.000000  0.000000"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist = pd.DataFrame(dist)\n",
    "dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}