KendallsTau_exclusiveBERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a8b8f532",
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats\n",
    "x1 = [12, 2, 1, 12, 2]\n",
    "x2 = [1, 4, 7, 1, 0]\n",
    "tau, p_value = stats.kendalltau(x1, x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7ab67d7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4666666666666666\n"
     ]
    }
   ],
   "source": [
    "x1 = [1,2,3,4,5,6]\n",
    "x2 = [3,1,4,2,6,5]\n",
    "tau, p_value = stats.kendalltau(x1, x2)\n",
    "print(tau)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6fd35ddc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.9999999999999999\n"
     ]
    }
   ],
   "source": [
    "x1 = [-0.2, -0.3, -0.4, -0.5, -0.6, -0.7]\n",
    "x2 = [0.3,2,3,4,5,6]\n",
    "tau, p_value = stats.kendalltau(x1, x2)\n",
    "print(tau)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "21e1b6ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import scipy.stats as stats\n",
    "import numpy as np\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "23c16ffe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(df):\n",
    "    trustworthiness = df['Trustworthiness'].tolist()\n",
    "    continuity = df['Continuity'].tolist()\n",
    "    shephard_diagram_correlation = df['Shephard Diagram Correlation'].tolist()\n",
    "    \n",
    "    result = []\n",
    "    for i in range(len(trustworthiness)):\n",
    "        result.append(round(trustworthiness[i]/4 + continuity[i]/4 + 0.5*(shephard_diagram_correlation[i]+1)/2,2))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "921f5603",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_perception(df):\n",
    "    neighborhood_hit = df['7-Neighborhood Hit'].tolist()\n",
    "    distance_consistency = df['Distance consistency'].tolist()\n",
    "\n",
    "    result = []\n",
    "    for i in range(len(neighborhood_hit)):\n",
    "        result.append(round((neighborhood_hit[i] + distance_consistency[i])/2,2))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b6e6a583",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_coherence_v(df):\n",
    "    coherence_c_v = df['coherence_c_v'].tolist()\n",
    "    return coherence_c_v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "87a6c388-4f12-4761-b054-a20d446f756d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_coherence_u(df):\n",
    "    coherence_u_mass = df['coherence_u_mass'].tolist()\n",
    "    return coherence_u_mass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a302f4e4-1db8-4da1-be60-e1d516104cc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_coherence_npmi(df):\n",
    "    coherence_c_npmi = df['coherence_c_npmi'].tolist()\n",
    "    return coherence_c_npmi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3930dd6b-8f08-4da8-a828-eea4a2a3be6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_coherence_uci(df):\n",
    "    coherence_c_uci = df['coherence_c_uci'].tolist()\n",
    "    return coherence_c_uci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bd9bd24d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def n_topics_factor(df):\n",
    "    n_topics_list = df[\"n_topics\"].tolist()\n",
    "    n_topics_sorted_values = sorted(list(set(n_topics_list)))\n",
    "    a_value = n_topics_sorted_values[0]\n",
    "    b_value = n_topics_sorted_values[1]\n",
    "    c_value = n_topics_sorted_values[2]\n",
    "    d_value = n_topics_sorted_values[3]\n",
    "    e_value = n_topics_sorted_values[4]\n",
    "    \n",
    "    n_topics_factor = []\n",
    "    for K in n_topics_list:\n",
    "        if K == a_value:\n",
    "            n_topics_factor.append(\"a\")\n",
    "        if K == b_value:\n",
    "            n_topics_factor.append(\"b\")\n",
    "        if K == c_value:\n",
    "            n_topics_factor.append(\"c\")\n",
    "        if K == d_value:\n",
    "            n_topics_factor.append(\"d\")\n",
    "        if K == e_value:\n",
    "            n_topics_factor.append(\"e\")\n",
    "    \n",
    "    return n_topics_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "758b2a3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_coherence_v(df_layout, df_models):\n",
    "    list_parameter = df_layout['Complete List of Hyperparameters'].tolist()\n",
    "    list_TM = df_layout['TM'].tolist()\n",
    "    list_DR = df_layout['DR'].tolist()\n",
    "    list_experiment = df_layout[\"Experiment\"].tolist()\n",
    "    \n",
    "    result = []\n",
    "    for i in range(len(list_parameter)):\n",
    "        parameters = ast.literal_eval(list_parameter[i]) # is a dict\n",
    "        if (list_TM[i] == 'lda') or (list_TM[i] == 'lda_linear_combined'):\n",
    "            TM = 'lda'\n",
    "            n_topics = parameters[TM]['n_topics']\n",
    "            alpha = parameters[TM]['alpha']\n",
    "\n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics) &\n",
    "                                  (df_models['alpha_lda'] == alpha)]['coherence_v'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        else:\n",
    "            TM = list_TM[i].replace('_linear_combined', '')\n",
    "            experiment = list_experiment[i]\n",
    "            n_topics = int(experiment.split(\"n_topics\")[1].split(\"_\")[1])\n",
    "            \n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics)]['coherence_v'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "85b522b2-d651-4eb0-99e4-dd5666e2457e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_coherence_u(df_layout, df_models):\n",
    "    list_parameter = df_layout['Complete List of Hyperparameters'].tolist()\n",
    "    list_TM = df_layout['TM'].tolist()\n",
    "    list_DR = df_layout['DR'].tolist()\n",
    "    list_experiment = df_layout[\"Experiment\"].tolist()\n",
    "    \n",
    "    result = []\n",
    "    for i in range(len(list_parameter)):\n",
    "        parameters = ast.literal_eval(list_parameter[i]) # is a dict\n",
    "        if (list_TM[i] == 'lda') or (list_TM[i] == 'lda_linear_combined'):\n",
    "            TM = 'lda'\n",
    "            n_topics = parameters[TM]['n_topics']\n",
    "            alpha = parameters[TM]['alpha']\n",
    "\n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics) &\n",
    "                                  (df_models['alpha_lda'] == alpha)]['coherence_u'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        else:\n",
    "            TM = list_TM[i].replace('_linear_combined', '')\n",
    "            experiment = list_experiment[i]\n",
    "            n_topics = int(experiment.split(\"n_topics\")[1].split(\"_\")[1])\n",
    "            \n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics)]['coherence_u'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "06774f49-b780-4b1a-a008-2489cfc4ee74",
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_coherence_npmi(df_layout, df_models):\n",
    "    list_parameter = df_layout['Complete List of Hyperparameters'].tolist()\n",
    "    list_TM = df_layout['TM'].tolist()\n",
    "    list_DR = df_layout['DR'].tolist()\n",
    "    list_experiment = df_layout[\"Experiment\"].tolist()\n",
    "    \n",
    "    result = []\n",
    "    for i in range(len(list_parameter)):\n",
    "        parameters = ast.literal_eval(list_parameter[i]) # is a dict\n",
    "        if (list_TM[i] == 'lda') or (list_TM[i] == 'lda_linear_combined'):\n",
    "            TM = 'lda'\n",
    "            n_topics = parameters[TM]['n_topics']\n",
    "            alpha = parameters[TM]['alpha']\n",
    "\n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics) &\n",
    "                                  (df_models['alpha_lda'] == alpha)]['coherence_npmi'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        else:\n",
    "            TM = list_TM[i].replace('_linear_combined', '')\n",
    "            experiment = list_experiment[i]\n",
    "            n_topics = int(experiment.split(\"n_topics\")[1].split(\"_\")[1])\n",
    "            \n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics)]['coherence_npmi'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "17e338cc-5602-4a2a-97cc-dec760035062",
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_coherence_uci(df_layout, df_models):\n",
    "    list_parameter = df_layout['Complete List of Hyperparameters'].tolist()\n",
    "    list_TM = df_layout['TM'].tolist()\n",
    "    list_DR = df_layout['DR'].tolist()\n",
    "    list_experiment = df_layout[\"Experiment\"].tolist()\n",
    "    \n",
    "    result = []\n",
    "    for i in range(len(list_parameter)):\n",
    "        parameters = ast.literal_eval(list_parameter[i]) # is a dict\n",
    "        if (list_TM[i] == 'lda') or (list_TM[i] == 'lda_linear_combined'):\n",
    "            TM = 'lda'\n",
    "            n_topics = parameters[TM]['n_topics']\n",
    "            alpha = parameters[TM]['alpha']\n",
    "\n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics) &\n",
    "                                  (df_models['alpha_lda'] == alpha)]['coherence_uci'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        else:\n",
    "            TM = list_TM[i].replace('_linear_combined', '')\n",
    "            experiment = list_experiment[i]\n",
    "            n_topics = int(experiment.split(\"n_topics\")[1].split(\"_\")[1])\n",
    "            \n",
    "            coherence = df_models[(df_models['model_type'] == TM) & (df_models['n_topics'] == n_topics)]['coherence_uci'].tolist()[0]\n",
    "            result.append(coherence)\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "cffcc59a-3e06-4a81-a7ca-ae24e56b8ef6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment</th>\n",
       "      <th>Trustworthiness</th>\n",
       "      <th>Continuity</th>\n",
       "      <th>Shephard Diagram Correlation</th>\n",
       "      <th>Normalized Stress</th>\n",
       "      <th>7-Neighborhood Hit</th>\n",
       "      <th>Calinski-Harabasz-Index</th>\n",
       "      <th>Silhouette coefficient</th>\n",
       "      <th>Davies-Bouldin-Index</th>\n",
       "      <th>SDBW validity index</th>\n",
       "      <th>Distance consistency</th>\n",
       "      <th>Complete List of Hyperparameters</th>\n",
       "      <th>DR</th>\n",
       "      <th>TM</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>perception</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1110</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.675698</td>\n",
       "      <td>0.633057</td>\n",
       "      <td>0.057643</td>\n",
       "      <td>45.592515</td>\n",
       "      <td>0.224273</td>\n",
       "      <td>59.170896</td>\n",
       "      <td>-0.349153</td>\n",
       "      <td>21.026834</td>\n",
       "      <td>1.105771</td>\n",
       "      <td>0.116758</td>\n",
       "      <td>{'som': {'n': 10, 'm': 30}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1111</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.739256</td>\n",
       "      <td>0.652365</td>\n",
       "      <td>0.038065</td>\n",
       "      <td>21.428914</td>\n",
       "      <td>0.231167</td>\n",
       "      <td>79.262727</td>\n",
       "      <td>-0.223375</td>\n",
       "      <td>23.159192</td>\n",
       "      <td>1.181976</td>\n",
       "      <td>0.112957</td>\n",
       "      <td>{'som': {'n': 15, 'm': 15}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.451663</td>\n",
       "      <td>0.530450</td>\n",
       "      <td>-0.035653</td>\n",
       "      <td>2.822163</td>\n",
       "      <td>0.084964</td>\n",
       "      <td>52.412127</td>\n",
       "      <td>-0.584931</td>\n",
       "      <td>27.484825</td>\n",
       "      <td>2.669238</td>\n",
       "      <td>0.071151</td>\n",
       "      <td>{'som': {'n': 25, 'm': 5}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1113</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.841250</td>\n",
       "      <td>0.688824</td>\n",
       "      <td>0.088636</td>\n",
       "      <td>81.198701</td>\n",
       "      <td>0.291017</td>\n",
       "      <td>62.116272</td>\n",
       "      <td>-0.216050</td>\n",
       "      <td>32.599910</td>\n",
       "      <td>1.454020</td>\n",
       "      <td>0.103147</td>\n",
       "      <td>{'som': {'n': 25, 'm': 20}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1114</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.822634</td>\n",
       "      <td>0.703322</td>\n",
       "      <td>0.095417</td>\n",
       "      <td>168.937709</td>\n",
       "      <td>0.289944</td>\n",
       "      <td>91.818637</td>\n",
       "      <td>-0.251051</td>\n",
       "      <td>27.098385</td>\n",
       "      <td>1.195803</td>\n",
       "      <td>0.096429</td>\n",
       "      <td>{'som': {'n': 15, 'm': 30}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1115</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.856899</td>\n",
       "      <td>0.712534</td>\n",
       "      <td>0.086700</td>\n",
       "      <td>147.692570</td>\n",
       "      <td>0.338178</td>\n",
       "      <td>88.774212</td>\n",
       "      <td>-0.190497</td>\n",
       "      <td>29.498811</td>\n",
       "      <td>1.127497</td>\n",
       "      <td>0.118791</td>\n",
       "      <td>{'som': {'n': 25, 'm': 25}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1116</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.725881</td>\n",
       "      <td>0.630795</td>\n",
       "      <td>0.064246</td>\n",
       "      <td>29.646196</td>\n",
       "      <td>0.208010</td>\n",
       "      <td>82.077472</td>\n",
       "      <td>-0.276572</td>\n",
       "      <td>27.206686</td>\n",
       "      <td>2.405901</td>\n",
       "      <td>0.121619</td>\n",
       "      <td>{'som': {'n': 10, 'm': 25}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1117</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.831014</td>\n",
       "      <td>0.693494</td>\n",
       "      <td>0.081198</td>\n",
       "      <td>95.075178</td>\n",
       "      <td>0.282027</td>\n",
       "      <td>118.388797</td>\n",
       "      <td>-0.219333</td>\n",
       "      <td>27.170243</td>\n",
       "      <td>1.106174</td>\n",
       "      <td>0.109687</td>\n",
       "      <td>{'som': {'n': 15, 'm': 25}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1118</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.842736</td>\n",
       "      <td>0.700860</td>\n",
       "      <td>0.106360</td>\n",
       "      <td>148.751285</td>\n",
       "      <td>0.314238</td>\n",
       "      <td>83.325297</td>\n",
       "      <td>-0.242774</td>\n",
       "      <td>22.136439</td>\n",
       "      <td>1.186895</td>\n",
       "      <td>0.126569</td>\n",
       "      <td>{'som': {'n': 30, 'm': 20}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1119</th>\n",
       "      <td>20_newsgroups_bow_som__10_0_0_0_0_0</td>\n",
       "      <td>0.568791</td>\n",
       "      <td>0.557862</td>\n",
       "      <td>0.016434</td>\n",
       "      <td>5.728002</td>\n",
       "      <td>0.131506</td>\n",
       "      <td>89.021928</td>\n",
       "      <td>-0.418337</td>\n",
       "      <td>17.090808</td>\n",
       "      <td>2.237856</td>\n",
       "      <td>0.082553</td>\n",
       "      <td>{'som': {'n': 5, 'm': 20}}</td>\n",
       "      <td>som</td>\n",
       "      <td>bow</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Experiment  Trustworthiness  Continuity  \\\n",
       "1110  20_newsgroups_bow_som__10_0_0_0_0_0         0.675698    0.633057   \n",
       "1111  20_newsgroups_bow_som__10_0_0_0_0_0         0.739256    0.652365   \n",
       "1112  20_newsgroups_bow_som__10_0_0_0_0_0         0.451663    0.530450   \n",
       "1113  20_newsgroups_bow_som__10_0_0_0_0_0         0.841250    0.688824   \n",
       "1114  20_newsgroups_bow_som__10_0_0_0_0_0         0.822634    0.703322   \n",
       "1115  20_newsgroups_bow_som__10_0_0_0_0_0         0.856899    0.712534   \n",
       "1116  20_newsgroups_bow_som__10_0_0_0_0_0         0.725881    0.630795   \n",
       "1117  20_newsgroups_bow_som__10_0_0_0_0_0         0.831014    0.693494   \n",
       "1118  20_newsgroups_bow_som__10_0_0_0_0_0         0.842736    0.700860   \n",
       "1119  20_newsgroups_bow_som__10_0_0_0_0_0         0.568791    0.557862   \n",
       "\n",
       "      Shephard Diagram Correlation  Normalized Stress  7-Neighborhood Hit  \\\n",
       "1110                      0.057643          45.592515            0.224273   \n",
       "1111                      0.038065          21.428914            0.231167   \n",
       "1112                     -0.035653           2.822163            0.084964   \n",
       "1113                      0.088636          81.198701            0.291017   \n",
       "1114                      0.095417         168.937709            0.289944   \n",
       "1115                      0.086700         147.692570            0.338178   \n",
       "1116                      0.064246          29.646196            0.208010   \n",
       "1117                      0.081198          95.075178            0.282027   \n",
       "1118                      0.106360         148.751285            0.314238   \n",
       "1119                      0.016434           5.728002            0.131506   \n",
       "\n",
       "      Calinski-Harabasz-Index  Silhouette coefficient  Davies-Bouldin-Index  \\\n",
       "1110                59.170896               -0.349153             21.026834   \n",
       "1111                79.262727               -0.223375             23.159192   \n",
       "1112                52.412127               -0.584931             27.484825   \n",
       "1113                62.116272               -0.216050             32.599910   \n",
       "1114                91.818637               -0.251051             27.098385   \n",
       "1115                88.774212               -0.190497             29.498811   \n",
       "1116                82.077472               -0.276572             27.206686   \n",
       "1117               118.388797               -0.219333             27.170243   \n",
       "1118                83.325297               -0.242774             22.136439   \n",
       "1119                89.021928               -0.418337             17.090808   \n",
       "\n",
       "      SDBW validity index  Distance consistency  \\\n",
       "1110             1.105771              0.116758   \n",
       "1111             1.181976              0.112957   \n",
       "1112             2.669238              0.071151   \n",
       "1113             1.454020              0.103147   \n",
       "1114             1.195803              0.096429   \n",
       "1115             1.127497              0.118791   \n",
       "1116             2.405901              0.121619   \n",
       "1117             1.106174              0.109687   \n",
       "1118             1.186895              0.126569   \n",
       "1119             2.237856              0.082553   \n",
       "\n",
       "     Complete List of Hyperparameters   DR   TM  accuracy  perception  \n",
       "1110      {'som': {'n': 10, 'm': 30}}  som  bow      0.59        0.17  \n",
       "1111      {'som': {'n': 15, 'm': 15}}  som  bow      0.61        0.17  \n",
       "1112       {'som': {'n': 25, 'm': 5}}  som  bow      0.49        0.08  \n",
       "1113      {'som': {'n': 25, 'm': 20}}  som  bow      0.65        0.20  \n",
       "1114      {'som': {'n': 15, 'm': 30}}  som  bow      0.66        0.19  \n",
       "1115      {'som': {'n': 25, 'm': 25}}  som  bow      0.66        0.23  \n",
       "1116      {'som': {'n': 10, 'm': 25}}  som  bow      0.61        0.16  \n",
       "1117      {'som': {'n': 15, 'm': 25}}  som  bow      0.65        0.20  \n",
       "1118      {'som': {'n': 30, 'm': 20}}  som  bow      0.66        0.22  \n",
       "1119       {'som': {'n': 5, 'm': 20}}  som  bow      0.54        0.11  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_20newsgroup[df_20newsgroup[\"TM\"] != \"bert\"].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f7788717",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Daniel Atzberger\\AppData\\Local\\Temp\\ipykernel_3520\\848114408.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_20newsgroup['accuracy'] = compute_accuracy(df_20newsgroup)\n",
      "C:\\Users\\Daniel Atzberger\\AppData\\Local\\Temp\\ipykernel_3520\\848114408.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_20newsgroup['perception'] = compute_perception(df_20newsgroup)\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[27], line 17\u001b[0m\n\u001b[0;32m     14\u001b[0m df_20newsgroup_models[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoherence_npmi\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m compute_coherence_npmi(df_20newsgroup_models)\n\u001b[0;32m     15\u001b[0m df_20newsgroup_models[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoherence_uci\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m compute_coherence_uci(df_20newsgroup_models)\n\u001b[1;32m---> 17\u001b[0m df_20newsgroup[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoherence_v\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m append_coherence_v(df_20newsgroup, df_20newsgroup_models)\n\u001b[0;32m     18\u001b[0m df_20newsgroup[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoherence_u\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m append_coherence_u(df_20newsgroup, df_20newsgroup_models)\n\u001b[0;32m     19\u001b[0m df_20newsgroup[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoherence_npmi\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m append_coherence_npmi(df_20newsgroup, df_20newsgroup_models)\n",
      "Cell \u001b[1;32mIn[15], line 21\u001b[0m, in \u001b[0;36mappend_coherence_v\u001b[1;34m(df_layout, df_models)\u001b[0m\n\u001b[0;32m     19\u001b[0m TM \u001b[38;5;241m=\u001b[39m list_TM[i]\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m_linear_combined\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m     20\u001b[0m experiment \u001b[38;5;241m=\u001b[39m list_experiment[i]\n\u001b[1;32m---> 21\u001b[0m n_topics \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(experiment\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mn_topics\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m.\u001b[39msplit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_\u001b[39m\u001b[38;5;124m\"\u001b[39m)[\u001b[38;5;241m1\u001b[39m])\n\u001b[0;32m     23\u001b[0m coherence \u001b[38;5;241m=\u001b[39m df_models[(df_models[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmodel_type\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m==\u001b[39m TM) \u001b[38;5;241m&\u001b[39m (df_models[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mn_topics\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m==\u001b[39m n_topics)][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcoherence_v\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mtolist()[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m     24\u001b[0m result\u001b[38;5;241m.\u001b[39mappend(coherence)\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# 20 newsgroup\n",
    "df_20newsgroup_with_BERT = pd.read_csv(\"results_cluster/cur_res/full_res_20_newsgroups.csv\")\n",
    "df_20newsgroup = df_20newsgroup_with_BERT[df_20newsgroup_with_BERT[\"TM\"] != \"bert\"]\n",
    "df_20newsgroup_models = pd.read_csv(\"model_evaluations/model_evaluations/20_newsgroups_model_evaluation.csv\")\n",
    "df_20newsgroup_models = df_20newsgroup_models.reset_index()  # make sure indexes pair with number of rows\n",
    "\n",
    "# append columns with aggregated quality metrics for layouts\n",
    "df_20newsgroup['accuracy'] = compute_accuracy(df_20newsgroup)\n",
    "df_20newsgroup['perception'] = compute_perception(df_20newsgroup)\n",
    "\n",
    "# append column with aggregated quality metric for coherence\n",
    "df_20newsgroup_models['coherence_v'] = compute_coherence_v(df_20newsgroup_models)\n",
    "df_20newsgroup_models['coherence_u'] = compute_coherence_u(df_20newsgroup_models)\n",
    "df_20newsgroup_models['coherence_npmi'] = compute_coherence_npmi(df_20newsgroup_models)\n",
    "df_20newsgroup_models['coherence_uci'] = compute_coherence_uci(df_20newsgroup_models)\n",
    "\n",
    "df_20newsgroup['coherence_v'] = append_coherence_v(df_20newsgroup, df_20newsgroup_models)\n",
    "df_20newsgroup['coherence_u'] = append_coherence_u(df_20newsgroup, df_20newsgroup_models)\n",
    "df_20newsgroup['coherence_npmi'] = append_coherence_npmi(df_20newsgroup, df_20newsgroup_models)\n",
    "df_20newsgroup['coherence_uci'] = append_coherence_uci(df_20newsgroup, df_20newsgroup_models)\n",
    "\n",
    "df_20newsgroup['corpus'] = \"20newsgroup\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46722c47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# emails\n",
    "df_emails = pd.read_csv(\"current_res/emails/full_res_emails.csv\")\n",
    "df_emails_models = pd.read_csv(\"current_res/emails/model_evaluation.csv\")\n",
    "df_emails_models = df_emails_models.reset_index()  # make sure indexes pair with number of rows\n",
    "\n",
    "# append columns with aggregated quality metrics for layouts\n",
    "df_emails['accuracy'] = compute_accuracy(df_emails)\n",
    "df_emails['perception'] = compute_perception(df_emails)\n",
    "\n",
    "# append column with aggregated quality metric for coherence\n",
    "df_emails_models['coherence'] = compute_coherence(df_emails_models)\n",
    "df_emails['coherence'] = append_coherence(df_emails, df_emails_models)\n",
    "df_emails['corpus'] = \"emails\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2aa1cb9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reuters\n",
    "df_reuters = pd.read_csv(\"current_res/reuters/full_res_reuters.csv\")\n",
    "df_reuters_models = pd.read_csv(\"current_res/reuters/model_evaluation.csv\")\n",
    "df_reuters_models = df_reuters_models.reset_index()  # make sure indexes pair with number of rows\n",
    "\n",
    "# append columns with aggregated quality metrics for layouts\n",
    "df_reuters['accuracy'] = compute_accuracy(df_reuters)\n",
    "df_reuters['perception'] = compute_perception(df_reuters)\n",
    "\n",
    "# append column with aggregated quality metric for coherence\n",
    "df_reuters_models['coherence'] = compute_coherence(df_reuters_models)\n",
    "df_reuters['coherence'] = append_coherence(df_reuters, df_reuters_models)\n",
    "df_reuters['corpus'] = \"reuters\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6c21648",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sevencategories\n",
    "df_seven_categories = pd.read_csv(\"current_res/seven_categories/full_res_seven_categories.csv\")\n",
    "df_seven_categories_models = pd.read_csv(\"current_res/seven_categories/model_evaluation.csv\")\n",
    "df_seven_categories_models = df_seven_categories_models.reset_index()  # make sure indexes pair with number of rows\n",
    "\n",
    "# append columns with aggregated quality metrics for layouts\n",
    "df_seven_categories['accuracy'] = compute_accuracy(df_seven_categories)\n",
    "df_seven_categories['perception'] = compute_perception(df_seven_categories)\n",
    "\n",
    "# append column with aggregated quality metric for coherence\n",
    "df_seven_categories_models['coherence'] = compute_coherence(df_seven_categories_models)\n",
    "df_seven_categories['coherence'] = append_coherence(df_seven_categories, df_seven_categories_models)\n",
    "df_seven_categories['corpus'] = \"7categories\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fb136f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = pd.concat([df_20newsgroup, df_emails, df_reuters, df_seven_categories])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c04383d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c297c7b2",
   "metadata": {},
   "source": [
    "### DR-TM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3523bb04",
   "metadata": {},
   "outputs": [],
   "source": [
    "DR_list = [\"mds\", \"som\", \"tsne\", \"umap\", \"total\"]\n",
    "TM_list = ['lda', 'nmf', 'lsi', 'total']\n",
    "\n",
    "result_TM_list = []\n",
    "result_DR_list = []\n",
    "result_acc_tau = []\n",
    "result_acc_p = []\n",
    "result_per_tau = []\n",
    "result_per_p = []\n",
    "\n",
    "for TM in TM_list:\n",
    "    for DR in DR_list:\n",
    "        \n",
    "        if DR == \"total\":\n",
    "            if TM == \"total\":\n",
    "                df_selected = df_total\n",
    "            elif TM == \"lda\":\n",
    "                df_selected = df_total[((df_total[\"TM\"] == \"lda\") | (df_total[\"TM\"] == \"lda_linear_combined\"))]\n",
    "            elif TM == \"nmf\":\n",
    "                df_selected = df_total[((df_total[\"TM\"] == \"nmf\") | (df_total[\"TM\"] == \"nmf_linear_combined\")\n",
    "                                           | (df_total[\"TM\"] == \"nmf_tfidf\") | (df_total[\"TM\"] == \"nmf_tfidf_linear_combined\"))]\n",
    "            else:\n",
    "                df_selected = df_total[ ((df_total[\"TM\"] == \"lsi\") | (df_total[\"TM\"] == \"lsi_linear_combined\")\n",
    "                                           | (df_total[\"TM\"] == \"lsi_tfidf\") | (df_total[\"TM\"] == \"lsi_tfidf_linear_combined\"))]\n",
    "        else:\n",
    "            if TM == \"total\":\n",
    "                df_selected = df_total[(df_total[\"DR\"] == DR)]\n",
    "            elif TM == \"lda\":\n",
    "                df_selected = df_total[(df_total[\"DR\"] == DR) &\n",
    "                                           ((df_total[\"TM\"] == \"lda\") | (df_total[\"TM\"] == \"lda_linear_combined\"))]\n",
    "            elif TM == \"nmf\":\n",
    "                df_selected = df_total[(df_total[\"DR\"] == DR) & \n",
    "                                          ((df_total[\"TM\"] == \"nmf\") | (df_total[\"TM\"] == \"nmf_linear_combined\")\n",
    "                                           | (df_total[\"TM\"] == \"nmf_tfidf\") | (df_total[\"TM\"] == \"nmf_tfidf_linear_combined\"))]\n",
    "            else:\n",
    "                df_selected = df_total[(df_total[\"DR\"] == DR) & \n",
    "                                          ((df_total[\"TM\"] == \"lsi\") | (df_total[\"TM\"] == \"lsi_linear_combined\")\n",
    "                                           | (df_total[\"TM\"] == \"lsi_tfidf\") | (df_total[\"TM\"] == \"lsi_tfidf_linear_combined\"))]\n",
    "                    \n",
    "        accuracy_list = df_selected[\"accuracy\"].tolist()\n",
    "        perception_list = df_selected[\"perception\"].tolist()\n",
    "        coherence_list = df_selected[\"coherence\"].tolist()\n",
    "            \n",
    "            \n",
    "        if (len(set(accuracy_list)) > 1) & (len(set(coherence_list)) > 1):\n",
    "            tau_accuracy, p_accuracy = stats.kendalltau(accuracy_list, coherence_list)\n",
    "            #print(corpus, DR, TM, round(tau_accuracy,2), round(p_accuracy,2))\n",
    "            tau_perception, p_perception = stats.kendalltau(perception_list, coherence_list)\n",
    "            #print(tau_perception, p_perception)\n",
    "\n",
    "        \n",
    "        result_TM_list.append(TM)\n",
    "        result_DR_list.append(DR)\n",
    "        result_acc_tau.append(round(tau_accuracy,2))\n",
    "        result_acc_p.append(round(p_accuracy,2))\n",
    "        result_per_tau.append(round(tau_perception,2))\n",
    "        result_per_p.append(round(p_perception,2))\n",
    "        \n",
    "df_result_TM_DR = pd.DataFrame({\"TM\": result_TM_list, \"DR\": result_DR_list, \"tau_acc\": result_acc_tau,\n",
    "                                \"p_acc\": result_acc_p, \"tau_per\": result_per_tau, \"p_per\": result_per_p})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac2ebbb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result_TM_DR.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "118348f4",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_result_TM_DR' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df_result_TM_DR\u001b[38;5;241m.\u001b[39mto_csv(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mKendall_TM_DR.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df_result_TM_DR' is not defined"
     ]
    }
   ],
   "source": [
    "df_result_TM_DR.to_csv(\"Kendall_TM_DR.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e286f6f-3081-4384-bbe1-66662ca10600",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}