DETAILED_FA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a796750e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from pyannote.core import Timeline, Segment, Annotation\n",
    "\n",
    "from inaGVAD.vad_metrics import VadEval, csv2annot, annot2vad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2a254ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VadFaEval(VadEval):\n",
    "    def compare_csv(self, ref_csv, pred_csv, reset = False):\n",
    "        predannot = annot2vad(csv2annot(pred_csv))\n",
    "        refannot = Annotation() ## NOTHING DETECTED - we'll iterate on non speech segments\n",
    "        \n",
    "        ref = pd.read_csv(ref_csv)\n",
    "        dret = {}\n",
    "        for cat in nonspeech:\n",
    "            uem = Timeline()\n",
    "            for t in ref[ref[cat] == True].itertuples():\n",
    "                uem.add(Segment(t.start, t.stop))\n",
    "            uem = uem.support()\n",
    "            #print(cat)\n",
    "            #print('uem dur', uem.duration())\n",
    "            ret = self(refannot, predannot, uem)\n",
    "            dret[(cat, 'total')] = uem.duration()\n",
    "            #print(ret)\n",
    "            dret[(cat, 'fa')] = ret['false positive']\n",
    "        return dret\n",
    "ve = VadFaEval(collar=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "afd61b78",
   "metadata": {},
   "outputs": [],
   "source": [
    "nonspeech = ['applause', 'noise', 'hubbub', 'jingle', 'fg_music', 'bg_music', 'respiration', 'laughers', 'other', 'empty']\n",
    "lsystems = ['inaspeechsegmenter', 'liumspkdiarization_csv', 'pyannote_vad', 'rvad_csv', 'silero_vad', 'speechbrain_vad']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f99ab0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def myeval(rdir, pdir):\n",
    "    ld = []\n",
    "    for f in os.listdir(rdir):\n",
    "        ld.append(ve.compare_csv(rdir + f, pdir + f))\n",
    "    stats = pd.DataFrame.from_dict(ld).sum()\n",
    "    d = {'system' : pdir}\n",
    "    for cat in nonspeech:\n",
    "        d[cat] = 100 * stats[(cat, 'fa')] / stats[(cat, 'total')]\n",
    "    return d\n",
    "#myeval(rdir, pdir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eaa794fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>applause</th>\n",
       "      <th>noise</th>\n",
       "      <th>hubbub</th>\n",
       "      <th>jingle</th>\n",
       "      <th>fg_music</th>\n",
       "      <th>bg_music</th>\n",
       "      <th>respiration</th>\n",
       "      <th>laughers</th>\n",
       "      <th>other</th>\n",
       "      <th>empty</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>system</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/inaspeechsegmenter/</th>\n",
       "      <td>15.433710</td>\n",
       "      <td>35.589857</td>\n",
       "      <td>35.950409</td>\n",
       "      <td>8.105043</td>\n",
       "      <td>1.509349</td>\n",
       "      <td>25.895633</td>\n",
       "      <td>95.051145</td>\n",
       "      <td>77.006912</td>\n",
       "      <td>21.973769</td>\n",
       "      <td>45.742934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/liumspkdiarization_csv/</th>\n",
       "      <td>28.013820</td>\n",
       "      <td>33.957232</td>\n",
       "      <td>35.218788</td>\n",
       "      <td>26.209908</td>\n",
       "      <td>31.051727</td>\n",
       "      <td>32.741995</td>\n",
       "      <td>95.993088</td>\n",
       "      <td>65.912187</td>\n",
       "      <td>22.174443</td>\n",
       "      <td>66.476826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/pyannote_vad/</th>\n",
       "      <td>7.506705</td>\n",
       "      <td>21.893214</td>\n",
       "      <td>30.829815</td>\n",
       "      <td>5.001881</td>\n",
       "      <td>27.709654</td>\n",
       "      <td>21.222281</td>\n",
       "      <td>88.955646</td>\n",
       "      <td>63.072211</td>\n",
       "      <td>35.913848</td>\n",
       "      <td>41.539049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/rvad_csv/</th>\n",
       "      <td>33.438827</td>\n",
       "      <td>33.771815</td>\n",
       "      <td>60.816923</td>\n",
       "      <td>83.775627</td>\n",
       "      <td>91.478572</td>\n",
       "      <td>53.774784</td>\n",
       "      <td>41.120480</td>\n",
       "      <td>85.735711</td>\n",
       "      <td>68.293557</td>\n",
       "      <td>21.737731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/silero_vad/</th>\n",
       "      <td>10.681225</td>\n",
       "      <td>25.862606</td>\n",
       "      <td>29.770845</td>\n",
       "      <td>20.121973</td>\n",
       "      <td>41.123387</td>\n",
       "      <td>25.681731</td>\n",
       "      <td>63.446152</td>\n",
       "      <td>76.040794</td>\n",
       "      <td>33.359612</td>\n",
       "      <td>35.384863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/speechbrain_vad/</th>\n",
       "      <td>30.329397</td>\n",
       "      <td>35.107573</td>\n",
       "      <td>39.993123</td>\n",
       "      <td>11.892817</td>\n",
       "      <td>9.293187</td>\n",
       "      <td>28.980360</td>\n",
       "      <td>97.736860</td>\n",
       "      <td>72.700005</td>\n",
       "      <td>21.237961</td>\n",
       "      <td>65.962379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              applause      noise     hubbub  \\\n",
       "system                                                                         \n",
       "automatic_baselines/inaspeechsegmenter/      15.433710  35.589857  35.950409   \n",
       "automatic_baselines/liumspkdiarization_csv/  28.013820  33.957232  35.218788   \n",
       "automatic_baselines/pyannote_vad/             7.506705  21.893214  30.829815   \n",
       "automatic_baselines/rvad_csv/                33.438827  33.771815  60.816923   \n",
       "automatic_baselines/silero_vad/              10.681225  25.862606  29.770845   \n",
       "automatic_baselines/speechbrain_vad/         30.329397  35.107573  39.993123   \n",
       "\n",
       "                                                jingle   fg_music   bg_music  \\\n",
       "system                                                                         \n",
       "automatic_baselines/inaspeechsegmenter/       8.105043   1.509349  25.895633   \n",
       "automatic_baselines/liumspkdiarization_csv/  26.209908  31.051727  32.741995   \n",
       "automatic_baselines/pyannote_vad/             5.001881  27.709654  21.222281   \n",
       "automatic_baselines/rvad_csv/                83.775627  91.478572  53.774784   \n",
       "automatic_baselines/silero_vad/              20.121973  41.123387  25.681731   \n",
       "automatic_baselines/speechbrain_vad/         11.892817   9.293187  28.980360   \n",
       "\n",
       "                                             respiration   laughers  \\\n",
       "system                                                                \n",
       "automatic_baselines/inaspeechsegmenter/        95.051145  77.006912   \n",
       "automatic_baselines/liumspkdiarization_csv/    95.993088  65.912187   \n",
       "automatic_baselines/pyannote_vad/              88.955646  63.072211   \n",
       "automatic_baselines/rvad_csv/                  41.120480  85.735711   \n",
       "automatic_baselines/silero_vad/                63.446152  76.040794   \n",
       "automatic_baselines/speechbrain_vad/           97.736860  72.700005   \n",
       "\n",
       "                                                 other      empty  \n",
       "system                                                             \n",
       "automatic_baselines/inaspeechsegmenter/      21.973769  45.742934  \n",
       "automatic_baselines/liumspkdiarization_csv/  22.174443  66.476826  \n",
       "automatic_baselines/pyannote_vad/            35.913848  41.539049  \n",
       "automatic_baselines/rvad_csv/                68.293557  21.737731  \n",
       "automatic_baselines/silero_vad/              33.359612  35.384863  \n",
       "automatic_baselines/speechbrain_vad/         21.237961  65.962379  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld = []\n",
    "for sys in lsystems:\n",
    "    ld.append(myeval('./annotations/detailed_csv/', 'automatic_baselines/' + sys  + '/'))\n",
    "df = pd.DataFrame.from_dict(ld).set_index('system')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "473dfa17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>applause</th>\n",
       "      <th>noise</th>\n",
       "      <th>hubbub</th>\n",
       "      <th>jingle</th>\n",
       "      <th>fg_music</th>\n",
       "      <th>bg_music</th>\n",
       "      <th>respiration</th>\n",
       "      <th>laughers</th>\n",
       "      <th>other</th>\n",
       "      <th>empty</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>system</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/inaspeechsegmenter/</th>\n",
       "      <td>15.433710</td>\n",
       "      <td>35.589857</td>\n",
       "      <td>35.950409</td>\n",
       "      <td>8.105043</td>\n",
       "      <td>1.509349</td>\n",
       "      <td>25.895633</td>\n",
       "      <td>95.051145</td>\n",
       "      <td>77.006912</td>\n",
       "      <td>21.973769</td>\n",
       "      <td>45.742934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/liumspkdiarization_csv/</th>\n",
       "      <td>28.013820</td>\n",
       "      <td>33.957232</td>\n",
       "      <td>35.218788</td>\n",
       "      <td>26.209908</td>\n",
       "      <td>31.051727</td>\n",
       "      <td>32.741995</td>\n",
       "      <td>95.993088</td>\n",
       "      <td>65.912187</td>\n",
       "      <td>22.174443</td>\n",
       "      <td>66.476826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/pyannote_vad/</th>\n",
       "      <td>7.506705</td>\n",
       "      <td>21.893214</td>\n",
       "      <td>30.829815</td>\n",
       "      <td>5.001881</td>\n",
       "      <td>27.709654</td>\n",
       "      <td>21.222281</td>\n",
       "      <td>88.955646</td>\n",
       "      <td>63.072211</td>\n",
       "      <td>35.913848</td>\n",
       "      <td>41.539049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/rvad_csv/</th>\n",
       "      <td>33.438827</td>\n",
       "      <td>33.771815</td>\n",
       "      <td>60.816923</td>\n",
       "      <td>83.775627</td>\n",
       "      <td>91.478572</td>\n",
       "      <td>53.774784</td>\n",
       "      <td>41.120480</td>\n",
       "      <td>85.735711</td>\n",
       "      <td>68.293557</td>\n",
       "      <td>21.737731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/silero_vad/</th>\n",
       "      <td>10.681225</td>\n",
       "      <td>25.862606</td>\n",
       "      <td>29.770845</td>\n",
       "      <td>20.121973</td>\n",
       "      <td>41.123387</td>\n",
       "      <td>25.681731</td>\n",
       "      <td>63.446152</td>\n",
       "      <td>76.040794</td>\n",
       "      <td>33.359612</td>\n",
       "      <td>35.384863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automatic_baselines/speechbrain_vad/</th>\n",
       "      <td>30.329397</td>\n",
       "      <td>35.107573</td>\n",
       "      <td>39.993123</td>\n",
       "      <td>11.892817</td>\n",
       "      <td>9.293187</td>\n",
       "      <td>28.980360</td>\n",
       "      <td>97.736860</td>\n",
       "      <td>72.700005</td>\n",
       "      <td>21.237961</td>\n",
       "      <td>65.962379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MEAN</th>\n",
       "      <td>20.900614</td>\n",
       "      <td>31.030383</td>\n",
       "      <td>38.763317</td>\n",
       "      <td>25.851208</td>\n",
       "      <td>33.694313</td>\n",
       "      <td>31.382797</td>\n",
       "      <td>80.383895</td>\n",
       "      <td>73.411303</td>\n",
       "      <td>33.825532</td>\n",
       "      <td>46.140630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DIFF</th>\n",
       "      <td>25.932122</td>\n",
       "      <td>13.696643</td>\n",
       "      <td>31.046078</td>\n",
       "      <td>78.773746</td>\n",
       "      <td>89.969222</td>\n",
       "      <td>32.552503</td>\n",
       "      <td>56.616380</td>\n",
       "      <td>22.663501</td>\n",
       "      <td>47.055596</td>\n",
       "      <td>44.739095</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              applause      noise     hubbub  \\\n",
       "system                                                                         \n",
       "automatic_baselines/inaspeechsegmenter/      15.433710  35.589857  35.950409   \n",
       "automatic_baselines/liumspkdiarization_csv/  28.013820  33.957232  35.218788   \n",
       "automatic_baselines/pyannote_vad/             7.506705  21.893214  30.829815   \n",
       "automatic_baselines/rvad_csv/                33.438827  33.771815  60.816923   \n",
       "automatic_baselines/silero_vad/              10.681225  25.862606  29.770845   \n",
       "automatic_baselines/speechbrain_vad/         30.329397  35.107573  39.993123   \n",
       "MEAN                                         20.900614  31.030383  38.763317   \n",
       "DIFF                                         25.932122  13.696643  31.046078   \n",
       "\n",
       "                                                jingle   fg_music   bg_music  \\\n",
       "system                                                                         \n",
       "automatic_baselines/inaspeechsegmenter/       8.105043   1.509349  25.895633   \n",
       "automatic_baselines/liumspkdiarization_csv/  26.209908  31.051727  32.741995   \n",
       "automatic_baselines/pyannote_vad/             5.001881  27.709654  21.222281   \n",
       "automatic_baselines/rvad_csv/                83.775627  91.478572  53.774784   \n",
       "automatic_baselines/silero_vad/              20.121973  41.123387  25.681731   \n",
       "automatic_baselines/speechbrain_vad/         11.892817   9.293187  28.980360   \n",
       "MEAN                                         25.851208  33.694313  31.382797   \n",
       "DIFF                                         78.773746  89.969222  32.552503   \n",
       "\n",
       "                                             respiration   laughers  \\\n",
       "system                                                                \n",
       "automatic_baselines/inaspeechsegmenter/        95.051145  77.006912   \n",
       "automatic_baselines/liumspkdiarization_csv/    95.993088  65.912187   \n",
       "automatic_baselines/pyannote_vad/              88.955646  63.072211   \n",
       "automatic_baselines/rvad_csv/                  41.120480  85.735711   \n",
       "automatic_baselines/silero_vad/                63.446152  76.040794   \n",
       "automatic_baselines/speechbrain_vad/           97.736860  72.700005   \n",
       "MEAN                                           80.383895  73.411303   \n",
       "DIFF                                           56.616380  22.663501   \n",
       "\n",
       "                                                 other      empty  \n",
       "system                                                             \n",
       "automatic_baselines/inaspeechsegmenter/      21.973769  45.742934  \n",
       "automatic_baselines/liumspkdiarization_csv/  22.174443  66.476826  \n",
       "automatic_baselines/pyannote_vad/            35.913848  41.539049  \n",
       "automatic_baselines/rvad_csv/                68.293557  21.737731  \n",
       "automatic_baselines/silero_vad/              33.359612  35.384863  \n",
       "automatic_baselines/speechbrain_vad/         21.237961  65.962379  \n",
       "MEAN                                         33.825532  46.140630  \n",
       "DIFF                                         47.055596  44.739095  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ld = []\n",
    "\n",
    "ld.append({'system' : 'MEAN'})\n",
    "ld[-1].update(df.mean())\n",
    "\n",
    "ld.append({'system' : 'DIFF'})\n",
    "ld[-1].update(df.max() - df.min())\n",
    "\n",
    "#ld.append({'system' : 'MEAN'})\n",
    "#ld[-1].update(df.mean())\n",
    "\n",
    "\n",
    "\n",
    "dmean = pd.DataFrame.from_dict(ld).set_index('system')\n",
    "dmean\n",
    "\n",
    "#d = {'system' : 'MAX'}\n",
    "#d.update(df.max())\n",
    "#dmean = pd.DataFrame.from_dict([d]).set_index('system')\n",
    "#dmean\n",
    "\n",
    "dall = pd.concat([df, dmean])\n",
    "#.to_latex()\n",
    "#df\n",
    "dall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "548b2e72",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrrrrrrr}\n",
      "\\toprule\n",
      " & applause & noise & hubbub & jingle & fgmusic & bgmusic & respiration & laughers & other & empty \\\\\n",
      "system &  &  &  &  &  &  &  &  &  &  \\\\\n",
      "\\midrule\n",
      "automaticbaselines/inaspeechsegmenter/ & 15.4 & 35.6 & 36.0 & 8.1 & 1.5 & 25.9 & 95.1 & 77.0 & 22.0 & 45.7 \\\\\n",
      "automaticbaselines/liumspkdiarizationcsv/ & 28.0 & 34.0 & 35.2 & 26.2 & 31.1 & 32.7 & 96.0 & 65.9 & 22.2 & 66.5 \\\\\n",
      "automaticbaselines/pyannotevad/ & 7.5 & 21.9 & 30.8 & 5.0 & 27.7 & 21.2 & 89.0 & 63.1 & 35.9 & 41.5 \\\\\n",
      "automaticbaselines/rvadcsv/ & 33.4 & 33.8 & 60.8 & 83.8 & 91.5 & 53.8 & 41.1 & 85.7 & 68.3 & 21.7 \\\\\n",
      "automaticbaselines/silerovad/ & 10.7 & 25.9 & 29.8 & 20.1 & 41.1 & 25.7 & 63.4 & 76.0 & 33.4 & 35.4 \\\\\n",
      "automaticbaselines/speechbrainvad/ & 30.3 & 35.1 & 40.0 & 11.9 & 9.3 & 29.0 & 97.7 & 72.7 & 21.2 & 66.0 \\\\\n",
      "MEAN & 20.9 & 31.0 & 38.8 & 25.9 & 33.7 & 31.4 & 80.4 & 73.4 & 33.8 & 46.1 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(dall.to_latex(float_format='%.1f').replace('_', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf3bbd08",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}