WordErrorRate.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "WordErrorRate.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "eTAMbarHpjMp",
        "FUt5XoH5v1ps",
        "8dFs5L385eUl",
        "6YqotxMs5v7x"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gZeX8t7FlO_S"
      },
      "source": [
        "# Installation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "31xjFdrBlSLz",
        "outputId": "340f7bff-a5d6-4581-ce83-c4da99d89291"
      },
      "source": [
        "! pip install jiwer\n",
        "\n",
        "! pip install SpeechRecognition"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: jiwer in /usr/local/lib/python3.7/dist-packages (2.2.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from jiwer) (1.19.5)\n",
            "Requirement already satisfied: python-Levenshtein in /usr/local/lib/python3.7/dist-packages (from jiwer) (0.12.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from python-Levenshtein->jiwer) (56.1.0)\n",
            "Requirement already satisfied: SpeechRecognition in /usr/local/lib/python3.7/dist-packages (3.8.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pZBXJJKnlU1e"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBB6lDt7lWEb"
      },
      "source": [
        "## Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zBOYvPBUTDQ3",
        "outputId": "80d0a8c6-4d84-4fc7-a907-12c21d419fb7"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "46EDKWj7lZwx"
      },
      "source": [
        "## General"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VD3FMjmfTtPf"
      },
      "source": [
        "import json\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import speech_recognition as sr\n",
        "\n",
        "from jiwer import wer\n",
        "from random import randint"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pmLz-Vbvlicn"
      },
      "source": [
        "# Code"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2aB6TTE-ljdl"
      },
      "source": [
        "## Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTMjEvjnUslb"
      },
      "source": [
        "df = pd.read_csv('path/to/test_csv.csv')\n",
        "df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eTAMbarHpjMp"
      },
      "source": [
        "## Google WER Result on Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llw-1MuDVDh3"
      },
      "source": [
        "wer_list = []\n",
        "print('-------------------------\\n')\n",
        "for i in range(len(df)):\n",
        "  filename = df.iloc[i]['wav_filename']\n",
        "  transcript = df.iloc[i]['transcript']\n",
        "  r = sr.Recognizer()\n",
        "  with sr.AudioFile(filename) as source:\n",
        "    text = r.listen(source)\n",
        "  try :\n",
        "    predicted_transcript = r.recognize_google(text , language = \"ta-IN\")\n",
        "    print('Audio File', str(i+1))\n",
        "    print('Transcript:', transcript)\n",
        "    print('Predicted Transcript: ' + predicted_transcript + '\\n')\n",
        "    wer_list.append(wer(transcript, predicted_transcript))\n",
        "    print('-------------------------\\n')\n",
        "  except :\n",
        "    print('Sorry.Try Again!\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zN_0dDeLrrE0",
        "outputId": "3d08740f-fd33-4f38-9229-2a2b17086871"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1497\n",
            "Average WER: 0.55\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FUt5XoH5v1ps"
      },
      "source": [
        "## WER of Models based on jiwer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LIO9qPU4nvhv"
      },
      "source": [
        "def get_wer(json_data):\n",
        "  wer_list = []\n",
        "  print('-------------------------\\n')\n",
        "  for i in range(len(df)):\n",
        "    ground_truth = data[i]['src']\n",
        "    prediction = data[i]['res']\n",
        "    print('Audio File', str(i+1))\n",
        "    print('Ground Truth:', ground_truth)\n",
        "    print('Model Prediction: ' + prediction + '\\n')\n",
        "    wer_list.append(wer(ground_truth, prediction))\n",
        "    print('-------------------------\\n')\n",
        "  return wer_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5mTfcN5kmFfT"
      },
      "source": [
        "### Trained on Common Voice Train"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s09RQEqHmOKZ"
      },
      "source": [
        "#### Common Voice Train LM (LM1)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZtRQuEImOKa"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KZF5p-lFn5ak"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7AiZelpCBhmZ",
        "outputId": "e039e548-a6da-4c1e-db17-26892ac8570f"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.253\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MJl7XJ44msF6"
      },
      "source": [
        "#### News LM (LM2)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMhvTvttmsGS"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zOVMltmvmsGT"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZBdZQpEPBgO_",
        "outputId": "8702f436-dbff-4f49-99a5-25b7862a8852"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.277\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xt_zL9zimFfU"
      },
      "source": [
        "#### Wiki LM (LM3)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pFkcHfPHmFfU"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJ8OJtE1mFfV"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2SdJiR5TBfjI",
        "outputId": "1dc473c8-e5da-4bc7-d4b5-e1cb96226301"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.378\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5vC85gmemFfa"
      },
      "source": [
        "#### Wiki and News LM (LM4)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LqWpVAgcmFfa"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eJX84XFbmFfb"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RK3ynMRnBg7f",
        "outputId": "31e1eca3-5ffc-4e37-b3b8-80d84a4f32b0"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.378\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8dFs5L385eUl"
      },
      "source": [
        "### Trained on Common Voice Train and Open SLR\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75v6Jz1lnleL"
      },
      "source": [
        "#### Common Voice Train and Open SLR LM (LM1)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gCEQI0tpnleM"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n8cq_hwknleN"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WY0xYI2cA_xw",
        "outputId": "461ad859-3906-4816-999b-64a5ecb02af9"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.255\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "txRTAHY9nleO"
      },
      "source": [
        " #### News LM (LM2)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sl7fvwmRnleP"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JF5IgT7YnleP"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AWD-sxA7Ayvs",
        "outputId": "a809a8d2-843a-42b5-b159-fb4446852be2"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.273\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rCqAYrBenVmZ"
      },
      "source": [
        "#### Wiki LM (LM3)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6FYblY67nVma"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IC-wunxdnVmb"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_c9EfW9WqhlH",
        "outputId": "d657d336-bb07-4bb5-c908-1c5d87b4af2c"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.374\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4HqrRoDdngmY"
      },
      "source": [
        "#### Wiki and News LM (LM4)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BSbDjWHDngmZ"
      },
      "source": [
        "with open('path/to/output_json.json') as f :\n",
        "  data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9hKloBnFngma"
      },
      "source": [
        "wer_list = get_wer(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GwWfYlK9A7c6",
        "outputId": "3a27d981-bc72-46ba-abf9-005dabc99da6"
      },
      "source": [
        "print('Length:', len(wer_list))\n",
        "average = sum(wer_list) / len(wer_list)\n",
        "print('Average WER:', round(average, 3))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Length: 1500\n",
            "Average WER: 0.374\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ObbrTBN9RXp"
      },
      "source": [
        "## Examples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TQpu3eTF0MHE"
      },
      "source": [
        "def get_index(data, filename) :\n",
        "  for idx in range(len(data)) :\n",
        "    if data[idx]['wav_filename'] == filename :\n",
        "      return idx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KRUwdsLPCKxx"
      },
      "source": [
        "def get_wer_data(filenames, flag = True) :\n",
        "  count = 0\n",
        "  print('---------------------------------------\\n')\n",
        "  if flag == True :\n",
        "    print('Trained on Common Voice Train and Open SLR\\n')\n",
        "  else :\n",
        "    print('Trained on Common Voice Train\\n')\n",
        "  index = randint(0, len(df) - 1)\n",
        "  for lm, path in filenames.items() :\n",
        "    data = json.load(open(path))\n",
        "    print('LM:', lm)\n",
        "    if count == 0 :\n",
        "      filename = data[index]['wav_filename']\n",
        "      filename_to_be_checked = filename\n",
        "      count+= 1\n",
        "    else :\n",
        "      index = get_index(data, filename_to_be_checked)\n",
        "      filename = data[index]['wav_filename']\n",
        "    r = sr.Recognizer()\n",
        "    with sr.AudioFile(filename) as source:\n",
        "      text = r.listen(source)\n",
        "    try :\n",
        "      predicted_transcript = r.recognize_google(text , language = \"ta-IN\")\n",
        "      ground_truth = data[index]['src']\n",
        "      prediction = data[index]['res']\n",
        "      print('Filename:', filename)\n",
        "      print('Ground Truth:', ground_truth)\n",
        "      print('Model Prediction: ' + prediction)\n",
        "      print('Google Transcript: ' + predicted_transcript)\n",
        "      print('Model Prediction WER:', wer(ground_truth, prediction))\n",
        "      print('Google Prediction WER:', wer(ground_truth, predicted_transcript))\n",
        "    except :\n",
        "      print('Sorry.Try Again!\\n')\n",
        "    print('-------------------------\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RaORAlk2oYVZ"
      },
      "source": [
        "### Trained on Common Voice Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QU3_lguboYVa"
      },
      "source": [
        "filenames = {}\n",
        "\n",
        "filenames['Common Voice Train LM (LM1)'] = 'path/to/output_json.json' \n",
        "filenames['News LM (LM2)'] = 'path/to/output_json.json' \n",
        "filenames['Wiki LM (LM3)'] = 'path/to/output_json.json' \n",
        "filenames['Wiki and News LM (LM4)'] = 'path/to/output_json.json'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "abUb9vDPoYVb"
      },
      "source": [
        "#### Example 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LVIvVNpaoYVc",
        "outputId": "b605ec6c-e74d-4aba-9d7a-a3e021c46c76"
      },
      "source": [
        "get_wer_data(filenames, False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------------------------------\n",
            "\n",
            "Trained on Common Voice Train\n",
            "\n",
            "LM: Common Voice Train LM (LM1)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20400627.wav\n",
            "Ground Truth: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Model Prediction: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Google Transcript: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n",
            "LM: News LM (LM2)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20400627.wav\n",
            "Ground Truth: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Model Prediction: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Google Transcript: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki LM (LM3)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20400627.wav\n",
            "Ground Truth: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Model Prediction: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Google Transcript: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki and News LM (LM4)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20400627.wav\n",
            "Ground Truth: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Model Prediction: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க \n",
            "Google Transcript: யாராவது எடுத்துட்டு போயிட்டா என்ன பண்ணுவீங்க\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bmarVN6KoYVd"
      },
      "source": [
        "#### Example 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dNBliVDVoYVe",
        "outputId": "b29c8ea7-fbd2-45b9-f5bf-1a4677d68f44"
      },
      "source": [
        "get_wer_data(filenames, False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------------------------------\n",
            "\n",
            "Trained on Common Voice Train\n",
            "\n",
            "LM: Common Voice Train LM (LM1)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21279014.wav\n",
            "Ground Truth: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Model Prediction: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Google Transcript: ஆட்ட நடைநடந்து மண்ணை\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.3333333333333333\n",
            "-------------------------\n",
            "\n",
            "LM: News LM (LM2)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21279014.wav\n",
            "Ground Truth: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Model Prediction: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Google Transcript: ஆட்ட நடைநடந்து மண்ணை\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.3333333333333333\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki LM (LM3)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21279014.wav\n",
            "Ground Truth: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Model Prediction: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Google Transcript: ஆட்ட நடைநடந்து மண்ணை\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.3333333333333333\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki and News LM (LM4)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21279014.wav\n",
            "Ground Truth: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Model Prediction: ஆட்ட நடைநடந்தே மண்ணை\n",
            "Google Transcript: ஆட்ட நடைநடந்து மண்ணை\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.3333333333333333\n",
            "-------------------------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HiwmeOoioYVe"
      },
      "source": [
        "#### Example 3"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p3GLkg0foYVf",
        "outputId": "6cddc3a8-a8f7-4dd5-aa48-a994289cac79"
      },
      "source": [
        "get_wer_data(filenames, False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------------------------------\n",
            "\n",
            "Trained on Common Voice Train\n",
            "\n",
            "LM: Common Voice Train LM (LM1)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20529365.wav\n",
            "Ground Truth: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று \n",
            "Model Prediction: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று \n",
            "Google Transcript: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n",
            "LM: News LM (LM2)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20529365.wav\n",
            "Ground Truth: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று \n",
            "Model Prediction: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று \n",
            "Google Transcript: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki LM (LM3)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20529365.wav\n",
            "Ground Truth: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று \n",
            "Model Prediction: தேடலுக்கு அச்சம் தீர்ப்புக்கு \n",
            "Google Transcript: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று\n",
            "Model Prediction WER: 0.75\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki and News LM (LM4)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20529365.wav\n",
            "Ground Truth: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று \n",
            "Model Prediction: தேடலுக்கு அச்சம் தீர்ப்புக்கு \n",
            "Google Transcript: திருடனுக்கு அச்சம் தீர்ந்து போயிற்று\n",
            "Model Prediction WER: 0.75\n",
            "Google Prediction WER: 0.0\n",
            "-------------------------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R0R_GsKx9aoT"
      },
      "source": [
        "### Trained on Common Voice Train and Open SLR"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M8_2l-4mG4xR"
      },
      "source": [
        "filenames = {}\n",
        "\n",
        "filenames['Common Voice Train and Open SLR LM (LM1)'] = 'path/to/output_json.json' \n",
        "filenames['News LM (LM2)'] = 'path/to/output_json.json' \n",
        "filenames['Wiki LM (LM3)'] = 'path/to/output_json.json' \n",
        "filenames['Wiki and News LM (LM4)'] = 'path/to/output_json.json' "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mRH2fjgnHjTb"
      },
      "source": [
        "#### Example 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pY_Uujs9HznO",
        "outputId": "dacd7779-a322-444b-b1c3-df28a3a09713"
      },
      "source": [
        "get_wer_data(filenames)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------------------------------\n",
            "\n",
            "Trained on Common Voice Train and Open SLR\n",
            "\n",
            "LM: Common Voice Train and Open SLR LM (LM1)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_19093663.wav\n",
            "Ground Truth: முந்தஓர் பாட்டுரைத்தாள் அது\n",
            "Model Prediction: முந்தஓர் பாட்டுரைத்தாள் அது \n",
            "Google Transcript: முந்தை ஓர் பாட்டு உரைத்தால் அது\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 1.3333333333333333\n",
            "-------------------------\n",
            "\n",
            "LM: News LM (LM2)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_19093663.wav\n",
            "Ground Truth: முந்தஓர் பாட்டுரைத்தாள் அது\n",
            "Model Prediction: முந்தஓர் பாட்டுரைத்தாள் அது \n",
            "Google Transcript: முந்தை ஓர் பாட்டு உரைத்தால் அது\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 1.3333333333333333\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki LM (LM3)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_19093663.wav\n",
            "Ground Truth: முந்தஓர் பாட்டுரைத்தாள் அது\n",
            "Model Prediction: முந்தஓர் பாட்டுரைத்தாள் அது \n",
            "Google Transcript: முந்தை ஓர் பாட்டு உரைத்தால் அது\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 1.3333333333333333\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki and News LM (LM4)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_19093663.wav\n",
            "Ground Truth: முந்தஓர் பாட்டுரைத்தாள் அது\n",
            "Model Prediction: முந்தஓர் பாட்டுரைத்தாள் அது \n",
            "Google Transcript: முந்தை ஓர் பாட்டு உரைத்தால் அது\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 1.3333333333333333\n",
            "-------------------------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3cCb6KBSAmpG"
      },
      "source": [
        "#### Example 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j5Rlw2oRIPO-",
        "outputId": "7c583ca0-f252-4373-d448-5ca255aa7c1a"
      },
      "source": [
        "get_wer_data(filenames)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------------------------------\n",
            "\n",
            "Trained on Common Voice Train and Open SLR\n",
            "\n",
            "LM: Common Voice Train and Open SLR LM (LM1)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20591148.wav\n",
            "Ground Truth: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Model Prediction: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Google Transcript: நமது வீட்டின் நாலாவது வீட்டு\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.25\n",
            "-------------------------\n",
            "\n",
            "LM: News LM (LM2)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20591148.wav\n",
            "Ground Truth: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Model Prediction: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Google Transcript: நமது வீட்டின் நாலாவது வீட்டு\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.25\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki LM (LM3)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20591148.wav\n",
            "Ground Truth: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Model Prediction: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Google Transcript: நமது வீட்டின் நாலாவது வீட்டு\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.25\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki and News LM (LM4)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_20591148.wav\n",
            "Ground Truth: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Model Prediction: நமது வீட்டின் நாலாவது வீட்டுக் \n",
            "Google Transcript: நமது வீட்டின் நாலாவது வீட்டு\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.25\n",
            "-------------------------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yt-nh_2yAnnX"
      },
      "source": [
        "#### Example 3"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aV5cWlEPIUaj",
        "outputId": "abcd1cf5-1752-4786-ca81-3c8d2c5a07d2"
      },
      "source": [
        "get_wer_data(filenames)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "---------------------------------------\n",
            "\n",
            "Trained on Common Voice Train and Open SLR\n",
            "\n",
            "LM: Common Voice Train and Open SLR LM (LM1)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21281955.wav\n",
            "Ground Truth: இலையிந்த நாட்டினிலே அவனை ஒப்பார் \n",
            "Model Prediction: இலைஎன்ன நாட்டினிலே அவனை ஒப்பார் \n",
            "Google Transcript: இலை இந்த நாட்டினிலே அவனை ஒப்பார்\n",
            "Model Prediction WER: 0.25\n",
            "Google Prediction WER: 0.5\n",
            "-------------------------\n",
            "\n",
            "LM: News LM (LM2)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21281955.wav\n",
            "Ground Truth: இலையிந்த நாட்டினிலே அவனை ஒப்பார் \n",
            "Model Prediction: இலையிந்த நாட்டினிலே அவனை ஒப்பார் \n",
            "Google Transcript: இலை இந்த நாட்டினிலே அவனை ஒப்பார்\n",
            "Model Prediction WER: 0.0\n",
            "Google Prediction WER: 0.5\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki LM (LM3)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21281955.wav\n",
            "Ground Truth: இலையிந்த நாட்டினிலே அவனை ஒப்பார் \n",
            "Model Prediction: இலை என்ற நாட்டினிலே அவனை ஒப்பார் \n",
            "Google Transcript: இலை இந்த நாட்டினிலே அவனை ஒப்பார்\n",
            "Model Prediction WER: 0.5\n",
            "Google Prediction WER: 0.5\n",
            "-------------------------\n",
            "\n",
            "LM: Wiki and News LM (LM4)\n",
            "Filename: /content/drive/My Drive/deepspeech/data/CommonVoiceTamil/Audio16000/common_voice_ta_21281955.wav\n",
            "Ground Truth: இலையிந்த நாட்டினிலே அவனை ஒப்பார் \n",
            "Model Prediction: இலை என்ற நாட்டினிலே அவனை ஒப்பார் \n",
            "Google Transcript: இலை இந்த நாட்டினிலே அவனை ஒப்பார்\n",
            "Model Prediction WER: 0.5\n",
            "Google Prediction WER: 0.5\n",
            "-------------------------\n",
            "\n"
          ]
        }
      ]
    }
  ]
}