ESPnetASR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1HV5rvCGI1QwJ1wamEXi9XvuivexgH59_",
      "authorship_tag": "ABX9TyM+bxwRW+MDax3rdcpJkX3c",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kentamishima/kuromoji/blob/master/ESPnetASR.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6jsv77S80Gjj"
      },
      "outputs": [],
      "source": [
        "!pip install pydub\n",
        "!apt install ffmpeg\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install ffmpeg-python\n"
      ],
      "metadata": {
        "id": "oSO-2xya-3Nm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Google Driveと連携\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "iMg8p4MG_oxh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 作業ディレクトリの作成\n",
        "%cd /content/drive/MyDrive/\n",
        "!mkdir -p ASR\n",
        "%cd ASR\n",
        "!ls"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cCcm69miAZSB",
        "outputId": "5d921d51-029b-473d-d018-24a633205266"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive\n",
            "/content/drive/MyDrive/ASR\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# pip\n",
        "!pip install torch\n",
        "!pip install espnet_model_zoo"
      ],
      "metadata": {
        "id": "KAAoPhHv_wI3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import wave\n",
        "import struct\n",
        "import math\n",
        "import os\n",
        "from scipy import fromstring, int16\n",
        "\n",
        "import subprocess\n",
        "\n",
        "import shutil\n",
        "import glob\n",
        "\n",
        "import soundfile\n",
        "from espnet_model_zoo.downloader import ModelDownloader"
      ],
      "metadata": {
        "id": "xeRIQIUC_4aB"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 音声ファイルへの変換\n",
        "def mp4_to_wav(mp4f):\n",
        "    wavf = mp4f.replace('.mp4', '.wav')\n",
        "    subprocess.run(['ffmpeg', '-i', mp4f, '-ar', '16000', '-ac', '1', '-y', '-f', 'wav' ,wavf], \n",
        "                   encoding='utf-8', stdout=subprocess.PIPE)\n",
        "    return wavf"
      ],
      "metadata": {
        "id": "KtW8hF00Av1Q"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 音声ファイルの分割(デフォルト60秒)\n",
        "def cut_wav_espnet(wavf,time=600):\n",
        "    # timeの単位は[sec]\n",
        "    # ファイルを読み出し\n",
        "    wr = wave.open(wavf, 'r')\n",
        "\n",
        "    # waveファイルが持つ性質を取得\n",
        "    ch = wr.getnchannels()\n",
        "    width = wr.getsampwidth()\n",
        "    fr = wr.getframerate()\n",
        "    fn = wr.getnframes()\n",
        "    total_time = 1.0 * fn / fr\n",
        "    integer = math.floor(total_time) # 小数点以下切り捨て\n",
        "    t = int(time)  # 秒数[sec]\n",
        "    frames = int(ch * fr * t)\n",
        "    num_cut = int(integer//t)\n",
        "\n",
        "    # waveの実データを取得し、数値化\n",
        "    data = wr.readframes(wr.getnframes())\n",
        "    wr.close()\n",
        "    X = fromstring(data, dtype=int16)\n",
        "    \n",
        "    # wavファイルを削除\n",
        "    os.remove(wavf)\n",
        "    \n",
        "    wavf_list = []\n",
        "    for i in range(num_cut):\n",
        "        # 出力データを生成\n",
        "        output_dir = os.path.dirname(wavf) + '/output/cut_wav/'\n",
        "        os.makedirs(output_dir,exist_ok=True)\n",
        "        outf = output_dir + str(i).zfill(3) + '.wav'\n",
        "        start_cut = i*frames\n",
        "        end_cut = i*frames + frames\n",
        "        Y = X[start_cut:end_cut]\n",
        "        outd = struct.pack(\"h\" * len(Y), *Y)\n",
        "\n",
        "        # 書き出し\n",
        "        ww = wave.open(outf, 'w')\n",
        "        ww.setnchannels(ch)\n",
        "        ww.setsampwidth(width)\n",
        "        ww.setframerate(fr)\n",
        "        ww.writeframes(outd)\n",
        "        ww.close()\n",
        "        \n",
        "        # リストに追加\n",
        "        wavf_list.append(outf)\n",
        "    \n",
        "    return wavf_list"
      ],
      "metadata": {
        "id": "LLXTxVrKA2FO"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 複数ファイルの音声のテキスト変換\n",
        "def wavs_asr_espnet(wavf_list, asr_model):\n",
        "    output_text = ''\n",
        "    # 複数処理\n",
        "    print('音声のテキスト変換')\n",
        "    for wavf in wavf_list:\n",
        "        # wavファイルの読み込み\n",
        "        speech, _ = soundfile.read(wavf)\n",
        "        # 音声のテキスト変換\n",
        "        nbests = asr_model(speech)\n",
        "        text, *_ = nbests[0]\n",
        "\n",
        "        # 各ファイルの出力結果の結合\n",
        "        output_text = output_text + text + '\\n\\n'\n",
        "        # wavファイルを削除\n",
        "        os.remove(wavf)\n",
        "        print(wavf)\n",
        "        \n",
        "    return output_text"
      ],
      "metadata": {
        "id": "tfnPdbPTA9JD"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# mp4からwavへの変換から音声のテキスト変換まで\n",
        "def mp4_asr_espnet(mp4f):\n",
        "    # mp4のディレクトリ\n",
        "    input_dir = os.path.dirname(mp4f)\n",
        "    # # 学習済みをダウンロードし、音声認識モデルを作成\n",
        "    # d = ModelDownloader()\n",
        "    # speech2text = Speech2Text(\n",
        "    #         **d.download_and_unpack(\"kan-bayashi/csj_asr_train_asr_transformer_raw_char_sp_valid.acc.ave\"),\n",
        "    #         device=\"cuda\"  # CPU で認識を行う場合は省略\n",
        "    #         )\n",
        "\n",
        "    # 出力ディレクトリ\n",
        "    if os.path.exists(input_dir + '/output/cut_wav/'):\n",
        "        shutil.rmtree(input_dir + '/output/cut_wav/')\n",
        "        os.makedirs(input_dir + '/output/cut_wav/', exist_ok=True)\n",
        "    else:\n",
        "        os.makedirs(input_dir + '/output/cut_wav/', exist_ok=True)\n",
        "    \n",
        "    # 音声ファイルへの変換\n",
        "    wav_file = mp4_to_wav(mp4f)\n",
        "    \n",
        "    # 音声ファイルの分割(デフォルト60秒)\n",
        "    cut_wavs = cut_wav_espnet(wav_file)\n",
        "    \n",
        "    # # 複数ファイルの音声のテキスト変換\n",
        "    # out_text = wavs_asr_espnet(cut_wavs, speech2text)\n",
        "    \n",
        "    # # テキストファイルへの入力\n",
        "    # mp4f_name = os.path.basename(mp4f)\n",
        "    # txt_file = input_dir + '/output/' + mp4f_name.replace('.mp4', '.txt')\n",
        "    # print('テキスト出力')\n",
        "    # print(txt_file)\n",
        "    # f = open(txt_file, 'w')\n",
        "    # f.write(out_text)\n",
        "    # f.close()"
      ],
      "metadata": {
        "id": "FcFxKNKWBZ59"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mp4_files = glob.glob('./ASR_FILES/*.mp4')\n",
        "\n",
        "for mp4_file in mp4_files:\n",
        "    mp4_asr_espnet(mp4_file)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LalyPCNGBqj1",
        "outputId": "0eef43a7-3b26-440f-cb59-f6eebda3a0b2"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-32-0221bfccb932>:21: DeprecationWarning: scipy.fromstring is deprecated and will be removed in SciPy 2.0.0, use numpy.fromstring instead\n",
            "  X = fromstring(data, dtype=int16)\n"
          ]
        }
      ]
    }
  ]
}