From a4a73a68cd51ee5b2140f58e05e7fd1d57b480ae Mon Sep 17 00:00:00 2001 From: Aleksandr Mokrov Date: Tue, 16 Jul 2024 02:31:27 +0200 Subject: [PATCH] Add weight compression --- notebooks/openvoice/openvoice.ipynb | 345 ++++++++++++++++++++++------ 1 file changed, 274 insertions(+), 71 deletions(-) diff --git a/notebooks/openvoice/openvoice.ipynb b/notebooks/openvoice/openvoice.ipynb index d19a4d3ba27..456383df25b 100644 --- a/notebooks/openvoice/openvoice.ipynb +++ b/notebooks/openvoice/openvoice.ipynb @@ -60,12 +60,12 @@ "output_type": "stream", "text": [ "Cloning into 'OpenVoice'...\n", - "remote: Enumerating objects: 371, done.\u001b[K\n", - "remote: Counting objects: 100% (188/188), done.\u001b[K\n", - "remote: Compressing objects: 100% (90/90), done.\u001b[K\n", - "remote: Total 371 (delta 129), reused 127 (delta 98), pack-reused 183\u001b[K\n", - "Receiving objects: 100% (371/371), 2.92 MiB | 3.34 MiB/s, done.\n", - "Resolving deltas: 100% (182/182), done.\n", + "remote: Enumerating objects: 435, done.\u001b[K\n", + "remote: Counting objects: 100% (235/235), done.\u001b[K\n", + "remote: Compressing objects: 100% (110/110), done.\u001b[K\n", + "remote: Total 435 (delta 176), reused 126 (delta 125), pack-reused 200\u001b[K\n", + "Receiving objects: 100% (435/435), 3.82 MiB | 6.90 MiB/s, done.\n", + "Resolving deltas: 100% (219/219), done.\n", "Note: you may need to restart the kernel to use updated packages.\n" ] } @@ -92,7 +92,7 @@ "sys.path.append(str(repo_dir))\n", "\n", "%pip install -q \"librosa>=0.8.1\" \"wavmark>=0.0.3\" \"faster-whisper>=0.9.0\" \"pydub>=0.25.1\" \"whisper-timestamped>=1.14.2\" \"tqdm\" \"inflect>=7.0.0\" \"eng_to_ipa>=0.0.2\" \"pypinyin>=0.50.0\" \\\n", - "\"cn2an>=0.5.22\" \"jieba>=0.42.1\" \"langid>=1.1.6\" \"gradio>=4.15\" \"ipywebrtc\" \"anyascii\" \"openvino>=2023.3\" \"torch>=2.1\"" + "\"cn2an>=0.5.22\" \"jieba>=0.42.1\" \"langid>=1.1.6\" \"gradio>=4.15\" \"ipywebrtc\" \"anyascii\" \"openvino>=2023.3\" \"torch>=2.1\" \"nncf>=2.11.0\"" ] }, { @@ -167,7 +167,92 @@ "cell_type": "code", "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bc89db6688a843298892d2b431a5b9b6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "checkpoint.pth: 0%| | 0.00/131M [00:00 0:\n", - "/home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:114: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", " if torch.min(inputs) < left or torch.max(inputs) > right:\n", - "/home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:119: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", " if min_bin_width * num_bins > 1.0:\n", - "/home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:121: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:121: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", " if min_bin_height * num_bins > 1.0:\n", - "/home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:171: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/transforms.py:171: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", " assert (discriminant >= 0).all()\n", - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:\n", - "\t%3293 : Float(1, 2, 43, strides=[86, 43, 1], requires_grad=0, device=cpu) = aten::randn(%3288, %3289, %3290, %3291, %3292) # /home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/models.py:175:0\n", - "\t%5559 : Float(1, 192, 154, strides=[29568, 1, 192], requires_grad=0, device=cpu) = aten::randn_like(%m_p, %5554, %5555, %5556, %5557, %5558) # /home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/models.py:485:0\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/torch/jit/_trace.py:1116: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:\n", + "\t%3293 : Float(1, 2, 43, strides=[86, 43, 1], requires_grad=0, device=cpu) = aten::randn(%3288, %3289, %3290, %3291, %3292) # /home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/nncf/torch/dynamic_graph/wrappers.py:81:0\n", + "\t%5559 : Float(1, 192, 155, strides=[29760, 1, 192], requires_grad=0, device=cpu) = aten::randn_like(%m_p, %5554, %5555, %5556, %5557, %5558) # /home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/nncf/torch/dynamic_graph/wrappers.py:81:0\n", "This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace()\n", " _check_trace(\n", - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", - "The values for attribute 'shape' do not match: torch.Size([1, 1, 38400]) != torch.Size([1, 1, 38912]).\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", + "The values for attribute 'shape' do not match: torch.Size([1, 1, 39424]) != torch.Size([1, 1, 39168]).\n", " _check_trace(\n", - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 2. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", - "The values for attribute 'shape' do not match: torch.Size([1, 1, 150, 43]) != torch.Size([1, 1, 152, 43]).\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 2. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", + "The values for attribute 'shape' do not match: torch.Size([1, 1, 154, 43]) != torch.Size([1, 1, 153, 43]).\n", " _check_trace(\n", - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 3. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", - "The values for attribute 'shape' do not match: torch.Size([1, 1, 150]) != torch.Size([1, 1, 152]).\n", - " _check_trace(\n", - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:\n", - "\t%1596 : Float(1, 192, 238, strides=[91392, 238, 1], requires_grad=0, device=cpu) = aten::randn_like(%m, %1591, %1592, %1593, %1594, %1595) # /home/ea/work/openvino_notebooks/notebooks/openvoice/OpenVoice/openvoice/models.py:220:0\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 3. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", + "The values for attribute 'shape' do not match: torch.Size([1, 1, 154]) != torch.Size([1, 1, 153]).\n", + " _check_trace(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:nncf:Statistics of the bitwidth distribution:\n", + "┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑\n", + "│ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │\n", + "┝━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥\n", + "│ 8 │ 100% (199 / 199) │ 100% (199 / 199) │\n", + "┕━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8f42a300cba64e21b89e80f6badf0dca", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n",
+       "
\n" + ], + "text/plain": [ + "\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/torch/jit/_trace.py:1116: TracerWarning: Trace had nondeterministic nodes. Did you forget call .eval() on your model? Nodes:\n", + "\t%1596 : Float(1, 192, 238, strides=[91392, 238, 1], requires_grad=0, device=cpu) = aten::randn_like(%m, %1591, %1592, %1593, %1594, %1595) # /home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/nncf/torch/dynamic_graph/wrappers.py:81:0\n", "This may cause errors in trace checking. To disable trace checking, pass check_trace=False to torch.jit.trace()\n", " _check_trace(\n", - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/jit/_trace.py:1102: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", + "/home/maleksandr/test_notebooks/openvoice-weight-compression/openvino_notebooks/notebooks/openvoice/venv/lib/python3.10/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error:\n", "Tensor-likes are not close!\n", "\n", - "Mismatched elements: 57466 / 60928 (94.3%)\n", - "Greatest absolute difference: 0.029862759198294953 at index (0, 0, 56526) (up to 1e-05 allowed)\n", - "Greatest relative difference: 16600.53479994854 at index (0, 0, 33772) (up to 1e-05 allowed)\n", + "Mismatched elements: 27966 / 60928 (45.9%)\n", + "Greatest absolute difference: 0.012129556707805023 at index (0, 0, 37204) (up to 1e-05 allowed)\n", + "Greatest relative difference: 34172.769230769234 at index (0, 0, 34029) (up to 1e-05 allowed)\n", " _check_trace(\n" ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:nncf:Statistics of the bitwidth distribution:\n", + "┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑\n", + "│ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │\n", + "┝━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥\n", + "│ 8 │ 100% (194 / 194) │ 100% (194 / 194) │\n", + "┕━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cbf4bca691de4a92ae5328805a541750", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n",
+       "
\n" + ], + "text/plain": [ + "\n" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ + "import nncf\n", + "\n", + "\n", "IRS_PATH = \"openvino_irs/\"\n", "EN_TTS_IR = f\"{IRS_PATH}/openvoice_en_tts.xml\"\n", "ZH_TTS_IR = f\"{IRS_PATH}/openvoice_zh_tts.xml\"\n", @@ -397,6 +607,7 @@ "for model, path in zip(models, paths):\n", " if not os.path.exists(path):\n", " ov_model = ov.convert_model(model, example_input=model.get_example_input())\n", + " ov_model = nncf.compress_weights(ov_model)\n", " ov.save_model(ov_model, path)\n", " else:\n", " ov_model = core.read_model(path)\n", @@ -433,12 +644,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "da1af27d5ecf48e48dbd545bd3cd1574", + "model_id": "a7447a09a38245d3a7d48bf014f68ecd", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Dropdown(description='Device:', index=3, options=('CPU', 'GPU.0', 'GPU.1', 'AUTO'), value='AUTO')" + "Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO')" ] }, "execution_count": 9, @@ -483,12 +694,12 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e6146e4da9b14edc893e0fdbed9a87d8", + "model_id": "81d39354b2e74a2a84a5ea25dd72f9d7", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Dropdown(description='reference voice from which tone color will be copied', options=('demo_speaker2.mp3', 'ex…" + "Dropdown(description='reference voice from which tone color will be copied', options=('demo_speaker0.mp3', 'de…" ] }, "execution_count": 10, @@ -516,7 +727,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -526,7 +737,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -559,7 +770,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -576,7 +787,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -593,7 +804,7 @@ "" ] }, - "execution_count": 14, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -612,25 +823,17 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "OpenVoice version: v1\n", "[(0.0, 8.178), (9.326, 12.914), (13.262, 16.402), (16.654, 29.49225)]\n", "after vad: dur = 27.743990929705216\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/ea/work/my_optimum_intel/optimum_env/lib/python3.8/site-packages/torch/functional.py:660: UserWarning: stft with return_complex=False is deprecated. In a future pytorch release, stft will return complex tensors for all inputs, and return_complex=False will raise an error.\n", - "Note: you can still call torch.view_as_real on the complex output to recover the old return format. (Triggered internally at ../aten/src/ATen/native/SpectralOps.cpp:874.)\n", - " return _VF.stft(input, n_fft, hop_length, win_length, window, # type: ignore[attr-defined]\n" - ] } ], "source": [ @@ -653,7 +856,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -694,13 +897,13 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "611d1acfb7da46948fd70de48b98c956", + "model_id": "3ceddc53676c40a3bbd12ba12e88ca71", "version_major": 2, "version_minor": 0 }, @@ -708,7 +911,7 @@ "Dropdown(description='Voice source', options=('use TTS', 'choose_manually'), value='use TTS')" ] }, - "execution_count": 17, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -726,7 +929,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -741,7 +944,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -787,13 +990,13 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4202d735f73e4bdf916edb6f552c603e", + "model_id": "a490b4a80d4245beb7cbb0cd6f3c9e8b", "version_major": 2, "version_minor": 0 }, @@ -801,7 +1004,7 @@ "FloatSlider(value=0.3, description='tau', max=2.0, min=0.01, step=0.01)" ] }, - "execution_count": 20, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -821,7 +1024,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -839,7 +1042,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -847,7 +1050,7 @@ "text/html": [ "\n", " \n", " " @@ -856,7 +1059,7 @@ "" ] }, - "execution_count": 23, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -867,7 +1070,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -875,7 +1078,7 @@ "text/html": [ "\n", " \n", " " @@ -884,7 +1087,7 @@ "" ] }, - "execution_count": 24, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -1261,7 +1464,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.12" }, "openvino_notebooks": { "imageUrl": "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/48584f6c-5836-464e-b6d6-10d46511d961",