reduce quantization subset for validation (#1954)

openvinotoolkit · May 3, 2024 · dd2d088 · dd2d088
1 parent 5adfd4c
commit dd2d088
Show file tree

Hide file tree

Showing 28 changed files with 109 additions and 46 deletions.
diff --git a/...oks/big-transfer-quantization/tensorflow-bit-image-classification-nncf-quantization.ipynb b/...oks/big-transfer-quantization/tensorflow-bit-image-classification-nncf-quantization.ipynb
@@ -321,7 +321,12 @@
   {
    "cell_type": "code",
    "execution_count": 9,
-   "metadata": {},
+   "metadata": {
+     "test_replace": {
+     "train_dataset.take(3000)": "train_dataset.take(30)",
+     "validation_data=validation_dataset.take(1000)": "validation_data=validation_dataset.take(10)"
+      }
+   },
    "outputs": [
     {
      "name": "stdout",
@@ -397,7 +402,10 @@
   {
    "cell_type": "code",
    "execution_count": 11,
-   "metadata": {},
+   "metadata": {
+      "test_replace": {"for _, label in validation_dataset:": "for _, label in validation_dataset.take(2):",
+   "for img_batch, label_batch in validation_dataset": "for img_batch, label_batch in validation_dataset.take(2):"}
+   },
    "outputs": [
     {
      "name": "stderr",
@@ -467,7 +475,9 @@
   {
    "cell_type": "code",
    "execution_count": 12,
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"for img_batch, _ in validation_dataset:": "for img_batch, _ in validation_dataset.take(2):"}
+   },
    "outputs": [],
    "source": [
     "ov_fp32_model = core.read_model(\"./bit_ov_model/bit_m_r50x1_1.xml\")\n",
@@ -507,7 +517,9 @@
   {
    "cell_type": "code",
    "execution_count": 13,
-   "metadata": {},
+   "metadata": {"test_replace": {
+     "ov_int8_model = nncf.quantize(ov_fp32_model, calibration_dataset, fast_bias_correction=False)": "ov_int8_model = nncf.quantize(ov_fp32_model, calibration_dataset, fast_bias_correction=False, subset_size=10)"
+     }},
    "outputs": [
     {
      "data": {

diff --git a/notebooks/blip-visual-language-processing/blip-visual-language-processing.ipynb b/notebooks/blip-visual-language-processing/blip-visual-language-processing.ipynb
@@ -818,7 +818,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "6a97bea6-ace6-4f66-b823-710ff969e3e9",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"def prepare_dataset(vision_model, opt_init_steps=300, streaming=False):": "def prepare_dataset(vision_model, opt_init_steps=10, streaming=False):"}
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/controlnet-stable-diffusion/controlnet-stable-diffusion.ipynb b/notebooks/controlnet-stable-diffusion/controlnet-stable-diffusion.ipynb
@@ -1554,7 +1554,12 @@
    "cell_type": "code",
    "execution_count": 22,
    "id": "5736a460",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {
+   "num_inference_steps = 20": "num_inference_steps = 10",
+   "subset_size = 200": "subset_size = 20"
+   }
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/decidiffusion-image-generation/decidiffusion-image-generation.ipynb b/notebooks/decidiffusion-image-generation/decidiffusion-image-generation.ipynb
@@ -1357,7 +1357,9 @@
    "cell_type": "code",
    "execution_count": 24,
    "id": "e07943eb",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 300": "subset_size = 10"}
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/depth-anything/depth-anything.ipynb b/notebooks/depth-anything/depth-anything.ipynb
@@ -883,7 +883,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "fc6d3137",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 300": "subset_size = 10"}
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/efficient-sam/efficient-sam.ipynb b/notebooks/efficient-sam/efficient-sam.ipynb
@@ -922,7 +922,9 @@
        "\n"
       ]
      },
-     "metadata": {},
+     "metadata": {
+     "test_replace": {"subset_size=128)": "subset_size=12)"}
+     },
      "output_type": "display_data"
     },
     {

diff --git a/notebooks/fast-segment-anything/fast-segment-anything.ipynb b/notebooks/fast-segment-anything/fast-segment-anything.ipynb
@@ -448,7 +448,8 @@
    "cell_type": "code",
    "execution_count": 12,
    "metadata": {
-    "scrolled": true
+    "scrolled": true,
+    "test_replace": {"calibration_dataset_size = 128": "calibration_dataset_size = 8"}
    },
    "outputs": [
     {
@@ -699,7 +700,8 @@
    "cell_type": "code",
    "execution_count": 13,
    "metadata": {
-    "scrolled": true
+    "scrolled": true,
+    "test_replace": {"calibration_dataset_size = 128": "calibration_dataset_size = 8"}
    },
    "outputs": [
     {

diff --git a/notebooks/image-bind/image-bind.ipynb b/notebooks/image-bind/image-bind.ipynb
@@ -779,7 +779,9 @@
    "cell_type": "code",
    "execution_count": 19,
    "id": "c0a03782-f215-4d99-830a-a0e90313ef38",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"def prepare_vision_text_dataset(opt_init_steps=50):": "def prepare_vision_text_dataset(opt_init_steps=5):"}
+   },
    "outputs": [],
    "source": [
     "from datasets import load_dataset\n",

diff --git a/notebooks/image-classification-quantization/image-classification-quantization.ipynb b/notebooks/image-classification-quantization/image-classification-quantization.ipynb
@@ -258,7 +258,8 @@
    "cell_type": "code",
    "execution_count": 9,
    "metadata": {
-    "tags": []
+    "tags": [],
+    "test_replace": {"quant_ov_model = nncf.quantize(ov_model, quantization_dataset)": "quant_ov_model = nncf.quantize(ov_model, quantization_dataset, subset_size=10)"}
    },
    "outputs": [
     {

diff --git a/notebooks/instant-id/instant-id.ipynb b/notebooks/instant-id/instant-id.ipynb
@@ -2057,7 +2057,9 @@
    "cell_type": "code",
    "execution_count": 32,
    "id": "79e974fe",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 200": "subset_size = 4"}
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/instruct-pix2pix-image-editing/instruct-pix2pix-image-editing.ipynb b/notebooks/instruct-pix2pix-image-editing/instruct-pix2pix-image-editing.ipynb
@@ -1322,7 +1322,9 @@
    "cell_type": "code",
    "execution_count": 17,
    "id": "5b4b6944",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 300": "subset_size = 10"}
+   },
    "outputs": [
     {
      "name": "stderr",

diff --git a/notebooks/language-quantize-bert/language-quantize-bert.ipynb b/notebooks/language-quantize-bert/language-quantize-bert.ipynb
@@ -381,7 +381,11 @@
    "cell_type": "code",
    "execution_count": 7,
    "id": "e089ea99",
-   "metadata": {},
+   "metadata": {
+    "test_replace": {
+     "quantized_model = nncf.quantize(model, calibration_dataset, model_type=ModelType.TRANSFORMER)": "quantized_model = nncf.quantize(model, calibration_dataset, model_type=ModelType.TRANSFORMER, subset_size=10)"
+    }
+   },
    "outputs": [
     {
      "name": "stdout",
@@ -894,7 +898,7 @@
    ],
    "source": [
     "# Inference FP32 model (OpenVINO IR)\n",
-    "!benchmark_app -m $ir_model_xml -shape [1,128],[1,128],[1,128] -d device.value -api sync"
+    "!benchmark_app -m $ir_model_xml -shape [1,128],[1,128],[1,128] -d {device.value} -api sync"
    ]
   },
   {
@@ -932,7 +936,7 @@
    ],
    "source": [
     "# Inference INT8 model (OpenVINO IR)\n",
-    "! benchmark_app -m $compressed_model_xml -shape [1,128],[1,128],[1,128] -d device.value -api sync"
+    "! benchmark_app -m $compressed_model_xml -shape [1,128],[1,128],[1,128] -d {device.value} -api sync"
    ]
   }
  ],

diff --git a/...tent-consistency-models-image-generation/latent-consistency-models-image-generation.ipynb b/...tent-consistency-models-image-generation/latent-consistency-models-image-generation.ipynb
@@ -565,14 +565,6 @@
     "                return_tensors=\"pt\",\n",
     "            )\n",
     "            text_input_ids = text_inputs.input_ids\n",
-    "            untruncated_ids = self.tokenizer(prompt, padding=\"longest\", return_tensors=\"pt\").input_ids\n",
-    "\n",
-    "            if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):\n",
-    "                removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1])\n",
-    "                logger.warning(\n",
-    "                    \"The following part of your input was truncated because CLIP can only handle sequences up to\"\n",
-    "                    f\" {self.tokenizer.model_max_length} tokens: {removed_text}\"\n",
-    "                )\n",
     "\n",
     "            prompt_embeds = self.text_encoder(text_input_ids, share_inputs=True, share_outputs=True)\n",
     "            prompt_embeds = torch.from_numpy(prompt_embeds[0])\n",

diff --git a/notebooks/latent-consistency-models-image-generation/lcm-lora-controlnet.ipynb b/notebooks/latent-consistency-models-image-generation/lcm-lora-controlnet.ipynb
@@ -1485,7 +1485,9 @@
    "cell_type": "code",
    "execution_count": 19,
    "id": "3596d45c",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 200": "subset_size = 10"}
+   },
    "outputs": [
     {
      "data": {

diff --git a/notebooks/llava-next-multimodal-chatbot/llava-next-multimodal-chatbot.ipynb b/notebooks/llava-next-multimodal-chatbot/llava-next-multimodal-chatbot.ipynb
@@ -719,7 +719,9 @@
    "cell_type": "code",
    "execution_count": 13,
    "id": "0c8e4bd0-2107-498d-8326-8de43746e817",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"def prepare_dataset(opt_init_steps=50, max_train_samples=1000):": "def prepare_dataset(opt_init_steps=10, max_train_samples=50)"}
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/named-entity-recognition/named-entity-recognition.ipynb b/notebooks/named-entity-recognition/named-entity-recognition.ipynb
@@ -145,7 +145,9 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"num_samples=100,": "num_samples=10,"}
+   },
    "outputs": [
     {
      "name": "stderr",

diff --git a/notebooks/qrcode-monster/qrcode-monster.ipynb b/notebooks/qrcode-monster/qrcode-monster.ipynb
@@ -1159,7 +1159,9 @@
   {
    "cell_type": "code",
    "execution_count": 40,
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 200": "subset_size = 4"}
+   },
    "outputs": [
     {
      "data": {

diff --git a/notebooks/sdxl-turbo/sdxl-turbo.ipynb b/notebooks/sdxl-turbo/sdxl-turbo.ipynb
@@ -599,7 +599,7 @@
    "cell_type": "code",
    "execution_count": 16,
    "id": "6b62f498",
-   "metadata": {},
+   "metadata": {"test_replace": {"subset_size=200": "subset_size=10"}},
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/segment-anything/segment-anything.ipynb b/notebooks/segment-anything/segment-anything.ipynb
@@ -1901,7 +1901,9 @@
    "cell_type": "code",
    "execution_count": 44,
    "id": "d961679b",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size=128": "subset_size=10"}
+   },
    "outputs": [
     {
      "name": "stderr",

diff --git a/notebooks/siglip-zero-shot-image-classification/siglip-zero-shot-image-classification.ipynb b/notebooks/siglip-zero-shot-image-classification/siglip-zero-shot-image-classification.ipynb
@@ -432,7 +432,9 @@
   {
    "cell_type": "code",
    "execution_count": 11,
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"def prepare_dataset(opt_init_steps=300, max_train_samples=1000):": "def prepare_dataset(opt_init_steps=4, max_train_samples=40):"}
+   },
    "outputs": [],
    "source": [
     "import torch\n",

diff --git a/notebooks/stable-diffusion-keras-cv/stable-diffusion-keras-cv.ipynb b/notebooks/stable-diffusion-keras-cv/stable-diffusion-keras-cv.ipynb
@@ -55,7 +55,7 @@
     "%pip install -q \"tensorflow-macos>=2.15; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version > '3.8'\" # macOS M1 and M2\n",
     "%pip install -q \"tensorflow>=2.15; sys_platform == 'darwin' and platform_machine != 'arm64' and python_version > '3.8'\" # macOS x86\n",
     "%pip install -q \"tensorflow>=2.15; sys_platform != 'darwin' and python_version > '3.8'\"\n",
-    "%pip install -q keras-cv tf_keras numpy \"openvino>=2024.1.0\" \"gradio>=4.19\" datasets \"nncf>=2.9.0\"\n",
+    "%pip install -q keras-cv tf_keras numpy \"openvino>=2024.1.0\" \"gradio>=4.19\" datasets \"nncf>=2.10.0\"\n",
     "\n",
     "\n",
     "if platform.system() != \"Windows\":\n",
@@ -781,7 +781,12 @@
   {
    "cell_type": "code",
    "execution_count": 16,
-   "metadata": {},
+   "metadata": {
+   "test_replace": {
+       "subset_size = 200": "subset_size = 10",
+       "calibration_data = collect_calibration_data(ov_pipeline, calibration_dataset_size=subset_size)":"calibration_data = collect_calibration_data(ov_pipeline, calibration_dataset_size=subset_size, num_inference_steps=10)"
+    }
+   },
    "outputs": [
     {
      "name": "stderr",

diff --git a/notebooks/stable-diffusion-v2/stable-diffusion-v2-text-to-image.ipynb b/notebooks/stable-diffusion-v2/stable-diffusion-v2-text-to-image.ipynb
@@ -1174,6 +1174,10 @@
     "ExecuteTime": {
      "end_time": "2024-02-13T14:31:36.656460Z",
      "start_time": "2024-02-13T13:32:03.144519Z"
+    },
+    "test_replace": {
+    "calibration_dataset_size = 300": "calibration_dataset_size = 10",
+    "num_inference_steps=50)": "num_inference_steps=10)"
     }
    },
    "outputs": [

diff --git a/notebooks/stable-diffusion-xl/segmind-vegart.ipynb b/notebooks/stable-diffusion-xl/segmind-vegart.ipynb
@@ -544,7 +544,7 @@
    "cell_type": "code",
    "execution_count": 13,
    "id": "029cec2e-492c-4682-b9c4-07d772bb5985",
-   "metadata": {},
+   "metadata": {"test_replace": {"subset_size=200": "subset_size=10"}},
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",

diff --git a/notebooks/wuerstchen-image-generation/wuerstchen-image-generation.ipynb b/notebooks/wuerstchen-image-generation/wuerstchen-image-generation.ipynb
@@ -954,7 +954,9 @@
    "cell_type": "code",
    "execution_count": 34,
    "id": "7344f5fa",
-   "metadata": {},
+   "metadata": {
+   "test_replace": {"subset_size = 300": "subset_size = 5"}
+   },
    "outputs": [],
    "source": [
     "%%skip not $to_quantize.value\n",