Add support of ImageNet21K models for accuracy tests (#78)

openvinotoolkit · Oct 2, 2024 · 19e2e8f · 19e2e8f
1 parent 3a58970
commit 19e2e8f
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 8 deletions.
diff --git a/docs/source/user-guide.md b/docs/source/user-guide.md
@@ -394,7 +394,7 @@ Target layer is the part of the model graph where XAI branch will be inserted (a
 
 All supported methods are gradient-free, which suits deployment framework settings (e.g. OpenVINO™), where the model is in optimized or compiled representation.
 
-## Methods performance-accuracy comparison
+### Methods performance-accuracy comparison
 
 The table below compares accuracy and performace of different models and explain methods (learn more about [Quality Metrics](#measure-quality-metrics-of-saliency-maps)).
 
@@ -404,16 +404,18 @@ Metrics were measured on a 10% random subset of the [ILSVRC 2012](https://www.im
 |:---------------------------:|:------------:|:--------------:|:---------------------------------:|---|:-------------:|---|:---------:|:--------:|---|:--------:|:---------:|:----------:|:------------:|
 | deit - tiny   (transformer) |   White box  | VIT ReciproCAM |                 1*                |   |    **89.9**   |   |    22.4   |  **4.5** |   |   70.4   |    88.9   |  **38.1**  |     34.3     |
 |                             |              | Activation map |                 1                 |   |      56.6     |   |    7.8    |    7.0   |   |   46.9   |    74.0   |    53.7    |     65.4     |
-|                             |   Black Box  |      AISE      |                 60                |   |      73.9     |   |    15.9   |    8.9   |   |   66.6   |    73.9   |    44.3    |     26.0     |
+|                             |   Black Box**  |      AISE      |                 60                |   |      73.9     |   |    15.9   |    8.9   |   |   66.6   |    73.9   |    44.3    |     26.0     |
 |                             |              |      RISE      |                2000               |   |      85.5     |   |  **23.2** |    5.8   |   | **74.8** |  **92.5** |    42.3    |   **16.6**   |
 |                             |              |                |                                   |   |               |   |           |          |   |          |           |            |              |
 |           resnet18          |   White box  |   ReciproCAM   |                 1*                |   |    **89.5**   |   |    33.9   |  **5.9** |   | **77.3** |    91.1   |    30.2    |     25.9     |
 |                             |              | Activation map |                 1                 |   |      87.0     |   |  **36.3** |   10.5   |   |   74.4   |  **97.9** |  **25.2**  |     40.2     |
-|                             |   Black Box  |      AISE      |                 60                |   |      72.0     |   |    22.5   |   12.4   |   |   67.4   |    69.3   |    44.5    |     16.9     |
+|                             |   Black Box**  |      AISE      |                 60                |   |      72.0     |   |    22.5   |   12.4   |   |   67.4   |    69.3   |    44.5    |     16.9     |
 |                             |              |      RISE      |                2000               |   |      87.0     |   |    34.6   |    7.1   |   |   77.1   |    93.0   |    42.0    |    **8.3**   |
 
 \* Recipro-CAM re-infers part of the graph (usually neck + head or last transformer block) H*W times, where HxW is the feature map size of the target layer.
 
+\*\* For Black Box Methods preset = `SPEED`
+
 
 ### White-Box methods
 

diff --git a/tests/perf/test_accuracy.py b/tests/perf/test_accuracy.py
@@ -62,7 +62,6 @@
 ]
 
 TEST_MODELS = IMAGENET_MODELS + VOC_MODELS + TRANSFORMER_MODELS
-IMAGENET_LABELS = get_imagenet_labels()
 EXPLAIN_METHODS = [Method.RECIPROCAM, Method.AISE, Method.RISE, Method.ACTIVATIONMAP]
 
 
@@ -100,8 +99,9 @@ def setup_model(self, data_dir, model_name):
             return model, None
 
         elif model_name in IMAGENET_MODELS + TRANSFORMER_MODELS:
-            self.dataset_label_list = IMAGENET_LABELS
             _, model_cfg = convert_timm_to_ir(model_name, data_dir, self.supported_num_classes)
+            version = "1k" if model_cfg["num_classes"] == 1000 else "21k"
+            self.dataset_label_list = get_imagenet_labels(version)
             ir_path = data_dir / "timm_models" / "converted_models" / model_name / "model_fp32.xml"
             model = ov.Core().read_model(ir_path)
             return model, model_cfg
@@ -157,7 +157,7 @@ def setup_explainer(self, model, explain_method):
     def setup(self, fxt_data_root, fxt_output_root, fxt_dataset_parameters):
         self.data_dir = fxt_data_root
         self.output_dir = fxt_output_root
-        self.supported_num_classes = {1000: 1000}
+        self.supported_num_classes = {1000: 1000, 21841: 21841, 21843: 21843}
 
         self.setup_dataset(fxt_dataset_parameters)
         self.dataset_name = self.dataset_type.value

diff --git a/tests/unit/explainer/test_explanation_utils.py b/tests/unit/explainer/test_explanation_utils.py
@@ -84,8 +84,13 @@ def test_is_bhwc_layout():
     assert is_bhwc_layout(np.empty((1, 3, 224, 224))) == False
 
 
-def get_imagenet_labels(file_path="imagenet_2012.txt"):
-    url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt"
+def get_imagenet_labels(version="1k"):
+    if version == "1k":
+        url = "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/datasets/imagenet/imagenet_2012.txt"
+        file_path = "imagenet_2012.txt"
+    elif version == "21k":
+        url = "https://storage.googleapis.com/bit_models/imagenet21k_wordnet_ids.txt"
+        file_path = "imagenet21k_wordnet_ids.txt"
 
     if not os.path.exists(file_path):
         response = requests.get(url)