Rope testing in pre-commit (openvinotoolkit#25943)

### Details:
 - add precommit tests to check if RoPE fusion works

### Tickets:
 - 146982

---------

Co-authored-by: Ivan Tikhonov <ivan.tikhonov@intel.com>

.github/workflows/job_pytorch_models_tests.yml

@@ -160,6 +160,15 @@ jobs:
           TEST_DEVICE: CPU
           USE_SYSTEM_CACHE: False
 
+      - name: RoPE Test
+        if: ${{ inputs.model_scope == 'precommit' }}
+        run: |
+          export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH
+          python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_transformations.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_rope_tests.html --self-contained-html -v --tb=short -n 2
+        env:
+          TEST_DEVICE: CPU
+          USE_SYSTEM_CACHE: False
+
       - name: StatefulToStateless Test
         if: ${{ inputs.model_scope == 'precommit' }}
         run: |

tests/model_hub_tests/models_hub_common/utils.py

@@ -11,39 +11,45 @@
 from models_hub_common.constants import test_device
 
 
-def get_models_list(file_name: str):
-    models = []
-    with open(file_name) as f:
-        for model_info in f:
+def parse_list_file(file_name: str):
+    with open(file_name, 'r') as f_in:
+        for model_info in f_in:
+            if not model_info:
+                continue
             model_info = model_info.strip()
             # skip comment in model scope file
-            if model_info.startswith('#'):
+            if not model_info or model_info.startswith('#'):
                 continue
-            mark = None
-            reason = None
-            assert len(model_info.split(',')) == 2 or len(model_info.split(',')) == 4, \
-                "Incorrect model info `{}`. It must contain either 2 or 4 fields.".format(model_info)
-            if len(model_info.split(',')) == 2:
-                model_name, model_link = model_info.split(',')
-            elif len(model_info.split(',')) == 4:
-                model_name, model_link, mark, reason = model_info.split(',')
-            models.append((model_name, model_link, mark, reason))
+            yield model_info.split(',')
 
-    return models
 
+def get_models_list(file_name: str):
+    models = []
+    for line_items in parse_list_file(file_name):
+        if len(line_items) == 2:
+            model_name, model_link = line_items
+            models.append((model_name, model_link, None, None))
+        elif len(line_items) == 4:
+            model_name, model_link, mark, reason = line_items
+            models.append((model_name, model_link, mark, reason))
+        elif len(line_items) > 4:
+            model_name, model_link, mark, reason = line_items[:4]
+            if not mark:
+                mark = None
+            if not reason:
+                reason = None
+            other = line_items[4:]
+            transformations = [item[8:] for item in other if item.startswith('ts_name:')]
+            layers = [item[6:] for item in other if item.startswith('layer:')]
+            models.append((model_name, model_link, mark, reason, transformations, layers))
+        else:
+            items = ','.join(line_items)
+            assert False, \
+                f'Incorrect model info fields {items}. It must contain either 2 or 4 or more than 4 fields.'
+    return models
 
-def get_skipped_model_links(filename: str):
-    links = set()
-    if not os.path.exists(filename):
-        return links
-    with open(filename) as f:
-        for model_info in f:
-            model_info = model_info.strip()
-            if not model_info:
-                continue
-            model_name, model_link = model_info.split(',')
-            links.add(model_link)
-    return links
+def get_skipped_model_links(file_name: str):
+    return {line_items[1] for line_items in parse_list_file(file_name)}
 
 
 def get_models_list_not_skipped(model_list_file: str, skip_list_file: str):

tests/model_hub_tests/transformation_tests/models/transformations-models-precommit

@@ -0,0 +1,27 @@
+hf-internal-testing/tiny-random-LlamaForCausalLM,https://huggingface.co/trl-internal-testing/tiny-random-LlamaForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-GPTJForCausalLM,https://huggingface.co/trl-internal-testing/tiny-random-GPTJForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-GPTNeoXForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-GPTNeoXForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-MistralForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-MistralForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-CodeGenForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-CodeGenForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/Mixtral-tiny,https://huggingface.co/hf-internal-testing/Mixtral-tiny,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-Starcoder2ForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-Starcoder2ForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-PhiForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-PhiForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-StableLmForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-StableLmForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-PersimmonForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-PersimmonForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+hf-internal-testing/tiny-random-FalconForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-FalconForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-xverse,https://huggingface.co/katuni4ka/tiny-random-xverse,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-qwen,https://huggingface.co/katuni4ka/tiny-random-qwen,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-aquilachat,https://huggingface.co/katuni4ka/tiny-random-aquilachat,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-aquila2,https://huggingface.co/katuni4ka/tiny-random-aquila2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-qwen1.5-moe,https://huggingface.co/katuni4ka/tiny-random-qwen1.5-moe,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-olmo-hf,https://huggingface.co/katuni4ka/tiny-random-olmo-hf,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-baichuan2,https://huggingface.co/katuni4ka/tiny-random-baichuan2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-internlm,https://huggingface.co/katuni4ka/tiny-random-internlm,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-internlm2,https://huggingface.co/katuni4ka/tiny-random-internlm2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-minicpm,https://huggingface.co/katuni4ka/tiny-random-minicpm,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-falcon-40b,https://huggingface.co/katuni4ka/tiny-random-falcon-40b,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+katuni4ka/tiny-random-dbrx,https://huggingface.co/katuni4ka/tiny-random-dbrx,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+fxmarty/tiny-random-GemmaForCausalLM,https://huggingface.co/fxmarty/tiny-random-GemmaForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+fxmarty/tiny-dummy-qwen2,https://huggingface.co/fxmarty/tiny-dummy-qwen2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+fxmarty/really-tiny-falcon-testing,https://huggingface.co/fxmarty/really-tiny-falcon-testing,,,ts_name:ov::pass::RoPEFusion,layer:RoPE
+Xenova/tiny-random-Phi3ForCausalLM,https://huggingface.co/Xenova/tiny-random-Phi3ForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE

tests/model_hub_tests/transformation_tests/test_transformations.py

@@ -0,0 +1,98 @@
+# Copyright (C) 2018-2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+
+from optimum.intel import OVModelForCausalLM
+import models_hub_common.utils as utils
+import pytest
+import os
+import openvino as ov
+import tempfile
+from collections import deque
+import csv
+
+
+def parse_transformations_log(file_name):
+    with open(file_name, 'r') as f_in:
+        csv_reader = csv.reader(f_in, delimiter=';')
+        for line in csv_reader:
+            if line[0] != 't':
+                continue
+            ts_name = line[1]
+            status = line[4]
+            yield ts_name, status
+
+
+def check_transformations(file_name, ts_names):
+    not_executed = deque(ts_names)
+    false_executed = set()
+    for ts_name, status in parse_transformations_log(file_name):
+        if not not_executed and not false_executed:
+            break
+        for _ in range(len(not_executed)):
+            not_executed_name = not_executed.popleft()
+            if not_executed_name == ts_name:
+                if status != '1':
+                    false_executed.add(not_executed_name)
+                break
+            not_executed.append(not_executed_name)
+        if ts_name in false_executed and status == '1':
+            false_executed.remove(ts_name)
+    if not_executed or false_executed:
+        fail_text = ''
+        if not_executed:
+            not_executed_names = ','.join(not_executed)
+            fail_text = f'transformation(s) {not_executed_names} not executed'
+        if false_executed:
+            false_executed_names = ','.join(false_executed)
+            if bool(fail_text):
+                fail_text += '; '
+            fail_text += f'transformation(s) {false_executed_names} executed with false return'
+        pytest.fail(fail_text)
+
+
+def check_operations(actual_layer_types, expected_layer_types):
+    not_found = [layer for layer in expected_layer_types if layer not in actual_layer_types]
+    if not_found:
+        names = ','.join(not_found)
+        pytest.fail(f'operation(s) {names} not found in compiled model')
+
+
+class EnvVar:
+    def __init__(self, env_vars):
+        self.__vars = env_vars
+
+    def __enter__(self):
+        for name, value in self.__vars.items():
+            os.environ[name] = value
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for name in self.__vars:
+            del os.environ[name]
+
+
+def run_test(model_id, ie_device, ts_names, expected_layer_types):
+    model = OVModelForCausalLM.from_pretrained(model_id, export=True, trust_remote_code=True)
+
+    with tempfile.NamedTemporaryFile(delete=True) as temp_file, \
+            EnvVar({'OV_ENABLE_PROFILE_PASS': temp_file.name}):
+        core = ov.Core()
+        compiled = core.compile_model(model.model, ie_device)
+        check_transformations(temp_file.name, ts_names)
+        ov_model = compiled.get_runtime_model()
+        type_names = {op.get_rt_info()["layerType"] for op in ov_model.get_ordered_ops()}
+        check_operations(type_names, expected_layer_types)
+
+
+@pytest.mark.precommit
+@pytest.mark.parametrize("model_name, model_link, mark, reason, ts_names, layer_types", utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "transformations-models-precommit")))
+def test_transformations_precommit(tmp_path, model_name, model_link, mark, reason, ie_device, ts_names, layer_types):
+    assert mark is None or mark == 'skip' or mark == 'xfail', \
+        "Incorrect test case: {}, {}".format(model_name, model_link)
+    if mark == 'skip':
+        pytest.skip(reason)
+    elif mark == 'xfail':
+        pytest.xfail(reason)
+    if not ts_names and not layer_types:
+        return
+    run_test(model_name, ie_device, ts_names, layer_types)