From 2b6bcf182dda006b626931c05d8a8289cbaae610 Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Fri, 16 Aug 2024 17:01:45 +0200 Subject: [PATCH] Rope testing in pre-commit (#25943) ### Details: - add precommit tests to check if RoPE fusion works ### Tickets: - 146982 --------- Co-authored-by: Ivan Tikhonov --- .../workflows/job_pytorch_models_tests.yml | 9 ++ .../models_hub_common/utils.py | 60 +++++++----- .../models/transformations-models-precommit | 27 +++++ .../test_transformations.py | 98 +++++++++++++++++++ 4 files changed, 167 insertions(+), 27 deletions(-) create mode 100644 tests/model_hub_tests/transformation_tests/models/transformations-models-precommit create mode 100644 tests/model_hub_tests/transformation_tests/test_transformations.py diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 381b0a51eb49df..43e1d46319b089 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -160,6 +160,15 @@ jobs: TEST_DEVICE: CPU USE_SYSTEM_CACHE: False + - name: RoPE Test + if: ${{ inputs.model_scope == 'precommit' }} + run: | + export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_transformations.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_rope_tests.html --self-contained-html -v --tb=short -n 2 + env: + TEST_DEVICE: CPU + USE_SYSTEM_CACHE: False + - name: StatefulToStateless Test if: ${{ inputs.model_scope == 'precommit' }} run: | diff --git a/tests/model_hub_tests/models_hub_common/utils.py b/tests/model_hub_tests/models_hub_common/utils.py index b014c8dc64c2d8..428802c58463fc 100644 --- a/tests/model_hub_tests/models_hub_common/utils.py +++ b/tests/model_hub_tests/models_hub_common/utils.py @@ -11,39 +11,45 @@ from models_hub_common.constants import test_device -def get_models_list(file_name: str): - models = [] - with open(file_name) as f: - for model_info in f: +def parse_list_file(file_name: str): + with open(file_name, 'r') as f_in: + for model_info in f_in: + if not model_info: + continue model_info = model_info.strip() # skip comment in model scope file - if model_info.startswith('#'): + if not model_info or model_info.startswith('#'): continue - mark = None - reason = None - assert len(model_info.split(',')) == 2 or len(model_info.split(',')) == 4, \ - "Incorrect model info `{}`. It must contain either 2 or 4 fields.".format(model_info) - if len(model_info.split(',')) == 2: - model_name, model_link = model_info.split(',') - elif len(model_info.split(',')) == 4: - model_name, model_link, mark, reason = model_info.split(',') - models.append((model_name, model_link, mark, reason)) + yield model_info.split(',') - return models +def get_models_list(file_name: str): + models = [] + for line_items in parse_list_file(file_name): + if len(line_items) == 2: + model_name, model_link = line_items + models.append((model_name, model_link, None, None)) + elif len(line_items) == 4: + model_name, model_link, mark, reason = line_items + models.append((model_name, model_link, mark, reason)) + elif len(line_items) > 4: + model_name, model_link, mark, reason = line_items[:4] + if not mark: + mark = None + if not reason: + reason = None + other = line_items[4:] + transformations = [item[8:] for item in other if item.startswith('ts_name:')] + layers = [item[6:] for item in other if item.startswith('layer:')] + models.append((model_name, model_link, mark, reason, transformations, layers)) + else: + items = ','.join(line_items) + assert False, \ + f'Incorrect model info fields {items}. It must contain either 2 or 4 or more than 4 fields.' + return models -def get_skipped_model_links(filename: str): - links = set() - if not os.path.exists(filename): - return links - with open(filename) as f: - for model_info in f: - model_info = model_info.strip() - if not model_info: - continue - model_name, model_link = model_info.split(',') - links.add(model_link) - return links +def get_skipped_model_links(file_name: str): + return {line_items[1] for line_items in parse_list_file(file_name)} def get_models_list_not_skipped(model_list_file: str, skip_list_file: str): diff --git a/tests/model_hub_tests/transformation_tests/models/transformations-models-precommit b/tests/model_hub_tests/transformation_tests/models/transformations-models-precommit new file mode 100644 index 00000000000000..b3068ffbb52840 --- /dev/null +++ b/tests/model_hub_tests/transformation_tests/models/transformations-models-precommit @@ -0,0 +1,27 @@ +hf-internal-testing/tiny-random-LlamaForCausalLM,https://huggingface.co/trl-internal-testing/tiny-random-LlamaForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-GPTJForCausalLM,https://huggingface.co/trl-internal-testing/tiny-random-GPTJForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-GPTNeoXForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-GPTNeoXForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-MistralForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-MistralForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-CodeGenForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-CodeGenForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/Mixtral-tiny,https://huggingface.co/hf-internal-testing/Mixtral-tiny,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-Starcoder2ForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-Starcoder2ForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-PhiForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-PhiForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-StableLmForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-StableLmForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-PersimmonForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-PersimmonForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +hf-internal-testing/tiny-random-FalconForCausalLM,https://huggingface.co/hf-internal-testing/tiny-random-FalconForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-xverse,https://huggingface.co/katuni4ka/tiny-random-xverse,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-qwen,https://huggingface.co/katuni4ka/tiny-random-qwen,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-aquilachat,https://huggingface.co/katuni4ka/tiny-random-aquilachat,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-aquila2,https://huggingface.co/katuni4ka/tiny-random-aquila2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-qwen1.5-moe,https://huggingface.co/katuni4ka/tiny-random-qwen1.5-moe,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-olmo-hf,https://huggingface.co/katuni4ka/tiny-random-olmo-hf,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-baichuan2,https://huggingface.co/katuni4ka/tiny-random-baichuan2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-internlm,https://huggingface.co/katuni4ka/tiny-random-internlm,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-internlm2,https://huggingface.co/katuni4ka/tiny-random-internlm2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-minicpm,https://huggingface.co/katuni4ka/tiny-random-minicpm,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-falcon-40b,https://huggingface.co/katuni4ka/tiny-random-falcon-40b,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +katuni4ka/tiny-random-dbrx,https://huggingface.co/katuni4ka/tiny-random-dbrx,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +fxmarty/tiny-random-GemmaForCausalLM,https://huggingface.co/fxmarty/tiny-random-GemmaForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +fxmarty/tiny-dummy-qwen2,https://huggingface.co/fxmarty/tiny-dummy-qwen2,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +fxmarty/really-tiny-falcon-testing,https://huggingface.co/fxmarty/really-tiny-falcon-testing,,,ts_name:ov::pass::RoPEFusion,layer:RoPE +Xenova/tiny-random-Phi3ForCausalLM,https://huggingface.co/Xenova/tiny-random-Phi3ForCausalLM,,,ts_name:ov::pass::RoPEFusion,layer:RoPE diff --git a/tests/model_hub_tests/transformation_tests/test_transformations.py b/tests/model_hub_tests/transformation_tests/test_transformations.py new file mode 100644 index 00000000000000..235fb2cf10b6f3 --- /dev/null +++ b/tests/model_hub_tests/transformation_tests/test_transformations.py @@ -0,0 +1,98 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +from optimum.intel import OVModelForCausalLM +import models_hub_common.utils as utils +import pytest +import os +import openvino as ov +import tempfile +from collections import deque +import csv + + +def parse_transformations_log(file_name): + with open(file_name, 'r') as f_in: + csv_reader = csv.reader(f_in, delimiter=';') + for line in csv_reader: + if line[0] != 't': + continue + ts_name = line[1] + status = line[4] + yield ts_name, status + + +def check_transformations(file_name, ts_names): + not_executed = deque(ts_names) + false_executed = set() + for ts_name, status in parse_transformations_log(file_name): + if not not_executed and not false_executed: + break + for _ in range(len(not_executed)): + not_executed_name = not_executed.popleft() + if not_executed_name == ts_name: + if status != '1': + false_executed.add(not_executed_name) + break + not_executed.append(not_executed_name) + if ts_name in false_executed and status == '1': + false_executed.remove(ts_name) + if not_executed or false_executed: + fail_text = '' + if not_executed: + not_executed_names = ','.join(not_executed) + fail_text = f'transformation(s) {not_executed_names} not executed' + if false_executed: + false_executed_names = ','.join(false_executed) + if bool(fail_text): + fail_text += '; ' + fail_text += f'transformation(s) {false_executed_names} executed with false return' + pytest.fail(fail_text) + + +def check_operations(actual_layer_types, expected_layer_types): + not_found = [layer for layer in expected_layer_types if layer not in actual_layer_types] + if not_found: + names = ','.join(not_found) + pytest.fail(f'operation(s) {names} not found in compiled model') + + +class EnvVar: + def __init__(self, env_vars): + self.__vars = env_vars + + def __enter__(self): + for name, value in self.__vars.items(): + os.environ[name] = value + + def __exit__(self, exc_type, exc_val, exc_tb): + for name in self.__vars: + del os.environ[name] + + +def run_test(model_id, ie_device, ts_names, expected_layer_types): + model = OVModelForCausalLM.from_pretrained(model_id, export=True, trust_remote_code=True) + + with tempfile.NamedTemporaryFile(delete=True) as temp_file, \ + EnvVar({'OV_ENABLE_PROFILE_PASS': temp_file.name}): + core = ov.Core() + compiled = core.compile_model(model.model, ie_device) + check_transformations(temp_file.name, ts_names) + ov_model = compiled.get_runtime_model() + type_names = {op.get_rt_info()["layerType"] for op in ov_model.get_ordered_ops()} + check_operations(type_names, expected_layer_types) + + +@pytest.mark.precommit +@pytest.mark.parametrize("model_name, model_link, mark, reason, ts_names, layer_types", utils.get_models_list(os.path.join(os.path.dirname(__file__), "models", "transformations-models-precommit"))) +def test_transformations_precommit(tmp_path, model_name, model_link, mark, reason, ie_device, ts_names, layer_types): + assert mark is None or mark == 'skip' or mark == 'xfail', \ + "Incorrect test case: {}, {}".format(model_name, model_link) + if mark == 'skip': + pytest.skip(reason) + elif mark == 'xfail': + pytest.xfail(reason) + if not ts_names and not layer_types: + return + run_test(model_name, ie_device, ts_names, layer_types)