From 76dce7027aec3bcd801aa0b6921e35f37725b5d0 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Thu, 2 Jan 2025 14:08:46 +0300 Subject: [PATCH 1/3] init --- ydb/library/yql/tests/sql/dq_file.make | 4 ++-- ydb/library/yql/tests/sql/dq_file.py | 12 ++++++------ ydb/library/yql/tests/sql/dq_file/part0/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part1/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part10/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part11/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part12/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part13/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part14/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part15/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part16/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part17/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part18/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part19/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part2/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part3/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part4/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part5/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part6/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part7/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part8/test.py | 4 ++-- ydb/library/yql/tests/sql/dq_file/part9/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file.make | 4 ++-- ydb/library/yql/tests/sql/hybrid_file.py | 9 +++++---- ydb/library/yql/tests/sql/hybrid_file/part0/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part1/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part10/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part2/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part3/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part4/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part5/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part6/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part7/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part8/test.py | 4 ++-- ydb/library/yql/tests/sql/hybrid_file/part9/test.py | 4 ++-- 35 files changed, 77 insertions(+), 76 deletions(-) diff --git a/ydb/library/yql/tests/sql/dq_file.make b/ydb/library/yql/tests/sql/dq_file.make index 96649376d776..3f3317ae7cf6 100644 --- a/ydb/library/yql/tests/sql/dq_file.make +++ b/ydb/library/yql/tests/sql/dq_file.make @@ -1,4 +1,4 @@ -PY2TEST() +PY3TEST() TEST_SRCS( test.py @@ -31,7 +31,7 @@ DEPENDS( ) DATA( arcadia/ydb/library/yql/tests/sql # python files - arcadia/yql/essentials/tests/sql/suites + arcadia/yt/yql/tests/sql/suites arcadia/yql/essentials/mount arcadia/yql/essentials/cfg/tests ) diff --git a/ydb/library/yql/tests/sql/dq_file.py b/ydb/library/yql/tests/sql/dq_file.py index ce5cf966ba26..50bc4af464bd 100644 --- a/ydb/library/yql/tests/sql/dq_file.py +++ b/ydb/library/yql/tests/sql/dq_file.py @@ -8,18 +8,18 @@ dump_table_yson, get_gateway_cfg_suffix, do_custom_query_check, normalize_result, \ stable_result_file, stable_table_file, is_with_final_result_issues, log -from test_utils import get_config, DATA_PATH +from test_utils import get_config from test_file_common import run_file, run_file_no_cache ASTDIFF_PATH = yql_binary_path('yql/essentials/tools/astdiff/astdiff') DQRUN_PATH = yql_binary_path('ydb/library/yql/tools/dqrun/dqrun') - +DATA_PATH = yatest.common.source_path('yt/yql/tests/sql/suites') def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if get_gateway_cfg_suffix() != '' and what != 'Results': pytest.skip('non-trivial gateways.conf') - config = get_config(suite, case, cfg) + config = get_config(suite, case, cfg, data_path=DATA_PATH) program_sql = os.path.join(DATA_PATH, suite, '%s.sql' % case) with codecs.open(program_sql, encoding='utf-8') as program_file_descr: @@ -41,17 +41,17 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if is_with_final_result_issues(config): extra_args += ["--with-final-issues"] - (res, tables_res) = run_file('dq', suite, case, cfg, config, yql_http_file_server, DQRUN_PATH, extra_args=extra_args) + (res, tables_res) = run_file('dq', suite, case, cfg, config, yql_http_file_server, DQRUN_PATH, extra_args=extra_args, data_path=DATA_PATH) if what == 'Results' or force_blocks: if not xfail: if force_blocks: yqlrun_res, yqlrun_tables_res = run_file_no_cache('dq', suite, case, cfg, config, yql_http_file_server, DQRUN_PATH, \ - extra_args=["--emulate-yt"], force_blocks=True) + extra_args=["--emulate-yt"], force_blocks=True, data_path=DATA_PATH) dq_result_name = 'Scalar' yqlrun_result_name = 'Block' else: - yqlrun_res, yqlrun_tables_res = run_file_no_cache('yt', suite, case, cfg, config, yql_http_file_server) + yqlrun_res, yqlrun_tables_res = run_file_no_cache('yt', suite, case, cfg, config, yql_http_file_server, data_path=DATA_PATH) dq_result_name = 'DQFILE' yqlrun_result_name = 'YQLRUN' diff --git a/ydb/library/yql/tests/sql/dq_file/part0/test.py b/ydb/library/yql/tests/sql/dq_file/part0/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part0/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part0/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part1/test.py b/ydb/library/yql/tests/sql/dq_file/part1/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part1/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part1/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part10/test.py b/ydb/library/yql/tests/sql/dq_file/part10/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part10/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part10/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part11/test.py b/ydb/library/yql/tests/sql/dq_file/part11/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part11/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part11/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part12/test.py b/ydb/library/yql/tests/sql/dq_file/part12/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part12/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part12/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part13/test.py b/ydb/library/yql/tests/sql/dq_file/part13/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part13/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part13/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part14/test.py b/ydb/library/yql/tests/sql/dq_file/part14/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part14/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part14/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part15/test.py b/ydb/library/yql/tests/sql/dq_file/part15/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part15/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part15/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part16/test.py b/ydb/library/yql/tests/sql/dq_file/part16/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part16/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part16/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part17/test.py b/ydb/library/yql/tests/sql/dq_file/part17/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part17/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part17/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part18/test.py b/ydb/library/yql/tests/sql/dq_file/part18/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part18/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part18/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part19/test.py b/ydb/library/yql/tests/sql/dq_file/part19/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part19/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part19/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part2/test.py b/ydb/library/yql/tests/sql/dq_file/part2/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part2/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part2/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part3/test.py b/ydb/library/yql/tests/sql/dq_file/part3/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part3/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part3/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part4/test.py b/ydb/library/yql/tests/sql/dq_file/part4/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part4/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part4/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part5/test.py b/ydb/library/yql/tests/sql/dq_file/part5/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part5/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part5/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part6/test.py b/ydb/library/yql/tests/sql/dq_file/part6/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part6/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part6/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part7/test.py b/ydb/library/yql/tests/sql/dq_file/part7/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part7/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part7/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part8/test.py b/ydb/library/yql/tests/sql/dq_file/part8/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part8/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part8/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/dq_file/part9/test.py b/ydb/library/yql/tests/sql/dq_file/part9/test.py index 29d1e76d2e66..fe9d8cd409e1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part9/test.py +++ b/ydb/library/yql/tests/sql/dq_file/part9/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from dq_file import run_test +from dq_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results', 'ForceBlocks']) diff --git a/ydb/library/yql/tests/sql/hybrid_file.make b/ydb/library/yql/tests/sql/hybrid_file.make index 08124b9399f0..6656742950ce 100644 --- a/ydb/library/yql/tests/sql/hybrid_file.make +++ b/ydb/library/yql/tests/sql/hybrid_file.make @@ -1,4 +1,4 @@ -PY2TEST() +PY3TEST() TEST_SRCS( test.py @@ -31,7 +31,7 @@ DEPENDS( ) DATA( arcadia/ydb/library/yql/tests/sql # python files - arcadia/yql/essentials/tests/sql/suites + arcadia/yt/yql/tests/sql/suites arcadia/yql/essentials/mount arcadia/yql/essentials/cfg/tests ) diff --git a/ydb/library/yql/tests/sql/hybrid_file.py b/ydb/library/yql/tests/sql/hybrid_file.py index 30f6e49d767a..f3e3cce08b28 100644 --- a/ydb/library/yql/tests/sql/hybrid_file.py +++ b/ydb/library/yql/tests/sql/hybrid_file.py @@ -9,20 +9,21 @@ get_gateway_cfg_suffix, normalize_result, stable_result_file, stable_table_file, \ dump_table_yson, normalize_source_code_path -from test_utils import get_config, DATA_PATH +from test_utils import get_config from test_file_common import run_file, run_file_no_cache ASTDIFF_PATH = yql_binary_path('yql/essentials/tools/astdiff/astdiff') DQRUN_PATH = yql_binary_path('ydb/library/yql/tools/dqrun/dqrun') +DATA_PATH = yatest.common.source_path('yt/yql/tests/sql/suites') def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if get_gateway_cfg_suffix() != '' and what != 'Results': pytest.skip('non-trivial gateways.conf') - config = get_config(suite, case, cfg) + config = get_config(suite, case, cfg, data_path=DATA_PATH) xfail = is_xfail(config) - (res, tables_res) = run_file('hybrid', suite, case, cfg, config, yql_http_file_server, DQRUN_PATH, extra_args=["--emulate-yt", "--no-force-dq"]) + (res, tables_res) = run_file('hybrid', suite, case, cfg, config, yql_http_file_server, DQRUN_PATH, extra_args=["--emulate-yt", "--no-force-dq"], data_path=DATA_PATH) if what == 'Results': if not xfail: @@ -31,7 +32,7 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): sql_query = program_file_descr.read() # yqlrun run - yqlrun_res, yqlrun_tables_res = run_file_no_cache('yt', suite, case, cfg, config, yql_http_file_server) + yqlrun_res, yqlrun_tables_res = run_file_no_cache('yt', suite, case, cfg, config, yql_http_file_server, data_path=DATA_PATH) hybrid_result_name = 'HYBRIDFILE' yqlrun_result_name = 'YQLRUN' diff --git a/ydb/library/yql/tests/sql/hybrid_file/part0/test.py b/ydb/library/yql/tests/sql/hybrid_file/part0/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part0/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part0/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part1/test.py b/ydb/library/yql/tests/sql/hybrid_file/part1/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part1/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part1/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part10/test.py b/ydb/library/yql/tests/sql/hybrid_file/part10/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part10/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part10/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part2/test.py b/ydb/library/yql/tests/sql/hybrid_file/part2/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part2/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part2/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part3/test.py b/ydb/library/yql/tests/sql/hybrid_file/part3/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part3/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part3/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part4/test.py b/ydb/library/yql/tests/sql/hybrid_file/part4/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part4/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part4/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part5/test.py b/ydb/library/yql/tests/sql/hybrid_file/part5/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part5/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part5/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part6/test.py b/ydb/library/yql/tests/sql/hybrid_file/part6/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part6/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part6/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part7/test.py b/ydb/library/yql/tests/sql/hybrid_file/part7/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part7/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part7/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part8/test.py b/ydb/library/yql/tests/sql/hybrid_file/part8/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part8/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part8/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) diff --git a/ydb/library/yql/tests/sql/hybrid_file/part9/test.py b/ydb/library/yql/tests/sql/hybrid_file/part9/test.py index 0c8b1d70adc6..857e652c8170 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part9/test.py +++ b/ydb/library/yql/tests/sql/hybrid_file/part9/test.py @@ -1,14 +1,14 @@ import pytest import yatest -from hybrid_file import run_test +from hybrid_file import run_test, DATA_PATH from test_utils import pytest_generate_tests_for_part from yql_utils import pytest_get_current_part def pytest_generate_tests(metafunc): current_part, part_count = pytest_get_current_part(yatest.common.source_path(__file__)) - return pytest_generate_tests_for_part(metafunc, current_part, part_count) + return pytest_generate_tests_for_part(metafunc, current_part, part_count, data_path=DATA_PATH) @pytest.mark.parametrize('what', ['Results']) From 884978258817b8a0a05606f3a50348db4a7a7570 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Thu, 2 Jan 2025 15:28:50 +0300 Subject: [PATCH 2/3] py2 --- ydb/library/yql/tests/sql/dq_file.make | 2 +- ydb/library/yql/tests/sql/hybrid_file.make | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/library/yql/tests/sql/dq_file.make b/ydb/library/yql/tests/sql/dq_file.make index 3f3317ae7cf6..210bfeba8494 100644 --- a/ydb/library/yql/tests/sql/dq_file.make +++ b/ydb/library/yql/tests/sql/dq_file.make @@ -1,4 +1,4 @@ -PY3TEST() +PY2TEST() TEST_SRCS( test.py diff --git a/ydb/library/yql/tests/sql/hybrid_file.make b/ydb/library/yql/tests/sql/hybrid_file.make index 6656742950ce..76b3f298ebbb 100644 --- a/ydb/library/yql/tests/sql/hybrid_file.make +++ b/ydb/library/yql/tests/sql/hybrid_file.make @@ -1,4 +1,4 @@ -PY3TEST() +PY2TEST() TEST_SRCS( test.py From 9fb45b85b6a80bc11c49a3cac926f8a450fbbe5f Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Sat, 4 Jan 2025 01:14:55 +0300 Subject: [PATCH 3/3] stabilize --- ydb/library/yql/tests/sql/dq_file.py | 38 +++++++++++++++++++++--- ydb/library/yql/tests/sql/hybrid_file.py | 38 +++++++++++++++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/ydb/library/yql/tests/sql/dq_file.py b/ydb/library/yql/tests/sql/dq_file.py index 50bc4af464bd..fe901abf34c3 100644 --- a/ydb/library/yql/tests/sql/dq_file.py +++ b/ydb/library/yql/tests/sql/dq_file.py @@ -15,9 +15,37 @@ DQRUN_PATH = yql_binary_path('ydb/library/yql/tools/dqrun/dqrun') DATA_PATH = yatest.common.source_path('yt/yql/tests/sql/suites') + +# TODO move to yql_utils +def is_sorted_table(table): + import cyson + assert table.attr is not None + for column in cyson.loads(table.attr)[b'schema']: + if b'sort_order' in column: + return True + return False + + +# TODO move to yql_utils +def is_unordered_result(res): + import cyson + path = res.results_file + assert os.path.exists(path) + with open(path, 'rb') as f: + res = f.read() + res = cyson.loads(res) + for r in res: + for data in r[b'Write']: + if b'Unordered' in data: + return True + return False + + def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if get_gateway_cfg_suffix() != '' and what != 'Results': pytest.skip('non-trivial gateways.conf') + if (suite, case) in [('result_types','containers')]: + pytest.skip('TODO make sorted/stable dicts') config = get_config(suite, case, cfg, data_path=DATA_PATH) @@ -60,8 +88,8 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if os.path.exists(yqlrun_res.results_file): assert os.path.exists(res.results_file) - dq_res_yson = normalize_result(stable_result_file(res), False) - yqlrun_res_yson = normalize_result(stable_result_file(yqlrun_res), False) + dq_res_yson = normalize_result(stable_result_file(res), is_unordered_result(res)) + yqlrun_res_yson = normalize_result(stable_result_file(yqlrun_res), is_unordered_result(yqlrun_res)) # Compare results assert dq_res_yson == yqlrun_res_yson, 'RESULTS_DIFFER\n' \ @@ -73,8 +101,10 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if os.path.exists(yqlrun_tables_res[table].file): assert os.path.exists(tables_res[table].file) - yqlrun_table_yson = dump_table_yson(stable_table_file(yqlrun_tables_res[table]), False) - dq_table_yson = dump_table_yson(stable_table_file(tables_res[table]), False) + yqlrun_table_yson = dump_table_yson(stable_table_file(yqlrun_tables_res[table]), \ + not is_sorted_table(yqlrun_tables_res[table])) + dq_table_yson = dump_table_yson(stable_table_file(tables_res[table]), \ + not is_sorted_table(tables_res[table])) assert yqlrun_table_yson == dq_table_yson, \ 'OUT_TABLE_DIFFER: %(table)s\n' \ diff --git a/ydb/library/yql/tests/sql/hybrid_file.py b/ydb/library/yql/tests/sql/hybrid_file.py index f3e3cce08b28..11cd142bfe8c 100644 --- a/ydb/library/yql/tests/sql/hybrid_file.py +++ b/ydb/library/yql/tests/sql/hybrid_file.py @@ -16,9 +16,37 @@ DQRUN_PATH = yql_binary_path('ydb/library/yql/tools/dqrun/dqrun') DATA_PATH = yatest.common.source_path('yt/yql/tests/sql/suites') + +# TODO move to yql_utils +def is_sorted_table(table): + import cyson + assert table.attr is not None + for column in cyson.loads(table.attr)[b'schema']: + if b'sort_order' in column: + return True + return False + + +# TODO move to yql_utils +def is_unordered_result(res): + import cyson + path = res.results_file + assert os.path.exists(path) + with open(path, 'rb') as f: + res = f.read() + res = cyson.loads(res) + for r in res: + for data in r[b'Write']: + if b'Unordered' in data: + return True + return False + + def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if get_gateway_cfg_suffix() != '' and what != 'Results': pytest.skip('non-trivial gateways.conf') + if (suite, case) in [('result_types','containers')]: + pytest.skip('TODO make sorted/stable dicts') config = get_config(suite, case, cfg, data_path=DATA_PATH) xfail = is_xfail(config) @@ -39,8 +67,8 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if os.path.exists(yqlrun_res.results_file): assert os.path.exists(res.results_file) - hybrid_res_yson = normalize_result(stable_result_file(res), False) - yqlrun_res_yson = normalize_result(stable_result_file(yqlrun_res), False) + hybrid_res_yson = normalize_result(stable_result_file(res), is_unordered_result(res)) + yqlrun_res_yson = normalize_result(stable_result_file(yqlrun_res), is_unordered_result(yqlrun_res)) # Compare results assert hybrid_res_yson == yqlrun_res_yson, 'RESULTS_DIFFER\n' \ @@ -52,8 +80,10 @@ def run_test(suite, case, cfg, tmpdir, what, yql_http_file_server): if os.path.exists(yqlrun_tables_res[table].file): assert os.path.exists(tables_res[table].file) - yqlrun_table_yson = dump_table_yson(stable_table_file(yqlrun_tables_res[table]), False) - hybrid_table_yson = dump_table_yson(stable_table_file(tables_res[table]), False) + yqlrun_table_yson = dump_table_yson(stable_table_file(yqlrun_tables_res[table]),\ + not is_sorted_table(yqlrun_tables_res[table])) + hybrid_table_yson = dump_table_yson(stable_table_file(tables_res[table]),\ + not is_sorted_table(tables_res[table])) assert yqlrun_table_yson == hybrid_table_yson, \ 'OUT_TABLE_DIFFER: %(table)s\n' \