From 8af346c294a3e331637167d9b2c4f7048c716cf0 Mon Sep 17 00:00:00 2001 From: AshburnLee Date: Thu, 13 Jun 2024 06:13:02 +0000 Subject: [PATCH 1/3] [test-triton.sh]Add an option to unskip all pytest.skip() calls --- scripts/pytest-utils.sh | 7 ++++-- scripts/replace_skip_calls.py | 45 +++++++++++++++++++++++++++++++++++ scripts/test-triton.sh | 24 ++++++++++++++++--- 3 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 scripts/replace_skip_calls.py diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index 560ede0595..684624fd52 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -13,7 +13,7 @@ TRITON_TEST_SKIPLIST_DIR="$(cd "$TRITON_TEST_SKIPLIST_DIR" && pwd)" CURRENT_SKIPLIST_DIR="$SCRIPTS_DIR/skiplist/current" pytest() { - pytest_extra_args=() + pytest_extra_args=("--timeout=15") # need pytest-timeout installed if [[ -v TRITON_TEST_SUITE && $TRITON_TEST_REPORTS = true ]]; then mkdir -p "$TRITON_TEST_REPORTS_DIR" @@ -36,8 +36,11 @@ pytest() { sed -e '/^#/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" pytest_extra_args+=( "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - "--select-fail-on-missing" ) + # this argument results in execution error when enable unskip test + if [ "$TEST_UNSKIP" = false ]; then + pytest_extra_args+=("--select-fail-on-missing") + fi fi python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS diff --git a/scripts/replace_skip_calls.py b/scripts/replace_skip_calls.py new file mode 100644 index 0000000000..34bd9ff0d9 --- /dev/null +++ b/scripts/replace_skip_calls.py @@ -0,0 +1,45 @@ +"""Replace all 'pytest.skip()' to 'pass' calls.""" +import ast +import os + +class ReplaceToPassransformer(ast.NodeTransformer): + """Replace 'pytest.skip' calls to 'pass' """ + def visit_Call(self, node: ast.Call): + if isinstance(node.func, ast.Attribute): + if node.func.attr == 'skip' and isinstance(node.func.value, ast.Name) \ + and node.func.value.id == 'pytest': + return ast.Pass() + self.generic_visit(node) + return node + +def rewrite_test_scripts(src_dir: str, dest_dir: str): + """Traverse all .py files under src_dir and do the replacement, put all under the dest_dir""" + if os.path.exists(src_dir) and os.path.isdir(src_dir): + for dirpath, sub_dirname, filename in os.walk(src_dir): + # print(f"dirpath: {dirpath}") + # print(f"sub_dirname: {sub_dirname}") + # print(f"filename: {filename}") + for filename in filename: + if filename.endswith('.py'): + file_path = os.path.join(dirpath, filename) + with open(file_path, 'r') as file: + source_code = file.read() + + tree = ast.parse(source_code) + modified_tree = ReplaceToPassransformer().visit(tree) + modified_source_code = ast.unparse(modified_tree) + + relative_path = os.path.relpath(file_path, src_dir) + target_path = os.path.join(dest_dir, relative_path) + os.makedirs(os.path.dirname(target_path), exist_ok=True) + + with open(target_path, 'w') as file: + file.write(modified_source_code) + else: + print(f"{src_dir} not exist") + +if __name__ == '__main__': + triton_dir = os.getenv('TRITON_PROJ') + src_dir = os.path.join(triton_dir, "/python/test/") if triton_dir else None + dest_dir = os.path.join(triton_dir, "/tmp/python/test/") if triton_dir else None + rewrite_test_scripts(src_dir, dest_dir) diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index bb207b2cbc..a703d85a85 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -14,9 +14,14 @@ TRITON_TEST_REPORTS=false TRITON_TEST_WARNING_REPORTS=false TRITON_TEST_IGNORE_ERRORS=false SKIP_DEPS=false +TEST_UNSKIP=false ARGS= for arg in "$@"; do case $arg in + --unskip) + TEST_UNSKIP=true + shift + ;; --microbench) TEST_MICRO_BENCHMARKS=true shift @@ -85,7 +90,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd) -python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select setuptools==69.5.1 +python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout setuptools==69.5.1 if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python3 -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0 @@ -140,7 +145,12 @@ run_core_tests() { echo "***************************************************" echo "****** Running Triton Core tests ******" echo "***************************************************" - CORE_TEST_DIR=$TRITON_PROJ/python/test/unit + if [ "$TEST_UNSKIP" = true ]; then + CORE_TEST_DIR=$TRITON_PROJ/tmp/python/test/unit + else + CORE_TEST_DIR=$TRITON_PROJ/python/test/unit + fi + if [ ! -d "${CORE_TEST_DIR}" ]; then echo "Not found '${CORE_TEST_DIR}'. Build Triton please" ; exit 3 fi @@ -165,7 +175,11 @@ run_regression_tests() { echo "***************************************************" echo "****** Running Triton Regression tests ******" echo "***************************************************" - REGRESSION_TEST_DIR=$TRITON_PROJ/python/test/regression + if [ "$TEST_UNSKIP" = true ]; then + REGRESSION_TEST_DIR=$TRITON_PROJ/tmp/python/test/regression + else + REGRESSION_TEST_DIR=$TRITON_PROJ/python/test/regression + fi if [ ! -d "${REGRESSION_TEST_DIR}" ]; then echo "Not found '${REGRESSION_TEST_DIR}'. Build Triton please" ; exit 3 fi @@ -200,6 +214,10 @@ run_tutorial_tests() { } test_triton() { + # generate dir ${TRITON_PROJ}/tmp/python/test/ + if [ "$TEST_UNSKIP" = true ]; then + python ${SCRIPTS_DIR}/replace_skip_calls.py + fi if [ "$TEST_UNIT" = true ]; then run_unit_tests fi From 041545a909f02aaef7e46273bc000f245541be78 Mon Sep 17 00:00:00 2001 From: AshburnLee Date: Fri, 14 Jun 2024 07:51:51 +0000 Subject: [PATCH 2/3] version 2 --- python/test/conftest.py | 25 +++++++++++++++++++ scripts/pytest-utils.sh | 7 ++---- scripts/replace_skip_calls.py | 45 ----------------------------------- scripts/test-triton.sh | 21 +++++----------- 4 files changed, 33 insertions(+), 65 deletions(-) create mode 100644 python/test/conftest.py delete mode 100644 scripts/replace_skip_calls.py diff --git a/python/test/conftest.py b/python/test/conftest.py new file mode 100644 index 0000000000..8befe72450 --- /dev/null +++ b/python/test/conftest.py @@ -0,0 +1,25 @@ +# content of conftest.py +import os +import pytest + + +def pytest_configure(config): + if os.getenv('TEST_UNSKIP') == 'false': + pass + else: + # define a function that do nothing + def unskip(reason=None, allow_module_level=False): + pass + + # save the original 'pytest.skip' to config._skip_f + config._skip_f = pytest.skip + # replace 'pytest.skip' with 'pass' call + pytest.skip = unskip + + +def pytest_unconfigure(config): + if os.getenv('TEST_UNSKIP') == 'false': + pass + else: + # restore 'pytest.skip' + pytest.skip = config._skip_f diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index 684624fd52..560ede0595 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -13,7 +13,7 @@ TRITON_TEST_SKIPLIST_DIR="$(cd "$TRITON_TEST_SKIPLIST_DIR" && pwd)" CURRENT_SKIPLIST_DIR="$SCRIPTS_DIR/skiplist/current" pytest() { - pytest_extra_args=("--timeout=15") # need pytest-timeout installed + pytest_extra_args=() if [[ -v TRITON_TEST_SUITE && $TRITON_TEST_REPORTS = true ]]; then mkdir -p "$TRITON_TEST_REPORTS_DIR" @@ -36,11 +36,8 @@ pytest() { sed -e '/^#/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" pytest_extra_args+=( "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" + "--select-fail-on-missing" ) - # this argument results in execution error when enable unskip test - if [ "$TEST_UNSKIP" = false ]; then - pytest_extra_args+=("--select-fail-on-missing") - fi fi python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS diff --git a/scripts/replace_skip_calls.py b/scripts/replace_skip_calls.py deleted file mode 100644 index 34bd9ff0d9..0000000000 --- a/scripts/replace_skip_calls.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Replace all 'pytest.skip()' to 'pass' calls.""" -import ast -import os - -class ReplaceToPassransformer(ast.NodeTransformer): - """Replace 'pytest.skip' calls to 'pass' """ - def visit_Call(self, node: ast.Call): - if isinstance(node.func, ast.Attribute): - if node.func.attr == 'skip' and isinstance(node.func.value, ast.Name) \ - and node.func.value.id == 'pytest': - return ast.Pass() - self.generic_visit(node) - return node - -def rewrite_test_scripts(src_dir: str, dest_dir: str): - """Traverse all .py files under src_dir and do the replacement, put all under the dest_dir""" - if os.path.exists(src_dir) and os.path.isdir(src_dir): - for dirpath, sub_dirname, filename in os.walk(src_dir): - # print(f"dirpath: {dirpath}") - # print(f"sub_dirname: {sub_dirname}") - # print(f"filename: {filename}") - for filename in filename: - if filename.endswith('.py'): - file_path = os.path.join(dirpath, filename) - with open(file_path, 'r') as file: - source_code = file.read() - - tree = ast.parse(source_code) - modified_tree = ReplaceToPassransformer().visit(tree) - modified_source_code = ast.unparse(modified_tree) - - relative_path = os.path.relpath(file_path, src_dir) - target_path = os.path.join(dest_dir, relative_path) - os.makedirs(os.path.dirname(target_path), exist_ok=True) - - with open(target_path, 'w') as file: - file.write(modified_source_code) - else: - print(f"{src_dir} not exist") - -if __name__ == '__main__': - triton_dir = os.getenv('TRITON_PROJ') - src_dir = os.path.join(triton_dir, "/python/test/") if triton_dir else None - dest_dir = os.path.join(triton_dir, "/tmp/python/test/") if triton_dir else None - rewrite_test_scripts(src_dir, dest_dir) diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index a703d85a85..4900a6beaa 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -90,7 +90,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd) -python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout setuptools==69.5.1 +python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select setuptools==69.5.1 if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python3 -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0 @@ -145,16 +145,13 @@ run_core_tests() { echo "***************************************************" echo "****** Running Triton Core tests ******" echo "***************************************************" - if [ "$TEST_UNSKIP" = true ]; then - CORE_TEST_DIR=$TRITON_PROJ/tmp/python/test/unit - else - CORE_TEST_DIR=$TRITON_PROJ/python/test/unit - fi + CORE_TEST_DIR=$TRITON_PROJ/python/test/unit if [ ! -d "${CORE_TEST_DIR}" ]; then echo "Not found '${CORE_TEST_DIR}'. Build Triton please" ; exit 3 fi cd ${CORE_TEST_DIR} + export TEST_UNSKIP TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=language \ pytest -vvv -n 8 --device xpu language/ --ignore=language/test_line_info.py --ignore=language/test_subprocess.py @@ -175,11 +172,9 @@ run_regression_tests() { echo "***************************************************" echo "****** Running Triton Regression tests ******" echo "***************************************************" - if [ "$TEST_UNSKIP" = true ]; then - REGRESSION_TEST_DIR=$TRITON_PROJ/tmp/python/test/regression - else - REGRESSION_TEST_DIR=$TRITON_PROJ/python/test/regression - fi + REGRESSION_TEST_DIR=$TRITON_PROJ/python/test/regression + export TEST_UNSKIP + if [ ! -d "${REGRESSION_TEST_DIR}" ]; then echo "Not found '${REGRESSION_TEST_DIR}'. Build Triton please" ; exit 3 fi @@ -214,10 +209,6 @@ run_tutorial_tests() { } test_triton() { - # generate dir ${TRITON_PROJ}/tmp/python/test/ - if [ "$TEST_UNSKIP" = true ]; then - python ${SCRIPTS_DIR}/replace_skip_calls.py - fi if [ "$TEST_UNIT" = true ]; then run_unit_tests fi From 7fbf8f4127b3ec41c6d05621356d5cbee5167eab Mon Sep 17 00:00:00 2001 From: AshburnLee Date: Fri, 14 Jun 2024 09:24:57 +0000 Subject: [PATCH 3/3] supported on CI and support timeout, support pass_rate, restart crash --- .github/workflows/build-test-a770.yml | 6 ++++++ .github/workflows/build-test-no-ipex.yml | 5 +++++ .github/workflows/build-test-reusable.yml | 7 ++++++- .github/workflows/build-test.yml | 6 ++++++ python/test/conftest.py | 12 ++++++------ scripts/pass_rate.py | 11 ++++++++--- scripts/pytest-utils.sh | 15 +++++++++++---- scripts/test-triton.sh | 2 +- 8 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-test-a770.yml b/.github/workflows/build-test-a770.yml index 709b864cb0..4161a8bdf6 100644 --- a/.github/workflows/build-test-a770.yml +++ b/.github/workflows/build-test-a770.yml @@ -37,6 +37,11 @@ on: description: Custom run name type: string default: "Build and test on A770" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false + schedule: # Every midnight PST (UTC-8) - cron: "0 8 * * *" @@ -61,3 +66,4 @@ jobs: ignore_errors: ${{ inputs.ignore_errors || false }} skip_list: ${{ inputs.skip_list }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ inputs.enable_unskip || false }} diff --git a/.github/workflows/build-test-no-ipex.yml b/.github/workflows/build-test-no-ipex.yml index 859b5e4382..9e267e8775 100644 --- a/.github/workflows/build-test-no-ipex.yml +++ b/.github/workflows/build-test-no-ipex.yml @@ -28,6 +28,10 @@ on: description: Custom run name type: string default: "Build and test upstream pytorch with no IPEX" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false schedule: # Every midnight PST (UTC-8) - cron: "0 8 * * *" @@ -52,3 +56,4 @@ jobs: ignore_errors: ${{ inputs.ignore_errors || false }} skip_list: ${{ inputs.skip_list }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ inputs.enable_unskip || false }} diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml index b360e934a1..665d519d44 100644 --- a/.github/workflows/build-test-reusable.yml +++ b/.github/workflows/build-test-reusable.yml @@ -49,12 +49,17 @@ on: description: Build LLVM type: boolean default: false + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false permissions: read-all env: TRITON_DISABLE_LINE_INFO: 1 TORCH_XPU_OPS_COMMIT: 39522db63ce045f52c9d61a286018c266cd00479 + TEST_UNSKIP: ${{ inputs.enable_unskip }} jobs: integration-tests: @@ -177,7 +182,7 @@ jobs: run: | export DEBUG=1 cd python - pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select + pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout pip install --no-build-isolation '.[build,tests,tutorials]' pip install git+https://github.com/kwasd/pytest-capturewarnings-ng.git@v1.2.0 diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index baf8950179..6f47bbb1b2 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -37,6 +37,11 @@ on: description: Custom run name type: string default: "Build and test" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false + pull_request: branches: - llvm-target @@ -114,3 +119,4 @@ jobs: ignore_errors: ${{ inputs.ignore_errors || false }} skip_list: ${{ inputs.skip_list }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ inputs.enable_unskip || false }} diff --git a/python/test/conftest.py b/python/test/conftest.py index 8befe72450..9f9ca90723 100644 --- a/python/test/conftest.py +++ b/python/test/conftest.py @@ -4,9 +4,7 @@ def pytest_configure(config): - if os.getenv('TEST_UNSKIP') == 'false': - pass - else: + if os.getenv('TEST_UNSKIP') == 'true': # define a function that do nothing def unskip(reason=None, allow_module_level=False): pass @@ -15,11 +13,13 @@ def unskip(reason=None, allow_module_level=False): config._skip_f = pytest.skip # replace 'pytest.skip' with 'pass' call pytest.skip = unskip + else: + pass def pytest_unconfigure(config): - if os.getenv('TEST_UNSKIP') == 'false': - pass - else: + if os.getenv('TEST_UNSKIP') == 'true': # restore 'pytest.skip' pytest.skip = config._skip_f + else: + pass diff --git a/scripts/pass_rate.py b/scripts/pass_rate.py index 66f7835c39..f393fbaee6 100644 --- a/scripts/pass_rate.py +++ b/scripts/pass_rate.py @@ -76,9 +76,14 @@ def parse_report(report_path: pathlib.Path) -> ReportStats: except FileNotFoundError: pass stats.fixme += len(testsuite_fixme_tests) - deselected = get_deselected(report_path) - stats.skipped += deselected - stats.total += deselected + + test_unskip = os.getenv('TEST_UNSKIP') + if test_unskip not in ('true', 'false'): + raise ValueError('Error: please set TEST_UNSKIP true or false') + if test_unskip == 'false': + deselected = get_deselected(report_path) + stats.skipped += deselected + stats.total += deselected stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed return stats diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index 560ede0595..582cc89323 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -34,10 +34,17 @@ pytest() { mkdir -p "$CURRENT_SKIPLIST_DIR" # skip comments in the skiplist sed -e '/^#/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - pytest_extra_args+=( - "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - "--select-fail-on-missing" - ) + if [[ $TEST_UNSKIP = false ]]; then + pytest_extra_args+=( + "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" + "--select-fail-on-missing" + ) + else + pytest_extra_args+=( + "--timeout=500" + "--max-worker-restart=500" + ) + fi fi python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index 4900a6beaa..0e297c0b5f 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -90,7 +90,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd) -python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select setuptools==69.5.1 +python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout setuptools==69.5.1 if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python3 -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0