From f7b54b66175269832c9fc0433f57680bbcd9a555 Mon Sep 17 00:00:00 2001 From: AshburnLee Date: Fri, 14 Jun 2024 09:24:57 +0000 Subject: [PATCH] supported on CI and support timeout, support pass_rate --- .github/workflows/build-test-a770.yml | 6 ++++++ .github/workflows/build-test-no-ipex.yml | 5 +++++ .github/workflows/build-test-reusable.yml | 7 ++++++- .github/workflows/build-test.yml | 6 ++++++ scripts/pass_rate.py | 11 ++++++++--- scripts/pytest-utils.sh | 12 ++++++++---- scripts/test-triton.sh | 2 +- 7 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test-a770.yml b/.github/workflows/build-test-a770.yml index 0f735e33f9..931b1798ca 100644 --- a/.github/workflows/build-test-a770.yml +++ b/.github/workflows/build-test-a770.yml @@ -33,6 +33,11 @@ on: description: Custom run name type: string default: "Build and test on A770" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false + schedule: # Every midnight PST (UTC-8) - cron: "0 8 * * *" @@ -56,3 +61,4 @@ jobs: upload_test_reports: ${{ inputs.upload_test_reports }} ignore_errors: ${{ inputs.ignore_errors }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ (github.event_name == 'workflow_dispatch' && inputs.enable_unskip) || 'false' }} diff --git a/.github/workflows/build-test-no-ipex.yml b/.github/workflows/build-test-no-ipex.yml index 01c7a6e121..15553e270e 100644 --- a/.github/workflows/build-test-no-ipex.yml +++ b/.github/workflows/build-test-no-ipex.yml @@ -24,6 +24,10 @@ on: description: Custom run name type: string default: "Build and test upstream pytorch with no IPEX" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false schedule: # Every midnight PST (UTC-8) - cron: "0 8 * * *" @@ -47,3 +51,4 @@ jobs: upload_test_reports: ${{ inputs.upload_test_reports }} ignore_errors: ${{ inputs.ignore_errors }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ (github.event_name == 'workflow_dispatch' && inputs.enable_unskip) || 'false' }} diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml index b351c9eba2..8ce8dae528 100644 --- a/.github/workflows/build-test-reusable.yml +++ b/.github/workflows/build-test-reusable.yml @@ -41,12 +41,17 @@ on: description: Custom run name type: string default: "Build and test" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false permissions: read-all env: TRITON_DISABLE_LINE_INFO: 1 TORCH_XPU_OPS_COMMIT: 39522db63ce045f52c9d61a286018c266cd00479 + TEST_UNSKIP: ${{ inputs.enable_unskip }} jobs: print_inputs: @@ -118,7 +123,7 @@ jobs: run: | export DEBUG=1 cd python - pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select + pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout pip install --no-build-isolation '.[build,tests,tutorials]' pip install git+https://github.com/kwasd/pytest-capturewarnings-ng.git@v1.2.0 diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index da3632ef84..42afd04e4b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -33,6 +33,11 @@ on: description: Custom run name type: string default: "Build and test" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false + pull_request: branches: - llvm-target @@ -109,3 +114,4 @@ jobs: upload_test_reports: ${{ inputs.upload_test_reports || false }} ignore_errors: ${{ inputs.ignore_errors || false }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ (github.event_name == 'workflow_dispatch' && inputs.enable_unskip) || 'false' }} diff --git a/scripts/pass_rate.py b/scripts/pass_rate.py index 66f7835c39..5fdac3145b 100644 --- a/scripts/pass_rate.py +++ b/scripts/pass_rate.py @@ -76,9 +76,14 @@ def parse_report(report_path: pathlib.Path) -> ReportStats: except FileNotFoundError: pass stats.fixme += len(testsuite_fixme_tests) - deselected = get_deselected(report_path) - stats.skipped += deselected - stats.total += deselected + + test_unskip = os.getenv('TEST_UNSKIP') + if test_unskip is None or test_unskip not in ("true", "false"): + raise ValueError("Error: please set TEST_UNSKIP true or false") + if test_unskip == 'false': + deselected = get_deselected(report_path) + stats.skipped += deselected + stats.total += deselected stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed return stats diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index 560ede0595..dcbd3f2df4 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -34,10 +34,14 @@ pytest() { mkdir -p "$CURRENT_SKIPLIST_DIR" # skip comments in the skiplist sed -e '/^#/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - pytest_extra_args+=( - "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - "--select-fail-on-missing" - ) + if [[ $TEST_UNSKIP = false ]]; then + pytest_extra_args+=( + "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" + "--select-fail-on-missing" + ) + else + pytest_extra_args+=("--timeout=500") + fi fi python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index 8ce383dc44..8de88ae628 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -90,7 +90,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd) -python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select setuptools==69.5.1 +python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout setuptools==69.5.1 if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python3 -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0