diff --git a/.github/workflows/build-test-a770.yml b/.github/workflows/build-test-a770.yml index 709b864cb0..4161a8bdf6 100644 --- a/.github/workflows/build-test-a770.yml +++ b/.github/workflows/build-test-a770.yml @@ -37,6 +37,11 @@ on: description: Custom run name type: string default: "Build and test on A770" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false + schedule: # Every midnight PST (UTC-8) - cron: "0 8 * * *" @@ -61,3 +66,4 @@ jobs: ignore_errors: ${{ inputs.ignore_errors || false }} skip_list: ${{ inputs.skip_list }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ inputs.enable_unskip || false }} diff --git a/.github/workflows/build-test-no-ipex.yml b/.github/workflows/build-test-no-ipex.yml index 859b5e4382..9e267e8775 100644 --- a/.github/workflows/build-test-no-ipex.yml +++ b/.github/workflows/build-test-no-ipex.yml @@ -28,6 +28,10 @@ on: description: Custom run name type: string default: "Build and test upstream pytorch with no IPEX" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false schedule: # Every midnight PST (UTC-8) - cron: "0 8 * * *" @@ -52,3 +56,4 @@ jobs: ignore_errors: ${{ inputs.ignore_errors || false }} skip_list: ${{ inputs.skip_list }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ inputs.enable_unskip || false }} diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml index b360e934a1..665d519d44 100644 --- a/.github/workflows/build-test-reusable.yml +++ b/.github/workflows/build-test-reusable.yml @@ -49,12 +49,17 @@ on: description: Build LLVM type: boolean default: false + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false permissions: read-all env: TRITON_DISABLE_LINE_INFO: 1 TORCH_XPU_OPS_COMMIT: 39522db63ce045f52c9d61a286018c266cd00479 + TEST_UNSKIP: ${{ inputs.enable_unskip }} jobs: integration-tests: @@ -177,7 +182,7 @@ jobs: run: | export DEBUG=1 cd python - pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select + pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout pip install --no-build-isolation '.[build,tests,tutorials]' pip install git+https://github.com/kwasd/pytest-capturewarnings-ng.git@v1.2.0 diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index baf8950179..6f47bbb1b2 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -37,6 +37,11 @@ on: description: Custom run name type: string default: "Build and test" + enable_unskip: + description: Ignore pytest.skip + type: boolean + default: false + pull_request: branches: - llvm-target @@ -114,3 +119,4 @@ jobs: ignore_errors: ${{ inputs.ignore_errors || false }} skip_list: ${{ inputs.skip_list }} run_name: ${{ inputs.run_name }} + enable_unskip: ${{ inputs.enable_unskip || false }} diff --git a/python/test/conftest.py b/python/test/conftest.py new file mode 100644 index 0000000000..9f9ca90723 --- /dev/null +++ b/python/test/conftest.py @@ -0,0 +1,25 @@ +# content of conftest.py +import os +import pytest + + +def pytest_configure(config): + if os.getenv('TEST_UNSKIP') == 'true': + # define a function that do nothing + def unskip(reason=None, allow_module_level=False): + pass + + # save the original 'pytest.skip' to config._skip_f + config._skip_f = pytest.skip + # replace 'pytest.skip' with 'pass' call + pytest.skip = unskip + else: + pass + + +def pytest_unconfigure(config): + if os.getenv('TEST_UNSKIP') == 'true': + # restore 'pytest.skip' + pytest.skip = config._skip_f + else: + pass diff --git a/scripts/pass_rate.py b/scripts/pass_rate.py index 66f7835c39..f393fbaee6 100644 --- a/scripts/pass_rate.py +++ b/scripts/pass_rate.py @@ -76,9 +76,14 @@ def parse_report(report_path: pathlib.Path) -> ReportStats: except FileNotFoundError: pass stats.fixme += len(testsuite_fixme_tests) - deselected = get_deselected(report_path) - stats.skipped += deselected - stats.total += deselected + + test_unskip = os.getenv('TEST_UNSKIP') + if test_unskip not in ('true', 'false'): + raise ValueError('Error: please set TEST_UNSKIP true or false') + if test_unskip == 'false': + deselected = get_deselected(report_path) + stats.skipped += deselected + stats.total += deselected stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed return stats diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index 560ede0595..582cc89323 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -34,10 +34,17 @@ pytest() { mkdir -p "$CURRENT_SKIPLIST_DIR" # skip comments in the skiplist sed -e '/^#/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - pytest_extra_args+=( - "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" - "--select-fail-on-missing" - ) + if [[ $TEST_UNSKIP = false ]]; then + pytest_extra_args+=( + "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" + "--select-fail-on-missing" + ) + else + pytest_extra_args+=( + "--timeout=500" + "--max-worker-restart=500" + ) + fi fi python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index bb207b2cbc..0e297c0b5f 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -14,9 +14,14 @@ TRITON_TEST_REPORTS=false TRITON_TEST_WARNING_REPORTS=false TRITON_TEST_IGNORE_ERRORS=false SKIP_DEPS=false +TEST_UNSKIP=false ARGS= for arg in "$@"; do case $arg in + --unskip) + TEST_UNSKIP=true + shift + ;; --microbench) TEST_MICRO_BENCHMARKS=true shift @@ -85,7 +90,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd) -python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select setuptools==69.5.1 +python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout setuptools==69.5.1 if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then python3 -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0 @@ -141,10 +146,12 @@ run_core_tests() { echo "****** Running Triton Core tests ******" echo "***************************************************" CORE_TEST_DIR=$TRITON_PROJ/python/test/unit + if [ ! -d "${CORE_TEST_DIR}" ]; then echo "Not found '${CORE_TEST_DIR}'. Build Triton please" ; exit 3 fi cd ${CORE_TEST_DIR} + export TEST_UNSKIP TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=language \ pytest -vvv -n 8 --device xpu language/ --ignore=language/test_line_info.py --ignore=language/test_subprocess.py @@ -166,6 +173,8 @@ run_regression_tests() { echo "****** Running Triton Regression tests ******" echo "***************************************************" REGRESSION_TEST_DIR=$TRITON_PROJ/python/test/regression + export TEST_UNSKIP + if [ ! -d "${REGRESSION_TEST_DIR}" ]; then echo "Not found '${REGRESSION_TEST_DIR}'. Build Triton please" ; exit 3 fi