[test-triton.sh] Add an option to unskip all pytest.skip() calls

.github/workflows/build-test-a770.yml

@@ -37,6 +37,11 @@ on:
         description: Custom run name
         type: string
         default: "Build and test on A770"
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
+
   schedule:
     # Every midnight PST (UTC-8)
     - cron: "0 8 * * *"
@@ -61,3 +66,4 @@ jobs:
       ignore_errors: ${{ inputs.ignore_errors || false }}
       skip_list: ${{ inputs.skip_list }}
       run_name: ${{ inputs.run_name }}
+      enable_unskip: ${{ inputs.enable_unskip || false }}

.github/workflows/build-test-no-ipex.yml

@@ -28,6 +28,10 @@ on:
         description: Custom run name
         type: string
         default: "Build and test upstream pytorch with no IPEX"
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
   schedule:
     # Every midnight PST (UTC-8)
     - cron: "0 8 * * *"
@@ -52,3 +56,4 @@ jobs:
       ignore_errors: ${{ inputs.ignore_errors || false }}
       skip_list: ${{ inputs.skip_list }}
       run_name: ${{ inputs.run_name }}
+      enable_unskip: ${{ inputs.enable_unskip || false }}

.github/workflows/build-test-reusable.yml

@@ -49,12 +49,17 @@ on:
         description: Build LLVM
         type: boolean
         default: false
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
 
 permissions: read-all
 
 env:
   TRITON_DISABLE_LINE_INFO: 1
   TORCH_XPU_OPS_COMMIT: 39522db63ce045f52c9d61a286018c266cd00479
+  TEST_UNSKIP: ${{ inputs.enable_unskip }}
 
 jobs:
   integration-tests:
@@ -177,7 +182,7 @@ jobs:
         run: |
           export DEBUG=1
           cd python
-          pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select
+          pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout
           pip install --no-build-isolation '.[build,tests,tutorials]'
           pip install git+https://github.com/kwasd/pytest-capturewarnings-ng.git@v1.2.0
 

.github/workflows/build-test.yml

@@ -37,6 +37,11 @@ on:
         description: Custom run name
         type: string
         default: "Build and test"
+      enable_unskip:
+        description: Ignore pytest.skip
+        type: boolean
+        default: false
+
   pull_request:
     branches:
       - llvm-target
@@ -114,3 +119,4 @@ jobs:
       ignore_errors: ${{ inputs.ignore_errors || false }}
       skip_list: ${{ inputs.skip_list }}
       run_name: ${{ inputs.run_name }}
+      enable_unskip: ${{ inputs.enable_unskip || false }}

python/test/conftest.py

@@ -0,0 +1,25 @@
+# content of conftest.py
+import os
+import pytest
+
+
+def pytest_configure(config):
+    if os.getenv('TEST_UNSKIP') == 'true':
+        # define a function that do nothing
+        def unskip(reason=None, allow_module_level=False):
+            pass
+
+        # save the original 'pytest.skip' to config._skip_f
+        config._skip_f = pytest.skip
+        # replace 'pytest.skip' with 'pass' call
+        pytest.skip = unskip
+    else:
+        pass
+
+
+def pytest_unconfigure(config):
+    if os.getenv('TEST_UNSKIP') == 'true':
+        # restore 'pytest.skip'
+        pytest.skip = config._skip_f
+    else:
+        pass

scripts/pass_rate.py

@@ -76,9 +76,14 @@ def parse_report(report_path: pathlib.Path) -> ReportStats:
         except FileNotFoundError:
             pass
         stats.fixme += len(testsuite_fixme_tests)
-    deselected = get_deselected(report_path)
-    stats.skipped += deselected
-    stats.total += deselected
+
+    test_unskip = os.getenv('TEST_UNSKIP')
+    if test_unskip not in ('true', 'false'):
+        raise ValueError('Error: please set TEST_UNSKIP true or false')
+    if test_unskip == 'false':
+        deselected = get_deselected(report_path)
+        stats.skipped += deselected
+        stats.total += deselected
     stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed
     return stats
 

scripts/pytest-utils.sh

@@ -34,10 +34,17 @@ pytest() {
         mkdir -p "$CURRENT_SKIPLIST_DIR"
         # skip comments in the skiplist
         sed -e '/^#/d' "$TRITON_TEST_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt" > "$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"
-        pytest_extra_args+=(
-            "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"
-            "--select-fail-on-missing"
-        )
+        if [[ $TEST_UNSKIP = false ]]; then
+            pytest_extra_args+=(
+                "--deselect-from-file=$CURRENT_SKIPLIST_DIR/$TRITON_TEST_SUITE.txt"
+                "--select-fail-on-missing"
+            )
+        else
+            pytest_extra_args+=(
+                "--timeout=500"
+                "--max-worker-restart=500"
+            )
+        fi
     fi
 
     python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS

scripts/test-triton.sh

@@ -14,9 +14,14 @@ TRITON_TEST_REPORTS=false
 TRITON_TEST_WARNING_REPORTS=false
 TRITON_TEST_IGNORE_ERRORS=false
 SKIP_DEPS=false
+TEST_UNSKIP=false
 ARGS=
 for arg in "$@"; do
   case $arg in
+    --unskip)
+      TEST_UNSKIP=true
+      shift
+      ;;
     --microbench)
       TEST_MICRO_BENCHMARKS=true
       shift
@@ -85,7 +90,7 @@ export TRITON_PROJ=$BASE/intel-xpu-backend-for-triton
 export TRITON_PROJ_BUILD=$TRITON_PROJ/python/build
 export SCRIPTS_DIR=$(cd $(dirname "$0") && pwd)
 
-python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select setuptools==69.5.1
+python3 -m pip install lit pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout setuptools==69.5.1
 
 if [ "$TRITON_TEST_WARNING_REPORTS" == true ]; then
     python3 -m pip install git+https://github.com/kwasd/pytest-capturewarnings-ng@v1.2.0
@@ -141,10 +146,12 @@ run_core_tests() {
   echo "******      Running Triton Core tests        ******"
   echo "***************************************************"
   CORE_TEST_DIR=$TRITON_PROJ/python/test/unit
+
   if [ ! -d "${CORE_TEST_DIR}" ]; then
     echo "Not found '${CORE_TEST_DIR}'. Build Triton please" ; exit 3
   fi
   cd ${CORE_TEST_DIR}
+  export TEST_UNSKIP
 
   TRITON_DISABLE_LINE_INFO=1 TRITON_TEST_SUITE=language \
   pytest -vvv -n 8 --device xpu language/ --ignore=language/test_line_info.py --ignore=language/test_subprocess.py
@@ -166,6 +173,8 @@ run_regression_tests() {
   echo "******   Running Triton Regression tests     ******"
   echo "***************************************************"
   REGRESSION_TEST_DIR=$TRITON_PROJ/python/test/regression
+  export TEST_UNSKIP
+
   if [ ! -d "${REGRESSION_TEST_DIR}" ]; then
     echo "Not found '${REGRESSION_TEST_DIR}'. Build Triton please" ; exit 3
   fi