Inputs for build-test workflow (#1078)

* `runner_label` - a custom label for runner * `upload_test_reports` - upload tests junit xml reports as job artifact Adding an option to ignore test failures by setting `TRITON_TEST_IGNORE_ERRORS` to true.
intel · May 9, 2024 · 5231b4d · 5231b4d
1 parent 6ef229a
commit 5231b4d
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 13 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -2,6 +2,15 @@ name: Build and test
 
 on:
   workflow_dispatch:
+    inputs:
+      runner_label:
+        description: Runner label, keep empty for default
+        type: string
+        default: ""
+      upload_test_reports:
+        description: Upload test reports
+        type: boolean
+        default: false
   pull_request:
     branches:
       - llvm-target
@@ -61,9 +70,7 @@ jobs:
   integration-tests:
     name: Integration tests
     runs-on:
-      - glados
-      - spr
-      - runner-0.0.13
+      - ${{ inputs.runner_label || 'runner-0.0.13' }}
     strategy:
       matrix:
         python: ${{ github.ref_name == 'llvm-target' && fromJson('["3.9", "3.10", "3.11"]') || fromJson('["3.9"]') }}
@@ -115,10 +122,10 @@ jobs:
 
       - name: Create directory for tests reports
         run: |
-          mkdir ~/reports
+          mkdir reports
           echo "TRITON_TEST_REPORTS=true" >> $GITHUB_ENV
           echo "TRITON_TEST_WARNING_REPORTS=true" >> $GITHUB_ENV
-          echo "TRITON_TEST_REPORTS_DIR=$HOME/reports" >> $GITHUB_ENV
+          echo "TRITON_TEST_REPORTS_DIR=$GITHUB_WORKSPACE/reports" >> $GITHUB_ENV
 
       - name: Run core tests
         run: |
@@ -159,7 +166,7 @@ jobs:
           source ./scripts/pytest-utils.sh
           cd python/test/regression
           TRITON_TEST_SUITE=regression \
-            python3 -m pytest --junitxml=~/reports/regression.xml -vvv -s --device xpu . --reruns 10 --ignore=test_performance.py
+            pytest -vvv -s --device xpu . --reruns 10 --ignore=test_performance.py
 
       - name: Run XPU python tests
         run: |
@@ -207,8 +214,8 @@ jobs:
 
       - name: Pass rate
         run: |
-          python3 scripts/pass_rate.py --reports ~/reports
-          python3 scripts/pass_rate.py --reports ~/reports --json > pass_rate.json
+          python3 scripts/pass_rate.py --reports reports
+          python3 scripts/pass_rate.py --reports reports --json > pass_rate.json
 
       - name: Upload pass rate report
         # upload reports only for the default branch
@@ -217,3 +224,10 @@ jobs:
         with:
           name: pass_rate-${{ join(matrix.*, '-') }}
           path: pass_rate.json
+
+      - name: Upload test reports
+        if: inputs.upload_test_reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-reports-${{ join(matrix.*, '-') }}
+          path: reports
diff --git a/scripts/pass_rate.py b/scripts/pass_rate.py
@@ -16,6 +16,7 @@ class ReportStats:
     """Report stats."""
     name: str = ''
     passed: int = 0
+    failed: int = 0
     skipped: int = 0
     xfailed: int = 0
     total: int = 0
@@ -60,10 +61,14 @@ def parse_report(report_path: pathlib.Path) -> ReportStats:
                 stats.skipped += 1
             elif skipped.get('type') == 'pytest.xfail':
                 stats.xfailed += 1
+        for _ in testsuite.iter('failure'):
+            stats.failed += 1
+        for _ in testsuite.iter('error'):
+            stats.failed += 1
     deselected = get_deselected(report_path)
     stats.skipped += deselected
     stats.total += deselected
-    stats.passed = stats.total - stats.skipped - stats.xfailed
+    stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed
     return stats
 
 
@@ -72,6 +77,7 @@ def overall_stats(stats: List[ReportStats]) -> ReportStats:
     overall = ReportStats(name='all')
     for item in stats:
         overall.passed += item.passed
+        overall.failed += item.failed
         overall.skipped += item.skipped
         overall.xfailed += item.xfailed
         overall.total += item.total
@@ -88,6 +94,7 @@ def print_stats(stats: ReportStats):
     print(
         f'{stats.name}:'
         f' passed: {stats.passed},'
+        f' failed: {stats.failed},'
         f' skipped: {stats.skipped},'
         f' xfailed: {stats.xfailed},'
         f' total: {stats.total},'
@@ -112,6 +119,7 @@ def print_json_stats(stats: List[ReportStats]):
         'python_version': platform.python_version(),
         'testsuite': overall.name,
         'passed': overall.passed,
+        'failed': overall.failed,
         'skipped': overall.skipped,
         'xfailed': overall.xfailed,
         'total': overall.total,

diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh
@@ -1,10 +1,11 @@
 TIMESTAMP="$(date '+%Y%m%d%H%M%S')"
 
-SCRIPTS_DIR="${SCRIPTS_DIR:-$PWD/scripts}"
+SCRIPTS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 TRITON_TEST_REPORTS="${TRITON_TEST_REPORTS:-false}"
 TRITON_TEST_REPORTS_DIR="${TRITON_TEST_REPORTS_DIR:-$HOME/reports/$TIMESTAMP}"
 TRITON_TEST_SKIPLIST_DIR="${TRITON_TEST_SKIPLIST_DIR:-$SCRIPTS_DIR/skiplist/default}"
 TRITON_TEST_WARNING_REPORTS="${TRITON_TEST_WARNING_REPORTS:-false}"
+TRITON_TEST_IGNORE_ERRORS="${TRITON_TEST_IGNORE_ERRORS:-false}"
 
 # absolute path for the selected skip list
 TRITON_TEST_SKIPLIST_DIR="$(cd "$TRITON_TEST_SKIPLIST_DIR" && pwd)"
@@ -39,5 +40,5 @@ pytest() {
         )
     fi
 
-    python3 -m pytest "${pytest_extra_args[@]}" "$@"
+    python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS
 }
diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh
@@ -9,6 +9,7 @@ TEST_UNIT=false
 VENV=false
 TRITON_TEST_REPORTS=false
 TRITON_TEST_WARNING_REPORTS=false
+TRITON_TEST_IGNORE_ERRORS=false
 SKIP_DEPS=false
 ARGS=
 for arg in "$@"; do
@@ -41,8 +42,12 @@ for arg in "$@"; do
       TRITON_TEST_WARNING_REPORTS=true
       shift
       ;;
+    --ignore-errors)
+      TRITON_TEST_IGNORE_ERRORS=true
+      shift
+      ;;
     --help)
-      echo "Example usage: ./test-triton.sh [--core | --tutorial | --unit | --venv | --reports | --warning-reports]"
+      echo "Example usage: ./test-triton.sh [--core | --tutorial | --unit | --venv | --reports | --warning-reports | --ignore-errors]"
       exit 1
       ;;
     *)
@@ -158,7 +163,7 @@ run_tutorial_test() {
   echo
   echo "****** Running $1 test ******"
   echo
-  python $1.py
+  python $1.py || $TRITON_TEST_IGNORE_ERRORS
 }
 
 run_tutorial_tests() {