diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index a2a45d2710..dae0e8d7a7 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -2,6 +2,15 @@ name: Build and test on: workflow_dispatch: + inputs: + runner_label: + description: Runner label, keep empty for default + type: string + default: "" + upload_test_reports: + description: Upload test reports + type: boolean + default: false pull_request: branches: - llvm-target @@ -61,9 +70,7 @@ jobs: integration-tests: name: Integration tests runs-on: - - glados - - spr - - runner-0.0.13 + - ${{ inputs.runner_label || 'runner-0.0.13' }} strategy: matrix: python: ${{ github.ref_name == 'llvm-target' && fromJson('["3.9", "3.10", "3.11"]') || fromJson('["3.9"]') }} @@ -115,10 +122,10 @@ jobs: - name: Create directory for tests reports run: | - mkdir ~/reports + mkdir reports echo "TRITON_TEST_REPORTS=true" >> $GITHUB_ENV echo "TRITON_TEST_WARNING_REPORTS=true" >> $GITHUB_ENV - echo "TRITON_TEST_REPORTS_DIR=$HOME/reports" >> $GITHUB_ENV + echo "TRITON_TEST_REPORTS_DIR=$GITHUB_WORKSPACE/reports" >> $GITHUB_ENV - name: Run core tests run: | @@ -159,7 +166,7 @@ jobs: source ./scripts/pytest-utils.sh cd python/test/regression TRITON_TEST_SUITE=regression \ - python3 -m pytest --junitxml=~/reports/regression.xml -vvv -s --device xpu . --reruns 10 --ignore=test_performance.py + pytest -vvv -s --device xpu . --reruns 10 --ignore=test_performance.py - name: Run XPU python tests run: | @@ -207,8 +214,8 @@ jobs: - name: Pass rate run: | - python3 scripts/pass_rate.py --reports ~/reports - python3 scripts/pass_rate.py --reports ~/reports --json > pass_rate.json + python3 scripts/pass_rate.py --reports reports + python3 scripts/pass_rate.py --reports reports --json > pass_rate.json - name: Upload pass rate report # upload reports only for the default branch @@ -217,3 +224,10 @@ jobs: with: name: pass_rate-${{ join(matrix.*, '-') }} path: pass_rate.json + + - name: Upload test reports + if: inputs.upload_test_reports + uses: actions/upload-artifact@v4 + with: + name: test-reports-${{ join(matrix.*, '-') }} + path: reports diff --git a/scripts/pass_rate.py b/scripts/pass_rate.py index 0da08d35b0..db175745ef 100644 --- a/scripts/pass_rate.py +++ b/scripts/pass_rate.py @@ -16,6 +16,7 @@ class ReportStats: """Report stats.""" name: str = '' passed: int = 0 + failed: int = 0 skipped: int = 0 xfailed: int = 0 total: int = 0 @@ -60,10 +61,14 @@ def parse_report(report_path: pathlib.Path) -> ReportStats: stats.skipped += 1 elif skipped.get('type') == 'pytest.xfail': stats.xfailed += 1 + for _ in testsuite.iter('failure'): + stats.failed += 1 + for _ in testsuite.iter('error'): + stats.failed += 1 deselected = get_deselected(report_path) stats.skipped += deselected stats.total += deselected - stats.passed = stats.total - stats.skipped - stats.xfailed + stats.passed = stats.total - stats.failed - stats.skipped - stats.xfailed return stats @@ -72,6 +77,7 @@ def overall_stats(stats: List[ReportStats]) -> ReportStats: overall = ReportStats(name='all') for item in stats: overall.passed += item.passed + overall.failed += item.failed overall.skipped += item.skipped overall.xfailed += item.xfailed overall.total += item.total @@ -88,6 +94,7 @@ def print_stats(stats: ReportStats): print( f'{stats.name}:' f' passed: {stats.passed},' + f' failed: {stats.failed},' f' skipped: {stats.skipped},' f' xfailed: {stats.xfailed},' f' total: {stats.total},' @@ -112,6 +119,7 @@ def print_json_stats(stats: List[ReportStats]): 'python_version': platform.python_version(), 'testsuite': overall.name, 'passed': overall.passed, + 'failed': overall.failed, 'skipped': overall.skipped, 'xfailed': overall.xfailed, 'total': overall.total, diff --git a/scripts/pytest-utils.sh b/scripts/pytest-utils.sh index 3a2bc7fd9a..2d4345ddb3 100644 --- a/scripts/pytest-utils.sh +++ b/scripts/pytest-utils.sh @@ -1,10 +1,11 @@ TIMESTAMP="$(date '+%Y%m%d%H%M%S')" -SCRIPTS_DIR="${SCRIPTS_DIR:-$PWD/scripts}" +SCRIPTS_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" TRITON_TEST_REPORTS="${TRITON_TEST_REPORTS:-false}" TRITON_TEST_REPORTS_DIR="${TRITON_TEST_REPORTS_DIR:-$HOME/reports/$TIMESTAMP}" TRITON_TEST_SKIPLIST_DIR="${TRITON_TEST_SKIPLIST_DIR:-$SCRIPTS_DIR/skiplist/default}" TRITON_TEST_WARNING_REPORTS="${TRITON_TEST_WARNING_REPORTS:-false}" +TRITON_TEST_IGNORE_ERRORS="${TRITON_TEST_IGNORE_ERRORS:-false}" # absolute path for the selected skip list TRITON_TEST_SKIPLIST_DIR="$(cd "$TRITON_TEST_SKIPLIST_DIR" && pwd)" @@ -39,5 +40,5 @@ pytest() { ) fi - python3 -m pytest "${pytest_extra_args[@]}" "$@" + python3 -u -m pytest "${pytest_extra_args[@]}" "$@" || $TRITON_TEST_IGNORE_ERRORS } diff --git a/scripts/test-triton.sh b/scripts/test-triton.sh index faef4345e4..97d0d99ed8 100755 --- a/scripts/test-triton.sh +++ b/scripts/test-triton.sh @@ -9,6 +9,7 @@ TEST_UNIT=false VENV=false TRITON_TEST_REPORTS=false TRITON_TEST_WARNING_REPORTS=false +TRITON_TEST_IGNORE_ERRORS=false SKIP_DEPS=false ARGS= for arg in "$@"; do @@ -41,8 +42,12 @@ for arg in "$@"; do TRITON_TEST_WARNING_REPORTS=true shift ;; + --ignore-errors) + TRITON_TEST_IGNORE_ERRORS=true + shift + ;; --help) - echo "Example usage: ./test-triton.sh [--core | --tutorial | --unit | --venv | --reports | --warning-reports]" + echo "Example usage: ./test-triton.sh [--core | --tutorial | --unit | --venv | --reports | --warning-reports | --ignore-errors]" exit 1 ;; *) @@ -158,7 +163,7 @@ run_tutorial_test() { echo echo "****** Running $1 test ******" echo - python $1.py + python $1.py || $TRITON_TEST_IGNORE_ERRORS } run_tutorial_tests() {