Skip to content

Commit

Permalink
ci: fix status result determination (#25268)
Browse files Browse the repository at this point in the history 
Signed-off-by: Ryan Cragun <me@ryan.ec>
  • Loading branch information
@ryancragun
ryancragun authored Feb 8, 2024
1 parent f2638af commit 90c1515
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 34 deletions.
33 changes: 19 additions & 14 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,26 +223,31 @@ jobs:
- test-ui
if: always()
runs-on: ${{ github.repository == 'hashicorp/vault' && 'ubuntu-latest' || fromJSON('["self-hosted","linux","small"]') }}
permissions: write-all # Ensure we have id-token:write access for vault-auth.
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
# Determine the overall status of our required test jobs.
- name: Determine status
id: status
run: |
# Decode our test results and filter out unrequired jobs. Reformat the JSON
# body that we get from Github into a more useful shape.
if results=$(jq -er 'del(.["test-go-fips"], .["test-go-race"]) as $required
| $required | keys as $jobs
| reduce $jobs[] as $job ([]; . + [{job: $job}+$required[$job]])'
) <<< '${{ toJSON(needs.*.result) }}'; then
# Determine the overall status of the job. We allow fips and race tests to fail so we
# don't consider their result here.
#
# Encode the needs context into JSON, filter out unrequired workflows, shape the result
# into a more useful schema. Determine the overall status by comparing the total number of
# successful results with the number of required jobs.
if results=$(jq -rec 'del(.["test-go-fips"], .["test-go-race"]) as $required
| $required | keys as $jobs
| reduce $jobs[] as $job ([]; . + [{job: $job}+$required[$job]])' <<< '${{ toJSON(needs) }}'
); then
# Determine if all of our required jobs have succeeded.
if jq -er 'length as $expected_length
| [.[] | select(.result == "success")]
| length == $expected_length' <<< "$results"; then
msg=$(echo -e "All required test jobs succeeded!\n$results")
if jq -rec 'length as $expected
| [.[] | select((.result == "success") or (.result == "skipped"))] | length as $got
| $expected == $got' <<< "$results"; then
msg="All required test jobs succeeded!"
result="success"
else
msg=$(echo -e "One or more required test jobs failed!\n$results")
msg="One or more required test jobs failed!"
result="failed"
fi
else
Expand All @@ -253,7 +258,7 @@ jobs:
{
echo "msg=${msg}"
echo "result=${result}"
echo "results=${results}"
echo "results<<EOFRESULTS"$'\n'"${results}"$'\n'EOFRESULTS
} | tee -a "$GITHUB_OUTPUT"
- if: needs.setup.outputs.is-enterprise == 'true'
id: vault-auth
Expand Down Expand Up @@ -353,9 +358,9 @@ jobs:
echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"
fi
{
echo 'table-test-results<<EOF'
echo 'table-test-results<<EOFTABLE'
cat "$temp_file_name"
echo EOF
echo EOFTABLE
} | tee -a "$GITHUB_OUTPUT"
- name: Create comment
if: github.head_ref != '' && needs.setup.outputs.is-fork == 'false'
Expand Down
76 changes: 56 additions & 20 deletions .github/workflows/test-go.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,10 @@ on:
outputs:
data-race-output:
description: A textual output of any data race detector failures
value: ${{ jobs.test-go.outputs.data-race-output }}
value: ${{ jobs.status.outputs.data-race-output }}
data-race-result:
description: Whether or not there were any data races detected
value: ${{ jobs.test-go.outputs.data-race-result }}
value: ${{ jobs.status.outputs.data-race-result }}

env: ${{ fromJSON(inputs.env-vars) }}

Expand Down Expand Up @@ -228,9 +228,8 @@ jobs:
GOPRIVATE: github.com/hashicorp/*
TIMEOUT_IN_MINUTES: ${{ inputs.timeout-minutes }}
outputs:
data-race-result : ${{ steps.data-race-check.outputs.data-race-result }}
data-race-output : ${{ steps.data-race-check.outputs.data-race-output }}
go-test-results-download-pattern: ${{ steps.metadata.outputs.go-test-results-download-pattern }}
data-race-log-download-pattern: ${{ steps.metadata.outputs.data-race-log-download-pattern }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
Expand Down Expand Up @@ -260,9 +259,15 @@ jobs:
gotestsum_jsonfile=${go_test_dir}/${gotestsum_results_prefix}-${name}.json # test-results/go-test/results-1-testonly.json
gotestsum_timing_events=failure-summary-${name}.json # failure-summary-1-testonly.json
failure_summary_file_name="failure-summary-${name}.md" # failure-summary-1-testonly.md
data_race_log_file="data-race.log" # data-race.log
data_race_log_download_pattern="data-race-*.log" # data-race-*.log
data_race_log_upload_key="data-race-${name}.log" # data-race-1-testonly.log
{
echo "name=${name}"
echo "failure-summary-file-name=${failure_summary_file_name}"
echo "data-race-log-file=${data_race_log_file}"
echo "data-race-log-download-pattern=${data_race_log_download_pattern}"
echo "data-race-log-upload-key=${data_race_log_upload_key}"
echo "go-test-dir=${go_test_dir}"
echo "go-test-log-archive-name=${go_test_log_archive_name}"
echo "go-test-log-dir=${go_test_log_dir}"
Expand Down Expand Up @@ -478,14 +483,18 @@ jobs:
run: |
# Scan gotestsum output files for data race errors.
data_race_tests=()
data_race_log='${{ steps.metadata.outputs.data-race-log-file }}'
for file in *.json; do
# Check if test results contains offending phrase
if grep -q "WARNING: DATA RACE" "$file"; then
data_race_tests+=("test-go (${{ matrix.id }})")
touch "$data_race_log"
# Print failing matrix tests
echo "=============== test-go (${{ matrix.id }}) ==========================="
sed -n '/WARNING: DATA RACE/,/==================/p' "$file" | jq -r -j '.Output'
# Write output to our log file so we can aggregate it in the final workflow
{
echo "=============== test-go (${{ matrix.id }}) ==========================="
sed -n '/WARNING: DATA RACE/,/==================/p' "$file" | jq -r -j '.Output'
} | tee -a "$data_race_log"
fi
done
Expand All @@ -494,11 +503,17 @@ jobs:
if (("${#data_race_tests[@]}" > 0)); then
result="failure"
fi
{
echo "data-race-result=${result}"
echo -n "data-race-output=${data_race_tests[*]}"
} | tee -a "$GITHUB_OUTPUT"
echo "data-race-result=${result}" | tee -a "$GITHUB_OUTPUT"
- name: Upload data race detector failure log
if: steps.data-race-check.outputs.data-race-result == 'failure'
uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
with:
name: ${{ steps.metadata.outputs.data-race-log-upload-key }}
path: ${{ steps.metadata.outputs.go-test-dir }}/${{ steps.metadata.outputs.data-race-log-file }}
# Set the minimum retention possible. We only upload this because it's the only way to
# aggregate results from matrix workflows.
retention-days: 1
if-no-files-found: error # Make sure we always upload the data race logs if it failed
# GitHub Actions doesn't expose the job ID or the URL to the job execution,
# so we have to fetch it from the API
- name: Fetch job logs URL
Expand Down Expand Up @@ -555,7 +570,7 @@ jobs:
continue-on-error: true
run: |
# This jq query filters out successful tests, leaving only the failures.
# Then, it formats the results into rows of a Markdown table.
# Then, it formats the results into rows of a Markdown table.k
# An example row will resemble this:
# | github.com/hashicorp/vault/package | TestName | fips | 0 | 2 | [view results](github.com/link-to-logs) |
jq -r -n 'inputs
Expand All @@ -578,27 +593,48 @@ jobs:
- test-matrix
- test-go
runs-on: ${{ fromJSON(inputs.runs-on-small) }}
outputs:
data-race-output: ${{ steps.status.outputs.data-race-output }}
data-race-result: ${{ steps.status.outputs.data-race-result }}
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
with:
pattern: ${{ needs.test-go.outputs.data-race-log-download-pattern }}
path: data-race-logs
merge-multiple: true
# Determine our success/failure status by checking the result status and data race status.
- id: status
name: Determine status result
run: |
# Determine status result
result="success"
# Aggregate all of our test workflows and determine our Go test result from them.
test_go_results=$(tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | jq -Mrc)
if ! grep -q -v -E '(failure|cancelled)' <<< "$test_go_results"; then
test_go_result="failed"
result="failed"
else
test_go_result="success"
fi
data_race_output=""
data_race_results=$(tr -d '\n' <<< '${{ toJSON(needs.test-go.outputs.data-race-result) }}' | jq -Mrc)
if ! grep -q -v -E '(failure|cancelled)' <<< "$data_race_results"; then
data_race_output="${{ toJSON(needs.test-go.outputs.data-race-output) }}"
# If we have downloaded data race detector logs then at least one Go test job detected
# a data race during execution. We'll fail on that.
if [ -z "$(ls -A data-race-logs)" ]; then
data_race_output=""
data_race_result="success"
else
data_race_output="$(cat data-race-logs/*)"
data_race_result="failed"
result="failed"
fi
# Write Go and data race results to outputs.
{
echo "data-race-output=${data_race_output}"
echo "data-race-results=${data_race_results}"
echo "data-race-result=${data_race_result}"
echo "result=${result}"
echo "test-go-result=${test_go_result}"
echo "test-go-results=${test_go_results}"
} | tee -a "$GITHUB_OUTPUT"
# Aggregate, prune, and cache our timing data
Expand All @@ -616,7 +652,7 @@ jobs:
pattern: ${{ needs.test-go.outputs.go-test-results-download-pattern }}
merge-multiple: true
- if: ${{ ! cancelled() && needs.test-go.result == 'success' && inputs.test-timing-cache-enabled }}
name: Prune old timing
name: Prune any invalid timing files
run: |
ls -lhR '${{ needs.test-matrix.outputs.go-test-dir }}'
find '${{ needs.test-matrix.outputs.go-test-dir }}' -mindepth 1 -type f -mtime +3 -delete
Expand All @@ -628,7 +664,7 @@ jobs:
' shell {} \; > /dev/null 2>&1
ls -lhR '${{ needs.test-matrix.outputs.go-test-dir }}'
# Determine our overall pass/fail with our Go test results and our
# Determine our overall pass/fail with our Go test results
- if: always() && steps.status.outputs.result != 'success'
name: Check for failed status
run: |
Expand Down

0 comments on commit 90c1515

Please sign in to comment.