From 9258d883e7f94d040430850b75114877604c10fd Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 4 Jan 2025 00:41:23 +0000 Subject: [PATCH 1/3] fix: allure reporting --- .github/workflows/behave_pull_request.yml | 214 ++++++++++++++++++ .../{behave.yml => behave_schedule.yml} | 10 +- .../councils/AberdeenCityCouncil.py | 1 + 3 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/behave_pull_request.yml rename .github/workflows/{behave.yml => behave_schedule.yml} (95%) diff --git a/.github/workflows/behave_pull_request.yml b/.github/workflows/behave_pull_request.yml new file mode 100644 index 0000000000..fdc68d1dbf --- /dev/null +++ b/.github/workflows/behave_pull_request.yml @@ -0,0 +1,214 @@ +name: Test Councils (Pull Request Only) + +on: + pull_request: + branches: [ "master" ] + paths-ignore: + - "wiki/**" + - "**/*.md" + - "uk_bin_collection_api_server/**" + +jobs: + setup: + name: Setup Environment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Poetry + run: pipx install poetry==1.8.4 + + - uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Dependencies + run: make install-dev + + - name: Lint JSON + run: jq empty uk_bin_collection/tests/input.json + + - name: Get All Council Files That Have Changed + id: changed-council-files + uses: tj-actions/changed-files@v45 + with: + files: | + uk_bin_collection/uk_bin_collection/councils/**.py + + - name: Set Council Tests Environment Variable + id: set-council-tests + run: | + IFS=' ' read -ra FILES <<< "${{ steps.changed-council-files.outputs.all_changed_files }}" + COUNCIL_TESTS="" + for file in "${FILES[@]}"; do + FILENAME=$(basename "$file" .py) + if [ -z "$COUNCIL_TESTS" ]; then + COUNCIL_TESTS="$FILENAME" + else + COUNCIL_TESTS="$COUNCIL_TESTS or $FILENAME" + fi + done + echo "council_tests=$COUNCIL_TESTS" >> $GITHUB_OUTPUT + + outputs: + council_tests: ${{ steps.set-council-tests.outputs.council_tests }} + + unit-tests: + name: Run Unit Tests + needs: setup + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + poetry-version: [1.8.4] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: pipx install poetry==${{ matrix.poetry-version }} + + - name: Install Dependencies + run: make install-dev + + - name: Run Unit Tests + run: make unit-tests + + - name: Upload Test Results to Codecov + uses: codecov/codecov-action@v5 + with: + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.xml + + parity-check: + name: Parity Check + needs: setup + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + poetry-version: [1.8.4] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: pipx install poetry==${{ matrix.poetry-version }} + + - name: Install Dependencies + run: make install-dev + + - name: Check Parity of Councils / input.json / Feature file + run: | + repo=${{ github.event.pull_request.head.repo.full_name || 'robbrad/UKBinCollectionData' }} + branch=${{ github.event.pull_request.head.ref || 'master' }} + make parity-check repo=$repo branch=$branch + + integration-tests: + name: Run Integration Tests + needs: setup + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + poetry-version: [1.8.4] + services: + selenium: + image: selenium/standalone-chrome:latest + options: --shm-size=2gb --name selenium --hostname selenium + ports: + - 4444:4444 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: pipx install poetry==${{ matrix.poetry-version }} + + - name: Install Dependencies + run: make install-dev + + - name: Run Integration Tests + env: + HEADLESS: True + COUNCIL_TESTS: ${{ needs.setup.outputs.council_tests }} + run: make matrix=${{ matrix.python-version }} councils="${{ env.COUNCIL_TESTS }}" integration-tests + continue-on-error: true + + # Only generate and upload Allure reports on schedule + report: + name: Generate and Upload Reports + needs: [unit-tests, parity-check, integration-tests] + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.12] + steps: + - uses: actions/checkout@v4 + + # Fetch Allure history for the full report (optional) + - name: Get Allure history - Full Report + uses: actions/checkout@v4 + continue-on-error: true + with: + ref: gh-pages + path: gh-pages + + # Generate Allure Reports (Full) + - name: Allure report action for Full Run + uses: simple-elf/allure-report-action@master + with: + allure_results: build/${{ matrix.python-version }}/allure-results + subfolder: ${{ matrix.python-version }}/partial + allure_history: allure-history-partial + keep_reports: 20 + + # Archive the Full Report + - name: Tar full report + run: tar -cvf allure_partial_history_${{ matrix.python-version }}.tar allure-history-partial/${{ matrix.python-version }}/partial + + # Upload the Full Report artifact + - name: Upload artifact for Full Report + uses: actions/upload-artifact@v4 + with: + name: allure_partial_history_${{ matrix.python-version }} + path: allure_partial_history_${{ matrix.python-version }}.tar + + # Deploy only on schedule + master branch + deploy: + name: Deploy Reports + runs-on: ubuntu-latest + needs: report + steps: + # Download Full Artifacts + - uses: actions/download-artifact@v4 + name: Download Full Artifacts + with: + name: allure_full_history_3.12 + path: allure-history/tars + + - name: Untar reports + run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done + + - name: Remove tar reports + run: rm -rf allure-history/tars + + - name: Display structure of downloaded files + run: ls -R + + - name: Deploy + uses: peaceiris/actions-gh-pages@v4 + with: + PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PUBLISH_BRANCH: gh-pages + PUBLISH_DIR: allure-history-partial \ No newline at end of file diff --git a/.github/workflows/behave.yml b/.github/workflows/behave_schedule.yml similarity index 95% rename from .github/workflows/behave.yml rename to .github/workflows/behave_schedule.yml index d2a4212eaf..31f0ab6329 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave_schedule.yml @@ -179,20 +179,20 @@ jobs: uses: simple-elf/allure-report-action@master with: allure_results: build/${{ matrix.python-version }}/allure-results - subfolder: ${{ matrix.python-version }} - allure_history: allure-history + subfolder: ${{ matrix.python-version }}/full + allure_history: allure-history-full keep_reports: 20 # Archive the Full Report - name: Tar full report - run: tar -cvf allure_full_history_${{ matrix.python-version }}.tar allure-history/${{ matrix.python-version }} + run: tar -cvf allure_full_history_${{ matrix.python-version }}.tar allure-history-full/${{ matrix.python-version }}/full # Upload the Full Report artifact - name: Upload artifact for Full Report uses: actions/upload-artifact@v4 with: - name: allure_history_${{ matrix.python-version }} - path: allure_history_${{ matrix.python-version }}.tar + name: allure_full_history_${{ matrix.python-version }} + path: allure_full_history_${{ matrix.python-version }}.tar # Deploy only on schedule + master branch deploy: diff --git a/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py index 00ed29fe59..a69b3ab16b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py @@ -5,6 +5,7 @@ from uk_bin_collection.uk_bin_collection.common import * from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + # import the wonderful Beautiful Soup and the URL grabber class CouncilClass(AbstractGetBinDataClass): """ From ad8d9fe8ca13dd48ad16f76c8578d33dece34025 Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 4 Jan 2025 00:45:15 +0000 Subject: [PATCH 2/3] fix: allure reporting --- .github/workflows/behave_pull_request.yml | 3 ++- .github/workflows/behave_schedule.yml | 12 ------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/behave_pull_request.yml b/.github/workflows/behave_pull_request.yml index fdc68d1dbf..f766ab64d6 100644 --- a/.github/workflows/behave_pull_request.yml +++ b/.github/workflows/behave_pull_request.yml @@ -1,6 +1,7 @@ name: Test Councils (Pull Request Only) on: + workflow_dispatch: pull_request: branches: [ "master" ] paths-ignore: @@ -194,7 +195,7 @@ jobs: - uses: actions/download-artifact@v4 name: Download Full Artifacts with: - name: allure_full_history_3.12 + name: allure_partial_history_3.12 path: allure-history/tars - name: Untar reports diff --git a/.github/workflows/behave_schedule.yml b/.github/workflows/behave_schedule.yml index 31f0ab6329..8f9e53c3bb 100644 --- a/.github/workflows/behave_schedule.yml +++ b/.github/workflows/behave_schedule.yml @@ -2,18 +2,6 @@ name: Test Councils on: workflow_dispatch: - push: - paths-ignore: - - "wiki/**" - - "**/*.md" - - "uk_bin_collection_api_server/**" - branches: [ "master" ] - pull_request: - paths-ignore: - - "wiki/**" - - "**/*.md" - - "uk_bin_collection_api_server/**" - branches: [ "master" ] schedule: - cron: '0 0 * * *' # Nightly schedule for full test run From e644b40b9cf0ae141e952abb66c65741e7eab31b Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 4 Jan 2025 00:49:02 +0000 Subject: [PATCH 3/3] fix: allure reporting --- .github/workflows/behave_pull_request.yml | 8 +++----- .github/workflows/behave_schedule.yml | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/behave_pull_request.yml b/.github/workflows/behave_pull_request.yml index f766ab64d6..899373d1ba 100644 --- a/.github/workflows/behave_pull_request.yml +++ b/.github/workflows/behave_pull_request.yml @@ -157,16 +157,14 @@ jobs: steps: - uses: actions/checkout@v4 - # Fetch Allure history for the full report (optional) - - name: Get Allure history - Full Report + - name: Get Allure history - Partial Report uses: actions/checkout@v4 continue-on-error: true with: ref: gh-pages path: gh-pages - # Generate Allure Reports (Full) - - name: Allure report action for Full Run + - name: Allure report action for Partial Run uses: simple-elf/allure-report-action@master with: allure_results: build/${{ matrix.python-version }}/allure-results @@ -199,7 +197,7 @@ jobs: path: allure-history/tars - name: Untar reports - run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done + run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history-partial ;done - name: Remove tar reports run: rm -rf allure-history/tars diff --git a/.github/workflows/behave_schedule.yml b/.github/workflows/behave_schedule.yml index 8f9e53c3bb..7875dd1ba3 100644 --- a/.github/workflows/behave_schedule.yml +++ b/.github/workflows/behave_schedule.yml @@ -197,7 +197,7 @@ jobs: path: allure-history/tars - name: Untar reports - run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done + run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history-full ;done - name: Remove tar reports run: rm -rf allure-history/tars