Merge pull request #1120 from robbrad/test_behave

fix: allure reporting

.github/workflows/behave_pull_request.yml

@@ -0,0 +1,213 @@
+name: Test Councils (Pull Request Only)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ "master" ]
+    paths-ignore:
+      - "wiki/**"
+      - "**/*.md"
+      - "uk_bin_collection_api_server/**"
+
+jobs:
+  setup:
+    name: Setup Environment
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Poetry
+        run: pipx install poetry==1.8.4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install Dependencies
+        run: make install-dev
+
+      - name: Lint JSON
+        run: jq empty uk_bin_collection/tests/input.json
+
+      - name: Get All Council Files That Have Changed
+        id: changed-council-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            uk_bin_collection/uk_bin_collection/councils/**.py
+
+      - name: Set Council Tests Environment Variable
+        id: set-council-tests
+        run: |
+          IFS=' ' read -ra FILES <<< "${{ steps.changed-council-files.outputs.all_changed_files }}"
+          COUNCIL_TESTS=""
+          for file in "${FILES[@]}"; do
+            FILENAME=$(basename "$file" .py)
+            if [ -z "$COUNCIL_TESTS" ]; then
+              COUNCIL_TESTS="$FILENAME"
+            else
+              COUNCIL_TESTS="$COUNCIL_TESTS or $FILENAME"
+            fi
+          done
+          echo "council_tests=$COUNCIL_TESTS" >> $GITHUB_OUTPUT
+
+    outputs:
+      council_tests: ${{ steps.set-council-tests.outputs.council_tests }}
+
+  unit-tests:
+    name: Run Unit Tests
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.12]
+        poetry-version: [1.8.4]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: pipx install poetry==${{ matrix.poetry-version }}
+
+      - name: Install Dependencies
+        run: make install-dev
+
+      - name: Run Unit Tests
+        run: make unit-tests
+
+      - name: Upload Test Results to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: coverage.xml
+
+  parity-check:
+    name: Parity Check
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.12]
+        poetry-version: [1.8.4]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: pipx install poetry==${{ matrix.poetry-version }}
+
+      - name: Install Dependencies
+        run: make install-dev
+
+      - name: Check Parity of Councils / input.json / Feature file
+        run: |
+          repo=${{ github.event.pull_request.head.repo.full_name || 'robbrad/UKBinCollectionData' }}
+          branch=${{ github.event.pull_request.head.ref || 'master' }}
+          make parity-check repo=$repo branch=$branch
+
+  integration-tests:
+    name: Run Integration Tests
+    needs: setup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.12]
+        poetry-version: [1.8.4]
+    services:
+      selenium:
+        image: selenium/standalone-chrome:latest
+        options: --shm-size=2gb --name selenium --hostname selenium
+        ports:
+          - 4444:4444
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: pipx install poetry==${{ matrix.poetry-version }}
+
+      - name: Install Dependencies
+        run: make install-dev
+
+      - name: Run Integration Tests
+        env:
+          HEADLESS: True
+          COUNCIL_TESTS: ${{ needs.setup.outputs.council_tests }}
+        run: make matrix=${{ matrix.python-version }} councils="${{ env.COUNCIL_TESTS }}" integration-tests
+        continue-on-error: true
+
+  # Only generate and upload Allure reports on schedule
+  report:
+    name: Generate and Upload Reports
+    needs: [unit-tests, parity-check, integration-tests]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.12]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get Allure history - Partial Report
+        uses: actions/checkout@v4
+        continue-on-error: true
+        with:
+          ref: gh-pages
+          path: gh-pages
+
+      - name: Allure report action for Partial Run
+        uses: simple-elf/allure-report-action@master
+        with:
+          allure_results: build/${{ matrix.python-version }}/allure-results
+          subfolder: ${{ matrix.python-version }}/partial
+          allure_history: allure-history-partial
+          keep_reports: 20
+
+      # Archive the Full Report
+      - name: Tar full report
+        run: tar -cvf allure_partial_history_${{ matrix.python-version }}.tar allure-history-partial/${{ matrix.python-version }}/partial
+
+      # Upload the Full Report artifact
+      - name: Upload artifact for Full Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure_partial_history_${{ matrix.python-version }}
+          path: allure_partial_history_${{ matrix.python-version }}.tar
+
+  # Deploy only on schedule + master branch
+  deploy:
+    name: Deploy Reports
+    runs-on: ubuntu-latest
+    needs: report
+    steps:
+      # Download Full Artifacts
+      - uses: actions/download-artifact@v4
+        name: Download Full Artifacts
+        with:
+          name: allure_partial_history_3.12
+          path: allure-history/tars
+
+      - name: Untar reports
+        run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history-partial ;done
+
+      - name: Remove tar reports
+        run: rm -rf allure-history/tars
+
+      - name: Display structure of downloaded files
+        run: ls -R
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PUBLISH_BRANCH: gh-pages
+          PUBLISH_DIR: allure-history-partial

.github/workflows/behave.yml → .github/workflows/behave_schedule.yml

@@ -2,18 +2,6 @@ name: Test Councils
 
 on:
   workflow_dispatch:
-  push:
-    paths-ignore:
-      - "wiki/**"
-      - "**/*.md"
-      - "uk_bin_collection_api_server/**"
-    branches: [ "master" ]
-  pull_request:
-    paths-ignore:
-      - "wiki/**"
-      - "**/*.md"
-      - "uk_bin_collection_api_server/**"
-    branches: [ "master" ]
   schedule:
     - cron: '0 0 * * *' # Nightly schedule for full test run
 
@@ -179,20 +167,20 @@ jobs:
         uses: simple-elf/allure-report-action@master
         with:
           allure_results: build/${{ matrix.python-version }}/allure-results
-          subfolder: ${{ matrix.python-version }}
-          allure_history: allure-history
+          subfolder: ${{ matrix.python-version }}/full
+          allure_history: allure-history-full
           keep_reports: 20
 
       # Archive the Full Report
       - name: Tar full report
-        run: tar -cvf allure_full_history_${{ matrix.python-version }}.tar allure-history/${{ matrix.python-version }}
+        run: tar -cvf allure_full_history_${{ matrix.python-version }}.tar allure-history-full/${{ matrix.python-version }}/full
 
       # Upload the Full Report artifact
       - name: Upload artifact for Full Report
         uses: actions/upload-artifact@v4
         with:
-          name: allure_history_${{ matrix.python-version }}
-          path: allure_history_${{ matrix.python-version }}.tar
+          name: allure_full_history_${{ matrix.python-version }}
+          path: allure_full_history_${{ matrix.python-version }}.tar
 
   # Deploy only on schedule + master branch
   deploy:
@@ -209,7 +197,7 @@ jobs:
           path: allure-history/tars
 
       - name: Untar reports
-        run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done
+        run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history-full ;done
 
       - name: Remove tar reports
         run: rm -rf allure-history/tars

uk_bin_collection/uk_bin_collection/councils/AberdeenCityCouncil.py

@@ -5,6 +5,7 @@
 from uk_bin_collection.uk_bin_collection.common import *
 from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass
 
+
 # import the wonderful Beautiful Soup and the URL grabber
 class CouncilClass(AbstractGetBinDataClass):
     """