Generate coverage report for new code and show all reports in GH step summary

.github/workflows/largemodel_unit_test_CI.yml

@@ -150,6 +150,8 @@ jobs:
           echo "Deleted vespa_tester_app.zip"
 
       - name: Run Large Model Unit Tests
+        id: run_unit_tests
+        continue-on-error: true
         run: |
           # Define these for use by marqo
           export VESPA_CONFIG_URL=http://localhost:19071
@@ -162,8 +164,59 @@ jobs:
           export PRIVATE_MODEL_TESTS_AWS_SECRET_ACCESS_KEY=${{ secrets.PRIVATE_MODEL_TESTS_AWS_SECRET_ACCESS_KEY }}
           export PRIVATE_MODEL_TESTS_HF_TOKEN=${{ secrets.PRIVATE_MODEL_TESTS_HF_TOKEN }}
 
-          export PYTHONPATH="./marqo/tests:./marqo/src:./marqo"
-          pytest marqo/tests --largemodel --ignore=marqo/tests/test_documentation.py --ignore=marqo/tests/backwards_compatibility_tests 
+          cd marqo
+          export PYTHONPATH="./tests:./src:."
+          set -o pipefail
+          pytest --largemodel --ignore=tests/test_documentation.py --ignore=tests/backwards_compatibility_tests \
+            --durations=100 --cov=src --cov-branch --cov-context=test \
+            --cov-report=html:cov_html --cov-report=xml:cov.xml --cov-report term:skip-covered \
+            --md-report --md-report-flavor gfm --md-report-output pytest_result_summary.md \
+            tests | tee pytest_output.txt
+
+      - name: Check Test Coverage of New Code
+        id: check_test_coverage
+        continue-on-error: true
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+            export BASE_BRANCH="${{ github.event.pull_request.base.ref }}"
+          else
+            export BASE_BRANCH=mainline
+          fi
+
+          cd marqo
+          echo "Running diff-cover against branch $BASE_BRANCH"
+          git fetch origin $BASE_BRANCH:$BASE_BRANCH
+          diff-cover cov.xml --html-report diff_cov.html --markdown-report diff_cov.md \
+            --compare-branch $BASE_BRANCH
+
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v4
+        continue-on-error: true
+        with:
+          name: marqo-test-report
+          path: |
+            marqo/cov_html/
+            marqo/diff_cov.html
+
+      - name: Test and coverage report summary
+        continue-on-error: true
+        run: |
+          echo "# Test Summary" >> $GITHUB_STEP_SUMMARY
+          cat marqo/pytest_result_summary.md >> $GITHUB_STEP_SUMMARY
+
+          echo "# Coverage Summary, Slow Tests and Failed Tests" >> $GITHUB_STEP_SUMMARY
+          echo '```text' >> $GITHUB_STEP_SUMMARY
+          awk '/---------- coverage:/ {flag=1} flag' marqo/pytest_output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+          cat marqo/diff_cov.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Fail Job If Tests or Coverage Failed
+        if: ${{ steps.run_unit_tests.outcome == 'failure' || steps.check_test_coverage.outcome == 'failure' }}
+        run: |
+          echo "Tests or coverage checks failed. Marking job as failed."
+          exit 1
+        shell: bash
 
   Stop-Runner:
     name: Stop self-hosted EC2 runner

.github/workflows/unit_test_200gb_CI.yml

@@ -153,6 +153,8 @@ jobs:
           echo "Deleted vespa_tester_app.zip"
 
       - name: Run Unit Tests
+        id: run_unit_tests
+        continue-on-error: true
         run: |
           # Define these for use by marqo
           export VESPA_CONFIG_URL=http://localhost:19071
@@ -165,13 +167,57 @@ jobs:
 
           cd marqo
           export PYTHONPATH="./tests:./src:."
-          pytest --ignore=tests/test_documentation.py --ignore=tests/backwards_compatibility_tests --durations=100 --cov=src --cov-branch --cov-context=test --cov-report=html:cov_html --cov-report=lcov:lcov.info tests
+          set -o pipefail
+          pytest --ignore=tests/test_documentation.py --ignore=tests/backwards_compatibility_tests \
+            --durations=100 --cov=src --cov-branch --cov-context=test \
+            --cov-report=html:cov_html --cov-report=xml:cov.xml --cov-report term:skip-covered \
+            --md-report --md-report-flavor gfm --md-report-output pytest_result_summary.md \
+            tests | tee pytest_output.txt
+
+      - name: Check Test Coverage of New Code
+        id: check_test_coverage
+        continue-on-error: true
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+            export BASE_BRANCH="${{ github.event.pull_request.base.ref }}"
+          else
+            export BASE_BRANCH=mainline
+          fi
+
+          cd marqo
+          echo "Running diff-cover against branch $BASE_BRANCH"
+          git fetch origin $BASE_BRANCH:$BASE_BRANCH
+          diff-cover cov.xml --html-report diff_cov.html --markdown-report diff_cov.md \
+            --compare-branch $BASE_BRANCH
 
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
+        continue-on-error: true
         with:
           name: marqo-test-report
-          path: marqo/cov_html/
+          path: |
+            marqo/cov_html/
+            marqo/diff_cov.html
+
+      - name: Test and coverage report summary
+        continue-on-error: true
+        run: |
+          echo "# Test Summary" >> $GITHUB_STEP_SUMMARY
+          cat marqo/pytest_result_summary.md >> $GITHUB_STEP_SUMMARY
+
+          echo "# Coverage Summary, Slow Tests and Failed Tests" >> $GITHUB_STEP_SUMMARY
+          echo '```text' >> $GITHUB_STEP_SUMMARY
+          awk '/---------- coverage:/ {flag=1} flag' marqo/pytest_output.txt >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+          cat marqo/diff_cov.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Fail Job If Tests or Coverage Failed
+        if: ${{ steps.run_unit_tests.outcome == 'failure' || steps.check_test_coverage.outcome == 'failure' }}
+        run: |
+          echo "Tests or coverage checks failed. Marking job as failed."
+          exit 1
+        shell: bash
 
   Stop-Runner:
     name: Stop self-hosted EC2 runner

requirements.dev.txt

@@ -1,4 +1,6 @@
 # test requirements
 pyvespa==0.37.1
 pytest==7.4.3
-pytest-cov==5.0.0
+pytest-cov==5.0.0
+diff-cover==9.2.0
+pytest-md-report==0.6.2