sop-test-suite.yml

name: SOP interoperability test suite

on:
  pull_request:
    branches: [ main, v2 ]

jobs:

  build-gosop:
    name: Build gosop from branch
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Build gosop from branch
        uses: ./.github/actions/build-gosop
        with: 
          binary-location: ./gosop-${{ github.sha }}
      # Upload as artifact
      - name: Upload gosop artifact
        uses: actions/upload-artifact@v4
        with:
          name: gosop-${{ github.sha }}
          path: ./gosop-${{ github.sha }}

  build-gosop-target:
    name: Build gosop from target
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Build gosop from branch
        uses: ./.github/actions/build-gosop
        with: 
          gopenpgp-ref: ${{ github.base_ref }}
          binary-location: ./gosop-target
      # Upload as artifact
      - name: Upload gosop-target artifact
        uses: actions/upload-artifact@v4
        with:
          name: gosop-target
          path: ./gosop-target
    

  test-suite:
    name: Run interoperability test suite
    runs-on: ubuntu-latest
    container: 
      image: ghcr.io/protonmail/openpgp-interop-test-docker:v1.1.7
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.github_token }}
    needs: 
      - build-gosop
      - build-gosop-target
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      # Fetch gosop from target
      - name: Download gosop-target
        uses: actions/download-artifact@v4
        with:
          name: gosop-target
      # Test gosop-target
      - name: Make gosop-target executable
        run: chmod +x gosop-target
      - name: Print gosop-target version
        run: ./gosop-target version --extended
      # Fetch gosop from branch
      - name: Download gosop-branch
        uses: actions/download-artifact@v4
        with:
          name: gosop-${{ github.sha }}
      - name: Rename gosop-branch
        run: mv gosop-${{ github.sha }} gosop-branch
      # Test gosop-branch
      - name: Make gosop-branch executable
        run: chmod +x gosop-branch
      - name: Print gosop-branch version
        run: ./gosop-branch version --extended
      # Run test suite
      - name: Prepare test configuration
        run: ./.github/test-suite/prepare_config.sh $CONFIG_TEMPLATE $CONFIG_OUTPUT $GITHUB_WORKSPACE/gosop-branch $GITHUB_WORKSPACE/gosop-target
        env:
         CONFIG_TEMPLATE: .github/test-suite/config.json.template
         CONFIG_OUTPUT: .github/test-suite/config.json
      - name: Display configuration
        run: cat .github/test-suite/config.json
      - name: Run interoperability test suite
        run: cd $TEST_SUITE_DIR && $TEST_SUITE --config $GITHUB_WORKSPACE/$CONFIG --json-out $GITHUB_WORKSPACE/$RESULTS_JSON --html-out $GITHUB_WORKSPACE/$RESULTS_HTML
        env:
         CONFIG: .github/test-suite/config.json
         RESULTS_JSON: .github/test-suite/test-suite-results.json
         RESULTS_HTML: .github/test-suite/test-suite-results.html
      # Upload results
      - name: Upload test results json artifact
        uses: actions/upload-artifact@v4
        with:
          name: test-suite-results.json
          path: .github/test-suite/test-suite-results.json
      - name: Upload test results html artifact
        uses: actions/upload-artifact@v4
        with:
          name: test-suite-results.html
          path: .github/test-suite/test-suite-results.html

  compare-with-target:
    name: Compare with target
    runs-on: ubuntu-latest
    needs: test-suite
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download test results json artifact
        id: download-test-results
        uses: actions/download-artifact@v4
        with:
          name: test-suite-results.json
      - name: Compare with baseline
        uses: ProtonMail/openpgp-interop-test-analyzer@v2.0.0
        with: 
          results: ${{ steps.download-test-results.outputs.download-path }}/test-suite-results.json
          output: baseline-comparison.json
          baseline: gosop-target
          target: gosop-branch