rest-api-fuzzer.yml

name: REST API Fuzz Test

on:
  workflow_dispatch:
  # schedule:
  #   - cron: "15 4 3 * *" # Run at 4:15am on the 3rd of every month
  pull_request:
    paths:
      - '.github/workflows/rest-api-fuzzer.yml'
      - 'setup/fuzzer/**/*'

env:
  BUILD_CONFIGURATION: Release
  BUILD_PLATFORM: 'Any CPU'
  RESTLER_VERSION: '9.2.4'
  PYTHON_VERSION: '3.8'
  DOTNET_VERSION: '6.0.x'

jobs:
  build-restler-fuzzer:
    name: Fuzz test speckle-server REST API
    runs-on: ubuntu-latest
    permissions:
      contents: read

    steps:
      - uses: actions/checkout@v4
        name: Checkout speckle-server
        with:
          path: 'speckle-server'

      - uses: actions/checkout@v4
        name: Checkout RESTler Fuzzer
        with:
          repository: microsoft/restler-fuzzer
          ref: v${{ env.RESTLER_VERSION }}
          path: 'restler-fuzzer' # The path to clone the repository within the {{ github.workspace }} directory

      - name: Setup .NET ${{ env.DOTNET_VERSION }}
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: ${{ env.DOTNET_VERSION }}
          cache: true
          cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/src/Restler.sln

      - name: Restore NuGet packages
        run: dotnet restore ${{ github.workspace }}/restler-fuzzer/src/Restler.sln

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'
          cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt

      - name: Install engine (Python) dependencies
        run: |
          pip install -r ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt

      - name: Restore cached Restler binaries
        id: cache-restler-bin-restore
        uses: actions/cache/restore@v4
        with:
          path: |
            ${{ github.workspace }}/bin/restler
          key: restler-binaries-${{ env.RESTLER_VERSION }}-${{ hashFiles('speckle-server/.github/workflows/rest-api-fuzzer.yml') }}

      - name: Build RESTler
        if: steps.cache-restler-bin-restore.outputs.cache-hit != 'true'
        run: |
          python ${{ github.workspace }}/restler-fuzzer/build-restler.py --dest_dir ${{ github.workspace }}/bin
          python -m compileall -b ${{ github.workspace }}/bin/restler

      - name: Debug the built output
        run: |
          ls -la ${{ github.workspace }}/bin/restler
          ls -la ${{ github.workspace }}/bin/restler/Restler

      - name: Save Restler binaries to cache
        if: steps.cache-restler-bin-restore.outputs.cache-hit != 'true'
        id: cache-restler-bin-save
        uses: actions/cache/save@v4
        with:
          path: |
            ${{ github.workspace }}/bin/restler
          key: ${{ steps.cache-restler-bin-restore.outputs.cache-primary-key }}

      - name: Restore cached Restler configuration
        id: cache-config-restore
        uses: actions/cache/restore@v4
        with:
          path: |
            ${{ github.workspace }}/restlerConfig
          key: restler-config-${{ hashFiles('speckle-server/setup/fuzzer/speckle-server.openapi.json') }}

      - name: Generate RESTler config from OpenAPI specification
        if: steps.cache-config-restore.outputs.cache-hit != 'true'
        run: |
          ${{ github.workspace }}/bin/restler/Restler generate_config --specs ${{ github.workspace }}/speckle-server/setup/fuzzer/speckle-server.openapi.json

      - name: Print the Restler configuration
        run: |
          ls -la ${{ github.workspace }}
          ls -la ${{ github.workspace }}/restlerConfig
          echo ""
          echo "############################################"
          echo "# Engine settings                          #"
          echo "# To customize, copy and save this file to #"
          echo "# setup/fuzzer/settings.restler.json       #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/restlerConfig/engine_settings.json
          echo ""
          echo "############################################"
          echo "# Config                                   #"
          echo "# To customize, copy and save this file to #"
          echo "# setup/fuzzer/config.restler.json         #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/restlerConfig/config.json
          echo ""
          echo "############################################"
          echo "# Dictionary                               #"
          echo "# To customize, copy and save this file to #"
          echo "# setup/fuzzer/dictionary.restler.json     #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/restlerConfig/dict.json
          echo ""
          echo "############################################"
          echo "# Annotations                              #"
          echo "# To customize, copy and save this file to #"
          echo "# setup/fuzzer/annotations.restler.json    #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/restlerConfig/annotations.json

      - name: Save Restler Config
        if: steps.cache-config-restore.outputs.cache-hit != 'true'
        id: cache-config-save
        uses: actions/cache/save@v4
        with:
          path: |
            ${{ github.workspace }}/restlerConfig
          key: ${{ steps.cache-config-restore.outputs.cache-primary-key }}

      - name: Restore cached Restler grammar
        id: cache-grammar-restore
        uses: actions/cache/restore@v4
        with:
          path: |
            ${{ github.workspace }}/Compile
          key: restler-grammar-${{ hashFiles('speckle-server/setup/fuzzer/*.json') }}

      - name: Generate RESTler grammar from Restler config
        if: steps.cache-grammar-restore.outputs.cache-hit != 'true'
        run: |
          ${{ github.workspace }}/bin/restler/Restler compile ${{ github.workspace }}/speckle-server/setup/fuzzer/config.restler.json

      - name: Print the contents of the Restler compile directory
        if: always()
        run: |
          ls -la ${{ github.workspace }} || true
          ls -la ${{ github.workspace }}/Compile || true
          cat $(find ${{ github.workspace }}/Compile -type f -name "*.log") || true

      - name: Save Grammar
        if: steps.cache-grammar-restore.outputs.cache-hit != 'true'
        id: cache-grammar-save
        uses: actions/cache/save@v4
        with:
          path: |
            ${{ github.workspace }}/Compile
          key: ${{ steps.cache-grammar-restore.outputs.cache-primary-key }}

      - name: Deploy dependencies (docker compose)
        run: |
          docker compose --file ${{ github.workspace }}/speckle-server/docker-compose-deps.yml up --detach

      - name: Seed the database
        run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends postgresql-client
          PGPASSWORD=speckle psql -h 127.0.0.1 -U speckle -d speckle -p 5432 -w < ${{ github.workspace }}/speckle-server/setup/fuzzer/speckle.backup.sql

      - name: Deploy speckle-server (docker compose)
        working-directory: ${{ github.workspace }}
        timeout-minutes: 1
        run: |
          docker compose --file ${{ github.workspace }}/speckle-server/setup/fuzzer/docker-compose-speckle.yml up --detach

      - name: Run RESTler coverage test
        run: |
          until curl --output /dev/null --silent --head --fail http://127.0.0.1:3000/readiness; do
            echo "Waiting a further 3 seconds for speckle-server to start..."
            sleep 3
          done

          ${{ github.workspace }}/bin/restler/Restler test \
            --grammar_file "${{ github.workspace }}/Compile/grammar.py" \
            --dictionary_file "${{ github.workspace }}/speckle-server/setup/fuzzer/dictionary.restler.json" \
            --settings "${{ github.workspace }}/speckle-server/setup/fuzzer/settings.restler.json" \
            --no_ssl \
            --target_ip "127.0.0.1" \
            --target_port "3000"

      - name: Print the results
        if: always()
        run: |
          ls -la ${{ github.workspace }}/Test
          ls -la ${{ github.workspace }}/Test/RestlerResults || true
          ls -la ${{ github.workspace }}/Test/ResponseBuckets || true
          echo ""
          echo "############################################"
          echo "#            Engine stderr                 #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/Test/EngineStdErr.txt || true
          echo ""
          echo "############################################"
          echo "#     Engine stdout                        #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/Test/EngineStdOut.txt || true
          echo ""
          echo "############################################"
          echo "#     Results analyzer stderr              #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/Test/ResultsAnalyzerStdErr.txt || true
          echo ""
          echo "############################################"
          echo "#         Results analyzer stdout          #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/Test/ResultsAnalyzerStdOut.txt || true
          echo ""
          echo "############################################"
          echo "#    Coverage failures to investigate      #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/Test/coverage_failures_to_investigate.txt || true
          echo ""
          echo "############################################"
          echo "#              Restler logs                #"
          echo "############################################"
          echo ""
          cat ${{ github.workspace }}/Test/restler-*.log || true

      - uses: actions/upload-artifact@v4
        name: Store the Test output
        if: always()
        with:
          name: fuzz-test-rest-api-output
          path: ${{ github.workspace }}/Test
          if-no-files-found: error
          retention-days: 5
          overwrite: true

      - name: Print Docker Compose logs
        if: always()
        run: |
          docker compose --file ${{ github.workspace }}/speckle-server/docker-compose-deps.yml logs
          docker compose --file ${{ github.workspace }}/speckle-server/setup/fuzzer/docker-compose-speckle.yml logs