ci(tests): fuzz testing workflow for REST API #41
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: REST API Fuzz Test | |
| on: | |
| workflow_dispatch: | |
| # schedule: | |
| # - cron: "15 4 3 * *" # Run at 4:15am on the 3rd of every month | |
| pull_request: | |
| paths: | |
| - '.github/workflows/rest-api-fuzzer.yml' | |
| - 'setup/fuzzer/**/*' | |
| env: | |
| BUILD_CONFIGURATION: Release | |
| BUILD_PLATFORM: 'Any CPU' | |
| RESTLER_VERSION: '9.2.4' | |
| PYTHON_VERSION: '3.8' | |
| DOTNET_VERSION: '6.0.x' | |
| jobs: | |
| build-restler-fuzzer: | |
| name: Fuzz test speckle-server REST API | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| name: Checkout RESTler Fuzzer | |
| with: | |
| repository: microsoft/restler-fuzzer | |
| ref: v${{ env.RESTLER_VERSION }} | |
| path: 'restler-fuzzer' # The path to clone the repository within the {{ github.workspace }} directory | |
| - name: Setup .NET ${{ env.DOTNET_VERSION }} | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| cache: true | |
| cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/src/Restler.sln | |
| - name: Restore NuGet packages | |
| run: dotnet restore ${{ github.workspace }}/restler-fuzzer/src/Restler.sln | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt | |
| - name: Install engine (Python) dependencies | |
| run: | | |
| pip install -r ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt | |
| - name: Build RESTler | |
| run: | | |
| python ${{ github.workspace }}/restler-fuzzer/build-restler.py --dest_dir ${{ github.workspace }}/bin | |
| - name: Debug the built output | |
| run: | | |
| ls -la ${{ github.workspace }}/bin/restler | |
| ls -la ${{ github.workspace }}/bin/restler/Restler | |
| - uses: actions/checkout@v4 | |
| name: Checkout speckle-server | |
| with: | |
| path: 'speckle-server' | |
| - name: Restore cached Restler configuration | |
| id: cache-config-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/restlerConfig | |
| key: restler-config-${{ hashFiles('speckle-server/setup/fuzzer/speckle-server.openapi.json') }} | |
| - name: Generate RESTler config from OpenAPI specification | |
| if: steps.cache-config-restore.outputs.cache-hit != 'true' | |
| run: | | |
| ${{ github.workspace }}/bin/restler/Restler generate_config --specs ${{ github.workspace }}/speckle-server/setup/fuzzer/speckle-server.openapi.json | |
| - name: Print the Restler configuration | |
| run: | | |
| ls -la ${{ github.workspace }} | |
| ls -la ${{ github.workspace }}/restlerConfig | |
| echo "" | |
| echo "############################################" | |
| echo "# Engine settings #" | |
| echo "# To customize, copy and save this file to #" | |
| echo "# setup/fuzzer/settings.restler.json #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/restlerConfig/engine_settings.json | |
| echo "" | |
| echo "############################################" | |
| echo "# Config #" | |
| echo "# To customize, copy and save this file to #" | |
| echo "# setup/fuzzer/config.restler.json #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/restlerConfig/config.json | |
| echo "" | |
| echo "############################################" | |
| echo "# Dictionary #" | |
| echo "# To customize, copy and save this file to #" | |
| echo "# setup/fuzzer/dictionary.restler.json #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/restlerConfig/dict.json | |
| echo "" | |
| echo "############################################" | |
| echo "# Annotations #" | |
| echo "# To customize, copy and save this file to #" | |
| echo "# setup/fuzzer/annotations.restler.json #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/restlerConfig/annotations.json | |
| - name: Save Restler Config | |
| id: cache-config-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/restlerConfig | |
| key: ${{ steps.cache-config-restore.outputs.cache-primary-key }} | |
| - name: Restore cached Restler grammar | |
| id: cache-grammar-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/Compile | |
| key: restler-grammar-${{ hashFiles('speckle-server/setup/fuzzer/*.json') }} | |
| - name: Generate RESTler grammar from Restler config | |
| if: steps.cache-grammar-restore.outputs.cache-hit != 'true' | |
| run: | | |
| ${{ github.workspace }}/bin/restler/Restler compile ${{ github.workspace }}/speckle-server/setup/fuzzer/config.restler.json | |
| - name: Print the contents of the Restler compile directory | |
| if: always() | |
| run: | | |
| ls -la ${{ github.workspace }} || true | |
| ls -la ${{ github.workspace }}/Compile || true | |
| - name: Save Grammar | |
| id: cache-grammar-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/Compile | |
| key: ${{ steps.cache-grammar-restore.outputs.cache-primary-key }} | |
| - name: Deploy dependencies (docker compose) | |
| run: | | |
| docker compose --file ${{ github.workspace }}/speckle-server/docker-compose-deps.yml up --detach | |
| - name: Seed the database | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install --yes --no-install-recommends postgresql-client | |
| PGPASSWORD=speckle psql -h 127.0.0.1 -U speckle -d speckle -p 5432 -w < ${{ github.workspace }}/speckle-server/setup/fuzzer/speckle.backup.sql | |
| - name: Deploy speckle-server (docker compose) | |
| working-directory: ${{ github.workspace }} | |
| timeout-minutes: 1 | |
| run: | | |
| docker compose --file ${{ github.workspace }}/speckle-server/setup/fuzzer/docker-compose-speckle.yml up --detach | |
| - name: Run RESTler coverage test | |
| run: | | |
| until curl --output /dev/null --silent --head --fail http://127.0.0.1:3000/readiness; do | |
| echo "Waiting a further 3 seconds for speckle-server to start..." | |
| sleep 3 | |
| done | |
| ${{ github.workspace }}/bin/restler/Restler test \ | |
| --grammar_file "${{ github.workspace }}/Compile/grammar.py" \ | |
| --dictionary_file "${{ github.workspace }}/speckle-server/setup/fuzzer/dictionary.restler.json" \ | |
| --settings "${{ github.workspace }}/speckle-server/setup/fuzzer/settings.restler.json" \ | |
| --no_ssl \ | |
| --target_ip "127.0.0.1" \ | |
| --target_port "3000" | |
| - name: Print the results | |
| if: always() | |
| run: | | |
| ls -la ${{ github.workspace }}/Test | |
| ls -la ${{ github.workspace }}/Test/RestlerResults || true | |
| ls -la ${{ github.workspace }}/Test/ResponseBuckets || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Engine stderr #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/EngineStdErr.txt || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Engine stdout #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/EngineStdOut.txt || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Results analyzer stderr #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/ResultsAnalyzerStdErr.txt || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Results analyzer stdout #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/ResultsAnalyzerStdOut.txt || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Coverage failures to investigate #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/coverage_failures_to_investigate.txt || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Restler logs #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/restler-*.log || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Coverage report #" | |
| echo "############################################" | |
| echo "" | |
| cat "$(find ${{ github.workspace }}/Test -type f -name "speccov.json")" || true | |
| echo "" | |
| echo "############################################" | |
| echo "# Network testing logs #" | |
| echo "############################################" | |
| echo "" | |
| cat "$(find ${{ github.workspace }}/Test -type f -name "network.testing.*.txt")" || true | |
| - name: Print Docker Compose logs | |
| if: always() | |
| run: | | |
| docker compose --file ${{ github.workspace }}/speckle-server/docker-compose-deps.yml logs | |
| docker compose --file ${{ github.workspace }}/speckle-server/setup/fuzzer/docker-compose-speckle.yml logs |