ci(tests): fuzz testing workflow for REST API #21
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: REST API Fuzz Test | |
| on: | |
| workflow_dispatch: | |
| pull_request: # Pushing a new commit to the HEAD ref of a pull request will trigger the “synchronize” event | |
| paths: | |
| - .yarnrc.yml . | |
| - .yarn | |
| - package.json | |
| - '.github/workflows/rest-api-fuzzer.yml' | |
| - 'packages/server/**/*' | |
| - 'packages/shared/**/*' | |
| env: | |
| BUILD_CONFIGURATION: Release | |
| BUILD_PLATFORM: 'Any CPU' | |
| RESTLER_VERSION: '9.2.4' | |
| PYTHON_VERSION: '3.8' | |
| DOTNET_VERSION: '6.0.x' | |
| jobs: | |
| build-restler-fuzzer: | |
| name: Fuzz test speckle-server REST API | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| services: | |
| postgres: | |
| # Docker Hub image | |
| image: postgres:16.4-bookworm@sha256:91f464e7ba0ad91a106c94cff079fb4384139291b8c0502fd36989cf2c788bbb | |
| env: | |
| POSTGRES_DB: fuzz_test | |
| POSTGRES_PASSWORD: fuzz_test | |
| POSTGRES_USER: fuzz_test | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| redis: | |
| image: redis:7-alpine@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275 | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| name: Checkout RESTler Fuzzer | |
| with: | |
| repository: microsoft/restler-fuzzer | |
| ref: v${{ env.RESTLER_VERSION }} | |
| path: 'restler-fuzzer' # The path to clone the repository within the {{ github.workspace }} directory | |
| - name: Setup .NET ${{ env.DOTNET_VERSION }} | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| cache: true | |
| cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/src/Restler.sln | |
| - name: Restore NuGet packages | |
| run: dotnet restore ${{ github.workspace }}/restler-fuzzer/src/Restler.sln | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| cache-dependency-path: ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt | |
| - name: Install engine (Python) dependencies | |
| run: | | |
| pip install -r ${{ github.workspace }}/restler-fuzzer/restler/requirements.txt | |
| - name: Build RESTler | |
| run: | | |
| python ${{ github.workspace }}/restler-fuzzer/build-restler.py --dest_dir ${{ github.workspace }}/bin | |
| - name: Debug the built output | |
| run: | | |
| ls -la ${{ github.workspace }}/bin/restler | |
| ls -la ${{ github.workspace }}/bin/restler/Restler | |
| - uses: actions/checkout@v4 | |
| name: Checkout speckle-server | |
| with: | |
| path: 'speckle-server' | |
| - name: Restore cached Grammar | |
| id: cache-grammar-restore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/Compile | |
| key: restler-grammar-${{ hashFiles('speckle-server/utils/specifications/speckle-server.openapi.json') }} | |
| - name: Compile RESTler grammar from OpenAPI specification | |
| if: steps.cache-grammar-restore.outputs.cache-hit != 'true' | |
| run: | | |
| ${{ github.workspace }}/bin/restler/Restler compile --api_spec ${{ github.workspace }}/speckle-server/utils/specifications/speckle-server.openapi.json | |
| - name: Print the Restler configuration | |
| run: | | |
| ls -la ${{ github.workspace }}/Compile | |
| echo "" | |
| echo "Engine settings:" | |
| cat ${{ github.workspace }}/Compile/engine_settings.json | |
| echo "" | |
| echo "Config:" | |
| cat ${{ github.workspace }}/Compile/config.json | |
| echo "" | |
| echo "Dictionary:" | |
| cat ${{ github.workspace }}/Compile/dict.json | |
| - name: Save Grammar | |
| id: cache-grammar-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| ${{ github.workspace }}/Compile | |
| key: ${{ steps.cache-grammar-restore.outputs.cache-primary-key }} | |
| - name: Minio | |
| # Minio requires a command to be passed to the container, and github action services do not allow commands to be passed to containers. So we create the container manually. | |
| run: | | |
| docker create --name minio -p 9000:9000 minio/minio@sha256:1dce27c494a16bae114774f1cec295493f3613142713130c2d22dd5696be6ad3 server | |
| - name: Install Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 22 | |
| cache: 'yarn' | |
| cache-dependency-path: ${{ github.workspace }}/speckle-server/yarn.lock | |
| - name: Install dependencies | |
| working-directory: ${{ github.workspace }}/speckle-server/packages/server | |
| run: yarn install | |
| - name: Build public packages | |
| working-directory: ${{ github.workspace }}/speckle-server/packages/server | |
| run: yarn build:public | |
| - name: Build speckle-server | |
| working-directory: ${{ github.workspace }}/speckle-server/packages/server | |
| run: yarn build | |
| - name: Configure speckle-server | |
| working-directory: ${{ github.workspace }}/speckle-server/packages/server | |
| run: | | |
| cp ${{ github.workspace }}/speckle-server/setup/fuzzer/.env.fuzz-test-example .env | |
| - name: Run speckle-server | |
| working-directory: ${{ github.workspace }}/speckle-server/packages/server | |
| run: yarn start & | |
| - name: Run RESTler coverage test | |
| run: | | |
| ${{ github.workspace }}/bin/restler/Restler test \ | |
| --grammar_file ${{ github.workspace }}/Compile/grammar.py \ | |
| --dictionary_file ${{ github.workspace }}/Compile/dict.json \ | |
| --settings ${{ github.workspace }}/speckle-server/setup/fuzzer/settings.restler.json \ | |
| --no_ssl \ | |
| --target_ip 127.0.0.1 \ | |
| --target_port 3000 | |
| - name: Print the results | |
| run: | | |
| ls -la ${{ github.workspace }}/Test | |
| echo "" | |
| echo "############################################" | |
| echo "# Engine stderr #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/EngineStdErr.txt | |
| echo "" | |
| echo "############################################" | |
| echo "# Engine stdout #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/EngineStdOut.txt | |
| echo "" | |
| echo "############################################" | |
| echo "# Results analyzer stderr #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/ResultsAnalyzerStdErr.txt | |
| echo "" | |
| echo "############################################" | |
| echo "# Results analyzer stdout #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/ResultsAnalyzerStdOut.txt | |
| echo "" | |
| echo "############################################" | |
| echo "# Coverage failures to investigate #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/coverage_failures_to_investigate.txt | |
| echo "" | |
| echo "############################################" | |
| echo "# Restler logs #" | |
| echo "############################################" | |
| echo "" | |
| cat ${{ github.workspace }}/Test/restler-*.log | |
| echo "" | |
| echo "############################################" | |
| echo "# Coverage report #" | |
| echo "############################################" | |
| echo "" | |
| cat "$(find ${{ github.workspace }}/Test -type f -name "speccov.json")" | |
| - name: Print Minio logs | |
| if: always() | |
| run: | | |
| docker logs minio |