benchmarks.yml

name: Benchmarks

on:
  workflow_dispatch:
    inputs:
      change-type:
        description: How should changes be made after the benchmarks are completed?
        required: true
        default: ignore
        type: choice
        options:
          - ignore
          - commit
          - pull_request
      instance_type:
        description: Instance type to provision for benchmarking.
        required: true
        default: 'c6i.metal'
      instance_disk_size:
        description: Disk size (in GB) for the instance.
        required: true
        default: 100

concurrency:
  group: benchmarks
  cancel-in-progress: true

env:
  CARGO_TERM_COLOR: always
  TERM: xterm-256color

jobs:
  provision-runner:
    runs-on: ubuntu-latest
    outputs:
      available_region: ${{ steps.get-region.outputs.available_region }}
    steps:
    - name: AWS Login
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: us-west-2

    - name: Generate Runner Registration Token
      id: get-runner-token
      run: |
        RESPONSE=$(curl -L -X POST \
        -H "Accept: application/vnd.github+json" \
        -H "Authorization: Bearer ${{ secrets.RUNNER_TOKEN }}" \
        -H "X-GitHub-Api-Version: 2022-11-28" \
        https://api.github.com/repos/${{ github.repository }}/actions/runners/registration-token)
        TOKEN=$(echo "$RESPONSE" | jq -r .token)
        echo "REGISTER_TOKEN=$TOKEN" >> $GITHUB_ENV
        echo "::add-mask::$TOKEN"

    - name: Checkout
      uses: actions/checkout@v4

    - name: Check Capacity
      id: get-region
      run: |
        regions=(us-east-1 us-east-2 us-west-1 us-west-2 ap-south-1 ap-southeast-1 ap-southeast-2 ap-southeast-3 ap-northeast-1 ap-northeast-2 ap-northeast-3 ca-central-1 eu-central-1 eu-west-1 eu-west-2 eu-west-3 eu-north-1 sa-east-1)
        instance_type="${{ inputs.instance_type }}"
        available_region=""
        for region in "${regions[@]}"
        do
          available_region=$(aws ec2 describe-instance-type-offerings \
            --location-type availability-zone \
            --filters Name=instance-type,Values=$instance_type \
            --region $region \
            --query "InstanceTypeOfferings[?InstanceType=='$instance_type'].InstanceType" \
            --output text)
          if [ -n "$available_region" ]; then
            echo "AVAILABLE_REGION=$region" >> $GITHUB_ENV
            echo "available_region=$region" >> $GITHUB_OUTPUT
            break
          fi
        done

    - name: Check Region
      run: |
        if [ -z "${{ env.AVAILABLE_REGION }}" ]; then
          echo "No available regions for instance type ${{ inputs.instance_type }}"
          exit 1
        fi

    - name: Setup Terraform
      uses: hashicorp/setup-terraform@v3

    - name: Terraform Apply
      timeout-minutes: 30
      run: |
        cd terraform/bench-runner
        terraform init
        terraform apply -auto-approve -var="aws_region=${{ env.AVAILABLE_REGION }}" -var="instance_type=${{ inputs.instance_type }}" -var="instance_disk_size=${{ inputs.instance_disk_size }}" -var="registration_token=${{ env.REGISTER_TOKEN }}" -var="github_run_id=${{ github.run_id }}"

    - name: Notify on Failure
      if: failure()
      uses: appleboy/telegram-action@master
      with:
        to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }}
        token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }}
        format: markdown
        disable_web_page_preview: true
        message: |
          *Status*: 🔥
          *Problem*: Issue with server provisioning - bench-runner-${{ github.run_id }}
          *Details*: Check [Benchmark Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})

  benchmarks:
    needs: provision-runner
    runs-on: [bench]
    permissions:
      contents: write
      pull-requests: write
    env:
      RUSTUP_HOME: /tmp/rustup_home
    steps:
      - name: "ACTIONS: Checkout"
        uses: actions/checkout@v4

      - name: "Install: Rust toolchain"
        uses: dsherret/rust-toolchain-file@v1

      - name: "Install: Build deps"
        run: |
          sudo apt update
          sudo apt install -y git clang curl libssl-dev llvm libudev-dev cmake wabt protobuf-compiler wget bzip2
          curl -o wasm-opt-linux-x64.tar.gz -L `curl -s https://api.github.com/repos/WebAssembly/binaryen/releases/latest | jq -r '.assets[] | select(.name | contains("x86_64-linux")) | .browser_download_url' |grep -v sha256`
          tar xzf wasm-opt-linux-x64.tar.gz && sudo mv binaryen-version_*/bin/* /usr/local/bin/
          cargo install cargo-script
          wasm-opt --version

      - name: Run all benchmarks
        run: |
          INSTANCE_TYPE="${{ inputs.instance_type }}"
          INSTANCE_TYPE=${INSTANCE_TYPE} ./scripts/benchmarking/run_all_benchmarks.sh -c vara-dev -m
          cp ./scripts/benchmarking/weights-output/* runtime/vara/src/weights

          # apply weight adjustments for baremetal runner to `runtime/vara/src/weights/pallet_gear.rs`
          if [ "$INSTANCE_TYPE" = "c6i.metal" ]; then
            echo "Applying baremetal weight patch"
            ./scripts/benchmarking/apply_baremetal_weight_patch.rs -i runtime/vara/src/weights/pallet_gear.rs -o runtime/vara/src/weights/pallet_gear_patched.rs
            mv runtime/vara/src/weights/pallet_gear_patched.rs runtime/vara/src/weights/pallet_gear.rs
          fi

          # apply some patches for `pallets/gear/src/weights.rs`
          cp runtime/vara/src/weights/pallet_gear.rs pallets/gear/src/weights.rs
          sed -i -E 's/\w+::WeightInfo for SubstrateWeight/WeightInfo for SubstrateWeight/' pallets/gear/src/weights.rs

          # apply some patches for `pallets/gear-builtin/src/weights.rs`
          cp runtime/vara/src/weights/pallet_gear_builtin.rs pallets/gear-builtin/src/weights.rs
          sed -i -E 's/\w+::WeightInfo for SubstrateWeight/WeightInfo for SubstrateWeight/' pallets/gear-builtin/src/weights.rs

          # generate code for lightweight scheduler that is used in gtest and other crates
          ./scripts/weight-dump.sh

      - name: "ACTIONS: Upload artifact with benchmarking errors (if exist)"
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: benchmarking-errors
          path: scripts/benchmarking/benchmarking_errors.txt
          if-no-files-found: ignore

      - name: "ACTIONS: Upload artifact with updated weights (vara)"
        uses: actions/upload-artifact@v4
        with:
          name: weights-vara
          path: |
            runtime/vara/src/weights/
            utils/wasm-instrument/src/gas_metering/

      - name: "TIP: download the artifacts and use `./scripts/unpack-weights.sh`"
        run: |
          echo "::notice::You can download the artifacts and use script \`./scripts/unpack-weights.sh\` to apply the changes to your branch"

      - name: Create commit
        if: ${{ inputs.change-type == 'commit' }}
        uses: stefanzweifel/git-auto-commit-action@v5
        with:
          commit_message: >-
            chore(runtime): update weights
          file_pattern: "pallets/gear/src/ pallets/gear-builtin/src/ runtime/vara/src/weights/ utils/wasm-instrument/src/gas_metering/"

      - name: Create Pull Request
        if: ${{ inputs.change-type == 'pull_request' }}
        uses: peter-evans/create-pull-request@v7
        with:
          add-paths: |
            pallets/gear/src/
            pallets/gear-builtin/src/
            runtime/vara/src/weights/
            utils/wasm-instrument/src/gas_metering/
          commit-message: update weights
          branch: weights/patch
          branch-suffix: short-commit-hash
          title: >-
            chore(runtime): update weights
          body: |
            `run_all_benchmarks.sh` script is applied to update weights (branch: `${{ github.ref_name }}`, commit ${{ github.sha }})
            **Note:** If CI fails, try manually updating the heuristic tests: `runtime/vara/src/tests.rs`
          labels: |
            A0-pleasereview
            A4-insubstantial

  destroy-runner:
    if: always()
    needs: [provision-runner, benchmarks]
    runs-on: ubuntu-latest
    steps:
    - name: AWS Login
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: us-west-2

    - name: Checkout
      uses: actions/checkout@v4
      with:
        sparse-checkout: terraform

    - name: Setup Terraform
      uses: hashicorp/setup-terraform@v3

    - name: Terraform Destroy
      timeout-minutes: 60
      run: |
        cd terraform/bench-runner
        terraform init
        terraform destroy -auto-approve -var="aws_region=${{ needs.provision-runner.outputs.available_region }}" -var="instance_type=${{ inputs.instance_type }}" -var="instance_disk_size=${{ inputs.instance_disk_size }}" -var="registration_token=${{ env.REGISTER_TOKEN }}" -var="github_run_id=${{ github.run_id }}"

    - name: Notify on Failure
      if: failure()
      uses: appleboy/telegram-action@master
      with:
        to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }}
        token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }}
        format: markdown
        disable_web_page_preview: true
        message: |
          *Status*: 🔥
          *Problem*: Issue with server destroying - bench-runner-${{ github.run_id }}
          *Details*: Check [Benchmark Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})