ci.yml

name: CI

env:
  GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
  PROJECT_NAME: ${{ secrets.PROJECT_NAME }}
  CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }}
  CLUSTER_ZONE: ${{ secrets.CLUSTER_ZONE }}
  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
  ES_USER: ${{ secrets.ES_USER }}
  ES_PASS: ${{ secrets.ES_PASS }}
  MAIN_ES_IP: ${{ secrets.MAIN_ES_IP }}
  TD_QUEUE_NAME: ${{ secrets.TD_QUEUE_NAME }}
  TD_QUEUE_ZONE: ${{ secrets.TD_QUEUE_ZONE }}
  DUMP_QUEUE_NAME: ${{ secrets.DUMP_QUEUE_NAME }}
  DUMP_QUEUE_ZONE: ${{ secrets.DUMP_QUEUE_ZONE }}

# Trigger the workflow on all pull requests, and on push to specific branches
on:
  # Allow manually triggering this workflow
  workflow_dispatch:
  # run for all pull requests and pushes to certain branches
  pull_request:
  push:
    branches:
      - staging
      - trying

concurrency:
  group: ${{ github.base_ref == 'staging' && 'smci-staging' || format('smci-{0}-{1}', github.workflow, github.ref) }}
  cancel-in-progress: ${{ github.base_ref == 'staging' && false || true }}

jobs:
  ## stage 0: check which files were changed
  filter-changes:
    runs-on: ubuntu-latest
    outputs:
      nondocchanges: ${{ steps.filter.outputs.nondoc }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          # this pattern matches using picomatch syntax (used by this third party Action), which is slightly
          # different than GitHub syntax: it matches any file in any path ending in '.md'. this checks if
          # any non-markdown files were changed.
          filters: |
            nondoc:
              - '!**/*.md'

  ## stage 1: run unit tests and app tests as a prerequisite
  ## these run on all pushes to all pull requests, all branches
  ## note that secrets may not be accessible in this phase
  quicktests:
    runs-on: ubuntu-latest
    needs: filter-changes
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    # should not take more than 7-8 mins
    timeout-minutes: 10
    steps:
      - name: Add OpenCL support
        run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2
      - name: checkout
        uses: actions/checkout@v3
      - name: set up go
        uses: actions/setup-go@v4
        with:
          check-latest: true
          go-version-file: "go.mod"
      - name: fmt, tidy, generate
        run: |
          make install
          make test-fmt
          make test-tidy
          make test-generate
      - name: staticcheck
        run: make staticcheck

  lint:
    runs-on: ubuntu-latest
    needs: filter-changes
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    # should not take more than 4-6 mins
    timeout-minutes: 10
    steps:
      - name: checkout
        uses: actions/checkout@v3
      - name: set up go
        uses: actions/setup-go@v4
        with:
          check-latest: true
          go-version-file: "go.mod"
      - name: setup env
        run: make install
      - name: lint
        run: make lint-github-action

  build:
    runs-on: ${{ matrix.os }}
    needs: filter-changes
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    strategy:
      fail-fast: true
      matrix:
        os:
          - ubuntu-latest
          - [self-hosted, linux, arm64]
          - macos-latest
          - [self-hosted, macos, arm64]
          - windows-latest
    steps:
      - name: Add OpenCL support - Ubuntu
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2
      - name: Override SDKROOT for macOS
        if: ${{ contains(matrix.os, 'macos') && runner.arch == 'arm64' }}
        run: echo "SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.3.sdk" >> $GITHUB_ENV
      - name: disable Windows Defender - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: |
          Set-MpPreference -DisableRealtimeMonitoring $true
      - name: checkout
        uses: actions/checkout@v3
      - name: set up go
        uses: actions/setup-go@v4
        with:
          check-latest: true
          go-version-file: "go.mod"
          cache: ${{ runner.arch != 'arm64' }}
      - name: setup env
        run: make install
      - name: build
        timeout-minutes: 5
        run: make build

  unittests:
    runs-on: ${{ matrix.os }}
    needs: filter-changes
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    strategy:
      fail-fast: true
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
          - [self-hosted, macos, arm64]
    steps:
      # as we use some request to localhost, sometimes it gives us flaky tests. try to disable tcp offloading for fix it
      # https://github.com/actions/virtual-environments/issues/1187
      - name: disable TCP/UDP offload - Ubuntu
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo ethtool -K eth0 tx off
          sudo ethtool -K eth0 rx off
      - name: disable TCP/UDP offload - MacOS
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          sudo sysctl -w net.link.generic.system.hwcksum_tx=0
          sudo sysctl -w net.link.generic.system.hwcksum_rx=0
      - name: disable TCP/UDP offload - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: |
          Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6 -UdpIPv6
      - name: disable Windows Defender - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: |
          Set-MpPreference -DisableRealtimeMonitoring $true
      - name: checkout
        uses: actions/checkout@v3
        with:
          lfs: true
      - name: set up go
        uses: actions/setup-go@v4
        with:
          check-latest: true
          go-version-file: "go.mod"
          cache: ${{ runner.arch != 'arm64' }}
      - name: Add OpenCL support - Ubuntu
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2
      - name: Override SDKROOT for macOS
        if: ${{ contains(matrix.os, 'macos') && runner.arch == 'arm64' }}
        run: echo "SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.3.sdk" >> $GITHUB_ENV
      - name: Add OpenCL support - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: choco install opencl-intel-cpu-runtime
      - name: setup env
        run: make install
      - name: Clear test cache
        run: make clear-test-cache
      - name: unit tests
        timeout-minutes: 25
        env:
          GOTESTSUM_FORMAT: standard-verbose
          GOTESTSUM_JUNITFILE: unit-tests.xml
        run: make test
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        # always run even if the previous step fails
        if: always()
        with:
          report_paths: "**/unit-tests.xml"
          annotate_only: true

  unittests-slower:
    runs-on: ${{ matrix.os }}
    needs: filter-changes
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - [self-hosted, linux, arm64]
          - windows-latest
    steps:
      # as we use some request to localhost, sometimes it gives us flaky tests. try to disable tcp offloading for fix it
      # https://github.com/actions/virtual-environments/issues/1187
      - name: disable TCP/UDP offload - Ubuntu
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          sudo ethtool -K eth0 tx off
          sudo ethtool -K eth0 rx off
      - name: disable TCP/UDP offload - MacOS
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          sudo sysctl -w net.link.generic.system.hwcksum_tx=0
          sudo sysctl -w net.link.generic.system.hwcksum_rx=0
      - name: disable TCP/UDP offload - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: |
          Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6 -UdpIPv6
      - name: disable Windows Defender - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: |
          Set-MpPreference -DisableRealtimeMonitoring $true
      - name: checkout
        uses: actions/checkout@v3
        with:
          lfs: true
      - name: set up go
        uses: actions/setup-go@v4
        with:
          check-latest: true
          go-version-file: "go.mod"
          cache: ${{ runner.arch != 'arm64' }}
      - name: Add OpenCL support - Ubuntu
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: sudo apt-get update -q && sudo apt-get install -qy ocl-icd-opencl-dev libpocl2
      - name: Override SDKROOT for macOS
        if: ${{ contains(matrix.os, 'macos') && runner.arch == 'arm64' }}
        run: echo "SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX12.3.sdk" >> $GITHUB_ENV
      - name: Add OpenCL support - Windows
        if: ${{ matrix.os == 'windows-latest' }}
        run: choco install opencl-intel-cpu-runtime
      - name: setup env
        run: make install
      - name: Clear test cache
        run: make clear-test-cache
      - name: unit tests
        timeout-minutes: 35
        env:
          GOTESTSUM_FORMAT: standard-verbose
          GOTESTSUM_JUNITFILE: unit-tests.xml
        run: make test
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        # always run even if the previous step fails
        if: always()
        with:
          report_paths: "**/unit-tests.xml"
          annotate_only: true

  ci-status:
    # run regardless of status of previous jobs but skip if the required secret is not accessible
    if: always()
    needs:
      - filter-changes
      - quicktests
      - lint
      - build
      - unittests
    runs-on: ubuntu-latest
    env:
      # short-circuit success if no non-doc files were modified
      # this is the easiest way to access success/failure state of previous jobs in this workflow
      status: ${{ (needs.filter-changes.outputs.nondocchanges == 'false' || (needs.quicktests.result == 'success' && needs.build.result == 'success' && needs.lint.result == 'success' && needs.unittests.result == 'success')) && 'success' || 'failure' }}
    steps:
      - uses: act10ns/slack@v2
        name: Slack notification
        # skip if the secret is not accessible
        if: env.SLACK_WEBHOOK_URL
        with:
          status: ${{ env.status }}
      - name: Mark the job as succeeded
        if: env.status == 'success'
        run: exit 0
      - name: Mark the job as failed
        if: env.status != 'success'
        run: exit 1