tests.yml

name: Tests
on:
  push:
    branches:
      - main
      - stable-*
  pull_request:
  workflow_dispatch:
  schedule:
    - cron: '0 0 * * *'  # Test TICS daily

env:
  LXD_REQUIRED_TESTS: "storage_buckets"
  LXD_SKIP_TESTS: "clustering_upgrade clustering_upgrade_large"
  GOCOVERDIR: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && '/home/runner/work/lxd/lxd/coverage' || '' }}

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

defaults:
  run:
    # Make sure bash is always invoked with `-eo pipefail`
    # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
    shell: bash

jobs:
  code-tests:
    env:
      CGO_CFLAGS: "-I/home/runner/go/bin/dqlite/include/"
      CGO_LDFLAGS: "-L/home/runner/go/bin/dqlite/libs/"
      LD_LIBRARY_PATH: "/home/runner/go/bin/dqlite/libs/"
      LD_RUN_PATH: "/home/runner/go/bin/dqlite/libs/"
      CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
    name: Code
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # A non-shallow clone is needed for the Differential ShellCheck
          fetch-depth: 0

      - name: Tune disk performance
        uses: ./.github/actions/tune-disk-performance

      - name: Dependency Review
        uses: actions/dependency-review-action@v4
        if: github.event_name == 'pull_request'

      # XXX: `make static-analysis` also run shellcheck but this one provides
      #      useful feedback in the PR through github-code-scanning bot
      - id: ShellCheck
        name: Differential ShellCheck
        uses: redhat-plumbers-in-action/differential-shellcheck@v5
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          strict-check-on-push: true
        if: github.event_name == 'pull_request'

      - name: Upload artifact with ShellCheck defects in SARIF format
        uses: actions/upload-artifact@v4
        with:
          name: Differential ShellCheck SARIF
          path: ${{ steps.ShellCheck.outputs.sarif }}
        if: github.event_name == 'pull_request'

      - name: Install Go (1.22)
        uses: actions/setup-go@v5
        with:
          go-version: 1.22.x

      - name: Install build dependencies
        uses: ./.github/actions/install-lxd-builddeps

      - name: Install test dependencies
        run: |
          python3 -m pip install flake8

          # Download minio ready to include in dependencies for system tests.
          mkdir -p "$(go env GOPATH)/bin"
          curl -sSfL https://dl.min.io/server/minio/release/linux-amd64/minio --output "$(go env GOPATH)/bin/minio"
          chmod +x "$(go env GOPATH)/bin/minio"

          # Also grab the latest minio client to maintain compatibility with the server.
          curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc --output "$(go env GOPATH)/bin/mc"
          chmod +x "$(go env GOPATH)/bin/mc"

      - name: Download go dependencies
        run: |
          set -eux
          go mod download

      - name: Check compatibility with min Go version
        run: |
          set -eux
          GOMIN="$(sed -n 's/^GOMIN=\([0-9.]\+\)$/\1/p' Makefile)"
          sudo chmod o+w {go.mod,go.sum}
          go mod tidy -go="${GOMIN}"

          DOC_GOMIN="$(sed -n 's/^LXD requires Go \([0-9.]\+\) .*/\1/p' doc/requirements.md)"
          [ "${GOMIN}" = "${DOC_GOMIN}" ]

      - name: Make LXD tarball and unpack it
        env:
          CUSTOM_VERSION: "test"
        run: |
          set -eux
          make dist
          tar -xzf lxd-test.tar.gz -C /home/runner/work/lxd/
          rm lxd-test.tar.gz

      - name: Build LXD dependencies
        run: |
          set -eux

          # Build from unpacked dist tarball.
          cd /home/runner/work/lxd/lxd-test
          make deps

          # Include dqlite libs in dependencies for system tests.
          mkdir /home/runner/go/bin/dqlite
          mv /home/runner/work/lxd/lxd-test/vendor/dqlite/include /home/runner/go/bin/dqlite/include
          mv /home/runner/work/lxd/lxd-test/vendor/dqlite/.libs /home/runner/go/bin/dqlite/libs

      - name: Build binaries
        run: |
          set -eux

          # Build from unpacked dist tarball.
          cd /home/runner/work/lxd/lxd-test
          make

          strip --strip-all /home/runner/go/bin/{lxc*,lxd*} -v

      - name: Check lxc/lxd-agent binary sizes
        run: |
          set -eux

          # Build lxc/lxd-agent the same way as done in the snap
          go build -trimpath -o "/tmp/bin/lxc" github.com/canonical/lxd/lxc
          CGO_ENABLED=0 go build -trimpath -o "/tmp/bin/lxd-agent" -tags=agent,netgo github.com/canonical/lxd/lxd-agent
          strip -s /tmp/bin/*

          # bin/max (sizes are in MiB)
          SIZES="lxc 15
                 lxd-agent 13"
          MIB="$((1024 * 1024))"

          while read -r bin max; do
            cur="$(stat --format=%s "/tmp/bin/${bin}")"
            min=$((max - 1))
            min_mib="$((min * MIB))"
            max_mib="$((max * MIB))"
            rm -f "/tmp/bin/${bin}"

            if [ "${cur}" -gt "${max_mib}" ]; then
              echo "FAIL: ${bin} binary size exceeds ${max}MiB"
              exit 1
            fi

            # XXX: check for when we need to lower the min/max sizes
            if [ "${cur}" -lt "${min_mib}" ]; then
              echo "Congratulations: ${bin} binary size reduced below ${min}MiB"
              echo "It is now time to edit the workflow job to use smaller min/max sizes for ${bin}"
              exit 1
            fi

            echo "OK: ${bin} is between ${min} and ${max}MiB"
          done <<< ${SIZES}

      - name: Make GOCOVERDIR
        run: mkdir -p "${GOCOVERDIR}"
        if: env.GOCOVERDIR != ''

      - name: Run static analysis
        env:
          GITHUB_BEFORE: ${{ github.event.before }}
        run: |
          set -eux
          sudo chmod o+w ./lxd/metadata/configuration.json
          sudo chmod o+w ./doc/metadata.txt
          sudo chmod o+w ./po/*
          sudo chmod o+w ./lxd/auth/entitlements_generated.go
          sudo chmod o+w ./lxd/auth/drivers/openfga_model.openfga
          make static-analysis

      - name: Unit tests (all)
        run: |
          set -eux
          sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,GOCOVERDIR,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" make check-unit

      - name: Upload coverage data
        uses: actions/upload-artifact@v4
        with:
          name: coverage-unit
          path: ${{env.GOCOVERDIR}}
        if: env.GOCOVERDIR != ''

      - name: Upload system test dependencies
        uses: actions/upload-artifact@v4
        with:
          name: system-test-deps
          path: |
            /home/runner/go/bin/lxc*
            /home/runner/go/bin/lxd*
            /home/runner/go/bin/mc
            /home/runner/go/bin/minio
            /home/runner/go/bin/dqlite
          retention-days: 1

  system-tests:
    env:
      LXD_CEPH_CLUSTER: "ceph"
      LXD_CEPH_CEPHFS: "cephfs"
      LXD_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1"
      LXD_CONCURRENT: "1"
      LXD_VERBOSE: "1"
      LXD_OFFLINE: "1"
      LXD_TMPFS: "1"
    name: System
    runs-on: ubuntu-22.04
    needs: code-tests
    strategy:
      fail-fast: false
      matrix:
        go: ["1.22.x"]
        suite: ["cluster", "standalone"]
        backend: ["dir", "btrfs", "lvm", "zfs", "ceph", "random"]
        include:
          - go: stable
            suite: cluster
            backend: dir
          - go: stable
            suite: standalone
            backend: dir

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Tune disk performance
        uses: ./.github/actions/tune-disk-performance

      - name: Reclaim disk space
        uses: ./.github/actions/reclaim-disk-space

      - name: Remove docker
        uses: ./.github/actions/disable-docker

      - name: Install Go (${{ matrix.go }})
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go }}

      - name: Install dependencies
        run: |
          set -eux
          sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
          sudo apt-get update

          sudo systemctl mask lxc.service lxc-net.service

          sudo apt-get install --no-install-recommends -y \
            curl \
            git \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            liblxc-dev \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            make \
            pkg-config\
            acl \
            attr \
            bind9-dnsutils \
            btrfs-progs \
            busybox-static \
            dnsmasq-base \
            easy-rsa \
            gettext \
            jq \
            lxc-utils \
            lvm2 \
            nftables \
            quota \
            rsync \
            s3cmd \
            socat \
            sqlite3 \
            squashfs-tools \
            tar \
            tcl \
            thin-provisioning-tools \
            uuid-runtime \
            xfsprogs \
            xz-utils \
            zfsutils-linux

          # reclaim some space
          sudo apt-get clean

      - name: Download system test dependencies
        uses: actions/download-artifact@v4
        with:
          name: system-test-deps
          merge-multiple: true
          path: /home/runner/go/bin

      - name: Set exec perms on LXD binaries
        run: |
          set -eux
          ls -lR /home/runner/go/bin/
          chmod uog+x /home/runner/go/bin/*

      - name: Setup MicroCeph
        if: ${{ matrix.backend == 'ceph' }}
        run: |
          set -eux

          cleanup() {
            set +e
            # dmesg may contain oops, IO errors, crashes, etc
            echo "::group::dmesg logs"
            journalctl --quiet --no-hostname --no-pager --boot=0 --lines=100 --dmesg
            echo "::endgroup::"
            exit 1
          }
          trap cleanup ERR HUP INT TERM

          # If the rootfs and the ephemeral part are on the same physical disk, giving the whole
          # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
          # runners to have a single disk, we immediately bail rather than trying to gracefully
          # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
          # we will be able to stop worrying about that special case.
          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
            exit 1
          fi

          # Free-up the ephemeral disk to use it as ceph OSD.
          # https://github.com/canonical/microceph/issues/288 and https://github.com/canonical/microceph/issues/289
          sudo swapoff /mnt/swapfile
          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
          sudo umount /mnt

          sudo snap install microceph --edge
          sudo snap connect microceph:mount-observe
          sudo apt-get install --no-install-recommends -y ceph-common
          sudo microceph cluster bootstrap
          sudo microceph.ceph config set global osd_pool_default_size 1
          sudo microceph.ceph config set global mon_allow_pool_delete true
          sudo microceph.ceph config set global osd_memory_target 939524096
          sudo microceph.ceph osd crush rule rm replicated_rule
          sudo microceph.ceph osd crush rule create-replicated replicated default osd
          for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
              sudo microceph.ceph osd set $flag
          done
          sudo microceph disk add --wipe "${ephemeral_disk}"
          sudo rm -rf /etc/ceph
          sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
          sudo microceph enable rgw || true # workaround to ignore already enabled rgw
          sudo microceph.ceph osd pool create cephfs_meta 32
          sudo microceph.ceph osd pool create cephfs_data 32
          sudo microceph.ceph fs new cephfs cephfs_meta cephfs_data
          sudo microceph.ceph fs ls
          sleep 30
          sudo microceph.ceph status
          # Wait until there are no more "unkowns" pgs
          for _ in $(seq 60); do
            if sudo microceph.ceph pg stat | grep -wF unknown; then
              sleep 1
            else
              break
            fi
          done
          sudo microceph.ceph status
          sudo rm -f /snap/bin/rbd

          # reclaim some space
          sudo apt-get clean

      - name: Make GOCOVERDIR
        run: |
          set -eux
          mkdir -p "${GOCOVERDIR}"
        if: env.GOCOVERDIR != ''

      - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }}, ${{ matrix.backend }})"
        run: |
          set -eux
          chmod +x ~
          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
          cd test
          sudo --preserve-env=PATH,GOPATH,GOCOVERDIR,GITHUB_ACTIONS,LXD_VERBOSE,LXD_BACKEND,LXD_CEPH_CLUSTER,LXD_CEPH_CEPHFS,LXD_CEPH_CEPHOBJECT_RADOSGW,LXD_OFFLINE,LXD_SKIP_TESTS,LXD_REQUIRED_TESTS, LXD_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}

      - name: Upload coverage data
        uses: actions/upload-artifact@v4
        with:
          name: coverage-${{ matrix.go }}-${{ matrix.suite }}-${{ matrix.backend }}
          path: ${{env.GOCOVERDIR}}
        if: env.GOCOVERDIR != ''

  tics:
    name: Tiobe TICS
    runs-on: ubuntu-22.04
    needs: system-tests
    env:
      CGO_CFLAGS: "-I/home/runner/go/bin/dqlite/include/"
      CGO_LDFLAGS: "-L/home/runner/go/bin/dqlite/libs/"
      LD_LIBRARY_PATH: "/home/runner/go/bin/dqlite/libs/"
      CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
    if: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # A non-shallow clone is needed for the Differential ShellCheck
          fetch-depth: 0

      - name: Tune disk performance
        uses: ./.github/actions/tune-disk-performance

      - name: Reclaim disk space
        uses: ./.github/actions/reclaim-disk-space

      - name: Remove docker
        uses: ./.github/actions/disable-docker

      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: 1.22.x

      - name: Download coverage data
        uses: actions/download-artifact@v4
        with:
          pattern: coverage-*
          path: ${{env.GOCOVERDIR}}
          merge-multiple: true

      - name: Download system test dependencies
        uses: actions/download-artifact@v4
        with:
          name: system-test-deps
          merge-multiple: true
          path: /home/runner/go/bin

      - name: Install build dependencies
        uses: ./.github/actions/install-lxd-builddeps

      - name: Install dependencies
        run: |
          go install github.com/axw/gocov/gocov@latest
          go install github.com/AlekSi/gocov-xml@latest
          go install honnef.co/go/tools/cmd/staticcheck@latest

      - name: Convert coverage files
        run: |
          go tool covdata textfmt -i="${GOCOVERDIR}" -o "${GOCOVERDIR}"/coverage.out
          gocov convert "${GOCOVERDIR}"/coverage.out > "${GOCOVERDIR}"/coverage.json
          gocov-xml < "${GOCOVERDIR}"/coverage.json > "${GOCOVERDIR}"/coverage-go.xml

      - name: Run TICS
        uses: tiobe/tics-github-action@v3
        with:
          mode: qserver
          project: LXD
          viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default
          branchdir: ${{ github.workspace }}
          ticsAuthToken: ${{ secrets.TICS_AUTH_TOKEN }}
          installTics: true
          calc: ALL
          tmpdir: /tmp/tics

  client:
    name: Client
    strategy:
      fail-fast: false
      matrix:
        go:
          - 1.22.x
        os:
          - ubuntu-latest
          - macos-latest
          - windows-latest
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Go (${{ matrix.go }})
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go }}

      - name: Create build directory
        run: |
          set -eux
          mkdir bin

      - name: Build static lxc (x86_64)
        env:
          CGO_ENABLED: 0
          GOARCH: amd64
        run: |
          set -eux
          go build -ldflags "-s -w" -o trimpath -o bin/lxc.x86_64 ./lxc

      - name: Build static lxc (aarch64)
        env:
          CGO_ENABLED: 0
          GOARCH: arm64
        run: |
          set -eux
          go build -ldflags "-s -w" -o trimpath -o bin/lxc.aarch64 ./lxc

      - name: Build static lxd-benchmark
        if: runner.os == 'Linux'
        env:
          CGO_ENABLED: 0
        run: |
          set -eux
          GOARCH=amd64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-benchmark.x86_64 ./lxd-benchmark
          GOARCH=arm64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-benchmark.aarch64 ./lxd-benchmark

      - name: Build static lxd-migrate
        if: runner.os == 'Linux'
        env:
          CGO_ENABLED: 0
        run: |
          set -eux
          GOARCH=amd64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-migrate.x86_64 ./lxd-migrate
          GOARCH=arm64 go build -ldflags "-s -w" -o trimpath -o bin/lxd-migrate.aarch64 ./lxd-migrate

      - name: Unit tests (client)
        env:
          CGO_ENABLED: 0
        run: |
          set -eux
          go test -v ./client/...

      - name: Unit tests (lxc)
        env:
          CGO_ENABLED: 0
        run: |
          set -eux
          go test -v ./lxc/...

      - name: Unit tests (shared)
        env:
          CGO_ENABLED: 0
        run: |
          set -eux
          go test -v ./shared/...

      - name: Upload lxc client artifacts
        uses: actions/upload-artifact@v4
        continue-on-error: true
        with:
          name: lxd-clients-${{ runner.os }}
          path: bin/

  documentation:
    name: Documentation
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Go (1.22)
        uses: actions/setup-go@v5
        with:
          go-version: 1.22.x

      - name: Install dependencies
        run: |
          set -eux
          sudo apt-get install aspell aspell-en
          sudo snap install mdl

      - name: Run markdown linter
        run: |
          set -eux
          make doc-lint

      - name: Build docs (Sphinx)
        shell: 'script -q -e -c "export TERM=xterm-256color; bash {0}"'
        run: |
          set -eux
          make doc
          if [ -s doc/.sphinx/warnings.txt ]; then cat doc/.sphinx/warnings.txt; exit 1; fi

      - name: Run spell checker
        run: |
          set -eux
          make doc-spellcheck

      - name: Run inclusive naming checker
        uses: get-woke/woke-action@v0
        with:
          fail-on-error: true
          woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml"

      - name: Run link checker
        # Run link checker on PRs only
        if: ${{ github.event_name != 'push' }}
        shell: 'script -q -e -c "export TERM=xterm-256color; bash {0}"'
        run: |
          set -eux
          make doc-linkcheck

      - name: Upload documentation artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: documentation
          path: doc/_build

  snap:
    name: Trigger snap edge build
    runs-on: ubuntu-22.04
    needs: [code-tests, system-tests, client, documentation]
    if: ${{ github.repository == 'canonical/lxd' && github.event_name == 'push' && github.actor != 'dependabot[bot]' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Launchpad SSH access
        env:
          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
          LAUNCHPAD_LXD_BOT_KEY: ${{ secrets.LAUNCHPAD_LXD_BOT_KEY }}
        run: |
          set -eux
          mkdir -m 0700 -p ~/.ssh/
          ssh-agent -a "${SSH_AUTH_SOCK}" > /dev/null
          ssh-add - <<< "${{ secrets.LAUNCHPAD_LXD_BOT_KEY }}"
          ssh-add -L > ~/.ssh/id_ed25519.pub
          # In ephemeral environments like GitHub Action runners, relying on TOFU isn't providing any security
          # so require the key obtained by `ssh-keyscan` to match the expected hash from https://help.launchpad.net/SSHFingerprints
          ssh-keyscan git.launchpad.net >> ~/.ssh/known_hosts
          ssh-keygen -qlF git.launchpad.net | grep -xF 'git.launchpad.net RSA SHA256:UNOzlP66WpDuEo34Wgs8mewypV0UzqHLsIFoqwe8dYo'

      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: 1.22.x

      - name: Trigger Launchpad snap build
        env:
          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
          TARGET: >-
            ${{ fromJson('{
              "main": "latest-edge",
              "stable-5.0": "5.0-edge",
            }')[github.ref_name] }}
        run: |
          set -eux
          git config --global transfer.fsckobjects true
          git config --global user.name "Canonical LXD Bot"
          git config --global user.email "lxd@lists.canonical.com"
          git config --global commit.gpgsign true
          git config --global gpg.format "ssh"
          git config --global user.signingkey ~/.ssh/id_ed25519.pub
          localRev="$(git rev-parse HEAD)"
          go install github.com/canonical/lxd-ci/lxd-snapcraft@latest
          git clone -b "${TARGET}" git+ssh://lxdbot@git.launchpad.net/~canonical-lxd/lxd ~/lxd-pkg-snap-lp
          cd ~/lxd-pkg-snap-lp
          lxd-snapcraft -package lxd -set-version "git-${localRev:0:7}" -set-source-commit "${localRev}"
          git add --all
          git commit --all --quiet -s --allow-empty -m "Automatic upstream build (${TARGET})" -m "Upstream commit: ${localRev}"
          git show
          git push --quiet