ci.yaml

# Copyright 2024 NVIDIA CORPORATION
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: CI

on:
  pull_request:
    types:
      - opened
      - synchronize
    branches:
      - main
      - release-*
  push:
    branches:
      - main
      - release-*

jobs:
  ### Configuration checks ###
  helm-lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Install Helm
        uses: azure/setup-helm@v4.2.0
        id: install
      - run: helm lint deployments/gpu-operator/
  validate-csv:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Get Golang version
        id: vars
        run: |
          GOLANG_VERSION=$( grep "GOLANG_VERSION ?=" versions.mk )
          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> $GITHUB_ENV
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GOLANG_VERSION }}
      - run: make validate-csv
  validate-helm-values:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Get Golang version
        id: vars
        run: |
          GOLANG_VERSION=$( grep "GOLANG_VERSION ?=" versions.mk )
          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> $GITHUB_ENV
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GOLANG_VERSION }}
      - run: make validate-helm-values

  ### Golang checks and build ###
  go-check:
    needs: [helm-lint, validate-csv, validate-helm-values]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - name: Get Golang version
        id: vars
        run: |
          GOLANG_VERSION=$( grep "GOLANG_VERSION ?=" versions.mk )
          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> $GITHUB_ENV
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GOLANG_VERSION }}
      - name: Lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.60.1
          args: -v --timeout 5m
          skip-cache: true
      - run: make check
  go-test:
    needs: [helm-lint, validate-csv, validate-helm-values]
    name: unit tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Get Golang version
        id: vars
        run: |
          GOLANG_VERSION=$( grep "GOLANG_VERSION ?=" versions.mk )
          echo "GOLANG_VERSION=${GOLANG_VERSION##GOLANG_VERSION ?= }" >> $GITHUB_ENV
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GOLANG_VERSION }}
      - run: make coverage
  go-build:
    needs: [helm-lint, validate-csv, validate-helm-values]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Checkout code
      - run: make docker-build

  ### Image builds ###
  build-gpu-operator:
    needs: [go-check, go-test, go-build]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        dist: [ubi8]
    steps:
      - uses: actions/checkout@v4
        name: Check out code
      - name: Calculate build vars
        id: vars
        run: |
          echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV
          REPO_FULL_NAME="${{ github.event.pull_request.head.repo.full_name }}"
          echo "${REPO_FULL_NAME}"
          echo "LABEL_IMAGE_SOURCE=https://github.com/${REPO_FULL_NAME}" >> $GITHUB_ENV

          GENERATE_ARTIFACTS="false"
          if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then
            GENERATE_ARTIFACTS="false"
          elif [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.pull_request.head.repo.full_name }}" == "${{ github.repository }}" ]]; then
            GENERATE_ARTIFACTS="true"
          elif [[ "${{ github.event_name }}" == "push" ]]; then
            GENERATE_ARTIFACTS="true"
          fi
          echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
          echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build image
        env:
          IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator
          VERSION: ${COMMIT_SHORT_SHA}
        run: |
          echo "${VERSION}"
          make build-${{ matrix.dist }}
  build-gpu-operator-validator:
    needs: [go-check, go-test, go-build]
    runs-on: ubuntu-latest
    strategy:
      matrix:
        dist: [ubi8]
    steps:
      - uses: actions/checkout@v4
        name: Check out code
      - name: Calculate build vars
        id: vars
        run: |
          echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV
          REPO_FULL_NAME="${{ github.event.pull_request.head.repo.full_name }}"
          echo "${REPO_FULL_NAME}"
          echo "LABEL_IMAGE_SOURCE=https://github.com/${REPO_FULL_NAME}" >> $GITHUB_ENV

          GENERATE_ARTIFACTS="false"
          if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then
            GENERATE_ARTIFACTS="false"
          elif [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.pull_request.head.repo.full_name }}" == "${{ github.repository }}" ]]; then
            GENERATE_ARTIFACTS="true"
          elif [[ "${{ github.event_name }}" == "push" ]]; then
            GENERATE_ARTIFACTS="true"
          fi
          echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
          echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build image
        env:
          IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator/gpu-operator-validator
          VERSION: ${COMMIT_SHORT_SHA}
          SUBCOMPONENT: validator
        run: |
          echo "${VERSION}"
          make build-${{ matrix.dist }}

  ### e2e tests ###
  e2e-tests-containerd:
    needs: [build-gpu-operator, build-gpu-operator-validator]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Check out code
      - name: Set up Holodeck
        uses: NVIDIA/holodeck@v0.2.1
        with:
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
          holodeck_config: "tests/holodeck.yaml"
      - name: Get public dns name
        id: get_public_dns_name
        uses: mikefarah/yq@master
        with:
          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
      - name: Calculate test vars
        id: vars
        run: |
          COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}
          echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
          LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')
          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV

          echo "OPERATOR_VERSION=${COMMIT_SHORT_SHA}-ubi8" >> $GITHUB_ENV
          echo "OPERATOR_IMAGE=ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator" >> $GITHUB_ENV
          echo "VALIDATOR_VERSION=${COMMIT_SHORT_SHA}-ubi8" >> $GITHUB_ENV
          echo "VALIDATOR_IMAGE=ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator/gpu-operator-validator"  >> $GITHUB_ENV

          echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
      - name: Run e2e tests
        env:
          GPU_PRODUCT_NAME: "Tesla-T4"
          SKIP_LAUNCH: "true"
          CONTAINER_RUNTIME: "containerd"
          TEST_CASE: "./tests/cases/defaults.sh"
        run: |
          echo "${{ secrets.AWS_SSH_KEY }}" > ${private_key} && chmod 400 ${private_key}
          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${VALIDATOR_IMAGE} ${VALIDATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
          ./tests/scripts/pull.sh /tmp/logs logs
          exit $rc
      - name: Archive test logs
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: containerd-e2e-test-logs
          path: ./logs/
          retention-days: 15

  e2e-tests-nvidiadriver:
    needs: [build-gpu-operator, build-gpu-operator-validator]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        name: Check out code
      - name: Set up Holodeck
        uses: NVIDIA/holodeck@v0.2.1
        with:
          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
          holodeck_config: "tests/holodeck.yaml"
      - name: Get public dns name
        id: get_public_dns_name
        uses: mikefarah/yq@master
        with:
          cmd: yq '.status.properties[] | select(.name == "public-dns-name") | .value' /github/workspace/.cache/holodeck.yaml
      - name: Calculate test vars
        id: vars
        run: |
          COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}
          echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV
          LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')
          echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV

          echo "OPERATOR_VERSION=${COMMIT_SHORT_SHA}-ubi8" >> $GITHUB_ENV
          echo "OPERATOR_IMAGE=ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator" >> $GITHUB_ENV
          echo "VALIDATOR_VERSION=${COMMIT_SHORT_SHA}-ubi8" >> $GITHUB_ENV
          echo "VALIDATOR_IMAGE=ghcr.io/${LOWERCASE_REPO_OWNER}/gpu-operator/gpu-operator-validator"  >> $GITHUB_ENV

          echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
          echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
      - name: Run e2e tests
        env:
          GPU_PRODUCT_NAME: "Tesla-T4"
          SKIP_LAUNCH: "true"
          CONTAINER_RUNTIME: "containerd"
          TEST_CASE: "./tests/cases/nvidia-driver.sh"
        run: |
          echo "${{ secrets.AWS_SSH_KEY }}" > ${private_key} && chmod 400 ${private_key}
          ./tests/ci-run-e2e.sh ${OPERATOR_IMAGE} ${OPERATOR_VERSION} ${VALIDATOR_IMAGE} ${VALIDATOR_VERSION} ${GPU_PRODUCT_NAME} ${TEST_CASE} || rc=$?
          ./tests/scripts/pull.sh /tmp/logs logs
          exit $rc
      - name: Archive test logs
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: nvidiadriver-e2e-test-logs
          path: ./logs/
          retention-days: 15