.gitlab-ci.yml

variables:
  # NOTE: Custom variables should never start with CI_ prefix.
  #       This namespace belongs to Gitlab CI/CD.
  #       https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
  DOCKER_VERSION:
    value: "27"
    description: "Version of docker to use in pipelines"
  SKOPEO_VERSION:
    value: "v1.16.1"
    description: "Version of skopeo to use for publishing images"
  GOLANG_VERSION:
    value: "1.23.2"
    description: "Version of Golang to use in jobs"
  IMAGE_GOLANGCI_VERSION:
    value: "v1.60"
    description: "Version of image golangci/golangci-lint for static checks"
  DOCKER_PLATFORM:
    value: "linux/amd64,linux/arm64"
    description: "Platforms to build container images"

  RULES_CHANGES_COMPARE_TO_REF:
    value: "refs/heads/main"
    description: "Which reference to compare rules about changes (usually set by push option)"

  # Defines the docker tags of built artifacts objects
  MENDER_IMAGE_REGISTRY: "${CI_REGISTRY}"
  MENDER_IMAGE_REPOSITORY: "northern.tech/mender/${CI_PROJECT_NAME}"
  MENDER_IMAGE_TAG: "build-${CI_COMMIT_SHA}"
  MENDER_IMAGE_TAG_TEST: "test-${CI_COMMIT_SHA}"
  MENDER_IMAGE_TAG_BUILDER: "builder-${CI_COMMIT_SHA}"

  GOCOVERDIR: "${CI_PROJECT_DIR}/backend/tests/cover"

  # release and changelog generators
  GITHUB_REPO_URL:
    description: "The Github Repo URL for release-please, in the format of 'owner/repo'"
    value: "mendersoftware/mender-server"
  GITHUB_USER_NAME:
    description: "The Github username for release-please"
    value: "mender-test-bot"
  GITHUB_USER_EMAIL:
    description: "The Github user email for release-please"
    value: "mender@northern.tech"
  GIT_CLIFF:
    description: "Run git cliff to override the release-please changelog"
    value: "true"
    options:
      - "true"
      - "false"

include:
  - project: "Northern.tech/Mender/mendertesting"
    file:
      - ".gitlab-ci-check-commits.yml"
      - ".gitlab-ci-github-status-updates.yml"
  - local: "/frontend/pipeline.yml"

stages:
  - lint
  - build
  - test
  - publish
  - changelog

default:
  tags:
    - hetzner-amd-beefy

.dind-login: &dind-login
  - mkdir -p $HOME/.docker && echo $DOCKER_AUTH_CONFIG > $HOME/.docker/config.json
  - docker login --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY

.template:build:docker:
  stage: build
  needs: []
  tags:
    - hetzner-amd-beefy
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-cli
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  variables:
    DOCKER_BUILDARGS: "--push"
  before_script:
    - apk add make bash git
    - *dind-login
    # NOTE: If we're running on a PR, do not build multiplatform
    - test "$CI_COMMIT_REF_PROTECTED" != "true" && unset DOCKER_PLATFORM
    - if test -n "${DOCKER_PLATFORM}"; then
      docker context create ci;
      docker builder create --name ci-builder ci;
      export DOCKER_BUILDARGS="${DOCKER_BUILDARGS} --builder=ci-builder";
      unset DOCKER_HOST;
      fi

build:backend:docker:
  extends: .template:build:docker
  rules:
    - changes:
        paths: ["backend/**/*"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
      when: always
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: always
  script:
    # FIXME: Only exporting deployments build stage to run unit tests
    #        We're assuming the images have consistent GOTOOLCHAIN.
    #        Will be fixed once we optimize to template based pipeline.
    - |-
      make -C backend/services/deployments docker \
        DOCKER_BUILDARGS="${DOCKER_BUILDARGS} --target builder" \
        MENDER_IMAGE_TAG=${MENDER_IMAGE_TAG_BUILDER}
    - make -C backend docker

build:backend:docker-acceptance:
  extends: build:backend:docker
  before_script:
    - apk add make bash git
    - *dind-login
    # We're only building acceptance test images for CI runner platform.
    - unset DOCKER_PLATFORM
  script:
    # NOTE: Only build for test platform (default) for the acceptance test images
    - make -C backend docker-acceptance

test:backend:static:
  stage: test
  needs: []
  tags:
    - hetzner-amd-beefy
  rules:
    - changes:
        paths: ["backend/**/*.go"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
  image: "golangci/golangci-lint:${IMAGE_GOLANGCI_VERSION}"
  script:
    - cd backend
    - golangci-lint run -v

test:backend:validate-open-api:
  stage: test
  needs: []
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/alpine
  before_script:
    - apk add --no-cache curl
    - curl -L https://raw.github.com/stoplightio/spectral/master/scripts/install.sh -o install.sh
    - sh install.sh
  script:
    - |
      cat > .spectral.yaml << EOF
      extends: [['spectral:oas', all]]
      parserOptions:
        incompatibleValues: 1
      EOF
    - spectral lint -v -D -f text backend/services/**/docs/*.yml
    - spectral lint -v -D -f junit -o spectral-report.xml backend/services/**/docs/*.yml
  artifacts:
    when: always
    expire_in: 2 weeks
    reports:
      junit: $CI_PROJECT_DIR/spectral-report.xml

test:backend:unit:
  # FIXME: Using deployments build stage since we're running all tests
  image: "${CI_REGISTRY_IMAGE}/deployments:${MENDER_IMAGE_TAG_BUILDER}"
  stage: test
  needs:
    - job: build:backend:docker
      artifacts: false
  rules:
    - changes:
        paths: ["backend/**/*.go"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
      when: on_success
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: on_success
  tags:
    - hetzner-amd-beefy
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/mongo:6.0
      alias: mongo
  variables:
    TEST_MONGO_URL: "mongodb://mongo"
    WORKFLOWS_MONGO_URL: "mongodb://mongo"
  before_script:
    - mkdir -p $GOCOVERDIR
  script:
    - |
      make -C backend test-unit \
        TESTFLAGS="-cover -coverprofile=${GOCOVERDIR}/\$(COMPONENT)-unit.cover"
  artifacts:
    expire_in: 1w
    when: on_success
    paths:
      - ${GOCOVERDIR}/*-unit.cover

test:backend:acceptance:
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-cli
  stage: test
  rules:
    - changes:
        paths: ["backend/**/*"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
      when: on_success
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: on_success
  tags:
    - hetzner-amd-beefy
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  needs:
    - job: build:backend:docker
      artifacts: false
    - job: build:backend:docker-acceptance
      artifacts: false
  before_script:
    - apk add make bash git
    - *dind-login
    - make -C backend -j 4 docker-pull
    - make -C backend -j 4 docker-pull MENDER_IMAGE_TAG=${MENDER_IMAGE_TAG_TEST}
    - mkdir -p $GOCOVERDIR
  script:
    # NOTE: Setting GOCOVERDIR this way will group the coverage report per
    #       service (using make variable: COMPONENT).
    - make -C backend test-acceptance GOCOVERDIR="${GOCOVERDIR}/\$(COMPONENT)-acceptance"
  artifacts:
    expire_in: 1w
    when: on_success
    paths:
      - ${GOCOVERDIR}/*-acceptance

test:backend:integration:
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-cli
  stage: test
  rules:
    - changes:
        paths: ["backend/**/*"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
      when: on_success
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: on_success
  tags:
    - hetzner-amd-beefy
  services:
    - name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/docker:${DOCKER_VERSION}-dind
      alias: docker
  needs:
    - job: build:backend:docker
      artifacts: false
    - job: build:backend:docker-acceptance
      artifacts: false
  before_script:
    - apk add make bash git curl
    - *dind-login
    - mkdir -p ${GOCOVERDIR}/integration
    - make -C backend -j 4 docker-pull MENDER_IMAGE_TAG=$MENDER_IMAGE_TAG_TEST
  script:
    - make -C backend test-integration GOCOVERDIR=${GOCOVERDIR}/integration
  artifacts:
    expire_in: 1w
    when: always
    paths:
      - ${GOCOVERDIR}/integration
      - backend/logs.*
      - backend/results_integration_*.xml
      - backend/report_integration_*.html
    reports:
      junit: backend/results_integration_*.xml

test:integration:
  stage: test
  needs:
    - job: build:backend:docker
      artifacts: false
    - job: build:frontend:docker
      artifacts: false
  rules:
    - if: $CI_COMMIT_REF_PROTECTED == "true"
      when: manual
      allow_failure: true
  variables:
    # NOTE: Cannot use indirect values based off CI_* since these will be
    #       expanded in the downstream project context.
    MENDER_SERVER_REGISTRY: "${CI_REGISTRY}"
    MENDER_SERVER_REPOSITORY: "northern.tech/mender/${CI_PROJECT_NAME}"
    MENDER_SERVER_TAG: "build-${CI_COMMIT_SHA}"
    PYTEST_ADDOPTS: "-k 'not Enterprise'"
    RUN_TESTS_FULL_INTEGRATION: "true"
  trigger:
    project: "Northern.tech/Mender/integration"

publish:backend:coverage:
  stage: publish
  needs:
    - job: test:backend:unit
      artifacts: true
      optional: true
    - job: test:backend:acceptance
      artifacts: true
      optional: true
    - job: test:backend:integration
      artifacts: true
      optional: true
  rules:
    - changes:
        paths: ["backend/**/*"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
      when: on_success
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: on_success
  image: "golang:${GOLANG_VERSION}"
  variables:
    COVERALLS_TOKEN: "$COVERALLS_REPO_TOKEN"
  before_script:
    - go install github.com/mattn/goveralls@latest
    # Convert coverage directory (from acceptance/integration) to textfmt
    - find ${GOCOVERDIR} -mindepth 1 -maxdepth 1 -type d
      -exec go tool covdata textfmt -i {} -o {}.cover \;
  script:
    - cd backend
    # NOTE: All coverage files have the filename '<coveralls flag>.cover'
    - |
      for coverpath in $(find ${GOCOVERDIR} -type f -name '*.cover'); do
        coverfile=$(basename "$coverpath")
        goveralls -parallel \
          -service=gitlab \
          -flagname="${coverfile%.cover}" \
          -coverprofile="${coverpath}"
      done

publish:backend:docker:
  stage: publish
  tags:
    - hetzner-amd-beefy
  image:
    name: quay.io/skopeo/stable:${SKOPEO_VERSION}
    # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#override-the-entrypoint-of-an-image
    entrypoint: [""]
  rules:
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: on_success
    - when: never
  before_script:
    - skopeo login --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - skopeo login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD docker.io
    - dnf install -y make git-core
    - export MENDER_PUBLISH_TAG="${CI_COMMIT_REF_NAME}"
  script:
    - make -C backend -j 4 docker-publish NOASK=y
    - |
      if echo -n "${MENDER_PUBLISH_TAG}" | grep -E '^v[0-9]+\.v[0-9]+\.[0-9]+$'; then
         make -C backend -j 4 docker-publish NOASK=y \
            MENDER_PUBLISH_TAG=$(echo -n $MENDER_PUBLISH_TAG | cut -d . -f-2) # vX.Y
         make -C backend -j 4 docker-publish NOASK=y \
            MENDER_PUBLISH_TAG=$(echo -n $MENDER_PUBLISH_TAG | cut -d . -f-1) # vX
         make -C backend -j 4 docker-publish NOASK=y \
            MENDER_PUBLISH_TAG=latest
      fi

publish:backend:licenses:
  stage: publish
  tags:
    - hetzner-amd-beefy
  rules:
    - changes:
        paths: ["backend/**/*"]
        compare_to: "${RULES_CHANGES_COMPARE_TO_REF}"
    - if: '$CI_COMMIT_REF_PROTECTED == "true"'
      when: on_success
  image: golang:${GOLANG_VERSION}
  before_script:
    - go install github.com/google/go-licenses@v1.6.0
  script:
    - cd backend
    # HACK: go-licenses won't detect the license if it's present in the parent directory
    - ln -s ../LICENSE ./
    - |
      GOFLAGS='-tags=nopkcs11' go-licenses report \
        --template=./tests/go-licenses.gotpl \
        $(go list -f '{{ if eq .Name "main" }}{{println .Dir }}{{end}}' ./services/...) > licenses.md
    - |
      if grep -o '^LICENSE TEXT MISSING FOR.*$' licenses.md; then
        exit 1;
      fi
  artifacts:
    when: on_success
    expire_in: "1w"
    paths:
      - backend/licenses.md

coveralls:done:
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/curlimages/curl
  stage: .post
  script:
    - curl "https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN&carryforward=frontend-unit,frontend-e2e,frontend-e2e-enterprise,create-artifact-worker-unit,deployments-unit,deployments-acceptance,deviceauth-unit,deviceauth-acceptance,deviceconfig-unit,deviceconfig-acceptance,deviceconnect-unit,deviceconnect-acceptance,inventory-unit,inventory-acceptance,iot-manager-unit,iot-manager-acceptance,useradm-unit,useradm-acceptance,workflows-unit,workflows-acceptance,integration" -d "payload[build_num]=$CI_PIPELINE_ID&payload[status]=done"
  tags:
    - hetzner-amd-beefy


lint:commit:
  stage: lint
  needs: []
  image:
    name: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/commitlint/commitlint:latest
    entrypoint: [""]
  allow_failure: true
  tags:
    - hetzner-amd-beefy
  before_script:
    - npm install --global commitlint-plugin-selective-scope --save-dev
  script:
    - echo "${CI_COMMIT_MESSAGE}" | commitlint

changelog:
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/node:20
  stage: changelog
  variables:
    GIT_DEPTH: 0  # Always get the full history
    GIT_STRATEGY: clone  # Always get the full history
    GIT_CLIFF__BUMP__INITIAL_TAG: "4.0.0"  # TODO: after the new tag is created,
                                           # remove this variable
  tags:
    - hetzner-amd-beefy
  rules:
    # Only run for protected branches (main and maintenance branches)
    - if: $CI_COMMIT_REF_PROTECTED == "true" && $CI_COMMIT_BRANCH != ""
  before_script:
    # install release-please
    - npm install -g release-please
    # install github-cli
    - mkdir -p -m 755 /etc/apt/keyrings
    - wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
    - chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
    - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null
    - apt update
    - apt install gh jq -y
    # Setting up git
    - git config --global user.email "${GITHUB_USER_EMAIL}"
    - git config --global user.name "${GITHUB_USER_NAME}"
    - npm install -g git-cliff
    # GITHUB_TOKEN for Github cli authentication
    - export GITHUB_TOKEN=${GITHUB_CLI_TOKEN}
    # getting the centralized git cliff config
  script:
    - release-please release-pr
        --token=${GITHUB_BOT_TOKEN_REPO_FULL}
        --repo-url=${GITHUB_REPO_URL}
        --target-branch=${CI_COMMIT_REF_NAME} || echo "INFO - release already exists" # workaround because we shifted to prerelease versioning strategy and there's already a PR open
    # git cliff: override the changelog
    - test $GIT_CLIFF == "false" && echo "INFO - Skipping git-cliff" && exit 0
    - git remote add github-${CI_JOB_ID} https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL} || true  # Ignore already existing remote
    - gh repo set-default https://${GITHUB_USER_NAME}:${GITHUB_BOT_TOKEN_REPO_FULL}@github.com/${GITHUB_REPO_URL}
    - RELEASE_PLEASE_PR=$(gh pr list --author "${GITHUB_USER_NAME}" --head "release-please--branches--${CI_COMMIT_REF_NAME}" --json number | jq -r '.[0].number // empty')
    - test -z "$RELEASE_PLEASE_PR" && echo "No release-please PR found" && exit 0
    - gh pr checkout --force $RELEASE_PLEASE_PR
    - wget --output-document cliff.toml https://raw.githubusercontent.com/mendersoftware/mendertesting/master/utils/cliff.toml
    - git cliff --bump --output CHANGELOG.md --github-repo ${GITHUB_REPO_URL}
    - git add CHANGELOG.md
    - git commit --amend -s --no-edit
    - git push github-${CI_JOB_ID} --force
    # Update the PR body
    - git cliff --unreleased --bump -o tmp_pr_body.md --github-repo ${GITHUB_REPO_URL}
    - gh pr edit $RELEASE_PLEASE_PR --body-file tmp_pr_body.md
    - rm tmp_pr_body.md
  after_script:
    - git remote remove github-${CI_JOB_ID}

release:github:
  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/node:20
  stage: .post
  tags:
    - hetzner-amd-beefy
  rules:
    # Only make available for protected branches (main and maintenance branches)
    - if: $CI_COMMIT_REF_PROTECTED == "true" && $CI_COMMIT_BRANCH != ""
      when: manual
      allow_failure: true
  needs:
    - job: changelog
  script:
    - npm install -g release-please
    - release-please github-release
        --token=${GITHUB_BOT_TOKEN_REPO_FULL}
        --repo-url=${GITHUB_REPO_URL}
        --target-branch=${CI_COMMIT_REF_NAME}