migrate metricbeat pipeline to Buildkite (#37592)

migrate the metricbeat pipeline to Buildkite
elastic · Feb 2, 2024 · a611494 · a611494
1 parent aeb93a0
commit a611494
Show file tree

Hide file tree

Showing 13 changed files with 727 additions and 7 deletions.
diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -euo pipefail
+
+checkout_merge() {
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    git checkout FETCH_HEAD
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH="pr_merge_${PR_ID}"
+
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
+
+echo "Commit information"
+git --no-pager log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -2,12 +2,11 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/env.sh
-source .buildkite/env-scripts/util.sh
-source .buildkite/env-scripts/win-env.sh
-
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
+  source .buildkite/env-scripts/env.sh
+  source .buildkite/env-scripts/util.sh
+  source .buildkite/env-scripts/win-env.sh
+
   if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
     install_python_win
   fi
@@ -16,3 +15,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+  source .buildkite/scripts/setenv.sh
+  if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
+    echo "Skipped pre-command when running the Upload pipeline"
+    exit 0
+  fi
+fi
diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
@@ -1,5 +1,46 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_10: "family/general-windows-10"
+  IMAGE_WIN_11: "family/general-windows-11"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
+  BEATS_PROJECT_NAME: "metricbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "input-run-all-stages"
+    fields:
+    - select: "Metricbeat - runAllStages"
+      key: "runAllStages"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Metricbeat - runMacOsTests"
+      key: "UI_MACOS_TESTS"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic metricbeat pipeline"
+    key: "metricbeat-pipeline"
+    command: ".buildkite/scripts/generate_metricbeat_pipeline.sh"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
@@ -0,0 +1,252 @@
+#!/bin/bash
+set -euo pipefail
+
+WORKSPACE=${WORKSPACE:-"$(pwd)"}
+BIN="${WORKSPACE}/bin"
+platform_type="$(uname)"
+platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
+arch_type="$(uname -m)"
+GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
+ONLY_DOCS=${ONLY_DOCS:-"true"}
+UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})"
+runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})"
+metricbeat_changeset=(
+  "^metricbeat/.*"
+  "^go.mod"
+  "^pytest.ini"
+  "^dev-tools/.*"
+  "^libbeat/.*"
+  "^testing/.*"
+  )
+oss_changeset=(
+  "^go.mod"
+  "^pytest.ini"
+  "^dev-tools/.*"
+  "^libbeat/.*"
+  "^testing/.*"
+)
+ci_changeset=(
+  "^.buildkite/.*"
+)
+go_mod_changeset=(
+  "^go.mod"
+  )
+docs_changeset=(
+  ".*\\.(asciidoc|md)"
+  "deploy/kubernetes/.*-kubernetes\\.yaml"
+  )
+packaging_changeset=(
+  "^dev-tools/packaging/.*"
+  ".go-version"
+  )
+
+with_docker_compose() {
+  local version=$1
+  echo "Setting up the Docker-compose environment..."
+  create_workspace
+  retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}"
+  chmod +x ${BIN}/docker-compose
+  export PATH="${BIN}:${PATH}"
+  docker-compose version
+}
+
+create_workspace() {
+  if [[ ! -d "${BIN}" ]]; then
+    mkdir -p "${BIN}"
+  fi
+}
+
+add_bin_path() {
+  echo "Adding PATH to the environment variables..."
+  create_workspace
+  export PATH="${BIN}:${PATH}"
+}
+
+check_platform_architeture() {
+  case "${arch_type}" in
+    "x86_64")
+      go_arch_type="amd64"
+      ;;
+    "aarch64")
+      go_arch_type="arm64"
+      ;;
+    "arm64")
+      go_arch_type="arm64"
+      ;;
+    *)
+    echo "The current platform/OS type is unsupported yet"
+    ;;
+  esac
+}
+
+with_mage() {
+  local install_packages=(
+    "github.com/magefile/mage"
+    "github.com/elastic/go-licenser"
+    "golang.org/x/tools/cmd/goimports"
+    "github.com/jstemmer/go-junit-report"
+    "gotest.tools/gotestsum"
+  )
+  create_workspace
+  for pkg in "${install_packages[@]}"; do
+    go install "${pkg}@latest"
+  done
+}
+
+with_go() {
+  echo "Setting up the Go environment..."
+  create_workspace
+  check_platform_architeture
+  retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}"
+  chmod +x "${BIN}/gvm"
+  eval "$(gvm $GO_VERSION)"
+  go version
+  which go
+  local go_path="$(go env GOPATH):$(go env GOPATH)/bin"
+  export PATH="${go_path}:${PATH}"
+}
+
+with_python() {
+  if [ "${platform_type}" == "Linux" ]; then
+    #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
+    sudo apt-get update
+    sudo apt-get install -y python3-pip python3-venv
+  elif [ "${platform_type}" == "Darwin" ]; then
+    brew update
+    pip3 install virtualenv
+    ulimit -Sn 10000
+  fi
+}
+
+with_dependencies() {
+  if [ "${platform_type}" == "Linux" ]; then
+    #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
+    sudo apt-get update
+    sudo apt-get install -y libsystemd-dev libpcap-dev
+  elif [ "${platform_type}" == "Darwin" ]; then
+    pip3 install libpcap
+  fi
+}
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
+are_paths_changed() {
+  local patterns=("${@}")
+  local changelist=()
+
+  for pattern in "${patterns[@]}"; do
+    changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern"))
+    if [ "${#changed_files[@]}" -gt 0 ]; then
+      changelist+=("${changed_files[@]}")
+    fi
+  done
+
+  if [ "${#changelist[@]}" -gt 0 ]; then
+    echo "Files changed:"
+    echo "${changelist[*]}"
+    return 0
+  else
+    echo "No files changed within specified changeset:"
+    echo "${patterns[*]}"
+    return 1
+  fi
+}
+
+are_changed_only_paths() {
+  local patterns=("${@}")
+  local changelist=()
+  local changed_files=$(git diff --name-only HEAD@{1} HEAD)
+  if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+are_conditions_met_mandatory_tests() {
+  if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then     # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137
+    if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+      return 0
+    else
+      return 1
+    fi
+  else
+    return 1
+  fi
+}
+
+are_conditions_met_extended_tests() {
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    return 0
+  else
+    return 1
+  fi
+}
+
+are_conditions_met_macos_tests() {
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+      return 0
+    else
+      return 1
+    fi
+  else
+    return 1
+  fi
+}
+
+are_conditions_met_extended_windows_tests() {
+  if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+      return 0
+    else
+      return 1
+    fi
+  else
+    return 1
+  fi
+}
+
+are_conditions_met_packaging() {
+  if are_conditions_met_extended_windows_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
+      return 0
+    else
+      return 1
+    fi
+  else
+    return 1
+  fi
+}
+
+if ! are_changed_only_paths "${docs_changeset[@]}" ; then
+  ONLY_DOCS="false"
+  echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS."
+else
+  echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS."
+fi
+
+if are_paths_changed "${go_mod_changeset[@]}" ; then
+  GO_MOD_CHANGES="true"
+fi
+
+if are_paths_changed "${packaging_changeset[@]}" ; then
+  PACKAGING_CHANGES="true"
+fi
diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/install_tools.sh
+
+set -euo pipefail
+
+echo "--- Run Crosscompile for $BEATS_PROJECT_NAME"
+make -C "${BEATS_PROJECT_NAME}" crosscompile