migrate libbeat pipeline

.buildkite/hooks/post-checkout

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -euo pipefail
+
+checkout_merge() {
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    git checkout FETCH_HEAD
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH="pr_merge_${PR_ID}"
+
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
+
+echo "Commit information"
+git --no-pager log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""

.buildkite/hooks/pre-command

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "Golang version:"
+GO_VERSION=$(cat .go-version)
+export GO_VERSION
+echo "GO_VERSION: ${GO_VERSION}"

.buildkite/libbeat/pipeline.libbeat.yml

@@ -1,5 +1,76 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  SETUP_MAGE_VERSION: "1.13.0"
+  SETUP_WIN_PYTHON_VERSION: "3.11.0"
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
+  DOCKER_COMPOSE_VERSION: "1.21.0"
+  PIPELINE_NAME: "libbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths: "metricbeat/build/*.*"
+
+      - label: ":go: Go Intergration Tests"
+        key: "mandatory-int-test"
+        command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths: "metricbeat/build/*.*"
+
+      - label: ":python: Python Integration Tests"
+        key: "mandatory-python-int-test"
+        command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths: "metricbeat/build/*.*"
+
+      - label: ":negative_squared_cross_mark: Cross compile"
+        key: "mandatory-cross-compile"
+        command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths: "metricbeat/build/*.*"
+
+      - label: ":negative_squared_cross_mark: Cross compile"
+        key: "mandatory-stress-test"
+        command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths: "metricbeat/build/*.*"
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+      - label: ":mac: Arm64 Unit Tests"
+        key: "extended-arm64-unit-tests"
+        command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.xlarge"
+        artifact_paths: "metricbeat/build/*.*"

.buildkite/scripts/common.sh

@@ -0,0 +1,138 @@
+#!/bin/bash
+set -euo pipefail
+
+WORKSPACE=${WORKSPACE:-"$(pwd)"}
+BIN="${WORKSPACE}/bin"
+platform_type="$(uname)"
+platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
+arch_type="$(uname -m)"
+pipeline_name="metricbeat"
+DEBIAN_FRONTEND="noninteractive"
+sudo mkdir -p /etc/needrestart
+echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
+
+with_docker_compose() {
+  local version=$1
+  echo "Setting up the Docker-compose environment..."
+  create_workspace
+  retry 5 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}"
+  chmod +x ${BIN}/docker-compose
+  docker-compose version
+}
+
+create_workspace() {
+  if [[ ! -d "${BIN}" ]]; then
+    mkdir -p "${BIN}"
+  fi
+}
+
+add_bin_path() {
+  echo "Adding PATH to the environment variables..."
+  create_workspace
+  export PATH="${PATH}:${BIN}"
+}
+
+check_platform_architeture() {
+  case "${arch_type}" in
+    "x86_64")
+      arch_type="amd64"
+      ;;
+    "aarch64")
+      arch_type="arm64"
+      ;;
+    "arm64")
+      arch_type="arm64"
+      ;;
+    *)
+    echo "The current platform/OS type is unsupported yet"
+    ;;
+  esac
+}
+
+with_mage() {
+  local install_packages=(
+    "github.com/magefile/mage"
+    "github.com/elastic/go-licenser"
+    "golang.org/x/tools/cmd/goimports"
+    "github.com/jstemmer/go-junit-report"
+    "gotest.tools/gotestsum"
+  )
+  create_workspace
+  for pkg in "${install_packages[@]}"; do
+    go install "${pkg}@latest"
+  done
+}
+
+with_go() {
+  echo "Setting up the Go environment..."
+  create_workspace
+  check_platform_architeture
+  retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}"
+  chmod +x "${BIN}/gvm"
+  eval "$(gvm $GO_VERSION)"
+  go version
+  which go
+  local go_path="$(go env GOPATH):$(go env GOPATH)/bin"
+  export PATH="${PATH}:${go_path}"
+}
+
+with_python() {
+  if [ "${platform_type}" == "Linux" ]; then
+    sudo apt-get update
+    sudo apt-get install -y python3-pip python3-venv libsystemd-dev
+  elif [ "${platform_type}" == "Darwin" ]; then
+    brew update
+    pip3 install virtualenv
+    ulimit -Sn 10000
+  fi
+}
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
+are_files_changed() {
+  changeset=$1
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}
+
+echo "--- Env preparation"
+
+if command -v docker-compose &> /dev/null
+then
+  set +e
+  echo "Found docker-compose. Checking version.."
+  FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//)
+  if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then
+    echo "Versions match. No need to install docker-compose. Exiting."
+  else
+    echo "Versions don't match. Need to install the correct version of docker-compose."
+    with_docker_compose "${DOCKER_COMPOSE_VERSION}"
+  fi
+  set -e
+fi
+
+add_bin_path
+with_go "${GO_VERSION}"
+with_mage
+with_python

.buildkite/scripts/crosscompile.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+beats_subfilder=$1
+
+echo "--- Run Crosscompile for $beats_subfilder"
+make -C "${beats_subfilder}" crosscompile

.buildkite/scripts/go_int_tests.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+beats_subfilder=$1
+
+echo "--- Run Go Intergration Tests for $beats_subfilder"
+pushd "${beats_subfilder}" > /dev/null
+
+mage goIntegTest
+
+popd > /dev/null

.buildkite/scripts/py_int_tests.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+beats_subfilder=$1
+
+echo "--- Run Python Intergration Tests for $beats_subfilder"
+pushd "${beats_subfilder}" > /dev/null
+
+mage pythonIntegTest
+
+popd > /dev/null

.buildkite/scripts/stress_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+beats_subfilder=$1
+
+echo "--- Run Stress Tests for $beats_subfilder"
+make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C $beats_subfilder stress-tests

.buildkite/scripts/unit_tests.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+beats_subfilder=$1
+
+echo "--- Run Unit Tests"
+sudo chmod -R go-w "${beats_subfilder}/"
+pushd "${beats_subfilder}" > /dev/null
+umask 0022
+mage build unitTest
+popd > /dev/null