[7.17](backport #37283) Filebeat pipeline migration to Buildkite

.buildkite/env-scripts/env.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SETUP_GVM_VERSION="v0.5.1"
+WORKSPACE="$(pwd)"
+BIN="${WORKSPACE}/bin"
+HW_TYPE="$(uname -m)"
+PLATFORM_TYPE="$(uname)"
+
+export SETUP_GVM_VERSION
+export WORKSPACE
+export BIN
+export HW_TYPE
+export PLATFORM_TYPE

.buildkite/env-scripts/linux-env.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+source .buildkite/env-scripts/util.sh
+
+DEBIAN_FRONTEND="noninteractive"
+
+export DEBIAN_FRONTEND
+
+sudo mkdir -p /etc/needrestart
+echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
+
+# Remove this code once beats specific agent is set up
+if [[ $PLATFORM_TYPE == "Linux" ]]; then
+  echo ":: Installing libs ::"
+  sudo apt-get update
+  sudo apt-get install -y libsystemd-dev
+  sudo apt install -y python3-pip
+  sudo apt-get install -y python3-venv
+fi
+
+echo ":: Setting up environment ::"
+add_bin_path
+with_go
+with_mage

.buildkite/env-scripts/macos-env.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: Setting larger ulimit on MacOS ::"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 50000
+  echo ":: ULIMIT :: $(ulimit -n)"
+fi

.buildkite/env-scripts/util.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+add_bin_path() {
+    echo "Adding PATH to the environment variables..."
+    create_bin
+    export PATH="${PATH}:${BIN}"
+}
+
+with_go() {
+    local go_version="${GOLANG_VERSION}"
+    echo "Setting up the Go environment..."
+    create_bin
+    check_platform_architecture
+    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
+    export PATH="${PATH}:${BIN}"
+    chmod +x ${BIN}/gvm
+    eval "$(gvm "$go_version")"
+    go version
+    which go
+    export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
+}
+
+with_mage() {
+    local install_packages=(
+            "github.com/magefile/mage"
+            "github.com/elastic/go-licenser"
+            "golang.org/x/tools/cmd/goimports"
+            "github.com/jstemmer/go-junit-report"
+            "gotest.tools/gotestsum"
+    )
+    create_bin
+    for pkg in "${install_packages[@]}"; do
+        go install "${pkg}@latest"
+    done
+}
+
+create_bin() {
+    if [[ ! -d "${BIN}" ]]; then
+    mkdir -p ${BIN}
+    fi
+}
+
+check_platform_architecture() {
+# for downloading the GVM and Terraform packages
+  case "${HW_TYPE}" in
+   "x86_64")
+        arch_type="amd64"
+        ;;
+    "aarch64")
+        arch_type="arm64"
+        ;;
+    "arm64")
+        arch_type="arm64"
+        ;;
+    *)
+    echo "The current platform/OS type is unsupported yet"
+    ;;
+  esac
+}
+
+retry() {
+    local retries=$1
+    shift
+    local count=0
+    until "$@"; do
+        exit=$?
+        wait=$((2 ** count))
+        count=$((count + 1))
+        if [ $count -lt "$retries" ]; then
+            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+            sleep $wait
+        else
+            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+            return $exit
+        fi
+    done
+    return 0
+}
+
+are_files_changed() {
+  local changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}

.buildkite/env-scripts/win-env.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+install_python_win() {
+  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+    choco install mingw -y
+    choco install python --version=3.11.0 -y
+  fi
+}

.buildkite/filebeat/filebeat-pipeline.yml

@@ -1,5 +1,142 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: "Filebeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command:
+          - ".buildkite/filebeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+      - label: ":ubuntu: Go Integration Tests"
+        command:
+          - ".buildkite/filebeat/scripts/integration-gotests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+      - label: ":ubuntu: Python Integration Tests"
+        command:
+          - ".buildkite/filebeat/scripts/integration-pytests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Python Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+      - label: ":windows:-{{matrix.image}} Unit Tests"
+        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Filebeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/filebeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "filebeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/filebeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "filebeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+
+  - group: "Packaging"
+    key: "packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "mandatory-tests"
+      - "extended-tests"
+      - "extended-tests-win"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload"

.buildkite/filebeat/scripts/integration-gotests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo ":: Execute Integration Tests ::"
+sudo chmod -R go-w filebeat/
+
+cd filebeat
+umask 0022
+mage goIntegTest

.buildkite/filebeat/scripts/integration-pytests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo ":: Execute Integration Tests ::"
+sudo chmod -R go-w filebeat/
+
+cd filebeat
+umask 0022
+mage pythonIntegTest

.buildkite/filebeat/scripts/package-step.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+changeset="^filebeat/
+    ^go.mod
+    ^pytest.ini
+    ^dev-tools/
+    ^libbeat/
+    ^testing/
+    ^\.buildkite/filebeat/"
+
+if are_files_changed "$changeset"; then
+  cat <<-EOF
+    steps:
+      - label: ":ubuntu: Packaging Linux X86"
+        key: "package-linux-x86"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+        command:
+          - ".buildkite/filebeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Packaging: Linux X86"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+      - label: ":linux: Packaging Linux ARM"
+        key: "package-linux-arm"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        command:
+          - ".buildkite/filebeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "Filebeat/Packaging: ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+EOF
+fi

.buildkite/filebeat/scripts/package.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo ":: Evaluate Filebeat Changes ::"
+
+echo ":: Start Packaging ::"
+cd filebeat
+umask 0022
+mage package