Auditbeat pipeline migration to Buildkite (#37668)

* wip: getting GH message

* added test scripts

* added windows tests

* added packaging step

* updated packaging execution conditions

* win-test failure: updated artifact path

* pr fixes

* changed group_test.go

* moved env setup to separate script

* added dynamic step for packaging

* added tests and scrosscompile scripts for auditbeat

* fixes for crosscompile and macos steps

* added env vars for linux packaging

* added env vars for linux packaging

* enabled packaging step

* added docker login

* debugging arm packaing

* enabled synamic packaging step

* updated packaging group steps

* fixed conflicts

* updated win test script

(cherry picked from commit 31819ec)

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -1,5 +1,137 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_RHEL9: "family/core-rhel-9"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: "Auditbeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":rhel: Unit Tests"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows:-{{matrix.image}} Unit Tests"
+        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":linux: Crosscompile"
+        command:
+          - ".buildkite/auditbeat/scripts/crosscompile.sh"
+        env:
+          GOX_FLAGS: "-arch amd64"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for extended support"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "auditbeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "auditbeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+  - group: "Packaging"
+    key: "packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "mandatory-tests"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload"

.buildkite/auditbeat/scripts/crosscompile.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Executing Crosscompile"
+make -C auditbeat crosscompile

.buildkite/auditbeat/scripts/package-step.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+changeset="^auditbeat/
+^go.mod
+^pytest.ini
+^dev-tools/
+^libbeat/
+^testing/
+^\.buildkite/auditbeat/"
+
+if are_files_changed "$changeset"; then
+  cat <<-EOF
+    steps:
+      - label: ":ubuntu: Packaging Linux X86"
+        key: "package-linux-x86"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+        command:
+          - ".buildkite/auditbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat/Packaging: Linux X86"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+      - label: ":linux: Packaging Linux ARM"
+        key: "package-linux-arm"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        command:
+          - ".buildkite/auditbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "Auditbeat/Packaging: ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+EOF
+fi

.buildkite/auditbeat/scripts/package.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Docker Version: $(docker --version)"
+
+echo "--- Start Packaging"
+cd auditbeat
+umask 0022
+mage package
+

.buildkite/auditbeat/scripts/unit-tests-win.ps1

@@ -0,0 +1,51 @@
+$ErrorActionPreference = "Stop" # set -e
+$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
+
+# Forcing to checkout again all the files with a correct autocrlf.
+# Doing this here because we cannot set git clone options before.
+function fixCRLF() {
+    Write-Host "--- Fixing CRLF in git checkout --"
+    git config core.autocrlf false
+    git rm --quiet --cached -r .
+    git reset --quiet --hard
+}
+
+function withGolang() {
+    Write-Host "--- Install golang $GoVersion --"
+    choco install golang -y --version $GoVersion
+
+    $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
+    Import-Module "$choco\helpers\chocolateyProfile.psm1"
+    refreshenv
+    go version
+    go env
+}
+
+function installGoDependencies() {
+    $installPackages = @(
+        "github.com/magefile/mage"
+        "github.com/elastic/go-licenser"
+        "golang.org/x/tools/cmd/goimports"
+        "github.com/jstemmer/go-junit-report"
+        "github.com/tebeka/go2xunit"
+    )
+    foreach ($pkg in $installPackages) {
+        go install "$pkg"
+    }
+}
+
+fixCRLF
+
+$ErrorActionPreference = "Continue" # set +e
+
+Set-Location -Path auditbeat
+New-Item -ItemType Directory -Force -Path "build"
+withGolang
+installGoDependencies
+
+mage build unitTest
+
+$EXITCODE=$LASTEXITCODE
+$ErrorActionPreference = "Stop"
+
+Exit $EXITCODE

.buildkite/auditbeat/scripts/unit-tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Running Unit Tests"
+sudo chmod -R go-w auditbeat/
+
+cd auditbeat
+umask 0022
+mage build unitTest

.buildkite/env-scripts/env.sh

@@ -5,9 +5,15 @@ WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
+REPO="beats"
+TMP_FOLDER="tmp.${REPO}"
+DOCKER_REGISTRY="docker.elastic.co"
 
 export SETUP_GVM_VERSION
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
+export REPO
+export TMP_FOLDER
+export DOCKER_REGISTRY

.buildkite/env-scripts/linux-env.sh

@@ -1,24 +1,47 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 source .buildkite/env-scripts/util.sh
 
 DEBIAN_FRONTEND="noninteractive"
 
-export DEBIAN_FRONTEND
-
 sudo mkdir -p /etc/needrestart
 echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
 
-# Remove this code once beats specific agent is set up
+echo "--- PLATFORM TYPE $PLATFORM_TYPE"
+
 if [[ $PLATFORM_TYPE == "Linux" ]]; then
-  echo ":: Installing libs ::"
-  sudo apt-get update
-  sudo apt-get install -y libsystemd-dev
-  sudo apt install -y python3-pip
-  sudo apt-get install -y python3-venv
+  # Remove this code once beats specific agent is set up
+  if grep -q 'Ubuntu' /etc/*release; then
+    export DEBIAN_FRONTEND
+
+    echo "--- Ubuntu - Installing libs"
+    sudo apt-get update
+    sudo apt-get install -y libsystemd-dev
+    sudo apt install -y python3-pip
+    sudo apt-get install -y python3-venv
+  fi
+
+  # Remove this code once beats specific agent is set up
+  if grep -q 'Red Hat' /etc/*release; then
+    echo "--- RHL - Installing libs"
+    sudo yum update -y
+    sudo yum install -y systemd-devel
+    sudo yum install -y python3-pip
+    sudo yum install -y python3
+    pip3 install virtualenv
+  fi
+fi
+
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo "--- Setting larger ulimit on MacOS"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 50000
+  echo "--- ULIMIT: $(ulimit -n)"
 fi
 
-echo ":: Setting up environment ::"
+echo "--- Setting up environment"
 add_bin_path
 with_go
 with_mage

.buildkite/env-scripts/util.sh

@@ -89,3 +89,17 @@ are_files_changed() {
     return 1;
   fi
 }
+
+cleanup() {
+  echo "Deleting temporary files..."
+  rm -rf ${BIN}/${TMP_FOLDER}.*
+  echo "Done."
+}
+
+unset_secrets () {
+  for var in $(printenv | sed 's;=.*;;' | sort); do
+    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
+      unset "$var"
+    fi
+  done
+}

.buildkite/env-scripts/win-env.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-install_python_win() {
-  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
-    choco install mingw -y
-    choco install python --version=3.11.0 -y
-  fi
-}
+echo "--- PLATFORM TYPE: ${PLATFORM_TYPE}"
+if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+  echo "--- Installing Python on Win"
+  choco install mingw -y
+  choco install python --version=3.11.0 -y
+fi

.buildkite/filebeat/filebeat-pipeline.yml

@@ -134,8 +134,6 @@ steps:
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on:
       - "mandatory-tests"
-      - "extended-tests"
-      - "extended-tests-win"
 
     steps:
       - label: Package pipeline