added tests and scrosscompile scripts for auditbeat

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -1,5 +1,139 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_RHEL9: "family/core-rhel-9"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: "Auditbeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":rhel: Unit Tests"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows:-{{matrix.image}} Unit Tests"
+        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":linux: Crosscompile"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat extended tests"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "auditbeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "generic-13-ventura-x64"
+        artifact_paths: "auditbeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+  - group: "Packaging"
+    key: "packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "mandatory-tests"
+      - "extended-tests"
+      - "extended-tests-win"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload"

.buildkite/auditbeat/scripts/crosscompile.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/unix-env.sh
+
+echo ":: Executing Crosscompile ::"
+cd auditbeat
+make -C auditbeat crosscompile
+

.buildkite/auditbeat/scripts/package-step.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+changeset="^auditbeat/
+    ^go.mod
+    ^pytest.ini
+    ^dev-tools/
+    ^libbeat/
+    ^testing/
+    ^\.buildkite/auditbeat/"
+
+if are_files_changed "$changeset"; then
+  cat <<-EOF
+    steps:
+      - label: ":ubuntu: Packaging Linux X86"
+        key: "package-linux-x86"
+        command:
+          - ".buildkite/auditbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Packaging: Linux X86"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+      - label: ":linux: Packaging Linux ARM"
+        key: "package-linux-arm"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        command:
+          - ".buildkite/auditbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Packaging: ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+EOF
+fi
+

.buildkite/auditbeat/scripts/package.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/unix-env.sh
+
+echo ":: Evaluate Auditbeat Changes ::"
+
+echo ":: Start Packaging ::"
+cd auditbeat
+umask 0022
+mage package
+

.buildkite/auditbeat/scripts/unit-tests-win.ps1

@@ -0,0 +1,51 @@
+$ErrorActionPreference = "Stop" # set -e
+$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
+
+# Forcing to checkout again all the files with a correct autocrlf.
+# Doing this here because we cannot set git clone options before.
+function fixCRLF() {
+    Write-Host "-- Fixing CRLF in git checkout --"
+    git config core.autocrlf false
+    git rm --quiet --cached -r .
+    git reset --quiet --hard
+}
+
+function withGolang() {
+    Write-Host "-- Install golang $GoVersion --"
+    choco install golang -y --version $GoVersion
+
+    $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
+    Import-Module "$choco\helpers\chocolateyProfile.psm1"
+    refreshenv
+    go version
+    go env
+}
+
+function installGoDependencies() {
+    $installPackages = @(
+        "github.com/magefile/mage"
+        "github.com/elastic/go-licenser"
+        "golang.org/x/tools/cmd/goimports"
+        "github.com/jstemmer/go-junit-report"
+        "github.com/tebeka/go2xunit"
+    )
+    foreach ($pkg in $installPackages) {
+        go install "$pkg"
+    }
+}
+
+fixCRLF
+
+$ErrorActionPreference = "Continue" # set +e
+
+Set-Location -Path auditbeat
+New-Item -ItemType Directory -Force -Path "build"
+withGolang
+installGoDependencies
+
+mage build unitTest
+
+$EXITCODE=$LASTEXITCODE
+$ErrorActionPreference = "Stop"
+
+Exit $EXITCODE

.buildkite/auditbeat/scripts/unit-tests.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/unix-env.sh
+
+echo ":: Running Unit Tests ::"
+sudo chmod -R go-w auditbeat/
+
+cd auditbeat
+umask 0022
+mage build unitTest

.buildkite/env-scripts/unix-env.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+source .buildkite/env-scripts/util.sh
+
+DEBIAN_FRONTEND="noninteractive"
+#IS_UBUNTU=$(grep -q 'Ubuntu' /etc/*release)
+#IS_RHL=$(grep -q 'Red Hat' /etc/*release)
+
+if [[ $PLATFORM_TYPE == "Linux" ]]; then
+  IS_UBUNTU=$(grep -q 'Ubuntu' /etc/*release)
+  IS_RHL=$(grep -q 'Red Hat' /etc/*release)
+
+  # Remove this code once beats specific agent is set up
+  if [[ $IS_UBUNTU ]]; then
+    export DEBIAN_FRONTEND
+
+    echo ":: Ubuntu - Installing libs ::"
+    sudo apt-get update
+    sudo apt-get install -y libsystemd-dev
+    sudo apt install -y python3-pip
+    sudo apt-get install -y python3-venv
+  fi
+
+  # Remove this code once beats specific agent is set up
+  if [[ $IS_RHL ]]; then
+    echo ":: RHL - Installing libs ::"
+    sudo yum update -y
+    sudo yum install -y systemd-devel
+    sudo yum install -y python3-pip
+    sudo yum install -y python3
+    pip3 install virtualenv
+  fi
+fi
+
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: Setting larger ulimit on MacOS ::"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 50000
+  echo ":: ULIMIT :: $(ulimit -n)"
+fi
+
+echo ":: Setting up environment ::"
+add_bin_path
+with_go
+with_mage

.buildkite/filebeat/scripts/integration-gotests.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
+source .buildkite/env-scripts/unix-env.sh
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

.buildkite/filebeat/scripts/integration-pytests.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
+source .buildkite/env-scripts/unix-env.sh
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

.buildkite/filebeat/scripts/package.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
+source .buildkite/env-scripts/unix-env.sh
 
 echo ":: Evaluate Filebeat Changes ::"
 

.buildkite/filebeat/scripts/unit-tests.sh

@@ -2,8 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-source .buildkite/env-scripts/macos-env.sh
+source .buildkite/env-scripts/unix-env.sh
 
 echo ":: Execute Unit Tests ::"
 sudo chmod -R go-w filebeat/

.buildkite/hooks/pre-command

@@ -6,8 +6,7 @@ source .buildkite/env-scripts/env.sh
 source .buildkite/env-scripts/util.sh
 source .buildkite/env-scripts/win-env.sh
 
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
+if [[ ( "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ) ]]; then
   if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
     install_python_win
   fi