diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 34321b61161b..147ca45ced16 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,5 +1,137 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Auditbeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows:-{{matrix.image}} Unit Tests" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Crosscompile" + command: + - ".buildkite/auditbeat/scripts/crosscompile.sh" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "auditbeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "auditbeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "auditbeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload" diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh new file mode 100755 index 000000000000..866d6be42239 --- /dev/null +++ b/.buildkite/auditbeat/scripts/crosscompile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Crosscompile" +make -C auditbeat crosscompile diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh new file mode 100755 index 000000000000..021240589923 --- /dev/null +++ b/.buildkite/auditbeat/scripts/package-step.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^auditbeat/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/auditbeat/" + +if are_files_changed "$changeset"; then + cat <<-EOF + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Auditbeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "Auditbeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +EOF +fi diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh new file mode 100755 index 000000000000..71872ca15a35 --- /dev/null +++ b/.buildkite/auditbeat/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Docker Version: $(docker --version)" + +echo "--- Start Packaging" +cd auditbeat +umask 0022 +mage package + diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 new file mode 100644 index 000000000000..200627d518f0 --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "--- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "--- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path auditbeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh new file mode 100755 index 000000000000..c1f5685c77fe --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Running Unit Tests" +sudo chmod -R go-w auditbeat/ + +cd auditbeat +umask 0022 +mage build unitTest diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index d94d03aad53b..4dfc01bafc34 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -5,9 +5,15 @@ WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" PLATFORM_TYPE="$(uname)" +REPO="beats" +TMP_FOLDER="tmp.${REPO}" +DOCKER_REGISTRY="docker.elastic.co" export SETUP_GVM_VERSION export WORKSPACE export BIN export HW_TYPE export PLATFORM_TYPE +export REPO +export TMP_FOLDER +export DOCKER_REGISTRY diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh index edaf1a3100c2..5e6e5f7cbf05 100644 --- a/.buildkite/env-scripts/linux-env.sh +++ b/.buildkite/env-scripts/linux-env.sh @@ -1,24 +1,47 @@ #!/usr/bin/env bash +set -euo pipefail + source .buildkite/env-scripts/util.sh DEBIAN_FRONTEND="noninteractive" -export DEBIAN_FRONTEND - sudo mkdir -p /etc/needrestart echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null -# Remove this code once beats specific agent is set up +echo "--- PLATFORM TYPE $PLATFORM_TYPE" + if [[ $PLATFORM_TYPE == "Linux" ]]; then - echo ":: Installing libs ::" - sudo apt-get update - sudo apt-get install -y libsystemd-dev - sudo apt install -y python3-pip - sudo apt-get install -y python3-venv + # Remove this code once beats specific agent is set up + if grep -q 'Ubuntu' /etc/*release; then + export DEBIAN_FRONTEND + + echo "--- Ubuntu - Installing libs" + sudo apt-get update + sudo apt-get install -y libsystemd-dev + sudo apt install -y python3-pip + sudo apt-get install -y python3-venv + fi + + # Remove this code once beats specific agent is set up + if grep -q 'Red Hat' /etc/*release; then + echo "--- RHL - Installing libs" + sudo yum update -y + sudo yum install -y systemd-devel + sudo yum install -y python3-pip + sudo yum install -y python3 + pip3 install virtualenv + fi +fi + +if [[ $PLATFORM_TYPE == Darwin* ]]; then + echo "--- Setting larger ulimit on MacOS" + # To bypass file descriptor errors like "Too many open files error" on MacOS + ulimit -Sn 50000 + echo "--- ULIMIT: $(ulimit -n)" fi -echo ":: Setting up environment ::" +echo "--- Setting up environment" add_bin_path with_go with_mage diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh deleted file mode 100644 index ac1486b64fdd..000000000000 --- a/.buildkite/env-scripts/macos-env.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $PLATFORM_TYPE == Darwin* ]]; then - echo ":: Setting larger ulimit on MacOS ::" - # To bypass file descriptor errors like "Too many open files error" on MacOS - ulimit -Sn 50000 - echo ":: ULIMIT :: $(ulimit -n)" -fi diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 157a5aff37af..7aef69cff389 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -89,3 +89,17 @@ are_files_changed() { return 1; fi } + +cleanup() { + echo "Deleting temporary files..." + rm -rf ${BIN}/${TMP_FOLDER}.* + echo "Done." +} + +unset_secrets () { + for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi + done +} diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh index aa5f67ca4cee..ccf5479b46e1 100644 --- a/.buildkite/env-scripts/win-env.sh +++ b/.buildkite/env-scripts/win-env.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -install_python_win() { - if [[ ${PLATFORM_TYPE} = MINGW* ]]; then - choco install mingw -y - choco install python --version=3.11.0 -y - fi -} +echo "--- PLATFORM TYPE: ${PLATFORM_TYPE}" +if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + echo "--- Installing Python on Win" + choco install mingw -y + choco install python --version=3.11.0 -y +fi diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e3d7384a71ea..eda9fb93a669 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -134,8 +134,6 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - "mandatory-tests" - - "extended-tests" - - "extended-tests-win" steps: - label: Package pipeline diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index a3eabf70c0d3..d64ce7c98eb2 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -4,7 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Execute Integration Tests ::" +echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ cd filebeat diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh index 5e2e403dda87..b51e8ae18a68 100755 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -4,7 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Execute Integration Tests ::" +echo "--- Executing Integration Tests" sudo chmod -R go-w filebeat/ cd filebeat diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh index a4127c3cd1d6..985125433cec 100755 --- a/.buildkite/filebeat/scripts/package-step.sh +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -5,12 +5,12 @@ set -euo pipefail source .buildkite/env-scripts/util.sh changeset="^filebeat/ - ^go.mod - ^pytest.ini - ^dev-tools/ - ^libbeat/ - ^testing/ - ^\.buildkite/filebeat/" +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/filebeat/" if are_files_changed "$changeset"; then cat <<-EOF diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index 2ae226eb739c..0bb03250348c 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -4,9 +4,7 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -echo ":: Evaluate Filebeat Changes ::" - -echo ":: Start Packaging ::" +echo "--- Start Packaging" cd filebeat umask 0022 mage package diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index cda1dd85aea2..08ce9d4ea1c6 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -3,9 +3,8 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh -source .buildkite/env-scripts/macos-env.sh -echo ":: Execute Unit Tests ::" +echo "--- Executing Unit Tests" sudo chmod -R go-w filebeat/ umask 0022 diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index e10f15de7b65..b6cc7ad60bda 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ef38478a4327..c448a7102512 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,15 +2,11 @@ set -euo pipefail -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh - if [[ ${PLATFORM_TYPE} = MINGW* ]]; then - install_python_win - fi - if [[ -z "${GOLANG_VERSION-""}" ]]; then export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") fi diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 4607a0576d0b..abe0d20f6805 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -57,8 +57,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test auditbeat$", - "always_trigger_comment_regex": "^/test auditbeat$", + "trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", + "always_trigger_comment_regex": "^/test auditbeat(for (arm|macos|windows|extended support))?$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ],