From 10682b29ed5a2b05d2254de3062b15d2e3ce29b6 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Thu, 18 Jan 2024 15:46:07 +0200 Subject: [PATCH] added tests and scrosscompile scripts for auditbeat --- .buildkite/auditbeat/auditbeat-pipeline.yml | 138 +++++++++++++++++- .buildkite/auditbeat/scripts/crosscompile.sh | 10 ++ .buildkite/auditbeat/scripts/package-step.sh | 43 ++++++ .buildkite/auditbeat/scripts/package.sh | 13 ++ .../auditbeat/scripts/unit-tests-win.ps1 | 51 +++++++ .buildkite/auditbeat/scripts/unit-tests.sh | 12 ++ .buildkite/env-scripts/linux-env.sh | 21 --- .buildkite/env-scripts/macos-env.sh | 8 - .buildkite/env-scripts/unix-env.sh | 45 ++++++ .../filebeat/scripts/integration-gotests.sh | 2 +- .../filebeat/scripts/integration-pytests.sh | 2 +- .buildkite/filebeat/scripts/package.sh | 2 +- .buildkite/filebeat/scripts/unit-tests.sh | 3 +- .buildkite/hooks/pre-command | 3 +- 14 files changed, 315 insertions(+), 38 deletions(-) create mode 100755 .buildkite/auditbeat/scripts/crosscompile.sh create mode 100755 .buildkite/auditbeat/scripts/package-step.sh create mode 100755 .buildkite/auditbeat/scripts/package.sh create mode 100644 .buildkite/auditbeat/scripts/unit-tests-win.ps1 create mode 100755 .buildkite/auditbeat/scripts/unit-tests.sh delete mode 100644 .buildkite/env-scripts/linux-env.sh delete mode 100644 .buildkite/env-scripts/macos-env.sh create mode 100644 .buildkite/env-scripts/unix-env.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 34321b61161b..0ce17fb8003f 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -1,5 +1,139 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_RHEL9: "family/core-rhel-9" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Auditbeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":ubuntu: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":rhel: Unit Tests" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":windows:-{{matrix.image}} Unit Tests" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - label: ":linux: Crosscompile" + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat: Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat extended tests" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "auditbeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/auditbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "auditbeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "generic-13-ventura-x64" + artifact_paths: "auditbeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "auditbeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + - "extended-tests" + - "extended-tests-win" + + steps: + - label: Package pipeline + commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload" diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh new file mode 100755 index 000000000000..a727816e5214 --- /dev/null +++ b/.buildkite/auditbeat/scripts/crosscompile.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/unix-env.sh + +echo ":: Executing Crosscompile ::" +cd auditbeat +make -C auditbeat crosscompile + diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh new file mode 100755 index 000000000000..34ecc3a77f8f --- /dev/null +++ b/.buildkite/auditbeat/scripts/package-step.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +changeset="^auditbeat/ + ^go.mod + ^pytest.ini + ^dev-tools/ + ^libbeat/ + ^testing/ + ^\.buildkite/auditbeat/" + +if are_files_changed "$changeset"; then + cat <<-EOF + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "auditbeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/auditbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "auditbeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +EOF +fi + diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh new file mode 100755 index 000000000000..868897e54717 --- /dev/null +++ b/.buildkite/auditbeat/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/unix-env.sh + +echo ":: Evaluate Auditbeat Changes ::" + +echo ":: Start Packaging ::" +cd auditbeat +umask 0022 +mage package + diff --git a/.buildkite/auditbeat/scripts/unit-tests-win.ps1 b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 new file mode 100644 index 000000000000..b3bc4ef348ed --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "-- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path auditbeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh new file mode 100755 index 000000000000..d63365d6dc4d --- /dev/null +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/unix-env.sh + +echo ":: Running Unit Tests ::" +sudo chmod -R go-w auditbeat/ + +cd auditbeat +umask 0022 +mage build unitTest diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh deleted file mode 100644 index b4de60bc2912..000000000000 --- a/.buildkite/env-scripts/linux-env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/env-scripts/util.sh - -DEBIAN_FRONTEND="noninteractive" - -export DEBIAN_FRONTEND - -# Remove this code once beats specific agent is set up -if [[ $PLATFORM_TYPE == "Linux" ]]; then - echo ":: Installing libs ::" - sudo apt-get update - sudo apt-get install -y libsystemd-dev - sudo apt install -y python3-pip - sudo apt-get install -y python3-venv -fi - -echo ":: Setting up environment ::" -add_bin_path -with_go -with_mage diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh deleted file mode 100644 index ac1486b64fdd..000000000000 --- a/.buildkite/env-scripts/macos-env.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $PLATFORM_TYPE == Darwin* ]]; then - echo ":: Setting larger ulimit on MacOS ::" - # To bypass file descriptor errors like "Too many open files error" on MacOS - ulimit -Sn 50000 - echo ":: ULIMIT :: $(ulimit -n)" -fi diff --git a/.buildkite/env-scripts/unix-env.sh b/.buildkite/env-scripts/unix-env.sh new file mode 100644 index 000000000000..5b569807fe84 --- /dev/null +++ b/.buildkite/env-scripts/unix-env.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +source .buildkite/env-scripts/util.sh + +DEBIAN_FRONTEND="noninteractive" +#IS_UBUNTU=$(grep -q 'Ubuntu' /etc/*release) +#IS_RHL=$(grep -q 'Red Hat' /etc/*release) + +if [[ $PLATFORM_TYPE == "Linux" ]]; then + IS_UBUNTU=$(grep -q 'Ubuntu' /etc/*release) + IS_RHL=$(grep -q 'Red Hat' /etc/*release) + + # Remove this code once beats specific agent is set up + if [[ $IS_UBUNTU ]]; then + export DEBIAN_FRONTEND + + echo ":: Ubuntu - Installing libs ::" + sudo apt-get update + sudo apt-get install -y libsystemd-dev + sudo apt install -y python3-pip + sudo apt-get install -y python3-venv + fi + + # Remove this code once beats specific agent is set up + if [[ $IS_RHL ]]; then + echo ":: RHL - Installing libs ::" + sudo yum update -y + sudo yum install -y systemd-devel + sudo yum install -y python3-pip + sudo yum install -y python3 + pip3 install virtualenv + fi +fi + +if [[ $PLATFORM_TYPE == Darwin* ]]; then + echo ":: Setting larger ulimit on MacOS ::" + # To bypass file descriptor errors like "Too many open files error" on MacOS + ulimit -Sn 50000 + echo ":: ULIMIT :: $(ulimit -n)" +fi + +echo ":: Setting up environment ::" +add_bin_path +with_go +with_mage diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index a3eabf70c0d3..6a48af79af7b 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -2,7 +2,7 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh +source .buildkite/env-scripts/unix-env.sh echo ":: Execute Integration Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh index 5e2e403dda87..f5c9dfa615b3 100755 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -2,7 +2,7 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh +source .buildkite/env-scripts/unix-env.sh echo ":: Execute Integration Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index 2ae226eb739c..cdc1146e7742 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -2,7 +2,7 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh +source .buildkite/env-scripts/unix-env.sh echo ":: Evaluate Filebeat Changes ::" diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index 2987f0930c8e..d4f321243375 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -2,8 +2,7 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh -source .buildkite/env-scripts/macos-env.sh +source .buildkite/env-scripts/unix-env.sh echo ":: Execute Unit Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0a1567e53cd5..268d0acf07ed 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -6,8 +6,7 @@ source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then +if [[ ( "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ) ]]; then if [[ ${PLATFORM_TYPE} = MINGW* ]]; then install_python_win fi