From d85e193a7fe5360d8698a2d270760eaceca150c2 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 24 Jan 2024 17:25:40 +0200 Subject: [PATCH 01/52] migrate libbeat pipeline --- .buildkite/libbeat/pipeline.libbeat.yml | 75 ++++++++++++- .buildkite/scripts/common.sh | 138 ++++++++++++++++++++++++ .buildkite/scripts/crosscompile.sh | 10 ++ .buildkite/scripts/go_int_tests.sh | 14 +++ .buildkite/scripts/packaging.sh | 77 +++++++++++++ .buildkite/scripts/py_int_tests.sh | 14 +++ .buildkite/scripts/stress_tests.sh | 10 ++ .buildkite/scripts/unit_tests.sh | 14 +++ .buildkite/scripts/win_unit_tests.ps1 | 70 ++++++++++++ 9 files changed, 420 insertions(+), 2 deletions(-) create mode 100755 .buildkite/scripts/common.sh create mode 100755 .buildkite/scripts/crosscompile.sh create mode 100755 .buildkite/scripts/go_int_tests.sh create mode 100755 .buildkite/scripts/packaging.sh create mode 100755 .buildkite/scripts/py_int_tests.sh create mode 100755 .buildkite/scripts/stress_tests.sh create mode 100755 .buildkite/scripts/unit_tests.sh create mode 100644 .buildkite/scripts/win_unit_tests.ps1 diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 34321b61161b..cc386aacc928 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -1,5 +1,76 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + SETUP_GVM_VERSION: "v0.5.1" + SETUP_MAGE_VERSION: "1.13.0" + SETUP_WIN_PYTHON_VERSION: "3.11.0" + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GO_AGENT_IMAGE: "golang:${GO_VERSION}" + DOCKER_COMPOSE_VERSION: "1.21.0" + PIPELINE_NAME: "metricbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "metricbeat/build/*.*" + + - label: ":go: Go Intergration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "metricbeat/build/*.*" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "metricbeat/build/*.*" + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "metricbeat/build/*.*" + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-stress-test" + command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "metricbeat/build/*.*" + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + artifact_paths: "metricbeat/build/*.*" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh new file mode 100755 index 000000000000..26360423f760 --- /dev/null +++ b/.buildkite/scripts/common.sh @@ -0,0 +1,138 @@ +#!/bin/bash +set -euo pipefail + +WORKSPACE=${WORKSPACE:-"$(pwd)"} +BIN="${WORKSPACE}/bin" +platform_type="$(uname)" +platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') +arch_type="$(uname -m)" +pipeline_name="metricbeat" +DEBIAN_FRONTEND="noninteractive" +sudo mkdir -p /etc/needrestart +echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + +with_docker_compose() { + local version=$1 + echo "Setting up the Docker-compose environment..." + create_workspace + retry 5 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" + chmod +x ${BIN}/docker-compose + docker-compose version +} + +create_workspace() { + if [[ ! -d "${BIN}" ]]; then + mkdir -p "${BIN}" + fi +} + +add_bin_path() { + echo "Adding PATH to the environment variables..." + create_workspace + export PATH="${PATH}:${BIN}" +} + +check_platform_architeture() { + case "${arch_type}" in + "x86_64") + arch_type="amd64" + ;; + "aarch64") + arch_type="arm64" + ;; + "arm64") + arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} + +with_mage() { + local install_packages=( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "gotest.tools/gotestsum" + ) + create_workspace + for pkg in "${install_packages[@]}"; do + go install "${pkg}@latest" + done +} + +with_go() { + echo "Setting up the Go environment..." + create_workspace + check_platform_architeture + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${arch_type}" + chmod +x "${BIN}/gvm" + eval "$(gvm $GO_VERSION)" + go version + which go + local go_path="$(go env GOPATH):$(go env GOPATH)/bin" + export PATH="${PATH}:${go_path}" +} + +with_python() { + if [ "${platform_type}" == "Linux" ]; then + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv libsystemd-dev + elif [ "${platform_type}" == "Darwin" ]; then + brew update + pip3 install virtualenv + ulimit -Sn 10000 + fi +} + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +are_files_changed() { + changeset=$1 + if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then + return 0; + else + echo "WARN! No files changed in $changeset" + return 1; + fi +} + +echo "--- Env preparation" + +if command -v docker-compose &> /dev/null +then + set +e + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then + echo "Versions match. No need to install docker-compose. Exiting." + else + echo "Versions don't match. Need to install the correct version of docker-compose." + with_docker_compose "${DOCKER_COMPOSE_VERSION}" + fi + set -e +fi + +add_bin_path +with_go "${GO_VERSION}" +with_mage +with_python diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh new file mode 100755 index 000000000000..92da62edb6c1 --- /dev/null +++ b/.buildkite/scripts/crosscompile.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +beats_subfilder=$1 + +echo "--- Run Crosscompile for $beats_subfilder" +make -C "${beats_subfilder}" crosscompile diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh new file mode 100755 index 000000000000..d58f43435509 --- /dev/null +++ b/.buildkite/scripts/go_int_tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +beats_subfilder=$1 + +echo "--- Run Go Intergration Tests for $beats_subfilder" +pushd "${beats_subfilder}" > /dev/null + +mage goIntegTest + +popd > /dev/null diff --git a/.buildkite/scripts/packaging.sh b/.buildkite/scripts/packaging.sh new file mode 100755 index 000000000000..684c982a3cdd --- /dev/null +++ b/.buildkite/scripts/packaging.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common.sh + +echo "--- Evaluate Metricbeat Changes" + +changeset=( + "^metricbeat/.*" + "^go.mod" + "^pytest.ini" + "^dev-tools/.*" + "^libbeat/.*" + "^testing/.*" + ) + +are_paths_changed() { + local inputs=("${@}") + local changelist=() + for change in "${inputs[@]}"; do + changed_files=$(git diff --name-only HEAD@{1} HEAD | grep -E "$change") + if [ -n "$changed_files" ]; then + changelist+=("${changed_files}") + fi + done + if [[ -n "${changelist[@]}" ]]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + local message="No files changed within Metricbeat changeset" + echo "$message" + buildkite-agent annotate "$message" --style "info" --context "$BUILDKITE_STEP_KEY" + return 1 + fi +} + +if are_paths_changed "${changeset[@]}" ; then + cat <<- YAML | buildkite-agent pipeline upload + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + +steps: + + - group: "Packaging" + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "pushd "metricbeat" > /dev/null && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "pushd "metricbeat" > /dev/null && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + +YAML + +else + echo "Nothing has changed or it's not a pull request. Skipping packaging..." + exit 0 +fi diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh new file mode 100755 index 000000000000..40f8faece2a4 --- /dev/null +++ b/.buildkite/scripts/py_int_tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +beats_subfilder=$1 + +echo "--- Run Python Intergration Tests for $beats_subfilder" +pushd "${beats_subfilder}" > /dev/null + +mage pythonIntegTest + +popd > /dev/null diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh new file mode 100755 index 000000000000..d614a69d70d9 --- /dev/null +++ b/.buildkite/scripts/stress_tests.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +beats_subfilder=$1 + +echo "--- Run Stress Tests for $beats_subfilder" +make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C $beats_subfilder stress-tests diff --git a/.buildkite/scripts/unit_tests.sh b/.buildkite/scripts/unit_tests.sh new file mode 100755 index 000000000000..578e2ceb124b --- /dev/null +++ b/.buildkite/scripts/unit_tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +beats_subfilder=$1 + +echo "--- Run Unit Tests" +sudo chmod -R go-w "${beats_subfilder}/" +pushd "${beats_subfilder}" > /dev/null +umask 0022 +mage build unitTest +popd > /dev/null diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 new file mode 100644 index 000000000000..34833d183ffa --- /dev/null +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -0,0 +1,70 @@ +$ErrorActionPreference = "Stop" # set -e +$WorkFolder = "metricbeat" +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} +function withChoco { + Write-Host "-- Configure Choco --" + $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +} +function withGolang($version) { + Write-Host "-- Install golang $version --" + choco install -y golang --version=$version + refreshenv + go version +} +function withPython($version) { + Write-Host "-- Install Python $version --" + choco install python --version=$version + refreshenv + python --version +} +function withMinGW { + Write-Host "-- Install MinGW --" + choco install mingw -y + refreshenv +} +function installGoDependencies { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report/v2" + "gotest.tools/gotestsum" + ) + foreach ($pkg in $installPackages) { + go install "$pkg@latest" + } +} + +fixCRLF + +withChoco + +withGolang $env:GO_VERSION + +installGoDependencies + +withPython $env:SETUP_WIN_PYTHON_VERSION + +withMinGW + +$ErrorActionPreference = "Continue" # set +e + +Push-Location $WorkFolder + +New-Item -ItemType Directory -Force -Path "build" +mage build unitTest + +Pop-Location + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE From 35fa2031a672aa4e3ce701eddaa41fab3b359089 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 24 Jan 2024 17:28:48 +0200 Subject: [PATCH 02/52] migrate libbeat pipeline --- .buildkite/libbeat/pipeline.libbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index cc386aacc928..b70f01e0f99f 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -11,7 +11,7 @@ env: IMAGE_MACOS_X86_64: "generic-13-ventura-x64" GO_AGENT_IMAGE: "golang:${GO_VERSION}" DOCKER_COMPOSE_VERSION: "1.21.0" - PIPELINE_NAME: "metricbeat" + PIPELINE_NAME: "libbeat" steps: From ac9ac7b6dd64ddc31ca35792471e215da62de83d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 24 Jan 2024 17:52:48 +0200 Subject: [PATCH 03/52] add hooks --- .buildkite/hooks/post-checkout | 53 +++++++++++++++++++++++ .buildkite/hooks/pre-command | 8 ++++ .buildkite/scripts/packaging.sh | 77 --------------------------------- 3 files changed, 61 insertions(+), 77 deletions(-) create mode 100644 .buildkite/hooks/post-checkout create mode 100644 .buildkite/hooks/pre-command delete mode 100755 .buildkite/scripts/packaging.sh diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100644 index 000000000000..e10f15de7b65 --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "Commit information" +git --no-pager log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 000000000000..7689703c0cb1 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +echo "Golang version:" +GO_VERSION=$(cat .go-version) +export GO_VERSION +echo "GO_VERSION: ${GO_VERSION}" diff --git a/.buildkite/scripts/packaging.sh b/.buildkite/scripts/packaging.sh deleted file mode 100755 index 684c982a3cdd..000000000000 --- a/.buildkite/scripts/packaging.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/common.sh - -echo "--- Evaluate Metricbeat Changes" - -changeset=( - "^metricbeat/.*" - "^go.mod" - "^pytest.ini" - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" - ) - -are_paths_changed() { - local inputs=("${@}") - local changelist=() - for change in "${inputs[@]}"; do - changed_files=$(git diff --name-only HEAD@{1} HEAD | grep -E "$change") - if [ -n "$changed_files" ]; then - changelist+=("${changed_files}") - fi - done - if [[ -n "${changelist[@]}" ]]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - local message="No files changed within Metricbeat changeset" - echo "$message" - buildkite-agent annotate "$message" --style "info" --context "$BUILDKITE_STEP_KEY" - return 1 - fi -} - -if are_paths_changed "${changeset[@]}" ; then - cat <<- YAML | buildkite-agent pipeline upload - -env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - -steps: - - - group: "Packaging" - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "pushd "metricbeat" > /dev/null && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "pushd "metricbeat" > /dev/null && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - -YAML - -else - echo "Nothing has changed or it's not a pull request. Skipping packaging..." - exit 0 -fi From 8efb4f0d5d0195723af52dbe326cc5c81493c4f7 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 24 Jan 2024 18:57:47 +0200 Subject: [PATCH 04/52] fix go-int tests --- .buildkite/libbeat/pipeline.libbeat.yml | 14 +++++++------- .buildkite/scripts/common.sh | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index b70f01e0f99f..aa7f436180f1 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -25,7 +25,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "metricbeat/build/*.*" + artifact_paths: "libbeat/build/*.*" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" @@ -34,7 +34,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "metricbeat/build/*.*" + artifact_paths: "libbeat/build/*.*" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" @@ -43,7 +43,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "metricbeat/build/*.*" + artifact_paths: "libbeat/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" @@ -52,16 +52,16 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "metricbeat/build/*.*" + artifact_paths: "libbeat/build/*.*" - - label: ":negative_squared_cross_mark: Cross compile" + - label: ":negative_squared_cross_mark: Stress Tests" key: "mandatory-stress-test" command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "metricbeat/build/*.*" + artifact_paths: "libbeat/build/*.*" - group: "Extended Tests" key: "extended-tests" @@ -73,4 +73,4 @@ steps: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.xlarge" - artifact_paths: "metricbeat/build/*.*" + artifact_paths: "libbeat/build/*.*" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 26360423f760..b249d9bbb52c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -79,10 +79,10 @@ with_go() { with_python() { if [ "${platform_type}" == "Linux" ]; then sudo apt-get update - sudo apt-get install -y python3-pip python3-venv libsystemd-dev + sudo apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev elif [ "${platform_type}" == "Darwin" ]; then brew update - pip3 install virtualenv + pip3 install virtualenv libpcap ulimit -Sn 10000 fi } From 70bfa4ee96e7c72162b9ccbd67e883fe13e5b98e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 14:15:06 +0200 Subject: [PATCH 05/52] fix go-int tests --- .buildkite/scripts/common.sh | 16 +++++++++------- .buildkite/scripts/crosscompile.sh | 2 +- .buildkite/scripts/go_int_tests.sh | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b249d9bbb52c..d899fc9b82a9 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -74,6 +74,7 @@ with_go() { which go local go_path="$(go env GOPATH):$(go env GOPATH)/bin" export PATH="${PATH}:${go_path}" + go mod download } with_python() { @@ -106,16 +107,14 @@ retry() { return 0 } -are_files_changed() { - changeset=$1 - if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then - return 0; - else - echo "WARN! No files changed in $changeset" - return 1; +config_git() { + if [ -z "$(git config --get user.email)" ]; then + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" fi } + echo "--- Env preparation" if command -v docker-compose &> /dev/null @@ -136,3 +135,6 @@ add_bin_path with_go "${GO_VERSION}" with_mage with_python +config_git + +mage dumpVariables diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh index 92da62edb6c1..8050026e33b1 100755 --- a/.buildkite/scripts/crosscompile.sh +++ b/.buildkite/scripts/crosscompile.sh @@ -7,4 +7,4 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Crosscompile for $beats_subfilder" -make -C "${beats_subfilder}" crosscompile +make -C $beats_subfilder crosscompile diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index d58f43435509..f91eb1316799 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -7,8 +7,8 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Go Intergration Tests for $beats_subfilder" +sudo chmod -R go-w "${beats_subfilder}/" pushd "${beats_subfilder}" > /dev/null - mage goIntegTest popd > /dev/null From 9f5fd0d693761e140b155601ffaa3590e1d935fd Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 14:37:41 +0200 Subject: [PATCH 06/52] revert mage dumpVariables --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d899fc9b82a9..9e472d1772b5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -120,8 +120,8 @@ echo "--- Env preparation" if command -v docker-compose &> /dev/null then set +e - echo "Found docker-compose. Checking version.." FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//) + echo "Found docker-compose version:$FOUND_DOCKER_COMPOSE_VERSION" if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then echo "Versions match. No need to install docker-compose. Exiting." else @@ -137,4 +137,4 @@ with_mage with_python config_git -mage dumpVariables +# mage dumpVariables From e76a97a5d74ace3ee30e76c5b390ba614980d067 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 15:48:29 +0200 Subject: [PATCH 07/52] debug --- .buildkite/hooks/pre-command | 5 +- .buildkite/libbeat/pipeline.libbeat.yml | 10 +--- .buildkite/scripts/common.sh | 6 +-- .buildkite/scripts/setenv.sh | 25 +++++++++ .buildkite/scripts/win_unit_tests.ps1 | 70 ------------------------- 5 files changed, 29 insertions(+), 87 deletions(-) create mode 100644 .buildkite/scripts/setenv.sh delete mode 100644 .buildkite/scripts/win_unit_tests.ps1 diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7689703c0cb1..80198244cc02 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,7 +2,4 @@ set -euo pipefail -echo "Golang version:" -GO_VERSION=$(cat .go-version) -export GO_VERSION -echo "GO_VERSION: ${GO_VERSION}" +source .buildkite/scripts/setenv.sh diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index aa7f436180f1..39a1f7e6005a 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -1,16 +1,8 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + env: - SETUP_GVM_VERSION: "v0.5.1" - SETUP_MAGE_VERSION: "1.13.0" - SETUP_WIN_PYTHON_VERSION: "3.11.0" IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - GO_AGENT_IMAGE: "golang:${GO_VERSION}" - DOCKER_COMPOSE_VERSION: "1.21.0" PIPELINE_NAME: "libbeat" steps: diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9e472d1772b5..e8e8bc791384 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -6,7 +6,7 @@ BIN="${WORKSPACE}/bin" platform_type="$(uname)" platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') arch_type="$(uname -m)" -pipeline_name="metricbeat" + DEBIAN_FRONTEND="noninteractive" sudo mkdir -p /etc/needrestart echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null @@ -74,7 +74,7 @@ with_go() { which go local go_path="$(go env GOPATH):$(go env GOPATH)/bin" export PATH="${PATH}:${go_path}" - go mod download + # go mod download } with_python() { @@ -136,5 +136,3 @@ with_go "${GO_VERSION}" with_mage with_python config_git - -# mage dumpVariables diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh new file mode 100644 index 000000000000..9d9f6cb0abe1 --- /dev/null +++ b/.buildkite/scripts/setenv.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +SETUP_GVM_VERSION="v0.5.1" +DOCKER_COMPOSE_VERSION="1.21.0" +GO_VERSION=$(cat .go-version) +echo "GO_VERSION: ${GO_VERSION}" + +os_name=$(uname) + +case "$os_name" in + Darwin | Linux) + export DOCKER_COMPOSE_VERSION + export SETUP_GVM_VERSION + export GO_VERSION + mage dumpVariables + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) + # TODO: Add environment variables from other pipelines, at least from metricbeat + ;; + *) + echo "Unsupported operating system" + exit 1 + ;; +esac diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 deleted file mode 100644 index 34833d183ffa..000000000000 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ /dev/null @@ -1,70 +0,0 @@ -$ErrorActionPreference = "Stop" # set -e -$WorkFolder = "metricbeat" -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF { - Write-Host "-- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} -function withChoco { - Write-Host "-- Configure Choco --" - $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" -} -function withGolang($version) { - Write-Host "-- Install golang $version --" - choco install -y golang --version=$version - refreshenv - go version -} -function withPython($version) { - Write-Host "-- Install Python $version --" - choco install python --version=$version - refreshenv - python --version -} -function withMinGW { - Write-Host "-- Install MinGW --" - choco install mingw -y - refreshenv -} -function installGoDependencies { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report/v2" - "gotest.tools/gotestsum" - ) - foreach ($pkg in $installPackages) { - go install "$pkg@latest" - } -} - -fixCRLF - -withChoco - -withGolang $env:GO_VERSION - -installGoDependencies - -withPython $env:SETUP_WIN_PYTHON_VERSION - -withMinGW - -$ErrorActionPreference = "Continue" # set +e - -Push-Location $WorkFolder - -New-Item -ItemType Directory -Force -Path "build" -mage build unitTest - -Pop-Location - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE From bbe62dcf95eb9efe4c58eac66d338a3a5916ce06 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 15:50:58 +0200 Subject: [PATCH 08/52] debug --- .buildkite/scripts/common.sh | 1 + .buildkite/scripts/setenv.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 .buildkite/scripts/setenv.sh diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e8e8bc791384..5bd014d87c87 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -136,3 +136,4 @@ with_go "${GO_VERSION}" with_mage with_python config_git +mage dumpVariables diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh old mode 100644 new mode 100755 index 9d9f6cb0abe1..440808ba8b8f --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -13,7 +13,6 @@ case "$os_name" in export DOCKER_COMPOSE_VERSION export SETUP_GVM_VERSION export GO_VERSION - mage dumpVariables ;; MINGW* | MSYS* | CYGWIN* | Windows_NT) # TODO: Add environment variables from other pipelines, at least from metricbeat From 4b9b8010da77470f5e2075547d3b3588caac5960 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 16:26:52 +0200 Subject: [PATCH 09/52] debug --- .buildkite/libbeat/pipeline.libbeat.yml | 94 +++++++++++++------------ .buildkite/scripts/common.sh | 8 ++- .buildkite/scripts/setenv.sh | 5 +- 3 files changed, 55 insertions(+), 52 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 39a1f7e6005a..de26f338105c 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -3,6 +3,7 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + GO_AGENT_IMAGE: "golang:${GO_VERSION}" PIPELINE_NAME: "libbeat" steps: @@ -10,59 +11,60 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + #OK + # - label: ":linux: Ubuntu Unit Tests" + # key: "mandatory-linux-unit-test" + # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # provider: "gcp" + image: "${GO_AGENT_IMAGE}" + # machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":python: Python Integration Tests" + # key: "mandatory-python-int-test" + # command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - - label: ":negative_squared_cross_mark: Cross compile" - key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":negative_squared_cross_mark: Cross compile" + # key: "mandatory-cross-compile" + # command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - - label: ":negative_squared_cross_mark: Stress Tests" - key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":negative_squared_cross_mark: Stress Tests" + # key: "mandatory-stress-test" + # command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":mac: Arm64 Unit Tests" - key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - artifact_paths: "libbeat/build/*.*" + # - group: "Extended Tests" + # key: "extended-tests" + # steps: + # - label: ":mac: Arm64 Unit Tests" + # key: "extended-arm64-unit-tests" + # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "aws" + # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + # instanceType: "t4g.xlarge" + # artifact_paths: "libbeat/build/*.*" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 5bd014d87c87..33cccd288e7a 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -17,6 +17,8 @@ with_docker_compose() { create_workspace retry 5 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" chmod +x ${BIN}/docker-compose + ls -la ${BIN} + ${BIN}/docker-compose version docker-compose version } @@ -74,7 +76,7 @@ with_go() { which go local go_path="$(go env GOPATH):$(go env GOPATH)/bin" export PATH="${PATH}:${go_path}" - # go mod download + go mod download } with_python() { @@ -121,7 +123,7 @@ if command -v docker-compose &> /dev/null then set +e FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//) - echo "Found docker-compose version:$FOUND_DOCKER_COMPOSE_VERSION" + echo "Found docker-compose version: $FOUND_DOCKER_COMPOSE_VERSION" if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then echo "Versions match. No need to install docker-compose. Exiting." else @@ -132,7 +134,7 @@ then fi add_bin_path -with_go "${GO_VERSION}" +# with_go "${GO_VERSION}" with_mage with_python config_git diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 440808ba8b8f..a8cf7512c37a 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -5,10 +5,9 @@ SETUP_GVM_VERSION="v0.5.1" DOCKER_COMPOSE_VERSION="1.21.0" GO_VERSION=$(cat .go-version) echo "GO_VERSION: ${GO_VERSION}" +os_type=$(uname) -os_name=$(uname) - -case "$os_name" in +case "$os_type" in Darwin | Linux) export DOCKER_COMPOSE_VERSION export SETUP_GVM_VERSION From 857f48d9990459bbc0fc9be7cc3efcb654b3b4ee Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 16:30:34 +0200 Subject: [PATCH 10/52] debug --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 33cccd288e7a..248bcb6f9f5e 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -8,8 +8,8 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') arch_type="$(uname -m)" DEBIAN_FRONTEND="noninteractive" -sudo mkdir -p /etc/needrestart -echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null +# sudo mkdir -p /etc/needrestart +# echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null with_docker_compose() { local version=$1 From 36ffb33ef46f08837e319da7e157f6ff79cd20aa Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 16:39:49 +0200 Subject: [PATCH 11/52] debug --- .buildkite/scripts/common.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 248bcb6f9f5e..5dd38cc3010a 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,7 +9,8 @@ arch_type="$(uname -m)" DEBIAN_FRONTEND="noninteractive" # sudo mkdir -p /etc/needrestart -# echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null +mkdir -p /etc/needrestart +echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null with_docker_compose() { local version=$1 @@ -81,8 +82,10 @@ with_go() { with_python() { if [ "${platform_type}" == "Linux" ]; then - sudo apt-get update - sudo apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev + apt-get update + apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev + # sudo apt-get update + # sudo apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev elif [ "${platform_type}" == "Darwin" ]; then brew update pip3 install virtualenv libpcap From 22f445508eefab490edde1805c36ebca72a33f0e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 16:43:38 +0200 Subject: [PATCH 12/52] debug --- .buildkite/scripts/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 5dd38cc3010a..7625acdf0ab1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,8 +9,9 @@ arch_type="$(uname -m)" DEBIAN_FRONTEND="noninteractive" # sudo mkdir -p /etc/needrestart +# echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null mkdir -p /etc/needrestart -echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null +echo "\$nrconf{restart} = 'a';" | tee -a /etc/needrestart/needrestart.conf > /dev/null with_docker_compose() { local version=$1 From e64c72c58a1803031c5d38ecdba944740a2a410f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 16:51:33 +0200 Subject: [PATCH 13/52] debug Go Int Tests --- .buildkite/scripts/common.sh | 2 ++ .buildkite/scripts/go_int_tests.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7625acdf0ab1..85416d398170 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -135,6 +135,8 @@ then with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi set -e +else + with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi add_bin_path diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index f91eb1316799..044e19bdfdd8 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -7,7 +7,8 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Go Intergration Tests for $beats_subfilder" -sudo chmod -R go-w "${beats_subfilder}/" +# sudo chmod -R go-w "${beats_subfilder}/" +chmod -R go-w "${beats_subfilder}/" pushd "${beats_subfilder}" > /dev/null mage goIntegTest From 6c1747f89527cffaa662ffa3990b7b64a6f0e265 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 17:09:50 +0200 Subject: [PATCH 14/52] debug Go Int Tests --- .buildkite/scripts/common.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 85416d398170..c0d58b85b0bc 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -19,8 +19,7 @@ with_docker_compose() { create_workspace retry 5 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" chmod +x ${BIN}/docker-compose - ls -la ${BIN} - ${BIN}/docker-compose version + export PATH="${PATH}:${BIN}" docker-compose version } @@ -139,7 +138,7 @@ else with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi -add_bin_path +# add_bin_path # with_go "${GO_VERSION}" with_mage with_python From 6180a26cc42ee32171b8d9e28e81502e79ff4c25 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 17:24:16 +0200 Subject: [PATCH 15/52] debug Go Int Tests --- .buildkite/libbeat/pipeline.libbeat.yml | 7 +++---- .buildkite/scripts/common.sh | 18 +++++++----------- .buildkite/scripts/go_int_tests.sh | 3 +-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index de26f338105c..e58912184fe4 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -3,7 +3,6 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - GO_AGENT_IMAGE: "golang:${GO_VERSION}" PIPELINE_NAME: "libbeat" steps: @@ -25,9 +24,9 @@ steps: key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" agents: - # provider: "gcp" - image: "${GO_AGENT_IMAGE}" - # machineType: "c2-standard-16" + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" # - label: ":python: Python Integration Tests" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index c0d58b85b0bc..4a09d18ad4c4 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -8,10 +8,8 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') arch_type="$(uname -m)" DEBIAN_FRONTEND="noninteractive" -# sudo mkdir -p /etc/needrestart -# echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null -mkdir -p /etc/needrestart -echo "\$nrconf{restart} = 'a';" | tee -a /etc/needrestart/needrestart.conf > /dev/null +sudo mkdir -p /etc/needrestart +echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null with_docker_compose() { local version=$1 @@ -82,10 +80,8 @@ with_go() { with_python() { if [ "${platform_type}" == "Linux" ]; then - apt-get update - apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev - # sudo apt-get update - # sudo apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv libsystemd-dev libpcap-dev elif [ "${platform_type}" == "Darwin" ]; then brew update pip3 install virtualenv libpcap @@ -125,7 +121,7 @@ echo "--- Env preparation" if command -v docker-compose &> /dev/null then set +e - FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//) + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version | awk '{print $3}' | sed s/\,//) echo "Found docker-compose version: $FOUND_DOCKER_COMPOSE_VERSION" if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then echo "Versions match. No need to install docker-compose. Exiting." @@ -138,8 +134,8 @@ else with_docker_compose "${DOCKER_COMPOSE_VERSION}" fi -# add_bin_path -# with_go "${GO_VERSION}" +add_bin_path +with_go "${GO_VERSION}" with_mage with_python config_git diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index 044e19bdfdd8..f91eb1316799 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -7,8 +7,7 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Go Intergration Tests for $beats_subfilder" -# sudo chmod -R go-w "${beats_subfilder}/" -chmod -R go-w "${beats_subfilder}/" +sudo chmod -R go-w "${beats_subfilder}/" pushd "${beats_subfilder}" > /dev/null mage goIntegTest From 8282de05250b1fa649bf4a6147b5cb5e67c95dc4 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 17:49:00 +0200 Subject: [PATCH 16/52] debug Go Int Tests --- .buildkite/scripts/go_int_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index f91eb1316799..36179ca45929 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -7,8 +7,9 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Go Intergration Tests for $beats_subfilder" -sudo chmod -R go-w "${beats_subfilder}/" +sudo chmod -R go-w ${beats_subfilder}/ pushd "${beats_subfilder}" > /dev/null +umask 0022 mage goIntegTest popd > /dev/null From 01a3e6163ebfe307a0916c9151ed05a7b6a3fc24 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 17:55:35 +0200 Subject: [PATCH 17/52] debug Go Int Tests --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 4a09d18ad4c4..84ac20852bc1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -17,7 +17,7 @@ with_docker_compose() { create_workspace retry 5 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" chmod +x ${BIN}/docker-compose - export PATH="${PATH}:${BIN}" + export PATH="${BIN}:${PATH}" docker-compose version } @@ -30,7 +30,7 @@ create_workspace() { add_bin_path() { echo "Adding PATH to the environment variables..." create_workspace - export PATH="${PATH}:${BIN}" + export PATH="${BIN}:${PATH}" } check_platform_architeture() { From 383320e1d27d7726f4735bad9fab1636f59ce1c7 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 18:17:49 +0200 Subject: [PATCH 18/52] full test --- .buildkite/libbeat/pipeline.libbeat.yml | 87 ++++++++++++------------- .buildkite/scripts/go_int_tests.sh | 2 +- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index e58912184fe4..39a1f7e6005a 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -10,15 +10,14 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - #OK - # - label: ":linux: Ubuntu Unit Tests" - # key: "mandatory-linux-unit-test" - # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" @@ -29,41 +28,41 @@ steps: machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" - # - label: ":python: Python Integration Tests" - # key: "mandatory-python-int-test" - # command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - label: ":negative_squared_cross_mark: Cross compile" - # key: "mandatory-cross-compile" - # command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - label: ":negative_squared_cross_mark: Stress Tests" - # key: "mandatory-stress-test" - # command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":negative_squared_cross_mark: Stress Tests" + key: "mandatory-stress-test" + command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - group: "Extended Tests" - # key: "extended-tests" - # steps: - # - label: ":mac: Arm64 Unit Tests" - # key: "extended-arm64-unit-tests" - # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "aws" - # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - # instanceType: "t4g.xlarge" - # artifact_paths: "libbeat/build/*.*" + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + artifact_paths: "libbeat/build/*.*" diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index 36179ca45929..f22027655956 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -9,7 +9,7 @@ beats_subfilder=$1 echo "--- Run Go Intergration Tests for $beats_subfilder" sudo chmod -R go-w ${beats_subfilder}/ pushd "${beats_subfilder}" > /dev/null -umask 0022 +# umask 0022 mage goIntegTest popd > /dev/null From d95c25f37eb3206158e321cb1945e971da0885c8 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 18:47:07 +0200 Subject: [PATCH 19/52] debug Python Int Tests --- .buildkite/libbeat/pipeline.libbeat.yml | 50 ++++++++++++------------- .buildkite/scripts/go_int_tests.sh | 3 +- .buildkite/scripts/py_int_tests.sh | 3 ++ 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 39a1f7e6005a..34493b74e4ee 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -10,23 +10,23 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":linux: Ubuntu Unit Tests" + # key: "mandatory-linux-unit-test" + # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - - label: ":go: Go Intergration Tests" - key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":go: Go Intergration Tests" + # key: "mandatory-int-test" + # command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" @@ -46,19 +46,19 @@ steps: machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" - - label: ":negative_squared_cross_mark: Stress Tests" - key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":testengine: Stress Tests" + # key: "mandatory-stress-test" + # command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - group: "Extended Tests" key: "extended-tests" steps: - - label: ":mac: Arm64 Unit Tests" + - label: ":linux: Arm64 Unit Tests" key: "extended-arm64-unit-tests" command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" agents: diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index f22027655956..8dea561e9413 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -9,7 +9,8 @@ beats_subfilder=$1 echo "--- Run Go Intergration Tests for $beats_subfilder" sudo chmod -R go-w ${beats_subfilder}/ pushd "${beats_subfilder}" > /dev/null -# umask 0022 + +umask 0022 mage goIntegTest popd > /dev/null diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh index 40f8faece2a4..3e0d10bb29ec 100755 --- a/.buildkite/scripts/py_int_tests.sh +++ b/.buildkite/scripts/py_int_tests.sh @@ -7,8 +7,11 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Python Intergration Tests for $beats_subfilder" + +sudo chmod -R go-w ${beats_subfilder}/ pushd "${beats_subfilder}" > /dev/null +umask 0022 mage pythonIntegTest popd > /dev/null From c1b2b35669190282491a0b6dca4df2b8a14ce770 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 19:26:14 +0200 Subject: [PATCH 20/52] debug Crosscompile --- .buildkite/scripts/common.sh | 2 +- .buildkite/scripts/crosscompile.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 84ac20852bc1..a39ca7a88051 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -85,7 +85,7 @@ with_python() { elif [ "${platform_type}" == "Darwin" ]; then brew update pip3 install virtualenv libpcap - ulimit -Sn 10000 + ulimit -Sn 50000 fi } diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh index 8050026e33b1..a96cd2102f9a 100755 --- a/.buildkite/scripts/crosscompile.sh +++ b/.buildkite/scripts/crosscompile.sh @@ -7,4 +7,8 @@ set -euo pipefail beats_subfilder=$1 echo "--- Run Crosscompile for $beats_subfilder" +pushd "${beats_subfilder}" > /dev/null + make -C $beats_subfilder crosscompile + +popd > /dev/null From d688a5c2671de7e42093b1b93ce64d813b3bb96a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 19:50:45 +0200 Subject: [PATCH 21/52] debug Crosscompile --- .buildkite/libbeat/pipeline.libbeat.yml | 38 ++++++++++++------------- .buildkite/scripts/crosscompile.sh | 9 +++--- .buildkite/scripts/go_int_tests.sh | 8 +++--- .buildkite/scripts/py_int_tests.sh | 8 +++--- .buildkite/scripts/stress_tests.sh | 6 ++-- .buildkite/scripts/unit_tests.sh | 6 ++-- 6 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 34493b74e4ee..e3519835e9b3 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -28,14 +28,14 @@ steps: # machineType: "c2-standard-16" # artifact_paths: "libbeat/build/*.*" - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":python: Python Integration Tests" + # key: "mandatory-python-int-test" + # command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" @@ -55,14 +55,14 @@ steps: # machineType: "c2-standard-16" # artifact_paths: "libbeat/build/*.*" - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":linux: Arm64 Unit Tests" - key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - artifact_paths: "libbeat/build/*.*" + # - group: "Extended Tests" + # key: "extended-tests" + # steps: + # - label: ":linux: Arm64 Unit Tests" + # key: "extended-arm64-unit-tests" + # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "aws" + # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + # instanceType: "t4g.xlarge" + # artifact_paths: "libbeat/build/*.*" diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh index a96cd2102f9a..20fa2bcf2296 100755 --- a/.buildkite/scripts/crosscompile.sh +++ b/.buildkite/scripts/crosscompile.sh @@ -4,11 +4,12 @@ source .buildkite/scripts/common.sh set -euo pipefail -beats_subfilder=$1 +beats_subfolder=$1 -echo "--- Run Crosscompile for $beats_subfilder" -pushd "${beats_subfilder}" > /dev/null +echo "--- Run Crosscompile for $beats_subfolder" -make -C $beats_subfilder crosscompile +pushd "${beats_subfolder}" > /dev/null + +make crosscompile popd > /dev/null diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index 8dea561e9413..af662cc671bd 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -4,11 +4,11 @@ source .buildkite/scripts/common.sh set -euo pipefail -beats_subfilder=$1 +beats_subfolder=$1 -echo "--- Run Go Intergration Tests for $beats_subfilder" -sudo chmod -R go-w ${beats_subfilder}/ -pushd "${beats_subfilder}" > /dev/null +echo "--- Run Go Intergration Tests for $beats_subfolder" +sudo chmod -R go-w ${beats_subfolder}/ +pushd "${beats_subfolder}" > /dev/null umask 0022 mage goIntegTest diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh index 3e0d10bb29ec..3e6f21f78bfd 100755 --- a/.buildkite/scripts/py_int_tests.sh +++ b/.buildkite/scripts/py_int_tests.sh @@ -4,12 +4,12 @@ source .buildkite/scripts/common.sh set -euo pipefail -beats_subfilder=$1 +beats_subfolder=$1 -echo "--- Run Python Intergration Tests for $beats_subfilder" +echo "--- Run Python Intergration Tests for $beats_subfolder" -sudo chmod -R go-w ${beats_subfilder}/ -pushd "${beats_subfilder}" > /dev/null +sudo chmod -R go-w ${beats_subfolder}/ +pushd "${beats_subfolder}" > /dev/null umask 0022 mage pythonIntegTest diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh index d614a69d70d9..ed1b4e4868b5 100755 --- a/.buildkite/scripts/stress_tests.sh +++ b/.buildkite/scripts/stress_tests.sh @@ -4,7 +4,7 @@ source .buildkite/scripts/common.sh set -euo pipefail -beats_subfilder=$1 +beats_subfolder=$1 -echo "--- Run Stress Tests for $beats_subfilder" -make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C $beats_subfilder stress-tests +echo "--- Run Stress Tests for $beats_subfolder" +make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C $beats_subfolder stress-tests diff --git a/.buildkite/scripts/unit_tests.sh b/.buildkite/scripts/unit_tests.sh index 578e2ceb124b..a17342cd6c1b 100755 --- a/.buildkite/scripts/unit_tests.sh +++ b/.buildkite/scripts/unit_tests.sh @@ -4,11 +4,11 @@ source .buildkite/scripts/common.sh set -euo pipefail -beats_subfilder=$1 +beats_subfolder=$1 echo "--- Run Unit Tests" -sudo chmod -R go-w "${beats_subfilder}/" -pushd "${beats_subfilder}" > /dev/null +sudo chmod -R go-w "${beats_subfolder}/" +pushd "${beats_subfolder}" > /dev/null umask 0022 mage build unitTest popd > /dev/null From 7ab97188274df2803157cd9d476905b215de5733 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 20:04:51 +0200 Subject: [PATCH 22/52] debug Crosscompile --- libbeat/scripts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 100ccd3f0137..7a5cef960ebe 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/arm64 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 From b23fc85947c78da4927789debf745256d891b2b1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 20:17:34 +0200 Subject: [PATCH 23/52] debug Crosscompile --- libbeat/scripts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 7a5cef960ebe..86862c38b7bf 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 !linux/386 !linux/arm !linux/arm64 !linux/mipsle !linux/mips64le !linux/ppc64 !linux/ppc64le !linux/mips64 !linux/mips !linux/s390x !windows/386 !freebsd/386 !freebsd/arm !netbsd/386 !netbsd/arm ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 From dec018cc9b871d13f2a05dcd1c003a1ffe32c57e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 25 Jan 2024 22:06:44 +0200 Subject: [PATCH 24/52] debug Crosscompile --- libbeat/scripts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 86862c38b7bf..13c955ea1793 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 !linux/386 !linux/arm !linux/arm64 !linux/mipsle !linux/mips64le !linux/ppc64 !linux/ppc64le !linux/mips64 !linux/mips !linux/s390x !windows/386 !freebsd/386 !freebsd/arm !netbsd/386 !netbsd/arm ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 !linux/386 !linux/mipsle !linux/mips64le !linux/ppc64 !linux/ppc64le !linux/mips64 !linux/mips !linux/s390x !windows/386 !freebsd/386 !freebsd/arm !netbsd/386 !netbsd/arm !openbsd/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 From 3ffa0e72a20b865e180f4e13622d72cd31e058c7 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 12:24:13 +0200 Subject: [PATCH 25/52] full test --- .buildkite/libbeat/pipeline.libbeat.yml | 86 ++++++++++++------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index e3519835e9b3..edb6cb85ef32 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -10,32 +10,32 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - # - label: ":linux: Ubuntu Unit Tests" - # key: "mandatory-linux-unit-test" - # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - label: ":go: Go Intergration Tests" - # key: "mandatory-int-test" - # command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":go: Go Intergration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - label: ":python: Python Integration Tests" - # key: "mandatory-python-int-test" - # command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" @@ -46,23 +46,23 @@ steps: machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" - # - label: ":testengine: Stress Tests" - # key: "mandatory-stress-test" - # command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":testengine: Stress Tests" + key: "mandatory-stress-test" + command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - group: "Extended Tests" - # key: "extended-tests" - # steps: - # - label: ":linux: Arm64 Unit Tests" - # key: "extended-arm64-unit-tests" - # command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "aws" - # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - # instanceType: "t4g.xlarge" - # artifact_paths: "libbeat/build/*.*" + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + artifact_paths: "libbeat/build/*.*" From 0d5c9f8928c770777fc458a140165a0cd3227c93 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 13:21:39 +0200 Subject: [PATCH 26/52] debug stress tests --- .buildkite/libbeat/pipeline.libbeat.yml | 48 ++++++++++++------------- libbeat/scripts/Makefile | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index edb6cb85ef32..ec6c92454902 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -19,32 +19,32 @@ steps: machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" - - label: ":go: Go Intergration Tests" - key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":go: Go Intergration Tests" + # key: "mandatory-int-test" + # command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":python: Python Integration Tests" + # key: "mandatory-python-int-test" + # command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - - label: ":negative_squared_cross_mark: Cross compile" - key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + # - label: ":negative_squared_cross_mark: Cross compile" + # key: "mandatory-cross-compile" + # command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" + # artifact_paths: "libbeat/build/*.*" - label: ":testengine: Stress Tests" key: "mandatory-stress-test" diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 13c955ea1793..a50dd050795f 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -46,7 +46,7 @@ export PATH := ./bin:$(PATH) GOFILES = $(shell find . -type f -name '*.go' 2>/dev/null) GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*" 2>/dev/null) GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go' 2>/dev/null) -GOPACKAGES_STRESSTESTS=$(shell find . -name '*.go' 2>/dev/null | xargs grep -l '\+build.*stresstest' | xargs -n1 dirname | uniq) +GOPACKAGES_STRESSTESTS=$(find . -type d \( -name "stress" \)) SHELL=bash ES_HOST?=elasticsearch ES_PORT?=9200 From 1fe6f2029c2c61832487bbda41b7303e77fd515e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 14:03:55 +0200 Subject: [PATCH 27/52] debug stress tests --- libbeat/scripts/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index a50dd050795f..2ab7331de168 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -46,7 +46,7 @@ export PATH := ./bin:$(PATH) GOFILES = $(shell find . -type f -name '*.go' 2>/dev/null) GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*" 2>/dev/null) GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go' 2>/dev/null) -GOPACKAGES_STRESSTESTS=$(find . -type d \( -name "stress" \)) +GOPACKAGES_STRESSTESTS=$(shell find . -type d \( -name "stress" \)) SHELL=bash ES_HOST?=elasticsearch ES_PORT?=9200 From 4af82332e337987b5d47a83218ba413b80c4ed93 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 14:11:51 +0200 Subject: [PATCH 28/52] debug stress tests --- .buildkite/scripts/stress_tests.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh index ed1b4e4868b5..cb39e92c5cf6 100755 --- a/.buildkite/scripts/stress_tests.sh +++ b/.buildkite/scripts/stress_tests.sh @@ -6,5 +6,12 @@ set -euo pipefail beats_subfolder=$1 +sudo chmod -R go-w ${beats_subfolder}/ + echo "--- Run Stress Tests for $beats_subfolder" -make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C $beats_subfolder stress-tests +pushd "${beats_subfolder}" > /dev/null + +umask 0022 +make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' stress-tests + +popd > /dev/null From d1e5d5596454505cecaaaaab28ea9523aa27584c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 14:33:35 +0200 Subject: [PATCH 29/52] full test --- .buildkite/libbeat/pipeline.libbeat.yml | 48 ++++++++++++------------- .buildkite/scripts/crosscompile.sh | 4 ++- .buildkite/scripts/go_int_tests.sh | 3 +- .buildkite/scripts/py_int_tests.sh | 4 +-- .buildkite/scripts/unit_tests.sh | 5 ++- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index ec6c92454902..edb6cb85ef32 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -19,32 +19,32 @@ steps: machineType: "c2-standard-16" artifact_paths: "libbeat/build/*.*" - # - label: ":go: Go Intergration Tests" - # key: "mandatory-int-test" - # command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":go: Go Intergration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - label: ":python: Python Integration Tests" - # key: "mandatory-python-int-test" - # command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - # - label: ":negative_squared_cross_mark: Cross compile" - # key: "mandatory-cross-compile" - # command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" - # artifact_paths: "libbeat/build/*.*" + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "libbeat/build/*.*" - label: ":testengine: Stress Tests" key: "mandatory-stress-test" diff --git a/.buildkite/scripts/crosscompile.sh b/.buildkite/scripts/crosscompile.sh index 20fa2bcf2296..49c3c4f6a215 100755 --- a/.buildkite/scripts/crosscompile.sh +++ b/.buildkite/scripts/crosscompile.sh @@ -6,10 +6,12 @@ set -euo pipefail beats_subfolder=$1 -echo "--- Run Crosscompile for $beats_subfolder" +sudo chmod -R go-w ${beats_subfolder}/ +echo "--- Run Crosscompile for $beats_subfolder" pushd "${beats_subfolder}" > /dev/null +umask 0022 make crosscompile popd > /dev/null diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh index af662cc671bd..9e9a63f23868 100755 --- a/.buildkite/scripts/go_int_tests.sh +++ b/.buildkite/scripts/go_int_tests.sh @@ -6,8 +6,9 @@ set -euo pipefail beats_subfolder=$1 -echo "--- Run Go Intergration Tests for $beats_subfolder" sudo chmod -R go-w ${beats_subfolder}/ + +echo "--- Run Go Intergration Tests for $beats_subfolder" pushd "${beats_subfolder}" > /dev/null umask 0022 diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh index 3e6f21f78bfd..00c97521d1bc 100755 --- a/.buildkite/scripts/py_int_tests.sh +++ b/.buildkite/scripts/py_int_tests.sh @@ -6,9 +6,9 @@ set -euo pipefail beats_subfolder=$1 -echo "--- Run Python Intergration Tests for $beats_subfolder" - sudo chmod -R go-w ${beats_subfolder}/ + +echo "--- Run Python Intergration Tests for $beats_subfolder" pushd "${beats_subfolder}" > /dev/null umask 0022 diff --git a/.buildkite/scripts/unit_tests.sh b/.buildkite/scripts/unit_tests.sh index a17342cd6c1b..fad308e30d1a 100755 --- a/.buildkite/scripts/unit_tests.sh +++ b/.buildkite/scripts/unit_tests.sh @@ -6,9 +6,12 @@ set -euo pipefail beats_subfolder=$1 +sudo chmod -R go-w ${beats_subfolder}/ + echo "--- Run Unit Tests" -sudo chmod -R go-w "${beats_subfolder}/" pushd "${beats_subfolder}" > /dev/null + umask 0022 mage build unitTest + popd > /dev/null From 87ce234f462e2ae3400eb2076dec99fb192ecbec Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 15:02:17 +0200 Subject: [PATCH 30/52] full test --- libbeat/scripts/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 2ab7331de168..6155b664222c 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -46,7 +46,7 @@ export PATH := ./bin:$(PATH) GOFILES = $(shell find . -type f -name '*.go' 2>/dev/null) GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*" 2>/dev/null) GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go' 2>/dev/null) -GOPACKAGES_STRESSTESTS=$(shell find . -type d \( -name "stress" \)) +GOPACKAGES_STRESSTESTS=$(shell find . -type d \( -name "stress" \) 2>/dev/null) SHELL=bash ES_HOST?=elasticsearch ES_PORT?=9200 @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/arm64 !darwin/386 !linux/386 !linux/mipsle !linux/mips64le !linux/ppc64 !linux/ppc64le !linux/mips64 !linux/mips !linux/s390x !windows/386 !freebsd/386 !freebsd/arm !netbsd/386 !netbsd/arm !openbsd/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 From d684936439b54982f605040a0b5b2de7c3086d10 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 15:30:54 +0200 Subject: [PATCH 31/52] full test --- .buildkite/libbeat/pipeline.libbeat.yml | 2 +- .buildkite/scripts/stress_tests.sh | 2 +- libbeat/scripts/Makefile | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index edb6cb85ef32..1f193e206b1a 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -53,7 +53,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + artifact_paths: "libbeat/*.*" - group: "Extended Tests" key: "extended-tests" diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh index cb39e92c5cf6..8800278ea2c5 100755 --- a/.buildkite/scripts/stress_tests.sh +++ b/.buildkite/scripts/stress_tests.sh @@ -12,6 +12,6 @@ echo "--- Run Stress Tests for $beats_subfolder" pushd "${beats_subfolder}" > /dev/null umask 0022 -make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' stress-tests +make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests popd > /dev/null diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 6155b664222c..52aab111d45a 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -87,8 +87,9 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". +GOTEST_OUTPUT_OPTIONS?= # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 TESTING_ENVIRONMENT?=snapshot## @testing The name of the environment under test @@ -255,7 +256,7 @@ fast-system-tests: ${BEAT_NAME}.test python-env stress-tests: ## @testing Runs the stress tests with race detector enabled stress-tests: if [ -n '${GOPACKAGES_STRESSTESTS}' ]; then \ - go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS}; \ + go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS} ${GOTEST_OUTPUT_OPTIONS}; \ fi # Run benchmark tests From da7b08adc38a515526f5f51a84ed58d1627a34ba Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 26 Jan 2024 16:09:14 +0200 Subject: [PATCH 32/52] full test - changed crosscompile options --- .buildkite/libbeat/pipeline.libbeat.yml | 2 +- libbeat/scripts/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 1f193e206b1a..66eb0be3aca6 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -53,7 +53,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "libbeat/*.*" + artifact_paths: "libbeat/libbeat-stress-test.xml" - group: "Extended Tests" key: "extended-tests" diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 52aab111d45a..cb52b027df01 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -87,7 +87,7 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 !linux/ppc64 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". GOTEST_OUTPUT_OPTIONS?= # XXX: Should be switched back to `snapshot` once the Elasticsearch From 0cf08cd8e868be8de4a8dbc7636ccdc12f808e01 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 29 Jan 2024 08:26:29 +0200 Subject: [PATCH 33/52] updates artifacts paths --- .buildkite/libbeat/pipeline.libbeat.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 66eb0be3aca6..c6edd53153fe 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -17,7 +17,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + artifact_paths: "libbeat/build/*.xml" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" @@ -26,7 +26,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + artifact_paths: "libbeat/build/*.xml" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" @@ -35,7 +35,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + artifact_paths: "libbeat/build/*.xml" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" @@ -44,7 +44,7 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.*" + artifact_paths: "libbeat/build/*.xml" - label: ":testengine: Stress Tests" key: "mandatory-stress-test" @@ -65,4 +65,4 @@ steps: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.xlarge" - artifact_paths: "libbeat/build/*.*" + artifact_paths: "libbeat/build/*.xml" From 9aa7c6c6a941a7a8b7d7dbe119dbf4547ca6bda1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 3 Feb 2024 10:41:51 +0200 Subject: [PATCH 34/52] conflict solving --- .buildkite/hooks/post-checkout | 2 +- .buildkite/scripts/install_tools.sh | 2 ++ .buildkite/scripts/stress_tests.sh | 7 ++----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index e10f15de7b65..b6cc7ad60bda 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 796892341d30..1a1e3a29f54e 100644 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -36,6 +36,8 @@ with_go "${GO_VERSION}" with_mage with_python with_dependencies +config_git +mage dumpVariables #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #TODO: Remove when the issue is solved https://github.com/elastic/beats/issues/37838 diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh index 8800278ea2c5..cb64dc79290a 100755 --- a/.buildkite/scripts/stress_tests.sh +++ b/.buildkite/scripts/stress_tests.sh @@ -1,17 +1,14 @@ -#!/bin/bash +#!/usr/bin/env bash -source .buildkite/scripts/common.sh +source .buildkite/scripts/install_tools.sh set -euo pipefail beats_subfolder=$1 -sudo chmod -R go-w ${beats_subfolder}/ - echo "--- Run Stress Tests for $beats_subfolder" pushd "${beats_subfolder}" > /dev/null -umask 0022 make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests popd > /dev/null From 411d9d46888967a3f92925a9fdb79054cc7a7083 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 3 Feb 2024 10:47:28 +0200 Subject: [PATCH 35/52] change pre-command --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/setenv.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ef38478a4327..d9475f85b997 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -16,7 +16,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index b05bb27fba93..901ba9891c20 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env bash set -euo pipefail From 708000dba2528acbe824c888041ce155f298a0e1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 3 Feb 2024 10:54:36 +0200 Subject: [PATCH 36/52] add BEATS_PROJECT_NAME --- .buildkite/libbeat/pipeline.libbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index c6edd53153fe..f0a65097e354 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -3,7 +3,7 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - PIPELINE_NAME: "libbeat" + BEATS_PROJECT_NAME: "libbeat" steps: From af0b9622b686d33baa76d217549c1ab54693fdde Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 08:42:55 +0200 Subject: [PATCH 37/52] refactor libbeat --- .buildkite/libbeat/pipeline.libbeat.yml | 91 +++++++------------ .buildkite/scripts/common.sh | 57 +++++++----- .../scripts/generate_libbeat_pipeline.sh | 91 +++++++++++++++++++ .buildkite/scripts/stress_tests.sh | 5 +- 4 files changed, 158 insertions(+), 86 deletions(-) create mode 100755 .buildkite/scripts/generate_libbeat_pipeline.sh diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index f0a65097e354..4efb099df682 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -7,62 +7,35 @@ env: steps: - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.xml" - - - label: ":go: Go Intergration Tests" - key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.xml" - - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.xml" - - - label: ":negative_squared_cross_mark: Cross compile" - key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/build/*.xml" - - - label: ":testengine: Stress Tests" - key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: "libbeat/libbeat-stress-test.xml" - - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":linux: Arm64 Unit Tests" - key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - artifact_paths: "libbeat/build/*.xml" + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Libbeat - runLibbeat" + key: "runLibbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Libbeat - runLibBeatArmTest" + key: "runLibBeatArmTest" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + + - label: ":linux: Load dynamic Libbeat pipeline" + key: "libbeat-pipeline" + command: ".buildkite/scripts/generate_libbeat_pipeline.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 03c05de926d5..1b75cea62362 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -10,6 +10,8 @@ GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})" runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})" +runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" metricbeat_changeset=( "^metricbeat/.*" "^go.mod" @@ -175,65 +177,72 @@ are_changed_only_paths() { local changed_files=$(git diff --name-only HEAD@{1} HEAD) if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then return 0 - else - return 1 fi + return 1 } are_conditions_met_mandatory_tests() { if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - else - return 1 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then + return 0 + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then + return 0 + fi + fi + fi fi - else - return 1 fi + return 1 } are_conditions_met_extended_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 return 0 - else - return 1 fi + return 1 +} + +are_conditions_met_arm_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then + return 0 + fi + fi + fi + return 1 } are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - else - return 1 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi fi - else - return 1 fi + return 1 } are_conditions_met_extended_windows_tests() { if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 - else - return 1 fi - else - return 1 fi + return 1 } are_conditions_met_packaging() { if are_conditions_met_extended_windows_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 return 0 - else - return 1 fi - else - return 1 fi + return 1 } config_git() { diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh new file mode 100755 index 000000000000..4aecc0f046a9 --- /dev/null +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.libbeat-dynamic.yml" + +cat > $pipelineName <<- YAML + +steps: + +YAML + +if are_conditions_met_mandatory_tests; then + cat >> $pipelineName <<- YAML + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Intergration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":negative_squared_cross_mark: Cross compile" + key: "mandatory-cross-compile" + command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":testengine: Stress Tests" + key: "mandatory-stress-test" + command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" + +YAML +fi + +if are_conditions_met_extended_tests && are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/stress_tests.sh b/.buildkite/scripts/stress_tests.sh index cb64dc79290a..b177eb53ea6b 100755 --- a/.buildkite/scripts/stress_tests.sh +++ b/.buildkite/scripts/stress_tests.sh @@ -4,10 +4,9 @@ source .buildkite/scripts/install_tools.sh set -euo pipefail -beats_subfolder=$1 +echo "--- Run Stress Tests for $BEATS_PROJECT_NAME" -echo "--- Run Stress Tests for $beats_subfolder" -pushd "${beats_subfolder}" > /dev/null +pushd "${BEATS_PROJECT_NAME}" > /dev/null make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests From a56273377e58bfcf828f52205d4ae4dbbc7e77f3 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 10:04:53 +0200 Subject: [PATCH 38/52] refactor libbeat/metricbeat conditions --- .buildkite/scripts/common.sh | 71 ++++++++++--------- .../scripts/generate_libbeat_pipeline.sh | 10 +-- .../scripts/generate_metricbeat_pipeline.sh | 20 +++--- 3 files changed, 48 insertions(+), 53 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 1b75cea62362..027856ac523a 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -181,31 +181,32 @@ are_changed_only_paths() { return 1 } -are_conditions_met_mandatory_tests() { - if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then - return 0 - elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then - return 0 - fi - fi - fi - fi - fi - return 1 -} -are_conditions_met_extended_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - return 0 - fi - return 1 -} +# are_conditions_met_mandatory_tests() { +# if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 +# if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 +# if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then +# if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then +# return 0 +# elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then +# if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then +# return 0 +# fi +# fi +# fi +# fi +# fi +# return 1 +# } -are_conditions_met_arm_tests() { +# are_conditions_met_extended_tests() { +# if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 +# return 0 +# fi +# return 1 +# } + +are_conditions_met_libbeat_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then @@ -216,18 +217,18 @@ are_conditions_met_arm_tests() { return 1 } -are_conditions_met_macos_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then - if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi +are_conditions_met_metricbeat_macos_tests() { + # if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + # if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 fi return 1 + # fi + # return 1 } -are_conditions_met_extended_windows_tests() { +are_conditions_met_metricbeat_extended_windows_tests() { if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 @@ -237,11 +238,11 @@ are_conditions_met_extended_windows_tests() { } are_conditions_met_packaging() { - if are_conditions_met_extended_windows_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 - return 0 - fi + # if are_conditions_met_metricbeat_extended_windows_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 + return 0 fi + # fi return 1 } diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 4aecc0f046a9..bea9f10bf14c 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -6,15 +6,11 @@ set -euo pipefail pipelineName="pipeline.libbeat-dynamic.yml" +echo "Adding the mandatory tests into the pipeline" cat > $pipelineName <<- YAML steps: -YAML - -if are_conditions_met_mandatory_tests; then - cat >> $pipelineName <<- YAML - - group: "Mandatory Tests" key: "mandatory-tests" steps: @@ -64,9 +60,9 @@ if are_conditions_met_mandatory_tests; then artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML -fi -if are_conditions_met_extended_tests && are_conditions_met_arm_tests; then +echo "Adding the additional tests into the pipeline" +if are_conditions_met_libbeat_arm_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index a15447ba4bf6..5e479a918fb7 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -6,15 +6,11 @@ set -euo pipefail pipelineName="pipeline.metricbeat-dynamic.yml" +echo "Adding the mandatory tests into the pipeline" cat > $pipelineName <<- YAML steps: -YAML - -if are_conditions_met_mandatory_tests; then - cat >> $pipelineName <<- YAML - - group: "Mandatory Tests" key: "mandatory-tests" steps: @@ -71,9 +67,8 @@ if are_conditions_met_mandatory_tests; then artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML -fi -if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then +if are_conditions_met_metricbeat_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -90,8 +85,11 @@ if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then YAML fi -if are_conditions_met_extended_windows_tests; then - cat >> $pipelineName <<- YAML +echo "Adding the additional tests into the pipeline" +#TODO: ADD conditions from the main pipeline + +# if are_conditions_met_extended_windows_tests; then +cat >> $pipelineName <<- YAML - group: "Extended Windowds Tests" key: "extended-win-tests" @@ -131,9 +129,9 @@ if are_conditions_met_extended_windows_tests; then artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML -fi +# fi -if are_conditions_met_extended_windows_tests; then +if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 From b4efe613d9d39bb48b68ed104cac1fb807e36c75 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 10:11:36 +0200 Subject: [PATCH 39/52] fix dinamyc libbeat pipeline --- .buildkite/scripts/generate_libbeat_pipeline.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index bea9f10bf14c..127e602a07b2 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -16,7 +16,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + command: ".buildkite/scripts/unit_tests.sh" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -25,7 +25,7 @@ steps: - label: ":go: Go Intergration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh ${PIPELINE_NAME}" + command: ".buildkite/scripts/go_int_tests.sh" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -34,7 +34,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh ${PIPELINE_NAME}" + command: ".buildkite/scripts/py_int_tests.sh" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -43,7 +43,7 @@ steps: - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh ${PIPELINE_NAME}" + command: ".buildkite/scripts/crosscompile.sh" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -52,7 +52,7 @@ steps: - label: ":testengine: Stress Tests" key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh ${PIPELINE_NAME}" + command: ".buildkite/scripts/stress_tests.sh" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -70,7 +70,7 @@ if are_conditions_met_libbeat_arm_tests; then steps: - label: ":linux: Arm64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh ${PIPELINE_NAME}" + command: ".buildkite/scripts/unit_tests.sh" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" From 6cd9880b7c8916f7091c55f7b31cda26babf9153 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 11:55:55 +0200 Subject: [PATCH 40/52] change metricbeat dunamyc pipeline --- .buildkite/libbeat/pipeline.libbeat.yml | 1 - .buildkite/scripts/common.sh | 1 - .../scripts/generate_metricbeat_pipeline.sh | 51 ++++++++----------- 3 files changed, 22 insertions(+), 31 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 4efb099df682..eefe9e07446d 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -32,7 +32,6 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic Libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_libbeat_pipeline.sh" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 027856ac523a..7f66a2955741 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -181,7 +181,6 @@ are_changed_only_paths() { return 1 } - # are_conditions_met_mandatory_tests() { # if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 # if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 5e479a918fb7..4a09c546e118 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -68,27 +68,8 @@ steps: YAML -if are_conditions_met_metricbeat_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -echo "Adding the additional tests into the pipeline" +echo "Adding the extended windows tests into the pipeline" #TODO: ADD conditions from the main pipeline - -# if are_conditions_met_extended_windows_tests; then cat >> $pipelineName <<- YAML - group: "Extended Windowds Tests" @@ -127,13 +108,33 @@ cat >> $pipelineName <<- YAML disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +YAML +if are_conditions_met_metricbeat_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML -# fi + +fi if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 key: "packaging" steps: @@ -158,14 +159,6 @@ if are_conditions_met_packaging; then PLATFORMS: "linux/arm64" PACKAGES: "docker" - depends_on: - - step: "mandatory-tests" - allow_failure: false - - step: "extended-tests" - allow_failure: true - - step: "extended-win-tests" - allow_failure: true - YAML fi From a2c8826586b4d32a71d5c2bf34d8c1563e87cb2a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 12:52:45 +0200 Subject: [PATCH 41/52] full test --- .buildkite/metricbeat/pipeline.yml | 6 +-- .buildkite/scripts/common.sh | 52 +++++-------------- .../scripts/generate_libbeat_pipeline.sh | 8 +-- .../scripts/generate_metricbeat_pipeline.sh | 15 +++--- 4 files changed, 30 insertions(+), 51 deletions(-) diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 0abc58a85ae5..99134bf83921 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -14,10 +14,10 @@ env: steps: - input: "Input Parameters" - key: "input-run-all-stages" + key: "runMetricbeat" fields: - - select: "Metricbeat - runAllStages" - key: "runAllStages" + - select: "Metricbeat - runMetricbeat" + key: "runMetricbeat" options: - label: "True" value: "true" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7f66a2955741..f0d055acbf8f 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -181,29 +181,20 @@ are_changed_only_paths() { return 1 } -# are_conditions_met_mandatory_tests() { -# if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 -# if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 -# if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then -# if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then -# return 0 -# elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then -# if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then -# return 0 -# fi -# fi -# fi -# fi -# fi -# return 1 -# } - -# are_conditions_met_extended_tests() { -# if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 -# return 0 -# fi -# return 1 -# } +are_conditions_met_mandatory_tests() { + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then + return 0 + fi + fi + fi + return 1 +} are_conditions_met_libbeat_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 @@ -217,31 +208,16 @@ are_conditions_met_libbeat_arm_tests() { } are_conditions_met_metricbeat_macos_tests() { - # if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - # if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi - return 1 - # fi - # return 1 -} - -are_conditions_met_metricbeat_extended_windows_tests() { - if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - fi return 1 } are_conditions_met_packaging() { - # if are_conditions_met_metricbeat_extended_windows_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 return 0 fi - # fi return 1 } diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 127e602a07b2..e5e6baefaca3 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -6,8 +6,9 @@ set -euo pipefail pipelineName="pipeline.libbeat-dynamic.yml" -echo "Adding the mandatory tests into the pipeline" -cat > $pipelineName <<- YAML +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML steps: @@ -60,8 +61,9 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML +fi -echo "Adding the additional tests into the pipeline" +echo "Check and add the Extended Tests into the pipeline" if are_conditions_met_libbeat_arm_tests; then cat >> $pipelineName <<- YAML diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 4a09c546e118..9c2767edea6e 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -6,8 +6,9 @@ set -euo pipefail pipelineName="pipeline.metricbeat-dynamic.yml" -echo "Adding the mandatory tests into the pipeline" -cat > $pipelineName <<- YAML +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML steps: @@ -66,11 +67,8 @@ steps: - "${IMAGE_WIN_2022}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML - -echo "Adding the extended windows tests into the pipeline" -#TODO: ADD conditions from the main pipeline -cat >> $pipelineName <<- YAML +# echo "Add the extended windows tests into the pipeline" +# TODO: ADD conditions from the main pipeline - group: "Extended Windowds Tests" key: "extended-win-tests" @@ -109,7 +107,9 @@ cat >> $pipelineName <<- YAML disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML +fi +echo "Check and add the Extended Tests into the pipeline" if are_conditions_met_metricbeat_macos_tests; then cat >> $pipelineName <<- YAML @@ -127,6 +127,7 @@ YAML fi +echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML From e03d2b3e598cb7681790278be58684020e588e0a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 13:04:56 +0200 Subject: [PATCH 42/52] full test --- .buildkite/metricbeat/pipeline.yml | 4 ++-- .buildkite/scripts/common.sh | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 99134bf83921..430693dece9d 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -24,8 +24,8 @@ steps: - label: "False" value: "false" default: "false" - - select: "Metricbeat - runMacOsTests" - key: "UI_MACOS_TESTS" + - select: "Metricbeat - runMetricbeatMacOsTests" + key: "runMetricbeatMacOsTests" options: - label: "True" value: "true" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index f0d055acbf8f..e3dd2ec4ac41 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -8,10 +8,11 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})" -runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})" runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" + metricbeat_changeset=( "^metricbeat/.*" "^go.mod" @@ -208,7 +209,7 @@ are_conditions_met_libbeat_arm_tests() { } are_conditions_met_metricbeat_macos_tests() { - if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi return 1 From 823a27c180f58cb8a0d75b29e6913f84daaea263 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 13:16:06 +0200 Subject: [PATCH 43/52] full test --- .buildkite/scripts/generate_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 9c2767edea6e..c4bf99947880 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -128,7 +128,7 @@ YAML fi echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then +if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - wait: ~ From b1155e8b3e0a30e2257b07193e4876b4d7dbf706 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 15:10:25 +0200 Subject: [PATCH 44/52] update catalog-info.yaml --- catalog-info.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index f3dd3094788e..f1a04651fc7a 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -99,9 +99,8 @@ spec: cancel_intermediate_builds_branch_filter: '!main !7.* !8.*' skip_intermediate_builds: true skip_intermediate_builds_branch_filter: '!main !7.* !8.*' - # TODO uncomment this environment variable when pipeline definition is updated - # env: - # ELASTIC_PR_COMMENTS_ENABLED: 'true' + env: + ELASTIC_PR_COMMENTS_ENABLED: 'true' teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -310,9 +309,9 @@ spec: name: beats-libbeat description: "Beats libbeat pipeline" spec: -# branch_configuration: "main 7.17 8.* v7.17 v8.*" TODO: temporarily commented to build PRs from forks + branch_configuration: "main 7.* 8.* v7.* v8.*" pipeline_file: ".buildkite/libbeat/pipeline.libbeat.yml" -# maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + maximum_timeout_in_minutes: 120 provider_settings: build_pull_request_forks: false build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot @@ -322,11 +321,11 @@ spec: build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) repository: elastic/beats cancel_intermediate_builds: true - cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" + cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" - # env: - # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + env: + ELASTIC_PR_COMMENTS_ENABLED: "true" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ @@ -413,9 +412,9 @@ spec: cancel_intermediate_builds: true cancel_intermediate_builds_branch_filter: "!main !7.17 !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ everyone: - access_level: READ_ONLY \ No newline at end of file + access_level: READ_ONLY From 0c4becf11b563467c308093b3792ecd69ded09f9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 15:20:38 +0200 Subject: [PATCH 45/52] change the instance type and fix typos --- .buildkite/scripts/generate_libbeat_pipeline.sh | 12 ++++++------ .buildkite/scripts/generate_metricbeat_pipeline.sh | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index e5e6baefaca3..0c0d2ae58fab 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -21,16 +21,16 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - label: ":go: Go Intergration Tests" + - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -39,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":negative_squared_cross_mark: Cross compile" @@ -48,7 +48,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" - label: ":testengine: Stress Tests" @@ -57,7 +57,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index c4bf99947880..eab25e36d51a 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -21,7 +21,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":go: Go Intergration Tests" @@ -30,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":python: Python Integration Tests" @@ -39,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" @@ -48,7 +48,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" @@ -70,7 +70,7 @@ steps: # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline - - group: "Extended Windowds Tests" + - group: "Extended Windows Tests" key: "extended-win-tests" steps: - label: ":windows: Windows 2019 Unit Tests" @@ -145,7 +145,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # machineType: "c2-standard-16" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" From ca6c24192875a5394cf8cb8a2177f5f76b1075bf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 17:20:21 +0200 Subject: [PATCH 46/52] change instance types to c2d-highcpu-16 --- .buildkite/scripts/generate_libbeat_pipeline.sh | 8 +++----- .buildkite/scripts/generate_metricbeat_pipeline.sh | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 0c0d2ae58fab..ecf9f3a0a17d 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -21,7 +21,6 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":go: Go Integration Tests" @@ -30,7 +29,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" + machineType: "c2d-highcpu-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -39,7 +38,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" + machineType: "c2d-highcpu-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":negative_squared_cross_mark: Cross compile" @@ -48,7 +47,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" + machineType: "c2d-highcpu-16" artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" - label: ":testengine: Stress Tests" @@ -57,7 +56,6 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index eab25e36d51a..100c215182e6 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -21,7 +21,6 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":go: Go Intergration Tests" @@ -30,7 +29,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" + machineType: "c2d-highcpu-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":python: Python Integration Tests" @@ -39,7 +38,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" + machineType: "c2d-highcpu-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" @@ -48,7 +47,6 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" @@ -145,7 +143,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2-standard-16" + # machineType: "c2d-highcpu-16" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" From 3774483f8c2cbdb7d0dc18a6fdc610933022dbbc Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 18:18:07 +0200 Subject: [PATCH 47/52] change instance type for packaging --- .buildkite/scripts/generate_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 100c215182e6..361a5eeb1380 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -143,7 +143,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "c2d-highcpu-16" + machineType: "c2d-highcpu-16" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" From d5194bf852032b241576c0050fd477872424bc1a Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:54:26 +0200 Subject: [PATCH 48/52] Update catalog-info.yaml Co-authored-by: Victor Martinez --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index f1a04651fc7a..8feed765238a 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -309,7 +309,7 @@ spec: name: beats-libbeat description: "Beats libbeat pipeline" spec: - branch_configuration: "main 7.* 8.* v7.* v8.*" + branch_configuration: "main 7.17 8.* pipeline_file: ".buildkite/libbeat/pipeline.libbeat.yml" maximum_timeout_in_minutes: 120 provider_settings: From f3317c6d744202e97dde312026cd07b0c2f31e92 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 12:00:38 +0200 Subject: [PATCH 49/52] fix remarks --- .buildkite/libbeat/pipeline.libbeat.yml | 5 +++++ .buildkite/metricbeat/pipeline.yml | 6 ++++++ .../scripts/generate_libbeat_pipeline.sh | 10 ++++++---- .../scripts/generate_metricbeat_pipeline.sh | 20 ++++++++++--------- libbeat/scripts/Makefile | 4 ++-- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index eefe9e07446d..1fb185b59d19 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -3,6 +3,10 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "libbeat" steps: @@ -38,3 +42,4 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 430693dece9d..c42f17d2a363 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -10,6 +10,11 @@ env: IMAGE_MACOS_X86_64: "generic-13-ventura-x64" GO_AGENT_IMAGE: "golang:${GO_VERSION}" BEATS_PROJECT_NAME: "metricbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + steps: @@ -44,3 +49,4 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index ecf9f3a0a17d..6da1bef711dc 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -21,6 +21,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":go: Go Integration Tests" @@ -29,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MASHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -38,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MASHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":negative_squared_cross_mark: Cross compile" @@ -47,7 +48,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MASHINE_TYPE}" artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" - label: ":testengine: Stress Tests" @@ -56,6 +57,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML @@ -74,7 +76,7 @@ if are_conditions_met_libbeat_arm_tests; then agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" YAML diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 361a5eeb1380..1188871954d5 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -21,6 +21,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":go: Go Intergration Tests" @@ -29,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MASHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":python: Python Integration Tests" @@ -38,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MASHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" @@ -47,6 +48,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" @@ -55,7 +57,7 @@ steps: agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" matrix: @@ -77,19 +79,19 @@ steps: agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # Temporary disabled https://github.com/elastic/beats/issues/37841 + # # Temporary disabled https://github.com/elastic/beats/issues/37841 # - label: ":windows: Windows 10 Unit Tests" # key: "extended-win-10-unit-tests" # command: ".buildkite/scripts/win_unit_tests.ps1" # agents: # provider: "gcp" # image: "${IMAGE_WIN_10}" - # machine_type: "n2-standard-8" + # machine_type: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" @@ -100,7 +102,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_WIN_11}" - machine_type: "n2-standard-8" + machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" @@ -143,7 +145,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MASHINE_TYPE}" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -153,7 +155,7 @@ if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index cb52b027df01..7f83db510141 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -87,9 +87,9 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 !linux/ppc64 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". +GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 !linux/ppc64 ## @building Space-separated list of GOOS/GOARCH pairs to exclude (unsupported by GO and generated by GOX) in the "make crosscompile" build. GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". -GOTEST_OUTPUT_OPTIONS?= +GOTEST_OUTPUT_OPTIONS?= # The variable could be used for getting the output from the "go-test..." command, example: GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 TESTING_ENVIRONMENT?=snapshot## @testing The name of the environment under test From f151f981c4940e3c004e15725edf25bd3c4e1d19 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 12:30:17 +0200 Subject: [PATCH 50/52] revert makifile changes --- libbeat/scripts/Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 7f83db510141..100ccd3f0137 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -46,7 +46,7 @@ export PATH := ./bin:$(PATH) GOFILES = $(shell find . -type f -name '*.go' 2>/dev/null) GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*" 2>/dev/null) GOFILES_ALL = $(GOFILES) $(shell find $(ES_BEATS) -type f -name '*.go' 2>/dev/null) -GOPACKAGES_STRESSTESTS=$(shell find . -type d \( -name "stress" \) 2>/dev/null) +GOPACKAGES_STRESSTESTS=$(shell find . -name '*.go' 2>/dev/null | xargs grep -l '\+build.*stresstest' | xargs -n1 dirname | uniq) SHELL=bash ES_HOST?=elasticsearch ES_PORT?=9200 @@ -87,9 +87,8 @@ SYSTEM_TESTS?=false ## @testing if true, "make test" and "make testsuite" run un STRESS_TESTS?=false ## @testing if true, "make test" and "make testsuite" run also run the stress tests STRESS_TEST_OPTIONS?=-timeout=20m -race -v GOX_OS?=linux darwin windows freebsd netbsd openbsd ## @Building List of all OS to be supported by "make crosscompile". -GOX_OSARCH?=!darwin/arm !darwin/386 !linux/386 !windows/386 !freebsd/386 !netbsd/386 !openbsd/386 !linux/ppc64 ## @building Space-separated list of GOOS/GOARCH pairs to exclude (unsupported by GO and generated by GOX) in the "make crosscompile" build. +GOX_OSARCH?=!darwin/arm !darwin/arm64 ## @building Space separated list of GOOS/GOARCH pairs to build by "make crosscompile". GOX_FLAGS?= ## @building Additional flags to append to the gox command used by "make crosscompile". -GOTEST_OUTPUT_OPTIONS?= # The variable could be used for getting the output from the "go-test..." command, example: GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' # XXX: Should be switched back to `snapshot` once the Elasticsearch # snapshots are working. https://github.com/elastic/beats/pull/6416 TESTING_ENVIRONMENT?=snapshot## @testing The name of the environment under test @@ -256,7 +255,7 @@ fast-system-tests: ${BEAT_NAME}.test python-env stress-tests: ## @testing Runs the stress tests with race detector enabled stress-tests: if [ -n '${GOPACKAGES_STRESSTESTS}' ]; then \ - go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS} ${GOTEST_OUTPUT_OPTIONS}; \ + go test --tags=stresstest ${STRESS_TEST_OPTIONS} ${GOPACKAGES_STRESSTESTS}; \ fi # Run benchmark tests From 6ee43ff93672bf5ec66d07ecccb04278c6a15c87 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 6 Feb 2024 12:32:03 +0200 Subject: [PATCH 51/52] Update catalog-info.yaml Co-authored-by: Victor Martinez --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 8feed765238a..6904b0ffbd02 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -323,7 +323,7 @@ spec: cancel_intermediate_builds: true cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" skip_intermediate_builds: true - skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds_branch_filter: "!main !7.17 !8.*" env: ELASTIC_PR_COMMENTS_ENABLED: "true" teams: From e2a2afb8313ef07274207ec2b8669cb0b09fe97f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 14:50:29 +0200 Subject: [PATCH 52/52] windows 10 test --- .../scripts/generate_metricbeat_pipeline.sh | 22 +++++++++---------- metricbeat/tests/system/test_reload.py | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 1188871954d5..0ea19734c4fd 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -84,17 +84,17 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # # Temporary disabled https://github.com/elastic/beats/issues/37841 - # - label: ":windows: Windows 10 Unit Tests" - # key: "extended-win-10-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machine_type: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # Temporary disabled https://github.com/elastic/beats/issues/37841 + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index 29d82bbf82b2..aa0b8e8a5373 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -42,7 +42,7 @@ def test_reload(self): self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") def test_start_stop(self): """ Test if module is properly started and stopped