From 10b9311e44ade72f365d9f33fb9b49bcae270862 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 17 Jan 2024 15:48:22 +0200 Subject: [PATCH] added dynamic step for packaging --- .buildkite/env-scripts/linux-env.sh | 2 +- .buildkite/env-scripts/macos-env.sh | 5 +- .../scripts/common.sh => env-scripts/util.sh} | 14 ----- .buildkite/filebeat/filebeat-pipeline.yml | 60 +++++-------------- .../filebeat/scripts/integration-gotests.sh | 8 --- .buildkite/filebeat/scripts/package-step.sh | 14 ----- .buildkite/filebeat/scripts/package.sh | 23 ------- .buildkite/filebeat/scripts/unit-tests.sh | 4 -- .buildkite/hooks/pre-command | 8 +-- 9 files changed, 21 insertions(+), 117 deletions(-) rename .buildkite/{filebeat/scripts/common.sh => env-scripts/util.sh} (88%) mode change 100644 => 100755 .buildkite/filebeat/scripts/package-step.sh diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh index 1453807a073d..b4de60bc2912 100644 --- a/.buildkite/env-scripts/linux-env.sh +++ b/.buildkite/env-scripts/linux-env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source .buildkite/filebeat/scripts/common.sh +source .buildkite/env-scripts/util.sh DEBIAN_FRONTEND="noninteractive" diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh index c247c5a03a73..ac1486b64fdd 100644 --- a/.buildkite/env-scripts/macos-env.sh +++ b/.buildkite/env-scripts/macos-env.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then +if [[ $PLATFORM_TYPE == Darwin* ]]; then echo ":: Setting larger ulimit on MacOS ::" # To bypass file descriptor errors like "Too many open files error" on MacOS - ulimit -Sn 10000 + ulimit -Sn 50000 + echo ":: ULIMIT :: $(ulimit -n)" fi diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/env-scripts/util.sh similarity index 88% rename from .buildkite/filebeat/scripts/common.sh rename to .buildkite/env-scripts/util.sh index efbb81e66aab..157a5aff37af 100644 --- a/.buildkite/filebeat/scripts/common.sh +++ b/.buildkite/env-scripts/util.sh @@ -2,11 +2,6 @@ set -euo pipefail -#WORKSPACE="$(pwd)" -#BIN="${WORKSPACE}/bin" -#HW_TYPE="$(uname -m)" -#PLATFORM_TYPE="$(uname)" - add_bin_path() { echo "Adding PATH to the environment variables..." create_bin @@ -94,12 +89,3 @@ are_files_changed() { return 1; fi } - -#prepare_win() { -## local os -## os="$(uname)" -# if [[ ${PLATFORM_TYPE} = MINGW* ]]; then -# choco install mingw -y -# choco install python --version=3.11.0 -y -# fi -#} diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 800ae48eb1a9..d3ca097f1bb0 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,13 +1,11 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - SETUP_GVM_VERSION: "v0.5.1" IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" - DEBIAN_FRONTEND: "noninteractive" steps: - group: "Filebeat Mandatory Testing" @@ -20,7 +18,7 @@ steps: - ".buildkite/filebeat/scripts/unit-tests.sh" notify: - github_commit_status: - context: "Filebeat/Mandatory: Unit Tests" + context: "Filebeat: Unit Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -33,7 +31,7 @@ steps: - ".buildkite/filebeat/scripts/integration-gotests.sh" notify: - github_commit_status: - context: "Filebeat/Mandatory: Integration Tests" + context: "Filebeat: Integration Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -46,7 +44,7 @@ steps: - ".buildkite/filebeat/scripts/integration-pytests.sh" notify: - github_commit_status: - context: "Filebeat/Mandatory: Python Integration Tests" + context: "Filebeat: Python Integration Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -58,7 +56,7 @@ steps: command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" notify: - github_commit_status: - context: "Filebeat/Mandatory: {{matrix.image}} Unit Tests" + context: "Filebeat: Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" @@ -81,7 +79,7 @@ steps: steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" -# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: - ".buildkite/filebeat/scripts/unit-tests.sh" notify: @@ -95,7 +93,7 @@ steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" -# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: - ".buildkite/filebeat/scripts/unit-tests.sh" notify: @@ -108,7 +106,7 @@ steps: - group: "Windows Extended Testing" key: "extended-tests-win" -# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ steps: - label: ":windows: Win 2019 Unit Tests" @@ -127,40 +125,14 @@ steps: - "filebeat/build/*.xml" - "filebeat/build/*.json" + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + - "extended-tests" + - "extended-tests-win" + + steps: - label: Package pipeline commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload" - -# - group: "Packaging" -# key: "packaging" -# if: build.env("BUILDKITE_PULL_REQUEST") != "false" -# depends_on: -# - "mandatory-tests" -# - "extended-tests" -# - "extended-tests-win" -# -# steps: -# - label: ":ubuntu: Packaging Linux X86" -# key: "package-linux-x86" -# command: -# - ".buildkite/filebeat/scripts/package.sh" -# notify: -# - github_commit_status: -# context: "Filebeat/Packaging: Linux X86" -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# -# - label: ":linux: Packaging Linux ARM" -# key: "package-linux-arm" -# env: -# PLATFORMS: "linux/arm64" -# PACKAGES: "docker" -# command: -# - ".buildkite/filebeat/scripts/package.sh" -# notify: -# - github_commit_status: -# context: "Filebeat/Packaging: ARM" -# agents: -# provider: "aws" -# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" -# instanceType: "t4g.large" diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index 2a3d19e5ead8..a3eabf70c0d3 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -2,15 +2,7 @@ set -euo pipefail -source .buildkite/filebeat/scripts/common.sh source .buildkite/env-scripts/linux-env.sh -# -## ToDo - remove after Beats agent is created" -#echo ":: Setup Env ::" -#add_bin_path -#with_go -#with_mage -## ToDo - end echo ":: Execute Integration Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh old mode 100644 new mode 100755 index d53c3d7adebb..ef51af293ef8 --- a/.buildkite/filebeat/scripts/package-step.sh +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -2,8 +2,6 @@ set -euo pipefail -source .buildkite/filebeat/scripts/common.sh - changeset="^filebeat/ ^go.mod ^pytest.ini @@ -14,20 +12,10 @@ changeset="^filebeat/ if are_files_changed "$changeset"; then cat <<-EOF - - group: "Packaging" - key: "packaging" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" - - "extended-tests" - - "extended-tests-win" - steps: - label: ":ubuntu: Packaging Linux X86" key: "package-linux-x86" command: - - "sudo apt-get update" - - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/package.sh" notify: - github_commit_status: @@ -42,8 +30,6 @@ if are_files_changed "$changeset"; then PLATFORMS: "linux/arm64" PACKAGES: "docker" command: - - "sudo apt-get update" - - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/package.sh" notify: - github_commit_status: diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index f257f0645632..2ae226eb739c 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -6,29 +6,6 @@ source .buildkite/env-scripts/linux-env.sh echo ":: Evaluate Filebeat Changes ::" -changeset="^filebeat/ - ^go.mod - ^pytest.ini - ^dev-tools/ - ^libbeat/ - ^testing/ - ^\.buildkite/filebeat/" - -#if ! are_files_changed "$changeset" ; then -# message="No files changed within Filebeat changeset" -# echo "$message" -# buildkite-agent annotate "$message" --style 'warning' --context 'ctx-warn' -# # This should return any error but skip the packaging. -# exit 0 -#fi - -## ToDo - remove after Beats agent is created" -#echo ":: Setup Env ::" -#add_bin_path -#with_go -#with_mage -## ToDo - end - echo ":: Start Packaging ::" cd filebeat umask 0022 diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index 583a9a30231a..2987f0930c8e 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -5,10 +5,6 @@ set -euo pipefail source .buildkite/env-scripts/linux-env.sh source .buildkite/env-scripts/macos-env.sh -if [[ $PLATFORM_TYPE == Darwin* ]]; then - echo ":: ULIMIT :: $(ulimit -n)" -fi - echo ":: Execute Unit Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 4338c4016fbc..0a1567e53cd5 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -3,20 +3,14 @@ set -euo pipefail source .buildkite/env-scripts/env.sh -source .buildkite/filebeat/scripts/common.sh +source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh -#SETUP_GVM_VERSION="v0.5.1" -#WORKSPACE="$(pwd)" -# -#export SETUP_GVM_VERSION -#export WORKSPACE if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ ${PLATFORM_TYPE} = MINGW* ]]; then install_python_win fi - # prepare_win if [[ -z "${GOLANG_VERSION-""}" ]]; then export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")