diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh index 4dfc01bafc34..1a8bf2c2f0b8 100644 --- a/.buildkite/env-scripts/env.sh +++ b/.buildkite/env-scripts/env.sh @@ -1,6 +1,19 @@ #!/usr/bin/env bash -SETUP_GVM_VERSION="v0.5.1" +source .buildkite/env-scripts/util.sh + +DOCS_CHANGESET="^.*\.(asciidoc|md)$ +deploy/kubernetes/.*-kubernetes.yaml" +PACKAGING_CHANGESET="^dev-tools/packaging/ +^.go-version" + +REPO="beats" +SNAPSHOT="true" + +ASDF_MAGE_VERSION="1.15.0" + +# Docker & DockerHub +DOCKER_COMPOSE_VERSION="1.21.0" WORKSPACE="$(pwd)" BIN="${WORKSPACE}/bin" HW_TYPE="$(uname -m)" @@ -9,7 +22,6 @@ REPO="beats" TMP_FOLDER="tmp.${REPO}" DOCKER_REGISTRY="docker.elastic.co" -export SETUP_GVM_VERSION export WORKSPACE export BIN export HW_TYPE @@ -17,3 +29,6 @@ export PLATFORM_TYPE export REPO export TMP_FOLDER export DOCKER_REGISTRY +export SNAPSHOT +export ASDF_MAGE_VERSION +export DOCKER_COMPOSE_VERSION diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 6a5c36bcd047..2076a0af7da7 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -89,16 +89,11 @@ are_files_changed() { fi } -cleanup() { - echo "Deleting temporary files..." - rm -rf ${BIN}/${TMP_FOLDER}.* - echo "Done." -} - -unset_secrets () { - for var in $(printenv | sed 's;=.*;;' | sort); do - if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then - unset "$var" - fi - done +changeset_applies() { + local changeset=$1 + if are_files_changed "$changeset"; then + echo true + else + echo false + fi } diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index e811d286953c..a769edf2f4a7 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -1,140 +1,23 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "filebeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" steps: - - group: "Filebeat Mandatory Testing" - key: "mandatory-tests" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" - - steps: - - label: ":ubuntu: Unit Tests" - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Go Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-gotests.sh" - notify: - - github_commit_status: - context: "Filebeat: Go Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Python Integration Tests" - command: - - ".buildkite/filebeat/scripts/integration-pytests.sh" - notify: - - github_commit_status: - context: "Filebeat: Python Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows:-{{matrix.image}} Unit Tests" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" - notify: - - github_commit_status: - context: "Filebeat: windows/Unit Tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - group: "Extended Testing" - key: "extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support" - - steps: - - label: ":linux: ARM64 Unit Tests" - key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" - artifact_paths: "filebeat/build/*.xml" - - - label: ":mac: MacOS Unit Tests" - key: "macos-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: - - ".buildkite/filebeat/scripts/unit-tests.sh" - notify: - - github_commit_status: - context: "Filebeat/Extended: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "filebeat/build/*.xml" - - - group: "Windows Extended Testing" - key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ - - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "win-extended-2019" - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" - notify: - - github_commit_status: - context: "Filebeat/Extended: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - group: "Packaging" key: "packaging" if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - "mandatory-tests" - +# depends_on: +# - "mandatory-tests" steps: - label: Package pipeline commands: ".buildkite/filebeat/scripts/package-step.sh" diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh index f8fa02db81d5..f84437064c2f 100755 --- a/.buildkite/filebeat/scripts/package-step.sh +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -4,46 +4,37 @@ set -euo pipefail source .buildkite/env-scripts/util.sh -changeset="^filebeat/ +changeset="^${BEATS_PROJECT_NAME}/ ^go.mod ^pytest.ini ^dev-tools/ ^libbeat/ ^testing/ -^\.buildkite/filebeat/" +^\.buildkite/${BEATS_PROJECT_NAME}/" if are_files_changed "$changeset"; then bk_pipeline=$(cat <<-YAML steps: - - label: ":ubuntu: Packaging Linux X86" - key: "package-linux-x86" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - command: - - ".buildkite/filebeat/scripts/package.sh" - notify: - - github_commit_status: - context: "Filebeat/Packaging: Linux X86" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - - - label: ":linux: Packaging Linux ARM" - key: "package-linux-arm" + - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" + key: "package-${BEATS_PROJECT_NAME}-linux-arm" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" - command: - - ".buildkite/filebeat/scripts/package.sh" + SNAPSHOT: true + GOX_FLAGS: "-arch arm" + command: ".buildkite/filebeat/scripts/package.sh" notify: - github_commit_status: - context: "Filebeat/Packaging: ARM" + context: "${BEATS_PROJECT_NAME}/Packaging: ARM" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" YAML ) + echo "--- Printing steps" + echo "${bk_pipeline}" + echo "--- Uploading pipeline" echo "${bk_pipeline}" | buildkite-agent pipeline upload else buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index 0bb03250348c..d6b7b68a8b76 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -2,9 +2,18 @@ set -euo pipefail -source .buildkite/env-scripts/linux-env.sh +source .buildkite/scripts/packaging/package-util.sh + +IMG_POSTFIX="-SNAPSHOT" +VARIANTS=("" "-ubi" "-oss") +VERSION="$(make get-version)" +SOURCE_TAG+="${VERSION}${IMG_POSTFIX}" +TARGET="observability-ci/${BEATS_PROJECT_NAME}" + +echo "--- Creating package" +mage -d "${BEATS_PROJECT_NAME}" package + +echo "--- Distribution list" +dir="${BEATS_PROJECT_NAME}/build/distributions" +buildkite-agent artifact upload "$dir/*.tar.gz;$dir/*.tar.gz.sha512" -echo "--- Start Packaging" -cd filebeat -umask 0022 -mage package