Skip to content

Commit

Permalink
Small reproducer for go vcs issues with filebeat/arm packaging
Browse files Browse the repository at this point in the history
  • Loading branch information
dliappis committed Mar 18, 2024
1 parent 9c9ae35 commit 0a24335
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 160 deletions.
19 changes: 17 additions & 2 deletions .buildkite/env-scripts/env.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,19 @@
#!/usr/bin/env bash

SETUP_GVM_VERSION="v0.5.1"
source .buildkite/env-scripts/util.sh

DOCS_CHANGESET="^.*\.(asciidoc|md)$
deploy/kubernetes/.*-kubernetes.yaml"
PACKAGING_CHANGESET="^dev-tools/packaging/
^.go-version"

REPO="beats"
SNAPSHOT="true"

ASDF_MAGE_VERSION="1.15.0"

# Docker & DockerHub
DOCKER_COMPOSE_VERSION="1.21.0"
WORKSPACE="$(pwd)"
BIN="${WORKSPACE}/bin"
HW_TYPE="$(uname -m)"
Expand All @@ -9,11 +22,13 @@ REPO="beats"
TMP_FOLDER="tmp.${REPO}"
DOCKER_REGISTRY="docker.elastic.co"

export SETUP_GVM_VERSION
export WORKSPACE
export BIN
export HW_TYPE
export PLATFORM_TYPE
export REPO
export TMP_FOLDER
export DOCKER_REGISTRY
export SNAPSHOT
export ASDF_MAGE_VERSION
export DOCKER_COMPOSE_VERSION
19 changes: 7 additions & 12 deletions .buildkite/env-scripts/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,11 @@ are_files_changed() {
fi
}

cleanup() {
echo "Deleting temporary files..."
rm -rf ${BIN}/${TMP_FOLDER}.*
echo "Done."
}

unset_secrets () {
for var in $(printenv | sed 's;=.*;;' | sort); do
if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
unset "$var"
fi
done
changeset_applies() {
local changeset=$1
if are_files_changed "$changeset"; then
echo true
else
echo false
fi
}
133 changes: 8 additions & 125 deletions .buildkite/filebeat/filebeat-pipeline.yml
Original file line number Diff line number Diff line change
@@ -1,140 +1,23 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json

env:
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
BEATS_PROJECT_NAME: "filebeat"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
IMAGE_WIN_2016: "family/core-windows-2016"
IMAGE_WIN_2019: "family/core-windows-2019"
IMAGE_WIN_2022: "family/core-windows-2022"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"

GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
steps:
- group: "Filebeat Mandatory Testing"
key: "mandatory-tests"
if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"

steps:
- label: ":ubuntu: Unit Tests"
command:
- ".buildkite/filebeat/scripts/unit-tests.sh"
notify:
- github_commit_status:
context: "Filebeat: linux/Unit Tests"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"

- label: ":ubuntu: Go Integration Tests"
command:
- ".buildkite/filebeat/scripts/integration-gotests.sh"
notify:
- github_commit_status:
context: "Filebeat: Go Integration Tests"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"

- label: ":ubuntu: Python Integration Tests"
command:
- ".buildkite/filebeat/scripts/integration-pytests.sh"
notify:
- github_commit_status:
context: "Filebeat: Python Integration Tests"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"

- label: ":windows:-{{matrix.image}} Unit Tests"
command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
notify:
- github_commit_status:
context: "Filebeat: windows/Unit Tests"
agents:
provider: "gcp"
image: "{{matrix.image}}"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
matrix:
setup:
image:
- "${IMAGE_WIN_2016}"
- "${IMAGE_WIN_2022}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"

- group: "Extended Testing"
key: "extended-tests"
if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for extended support"

steps:
- label: ":linux: ARM64 Unit Tests"
key: "arm-extended"
if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
command:
- ".buildkite/filebeat/scripts/unit-tests.sh"
notify:
- github_commit_status:
context: "Filebeat/Extended: Unit Tests ARM"
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.large"
artifact_paths: "filebeat/build/*.xml"

- label: ":mac: MacOS Unit Tests"
key: "macos-extended"
if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
command:
- ".buildkite/filebeat/scripts/unit-tests.sh"
notify:
- github_commit_status:
context: "Filebeat/Extended: MacOS Unit Tests"
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_X86_64}"
artifact_paths: "filebeat/build/*.xml"

- group: "Windows Extended Testing"
key: "extended-tests-win"
if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/

steps:
- label: ":windows: Win 2019 Unit Tests"
key: "win-extended-2019"
command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
notify:
- github_commit_status:
context: "Filebeat/Extended: Win-2019 Unit Tests"
agents:
provider: "gcp"
image: "${IMAGE_WIN_2019}"
machine_type: "n2-standard-8"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"

- group: "Packaging"
key: "packaging"
if: build.env("BUILDKITE_PULL_REQUEST") != "false"
depends_on:
- "mandatory-tests"

steps:
- label: Package pipeline
commands: ".buildkite/filebeat/scripts/package-step.sh"
commands: ".buildkite/scripts/packaging/package-step.sh"
30 changes: 9 additions & 21 deletions .buildkite/filebeat/scripts/package-step.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,44 +4,32 @@ set -euo pipefail

source .buildkite/env-scripts/util.sh

changeset="^filebeat/
changeset="^${BEATS_PROJECT_NAME}/
^go.mod
^pytest.ini
^dev-tools/
^libbeat/
^testing/
^\.buildkite/filebeat/"
^\.buildkite/${BEATS_PROJECT_NAME}/"

if are_files_changed "$changeset"; then
bk_pipeline=$(cat <<-YAML
steps:
- label: ":ubuntu: Packaging Linux X86"
key: "package-linux-x86"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
command:
- ".buildkite/filebeat/scripts/package.sh"
notify:
- github_commit_status:
context: "Filebeat/Packaging: Linux X86"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
- label: ":linux: Packaging Linux ARM"
- label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM"
key: "package-linux-arm"
env:
PLATFORMS: "linux/arm64"
PACKAGES: "docker"
command:
- ".buildkite/filebeat/scripts/package.sh"
SNAPSHOT: true
GOX_FLAGS: "-arch arm"
command: ".buildkite/scripts/packaging/package.sh"
notify:
- github_commit_status:
context: "Filebeat/Packaging: ARM"
context: "${BEATS_PROJECT_NAME}/Packaging: ARM"
agents:
provider: "aws"
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.large"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
YAML
)
echo "${bk_pipeline}" | buildkite-agent pipeline upload
Expand Down

0 comments on commit 0a24335

Please sign in to comment.