added dynamic step for packaging

elastic · Jan 18, 2024 · 10b9311 · 10b9311
1 parent fb6d851
commit 10b9311
Show file tree

Hide file tree

Showing 9 changed files with 21 additions and 117 deletions.
diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-source .buildkite/filebeat/scripts/common.sh
+source .buildkite/env-scripts/util.sh
 
 DEBIAN_FRONTEND="noninteractive"
 

diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh
@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 
-if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
   echo ":: Setting larger ulimit on MacOS ::"
   # To bypass file descriptor errors like "Too many open files error" on MacOS
-  ulimit -Sn 10000
+  ulimit -Sn 50000
+  echo ":: ULIMIT :: $(ulimit -n)"
 fi
diff --git a/.buildkite/filebeat/scripts/common.sh → .buildkite/env-scripts/util.sh b/.buildkite/filebeat/scripts/common.sh → .buildkite/env-scripts/util.sh
@@ -2,11 +2,6 @@
 
 set -euo pipefail
 
-#WORKSPACE="$(pwd)"
-#BIN="${WORKSPACE}/bin"
-#HW_TYPE="$(uname -m)"
-#PLATFORM_TYPE="$(uname)"
-
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
     create_bin
@@ -94,12 +89,3 @@ are_files_changed() {
     return 1;
   fi
 }
-
-#prepare_win() {
-##  local os
-##  os="$(uname)"
-#  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
-#    choco install mingw -y
-#    choco install python --version=3.11.0 -y
-#  fi
-#}
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -1,13 +1,11 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  SETUP_GVM_VERSION: "v0.5.1"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
-  DEBIAN_FRONTEND: "noninteractive"
 
 steps:
   - group: "Filebeat Mandatory Testing"
@@ -20,7 +18,7 @@ steps:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat/Mandatory: Unit Tests"
+              context: "Filebeat: Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -33,7 +31,7 @@ steps:
           - ".buildkite/filebeat/scripts/integration-gotests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat/Mandatory: Integration Tests"
+              context: "Filebeat: Integration Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -46,7 +44,7 @@ steps:
           - ".buildkite/filebeat/scripts/integration-pytests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat/Mandatory: Python Integration Tests"
+              context: "Filebeat: Python Integration Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -58,7 +56,7 @@ steps:
         command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "Filebeat/Mandatory: {{matrix.image}} Unit Tests"
+              context: "Filebeat: Unit Tests"
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
@@ -81,7 +79,7 @@ steps:
     steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
@@ -95,7 +93,7 @@ steps:
 
       - label: ":mac: MacOS Unit Tests"
         key: "macos-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
@@ -108,7 +106,7 @@ steps:
 
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
-#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
       - label: ":windows: Win 2019 Unit Tests"
@@ -127,40 +125,14 @@ steps:
           - "filebeat/build/*.xml"
           - "filebeat/build/*.json"
 
+  - group: "Packaging"
+    key: "packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "mandatory-tests"
+      - "extended-tests"
+      - "extended-tests-win"
+
+    steps:
       - label: Package pipeline
         commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload"
-
-#  - group: "Packaging"
-#    key: "packaging"
-#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-#    depends_on:
-#      - "mandatory-tests"
-#      - "extended-tests"
-#      - "extended-tests-win"
-#
-#    steps:
-#      - label: ":ubuntu: Packaging Linux X86"
-#        key: "package-linux-x86"
-#        command:
-#          - ".buildkite/filebeat/scripts/package.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat/Packaging: Linux X86"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#
-#      - label: ":linux: Packaging Linux ARM"
-#        key: "package-linux-arm"
-#        env:
-#          PLATFORMS: "linux/arm64"
-#          PACKAGES: "docker"
-#        command:
-#          - ".buildkite/filebeat/scripts/package.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "Filebeat/Packaging: ARM"
-#        agents:
-#          provider: "aws"
-#          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-#          instanceType: "t4g.large"
diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh
@@ -2,15 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/filebeat/scripts/common.sh
 source .buildkite/env-scripts/linux-env.sh
-#
-## ToDo - remove after Beats agent is created"
-#echo ":: Setup Env ::"
-#add_bin_path
-#with_go
-#with_mage
-## ToDo - end
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh
@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-source .buildkite/filebeat/scripts/common.sh
-
 changeset="^filebeat/
     ^go.mod
     ^pytest.ini
@@ -14,20 +12,10 @@ changeset="^filebeat/
 
 if are_files_changed "$changeset"; then
   cat <<-EOF
-  - group: "Packaging"
-    key: "packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "mandatory-tests"
-      - "extended-tests"
-      - "extended-tests-win"
-
     steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:
@@ -42,8 +30,6 @@ if are_files_changed "$changeset"; then
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:

diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh
@@ -6,29 +6,6 @@ source .buildkite/env-scripts/linux-env.sh
 
 echo ":: Evaluate Filebeat Changes ::"
 
-changeset="^filebeat/
-    ^go.mod
-    ^pytest.ini
-    ^dev-tools/
-    ^libbeat/
-    ^testing/
-    ^\.buildkite/filebeat/"
-
-#if ! are_files_changed "$changeset" ; then
-#  message="No files changed within Filebeat changeset"
-#  echo "$message"
-#  buildkite-agent annotate "$message" --style 'warning' --context 'ctx-warn'
-#  # This should return any error but skip the packaging.
-#  exit 0
-#fi
-
-## ToDo - remove after Beats agent is created"
-#echo ":: Setup Env ::"
-#add_bin_path
-#with_go
-#with_mage
-## ToDo - end
-
 echo ":: Start Packaging ::"
 cd filebeat
 umask 0022

diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh
@@ -5,10 +5,6 @@ set -euo pipefail
 source .buildkite/env-scripts/linux-env.sh
 source .buildkite/env-scripts/macos-env.sh
 
-if [[ $PLATFORM_TYPE == Darwin* ]]; then
-  echo ":: ULIMIT :: $(ulimit -n)"
-fi
-
 echo ":: Execute Unit Tests ::"
 sudo chmod -R go-w filebeat/
 

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -3,20 +3,14 @@
 set -euo pipefail
 
 source .buildkite/env-scripts/env.sh
-source .buildkite/filebeat/scripts/common.sh
+source .buildkite/env-scripts/util.sh
 source .buildkite/env-scripts/win-env.sh
 
-#SETUP_GVM_VERSION="v0.5.1"
-#WORKSPACE="$(pwd)"
-#
-#export SETUP_GVM_VERSION
-#export WORKSPACE
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
     install_python_win
   fi
-  #  prepare_win
 
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")