[CI] Package elastic agent job

.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml

@@ -0,0 +1,127 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - input: "Build parameters"
+    if: build.env("ManifestURL") == null
+    fields:
+      - text: "ManifestURL"
+        key: "ManifestURL"
+        default: ""
+        required: true
+        hint: "Link to the build manifest URL."
+      - select: "Mage verbose"
+        key: "MAGEFILE_VERBOSE"
+        required: false
+        options:
+          - label: "True"
+            value: "1"
+          - label: "False"
+            value: "0"
+        hint: "Increase verbosity of the mage commands, defaults to False"
+      - select: "DRA Workflow"
+        key: "DRA_WORKFLOW"
+        required: true
+        options:
+          - label: "snapshot"
+            value: "snapshot"
+          - label: "staging"
+            value: "staging"
+        hint: "What workflow of the DRA release process this build is going to be triggered for"
+      - text: "DRA Version"
+        key: "DRA_VERSION"
+        required: true
+        default: ""
+        hint: "The packaging version to use"
+      - select: "DRA DRY-RUN"
+        key: "DRA_DRY_RUN"
+        required: false
+        options:
+          - label: "True"
+            value: "--dry-run"
+          - label: "False"
+            value: ""
+        hint: "If the DRA release manager script would actually publish anything or just print"
+
+  - wait: ~
+    if: build.env("ManifestURL") == null
+
+  - group: ":Packaging Artefacts"
+    key: "package"
+    steps:
+      - label: "Package elastic-agent"
+        key: package_elastic-agent
+        command: |
+          if [[ -z "$${ManifestURL}" ]]; then
+            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
+            if [[ -z "$${ManifestURL}" ]]; then
+              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+              echo "Provided ManifestURL: ${ManifestURL}"
+              exit 1
+            fi
+          fi
+          if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
+            export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+          fi
+          .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+        artifact_paths:
+          - "x-pack/elastic-agent/build/distributions/**/*"
+        agents:
+          provider: "gcp"
+          image: "family/platform-ingest-beats-ubuntu-2204"
+
+      - label: "Package ARM elastic-agent"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        key: package_elastic-agent-arm
+        command: |
+          if [[ -z "$${ManifestURL}" ]]; then
+            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
+            if [[ -z "$${ManifestURL}" ]]; then
+              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+              echo "Provided ManifestURL: ${ManifestURL}"
+              exit 1
+            fi
+          fi
+          if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
+            export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+          fi
+          .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+        artifact_paths:
+          - "x-pack/elastic-agent/build/distributions/**/*"
+        agents:
+          provider: "aws"
+          instanceType: "t4g.2xlarge"
+          imagePrefix: "platform-ingest-beats-ubuntu-2004-aarch64"
+  #
+  - label: ":elastic-stack: Publishing to DRA"
+    key: dra-publish
+    depends_on: package
+    agents:
+      provider: "gcp"
+    env:
+      DRA_PROJECT_ID: "elastic-agent-package"
+      DRA_PROJECT_ARTIFACT_ID: "agent-package"
+    command: |
+      echo "+++ Restoring Artifacts"
+      mkdir build       
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" .
+      echo "+++ Changing permissions for the release manager"
+      sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
+      echo "+++ Running DRA publish step"
+      if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+        export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+      fi
+      if [[ -z "${DRA_DRY_RUN}" ]]; then
+        DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "")
+        export DRA_DRY_RUN
+      fi
+      if [[ -z "${DRA_VERSION}" ]]; then
+        DRA_VERSION=$(buildkite-agent meta-data get DRA_VERSION --default "")
+        export DRA_VERSION
+      fi
+      if [[ -z "${DRA_WORKFLOW}" ]]; then
+        DRA_WORKFLOW=$(buildkite-agent meta-data get DRA_WORKFLOW --default "")
+        export DRA_WORKFLOW
+      fi      
+      .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh

.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+set -uo pipefail
+
+DRY_RUN="${DRA_DRY_RUN:=""}"
+WORKFLOW="${DRA_WORKFLOW:=""}"
+COMMIT="${DRA_COMMIT:="${BUILDKITE_COMMIT:=""}"}"
+BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
+PACKAGE_VERSION="${DRA_VERSION:="${BEAT_VERSION:=""}"}"
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+# force main branch on PR's or it won't execute
+# because the PR branch does not have a project folder in release-manager
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then
+    BRANCH=7.17
+    DRY_RUN="--dry-run"
+    echo "+++ Running in PR or test branch and setting branch 7.17 and --dry-run"
+fi
+
+
+
+if [[ -z "${WORKFLOW}" ]]; then
+  echo "+++ Missing DRA workflow";
+  exit 1
+fi
+if [[ -z "${COMMIT:-""}" ]]; then
+  echo "+++ Missing DRA_COMMIT";
+  exit 1
+fi
+if [[ -z "${PACKAGE_VERSION:-""}" ]]; then
+  echo "+++ Missing DRA_VERSION";
+  exit 1
+fi
+if [[ -z "${BRANCH:-""}" ]]; then
+  echo "+++ Missing DRA_BRANCH";
+  exit 1
+fi
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+}
+
+# Listing Release manager
+function run_release_manager_list() {
+    local _project_id="${1}" _artifact_set="${2}" _workflow="${3}" _commit="${4}" _branch="${5}" _version="${6}"
+    echo "+++ :hammer_and_pick: Release manager listing ${_branch} ${_workflow} DRA artifacts..."
+    docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli list \
+        --project "${_project_id}" \
+        --branch "${_branch}" \
+        --commit "${_commit}" \
+        --workflow "${_workflow}" \
+        --version "${_version}" \
+        --artifact-set "${_artifact_set}"
+}
+
+# Publish DRA artifacts
+function run_release_manager_collect() {
+    local _project_id="${1}" _artifact_set="${2}" _workflow="${3}" _commit="${4}" _branch="${5}" _version="${6}" _dry_run="${7}"
+    echo "+++ :hammer_and_pick: Publishing ${_branch} ${_workflow} DRA artifacts..."
+    docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project "${_project_id}" \
+        --branch "${_branch}" \
+        --commit "${_commit}" \
+        --workflow "${_workflow}" \
+        --version "${_version}" \
+        --artifact-set "${_artifact_set}" \
+        ${_dry_run}
+}
+
+echo "+++ Release Manager ${WORKFLOW} / ${BRANCH} / ${COMMIT}";
+release_manager_login
+run_release_manager_list "${DRA_PROJECT_ID}" "${DRA_PROJECT_ARTIFACT_ID}" "${WORKFLOW}" "${COMMIT}" "${BRANCH}" "${PACKAGE_VERSION}"
+run_release_manager_collect "${DRA_PROJECT_ID}" "${DRA_PROJECT_ARTIFACT_ID}" "${WORKFLOW}" "${COMMIT}" "${BRANCH}" "${PACKAGE_VERSION}" "${DRY_RUN}"
+RM_EXIT_CODE=$?
+
+exit $RM_EXIT_CODE

.buildkite/x-pack/elastic-agent/scripts/steps/package.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -uo pipefail
+source ".buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh"
+source ".buildkite/x-pack/elastic-agent/scripts/steps/common.sh"
+
+DIST_PATH="x-pack/elastic-agent/build/distributions"
+
+set -x
+if test -z "${ManifestURL=:""}"; then
+  echo "Missing variable ManifestURL, export it before use."
+  exit 2
+fi
+
+VERSION="$(make get-version)"
+echo "--- Packaging Elastic Agent"
+
+echo $ManifestURL
+
+export AGENT_DROP_PATH=build/elastic-agent-drop
+mkdir -p $AGENT_DROP_PATH
+
+mage -v -d x-pack/elastic-agent clean downloadManifest package ironbank fixDRADockerArtifacts
+
+echo  "+++ Generate dependencies report"
+BEAT_VERSION_FULL=$(curl -s -XGET "${ManifestURL}" |jq '.version' -r )
+bash dev-tools/dependencies-report
+mkdir -p $DIST_PATH/reports
+mv dependencies.csv "$DIST_PATH/reports/dependencies-${BEAT_VERSION_FULL}.csv"

.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SETUP_GVM_VERSION="v0.5.1"
+DOCKER_COMPOSE_VERSION="1.21.0"
+DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
+SETUP_WIN_PYTHON_VERSION="3.11.0"
+NMAP_WIN_VERSION="7.12"           # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
+GO_VERSION=$(cat .go-version)
+ASDF_MAGE_VERSION="1.15.0"
+PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+PACKAGING_ARM_PLATFORMS="linux/arm64"
+
+export SETUP_GVM_VERSION
+export DOCKER_COMPOSE_VERSION
+export DOCKER_COMPOSE_VERSION_AARCH64
+export SETUP_WIN_PYTHON_VERSION
+export NMAP_WIN_VERSION
+export GO_VERSION
+export ASDF_MAGE_VERSION
+export PACKAGING_PLATFORMS
+export PACKAGING_ARM_PLATFORMS
+
+exportVars() {
+  local platform_type="$(uname)"
+  local arch_type="$(uname -m)"
+  if [ "${arch_type}" == "x86_64" ]; then
+    case "${platform_type}" in
+      Linux|Darwin)
+        export GOX_FLAGS="-arch amd64"
+        export testResults="**/build/TEST*.xml"
+        export artifacts="**/build/TEST*.out"
+        ;;
+      MINGW*)
+        export GOX_FLAGS="-arch 386"
+        export testResults="**\\build\\TEST*.xml"
+        export artifacts="**\\build\\TEST*.out"
+        ;;
+    esac
+  elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then
+    export GOX_FLAGS="-arch arm"
+    export testResults="**/build/TEST*.xml"
+    export artifacts="**/build/TEST*.out"
+  else
+    echo "Unsupported OS"
+  fi
+}
+
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+  exportVars
+  export RACE_DETECTOR="true"
+  export TEST_COVERAGE="true"
+  export DOCKER_PULL="0"
+fi