From 4c47fccce365f45ab979f5601291d3a1abb1aadc Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 14 Feb 2024 15:24:31 +0100
Subject: [PATCH 01/43] [CI] Package elastic agent job

---
 .../pipeline.xpack.elastic-agent.package.yml    | 17 +++++++++++++++++
 .../elastic-agent/scripts/steps/package.sh      |  9 +++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 .buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
 create mode 100644 .buildkite/x-pack/elastic-agent/scripts/steps/package.sh

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
new file mode 100644
index 00000000000..5f572f0c560
--- /dev/null
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Package elastic-agent"
+    env:
+      AGENT_DROP_PATH: "build/distributions/elastic-agent-drop"
+    key: "package-full"
+    command: ".buildkite/x-pack/elastic-agent/scripts/steps/package.sh"
+    artifact_paths:
+      - "build/distributions/elastic-agent-drop/**/*"
+    agents:
+      # TODO: use a container
+      provider: "gcp"
+      image: "family/core-ubuntu-2204"
+    retry:
+      manual:
+        allowed: true
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
new file mode 100644
index 00000000000..cd4720d49e7
--- /dev/null
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -uo pipefail
+source ".buildkite/x-pack/elastic-agent/scripts/steps/common.sh"
+
+VERSION="$(make get-version)"
+echo "--- Packaging Elastic Agent"
+
+mage -v -d x-pack/elastic-agent package
+

From 714155d822e8a4d2c4335e6b02c1849da2c555f0 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 15:40:39 +0100
Subject: [PATCH 02/43] Packaging

---
 .../pipeline.xpack.elastic-agent.package.yml  |   2 +-
 .../elastic-agent/scripts/steps/package.sh    |   8 +-
 dev-tools/mage/artifacts/artifacts_api.go     | 288 ++++++++++++
 dev-tools/mage/crossbuild.go                  |   1 +
 dev-tools/mage/manifest/manifest.go           | 161 +++++++
 dev-tools/mage/manifest/manifestspecs.go      |  95 ++++
 dev-tools/mage/pkg.go                         |  50 +++
 dev-tools/mage/platforms.go                   |   9 +
 dev-tools/mage/settings.go                    |   7 +
 dev-tools/mage/version/version_parser.go      | 190 ++++++++
 dev-tools/mage/version/version_parser_test.go | 413 ++++++++++++++++++
 x-pack/elastic-agent/magefile.go              |  93 +++-
 12 files changed, 1308 insertions(+), 9 deletions(-)
 create mode 100644 dev-tools/mage/artifacts/artifacts_api.go
 create mode 100644 dev-tools/mage/manifest/manifest.go
 create mode 100644 dev-tools/mage/manifest/manifestspecs.go
 create mode 100644 dev-tools/mage/version/version_parser.go
 create mode 100644 dev-tools/mage/version/version_parser_test.go

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 5f572f0c560..a1b543bdb67 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -7,7 +7,7 @@ steps:
     key: "package-full"
     command: ".buildkite/x-pack/elastic-agent/scripts/steps/package.sh"
     artifact_paths:
-      - "build/distributions/elastic-agent-drop/**/*"
+      - "build/elastic-agent-drop/**/*"
     agents:
       # TODO: use a container
       provider: "gcp"
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
index cd4720d49e7..396844d70ce 100644
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -2,8 +2,12 @@
 set -uo pipefail
 source ".buildkite/x-pack/elastic-agent/scripts/steps/common.sh"
 
+if test -z "${ManifestURL=:""}"; then
+  echo "Missing variable ManifestURL, export it before use."
+  exit 2
+fi
+
 VERSION="$(make get-version)"
 echo "--- Packaging Elastic Agent"
 
-mage -v -d x-pack/elastic-agent package
-
+mage -v -d x-pack/elastic-agent clean downloadManifest package ironbank fixDRADockerArtifacts
diff --git a/dev-tools/mage/artifacts/artifacts_api.go b/dev-tools/mage/artifacts/artifacts_api.go
new file mode 100644
index 00000000000..2c45a67cebb
--- /dev/null
+++ b/dev-tools/mage/artifacts/artifacts_api.go
@@ -0,0 +1,288 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package artifacts
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/version"
+)
+
+const (
+	defaultArtifactAPIURL = "https://artifacts-api.elastic.co/"
+
+	artifactsAPIV1VersionsEndpoint      = "v1/versions/"
+	artifactsAPIV1VersionBuildsEndpoint = "v1/versions/%s/builds/"
+	artifactAPIV1BuildDetailsEndpoint   = "v1/versions/%s/builds/%s"
+	// artifactAPIV1SearchVersionPackage = "v1/search/%s/%s"
+)
+
+var (
+	ErrLatestVersionNil        = errors.New("latest version is nil")
+	ErrSnapshotVersionsEmpty   = errors.New("snapshot list is nil")
+	ErrInvalidVersionRetrieved = errors.New("invalid version retrieved from artifact API")
+
+	ErrBadHTTPStatusCode = errors.New("bad http status code")
+)
+
+type Manifests struct {
+	LastUpdateTime         string `json:"last-update-time"`
+	SecondsSinceLastUpdate int    `json:"seconds-since-last-update"`
+}
+
+type VersionList struct {
+	Versions  []string  `json:"versions"`
+	Aliases   []string  `json:"aliases"`
+	Manifests Manifests `json:"manifests"`
+}
+
+type VersionBuilds struct {
+	Builds    []string  `json:"builds"`
+	Manifests Manifests `json:"manifests"`
+}
+
+type Package struct {
+	URL          string   `json:"url"`
+	ShaURL       string   `json:"sha_url"`
+	AscURL       string   `json:"asc_url"`
+	Type         string   `json:"type"`
+	Architecture string   `json:"architecture"`
+	Os           []string `json:"os"`
+	Classifier   string   `json:"classifier"`
+	Attributes   struct {
+		IncludeInRepo string `json:"include_in_repo"`
+		ArtifactNoKpi string `json:"artifactNoKpi"`
+		Internal      string `json:"internal"`
+		ArtifactID    string `json:"artifact_id"`
+		Oss           string `json:"oss"`
+		Group         string `json:"group"`
+	} `json:"attributes"`
+}
+
+type Dependency struct {
+	Prefix   string `json:"prefix"`
+	BuildUri string `json:"build_uri"`
+}
+
+type Project struct {
+	Branch                       string             `json:"branch"`
+	CommitHash                   string             `json:"commit_hash"`
+	CommitURL                    string             `json:"commit_url"`
+	ExternalArtifactsManifestURL string             `json:"external_artifacts_manifest_url"`
+	BuildDurationSeconds         int                `json:"build_duration_seconds"`
+	Packages                     map[string]Package `json:"packages"`
+	Dependencies                 []Dependency       `json:"dependencies"`
+}
+
+type Build struct {
+	Projects             map[string]Project `json:"projects"`
+	StartTime            string             `json:"start_time"`
+	ReleaseBranch        string             `json:"release_branch"`
+	Prefix               string             `json:"prefix"`
+	EndTime              string             `json:"end_time"`
+	ManifestVersion      string             `json:"manifest_version"`
+	Version              string             `json:"version"`
+	Branch               string             `json:"branch"`
+	BuildID              string             `json:"build_id"`
+	BuildDurationSeconds int                `json:"build_duration_seconds"`
+}
+
+type BuildDetails struct {
+	Build     Build
+	Manifests Manifests `json:"manifests"`
+}
+
+type SearchPackageResult struct {
+	Packages  map[string]Package `json:"packages"`
+	Manifests Manifests          `json:"manifests"`
+}
+
+type httpDoer interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+type ArtifactAPIClientOpt func(aac *ArtifactAPIClient)
+
+func WithUrl(url string) ArtifactAPIClientOpt {
+	return func(aac *ArtifactAPIClient) { aac.url = url }
+}
+
+func WithHttpClient(client httpDoer) ArtifactAPIClientOpt {
+	return func(aac *ArtifactAPIClient) { aac.c = client }
+}
+
+// ArtifactAPIClient is a small (and incomplete) client for the Elastic artifact API.
+// More information about the API can be found at https://artifacts-api.elastic.co/v1
+// which will print a list of available operations
+type ArtifactAPIClient struct {
+	c   httpDoer
+	url string
+}
+
+// NewArtifactAPIClient creates a new Artifact API client
+func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
+	c := &ArtifactAPIClient{
+		url: defaultArtifactAPIURL,
+		c:   new(http.Client),
+	}
+
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	return c
+}
+
+// GetVersions returns a list of versions as server by the Artifact API along with some aliases and manifest information
+func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList, err error) {
+	joinedURL, err := aac.composeURL(artifactsAPIV1VersionsEndpoint)
+	if err != nil {
+		return
+	}
+
+	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
+	if err != nil {
+		err = fmt.Errorf("getting versions: %w", err)
+		return
+	}
+
+	defer resp.Body.Close()
+	return checkResponseAndUnmarshal[VersionList](resp)
+}
+
+// GetBuildsForVersion returns a list of builds for a specific version.
+// version should be one of the version strings returned by the GetVersions (expected format is semver
+// with optional prerelease but no build metadata, for example 8.9.0-SNAPSHOT)
+func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version string) (builds *VersionBuilds, err error) {
+	joinedURL, err := aac.composeURL(fmt.Sprintf(artifactsAPIV1VersionBuildsEndpoint, version))
+	if err != nil {
+		return
+	}
+
+	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
+	if err != nil {
+		err = fmt.Errorf("getting builds for version %s: %w", version, err)
+		return
+	}
+
+	defer resp.Body.Close()
+	return checkResponseAndUnmarshal[VersionBuilds](resp)
+}
+
+// GetBuildDetails returns the list of project and artifacts related to a specific build.
+// Version parameter format follows semver (without build metadata) and buildID format is <major>.<minor>.<patch>-<buildhash> as returned by
+// GetBuildsForVersion()
+func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string, buildID string) (buildDetails *BuildDetails, err error) {
+	joinedURL, err := aac.composeURL(fmt.Sprintf(artifactAPIV1BuildDetailsEndpoint, version, buildID))
+	if err != nil {
+		return
+	}
+
+	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
+	if err != nil {
+		err = fmt.Errorf("getting build details for version %s buildID %s: %w", version, buildID, err)
+		return
+	}
+
+	defer resp.Body.Close()
+	return checkResponseAndUnmarshal[BuildDetails](resp)
+}
+
+func (aac ArtifactAPIClient) composeURL(relativePath string) (string, error) {
+	joinedURL, err := url.JoinPath(aac.url, relativePath)
+	if err != nil {
+		return "", fmt.Errorf("composing URL with %q %q: %w", aac.url, relativePath, err)
+	}
+
+	return joinedURL, nil
+}
+
+func (aac ArtifactAPIClient) createAndPerformRequest(ctx context.Context, URL string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, URL, nil)
+	if err != nil {
+		err = fmt.Errorf("composing request: %w", err)
+		return nil, err
+	}
+
+	resp, err := aac.c.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("executing http request %v: %w", req, err)
+	}
+
+	return resp, nil
+}
+
+func checkResponseAndUnmarshal[T any](resp *http.Response) (*T, error) {
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%d: %w", resp.StatusCode, ErrBadHTTPStatusCode)
+	}
+
+	respBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading response body: %w", err)
+	}
+	result := new(T)
+	err = json.Unmarshal(respBytes, result)
+
+	if err != nil {
+		return nil, fmt.Errorf("unmarshaling: %w", err)
+	}
+
+	return result, nil
+}
+
+type logger interface {
+	Logf(format string, args ...any)
+}
+
+func (aac ArtifactAPIClient) GetLatestSnapshotVersion(ctx context.Context, log logger) (*version.ParsedSemVer, error) {
+	vList, err := aac.GetVersions(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if vList == nil {
+		return nil, ErrSnapshotVersionsEmpty
+	}
+
+	sortedParsedVersions := make(version.SortableParsedVersions, 0, len(vList.Versions))
+	for _, v := range vList.Versions {
+		pv, err := version.ParseVersion(v)
+		if err != nil {
+			log.Logf("invalid version retrieved from artifact API: %q", v)
+			return nil, ErrInvalidVersionRetrieved
+		}
+		sortedParsedVersions = append(sortedParsedVersions, pv)
+	}
+
+	if len(sortedParsedVersions) == 0 {
+		return nil, ErrSnapshotVersionsEmpty
+	}
+
+	// normally the output of the versions returned by artifact API is already
+	// sorted in ascending order.If we want to sort in descending order we need
+	// to pass a sort.Reverse to sort.Sort.
+	sort.Sort(sort.Reverse(sortedParsedVersions))
+
+	var latestSnapshotVersion *version.ParsedSemVer
+	// fetch the latest SNAPSHOT build
+	for _, pv := range sortedParsedVersions {
+		if pv.IsSnapshot() {
+			latestSnapshotVersion = pv
+			break
+		}
+	}
+	if latestSnapshotVersion == nil {
+		return nil, ErrLatestVersionNil
+	}
+	return latestSnapshotVersion, nil
+}
diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go
index f59dc2e710e..34345d43d84 100644
--- a/dev-tools/mage/crossbuild.go
+++ b/dev-tools/mage/crossbuild.go
@@ -136,6 +136,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 
 	// Docker is required for this target.
 	if err := HaveDocker(); err != nil {
+		fmt.Printf("No docker found: %v\n", err)
 		return err
 	}
 
diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go
new file mode 100644
index 00000000000..d7ad7babd2f
--- /dev/null
+++ b/dev-tools/mage/manifest/manifest.go
@@ -0,0 +1,161 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package manifest
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	artifacts "github.com/elastic/beats/v7/dev-tools/mage/artifacts"
+
+	"github.com/magefile/mage/mg"
+	"golang.org/x/sync/errgroup"
+)
+
+// A backoff schedule for when and how often to retry failed HTTP
+// requests. The first element is the time to wait after the
+// first failure, the second the time to wait after the second
+// failure, etc. After reaching the last element, retries stop
+// and the request is considered failed.
+var backoffSchedule = []time.Duration{
+	1 * time.Second,
+	3 * time.Second,
+	10 * time.Second,
+}
+
+var errorInvalidManifestURL = errors.New("invalid ManifestURL provided")
+var errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not allowed URL")
+
+var AllowedManifestHosts = []string{"snapshots.elastic.co", "staging.elastic.co", "artifacts-staging.elastic.co"}
+
+// DownloadManifest is going to download the given manifest file and return the ManifestResponse
+func DownloadManifest(manifest string) (artifacts.Build, error) {
+	log.Printf(">>>> Downloading manifest %s", manifest)
+	manifestUrl, urlError := url.Parse(manifest)
+	if urlError != nil {
+		return artifacts.Build{}, errorInvalidManifestURL
+	}
+	var valid = false
+	for _, manifestHost := range AllowedManifestHosts {
+		if manifestHost == manifestUrl.Host {
+			valid = true
+		}
+	}
+	if !valid {
+		log.Printf("Not allowed %s, valid ones are %+v", manifestUrl.Host, AllowedManifestHosts)
+		return artifacts.Build{}, errorNotAllowedManifestURL
+	}
+	sanitizedUrl := fmt.Sprintf("https://%s%s", manifestUrl.Host, manifestUrl.Path)
+	f := func() (artifacts.Build, error) { return downloadManifestData(sanitizedUrl) }
+	manifestResponse, err := doWithRetries(f)
+	if err != nil {
+		return artifacts.Build{}, fmt.Errorf("downloading manifest: %w", err)
+	}
+	// if mg.Verbose() {
+	log.Printf(">>>> Downloaded manifest %s", manifest)
+	log.Printf(">>>> Packaing version: %s, build_id: %s, manifest_version:%s", manifestResponse.Version, manifestResponse.BuildID, manifestResponse.ManifestVersion)
+	// }
+	return manifestResponse, nil
+}
+
+func resolveManifestPackage(project artifacts.Project, pkg string, reqPackage string, version string) []string {
+	packageName := fmt.Sprintf("%s-%s-%s", pkg, version, reqPackage)
+	val, ok := project.Packages[packageName]
+	if !ok {
+		return nil
+	}
+	if mg.Verbose() {
+		log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash)
+	}
+	return []string{val.URL, val.ShaURL, val.AscURL}
+
+}
+
+// DownloadComponentsFromManifest is going to download a set of components from the given manifest into the destination
+// dropPath folder in order to later use that folder for packaging
+func DownloadComponentsFromManifest(manifest string, platforms []string, platformPackages map[string]string, dropPath string) error {
+	componentSpec := map[string][]string{
+		"apm-server":            {"apm-server"},
+		"beats":                 {"auditbeat", "filebeat", "heartbeat", "metricbeat", "osquerybeat", "packetbeat"},
+		"cloud-defend":          {"cloud-defend"},
+		"cloudbeat":             {"cloudbeat"},
+		"elastic-agent-shipper": {"elastic-agent-shipper"},
+		"endpoint-dev":          {"endpoint-security"},
+		"fleet-server":          {"fleet-server"},
+		"prodfiler":             {"pf-elastic-collector", "pf-elastic-symbolizer", "pf-host-agent"},
+	}
+
+	manifestResponse, err := DownloadManifest(manifest)
+	if err != nil {
+		return fmt.Errorf("failed to download remote manifest file %w", err)
+	}
+	projects := manifestResponse.Projects
+
+	errGrp, downloadsCtx := errgroup.WithContext(context.Background())
+	for component, pkgs := range componentSpec {
+		for _, platform := range platforms {
+			targetPath := filepath.Join(dropPath)
+			err := os.MkdirAll(targetPath, 0755)
+			if err != nil {
+				return fmt.Errorf("failed to create directory %s", targetPath)
+			}
+			log.Printf("+++ Prepare to download project [%s] for [%s]", component, platform)
+
+			for _, pkg := range pkgs {
+				reqPackage := platformPackages[platform]
+				pkgURL := resolveManifestPackage(projects[component], pkg, reqPackage, manifestResponse.Version)
+				if pkgURL != nil {
+					for _, p := range pkgURL {
+						log.Printf(">>>>>>>>> Downloading [%s] [%s] ", pkg, p)
+						pkgFilename := path.Base(p)
+						downloadTarget := filepath.Join(targetPath, pkgFilename)
+						if _, err := os.Stat(downloadTarget); err != nil {
+							errGrp.Go(func(ctx context.Context, url, target string) func() error {
+								return func() error { return downloadPackage(ctx, url, target) }
+							}(downloadsCtx, p, downloadTarget))
+						}
+					}
+				} else if mg.Verbose() {
+					log.Printf(">>>>>>>>> Project [%s] does not have [%s] ", pkg, platform)
+				}
+			}
+		}
+	}
+
+	err = errGrp.Wait()
+	if err != nil {
+		return fmt.Errorf("error downloading files: %w", err)
+	}
+
+	log.Printf("Downloads for manifest %q complete.", manifest)
+	return nil
+}
+
+func downloadPackage(ctx context.Context, downloadUrl string, target string) error {
+	parsedURL, errorUrl := url.Parse(downloadUrl)
+	if errorUrl != nil {
+		return errorInvalidManifestURL
+	}
+	var valid = false
+	for _, manifestHost := range AllowedManifestHosts {
+		if manifestHost == parsedURL.Host {
+			valid = true
+		}
+	}
+	if !valid {
+		log.Printf("Not allowed %s, valid ones are %+v", parsedURL.Host, AllowedManifestHosts)
+		return errorNotAllowedManifestURL
+	}
+	cleanUrl := fmt.Sprintf("https://%s%s", parsedURL.Host, parsedURL.Path)
+	_, err := doWithRetries(func() (string, error) { return downloadFile(ctx, cleanUrl, target) })
+	return err
+}
diff --git a/dev-tools/mage/manifest/manifestspecs.go b/dev-tools/mage/manifest/manifestspecs.go
new file mode 100644
index 00000000000..c076a637c47
--- /dev/null
+++ b/dev-tools/mage/manifest/manifestspecs.go
@@ -0,0 +1,95 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package manifest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/magefile/mage/mg"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/artifacts"
+)
+
+func doWithRetries[T any](f func() (T, error)) (T, error) {
+	var err error
+	var resp T
+	for _, backoff := range backoffSchedule {
+		resp, err = f()
+		if err == nil {
+			return resp, nil
+		}
+		if mg.Verbose() {
+			log.Printf("Request error: %+v\n", err)
+			log.Printf("Retrying in %v\n", backoff)
+		}
+		time.Sleep(backoff)
+	}
+
+	// All retries failed
+	return resp, err
+}
+
+func downloadFile(ctx context.Context, url string, filepath string) (string, error) {
+	outFile, fileErr := os.Create(filepath)
+	if fileErr != nil {
+		return "", fmt.Errorf("failed to create destination file %w", fileErr)
+	}
+	defer func() {
+		if err := outFile.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return "", fmt.Errorf("error creating request for %q: %w", url, err)
+	}
+
+	resp, reqErr := http.DefaultClient.Do(req)
+	if reqErr != nil {
+		return filepath, fmt.Errorf("failed to download manifest [%s]\n %w", url, err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	_, errCopy := io.Copy(outFile, resp.Body)
+	if errCopy != nil {
+		return "", fmt.Errorf("failed to decode manifest response [%s]\n %w", url, err)
+	}
+	if mg.Verbose() {
+		log.Printf("<<<<<<<<< Downloaded: %s to %s", url, filepath)
+	}
+
+	return outFile.Name(), nil
+}
+
+func downloadManifestData(url string) (artifacts.Build, error) {
+	var response artifacts.Build
+	resp, err := http.Get(url) //nolint // we should have already verified that this is a proper valid url
+	if err != nil {
+		return response, fmt.Errorf("failed to download manifest [%s]\n %w", url, err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		return response, fmt.Errorf("failed to decode manifest response [%s]\n %w", url, err)
+	}
+	return response, nil
+}
diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go
index 5827c806d8b..b44522df8dc 100644
--- a/dev-tools/mage/pkg.go
+++ b/dev-tools/mage/pkg.go
@@ -22,6 +22,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -387,3 +388,52 @@ func TestBinaryGLIBCVersion(elfPath, maxGlibcVersion string) error {
 	fmt.Printf(">> testBinaryGLIBCVersion: %q requires glibc %v or greater\n", elfPath, requiredGlibc)
 	return nil
 }
+
+// FixDRADockerArtifacts is a workaround for the DRA artifacts produced by the package target. We had to do
+// because the initial unified release manager DSL code required specific names that the package does not produce,
+// we wanted to keep backwards compatibility with the artifacts of the unified release and the DRA.
+// this follows the same logic as https://github.com/elastic/beats/blob/2fdefcfbc783eb4710acef07d0ff63863fa00974/.ci/scripts/prepare-release-manager.sh
+func FixDRADockerArtifacts() error {
+	fmt.Println("--- Fixing Docker DRA artifacts")
+	distributionsPath := filepath.Join("build", "distributions")
+	// Find all the files with the given name
+	matches, err := filepath.Glob(filepath.Join(distributionsPath, "*docker.tar.gz*"))
+	if err != nil {
+		return err
+	}
+	if mg.Verbose() {
+		log.Printf("--- Found artifacts to rename %s %d", distributionsPath, len(matches))
+	}
+	// Match the artifact name and break down into groups so that we can reconstruct the names as its expected by the DRA DSL
+	// As SNAPSHOT keyword or BUILDID are optional, capturing the separator - or + with the value.
+	artifactRegexp, err := regexp.Compile(`([\w+-]+)-(([0-9]+)\.([0-9]+)\.([0-9]+))([-|\+][\w]+)?-([\w]+)-([\w]+)\.([\w]+)\.([\w.]+)`)
+	if err != nil {
+		return err
+	}
+	for _, m := range matches {
+		artifactFile, err := os.Stat(m)
+		if err != nil {
+			return fmt.Errorf("failed stating file: %w", err)
+		}
+		if artifactFile.IsDir() {
+			continue
+		}
+		match := artifactRegexp.FindAllStringSubmatch(artifactFile.Name(), -1)
+		// The groups here is tightly coupled with the regexp above.
+		// match[0][6] already contains the separator so no need to add before the variable
+		targetName := fmt.Sprintf("%s-%s%s-%s-image-%s-%s.%s", match[0][1], match[0][2], match[0][6], match[0][9], match[0][7], match[0][8], match[0][10])
+		if mg.Verbose() {
+			fmt.Printf("%#v\n", match)
+			fmt.Printf("Artifact: %s \n", artifactFile.Name())
+			fmt.Printf("Renamed:  %s \n", targetName)
+		}
+		renameErr := os.Rename(filepath.Join(distributionsPath, artifactFile.Name()), filepath.Join(distributionsPath, targetName))
+		if renameErr != nil {
+			return renameErr
+		}
+		if mg.Verbose() {
+			fmt.Println("Renamed artifact")
+		}
+	}
+	return nil
+}
diff --git a/dev-tools/mage/platforms.go b/dev-tools/mage/platforms.go
index 4be617ada83..d47b03d3061 100644
--- a/dev-tools/mage/platforms.go
+++ b/dev-tools/mage/platforms.go
@@ -202,6 +202,15 @@ func (p PlatformAttributes) String() string {
 // BuildPlatformList is a list of BuildPlatforms that supports filtering.
 type BuildPlatformList []BuildPlatform
 
+// Returns all BuildPlatform names
+func (list BuildPlatformList) Names() []string {
+	platforms := make([]string, len(list))
+	for i, bp := range list {
+		platforms[i] = bp.Name
+	}
+	return platforms
+}
+
 // Get returns the BuildPlatform matching the given name.
 func (list BuildPlatformList) Get(name string) (BuildPlatform, bool) {
 	for _, bp := range list {
diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go
index 1b6e07f7bac..213dc7ebd19 100644
--- a/dev-tools/mage/settings.go
+++ b/dev-tools/mage/settings.go
@@ -84,6 +84,11 @@ var (
 	versionQualified bool
 	versionQualifier string
 
+	// PackagingFromManifest This value is set to tru when we have defined a ManifestURL variable
+	PackagingFromManifest bool
+	// ManifestURL Location of the manifest file to package
+	ManifestURL string
+
 	FuncMap = map[string]interface{}{
 		"beat_doc_branch":   BeatDocBranch,
 		"beat_version":      BeatQualifiedVersion,
@@ -126,6 +131,8 @@ func init() {
 	}
 
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
+	ManifestURL = EnvOr("ManifestURL", "")
+	PackagingFromManifest = ManifestURL != ""
 }
 
 // ProjectType specifies the type of project (OSS vs X-Pack).
diff --git a/dev-tools/mage/version/version_parser.go b/dev-tools/mage/version/version_parser.go
new file mode 100644
index 00000000000..a06b1c0533f
--- /dev/null
+++ b/dev-tools/mage/version/version_parser.go
@@ -0,0 +1,190 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package version
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// regexp taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?)
+const semVerFormat = `^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`
+const preReleaseSeparator = "-"
+const metadataSeparator = "+"
+
+var semVerFmtRegEx *regexp.Regexp
+var namedGroups map[string]int
+
+func init() {
+	// small init to compile the regex and build a map of named groups and indexes
+	semVerFmtRegEx = regexp.MustCompile(semVerFormat)
+	groups := semVerFmtRegEx.SubexpNames()
+	namedGroups = make(map[string]int, len(groups))
+	for i, groupName := range groups {
+		namedGroups[groupName] = i
+	}
+}
+
+var ErrNoMatch = errors.New("version string does not match expected format")
+
+type ParsedSemVer struct {
+	original      string
+	major         int
+	minor         int
+	patch         int
+	prerelease    string
+	buildMetadata string
+}
+
+func (psv ParsedSemVer) Original() string {
+	return psv.original
+}
+
+func (psv ParsedSemVer) Major() int {
+	return psv.major
+}
+
+func (psv ParsedSemVer) Minor() int {
+	return psv.minor
+}
+
+func (psv ParsedSemVer) Patch() int {
+	return psv.patch
+}
+
+func (psv ParsedSemVer) CoreVersion() string {
+	return fmt.Sprintf("%d.%d.%d", psv.Major(), psv.Minor(), psv.Patch())
+}
+
+func (psv ParsedSemVer) Prerelease() string {
+	return psv.prerelease
+}
+
+func (psv ParsedSemVer) BuildMetadata() string {
+	return psv.buildMetadata
+}
+
+func (psv ParsedSemVer) VersionWithPrerelease() string {
+	b := new(strings.Builder)
+	b.WriteString(psv.CoreVersion())
+	if psv.prerelease != "" {
+		b.WriteString("-")
+		b.WriteString(psv.prerelease)
+	}
+	return b.String()
+}
+
+func (psv ParsedSemVer) IsSnapshot() bool {
+	return psv.prerelease == "SNAPSHOT" || strings.HasSuffix(psv.prerelease, "-SNAPSHOT")
+}
+
+func (psv ParsedSemVer) Less(other ParsedSemVer) bool {
+	// compare major version
+	if psv.major != other.major {
+		return psv.major < other.major
+	}
+
+	//same major, check minor
+	if psv.minor != other.minor {
+		return psv.minor < other.minor
+	}
+
+	//same minor, check patch
+	if psv.patch != other.patch {
+		return psv.patch < other.patch
+	}
+
+	// last resort check if one is prereleas and the other isn't
+	if psv.prerelease != "" && other.prerelease == "" {
+		return true
+	}
+
+	return false
+}
+
+func (psv ParsedSemVer) GetPreviousMinor() (*ParsedSemVer, error) {
+	major := psv.Major()
+	minor := psv.Minor()
+
+	if minor > 0 {
+		// We have at least one previous minor version in the current
+		// major version series. Set the patch to zero to guarnatee the
+		// version exists, the number of patch releases varies.
+		return NewParsedSemVer(major, minor-1, 0, psv.Prerelease(), psv.BuildMetadata()), nil
+	}
+
+	// We are at the first minor of the current major version series. To
+	// figure out the previous minor, we need to rely on knowledge of
+	// the release versions from the past major series'.
+	switch major {
+	case 8:
+		return NewParsedSemVer(7, 17, 10, psv.Prerelease(), psv.BuildMetadata()), nil
+	}
+
+	return nil, fmt.Errorf("unable to determine previous minor version for [%s]", psv.String())
+}
+
+func (psv ParsedSemVer) String() string {
+	b := new(strings.Builder)
+	b.WriteString(psv.CoreVersion())
+	if psv.Prerelease() != "" {
+		b.WriteString(preReleaseSeparator)
+		b.WriteString(psv.Prerelease())
+	}
+	if psv.BuildMetadata() != "" {
+		b.WriteString(metadataSeparator)
+		b.WriteString(psv.buildMetadata)
+	}
+	return b.String()
+}
+
+func NewParsedSemVer(major int, minor int, patch int, prerelease string, metadata string) *ParsedSemVer {
+	return &ParsedSemVer{
+		major:         major,
+		minor:         minor,
+		patch:         patch,
+		prerelease:    prerelease,
+		buildMetadata: metadata,
+	}
+}
+
+func ParseVersion(version string) (*ParsedSemVer, error) {
+	matches := semVerFmtRegEx.FindStringSubmatch(strings.TrimSpace(version))
+	if matches == nil {
+		return nil, ErrNoMatch
+	}
+
+	major, err := strconv.Atoi(matches[namedGroups["major"]])
+	if err != nil {
+		return nil, fmt.Errorf("parsing major version: %w", err)
+	}
+
+	minor, err := strconv.Atoi(matches[namedGroups["minor"]])
+	if err != nil {
+		return nil, fmt.Errorf("parsing minor version: %w", err)
+	}
+
+	patch, err := strconv.Atoi(matches[namedGroups["patch"]])
+	if err != nil {
+		return nil, fmt.Errorf("parsing patch version: %w", err)
+	}
+	return &ParsedSemVer{
+		original:      version,
+		major:         major,
+		minor:         minor,
+		patch:         patch,
+		prerelease:    matches[namedGroups["prerelease"]],
+		buildMetadata: matches[namedGroups["buildmetadata"]],
+	}, nil
+}
+
+type SortableParsedVersions []*ParsedSemVer
+
+func (spv SortableParsedVersions) Len() int           { return len(spv) }
+func (spv SortableParsedVersions) Swap(i, j int)      { spv[i], spv[j] = spv[j], spv[i] }
+func (spv SortableParsedVersions) Less(i, j int) bool { return spv[i].Less(*spv[j]) }
diff --git a/dev-tools/mage/version/version_parser_test.go b/dev-tools/mage/version/version_parser_test.go
new file mode 100644
index 00000000000..c7df78f5e63
--- /dev/null
+++ b/dev-tools/mage/version/version_parser_test.go
@@ -0,0 +1,413 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package version
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSemVerRegexpCompiles(t *testing.T) {
+	require.NotNil(t, semVerFmtRegEx)
+	assert.Contains(t, namedGroups, "major")
+	assert.Contains(t, namedGroups, "minor")
+	assert.Contains(t, namedGroups, "patch")
+	assert.Contains(t, namedGroups, "prerelease")
+	assert.Contains(t, namedGroups, "buildmetadata")
+}
+
+func TestParseVersion(t *testing.T) {
+	type expected struct {
+		parsed            *ParsedSemVer
+		versionPrerelease string
+		err               error
+	}
+
+	testcases := []struct {
+		name     string
+		input    string
+		expected expected
+	}{
+		{
+			name:  "Simple semver",
+			input: "1.2.3",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      "1.2.3",
+					major:         1,
+					minor:         2,
+					patch:         3,
+					prerelease:    "",
+					buildMetadata: "",
+				},
+				versionPrerelease: "1.2.3",
+				err:               nil,
+			},
+		},
+		{
+			name:  "Biiig semver",
+			input: "1111.2222.3333",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      "1111.2222.3333",
+					major:         1111,
+					minor:         2222,
+					patch:         3333,
+					prerelease:    "",
+					buildMetadata: "",
+				},
+				versionPrerelease: "1111.2222.3333",
+				err:               nil,
+			},
+		},
+		{
+			name:  "Simple semver with spaces around",
+			input: " \t1.2.3 \r\n ",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      " \t1.2.3 \r\n ",
+					major:         1,
+					minor:         2,
+					patch:         3,
+					prerelease:    "",
+					buildMetadata: "",
+				},
+				versionPrerelease: "1.2.3",
+				err:               nil,
+			},
+		},
+		{
+			name:  "Semver with prerelease",
+			input: "1.2.3-SNAPSHOT",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      "1.2.3-SNAPSHOT",
+					major:         1,
+					minor:         2,
+					patch:         3,
+					prerelease:    "SNAPSHOT",
+					buildMetadata: "",
+				},
+				versionPrerelease: "1.2.3-SNAPSHOT",
+				err:               nil,
+			},
+		},
+		{
+			name:  "Semver with versioned prerelease",
+			input: "1.2.3-er.1+abcdef",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      "1.2.3-er.1+abcdef",
+					major:         1,
+					minor:         2,
+					patch:         3,
+					prerelease:    "er.1",
+					buildMetadata: "abcdef",
+				},
+				versionPrerelease: "1.2.3-er.1",
+				err:               nil,
+			},
+		},
+		{
+			name:  "Semver with prerelease and build metadata",
+			input: "1.2.3-SNAPSHOT+abcdef",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      "1.2.3-SNAPSHOT+abcdef",
+					major:         1,
+					minor:         2,
+					patch:         3,
+					prerelease:    "SNAPSHOT",
+					buildMetadata: "abcdef",
+				},
+				versionPrerelease: "1.2.3-SNAPSHOT",
+				err:               nil,
+			},
+		},
+		{
+			name:  "Semver string version, with double prerelease(er and snapshot)",
+			input: "1.2.5-er.1-SNAPSHOT",
+			expected: expected{
+				parsed: &ParsedSemVer{
+					original:      "1.2.5-er.1-SNAPSHOT",
+					major:         1,
+					minor:         2,
+					patch:         5,
+					prerelease:    "er.1-SNAPSHOT",
+					buildMetadata: "",
+				},
+				versionPrerelease: "1.2.5-er.1-SNAPSHOT",
+			},
+		},
+		{
+			name:  "Error truncated semver",
+			input: "2.3",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Error missing prerelease type",
+			input: "1.2.3-",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Error missing build metadata",
+			input: "1.2.3-beta.22+",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Weird random string version",
+			input: "asdasdasdasdasd",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Almost semver string version, with double extra meta separator",
+			input: "1.2.3++",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Almost semver string version, with empty minor version",
+			input: "1..2+ab",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Almost semver string version, with patch containing non-digits",
+			input: "1.2.5ab0",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+		{
+			name:  "Split string version",
+			input: "4.5\r\n.6",
+			expected: expected{
+				parsed: nil,
+				err:    ErrNoMatch,
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			actualParsed, err := ParseVersion(tc.input)
+
+			if tc.expected.err != nil {
+				assert.ErrorIs(t, err, tc.expected.err)
+				return
+			}
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expected.parsed, actualParsed)
+
+			// assert getters
+			assert.Equal(t, tc.expected.parsed.original, actualParsed.Original())
+			assert.Equal(t, tc.expected.parsed.major, actualParsed.Major())
+			assert.Equal(t, tc.expected.parsed.minor, actualParsed.Minor())
+			assert.Equal(t, tc.expected.parsed.patch, actualParsed.Patch())
+			assert.Equal(t, tc.expected.parsed.prerelease, actualParsed.Prerelease())
+			assert.Equal(t, tc.expected.parsed.buildMetadata, actualParsed.BuildMetadata())
+			assert.Equal(t, tc.expected.versionPrerelease, actualParsed.VersionWithPrerelease())
+
+			// verify that String() method returns the same input string (after trimming)
+			assert.Equal(t, strings.TrimSpace(tc.input), actualParsed.String())
+		})
+	}
+}
+
+func TestIsSnapshot(t *testing.T) {
+	testcases := []struct {
+		name     string
+		input    string
+		snapshot bool
+	}{
+		{
+			name:     "Simple snapshot",
+			input:    "8.8.0-SNAPSHOT",
+			snapshot: true,
+		},
+		{
+			name:     "Snapshot with build meta",
+			input:    "8.8.0-SNAPSHOT+abcdef",
+			snapshot: true,
+		},
+		{
+			name:     "Snapshot comparison is case sensitive",
+			input:    "8.8.0-sNapShOt",
+			snapshot: false,
+		},
+		{
+			name:     "Only major minor patch",
+			input:    "8.8.0",
+			snapshot: false,
+		},
+		{
+			name:     "Alpha prerelease is not snapshot",
+			input:    "8.8.0-alpha",
+			snapshot: false,
+		},
+		{
+			name:     "Emergency release is not snapshot",
+			input:    "8.8.0-er.1",
+			snapshot: false,
+		},
+		{
+			name:     "Emergency release snapshot is actually a snapshot",
+			input:    "8.8.0-er.1-SNAPSHOT ",
+			snapshot: true,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			psv, err := ParseVersion(tc.input)
+			require.NoError(t, err)
+			require.NotNil(t, psv)
+			assert.Equal(t, tc.snapshot, psv.IsSnapshot())
+		})
+
+	}
+
+}
+
+func TestLess(t *testing.T) {
+	testcases := []struct {
+		name         string
+		leftVersion  string
+		rightVersion string
+		less         bool
+	}{
+		{
+			name:         "major version less than ours",
+			leftVersion:  "7.17.10",
+			rightVersion: "8.9.0",
+			less:         true,
+		},
+		{
+			name:         "minor version less than ours",
+			leftVersion:  "8.6.2",
+			rightVersion: "8.9.0",
+			less:         true,
+		},
+		{
+			name:         "patch version less than ours",
+			leftVersion:  "8.7.0",
+			rightVersion: "8.7.1",
+			less:         true,
+		},
+		{
+			name:         "prerelease is always less than non-prerelease",
+			leftVersion:  "8.9.0-SNAPSHOT",
+			rightVersion: "8.9.0",
+			less:         true,
+		},
+		{
+			name:         "2 prereleases have no specific order",
+			leftVersion:  "8.9.0-SNAPSHOT",
+			rightVersion: "8.9.0-er1",
+			less:         false,
+		},
+		{
+			name:         "2 prereleases have no specific order, reversed",
+			leftVersion:  "8.9.0-er1",
+			rightVersion: "8.9.0-SNAPSHOT",
+			less:         false,
+		},
+		{
+			name:         "build metadata have no influence on precedence",
+			leftVersion:  "8.9.0-SNAPSHOT+aaaaaa",
+			rightVersion: "8.9.0-SNAPSHOT+bbbbbb",
+			less:         false,
+		},
+		{
+			name:         "build metadata have no influence on precedence, reversed",
+			leftVersion:  "8.9.0-SNAPSHOT+bbbbbb",
+			rightVersion: "8.9.0-SNAPSHOT+aaaaaa",
+			less:         false,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			left, err := ParseVersion(tc.leftVersion)
+			require.NoError(t, err)
+			require.NotNil(t, left)
+			right, err := ParseVersion(tc.rightVersion)
+			require.NoError(t, err)
+			require.NotNil(t, right)
+			assert.Equal(t, left.Less(*right), tc.less)
+		})
+	}
+}
+
+func TestPreviousMinor(t *testing.T) {
+	testcases := []struct {
+		name             string
+		version          string
+		prevMinorVersion string
+	}{
+		{
+			name:             "basic release version",
+			version:          "8.7.0",
+			prevMinorVersion: "8.6.0",
+		},
+		{
+			name:             "snapshot release version",
+			version:          "8.9.3-SNAPSHOT",
+			prevMinorVersion: "8.8.0-SNAPSHOT",
+		},
+		{
+			name:             "emergency release version",
+			version:          "8.9.0-er1",
+			prevMinorVersion: "8.8.0-er1",
+		},
+		{
+			name:             "previous major version",
+			version:          "8.0.0",
+			prevMinorVersion: "7.17.10",
+		},
+		{
+			name:             "previous major snapshot",
+			version:          "8.0.0-SNAPSHOT",
+			prevMinorVersion: "7.17.10-SNAPSHOT",
+		},
+		{
+			name:             "snapshot version with metadata",
+			version:          "8.9.1-SNAPSHOT+aaaaaa",
+			prevMinorVersion: "8.8.0-SNAPSHOT+aaaaaa",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			parsed, err := ParseVersion(tc.version)
+			require.NoError(t, err)
+			require.NotNil(t, parsed)
+
+			prev, err := parsed.GetPreviousMinor()
+			require.NoError(t, err)
+			require.Equal(t, tc.prevMinorVersion, prev.String())
+		})
+	}
+}
diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go
index 6ac7d0db492..5cbb9d9d7ba 100644
--- a/x-pack/elastic-agent/magefile.go
+++ b/x-pack/elastic-agent/magefile.go
@@ -11,6 +11,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -23,8 +24,11 @@ import (
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 
+	"github.com/elastic/beats/v7/dev-tools/mage"
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+	"github.com/elastic/beats/v7/dev-tools/mage/manifest"
+	"github.com/elastic/beats/v7/dev-tools/mage/version"
+	bversion "github.com/elastic/beats/v7/libbeat/version"
 
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
@@ -53,6 +57,10 @@ var Aliases = map[string]interface{}{
 	"demo":  Demo.Enroll,
 }
 
+var errNoManifest = errors.New("missing ManifestURL environment variable")
+var errNoAgentDropPath = errors.New("missing AGENT_DROP_PATH environment variable")
+var errAtLeastOnePlatform = errors.New("elastic-agent package is expected to build at least one platform package")
+
 func init() {
 	common.RegisterCheckDeps(Update, Check.All)
 	test.RegisterDeps(UnitTest)
@@ -335,6 +343,52 @@ func Ironbank() error {
 	return devtools.Ironbank()
 }
 
+func FixDRADockerArtifacts() error {
+	return devtools.FixDRADockerArtifacts()
+}
+
+// DownloadManifest downloads the provided manifest file into the predefined folder
+func DownloadManifest() error {
+	fmt.Println("--- Downloading manifest")
+	start := time.Now()
+	defer func() { fmt.Println("Downloading manifest took", time.Since(start)) }()
+
+	dropPath, found := os.LookupEnv(agentDropPath)
+
+	if !found {
+		return errNoAgentDropPath
+	}
+
+	if !devtools.PackagingFromManifest {
+		return errNoManifest
+	}
+
+	platforms := devtools.Platforms.Names()
+	if len(platforms) == 0 {
+		return errAtLeastOnePlatform
+	}
+
+	platformPackages := map[string]string{
+		"darwin/amd64":  "darwin-x86_64.tar.gz",
+		"darwin/arm64":  "darwin-aarch64.tar.gz",
+		"linux/amd64":   "linux-x86_64.tar.gz",
+		"linux/arm64":   "linux-arm64.tar.gz",
+		"windows/amd64": "windows-x86_64.zip",
+	}
+
+	var requiredPackages []string
+	for _, p := range platforms {
+		requiredPackages = append(requiredPackages, platformPackages[p])
+	}
+
+	if e := manifest.DownloadComponentsFromManifest(devtools.ManifestURL, platforms, platformPackages, dropPath); e != nil {
+		return fmt.Errorf("failed to download the manifest file, %w", e)
+	}
+	log.Printf(">> Completed downloading packages from manifest into drop-in %s", dropPath)
+
+	return nil
+}
+
 func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool {
 	for _, pkg := range requiredPackages {
 		if _, ok := os.LookupEnv(snapshotEnv); ok {
@@ -348,6 +402,7 @@ func requiredPackagesPresent(basePath, beat, version string, requiredPackages []
 			return false
 		}
 	}
+	fmt.Printf("All packages for %s are present\n", beat)
 	return true
 }
 
@@ -392,6 +447,7 @@ func Update() {
 
 // CrossBuild cross-builds the beat for all target platforms.
 func CrossBuild() error {
+	fmt.Printf(">> Crossbuild")
 	return devtools.CrossBuild()
 }
 
@@ -566,10 +622,34 @@ func runAgent(env map[string]string) error {
 }
 
 func packageAgent(requiredPackages []string, packagingFn func()) {
-	version, found := os.LookupEnv("BEAT_VERSION")
-	if !found {
-		version = release.Version()
+	fmt.Println("--- Package Elastic-Agent")
+	var packageVersion string
+	// if we have defined a manifest URL to package Agent from, we should be using the same packageVersion of that manifest
+	if devtools.PackagingFromManifest {
+		fmt.Println(">>>> Using manifest to package Agent")
+		if manifestResponse, err := manifest.DownloadManifest(devtools.ManifestURL); err != nil {
+			log.Panicf("failed to download remote manifest file %s", err)
+		} else {
+			if parsedVersion, err := version.ParseVersion(manifestResponse.Version); err != nil {
+				log.Panicf("the manifest version from manifest is not semver, got %s", manifestResponse.Version)
+			} else {
+				// When getting the packageVersion from snapshot we should also update the env of SNAPSHOT=true which is
+				// something that we use as an implicit parameter to various functions
+				if parsedVersion.IsSnapshot() {
+					os.Setenv(snapshotEnv, "true")
+					mage.Snapshot = true
+				}
+				os.Setenv("BEAT_VERSION", parsedVersion.CoreVersion())
+			}
+		}
 	}
+	if beatVersion, found := os.LookupEnv("BEAT_VERSION"); !found {
+		packageVersion = bversion.GetDefaultVersion()
+	} else {
+		packageVersion = beatVersion
+	}
+
+	fmt.Printf(">>> BEAT_VERSION: %s\n", packageVersion)
 
 	// build deps only when drop is not provided
 	if dropPathEnv, found := os.LookupEnv(agentDropPath); !found || len(dropPathEnv) == 0 {
@@ -608,7 +688,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 				panic(err)
 			}
 
-			if !requiredPackagesPresent(pwd, b, version, requiredPackages) {
+			if !requiredPackagesPresent(pwd, b, packageVersion, requiredPackages) {
 				cmd := exec.Command("mage", "package")
 				cmd.Dir = pwd
 				cmd.Stdout = os.Stdout
@@ -617,7 +697,8 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 				if envVar := selectedPackageTypes(); envVar != "" {
 					cmd.Env = append(cmd.Env, envVar)
 				}
-
+				fmt.Println(">>> Running mage package for %s\n", b)
+				fmt.Println(cmd.String())
 				if err := cmd.Run(); err != nil {
 					panic(err)
 				}

From e95c73ca62a3e17c191102bb0d7e66e04e810e1b Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 15:48:44 +0100
Subject: [PATCH 03/43] Use own images

---
 .../pipeline.xpack.elastic-agent.package.yml  | 46 +++++++++++++++++--
 .../elastic-agent/scripts/steps/package.sh    |  3 ++
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index a1b543bdb67..6b8c51ee88b 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -1,17 +1,57 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 steps:
+  - input: "Build parameters"
+    if: build.env("ManifestURL") == null
+    fields:
+      - text: "ManifestURL"
+        key: "ManifestURL"
+        default: ""
+        required: true
+        hint: "Link to the build manifest URL."
+      - select: "Mage verbose"
+        key: "MAGEFILE_VERBOSE"
+        required: false
+        options:
+          - label: "True"
+            value: "1"
+          - label: "False"
+            value: "0"
+        hint: "Increase verbosity of the mage commands, defaults to 0"
+      - select: "DRA Workflow"
+        key: "DRA_WORKFLOW"
+        required: true
+        options:
+          - label: "snapshot"
+            value: "snapshot"
+          - label: "staging"
+            value: "staging"
+        hint: "What workflow of the DRA release process this build is going to be triggered for"
+      - text: "DRA Version"
+        key: "DRA_VERSION"
+        required: true
+        default: ""
+        hint: "The packaging version to use"
+      - select: "DRA DRY-RUN"
+        key: "DRA_DRY_RUN"
+        required: false
+        options:
+          - label: "True"
+            value: "--dry-run"
+          - label: "False"
+            value: ""
+        hint: "If the DRA release manager script would actually publish anything or just print"
+
   - label: "Package elastic-agent"
     env:
       AGENT_DROP_PATH: "build/distributions/elastic-agent-drop"
     key: "package-full"
     command: ".buildkite/x-pack/elastic-agent/scripts/steps/package.sh"
     artifact_paths:
-      - "build/elastic-agent-drop/**/*"
+      - "build/distributions/**/*"
     agents:
-      # TODO: use a container
       provider: "gcp"
-      image: "family/core-ubuntu-2204"
+      image: "family/platform-ingest-beats-ubuntu-2204"
     retry:
       manual:
         allowed: true
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
index 396844d70ce..8e1e6b4a638 100644
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -10,4 +10,7 @@ fi
 VERSION="$(make get-version)"
 echo "--- Packaging Elastic Agent"
 
+export AGENT_DROP_PATH=build/elastic-agent-drop
+mkdir -p $AGENT_DROP_PATH
+
 mage -v -d x-pack/elastic-agent clean downloadManifest package ironbank fixDRADockerArtifacts

From 6261bdc3c1b047f77417b7696048a08e67b324fa Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 16:46:40 +0100
Subject: [PATCH 04/43] Set mage verison

---
 .../elastic-agent/scripts/steps/package.sh    |  1 +
 .../elastic-agent/scripts/steps/setenv.sh     | 56 +++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 .buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
index 8e1e6b4a638..58ee789f32e 100644
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -uo pipefail
+source ".buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh"
 source ".buildkite/x-pack/elastic-agent/scripts/steps/common.sh"
 
 if test -z "${ManifestURL=:""}"; then
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh
new file mode 100644
index 00000000000..1af666be983
--- /dev/null
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SETUP_GVM_VERSION="v0.5.1"
+DOCKER_COMPOSE_VERSION="1.21.0"
+DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0"
+SETUP_WIN_PYTHON_VERSION="3.11.0"
+NMAP_WIN_VERSION="7.12"           # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
+GO_VERSION=$(cat .go-version)
+ASDF_MAGE_VERSION="1.15.0"
+PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+PACKAGING_ARM_PLATFORMS="linux/arm64"
+
+export SETUP_GVM_VERSION
+export DOCKER_COMPOSE_VERSION
+export DOCKER_COMPOSE_VERSION_AARCH64
+export SETUP_WIN_PYTHON_VERSION
+export NMAP_WIN_VERSION
+export GO_VERSION
+export ASDF_MAGE_VERSION
+export PACKAGING_PLATFORMS
+export PACKAGING_ARM_PLATFORMS
+
+exportVars() {
+  local platform_type="$(uname)"
+  local arch_type="$(uname -m)"
+  if [ "${arch_type}" == "x86_64" ]; then
+    case "${platform_type}" in
+      Linux|Darwin)
+        export GOX_FLAGS="-arch amd64"
+        export testResults="**/build/TEST*.xml"
+        export artifacts="**/build/TEST*.out"
+        ;;
+      MINGW*)
+        export GOX_FLAGS="-arch 386"
+        export testResults="**\\build\\TEST*.xml"
+        export artifacts="**\\build\\TEST*.out"
+        ;;
+    esac
+  elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then
+    export GOX_FLAGS="-arch arm"
+    export testResults="**/build/TEST*.xml"
+    export artifacts="**/build/TEST*.out"
+  else
+    echo "Unsupported OS"
+  fi
+}
+
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+  exportVars
+  export RACE_DETECTOR="true"
+  export TEST_COVERAGE="true"
+  export DOCKER_PULL="0"
+fi
\ No newline at end of file

From eccc68f896b5c94c7ca19ea74628df57f0507b03 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 16:50:49 +0100
Subject: [PATCH 05/43] Set mage verison

---
 .buildkite/x-pack/elastic-agent/scripts/steps/package.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
index 58ee789f32e..a29d681b8cc 100644
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -3,6 +3,7 @@ set -uo pipefail
 source ".buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh"
 source ".buildkite/x-pack/elastic-agent/scripts/steps/common.sh"
 
+set -x
 if test -z "${ManifestURL=:""}"; then
   echo "Missing variable ManifestURL, export it before use."
   exit 2

From 7766e8c9d1a10be3b3a406ddcb1a1f217d4ff982 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 17:21:25 +0100
Subject: [PATCH 06/43] Added wait step

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 6b8c51ee88b..4ca826f2a19 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -42,6 +42,9 @@ steps:
             value: ""
         hint: "If the DRA release manager script would actually publish anything or just print"
 
+  - wait: ~
+    if: build.env("ManifestURL") == null
+
   - label: "Package elastic-agent"
     env:
       AGENT_DROP_PATH: "build/distributions/elastic-agent-drop"

From d4236614ab0b67808832501c3a3acae69dd80728 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 17:30:19 +0100
Subject: [PATCH 07/43] Added wait step

---
 .../pipeline.xpack.elastic-agent.package.yml        | 13 ++++++++++++-
 .../x-pack/elastic-agent/scripts/steps/package.sh   |  2 ++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 4ca826f2a19..db52a267bf7 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -49,7 +49,18 @@ steps:
     env:
       AGENT_DROP_PATH: "build/distributions/elastic-agent-drop"
     key: "package-full"
-    command: ".buildkite/x-pack/elastic-agent/scripts/steps/package.sh"
+    command: |
+      if [[ -z "${ManifestURL}" ]]; then
+            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
+            if [[ -z "${ManifestURL}" ]]; then
+              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+              exit 1
+            fi
+          fi
+          if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+            export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+          fi
+      .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
     artifact_paths:
       - "build/distributions/**/*"
     agents:
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
index a29d681b8cc..0d6848e481a 100644
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -12,6 +12,8 @@ fi
 VERSION="$(make get-version)"
 echo "--- Packaging Elastic Agent"
 
+echo $ManifestURL
+
 export AGENT_DROP_PATH=build/elastic-agent-drop
 mkdir -p $AGENT_DROP_PATH
 

From 7014d5774ced76c2ad9f14eef7e9c51e8f043bfb Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 17:42:36 +0100
Subject: [PATCH 08/43] Added wait step

---
 .../pipeline.xpack.elastic-agent.package.yml  | 20 +++++++++----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index db52a267bf7..a4878862bf9 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -46,20 +46,18 @@ steps:
     if: build.env("ManifestURL") == null
 
   - label: "Package elastic-agent"
-    env:
-      AGENT_DROP_PATH: "build/distributions/elastic-agent-drop"
     key: "package-full"
     command: |
       if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
-              exit 1
-            fi
-          fi
-          if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
-            export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
-          fi
+        export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
+        if [[ -z "${ManifestURL}" ]]; then
+          echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          exit 1
+        fi
+      fi
+      if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+        export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+      fi
       .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
     artifact_paths:
       - "build/distributions/**/*"

From 750460c6114c3ebafc9439a67cc599867dd2f7f8 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 22 Feb 2024 19:15:44 +0100
Subject: [PATCH 09/43] Added package group

---
 .../pipeline.xpack.elastic-agent.package.yml  | 47 ++++++++++---------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index a4878862bf9..eaab54c4f24 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -45,25 +45,28 @@ steps:
   - wait: ~
     if: build.env("ManifestURL") == null
 
-  - label: "Package elastic-agent"
-    key: "package-full"
-    command: |
-      if [[ -z "${ManifestURL}" ]]; then
-        export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-        if [[ -z "${ManifestURL}" ]]; then
-          echo ":broken_heart: Missing ManifestURL variable or empty string provided"
-          exit 1
-        fi
-      fi
-      if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
-        export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
-      fi
-      .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
-    artifact_paths:
-      - "build/distributions/**/*"
-    agents:
-      provider: "gcp"
-      image: "family/platform-ingest-beats-ubuntu-2204"
-    retry:
-      manual:
-        allowed: true
+  - group: ":Packaging Artefacts"
+    key: "package"
+    steps:
+      - label: "Package elastic-agent"
+        key: "package-full"
+        command: |
+          if [[ -z "${ManifestURL}" ]]; then
+            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
+            if [[ -z "${ManifestURL}" ]]; then
+              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+              exit 1
+            fi
+          fi
+          if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+            export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+          fi
+          .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+        artifact_paths:
+          - "build/distributions/**/*"
+        agents:
+          provider: "gcp"
+          image: "family/platform-ingest-beats-ubuntu-2204"
+        retry:
+          manual:
+            allowed: true

From 1091f85c5c1e4c73188174e421d56f9f8e3f35f8 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 11:44:16 +0100
Subject: [PATCH 10/43] Added package group

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index eaab54c4f24..2f97358716e 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -51,14 +51,15 @@ steps:
       - label: "Package elastic-agent"
         key: "package-full"
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
+          if [[ -z "$${ManifestURL}" ]]; then
             export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
+            if [[ -z "$${ManifestURL}" ]]; then
               echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+              echo "Provided ManifestURL: ${ManifestURL}"
               exit 1
             fi
           fi
-          if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+          if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
             export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
           fi
           .buildkite/x-pack/elastic-agent/scripts/steps/package.sh

From 64c72fd797e938f709ecd95b3b7348f37f26fb86 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 11:49:57 +0100
Subject: [PATCH 11/43] Added package group

---
 .buildkite/x-pack/elastic-agent/scripts/steps/package.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 .buildkite/x-pack/elastic-agent/scripts/steps/package.sh

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
old mode 100644
new mode 100755

From b15e604b23b36b32bd78a27d869570241aec553f Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 14:24:07 +0100
Subject: [PATCH 12/43] Added package group

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 2f97358716e..9c28993d0a2 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -63,6 +63,7 @@ steps:
             export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
           fi
           .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+          ls -lah build/distributions/
         artifact_paths:
           - "build/distributions/**/*"
         agents:

From 127c0eeb876912f6585901c76cb18d2c9d6a3819 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 15:14:30 +0100
Subject: [PATCH 13/43] Added package group

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 9c28993d0a2..f01f39fb792 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -63,7 +63,8 @@ steps:
             export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
           fi
           .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
-          ls -lah build/distributions/
+          ls -lah build/distributions/ || echo "not found"
+          ls -lah x-pack/elastic-agent/build/distributions/ || echo "not found"
         artifact_paths:
           - "build/distributions/**/*"
         agents:

From 1110f2a45d6292dda2986b9fa5b5235f305aeffc Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 15:31:44 +0100
Subject: [PATCH 14/43] Added package group

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index f01f39fb792..d1ef7f20e82 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -66,7 +66,7 @@ steps:
           ls -lah build/distributions/ || echo "not found"
           ls -lah x-pack/elastic-agent/build/distributions/ || echo "not found"
         artifact_paths:
-          - "build/distributions/**/*"
+          - "x-pack/elastic-agent/build/distributions/**/*"
         agents:
           provider: "gcp"
           image: "family/platform-ingest-beats-ubuntu-2204"

From dec06a64d1d3bf2f44a068d0226d9985ba1a6134 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 15:40:01 +0100
Subject: [PATCH 15/43] Added dependencies report

---
 .buildkite/x-pack/elastic-agent/scripts/steps/package.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
index 0d6848e481a..a9c0d3763de 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/package.sh
@@ -3,6 +3,8 @@ set -uo pipefail
 source ".buildkite/x-pack/elastic-agent/scripts/steps/setenv.sh"
 source ".buildkite/x-pack/elastic-agent/scripts/steps/common.sh"
 
+DIST_PATH="x-pack/elastic-agent/build/distributions"
+
 set -x
 if test -z "${ManifestURL=:""}"; then
   echo "Missing variable ManifestURL, export it before use."
@@ -18,3 +20,9 @@ export AGENT_DROP_PATH=build/elastic-agent-drop
 mkdir -p $AGENT_DROP_PATH
 
 mage -v -d x-pack/elastic-agent clean downloadManifest package ironbank fixDRADockerArtifacts
+
+echo  "+++ Generate dependencies report"
+BEAT_VERSION_FULL=$(curl -s -XGET "${ManifestURL}" |jq '.version' -r )
+bash dev-tools/dependencies-report
+mkdir -p $DIST_PATH/reports
+mv dependencies.csv "$DIST_PATH/reports/dependencies-${BEAT_VERSION_FULL}.csv"
\ No newline at end of file

From aadc408d99fa4ad96f3fdb24beae4c7338f8bd91 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 16:11:17 +0100
Subject: [PATCH 16/43] publish step

---
 .../pipeline.xpack.elastic-agent.package.yml  | 27 ++++++
 .../scripts/steps/dra-publish.sh              | 92 +++++++++++++++++++
 2 files changed, 119 insertions(+)
 create mode 100644 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index d1ef7f20e82..6c433091acb 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -73,3 +73,30 @@ steps:
         retry:
           manual:
             allowed: true
+
+  - label: ":elastic-stack: Publishing to DRA"
+    key: dra-publish
+    depends_on: package
+    agents:
+      provider: "gcp"
+    env:
+      DRA_PROJECT_ID: "elastic-agent-package"
+      DRA_PROJECT_ARTIFACT_ID: "agent-package"
+    command: |
+      echo "+++ Restoring Artifacts"
+      buildkite-agent artifact download "build/**/*" .
+      echo "+++ Changing permissions for the release manager"
+      sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
+      echo "+++ Running DRA publish step"
+      if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
+        export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+      fi
+      if [[ -z "${DRA_DRY_RUN}" ]]; then
+        DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "")
+        export DRA_DRY_RUN
+      fi
+      if [[ -z "${DRA_VERSION}" ]]; then
+        DRA_VERSION=$(buildkite-agent meta-data get DRA_VERSION --default "")
+        export DRA_VERSION
+      fi
+      .buildkite/scripts/steps/dra-publish.sh
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
new file mode 100644
index 00000000000..136de358d70
--- /dev/null
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -uo pipefail
+
+DRY_RUN="${DRA_DRY_RUN:=""}"
+WORKFLOW="${DRA_WORKFLOW:=""}"
+COMMIT="${DRA_COMMIT:="${BUILDKITE_COMMIT:=""}"}"
+BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
+PACKAGE_VERSION="${DRA_VERSION:="${BEAT_VERSION:=""}"}"
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+# force main branch on PR's or it won't execute
+# because the PR branch does not have a project folder in release-manager
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then
+    BRANCH=main
+    DRY_RUN="--dry-run"
+    echo "+++ Running in PR and setting branch main and --dry-run"
+fi
+
+if [[ -z "${WORKFLOW}" ]]; then
+  echo "+++ Missing DRA workflow";
+  exit 1
+fi
+if [[ -z "${COMMIT:-""}" ]]; then
+  echo "+++ Missing DRA_COMMIT";
+  exit 1
+fi
+if [[ -z "${PACKAGE_VERSION:-""}" ]]; then
+  echo "+++ Missing DRA_VERSION";
+  exit 1
+fi
+if [[ -z "${BRANCH:-""}" ]]; then
+  echo "+++ Missing DRA_BRANCH";
+  exit 1
+fi
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+}
+
+# Listing Release manager
+function run_release_manager_list() {
+    local _project_id="${1}" _artifact_set="${2}" _workflow="${3}" _commit="${4}" _branch="${5}" _version="${6}"
+    echo "+++ :hammer_and_pick: Release manager listing ${_branch} ${_workflow} DRA artifacts..."
+    docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli list \
+        --project "${_project_id}" \
+        --branch "${_branch}" \
+        --commit "${_commit}" \
+        --workflow "${_workflow}" \
+        --version "${_version}" \
+        --artifact-set "${_artifact_set}"
+}
+
+# Publish DRA artifacts
+function run_release_manager_collect() {
+    local _project_id="${1}" _artifact_set="${2}" _workflow="${3}" _commit="${4}" _branch="${5}" _version="${6}" _dry_run="${7}"
+    echo "+++ :hammer_and_pick: Publishing ${_branch} ${_workflow} DRA artifacts..."
+    docker run --rm \
+        --name release-manager \
+        -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+        -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+        -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+        cli collect \
+        --project "${_project_id}" \
+        --branch "${_branch}" \
+        --commit "${_commit}" \
+        --workflow "${_workflow}" \
+        --version "${_version}" \
+        --artifact-set "${_artifact_set}" \
+        ${_dry_run}
+}
+
+echo "+++ Release Manager ${WORKFLOW} / ${BRANCH} / ${COMMIT}";
+release_manager_login
+run_release_manager_list "${DRA_PROJECT_ID}" "${DRA_PROJECT_ARTIFACT_ID}" "${WORKFLOW}" "${COMMIT}" "${BRANCH}" "${PACKAGE_VERSION}"
+run_release_manager_collect "${DRA_PROJECT_ID}" "${DRA_PROJECT_ARTIFACT_ID}" "${WORKFLOW}" "${COMMIT}" "${BRANCH}" "${PACKAGE_VERSION}" "${DRY_RUN}"
+RM_EXIT_CODE=$?
+
+exit $RM_EXIT_CODE

From 6907a88dd83de7a8376e6632370672b53ff0a825 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 17:16:11 +0100
Subject: [PATCH 17/43] publish step

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 6c433091acb..2bde445dc6c 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -84,9 +84,9 @@ steps:
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
     command: |
       echo "+++ Restoring Artifacts"
-      buildkite-agent artifact download "build/**/*" .
+      buildkite-agent artifact download "pack/elastic-agent/build/**/*" ./build/
       echo "+++ Changing permissions for the release manager"
-      sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
+      sudo chown -R :1000 build/distributions/
       echo "+++ Running DRA publish step"
       if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
         export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")

From a6a31acec14eac0978489cb3de670d7c70f3e346 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 17:17:09 +0100
Subject: [PATCH 18/43] publish step

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 2bde445dc6c..e882bf9c766 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -84,7 +84,7 @@ steps:
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
     command: |
       echo "+++ Restoring Artifacts"
-      buildkite-agent artifact download "pack/elastic-agent/build/**/*" ./build/
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" ./build/
       echo "+++ Changing permissions for the release manager"
       sudo chown -R :1000 build/distributions/
       echo "+++ Running DRA publish step"

From 41aa643f7350bdbad30dc8092b42a213830cbe24 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Fri, 23 Feb 2024 18:00:46 +0100
Subject: [PATCH 19/43] publish step

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index e882bf9c766..87c44bb6e91 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -84,6 +84,7 @@ steps:
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
     command: |
       echo "+++ Restoring Artifacts"
+      mkdir build 
       buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" ./build/
       echo "+++ Changing permissions for the release manager"
       sudo chown -R :1000 build/distributions/

From f626acbcb1404ff0111543ba862303513c9fede4 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 4 Mar 2024 15:29:06 +0100
Subject: [PATCH 20/43] Fixed artifacts paths

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml    | 4 ++--
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 87c44bb6e91..cb5a766613a 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -85,9 +85,9 @@ steps:
     command: |
       echo "+++ Restoring Artifacts"
       mkdir build 
-      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" ./build/
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" .
       echo "+++ Changing permissions for the release manager"
-      sudo chown -R :1000 build/distributions/
+      sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
       echo "+++ Running DRA publish step"
       if [[ -z "${MAGEFILE_VERBOSE}" ]]; then
         export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index 136de358d70..21efa932170 100644
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -51,7 +51,7 @@ function run_release_manager_list() {
         -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
         -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
-        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
         cli list \
         --project "${_project_id}" \
@@ -71,7 +71,7 @@ function run_release_manager_collect() {
         -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
         -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
-        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
         cli collect \
         --project "${_project_id}" \

From 8637c8a96a957029ba5a046fe6f05686a9b191bb Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 4 Mar 2024 16:00:58 +0100
Subject: [PATCH 21/43] Fixed artifacts paths

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index cb5a766613a..b9b8494d235 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -100,4 +100,4 @@ steps:
         DRA_VERSION=$(buildkite-agent meta-data get DRA_VERSION --default "")
         export DRA_VERSION
       fi
-      .buildkite/scripts/steps/dra-publish.sh
+      .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh

From 717342f781e0acb30a8c9108229649fe93f3de24 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 4 Mar 2024 18:15:04 +0100
Subject: [PATCH 22/43] Fixed artifacts paths

---
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
old mode 100644
new mode 100755

From 87cd526bb1ee49282b64d2d365b0800c3e91a23a Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 5 Mar 2024 00:41:34 +0100
Subject: [PATCH 23/43] test: use artifacts from previous build

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index b9b8494d235..2a67ce1c434 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -47,6 +47,8 @@ steps:
 
   - group: ":Packaging Artefacts"
     key: "package"
+    # todo: remove
+    if: "false"
     steps:
       - label: "Package elastic-agent"
         key: "package-full"
@@ -82,10 +84,11 @@ steps:
     env:
       DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
+      # TODO: remove --build
     command: |
       echo "+++ Restoring Artifacts"
-      mkdir build 
-      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" .
+      mkdir build       
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" --build 018e09fc-f390-4b59-9bda-afbdc7602bcc .
       echo "+++ Changing permissions for the release manager"
       sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
       echo "+++ Running DRA publish step"

From 8fedace7c600585719d4b6538262573b6026e818 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 5 Mar 2024 00:54:18 +0100
Subject: [PATCH 24/43] test: use artifacts from previous build

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml    | 4 ++++
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh  | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 2a67ce1c434..973683f2524 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -103,4 +103,8 @@ steps:
         DRA_VERSION=$(buildkite-agent meta-data get DRA_VERSION --default "")
         export DRA_VERSION
       fi
+      if [[ -z "${DRA_WORKFLOW}" ]]; then
+        DRA_WORKFLOW=$(buildkite-agent meta-data get DRA_WORKFLOW --default "")
+        export DRA_VERSION
+      fi      
       .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index 21efa932170..86cfa48beaa 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -12,9 +12,9 @@ CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 # force main branch on PR's or it won't execute
 # because the PR branch does not have a project folder in release-manager
 if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then
-    BRANCH=main
+    BRANCH=7.17
     DRY_RUN="--dry-run"
-    echo "+++ Running in PR and setting branch main and --dry-run"
+    echo "+++ Running in PR and setting branch 7.17 and --dry-run"
 fi
 
 if [[ -z "${WORKFLOW}" ]]; then

From 261d9dd3752fe46d5483128f4e1f7882b6eb633f Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Tue, 5 Mar 2024 00:59:14 +0100
Subject: [PATCH 25/43] test: use artifacts from previous build

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 973683f2524..d695a8b93bd 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -105,6 +105,6 @@ steps:
       fi
       if [[ -z "${DRA_WORKFLOW}" ]]; then
         DRA_WORKFLOW=$(buildkite-agent meta-data get DRA_WORKFLOW --default "")
-        export DRA_VERSION
+        export DRA_WORKFLOW
       fi      
       .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh

From 682bc53e7effb71226085937a1128dbfe2eca98d Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 6 Mar 2024 01:41:52 +0100
Subject: [PATCH 26/43] arm packaging

---
 .../pipeline.xpack.elastic-agent.package.yml  | 35 +++++++++++++------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index d695a8b93bd..82b928f6283 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -47,11 +47,9 @@ steps:
 
   - group: ":Packaging Artefacts"
     key: "package"
-    # todo: remove
-    if: "false"
     steps:
       - label: "Package elastic-agent"
-        key: "package-full"
+        key: package_elastic-agent
         command: |
           if [[ -z "$${ManifestURL}" ]]; then
             export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
@@ -65,17 +63,32 @@ steps:
             export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
           fi
           .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
-          ls -lah build/distributions/ || echo "not found"
-          ls -lah x-pack/elastic-agent/build/distributions/ || echo "not found"
         artifact_paths:
           - "x-pack/elastic-agent/build/distributions/**/*"
         agents:
           provider: "gcp"
           image: "family/platform-ingest-beats-ubuntu-2204"
-        retry:
-          manual:
-            allowed: true
-
+      - label: "Package ARM elastic-agent"
+        key: package_elastic-agent-arm
+        command: |
+          if [[ -z "$${ManifestURL}" ]]; then
+            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
+            if [[ -z "$${ManifestURL}" ]]; then
+              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+              echo "Provided ManifestURL: ${ManifestURL}"
+              exit 1
+            fi
+          fi
+          if [[ -z "$${MAGEFILE_VERBOSE}" ]]; then
+            export MAGEFILE_VERBOSE=$(buildkite-agent meta-data get MAGEFILE_VERBOSE --default "0")
+          fi
+          .buildkite/x-pack/elastic-agent/scripts/steps/package.sh
+        artifact_paths:
+          - "x-pack/elastic-agent/build/distributions/**/*"
+        agents:
+          provider: "gcp"
+          image: "family/platform-ingest-beats-ubuntu-2004-aarch64"
+  #
   - label: ":elastic-stack: Publishing to DRA"
     key: dra-publish
     depends_on: package
@@ -84,11 +97,11 @@ steps:
     env:
       DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
-      # TODO: remove --build
+      # TODO: remove --build 018e09fc-f390-4b59-9bda-afbdc7602bcc
     command: |
       echo "+++ Restoring Artifacts"
       mkdir build       
-      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" --build 018e09fc-f390-4b59-9bda-afbdc7602bcc .
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" .
       echo "+++ Changing permissions for the release manager"
       sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
       echo "+++ Running DRA publish step"

From bb46afd79ced920924ff1a2bfb26e4ff4418a519 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 6 Mar 2024 02:17:17 +0100
Subject: [PATCH 27/43] arm packaging

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 82b928f6283..3917b11ef18 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -86,7 +86,7 @@ steps:
         artifact_paths:
           - "x-pack/elastic-agent/build/distributions/**/*"
         agents:
-          provider: "gcp"
+          provider: "aws"
           image: "family/platform-ingest-beats-ubuntu-2004-aarch64"
   #
   - label: ":elastic-stack: Publishing to DRA"

From e1f3976aa3d99b9a9b99c4c4d10a47a4f6419638 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 6 Mar 2024 02:28:11 +0100
Subject: [PATCH 28/43] arm packaging

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 3917b11ef18..f3e48980cda 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -87,7 +87,7 @@ steps:
           - "x-pack/elastic-agent/build/distributions/**/*"
         agents:
           provider: "aws"
-          image: "family/platform-ingest-beats-ubuntu-2004-aarch64"
+          imagePrefix: "platform-ingest-beats-ubuntu-2004-aarch64"
   #
   - label: ":elastic-stack: Publishing to DRA"
     key: dra-publish

From d5c2e269825b2a7b051eebffdcc0fe92e0229a95 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 6 Mar 2024 02:38:02 +0100
Subject: [PATCH 29/43] arm packaging

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index f3e48980cda..34a4646ba60 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -87,6 +87,7 @@ steps:
           - "x-pack/elastic-agent/build/distributions/**/*"
         agents:
           provider: "aws"
+          instanceType: "t4g.2xlarge"
           imagePrefix: "platform-ingest-beats-ubuntu-2004-aarch64"
   #
   - label: ":elastic-stack: Publishing to DRA"

From 643e3daf9bfcc050f5d88165017415bc5aec70fc Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Wed, 6 Mar 2024 03:16:23 +0100
Subject: [PATCH 30/43] arm packaging

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 34a4646ba60..d96bb407a9f 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -69,6 +69,9 @@ steps:
           provider: "gcp"
           image: "family/platform-ingest-beats-ubuntu-2204"
       - label: "Package ARM elastic-agent"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
         key: package_elastic-agent-arm
         command: |
           if [[ -z "$${ManifestURL}" ]]; then

From 3a07c727a94a5c06bb819b4395219d02049a1664 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 7 Mar 2024 12:16:21 +0100
Subject: [PATCH 31/43] arm packaging

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index d96bb407a9f..ba401edf4f9 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -50,6 +50,7 @@ steps:
     steps:
       - label: "Package elastic-agent"
         key: package_elastic-agent
+        if: "false"
         command: |
           if [[ -z "$${ManifestURL}" ]]; then
             export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
@@ -69,6 +70,7 @@ steps:
           provider: "gcp"
           image: "family/platform-ingest-beats-ubuntu-2204"
       - label: "Package ARM elastic-agent"
+        if: "false"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
@@ -101,11 +103,11 @@ steps:
     env:
       DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
-      # TODO: remove --build 018e09fc-f390-4b59-9bda-afbdc7602bcc
+      # TODO: remove --build 018e118c-7f65-478b-a519-608df4a59479
     command: |
       echo "+++ Restoring Artifacts"
       mkdir build       
-      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" .
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" --build 018e118c-7f65-478b-a519-608df4a59479 .
       echo "+++ Changing permissions for the release manager"
       sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
       echo "+++ Running DRA publish step"

From f12e95c983ba81e854da25d9ff6d4eb035d38a3d Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 7 Mar 2024 15:48:11 +0100
Subject: [PATCH 32/43] Added branch exclusion

---
 .../x-pack/elastic-agent/scripts/steps/dra-publish.sh       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index 86cfa48beaa..147e8154ee2 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -11,12 +11,14 @@ CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
 # force main branch on PR's or it won't execute
 # because the PR branch does not have a project folder in release-manager
-if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ] || "${BUILDKITE_BRANCH}" == "packaging_elastic_agent" ]; then
     BRANCH=7.17
     DRY_RUN="--dry-run"
-    echo "+++ Running in PR and setting branch 7.17 and --dry-run"
+    echo "+++ Running in PR or test branch and setting branch 7.17 and --dry-run"
 fi
 
+
+
 if [[ -z "${WORKFLOW}" ]]; then
   echo "+++ Missing DRA workflow";
   exit 1

From 6e0efc257a7719ef4b3d19a704731012ded5f925 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 7 Mar 2024 15:53:28 +0100
Subject: [PATCH 33/43] Added branch exclusion

---
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index 147e8154ee2..ab295165998 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -11,7 +11,7 @@ CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
 # force main branch on PR's or it won't execute
 # because the PR branch does not have a project folder in release-manager
-if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ] || "${BUILDKITE_BRANCH}" == "packaging_elastic_agent" ]; then
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "packaging_elastic_agent" ]]; then
     BRANCH=7.17
     DRY_RUN="--dry-run"
     echo "+++ Running in PR or test branch and setting branch 7.17 and --dry-run"

From a5dd15aca62638f877c1c3801378c0a90a8eb932 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 7 Mar 2024 16:17:13 +0100
Subject: [PATCH 34/43] Added branch exclusion

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index ba401edf4f9..8868ef1d23c 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -101,7 +101,7 @@ steps:
     agents:
       provider: "gcp"
     env:
-      DRA_PROJECT_ID: "elastic-agent-package"
+      DRA_PROJECT_ID: "elastic_agent_package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
       # TODO: remove --build 018e118c-7f65-478b-a519-608df4a59479
     command: |

From c577cb9f62e75bc73e3c2f7d28f1f73a45406e9a Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 7 Mar 2024 17:38:12 +0100
Subject: [PATCH 35/43] Added branch exclusion

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index 8868ef1d23c..ba401edf4f9 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -101,7 +101,7 @@ steps:
     agents:
       provider: "gcp"
     env:
-      DRA_PROJECT_ID: "elastic_agent_package"
+      DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
       # TODO: remove --build 018e118c-7f65-478b-a519-608df4a59479
     command: |

From 0a33699b181c2ece1f11a2d82e816904fa8f7894 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 11 Mar 2024 15:16:08 +0100
Subject: [PATCH 36/43] Fixed artifacts mout path

---
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index ab295165998..d11179c0c2c 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -53,7 +53,7 @@ function run_release_manager_list() {
         -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
         -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
-        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent",target=/artifacts \
+        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent/build/distributions/",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
         cli list \
         --project "${_project_id}" \

From 07be82cae3d6fd20f279958358ee3953ec122d44 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 11 Mar 2024 15:30:24 +0100
Subject: [PATCH 37/43] Fixed artifacts mout path

---
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index d11179c0c2c..fecb7f13eac 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -55,7 +55,7 @@ function run_release_manager_list() {
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
         --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent/build/distributions/",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
-        cli list \
+        ls -lah /artifacts || cli list \
         --project "${_project_id}" \
         --branch "${_branch}" \
         --commit "${_commit}" \

From 4e8cd40a7f90d2b8a6f184056633fe000af431df Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 11 Mar 2024 15:34:21 +0100
Subject: [PATCH 38/43] Fixed artifacts mout path

---
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index fecb7f13eac..47d74b38173 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -55,7 +55,7 @@ function run_release_manager_list() {
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
         --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent/build/distributions/",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
-        ls -lah /artifacts || cli list \
+        cli list \
         --project "${_project_id}" \
         --branch "${_branch}" \
         --commit "${_commit}" \
@@ -73,7 +73,7 @@ function run_release_manager_collect() {
         -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
         -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
-        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent",target=/artifacts \
+        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent/build/distributions/",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
         cli collect \
         --project "${_project_id}" \

From 8374cd1a1cfcea7fda71c53f5d00666a744207b6 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 11 Mar 2024 15:40:53 +0100
Subject: [PATCH 39/43] Fixed artifacts mout path

---
 .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index 47d74b38173..2ca9c7f4f15 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -53,7 +53,7 @@ function run_release_manager_list() {
         -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
         -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
-        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent/build/distributions/",target=/artifacts \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
         cli list \
         --project "${_project_id}" \
@@ -73,7 +73,7 @@ function run_release_manager_collect() {
         -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
         -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
         -e VAULT_SECRET_ID="${VAULT_SECRET}" \
-        --mount type=bind,readonly=false,src="${PWD}/x-pack/elastic-agent/build/distributions/",target=/artifacts \
+        --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
         docker.elastic.co/infra/release-manager:latest \
         cli collect \
         --project "${_project_id}" \

From 67e72b5cf2ac621311760cb9be948d6f5e649215 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Mon, 11 Mar 2024 17:47:27 +0100
Subject: [PATCH 40/43] cleanup

---
 .../elastic-agent/pipeline.xpack.elastic-agent.package.yml  | 6 ++----
 .../x-pack/elastic-agent/scripts/steps/dra-publish.sh       | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index ba401edf4f9..e3320557bc6 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -50,7 +50,6 @@ steps:
     steps:
       - label: "Package elastic-agent"
         key: package_elastic-agent
-        if: "false"
         command: |
           if [[ -z "$${ManifestURL}" ]]; then
             export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
@@ -69,8 +68,8 @@ steps:
         agents:
           provider: "gcp"
           image: "family/platform-ingest-beats-ubuntu-2204"
+
       - label: "Package ARM elastic-agent"
-        if: "false"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
@@ -103,11 +102,10 @@ steps:
     env:
       DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"
-      # TODO: remove --build 018e118c-7f65-478b-a519-608df4a59479
     command: |
       echo "+++ Restoring Artifacts"
       mkdir build       
-      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" --build 018e118c-7f65-478b-a519-608df4a59479 .
+      buildkite-agent artifact download "x-pack/elastic-agent/build/**/*" .
       echo "+++ Changing permissions for the release manager"
       sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
       echo "+++ Running DRA publish step"
diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
index 2ca9c7f4f15..dc211189aff 100755
--- a/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+++ b/.buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
@@ -11,7 +11,7 @@ CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
 # force main branch on PR's or it won't execute
 # because the PR branch does not have a project folder in release-manager
-if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" || "${BUILDKITE_BRANCH}" == "packaging_elastic_agent" ]]; then
+if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then
     BRANCH=7.17
     DRY_RUN="--dry-run"
     echo "+++ Running in PR or test branch and setting branch 7.17 and --dry-run"

From 6dc5e06f5bf105d64b136832906681829cac96d0 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 14 Mar 2024 18:58:21 +0100
Subject: [PATCH 41/43] Fixes

---
 .../pipeline.xpack.elastic-agent.package.yml  |  2 +-
 dev-tools/mage/artifacts/artifacts_api.go     | 21 ++++++++-----------
 dev-tools/mage/crossbuild.go                  |  4 ++--
 dev-tools/mage/manifest/manifest.go           |  5 +++--
 dev-tools/mage/manifest/manifestspecs.go      | 10 +++------
 dev-tools/mage/settings.go                    |  2 +-
 dev-tools/mage/version/version_parser.go      |  3 +--
 7 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
index e3320557bc6..ff81812043f 100644
--- a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
+++ b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.package.yml
@@ -17,7 +17,7 @@ steps:
             value: "1"
           - label: "False"
             value: "0"
-        hint: "Increase verbosity of the mage commands, defaults to 0"
+        hint: "Increase verbosity of the mage commands, defaults to False"
       - select: "DRA Workflow"
         key: "DRA_WORKFLOW"
         required: true
diff --git a/dev-tools/mage/artifacts/artifacts_api.go b/dev-tools/mage/artifacts/artifacts_api.go
index 2c45a67cebb..93e244e9280 100644
--- a/dev-tools/mage/artifacts/artifacts_api.go
+++ b/dev-tools/mage/artifacts/artifacts_api.go
@@ -146,16 +146,15 @@ func NewArtifactAPIClient(opts ...ArtifactAPIClientOpt) *ArtifactAPIClient {
 func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList, err error) {
 	joinedURL, err := aac.composeURL(artifactsAPIV1VersionsEndpoint)
 	if err != nil {
-		return
+		return nil, fmt.Errorf("couldn't compose URL: %w", err)
 	}
 
 	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
+	defer resp.Body.Close()
 	if err != nil {
-		err = fmt.Errorf("getting versions: %w", err)
-		return
+		return nil, fmt.Errorf("getting versions: %w", err)
 	}
 
-	defer resp.Body.Close()
 	return checkResponseAndUnmarshal[VersionList](resp)
 }
 
@@ -165,16 +164,15 @@ func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList
 func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version string) (builds *VersionBuilds, err error) {
 	joinedURL, err := aac.composeURL(fmt.Sprintf(artifactsAPIV1VersionBuildsEndpoint, version))
 	if err != nil {
-		return
+		return nil, fmt.Errorf("couldn't compose URL: %w", err)
 	}
 
 	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
+	defer resp.Body.Close()
 	if err != nil {
-		err = fmt.Errorf("getting builds for version %s: %w", version, err)
-		return
+		return nil, fmt.Errorf("getting builds for version %s: %w", version, err)
 	}
 
-	defer resp.Body.Close()
 	return checkResponseAndUnmarshal[VersionBuilds](resp)
 }
 
@@ -184,16 +182,15 @@ func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version st
 func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string, buildID string) (buildDetails *BuildDetails, err error) {
 	joinedURL, err := aac.composeURL(fmt.Sprintf(artifactAPIV1BuildDetailsEndpoint, version, buildID))
 	if err != nil {
-		return
+		return nil, fmt.Errorf("couldn't compose URL: %w", err)
 	}
 
 	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
+	defer resp.Body.Close()
 	if err != nil {
-		err = fmt.Errorf("getting build details for version %s buildID %s: %w", version, buildID, err)
-		return
+		return nil, fmt.Errorf("getting build details for version %s buildID %s: %w", version, buildID, err)
 	}
 
-	defer resp.Body.Close()
 	return checkResponseAndUnmarshal[BuildDetails](resp)
 }
 
diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go
index 34345d43d84..2a633afc36f 100644
--- a/dev-tools/mage/crossbuild.go
+++ b/dev-tools/mage/crossbuild.go
@@ -136,7 +136,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 
 	// Docker is required for this target.
 	if err := HaveDocker(); err != nil {
-		fmt.Printf("No docker found: %v\n", err)
+		log.Printf("No docker found: %v\n", err)
 		return err
 	}
 
@@ -245,7 +245,7 @@ type GolangCrossBuilder struct {
 
 // Build executes the build inside of Docker.
 func (b GolangCrossBuilder) Build() error {
-	fmt.Printf(">> %v: Building for %v\n", b.Target, b.Platform)
+	log.Printf(">> %v: Building for %v\n", b.Target, b.Platform)
 
 	repoInfo, err := GetProjectRepoInfo()
 	if err != nil {
diff --git a/dev-tools/mage/manifest/manifest.go b/dev-tools/mage/manifest/manifest.go
index d7ad7babd2f..8fa7fe77952 100644
--- a/dev-tools/mage/manifest/manifest.go
+++ b/dev-tools/mage/manifest/manifest.go
@@ -48,6 +48,7 @@ func DownloadManifest(manifest string) (artifacts.Build, error) {
 	for _, manifestHost := range AllowedManifestHosts {
 		if manifestHost == manifestUrl.Host {
 			valid = true
+			break
 		}
 	}
 	if !valid {
@@ -60,10 +61,10 @@ func DownloadManifest(manifest string) (artifacts.Build, error) {
 	if err != nil {
 		return artifacts.Build{}, fmt.Errorf("downloading manifest: %w", err)
 	}
-	// if mg.Verbose() {
+
 	log.Printf(">>>> Downloaded manifest %s", manifest)
 	log.Printf(">>>> Packaing version: %s, build_id: %s, manifest_version:%s", manifestResponse.Version, manifestResponse.BuildID, manifestResponse.ManifestVersion)
-	// }
+
 	return manifestResponse, nil
 }
 
diff --git a/dev-tools/mage/manifest/manifestspecs.go b/dev-tools/mage/manifest/manifestspecs.go
index c076a637c47..31dcafb5c3a 100644
--- a/dev-tools/mage/manifest/manifestspecs.go
+++ b/dev-tools/mage/manifest/manifestspecs.go
@@ -38,15 +38,13 @@ func doWithRetries[T any](f func() (T, error)) (T, error) {
 	return resp, err
 }
 
-func downloadFile(ctx context.Context, url string, filepath string) (string, error) {
+func downloadFile(ctx context.Context, url string, filepath string) (path string, err error) {
 	outFile, fileErr := os.Create(filepath)
 	if fileErr != nil {
 		return "", fmt.Errorf("failed to create destination file %w", fileErr)
 	}
 	defer func() {
-		if err := outFile.Close(); err != nil {
-			panic(err)
-		}
+		err = outFile.Close()
 	}()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
@@ -59,9 +57,7 @@ func downloadFile(ctx context.Context, url string, filepath string) (string, err
 		return filepath, fmt.Errorf("failed to download manifest [%s]\n %w", url, err)
 	}
 	defer func() {
-		if err := resp.Body.Close(); err != nil {
-			panic(err)
-		}
+		err = resp.Body.Close()
 	}()
 
 	_, errCopy := io.Copy(outFile, resp.Body)
diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go
index 213dc7ebd19..75112ca149c 100644
--- a/dev-tools/mage/settings.go
+++ b/dev-tools/mage/settings.go
@@ -84,7 +84,7 @@ var (
 	versionQualified bool
 	versionQualifier string
 
-	// PackagingFromManifest This value is set to tru when we have defined a ManifestURL variable
+	// PackagingFromManifest This value is set to true when we have defined a ManifestURL variable
 	PackagingFromManifest bool
 	// ManifestURL Location of the manifest file to package
 	ManifestURL string
diff --git a/dev-tools/mage/version/version_parser.go b/dev-tools/mage/version/version_parser.go
index a06b1c0533f..2dc301d37eb 100644
--- a/dev-tools/mage/version/version_parser.go
+++ b/dev-tools/mage/version/version_parser.go
@@ -121,8 +121,7 @@ func (psv ParsedSemVer) GetPreviousMinor() (*ParsedSemVer, error) {
 	// We are at the first minor of the current major version series. To
 	// figure out the previous minor, we need to rely on knowledge of
 	// the release versions from the past major series'.
-	switch major {
-	case 8:
+	if major == 8 {
 		return NewParsedSemVer(7, 17, 10, psv.Prerelease(), psv.BuildMetadata()), nil
 	}
 

From 966a7633acc9d8badde10ac3036d7a35150e93a2 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 14 Mar 2024 19:44:12 +0100
Subject: [PATCH 42/43] Fixes: lint

---
 dev-tools/mage/artifacts/artifacts_api.go |  6 +++---
 dev-tools/mage/crossbuild.go              | 17 ++++++++---------
 dev-tools/mage/pkg.go                     | 16 ++++++++++++----
 3 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/dev-tools/mage/artifacts/artifacts_api.go b/dev-tools/mage/artifacts/artifacts_api.go
index 93e244e9280..5a8d67643bb 100644
--- a/dev-tools/mage/artifacts/artifacts_api.go
+++ b/dev-tools/mage/artifacts/artifacts_api.go
@@ -150,10 +150,10 @@ func (aac ArtifactAPIClient) GetVersions(ctx context.Context) (list *VersionList
 	}
 
 	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
-	defer resp.Body.Close()
 	if err != nil {
 		return nil, fmt.Errorf("getting versions: %w", err)
 	}
+	defer resp.Body.Close()
 
 	return checkResponseAndUnmarshal[VersionList](resp)
 }
@@ -168,10 +168,10 @@ func (aac ArtifactAPIClient) GetBuildsForVersion(ctx context.Context, version st
 	}
 
 	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
-	defer resp.Body.Close()
 	if err != nil {
 		return nil, fmt.Errorf("getting builds for version %s: %w", version, err)
 	}
+	defer resp.Body.Close()
 
 	return checkResponseAndUnmarshal[VersionBuilds](resp)
 }
@@ -186,10 +186,10 @@ func (aac ArtifactAPIClient) GetBuildDetails(ctx context.Context, version string
 	}
 
 	resp, err := aac.createAndPerformRequest(ctx, joinedURL)
-	defer resp.Body.Close()
 	if err != nil {
 		return nil, fmt.Errorf("getting build details for version %s buildID %s: %w", version, buildID, err)
 	}
+	defer resp.Body.Close()
 
 	return checkResponseAndUnmarshal[BuildDetails](resp)
 }
diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go
index 2a633afc36f..9bb5fed49f1 100644
--- a/dev-tools/mage/crossbuild.go
+++ b/dev-tools/mage/crossbuild.go
@@ -30,7 +30,6 @@ import (
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
-	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 	"github.com/elastic/beats/v7/libbeat/common/file"
@@ -43,7 +42,7 @@ const defaultCrossBuildTarget = "golangCrossBuild"
 // See NewPlatformList for details about platform filtering expressions.
 var Platforms = BuildPlatforms.Defaults()
 
-// Types is the list of package types
+// SelectedPackageTypes Types is the list of package types
 var SelectedPackageTypes []PackageType
 
 func init() {
@@ -163,8 +162,8 @@ func CrossBuild(options ...CrossBuildOption) error {
 		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir, params.ImageSelector}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
-				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v %v", params.ImageSelector,
-					params.Target, buildPlatform.Name)
+				return fmt.Errorf("failed cross-building target=%v for platform=%v %v. Err: %w", params.ImageSelector,
+					params.Target, buildPlatform.Name, err)
 			}
 		} else {
 			deps = append(deps, builder.Build)
@@ -249,7 +248,7 @@ func (b GolangCrossBuilder) Build() error {
 
 	repoInfo, err := GetProjectRepoInfo()
 	if err != nil {
-		return errors.Wrap(err, "failed to determine repo root and package sub dir")
+		return fmt.Errorf("failed to determine repo root and package sub dir. Err: %w", err)
 	}
 
 	mountPoint := filepath.ToSlash(filepath.Join("/go", "src", repoInfo.CanonicalRootImportPath))
@@ -263,13 +262,13 @@ func (b GolangCrossBuilder) Build() error {
 	builderArch := runtime.GOARCH
 	buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-"+builderArch))
 	if err != nil {
-		return errors.Wrap(err, "failed to determine mage-linux-"+builderArch+" relative path")
+		return fmt.Errorf("failed to determine mage-linux-"+builderArch+" relative path. Err: %w", err)
 	}
 
 	dockerRun := sh.RunCmd("docker", "run")
 	image, err := b.ImageSelector(b.Platform)
 	if err != nil {
-		return errors.Wrap(err, "failed to determine golang-crossbuild image tag")
+		return fmt.Errorf("failed to determine golang-crossbuild image tag. Err: %w", err)
 	}
 	verbose := ""
 	if mg.Verbose() {
@@ -337,7 +336,7 @@ func chownPaths(uid, gid int, path string) error {
 	start := time.Now()
 	numFixed := 0
 	defer func() {
-		log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
+		log.Printf("chown took: %v, changed %d files", time.Since(start), numFixed)
 	}()
 
 	return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
@@ -358,7 +357,7 @@ func chownPaths(uid, gid int, path string) error {
 		}
 
 		if err := os.Chown(name, uid, gid); err != nil {
-			return errors.Wrapf(err, "failed to chown path=%v", name)
+			return fmt.Errorf("failed to chown path=%v. Err: %w", name, err)
 		}
 		numFixed++
 		return nil
diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go
index b44522df8dc..569ab7b3bdd 100644
--- a/dev-tools/mage/pkg.go
+++ b/dev-tools/mage/pkg.go
@@ -18,6 +18,7 @@
 package mage
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -29,7 +30,6 @@ import (
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
-	"github.com/pkg/errors"
 )
 
 // Package packages the Beat for distribution. It generates packages based on
@@ -222,7 +222,12 @@ func saveIronbank() error {
 		return fmt.Errorf("cannot compress the tar.gz file: %+v", err)
 	}
 
-	return errors.Wrap(CreateSHA512File(tarGzFile), "failed to create .sha512 file")
+	err = CreateSHA512File(tarGzFile)
+	if err != nil {
+		return fmt.Errorf("failed to create the sha512 file: %+v", err)
+	}
+
+	return nil
 }
 
 // isPackageTypeSelected returns true if SelectedPackageTypes is empty or if
@@ -249,8 +254,11 @@ type packageBuilder struct {
 func (b packageBuilder) Build() error {
 	fmt.Printf(">> package: Building %v type=%v for platform=%v\n", b.Spec.Name, b.Type, b.Platform.Name)
 	log.Printf("Package spec: %+v", b.Spec)
-	return errors.Wrapf(b.Type.Build(b.Spec), "failed building %v type=%v for platform=%v",
-		b.Spec.Name, b.Type, b.Platform.Name)
+	err := b.Type.Build(b.Spec)
+	if err != nil {
+		return fmt.Errorf("failed building %v type=%v for platform=%v: %w", err)
+	}
+	return nil
 }
 
 type testPackagesParams struct {

From 14dcba0b0896cb82732e015cdbf15b6884975354 Mon Sep 17 00:00:00 2001
From: Pavel Zorin <pavel.zorin@elastic.co>
Date: Thu, 14 Mar 2024 19:56:49 +0100
Subject: [PATCH 43/43] Fixes: lint

---
 dev-tools/mage/pkg.go       | 8 ++++----
 dev-tools/mage/platforms.go | 7 +++----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/dev-tools/mage/pkg.go b/dev-tools/mage/pkg.go
index 569ab7b3bdd..761249de285 100644
--- a/dev-tools/mage/pkg.go
+++ b/dev-tools/mage/pkg.go
@@ -204,14 +204,14 @@ func saveIronbank() error {
 	ironbank := getIronbankContextName()
 	buildDir := filepath.Join("build", ironbank)
 	if _, err := os.Stat(buildDir); os.IsNotExist(err) {
-		return fmt.Errorf("cannot find the folder with the ironbank context: %+v", err)
+		return fmt.Errorf("cannot find the folder with the ironbank context: %w", err)
 	}
 
 	distributionsDir := "build/distributions"
 	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
 		err := os.MkdirAll(distributionsDir, 0750)
 		if err != nil {
-			return fmt.Errorf("cannot create folder for docker artifacts: %+v", err)
+			return fmt.Errorf("cannot create folder for docker artifacts: %w", err)
 		}
 	}
 	tarGzFile := filepath.Join(distributionsDir, ironbank+".tar.gz")
@@ -219,12 +219,12 @@ func saveIronbank() error {
 	// Save the build context as tar.gz artifact
 	err := TarWithOptions(buildDir, tarGzFile, true)
 	if err != nil {
-		return fmt.Errorf("cannot compress the tar.gz file: %+v", err)
+		return fmt.Errorf("cannot compress the tar.gz file: %w", err)
 	}
 
 	err = CreateSHA512File(tarGzFile)
 	if err != nil {
-		return fmt.Errorf("failed to create the sha512 file: %+v", err)
+		return fmt.Errorf("failed to create the sha512 file: %w", err)
 	}
 
 	return nil
diff --git a/dev-tools/mage/platforms.go b/dev-tools/mage/platforms.go
index d47b03d3061..a2211ba999a 100644
--- a/dev-tools/mage/platforms.go
+++ b/dev-tools/mage/platforms.go
@@ -18,10 +18,9 @@
 package mage
 
 import (
+	"fmt"
 	"sort"
 	"strings"
-
-	"github.com/pkg/errors"
 )
 
 // BuildPlatforms is a list of GOOS/GOARCH pairs supported by Go.
@@ -326,7 +325,7 @@ func newPlatformExpression(expr string) (*platformExpression, error) {
 		}
 
 		if !valid {
-			return nil, errors.Errorf("invalid platform in expression: %v", name)
+			return nil, fmt.Errorf("invalid platform in expression: %v", name)
 		}
 	}
 
@@ -420,7 +419,7 @@ func (list BuildPlatformList) Filter(expr string) BuildPlatformList {
 		return list
 	}
 	if len(pe.Add) > 0 {
-		panic(errors.Errorf("adds (%v) cannot be used in filter expressions",
+		panic(fmt.Errorf("adds (%v) cannot be used in filter expressions",
 			strings.Join(pe.Add, ", ")))
 	}