Elastic agent unit tests pipieline (#37750)

* Elastic agent unit tests pipieline * Removed unnecessary script functions * Removed unnecessary script functions * Removed unnecessary script functions * Removed unnecessary script functions * Corrected coverage path * Corrected coverage path * Fix pipeline slug * Corrected coverage path * Fix hooks * Fix hooks * Fix hooks * Fix hooks * Fix paths * Fix paths * Fix paths * fixed paths * fixed paths * fixed paths * extended windows tests * extended windows tests * macos tests * xpack to x-pack * xpack to x-pack
elastic · Feb 2, 2024 · 3e665b8 · 3e665b8
1 parent aba352d
commit 3e665b8
Show file tree

Hide file tree

Showing 11 changed files with 465 additions and 87 deletions.
diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -euo pipefail
+
+checkout_merge() {
+    local target_branch=$1
+    local pr_commit=$2
+    local merge_branch=$3
+
+    if [[ -z "${target_branch}" ]]; then
+        echo "No pull request target branch"
+        exit 1
+    fi
+
+    git fetch -v origin "${target_branch}"
+    git checkout FETCH_HEAD
+    echo "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # create temporal branch to merge the PR with the target branch
+    git checkout -b ${merge_branch}
+    echo "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # set author identity so it can be run git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit "${BUILDKITE_COMMIT}" || {
+        local merge_result=$?
+        echo "Merge failed: ${merge_result}"
+        git merge --abort
+        exit ${merge_result}
+    }
+}
+
+pull_request="${BUILDKITE_PULL_REQUEST:-false}"
+
+if [[ "${pull_request}" == "false" ]]; then
+    echo "Not a pull request, skipping"
+    exit 0
+fi
+
+TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}"
+PR_COMMIT="${BUILDKITE_COMMIT}"
+PR_ID=${BUILDKITE_PULL_REQUEST}
+MERGE_BRANCH="pr_merge_${PR_ID}"
+
+checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
+
+echo "Commit information"
+git --no-pager log --format=%B -n 1
+
+# Ensure buildkite groups are rendered
+echo ""
diff --git a/.buildkite/hooks/post-checkout.ps1 b/.buildkite/hooks/post-checkout.ps1
@@ -0,0 +1,60 @@
+# Set error handling
+$ErrorActionPreference = "Stop"
+
+# Define a function to checkout and merge
+function Checkout-Merge {
+    param (
+        [string]$targetBranch,
+        [string]$prCommit,
+        [string]$mergeBranch
+    )
+
+    if (-not $targetBranch) {
+        Write-Host "No pull request target branch"
+        exit 1
+    }
+
+    git fetch -v origin $targetBranch
+    git checkout FETCH_HEAD
+    Write-Host "Current branch: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Create a temporary branch to merge the PR with the target branch
+    git checkout -b $mergeBranch
+    Write-Host "New branch created: $(git rev-parse --abbrev-ref HEAD)"
+
+    # Set author identity so it can be used for git merge
+    git config user.name "github-merged-pr-post-checkout"
+    git config user.email "auto-merge@buildkite"
+
+    git merge --no-edit $prCommit
+
+    if ($LASTEXITCODE -ne 0) {
+        $mergeResult = $LASTEXITCODE
+        Write-Host "Merge failed: $mergeResult"
+        git merge --abort
+        exit $mergeResult
+    }
+}
+
+$pullRequest = $env:BUILDKITE_PULL_REQUEST
+
+if ($pullRequest -eq "false") {
+    Write-Host "Not a pull request, skipping"
+    exit 0
+}
+
+$targetBranch = $env:BUILDKITE_PULL_REQUEST_BASE_BRANCH
+$prCommit = $env:BUILDKITE_COMMIT
+$prId = $env:BUILDKITE_PULL_REQUEST
+$mergeBranch = "pr_merge_$prId"
+
+Checkout-Merge $targetBranch $prCommit $mergeBranch
+
+Write-Host "Commit information"
+git --no-pager log --format=%B -n 1
+
+Write-Host "Fixing CRLF in git checkout --"
+git config core.autocrlf true
+
+# Ensure Buildkite groups are rendered
+Write-Host ""
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
@@ -130,7 +130,7 @@
         },
         {
             "enabled": true,
-            "pipeline_slug": "xpack-elastic-agent",
+            "pipeline_slug": "beats-xpack-elastic-agent",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
             "allowed_list": [ ],
@@ -141,8 +141,8 @@
             "always_trigger_comment_regex": "^/test elastic-agent$",
             "skip_ci_labels": [  ],
             "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": ["^xpack/elastic-agent/README.md", "^xpack/elastic-agent/docs/.*", "^xpack/elastic-agent/devtools/.*" ],
-            "always_require_ci_on_changed": ["^xpack/elastic-agent/.*", ".buildkite/xpack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+            "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ],
+            "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/xpack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
         }
     ]
 }
diff --git a/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml b/.buildkite/x-pack/elastic-agent/pipeline.xpack.elastic-agent.yml
@@ -0,0 +1,174 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  DOCKER_REGISTRY: "docker.elastic.co"
+steps:
+  - group: "Unit tests"
+    key: "unit-tests"
+    steps:
+      - label: "Unit tests - Ubuntu 22.04"
+        key: "unit-tests-2204"
+        command: ".buildkite/x-pack/elastic-agent/scripts/steps/unit-tests.sh"
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/core-ubuntu-2204"
+        retry:
+          manual:
+            allowed: true
+
+      - label: "Unit tests - Ubuntu 22.04 ARM64"
+        key: "unit-tests-2204-arm64"
+        command: ".buildkite/x-pack/elastic-agent/scripts/steps/unit-tests.sh"
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "aws"
+          imagePrefix: "core-ubuntu-2204-aarch64"
+          diskSizeGb: 200
+          instanceType: "m6g.xlarge"
+        retry:
+          manual:
+            allowed: true
+
+      - label: "Unit tests - MacOS 13"
+        key: "unit-tests-macos-13"
+        command: ".buildkite/x-pack/elastic-agent/scripts/steps/unit-tests.sh"
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "coverage.out"
+        agents:
+          provider: orka
+          imagePrefix: generic-13-ventura-x64
+        retry:
+          manual:
+            allowed: true
+
+      # Runs inly on the 7.17 branch
+      - label: "Unit tests - MacOS 13 ARM"
+        key: "unit-tests-macos-13-arm"
+        command: ".buildkite/x-pack/elastic-agent/scripts/steps/unit-tests.sh"
+        branches: main
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: orka
+          imagePrefix: generic-13-ventura-arm
+        retry:
+          manual:
+            allowed: true
+
+      - label: "Unit tests - Windows 2022"
+        key: "unit-tests-win2022"
+        command: |
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\install_mage.ps1
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\unit-tests.ps1
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/core-windows-2022"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        retry:
+          manual:
+            allowed: true
+
+      - label: "Unit tests - Windows 2016"
+        key: "unit-tests-win2016"
+        command: |
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\install_mage.ps1
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\unit-tests.ps1
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/core-windows-2016"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        retry:
+          manual:
+            allowed: true
+
+      - label: "Unit tests - Windows 2019"
+        key: "unit-tests-win2019"
+        command: |
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\install_mage.ps1
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\unit-tests.ps1
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/core-windows-2019"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        retry:
+          manual:
+            allowed: true
+
+  - group: "Desktop Windows tests"
+    key: "extended-windows"
+    steps:
+      - label: "Unit tests - Windows 10"
+        key: "unit-tests-win10"
+        command: |
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\install_mage.ps1
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\unit-tests.ps1
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/general-windows-10"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        retry:
+          manual:
+            allowed: true
+
+      - label: "Unit tests - Windows 11"
+        key: "unit-tests-win11"
+        command: |
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\install_mage.ps1
+          .\\.buildkite\\x-pack\\elastic-agent\\scripts\\steps\\unit-tests.ps1
+        artifact_paths:
+          - "build/TEST-go-unit.html"
+          - "build/TEST-go-unit.xml"
+          - "build/diagnostics/*"
+          - "coverage.out"
+        agents:
+          provider: "gcp"
+          image: "family/general-windows-11"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        retry:
+          manual:
+            allowed: true