buildkite: python generator

.buildkite/buildkite.yml

@@ -0,0 +1,44 @@
+projects:
+    - "auditbeat"
+    - "deploy/kubernetes"
+    - "filebeat"
+    - "heartbeat"
+    - "libbeat"
+    - "metricbeat"
+    - "packetbeat"
+    - "winlogbeat"
+    - "x-pack/auditbeat"
+    - "x-pack/dockerlogbeat"
+    - "x-pack/filebeat"
+    - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
+    - "x-pack/libbeat"
+    - "x-pack/metricbeat"
+    - "x-pack/osquerybeat"
+    - "x-pack/packetbeat"
+    - "x-pack/winlogbeat"
+
+## Changeset macros that are defined here and used in each specific 3.0 pipeline.
+changeset:
+    ci:
+        - "^Jenkinsfile"
+        - "^\\.ci/scripts/.*"
+    oss:
+        - "^go.mod"
+        - "^pytest.ini"
+        - "^dev-tools/.*"
+        - "^libbeat/.*"
+        - "^testing/.*"
+    xpack:
+        - "^go.mod"
+        - "^pytest.ini"
+        - "^dev-tools/.*"
+        - "^libbeat/.*"
+        - "^testing/.*"
+        - "^x-pack/libbeat/.*"
+
+disabled:
+    when:
+        labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR.
+            - skip-ci
+        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.

.buildkite/pipeline.py

@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+import yaml
+import os
+from dataclasses import dataclass, field
+
+from jinja2 import Template
+from pathlib import Path
+
+
+@dataclass()
+class Pipeline:
+    """Buildkite Pipeline object"""
+    groups: list[str]
+
+    def create_entity(self):
+        data = """
+steps:
+{% for group in pipeline.groups -%}
+{{ group.create_entity() }}
+{% endfor -%}
+"""
+
+        tm = Template(data)
+        msg = tm.render(pipeline=self)
+        return msg
+
+
+@dataclass(unsafe_hash=True)
+class Group:
+    """Buildkite Group object"""
+
+    project: str
+    category: str
+    steps: list[str]
+
+    def __lt__(self, other):
+        return self.project < other.project
+
+    def create_entity(self):
+        data = """
+  - group: "{{ group.project }} {{ group.category }}"
+    key: "{{ group.project }}-{{ group.category }}"
+    steps:
+      {% for step in group.steps|sort -%}
+        {{ step.create_entity() }}
+      {% endfor -%}
+"""
+
+        tm = Template(data)
+        msg = tm.render(group=self)
+        return msg
+
+
+@dataclass(unsafe_hash=True)
+class Step:
+    """Buildkite Step object"""
+
+    command: str
+    name: str
+    runner: str
+    project: str
+    provider: str
+    label: str = field(init=False)
+    comment: str = field(init=False)
+
+    def __post_init__(self):
+        self.comment = "/test " + self.project + " " + self.name
+        self.label = self.name
+
+    def __lt__(self, other):
+        return self.name < other.name
+
+    def create_entity(self):
+        data = """
+      - label: "{{ stage.project }} {{ stage.name }}"
+        command:
+          - {{ stage.command }}
+        notify:
+          - github_commit_status:
+              context: "{{ stage.project }}: {{ stage.name }}"
+        agents:
+          provider: {{ stage.provider }}
+          image:{{ stage.runner }}
+"""
+
+        tm = Template(data)
+        msg = tm.render(stage=self)
+        return msg
+
+
+def is_step_enabled(step: Step, conditions) -> bool:
+    # TODO:
+    # If branch
+    # If PR then
+    #   If GitHub label matches project name
+    #   If GitHub comment
+    #   If Changeset
+    return True
+
+
+def is_group_enabled(group: Group, conditions) -> bool:
+    # TODO:
+    # If branch
+    # If PR then
+    #   If GitHub label matches project name + category
+    #   If GitHub comment
+    #   If Changeset
+    return True
+
+
+def fetch_stage(name: str, stage, project: str) -> Step:
+    """Create a step given the yaml object."""
+
+    # TODO: need to accomodate the provider type.
+    # maybe in the buildkite.yml or some dynamic analysis based on the
+    # name of the runners.
+    return Step(
+            command=stage["command"],
+            name=name,
+            runner=stage["platform"],
+            project=project,
+            provider="gcp")
+
+# TODO: validate unique stages!
+
+def main() -> None:
+
+    groups = []
+    extended_groups = []
+    with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file:
+        doc = yaml.load(file, yaml.FullLoader)
+
+        for project in doc["projects"]:
+            project_file = os.path.join(project, "buildkite.yml")
+            if not os.path.isfile(project_file):
+                continue
+            # TODO: data structure when things run.
+            conditions = None
+            with open(project_file, "r", encoding="utf8") as file:
+                steps = []
+                project_obj = yaml.load(file, yaml.FullLoader)
+
+                # Given the mandatory list first
+                mandatory = project_obj["stages"]["mandatory"]
+                for stage in mandatory:
+                    step = fetch_stage(
+                            name=stage,
+                            project=project,
+                            stage=mandatory[stage])
+
+                    if is_step_enabled(step, conditions):
+                        steps.append(step)
+
+                group = Group(
+                            project=project,
+                            category="mandatory",
+                            steps=steps
+                        )
+
+                if is_group_enabled(group, conditions):
+                    extended_groups.append(group)
+
+                # Given the extended list if needed
+                # TODO: Validate if included
+                extended_steps = []
+
+                extended = project_obj["stages"]["extended"]
+                for stage in extended:
+                    step = fetch_stage(
+                            name=stage,
+                            project=project,
+                            stage=extended[stage])
+
+                    if is_step_enabled(step, conditions):
+                        extended_steps.append(step)
+
+                group = Group(
+                            project=project,
+                            category="extended",
+                            steps=extended_steps
+                        )
+
+                if is_group_enabled(group, conditions):
+                    extended_groups.append(group)
+
+    # TODO: improve this merging lists
+    all_groups = []
+    for group in sorted(groups):
+        all_groups.append(group)
+    for group in sorted(extended_groups):
+        all_groups.append(group)
+
+    # Produce now the pipeline
+    print(Pipeline(all_groups).create_entity())
+
+
+if __name__ == "__main__":
+
+    # pylint: disable=E1120
+    main()

auditbeat/buildkite.yml

@@ -0,0 +1,50 @@
+when:
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^auditbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+stages:
+    # mandatory stage - it runs always for:
+    #   - branches/tags
+    #   - on PRs
+    #   - GitHub comment /test auditbeat
+    #   - GitHub label auditbeat
+    mandatory:
+        # NOTE: stage name should be unique!
+        unitTest:
+            command: "mage build unitTest"
+            platform: "family/core-ubuntu-2204"
+        crosscompile:
+            command: "make -C auditbeat crosscompile"
+            platform: "family/core-ubuntu-2204"
+        rhel-9:
+            command: "mage build unitTest"
+            platform: "family/core-rhel-9"
+        unitTest-windows-2022:
+            command: "mage build unitTest"
+            platform: "windows-2022"
+        unitTest-windows-2016:
+            command: "mage build unitTest"
+            platform: "family/core-windows-2016"
+    # optional stage - it runs on:
+    #   - branches/tags
+    #   - on PRs if:
+    #       - GitHub comment /test auditbeat <sub-stage-name> . i.e: /test auditbeat integTest
+    #       - GitHub label <sub-stage-name> . i.e: integTest or unitTest-arm or unitTest-macos ...
+    extended:
+        # NOTE: stage name should be unique!
+        integTest:
+            command: "mage build integTest"
+            platform: "core-ubuntu-2004-aarch64"
+        integTest-arm:
+            command: "mage build integTest"
+            platform: "core-ubuntu-2004-aarch64"
+        unitTest-arm:
+            command: "mage build unitTest"
+            platform: "core-ubuntu-2004-aarch64"
+        unitTest-macos:
+            command: "mage build unitTest"
+            platform: "generic-13-ventura-x64"
+        unitTest-windows-2019:
+            command: "mage build unitTest"
+            platform: "family/core-windows-2019"

filebeat/buildkite.yml

@@ -0,0 +1,49 @@
+when:
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^filebeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+stages:
+    # default stage - it runs always for:
+    #   - branches/tags
+    #   - on PRs
+    #   - GitHub comment /test filebeat
+    #   - GitHub label filebeat
+    mandatory:
+        # NOTE: stage name should be unique!
+        unitTest:
+            command: "mage build unitTest"
+            platform: "family/core-ubuntu-2204"
+        crosscompile:
+            command: "make -C filebeat crosscompile"
+            platform: "family/core-ubuntu-2204"
+        goIntegTest:
+            command: "mage goIntegTest"
+            platform: "family/core-ubuntu-2204"
+        pythonIntegTest:
+            command: "mage pythonIntegTest"
+            platform: "family/core-ubuntu-2204"
+        rhel-9:
+            command: "mage build unitTest"
+            platform: "family/core-rhel-9"
+        unitTest-windows-2022:
+            command: "mage build unitTest"
+            platform: "windows-2022"
+        unitTest-windows-2016:
+            command: "mage build unitTest"
+            platform: "family/core-windows-2016"
+    # optional stage - it runs on:
+    #   - branches/tags
+    #   - on PRs if:
+    #       - GitHub comment /test filebeat <sub-stage-name> . i.e: /test filebeat integTest
+    #       - GitHub label <sub-stage-name> . i.e: integTest or unitTest-arm or unitTest-macos ...
+    extended:
+        unitTest-arm:
+            command: "mage build unitTest"
+            platform: "core-ubuntu-2004-aarch64"
+        unitTest-macos:
+            command: "mage build unitTest"
+            platform: "generic-13-ventura-x64"
+        unitTest-windows-2019:
+            command: "mage build unitTest"
+            platform: "family/core-windows-2019"