From 05b112c956ede66ab9f58e683e7f7c5a6d0af531 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 1 Mar 2024 13:53:37 +0100 Subject: [PATCH] buildkite: python generator --- .buildkite/buildkite.yml | 44 +++++++++ .buildkite/pipeline.py | 200 +++++++++++++++++++++++++++++++++++++++ auditbeat/buildkite.yml | 50 ++++++++++ filebeat/buildkite.yml | 49 ++++++++++ 4 files changed, 343 insertions(+) create mode 100644 .buildkite/buildkite.yml create mode 100755 .buildkite/pipeline.py create mode 100644 auditbeat/buildkite.yml create mode 100644 filebeat/buildkite.yml diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml new file mode 100644 index 000000000000..4707707e07cf --- /dev/null +++ b/.buildkite/buildkite.yml @@ -0,0 +1,44 @@ +projects: + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "heartbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/heartbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/osquerybeat" + - "x-pack/packetbeat" + - "x-pack/winlogbeat" + +## Changeset macros that are defined here and used in each specific 3.0 pipeline. +changeset: + ci: + - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + xpack: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" + +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. + - skip-ci + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py new file mode 100755 index 000000000000..af0e0f048739 --- /dev/null +++ b/.buildkite/pipeline.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +import yaml +import os +from dataclasses import dataclass, field + +from jinja2 import Template +from pathlib import Path + + +@dataclass() +class Pipeline: + """Buildkite Pipeline object""" + groups: list[str] + + def create_entity(self): + data = """ +steps: +{% for group in pipeline.groups -%} +{{ group.create_entity() }} +{% endfor -%} +""" + + tm = Template(data) + msg = tm.render(pipeline=self) + return msg + + +@dataclass(unsafe_hash=True) +class Group: + """Buildkite Group object""" + + project: str + category: str + steps: list[str] + + def __lt__(self, other): + return self.project < other.project + + def create_entity(self): + data = """ + - group: "{{ group.project }} {{ group.category }}" + key: "{{ group.project }}-{{ group.category }}" + steps: + {% for step in group.steps|sort -%} + {{ step.create_entity() }} + {% endfor -%} +""" + + tm = Template(data) + msg = tm.render(group=self) + return msg + + +@dataclass(unsafe_hash=True) +class Step: + """Buildkite Step object""" + + command: str + name: str + runner: str + project: str + provider: str + label: str = field(init=False) + comment: str = field(init=False) + + def __post_init__(self): + self.comment = "/test " + self.project + " " + self.name + self.label = self.name + + def __lt__(self, other): + return self.name < other.name + + def create_entity(self): + data = """ + - label: "{{ stage.project }} {{ stage.name }}" + command: + - {{ stage.command }} + notify: + - github_commit_status: + context: "{{ stage.project }}: {{ stage.name }}" + agents: + provider: {{ stage.provider }} + image:{{ stage.runner }} +""" + + tm = Template(data) + msg = tm.render(stage=self) + return msg + + +def is_step_enabled(step: Step, conditions) -> bool: + # TODO: + # If branch + # If PR then + # If GitHub label matches project name + # If GitHub comment + # If Changeset + return True + + +def is_group_enabled(group: Group, conditions) -> bool: + # TODO: + # If branch + # If PR then + # If GitHub label matches project name + category + # If GitHub comment + # If Changeset + return True + + +def fetch_stage(name: str, stage, project: str) -> Step: + """Create a step given the yaml object.""" + + # TODO: need to accomodate the provider type. + # maybe in the buildkite.yml or some dynamic analysis based on the + # name of the runners. + return Step( + command=stage["command"], + name=name, + runner=stage["platform"], + project=project, + provider="gcp") + +# TODO: validate unique stages! + +def main() -> None: + + groups = [] + extended_groups = [] + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + doc = yaml.load(file, yaml.FullLoader) + + for project in doc["projects"]: + project_file = os.path.join(project, "buildkite.yml") + if not os.path.isfile(project_file): + continue + # TODO: data structure when things run. + conditions = None + with open(project_file, "r", encoding="utf8") as file: + steps = [] + project_obj = yaml.load(file, yaml.FullLoader) + + # Given the mandatory list first + mandatory = project_obj["stages"]["mandatory"] + for stage in mandatory: + step = fetch_stage( + name=stage, + project=project, + stage=mandatory[stage]) + + if is_step_enabled(step, conditions): + steps.append(step) + + group = Group( + project=project, + category="mandatory", + steps=steps + ) + + if is_group_enabled(group, conditions): + extended_groups.append(group) + + # Given the extended list if needed + # TODO: Validate if included + extended_steps = [] + + extended = project_obj["stages"]["extended"] + for stage in extended: + step = fetch_stage( + name=stage, + project=project, + stage=extended[stage]) + + if is_step_enabled(step, conditions): + extended_steps.append(step) + + group = Group( + project=project, + category="extended", + steps=extended_steps + ) + + if is_group_enabled(group, conditions): + extended_groups.append(group) + + # TODO: improve this merging lists + all_groups = [] + for group in sorted(groups): + all_groups.append(group) + for group in sorted(extended_groups): + all_groups.append(group) + + # Produce now the pipeline + print(Pipeline(all_groups).create_entity()) + + +if __name__ == "__main__": + + # pylint: disable=E1120 + main() diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml new file mode 100644 index 000000000000..c85c176947e2 --- /dev/null +++ b/auditbeat/buildkite.yml @@ -0,0 +1,50 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "^auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # mandatory stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test auditbeat + # - GitHub label auditbeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/core-ubuntu-2204" + crosscompile: + command: "make -C auditbeat crosscompile" + platform: "family/core-ubuntu-2204" + rhel-9: + command: "mage build unitTest" + platform: "family/core-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/core-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + # NOTE: stage name should be unique! + integTest: + command: "mage build integTest" + platform: "core-ubuntu-2004-aarch64" + integTest-arm: + command: "mage build integTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-arm: + command: "mage build unitTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml new file mode 100644 index 000000000000..faffac3313e2 --- /dev/null +++ b/filebeat/buildkite.yml @@ -0,0 +1,49 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "^filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # default stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test filebeat + # - GitHub label filebeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/core-ubuntu-2204" + crosscompile: + command: "make -C filebeat crosscompile" + platform: "family/core-ubuntu-2204" + goIntegTest: + command: "mage goIntegTest" + platform: "family/core-ubuntu-2204" + pythonIntegTest: + command: "mage pythonIntegTest" + platform: "family/core-ubuntu-2204" + rhel-9: + command: "mage build unitTest" + platform: "family/core-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/core-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test filebeat . i.e: /test filebeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + unitTest-arm: + command: "mage build unitTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019"