Skip to content

Commit

Permalink
buildkite: make build steps unique
Browse files Browse the repository at this point in the history
If we have more than one build step in a pipeline, Buildkite complains
that they are not unique.

    fatal: Failed to upload and process pipeline: Pipeline upload
    rejected: The key "build_x86_64" has already been used by another
    step in this build

We had a similar issue with shared build tarballs, so just reuse the
tarball name as key.

Fixes: 19e9051

Signed-off-by: Pablo Barbáchano <pablob@amazon.com>
  • Loading branch information
pb8o committed Sep 17, 2024
1 parent 474392d commit f70a87f
Showing 1 changed file with 12 additions and 8 deletions.
20 changes: 12 additions & 8 deletions .buildkite/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ def __init__(self, with_build_step=True, **kwargs):
if with_build_step:
build_cmds, self.shared_build = shared_build()
self.build_group_per_arch(
"🏗️ Build", build_cmds, depends_on_build=False, key_prefix="build"
"🏗️ Build", build_cmds, depends_on_build=False, set_key=True
)
else:
self.shared_build = None
Expand Down Expand Up @@ -297,8 +297,8 @@ def _adapt_group(self, group):
for step in group["steps"]:
step["command"] = prepend + step["command"]
if self.shared_build is not None:
step["depends_on"] = (
"build_" + DEFAULT_INSTANCES[step["agents"]["instance"]]
step["depends_on"] = self.build_key(
DEFAULT_INSTANCES[step["agents"]["instance"]]
)
return group

Expand All @@ -314,22 +314,26 @@ def build_group(self, *args, **kwargs):
group(*args, **combined), depends_on_build=depends_on_build
)

def build_key(self, arch):
"""Return the Buildkite key for the build step, for the specified arch"""
return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")

def build_group_per_arch(self, label, *args, **kwargs):
"""
Build a group, parametrizing over the architectures only.
kwargs consumed by this method and not passed down to `group`:
- `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
- `key_prefix`: If set, causes the generated steps to have a "key" field set to f"{key_prefix}_{$ARCH}".
- `set_key`: If True, causes the generated steps to have a "key" field
"""
depends_on_build = kwargs.pop("depends_on_build", True)
key_prefix = kwargs.pop("key_prefix", None)
set_key = kwargs.pop("set_key", None)
combined = overlay_dict(self.per_arch, kwargs)
grp = group(label, *args, **combined)
if key_prefix:
if set_key:
for step in grp["steps"]:
step["key"] = (
key_prefix + "_" + DEFAULT_INSTANCES[step["agents"]["instance"]]
step["key"] = self.build_key(
DEFAULT_INSTANCES[step["agents"]["instance"]]
)
return self.add_step(grp, depends_on_build=depends_on_build)

Expand Down

0 comments on commit f70a87f

Please sign in to comment.