diff --git a/manifests/1.1.1/opensearch-1.1.1.yml b/manifests/1.1.1/opensearch-1.1.1.yml index 3c126b0d32..3a1b69208c 100644 --- a/manifests/1.1.1/opensearch-1.1.1.yml +++ b/manifests/1.1.1/opensearch-1.1.1.yml @@ -3,6 +3,8 @@ schema-version: "1.0" build: name: OpenSearch version: 1.1.1 + patches: + - 1.1.0 components: - name: OpenSearch dist: https://ci.opensearch.org/ci/dbc/builds/1.1.0/405 diff --git a/src/build_workflow/build_target.py b/src/build_workflow/build_target.py index b9aa8d29ff..3bada95def 100644 --- a/src/build_workflow/build_target.py +++ b/src/build_workflow/build_target.py @@ -22,6 +22,7 @@ class BuildTarget: def __init__( self, version, + patches=[], platform=None, architecture=None, name=None, @@ -32,6 +33,7 @@ def __init__( self.build_id = os.getenv("BUILD_NUMBER") or build_id or uuid.uuid4().hex self.name = name self.version = version + self.patches = patches self.snapshot = snapshot self.architecture = architecture or current_architecture() self.platform = platform or current_platform() @@ -41,7 +43,22 @@ def __init__( def opensearch_version(self): return self.version + "-SNAPSHOT" if self.snapshot else self.version + @property + def compatible_opensearch_versions(self): + return list(map(lambda version: version + "-SNAPSHOT" if self.snapshot else version, self.compatible_versions)) + @property def component_version(self): # BUG: the 4th digit is dictated by the component, it's not .0, this will break for 1.1.0.1 return self.version + ".0-SNAPSHOT" if self.snapshot else f"{self.version}.0" + + @property + def compatible_component_versions(self): + # BUG: the 4th digit is dictated by the component, it's not .0, this will break for 1.1.0.1 + return list(map(lambda version: version + ".0-SNAPSHOT" if self.snapshot else f"{version}.0", self.compatible_versions)) + + @property + def compatible_versions(self): + versions = [self.version] + versions.extend(self.patches) + return versions diff --git a/src/build_workflow/opensearch/build_artifact_check_maven.py b/src/build_workflow/opensearch/build_artifact_check_maven.py index a424090ee9..82119a5e46 100644 --- a/src/build_workflow/opensearch/build_artifact_check_maven.py +++ b/src/build_workflow/opensearch/build_artifact_check_maven.py @@ -34,14 +34,10 @@ def check(self, path): data = zip.read("META-INF/MANIFEST.MF").decode("UTF-8") properties = PropertiesFile(data) try: - properties.check_value_in( - "Implementation-Version", - [ - self.target.component_version, - self.target.opensearch_version, - None, - ], - ) + versions = [None] + versions.extend(self.target.compatible_component_versions) + versions.extend(self.target.compatible_opensearch_versions) + properties.check_value_in("Implementation-Version", versions) except PropertiesFile.CheckError as e: raise BuildArtifactCheck.BuildArtifactInvalidError(path, str(e)) logging.info(f'Checked {path} ({properties.get_value("Implementation-Version", "N/A")})') diff --git a/src/build_workflow/opensearch/build_artifact_check_plugin.py b/src/build_workflow/opensearch/build_artifact_check_plugin.py index 6c1f5f271a..fbb23e0a62 100644 --- a/src/build_workflow/opensearch/build_artifact_check_plugin.py +++ b/src/build_workflow/opensearch/build_artifact_check_plugin.py @@ -16,14 +16,17 @@ class BuildArtifactOpenSearchCheckPlugin(BuildArtifactCheck): def check(self, path): if os.path.splitext(path)[1] != ".zip": raise BuildArtifactCheck.BuildArtifactInvalidError(path, "Not a zip file.") - if not path.endswith(f"-{self.target.component_version}.zip"): - raise BuildArtifactCheck.BuildArtifactInvalidError(path, f"Expected filename to include {self.target.component_version}.") + if not self.valid_path(path): + raise BuildArtifactCheck.BuildArtifactInvalidError(path, f"Expected filename to include one of {self.target.compatible_component_versions}.") with ZipFile(path, "r") as zip: data = zip.read("plugin-descriptor.properties").decode("UTF-8") properties = PropertiesFile(data) try: - properties.check_value("version", self.target.component_version) - properties.check_value("opensearch.version", self.target.version) + properties.check_value_in("version", self.target.compatible_component_versions) + properties.check_value_in("opensearch.version", self.target.compatible_versions) except PropertiesFile.CheckError as e: raise BuildArtifactCheck.BuildArtifactInvalidError(path, e.__str__()) logging.info(f'Checked {path} ({properties.get_value("version", "N/A")})') + + def valid_path(self, path): + return any(map(lambda version: path.endswith(f"-{version}.zip"), self.target.compatible_component_versions)) diff --git a/src/manifests/input_manifest.py b/src/manifests/input_manifest.py index 9622d3da8b..4fa2acd92f 100644 --- a/src/manifests/input_manifest.py +++ b/src/manifests/input_manifest.py @@ -14,6 +14,7 @@ build: name: string version: string + patches: optional list of compatible versions this build is patching ci: image: name: docker image name to pull @@ -47,6 +48,10 @@ class InputManifest(Manifest): "schema": { "name": {"required": True, "type": "string"}, "version": {"required": True, "type": "string"}, + "patches": { + "type": "list", + "schema": {"type": "string"}, + }, }, }, "ci": { @@ -131,9 +136,10 @@ class Build: def __init__(self, data): self.name = data["name"] self.version = data["version"] + self.patches = data.get("patches", []) def __to_dict__(self): - return {"name": self.name, "version": self.version} + return Manifest.compact({"name": self.name, "version": self.version, "patches": self.patches}) class Components(dict): def __init__(self, data): diff --git a/src/run_build.py b/src/run_build.py index 659ec6a9fd..45a0146505 100755 --- a/src/run_build.py +++ b/src/run_build.py @@ -31,6 +31,7 @@ def main(): target = BuildTarget( name=manifest.build.name, version=manifest.build.version, + patches=manifest.build.patches, snapshot=args.snapshot, output_dir=output_dir, platform=args.platform, diff --git a/tests/test_run_build.py b/tests/test_run_build.py index 4899ffa784..0d840830eb 100644 --- a/tests/test_run_build.py +++ b/tests/test_run_build.py @@ -7,7 +7,7 @@ import os import tempfile import unittest -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch import pytest diff --git a/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_maven.py b/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_maven.py index 5876e64bca..eb0161aa7f 100644 --- a/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_maven.py +++ b/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_maven.py @@ -24,6 +24,7 @@ def __mock(self, props="", snapshot=True): output_dir="output_dir", name="OpenSearch", version="1.1.0", + patches=["1.0.0"], architecture="x64", snapshot=snapshot, ) @@ -43,12 +44,16 @@ def test_record_maven_artifact_after_checking_maven_version_properties_snapshot( with self.__mock("Implementation-Version: 1.1.0.0-SNAPSHOT") as mock: mock.check("valid.jar") + def test_record_maven_artifact_after_checking_maven_version_properties_patch(self): + with self.__mock("Implementation-Version: 1.0.0.0", snapshot=False) as mock: + mock.check("valid.jar") + def test_build_artifact_check_maven_version_properties_mismatch(self): with self.assertRaises(BuildArtifactCheck.BuildArtifactInvalidError) as context: with self.__mock("Implementation-Version: 1.2.3.4", snapshot=False) as mock: mock.check("valid.jar") self.assertEqual( - "Artifact valid.jar is invalid. Expected to have Implementation-Version=any of ['1.1.0.0', '1.1.0', None], but was '1.2.3.4'.", + "Artifact valid.jar is invalid. Expected to have Implementation-Version=any of [None, '1.1.0.0', '1.0.0.0', '1.1.0', '1.0.0'], but was '1.2.3.4'.", str(context.exception), ) diff --git a/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_plugin.py b/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_plugin.py index 9adc5f03c1..c3f9507b85 100644 --- a/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_plugin.py +++ b/tests/tests_build_workflow/opensearch/test_build_artifact_opensearch_check_plugin.py @@ -24,6 +24,7 @@ def __mock(self, props="", snapshot=True): output_dir="output_dir", name="OpenSearch", version="1.1.0", + patches=["1.0.0"], architecture="x64", snapshot=snapshot, ) @@ -34,7 +35,7 @@ def test_check_plugin_zip_version_snapshot(self): with self.__mock() as mock: mock.check("invalid.zip") self.assertEqual( - "Artifact invalid.zip is invalid. Expected filename to include 1.1.0.0-SNAPSHOT.", + "Artifact invalid.zip is invalid. Expected filename to include one of ['1.1.0.0-SNAPSHOT', '1.0.0.0-SNAPSHOT'].", str(context.exception), ) @@ -43,7 +44,7 @@ def test_check_plugin_zip_version(self): with self.__mock(snapshot=False) as mock: mock.check("invalid.zip") self.assertEqual( - "Artifact invalid.zip is invalid. Expected filename to include 1.1.0.0.", + "Artifact invalid.zip is invalid. Expected filename to include one of ['1.1.0.0', '1.0.0.0'].", str(context.exception), ) @@ -52,7 +53,7 @@ def test_check_plugin_version_properties_missing(self, *mocks): with self.__mock("") as mock: mock.check("valid-1.1.0.0-SNAPSHOT.zip") self.assertEqual( - "Artifact valid-1.1.0.0-SNAPSHOT.zip is invalid. Expected to have version='1.1.0.0-SNAPSHOT', but none was found.", + "Artifact valid-1.1.0.0-SNAPSHOT.zip is invalid. Expected to have version=any of ['1.1.0.0-SNAPSHOT', '1.0.0.0-SNAPSHOT'], but none was found.", str(context.exception), ) @@ -61,10 +62,14 @@ def test_check_plugin_version_properties_mismatch(self, *mocks): with self.__mock("version=1.2.3.4") as mock: mock.check("valid-1.1.0.0-SNAPSHOT.zip") self.assertEqual( - "Artifact valid-1.1.0.0-SNAPSHOT.zip is invalid. Expected to have version='1.1.0.0-SNAPSHOT', but was '1.2.3.4'.", + "Artifact valid-1.1.0.0-SNAPSHOT.zip is invalid. Expected to have version=any of ['1.1.0.0-SNAPSHOT', '1.0.0.0-SNAPSHOT'], but was '1.2.3.4'.", str(context.exception), ) def test_check_plugin_version_properties(self, *mocks): with self.__mock("opensearch.version=1.1.0\nversion=1.1.0.0-SNAPSHOT") as mock: mock.check("valid-1.1.0.0-SNAPSHOT.zip") + + def test_check_plugin_version_properties_patch(self, *mocks): + with self.__mock("opensearch.version=1.1.0\nversion=1.0.0.0-SNAPSHOT") as mock: + mock.check("valid-1.0.0.0-SNAPSHOT.zip") diff --git a/tests/tests_build_workflow/test_build_target.py b/tests/tests_build_workflow/test_build_target.py index 608137c994..eba3c48b89 100644 --- a/tests/tests_build_workflow/test_build_target.py +++ b/tests/tests_build_workflow/test_build_target.py @@ -23,9 +23,7 @@ def test_build_id_from_env(self): self.assertEqual(BuildTarget(version="1.1.0", architecture="x86").build_id, "id") def test_build_id_from_arg(self): - self.assertEqual( - BuildTarget(version="1.1.0", architecture="x86", build_id="id").build_id, "id" - ) + self.assertEqual(BuildTarget(version="1.1.0", architecture="x86", build_id="id").build_id, "id") def test_opensearch_version(self): self.assertEqual( @@ -33,6 +31,18 @@ def test_opensearch_version(self): "1.1.0", ) + def test_compatible_opensearch_versions(self): + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False).compatible_opensearch_versions, + ["1.1.2", "1.1.0", "1.1.1"], + ) + + def test_compatible_opensearch_versions_snapshot(self): + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True).compatible_opensearch_versions, + ["1.1.2-SNAPSHOT", "1.1.0-SNAPSHOT", "1.1.1-SNAPSHOT"], + ) + def test_opensearch_version_snapshot(self): self.assertEqual( BuildTarget(version="1.1.0", architecture="x86", snapshot=True).opensearch_version, @@ -45,6 +55,18 @@ def test_component_version(self): "1.1.0.0", ) + def test_compatible_component_versions(self): + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=False).compatible_component_versions, + ["1.1.2.0", "1.1.0.0", "1.1.1.0"], + ) + + def test_compatible_component_versions_snapshot(self): + self.assertEqual( + BuildTarget(version="1.1.2", architecture="x86", patches=["1.1.0", "1.1.1"], snapshot=True).compatible_component_versions, + ["1.1.2.0-SNAPSHOT", "1.1.0.0-SNAPSHOT", "1.1.1.0-SNAPSHOT"], + ) + def test_component_version_snapshot(self): self.assertEqual( BuildTarget(version="1.1.0", architecture="x86", snapshot=True).component_version, @@ -56,15 +78,11 @@ def test_platform(self, *mock): self.assertEqual(BuildTarget(version="1.1.0", snapshot=False).platform, "value") def test_platform_value(self): - self.assertEqual( - BuildTarget(version="1.1.0", platform="value", snapshot=False).platform, "value" - ) + self.assertEqual(BuildTarget(version="1.1.0", platform="value", snapshot=False).platform, "value") @patch("build_workflow.build_target.current_architecture", return_value="value") def test_arch(self, *mock): self.assertEqual(BuildTarget(version="1.1.0", snapshot=False).architecture, "value") def test_arch_value(self): - self.assertEqual( - BuildTarget(version="1.1.0", architecture="value", snapshot=False).architecture, "value" - ) + self.assertEqual(BuildTarget(version="1.1.0", architecture="value", snapshot=False).architecture, "value") diff --git a/tests/tests_build_workflow/test_builder_from_dist.py b/tests/tests_build_workflow/test_builder_from_dist.py index 5a8fb455b5..0bff60f21f 100644 --- a/tests/tests_build_workflow/test_builder_from_dist.py +++ b/tests/tests_build_workflow/test_builder_from_dist.py @@ -6,13 +6,12 @@ import os import unittest -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch from build_workflow.build_target import BuildTarget from build_workflow.builder_from_dist import BuilderFromDist from manifests.build_manifest import BuildManifest from manifests.input_manifest import InputManifest -from paths.script_finder import ScriptFinder class TestBuilderFromDist(unittest.TestCase):