From 95fb6ce19189bda1c757ffb94ec9904d7c6c508e Mon Sep 17 00:00:00 2001 From: Robert Cerven Date: Tue, 22 Oct 2024 20:03:48 +0200 Subject: [PATCH] Allow konflux images with oci index, be used in osbs * STONEBLD-2896 Signed-off-by: Robert Cerven --- atomic_reactor/config.py | 2 +- atomic_reactor/plugins/check_base_image.py | 5 ++++ atomic_reactor/plugins/fetch_sources.py | 8 +++-- atomic_reactor/plugins/koji_parent.py | 5 ++-- atomic_reactor/plugins/resolve_composes.py | 11 +++++-- atomic_reactor/util.py | 34 ++++++++++++++++++---- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/atomic_reactor/config.py b/atomic_reactor/config.py index dd2df3a15..547444304 100644 --- a/atomic_reactor/config.py +++ b/atomic_reactor/config.py @@ -483,7 +483,7 @@ def hide_files(self): @property def skip_koji_check_for_base_image(self): return self._get_value(ReactorConfigKeys.SKIP_KOJI_CHECK_FOR_BASE_IMAGE_KEY, - fallback=False) + fallback=True) @property def deep_manifest_list_inspection(self): diff --git a/atomic_reactor/plugins/check_base_image.py b/atomic_reactor/plugins/check_base_image.py index 5923a7644..b02f07cc0 100644 --- a/atomic_reactor/plugins/check_base_image.py +++ b/atomic_reactor/plugins/check_base_image.py @@ -143,6 +143,11 @@ def _get_manifest_list(self, image: ImageName) -> requests.Response: reg_client = self._get_registry_client(image.registry) manifest_list = reg_client.get_manifest_list(image) + + if not manifest_list: + self.log.info('manifest list not found trying to get image index') + manifest_list = reg_client.get_manifest_index(image) + if '@sha256:' in str(image) and not manifest_list: # we want to adjust the tag only for manifest list fetching image = image.copy() diff --git a/atomic_reactor/plugins/fetch_sources.py b/atomic_reactor/plugins/fetch_sources.py index b7aea8deb..0ba7d3296 100644 --- a/atomic_reactor/plugins/fetch_sources.py +++ b/atomic_reactor/plugins/fetch_sources.py @@ -683,9 +683,11 @@ def get_srpm_urls(self, sigkeys=None, insecure=False): self.log.debug('Resolving SRPM for RPM ID: %s', rpm_id) if rpm['external_repo_name'] != 'INTERNAL': - msg = ('RPM comes from an external repo (RPM ID: {}). ' - 'External RPMs are currently not supported.').format(rpm_id) - raise RuntimeError(msg) + msg = ('RPM comes from an external repo (RPM ID: {}; NVR: {}). ' + 'External RPMs are currently not supported, ' + 'skipping').format(rpm_id, rpm['nvr']) + self.log.warning(msg) + continue rpm_hdr = self.session.getRPMHeaders(rpm_id, headers=['SOURCERPM']) if 'SOURCERPM' not in rpm_hdr: diff --git a/atomic_reactor/plugins/koji_parent.py b/atomic_reactor/plugins/koji_parent.py index f5df1b64f..05b323f30 100644 --- a/atomic_reactor/plugins/koji_parent.py +++ b/atomic_reactor/plugins/koji_parent.py @@ -23,7 +23,7 @@ import time -DEFAULT_POLL_TIMEOUT = 60 * 10 # 10 minutes +DEFAULT_POLL_TIMEOUT = 60 * 3 # 3 minutes DEFAULT_POLL_INTERVAL = 10 # 10 seconds @@ -295,7 +295,8 @@ def wait_for_parent_image_build(self, nvr: str) -> Dict[str, Any]: exc_msg = ('Parent image Koji build {} state is {}, not COMPLETE.') raise KojiParentBuildMissing(exc_msg.format(nvr, build_state)) time.sleep(self.poll_interval) - raise KojiParentBuildMissing('Parent image Koji build NOT found for {}!'.format(nvr)) + self.log.warning("Parent image Koji build NOT found for %s!", nvr) + return None def make_result(self) -> Optional[Dict[str, Any]]: """Construct the result dict to be preserved in the build metadata.""" diff --git a/atomic_reactor/plugins/resolve_composes.py b/atomic_reactor/plugins/resolve_composes.py index d2654aa0e..a71205359 100644 --- a/atomic_reactor/plugins/resolve_composes.py +++ b/atomic_reactor/plugins/resolve_composes.py @@ -148,7 +148,10 @@ def adjust_for_inherit(self): if not self.plugin_result: return - build_info = self.plugin_result[BASE_IMAGE_KOJI_BUILD] + build_info = self.plugin_result.get(BASE_IMAGE_KOJI_BUILD) + if not build_info: + self.log.warning('Parent koji build does not exist can not inherit from the parent') + return parent_repourls = [] try: @@ -218,7 +221,11 @@ def adjust_signing_intent_from_parent(self): PLUGIN_KOJI_PARENT_KEY) return - build_info = self.plugin_result[BASE_IMAGE_KOJI_BUILD] + build_info = self.plugin_result.get(BASE_IMAGE_KOJI_BUILD) + if not build_info: + self.log.warning('Parent koji build does not exist can not adjust ' + 'signing intent from the parent') + return try: parent_signing_intent_name = build_info['extra']['image']['odcs']['signing_intent'] diff --git a/atomic_reactor/util.py b/atomic_reactor/util.py index fa21e635d..3938a3866 100644 --- a/atomic_reactor/util.py +++ b/atomic_reactor/util.py @@ -762,6 +762,17 @@ def get_manifest_list(self, image: ImageName) -> Optional[requests.Response]: response, _ = self.get_manifest(image, version) return response + def get_manifest_index(self, image: ImageName) -> Optional[requests.Response]: + """Return manifest index for image. + + :param image: ImageName, the remote image to inspect + + :return: response, or None, with manifest list + """ + version = 'oci_index' + response, _ = self.get_manifest(image, version) + return response + def get_manifest_list_digest(self, image): """Return manifest list digest for image @@ -775,7 +786,7 @@ def get_manifest_list_digest(self, image): return 'sha256:{}'.format(digest_dict['sha256sum']) def get_all_manifests( - self, image: ImageName, versions: Sequence[str] = ('v1', 'v2', 'v2_list') + self, image: ImageName, versions: Sequence[str] = ('v1', 'v2', 'v2_list', 'oci_index') ) -> Dict[str, requests.Response]: """Return manifest digests for image. @@ -815,6 +826,11 @@ def get_inspect_for_image( blob_config, config_digest = self._config_and_id_from_manifest_list( image, manifest_list, arch ) + elif 'oci_index' in all_man_digests: + manifest_list = all_man_digests['oci_index'].json() + blob_config, config_digest = self._config_and_id_from_manifest_list( + image, manifest_list, arch + ) # get config for v2 digest elif 'v2' in all_man_digests: v2_json = all_man_digests['v2'].json() @@ -889,11 +905,17 @@ def _config_and_id_from_manifest_list( ) manifest = arch_manifests[0] - if manifest["mediaType"] != MEDIA_TYPE_DOCKER_V2_SCHEMA2: - raise RuntimeError(f"Image {image}: v2 schema 1 in manifest list") + if (manifest["mediaType"] != MEDIA_TYPE_DOCKER_V2_SCHEMA2 and + manifest["mediaType"] != MEDIA_TYPE_OCI_V1): + raise RuntimeError(f"Image {image}: v2 schema 1 in manifest list, " + f"oci in image index is missing") - v2_digest = manifest["digest"] - return self.get_config_and_id_from_registry(image, v2_digest, version='v2') + image_digest = manifest["digest"] + + if manifest["mediaType"] != MEDIA_TYPE_DOCKER_V2_SCHEMA2: + return self.get_config_and_id_from_registry(image, image_digest, version='v2') + else: + return self.get_config_and_id_from_registry(image, image_digest, version='oci') def get_config_and_id_from_registry(self, image, digest: str, version='v2') -> Tuple[Dict, str]: """Return image config by digest @@ -1115,7 +1137,7 @@ def get_manifest_list(image, registry, insecure=False, dockercfg_path=None): def get_all_manifests(image, registry, insecure=False, dockercfg_path=None, - versions=('v1', 'v2', 'v2_list')): + versions=('v1', 'v2', 'v2_list', 'oci_index')): """Return manifest digests for image. :param image: ImageName, the remote image to inspect