From bd3bc37556582b5fdbb76efbfe56c8d4835ff62a Mon Sep 17 00:00:00 2001 From: Silvio Giebl Date: Sat, 13 Jul 2024 22:30:08 +0200 Subject: [PATCH] Fix OciRepositoryHandler.getOrHeadMetadata: The image metadata can be different (missing indexAnnotations and manifestDescriptorAnnotations) depending on if it is served from cache or not, because the digest/size of the single image manifest is used. Instead, the digest that is referenced by the imageReference (could be an index) must be used. This requires to add the platform as a path parameter to the oci metadata endpoint to select the right metadata. --- .../internal/registry/OciMetadataRegistry.kt | 70 +++++++++--------- .../internal/registry/OciRepositoryHandler.kt | 71 ++++++++++--------- 2 files changed, 75 insertions(+), 66 deletions(-) diff --git a/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciMetadataRegistry.kt b/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciMetadataRegistry.kt index b4d1f8e3..a31b8e59 100644 --- a/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciMetadataRegistry.kt +++ b/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciMetadataRegistry.kt @@ -13,40 +13,40 @@ import java.util.* */ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { - data class Metadata(val metadata: OciMetadata, val platform: Platform, val digest: OciDigest, val size: Int) + class OciImageMetadata(val platformToMetadata: Map, val digest: OciDigest, val size: Int) - fun pullMetadataList( + fun pullImageMetadata( registry: String, imageReference: OciImageReference, credentials: Credentials?, - ): Mono> = + ): Mono = registryApi.pullManifest(registry, imageReference.name, imageReference.tag.replaceFirst('!', ':'), credentials) - .transformToMetadataList(registry, imageReference, credentials) + .transformToImageMetadata(registry, imageReference, credentials) - fun pullMetadataList( + fun pullImageMetadata( registry: String, imageReference: OciImageReference, digest: OciDigest, size: Int, credentials: Credentials?, - ): Mono> = registryApi.pullManifest(registry, imageReference.name, digest, size, credentials) - .transformToMetadataList(registry, imageReference, credentials) + ): Mono = registryApi.pullManifest(registry, imageReference.name, digest, size, credentials) + .transformToImageMetadata(registry, imageReference, credentials) - private fun Mono.transformToMetadataList( + private fun Mono.transformToImageMetadata( registry: String, imageReference: OciImageReference, credentials: Credentials?, - ): Mono> = flatMap { manifest -> - transformToMetadataList(registry, imageReference, manifest, credentials) + ): Mono = flatMap { manifest -> + transformToImageMetadata(registry, imageReference, manifest, credentials) } - private fun transformToMetadataList( + private fun transformToImageMetadata( registry: String, imageReference: OciImageReference, manifest: OciData, credentials: Credentials?, - ): Mono> = when (manifest.mediaType) { - INDEX_MEDIA_TYPE -> transformIndexToMetadataList( + ): Mono = when (manifest.mediaType) { + INDEX_MEDIA_TYPE -> transformIndexToImageMetadata( registry, imageReference, manifest, @@ -56,7 +56,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { LAYER_MEDIA_TYPE_PREFIX, ) - MANIFEST_MEDIA_TYPE -> transformManifestToMetadataList( + MANIFEST_MEDIA_TYPE -> transformManifestToImageMetadata( registry, imageReference, manifest, @@ -65,7 +65,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { LAYER_MEDIA_TYPE_PREFIX, ) - DOCKER_MANIFEST_LIST_MEDIA_TYPE -> transformIndexToMetadataList( + DOCKER_MANIFEST_LIST_MEDIA_TYPE -> transformIndexToImageMetadata( registry, imageReference, manifest, @@ -75,7 +75,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { DOCKER_LAYER_MEDIA_TYPE, ) - DOCKER_MANIFEST_MEDIA_TYPE -> transformManifestToMetadataList( + DOCKER_MANIFEST_MEDIA_TYPE -> transformManifestToImageMetadata( registry, imageReference, manifest, @@ -87,7 +87,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { else -> throw IllegalStateException("unsupported manifest media type '${manifest.mediaType}'") } - private fun transformIndexToMetadataList( + private fun transformIndexToImageMetadata( registry: String, imageReference: OciImageReference, index: OciData, @@ -95,12 +95,12 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { manifestMediaType: String, configMediaType: String, layerMediaTypePrefix: String, - ): Mono> { + ): Mono { val indexJsonObject = jsonObject(String(index.bytes)) val indexAnnotations = indexJsonObject.getStringMapOrEmpty("annotations") val metadataMonoList = indexJsonObject.get("manifests") { asArray().toList { - val (platform, manifestDescriptor) = asObject().decodeOciManifestDescriptor() + val (manifestDescriptorPlatform, manifestDescriptor) = asObject().decodeOciManifestDescriptor() if (manifestDescriptor.mediaType != manifestMediaType) { // TODO support nested index Mono.empty() } else { @@ -112,7 +112,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { credentials, ).flatMap { manifest -> if (manifest.mediaType != manifestMediaType) { - throw IllegalArgumentException("media type in manifest descriptor ($manifestMediaType) and manifest (${manifest.mediaType}) do not match") + throw IllegalStateException("media type in manifest descriptor ($manifestMediaType) and manifest (${manifest.mediaType}) do not match") } transformManifestToMetadata( registry, @@ -124,11 +124,10 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { configMediaType, layerMediaTypePrefix, ) - }.map { metadata -> - if ((platform != null) && (metadata.platform != platform)) { - throw IllegalArgumentException("platform in manifest descriptor ($platform) and config (${metadata.platform}) do not match") + }.doOnNext { (platform) -> + if ((manifestDescriptorPlatform != null) && (platform != manifestDescriptorPlatform)) { + throw IllegalStateException("platform in manifest descriptor ($manifestDescriptorPlatform) and config ($platform) do not match") } - metadata } } } @@ -136,17 +135,24 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { indexJsonObject.requireStringOrNull("mediaType", index.mediaType) indexJsonObject.requireLong("schemaVersion", 2) // the same order as in the manifest is guaranteed by mergeSequential - return Flux.mergeSequential(metadataMonoList).collectList() + return Flux.mergeSequential(metadataMonoList) + // linked to preserve the platform order + .collect({ LinkedHashMap() }) { map, (platform, metadata) -> + if (map.putIfAbsent(platform, metadata) != null) { + throw IllegalStateException("duplicate platform in image index: $platform") + } + } + .map { OciImageMetadata(it, index.digest, index.bytes.size) } } - private fun transformManifestToMetadataList( + private fun transformManifestToImageMetadata( registry: String, imageReference: OciImageReference, manifest: OciData, credentials: Credentials?, configMediaType: String, layerMediaTypePrefix: String, - ): Mono> = transformManifestToMetadata( + ): Mono = transformManifestToMetadata( registry, imageReference, manifest, @@ -155,7 +161,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { credentials, configMediaType, layerMediaTypePrefix, - ).map { listOf(it) } + ).map { OciImageMetadata(mapOf(it), manifest.digest, manifest.bytes.size) } private fun transformManifestToMetadata( registry: String, @@ -166,7 +172,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { credentials: Credentials?, configMediaType: String, layerMediaTypePrefix: String, - ): Mono { + ): Mono> { val manifestJsonObject = jsonObject(String(manifest.bytes)) val manifestAnnotations = manifestJsonObject.getStringMapOrEmpty("annotations") val configDescriptor = manifestJsonObject.get("config") { asObject().decodeOciDescriptor() } @@ -272,7 +278,8 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { ) } - Metadata( + Pair( + Platform(os, architecture, variant, osVersion, osFeatures), OciMetadata( imageReference, creationTime, @@ -292,9 +299,6 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) { indexAnnotations, layers, ), - Platform(os, architecture, variant, osVersion, osFeatures), - manifest.digest, - manifest.bytes.size, ) } } diff --git a/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciRepositoryHandler.kt b/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciRepositoryHandler.kt index e6f534dc..01748501 100644 --- a/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciRepositoryHandler.kt +++ b/src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciRepositoryHandler.kt @@ -14,6 +14,7 @@ import io.github.sgtsilvio.gradle.oci.mapping.VersionedCoordinates import io.github.sgtsilvio.gradle.oci.mapping.map import io.github.sgtsilvio.gradle.oci.metadata.* import io.github.sgtsilvio.gradle.oci.platform.Platform +import io.github.sgtsilvio.gradle.oci.platform.toPlatform import io.netty.handler.codec.http.HttpHeaderNames import io.netty.handler.codec.http.HttpHeaderValues import io.netty.handler.codec.http.HttpMethod @@ -38,7 +39,7 @@ import java.util.function.BiFunction /v2/repository/ / // / ///<...>oci-layer /v0.11/ / // / <...>.module -/v0.11/ / // / ///<...>.json +/v0.11/ / // / ////<...>.json /v0.11/ / // / ///<...>oci-layer */ @@ -51,10 +52,10 @@ internal class OciRepositoryHandler( private val credentials: Credentials?, ) : BiFunction> { - private val metadataCache: AsyncCache> = + private val imageMetadataCache: AsyncCache = Caffeine.newBuilder().maximumSize(100).expireAfterAccess(1, TimeUnit.MINUTES).buildAsync() - private data class ComponentCacheKey( + private data class ImageMetadataCacheKey( val registry: String, val imageReference: OciImageReference, val digest: OciDigest?, @@ -109,7 +110,7 @@ internal class OciRepositoryHandler( isGet: Boolean, response: HttpServerResponse, ): Publisher { - if (segments.size != 8) { + if (segments.size != 9) { return response.sendNotFound() } val imageReference = try { @@ -127,9 +128,20 @@ internal class OciRepositoryHandler( } catch (e: NumberFormatException) { return response.sendBadRequest() } - val metadataJsonMono = getMetadata(registryUri, imageReference, digest, size, credentials).map { (metadata) -> - metadata.encodeToJsonString().toByteArray() + val platform = try { + segments[7].toPlatform() + } catch (e: IllegalArgumentException) { + return response.sendBadRequest() } + val metadataJsonMono = + getImageMetadata(registryUri, imageReference, digest, size, credentials).handle { imageMetadata, sink -> + val metadata = imageMetadata.platformToMetadata[platform] + if (metadata == null) { + response.status(400) + } else { + sink.next(metadata.encodeToJsonString().toByteArray()) + } + } response.header(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON) return response.sendByteArray(metadataJsonMono, isGet) } @@ -176,8 +188,8 @@ internal class OciRepositoryHandler( ): Publisher { val componentId = mappedComponent.componentId val variantMetadataMonoList = mappedComponent.variants.map { (variantName, variant) -> - getMetadataList(registryUri, variant.imageReference, credentials).map { metadataList -> - Triple(variantName, variant.capabilities, metadataList) + getImageMetadata(registryUri, variant.imageReference, credentials).map { imageMetadata -> + Triple(variantName, variant.capabilities, imageMetadata) } } val moduleJsonMono = variantMetadataMonoList.zip { variantMetadataList -> @@ -193,18 +205,18 @@ internal class OciRepositoryHandler( } val fileNamePrefix = "${componentId.name}-${componentId.version}-" addArray("variants") { - for ((variantName, capabilities, metadataList) in variantMetadataList) { + for ((variantName, capabilities, imageMetadata) in variantMetadataList) { addObject { addString("name", createOciVariantName(variantName)) addOciVariantAttributes(MULTIPLE_PLATFORMS_ATTRIBUTE_VALUE) addCapabilities("capabilities", capabilities, componentId) addArray("dependencies") { - for ((_, platform) in metadataList) { + for (platform in imageMetadata.platformToMetadata.keys) { addDependency(componentId, capabilities, platform) } } } - for ((metadata, platform, digest, size) in metadataList) { + for ((platform, metadata) in imageMetadata.platformToMetadata) { addObject { addString("name", createOciVariantName(variantName, platform)) addOciVariantAttributes(platform.toString()) @@ -223,7 +235,7 @@ internal class OciRepositoryHandler( val metadataName = fileNamePrefix + createOciMetadataClassifier(variantName) + createPlatformPostfix(platform) + ".json" val escapedImageReference = metadata.imageReference.toString().escapePathSegment() addString("name", metadataName) - addString("url", "$escapedImageReference/$digest/$size/$metadataName") + addString("url", "$escapedImageReference/${imageMetadata.digest}/${imageMetadata.size}/$platform/$metadataName") addNumber("size", metadataJson.size.toLong()) addString("sha512", DigestUtils.sha512Hex(metadataJson)) addString("sha256", DigestUtils.sha256Hex(metadataJson)) @@ -256,41 +268,34 @@ internal class OciRepositoryHandler( return response.sendByteArray(moduleJsonMono, isGet) } - private fun getMetadataList( + private fun getImageMetadata( registryUri: URI, imageReference: OciImageReference, credentials: Credentials?, - ): Mono> { - return metadataCache.getMono( - ComponentCacheKey(registryUri.toString(), imageReference, null, -1, credentials?.hashed()) + ): Mono { + return imageMetadataCache.getMono( + ImageMetadataCacheKey(registryUri.toString(), imageReference, null, -1, credentials?.hashed()) ) { key -> - metadataRegistry.pullMetadataList(key.registry, key.imageReference, credentials).doOnNext { metadataList -> - for (metadata in metadataList) { - metadataCache.asMap().putIfAbsent( - key.copy(digest = metadata.digest, size = metadata.size), - CompletableFuture.completedFuture(listOf(metadata)), - ) - } + metadataRegistry.pullImageMetadata(key.registry, key.imageReference, credentials).doOnNext { + imageMetadataCache.asMap().putIfAbsent( + key.copy(digest = it.digest, size = it.size), + CompletableFuture.completedFuture(it), + ) } } } - private fun getMetadata( + private fun getImageMetadata( registryUri: URI, imageReference: OciImageReference, digest: OciDigest, size: Int, credentials: Credentials?, - ): Mono { - return metadataCache.getMono( - ComponentCacheKey(registryUri.toString(), imageReference, digest, size, credentials?.hashed()) + ): Mono { + return imageMetadataCache.getMono( + ImageMetadataCacheKey(registryUri.toString(), imageReference, digest, size, credentials?.hashed()) ) { (registry, imageReference) -> - metadataRegistry.pullMetadataList(registry, imageReference, digest, size, credentials) - }.map { metadataList -> - if (metadataList.size != 1) { - throw IllegalStateException() // TODO message - } - metadataList[0] + metadataRegistry.pullImageMetadata(registry, imageReference, digest, size, credentials) } }