From c3f6043440657681438491c0943eee6ed5fb2354 Mon Sep 17 00:00:00 2001 From: Lars Hagen Date: Thu, 19 Oct 2023 15:31:05 +0200 Subject: [PATCH 1/4] implement support for SharedKey signature This is the recommended signature scheme for Azure, and the only scheme that is supported by the Azurite emulator. https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key --- .../main/java/org/jclouds/http/HttpUtils.java | 4 + .../azure/storage/config/AuthType.java | 4 +- .../filters/SharedKeyLiteAuthentication.java | 122 ++++++++++++++++-- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/jclouds/http/HttpUtils.java b/core/src/main/java/org/jclouds/http/HttpUtils.java index 0d41d7c83ff..eb36a50c126 100644 --- a/core/src/main/java/org/jclouds/http/HttpUtils.java +++ b/core/src/main/java/org/jclouds/http/HttpUtils.java @@ -272,6 +272,10 @@ public static String nullToEmpty(byte[] md5) { return md5 != null ? base64().encode(md5) : ""; } + public static String nullOrZeroToEmpty(Long contentLength) { + return contentLength != null && contentLength > 0 ? contentLength.toString() : ""; + } + public static String nullToEmpty(Collection collection) { return (collection == null || collection.isEmpty()) ? "" : collection.iterator().next(); } diff --git a/providers/azureblob/src/main/java/org/jclouds/azure/storage/config/AuthType.java b/providers/azureblob/src/main/java/org/jclouds/azure/storage/config/AuthType.java index 986f21dfc6f..fd3b2ee05e2 100644 --- a/providers/azureblob/src/main/java/org/jclouds/azure/storage/config/AuthType.java +++ b/providers/azureblob/src/main/java/org/jclouds/azure/storage/config/AuthType.java @@ -24,7 +24,9 @@ public enum AuthType { /** Includes both the API key and SAS credentials */ AZURE_KEY, /** Azure AD credentials */ - AZURE_AD; + AZURE_AD, + /** Uses the SharedKey scheme, rather than SharedKeyLite */ + AZURE_SHARED_KEY; @Override public String toString() { diff --git a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java index 2d8f3b62e48..b1f259365d1 100644 --- a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java +++ b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java @@ -34,6 +34,14 @@ import javax.inject.Provider; import javax.inject.Singleton; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Multiset; +import com.google.common.collect.TreeMultiset; import org.jclouds.Constants; import org.jclouds.azure.storage.config.AuthType; import org.jclouds.azure.storage.util.storageurl.StorageUrlSupplier; @@ -56,13 +64,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; import com.google.common.io.ByteProcessor; import com.google.common.net.HttpHeaders; @@ -74,6 +76,9 @@ @Singleton public class SharedKeyLiteAuthentication implements HttpRequestFilter { private static final Collection FIRST_HEADERS_TO_SIGN = ImmutableList.of(HttpHeaders.DATE); + private static final Collection FIRST_HEADERS_TO_SIGN_FOR_SHARED_KEY = + ImmutableList.of(HttpHeaders.DATE, HttpHeaders.IF_MODIFIED_SINCE, HttpHeaders.IF_MATCH, + HttpHeaders.IF_NONE_MATCH, HttpHeaders.IF_UNMODIFIED_SINCE, HttpHeaders.RANGE); private final SignatureWire signatureWire; private final Supplier creds; private final Provider timeStampProvider; @@ -114,6 +119,8 @@ public SharedKeyLiteAuthentication(SignatureWire signatureWire, public HttpRequest filter(HttpRequest request) throws HttpException { if (this.authType == AuthType.AZURE_AD) { request = this.oAuthFilter.filter(request); + } else if (this.authType == AuthType.AZURE_SHARED_KEY){ + request = this.isSAS ? filterSAS(request, this.credential) : filterSharedKey(request); } else { request = this.isSAS ? filterSAS(request, this.credential) : filterKey(request); } @@ -153,7 +160,22 @@ public HttpRequest filterKey(HttpRequest request) throws HttpException { String signature = calculateSignature(createStringToSign(request)); return replaceAuthorizationHeader(request, signature); } - + + /** + * this is a 'standard' filter method, applied when SharedKey authentication is used. + */ + public HttpRequest filterSharedKey(HttpRequest request) throws HttpException { + request = replaceDateHeader(request); + String signature = calculateSignature(createStringToSignForSharedKey(request)); + return replaceAuthorizationHeaderForSharedKey(request, signature); + } + + HttpRequest replaceAuthorizationHeaderForSharedKey(HttpRequest request, String signature) { + return request.toBuilder() + .replaceHeader(HttpHeaders.AUTHORIZATION, "SharedKey " + creds.get().identity + ":" + signature) + .build(); + } + HttpRequest replaceAuthorizationHeader(HttpRequest request, String signature) { return request.toBuilder() .replaceHeader(HttpHeaders.AUTHORIZATION, "SharedKeyLite " + creds.get().identity + ":" + signature) @@ -187,7 +209,22 @@ public String[] cutUri(URI uri) throws IllegalArgumentException { throw new IllegalArgumentException("there is neither ContainerName nor BlobName in the URI path"); } return result; - } + } + + public String createStringToSignForSharedKey(HttpRequest request) { + utils.logRequest(signatureLog, request, ">>"); + StringBuilder buffer = new StringBuilder(); + // re-sign the request + appendMethod(request, buffer); + appendPayloadMetadataForSharedKey(request, buffer); + appendHttpHeadersForSharedKey(request, buffer); + appendCanonicalizedHeaders(request, buffer); + appendCanonicalizedResourceForSharedKey(request, buffer); + if (signatureWire.enabled()) + signatureWire.output(buffer.toString()); + System.out.println(buffer); + return buffer.toString(); + } public String createStringToSign(HttpRequest request) { utils.logRequest(signatureLog, request, ">>"); @@ -203,6 +240,24 @@ public String createStringToSign(HttpRequest request) { return buffer.toString(); } + private void appendPayloadMetadataForSharedKey(HttpRequest request, StringBuilder buffer) { + buffer.append( + Strings.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() + .getContentEncoding())).append("\n"); + buffer.append( + Strings.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() + .getContentLanguage())).append("\n"); + buffer.append( + HttpUtils.nullOrZeroToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() + .getContentLength())).append("\n"); + buffer.append( + HttpUtils.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() + .getContentMD5())).append("\n"); + buffer.append( + Strings.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() + .getContentType())).append("\n"); + } + private void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) { buffer.append( HttpUtils.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() @@ -216,6 +271,7 @@ public String calculateSignature(String toSign) throws HttpException { String signature = signString(toSign); if (signatureWire.enabled()) signatureWire.input(Strings2.toInputStream(signature)); + System.out.println(signature); return signature; } @@ -260,6 +316,11 @@ private void appendHttpHeaders(HttpRequest request, StringBuilder toSign) { toSign.append(HttpUtils.nullToEmpty(request.getHeaders().get(header))).append("\n"); } + private void appendHttpHeadersForSharedKey(HttpRequest request, StringBuilder toSign) { + for (String header : FIRST_HEADERS_TO_SIGN_FOR_SHARED_KEY) + toSign.append(HttpUtils.nullToEmpty(request.getHeaders().get(header))).append("\n"); + } + @VisibleForTesting void appendCanonicalizedResource(HttpRequest request, StringBuilder toSign) { // 1. Beginning with an empty string (""), append a forward slash (/), followed by the name of @@ -268,6 +329,51 @@ void appendCanonicalizedResource(HttpRequest request, StringBuilder toSign) { appendUriPath(request, toSign); } + void appendCanonicalizedResourceForSharedKey(HttpRequest request, StringBuilder toSign) { + // 1. Beginning with an empty string (""), append a forward slash (/), followed by the name of + // the identity that owns the resource being accessed. + toSign.append("/").append(creds.get().identity); + // 2. Append the resource's encoded URI path + toSign.append(request.getEndpoint().getRawPath()); + appendQueryParametersForSharedKey(request, toSign); + } + + void appendQueryParametersForSharedKey(HttpRequest request, StringBuilder toSign) { + // 3. Append each query parameter as a new line + Map> sortedParams = Maps.newTreeMap(); + if (request.getEndpoint().getQuery() != null) { + String[] params = request.getEndpoint().getQuery().split("&"); + for (String param : params) { + String[] paramNameAndValue = param.split("="); + String key = paramNameAndValue[0]; + String value = paramNameAndValue.length > 1 ? paramNameAndValue[1] : ""; + if (sortedParams.containsKey(key)) { + sortedParams.get(key).add(value); + } else { + Multiset values = TreeMultiset.create(); + values.add(value); + sortedParams.put(key, values); + } + } + } + + for (Entry> entry : sortedParams.entrySet()) { + String key = entry.getKey(); + Multiset values = entry.getValue(); + toSign.append("\n"); + toSign.append(key); + toSign.append(":"); + boolean first = true; + for (String value : values) { + if (!first) { + toSign.append(","); + } + toSign.append(value); + first = false; + } + } + } + @VisibleForTesting void appendUriPath(HttpRequest request, StringBuilder toSign) { // 2. Append the resource's encoded URI path From d48bd3537c4428ba7387104d9dd9e7bfc4cbc7fe Mon Sep 17 00:00:00 2001 From: Lars Hagen Date: Thu, 19 Oct 2023 17:13:52 +0200 Subject: [PATCH 2/4] Remove print statement Co-authored-by: Ignasi Barrera --- .../azure/storage/filters/SharedKeyLiteAuthentication.java | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java index b1f259365d1..22d80c768b9 100644 --- a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java +++ b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java @@ -222,7 +222,6 @@ public String createStringToSignForSharedKey(HttpRequest request) { appendCanonicalizedResourceForSharedKey(request, buffer); if (signatureWire.enabled()) signatureWire.output(buffer.toString()); - System.out.println(buffer); return buffer.toString(); } From e918c6a904b691db3f68809d069b722f7a9e9104 Mon Sep 17 00:00:00 2001 From: Lars Hagen Date: Thu, 19 Oct 2023 17:36:21 +0200 Subject: [PATCH 3/4] Remove print statement --- .../azure/storage/filters/SharedKeyLiteAuthentication.java | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java index 22d80c768b9..9188c92e447 100644 --- a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java +++ b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java @@ -270,7 +270,6 @@ public String calculateSignature(String toSign) throws HttpException { String signature = signString(toSign); if (signatureWire.enabled()) signatureWire.input(Strings2.toInputStream(signature)); - System.out.println(signature); return signature; } From 0d717a88770577b0dba174aed14ae295e614c2f2 Mon Sep 17 00:00:00 2001 From: Lars Hagen Date: Thu, 19 Oct 2023 17:42:23 +0200 Subject: [PATCH 4/4] simplify logic --- .../filters/SharedKeyLiteAuthentication.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java index 9188c92e447..cfa2d4997ee 100644 --- a/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java +++ b/providers/azureblob/src/main/java/org/jclouds/azure/storage/filters/SharedKeyLiteAuthentication.java @@ -55,6 +55,8 @@ import org.jclouds.http.Uris; import org.jclouds.http.Uris.UriBuilder; import org.jclouds.http.internal.SignatureWire; +import org.jclouds.io.ContentMetadata; +import org.jclouds.io.Payload; import org.jclouds.logging.Logger; import org.jclouds.oauth.v2.filters.OAuthFilter; import org.jclouds.util.Strings2; @@ -240,21 +242,23 @@ public String createStringToSign(HttpRequest request) { } private void appendPayloadMetadataForSharedKey(HttpRequest request, StringBuilder buffer) { - buffer.append( - Strings.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentEncoding())).append("\n"); - buffer.append( - Strings.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentLanguage())).append("\n"); - buffer.append( - HttpUtils.nullOrZeroToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentLength())).append("\n"); - buffer.append( - HttpUtils.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentMD5())).append("\n"); - buffer.append( - Strings.nullToEmpty(request.getPayload() == null ? null : request.getPayload().getContentMetadata() - .getContentType())).append("\n"); + Payload payload = request.getPayload(); + if (payload == null) { + buffer.append("\n\n\n\n\n"); + return; + } + + ContentMetadata contentMetadata = payload.getContentMetadata(); + buffer.append(Strings.nullToEmpty(contentMetadata.getContentEncoding())) + .append("\n"); + buffer.append(Strings.nullToEmpty(contentMetadata.getContentLanguage())) + .append("\n"); + buffer.append(HttpUtils.nullOrZeroToEmpty(contentMetadata.getContentLength())) + .append("\n"); + buffer.append(HttpUtils.nullToEmpty(contentMetadata.getContentMD5())) + .append("\n"); + buffer.append(Strings.nullToEmpty(contentMetadata.getContentType())) + .append("\n"); } private void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {