From 82e50eea0c9cd3a6626f7f3c731f41ef2d078362 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 20 Jul 2024 19:25:45 +0530 Subject: [PATCH] Add middleware to override storage class This is best-effort and some storage classes do not map exactly, particularly for non-S3 object stores. Fixes #625. --- README.md | 3 +- src/main/java/org/gaul/s3proxy/Main.java | 15 ++ .../org/gaul/s3proxy/S3ProxyConstants.java | 3 + .../gaul/s3proxy/StorageClassBlobStore.java | 114 +++++++++++++++ .../org/gaul/s3proxy/TierBlobStoreTest.java | 132 ++++++++++++++++++ 5 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/gaul/s3proxy/StorageClassBlobStore.java create mode 100644 src/test/java/org/gaul/s3proxy/TierBlobStoreTest.java diff --git a/README.md b/README.md index 1011fe03..7aed5457 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,9 @@ S3Proxy can modify its behavior based on middlewares: * [eventual consistency modeling](https://github.com/gaul/s3proxy/wiki/Middleware---eventual-consistency) * [large object mocking](https://github.com/gaul/s3proxy/wiki/Middleware-large-object-mocking) * [read-only](https://github.com/gaul/s3proxy/wiki/Middleware-read-only) -* [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend) * [regex rename blobs](https://github.com/gaul/s3proxy/wiki/Middleware-regex) +* [sharded backend containers](https://github.com/gaul/s3proxy/wiki/Middleware-sharded-backend) +* [storage class override](https://github.com/gaul/s3proxy/wiki/Middleware-storage-class-override) ## SSL Support diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index 7fea714b..35134cba 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -56,6 +56,7 @@ import org.jclouds.location.reference.LocationConstants; import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; import org.jclouds.openstack.swift.v1.blobstore.RegionScopedBlobStoreContext; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; @@ -271,6 +272,20 @@ private static BlobStore parseMiddlewareProperties(BlobStore blobStore, properties); } + var storageClass = properties.getProperty( + S3ProxyConstants.PROPERTY_STORAGE_CLASS_BLOBSTORE); + if (!Strings.isNullOrEmpty(storageClass)) { + System.err.println("Using storage class override backend"); + var storageClassBlobStore = + StorageClassBlobStore.newStorageClassBlobStore( + blobStore, storageClass); + blobStore = storageClassBlobStore; + System.err.println("Configuration storage class: " + storageClass); + // TODO: This only makes sense for S3 backends. + System.err.println("Mapping storage storage class to: " + + StorageClass.fromTier(storageClassBlobStore.getTier())); + } + return blobStore; } diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index fa615354..036b4e53 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -111,6 +111,9 @@ public final class S3ProxyConstants { /** Shard objects across a specified number of buckets. */ public static final String PROPERTY_SHARDED_BLOBSTORE = "s3proxy.sharded-blobstore"; + /** Override tier when creating blobs. */ + public static final String PROPERTY_STORAGE_CLASS_BLOBSTORE = + "s3proxy.storage-class-blobstore"; /** Maximum time skew allowed in signed requests. */ public static final String PROPERTY_MAXIMUM_TIME_SKEW = diff --git a/src/main/java/org/gaul/s3proxy/StorageClassBlobStore.java b/src/main/java/org/gaul/s3proxy/StorageClassBlobStore.java new file mode 100644 index 00000000..df545383 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/StorageClassBlobStore.java @@ -0,0 +1,114 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.Tier; +import org.jclouds.blobstore.domain.internal.BlobMetadataImpl; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.util.ForwardingBlobStore; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; + +/** + * This class implements a middleware to set the storage tier when creating + * objects. The class is configured via: + * + * s3proxy.storage-class-blobstore = VALUE + * + * VALUE can be anything from org.jclouds.s3.domain.StorageClass, e.g., + * STANDARD, STANDARD_IA, GLACIER_IR, DEEP_ARCHIVE. Some values do not + * translate exactly due to jclouds limitations, e.g., REDUCED_REDUNDANCY maps + * to STANDARD. This mapping is best effort especially for non-S3 object + * stores. + */ +public final class StorageClassBlobStore extends ForwardingBlobStore { + private final Tier tier; + + private StorageClassBlobStore(BlobStore delegate, + String storageClassString) { + super(delegate); + StorageClass storageClass; + try { + storageClass = StorageClass.valueOf( + storageClassString.toUpperCase()); + } catch (IllegalArgumentException iae) { + storageClass = StorageClass.STANDARD; + } + this.tier = storageClass.toTier(); + } + + static StorageClassBlobStore newStorageClassBlobStore(BlobStore blobStore, + String storageClass) { + return new StorageClassBlobStore(blobStore, storageClass); + } + + public Tier getTier() { + return tier; + } + + @Override + public String putBlob(String containerName, Blob blob) { + var newBlob = replaceTier(containerName, blob); + return delegate().putBlob(containerName, newBlob); + } + + @Override + public String putBlob(String containerName, Blob blob, + PutOptions options) { + var newBlob = replaceTier(containerName, blob); + return delegate().putBlob(containerName, newBlob, options); + } + + @Override + public MultipartUpload initiateMultipartUpload( + String container, BlobMetadata blobMetadata, PutOptions options) { + var newBlobMetadata = replaceTier(blobMetadata); + return delegate().initiateMultipartUpload(container, newBlobMetadata, + options); + } + + private Blob replaceTier(String containerName, Blob blob) { + var blobMeta = blob.getMetadata(); + var contentMeta = blob.getMetadata().getContentMetadata(); + return blobBuilder(containerName) + .name(blobMeta.getName()) + .type(blobMeta.getType()) + .tier(tier) + .userMetadata(blobMeta.getUserMetadata()) + .payload(blob.getPayload()) + .cacheControl(contentMeta.getCacheControl()) + .contentDisposition(contentMeta.getContentDisposition()) + .contentEncoding(contentMeta.getContentEncoding()) + .contentLanguage(contentMeta.getContentLanguage()) + .contentType(contentMeta.getContentType()) + .build(); + } + + private BlobMetadata replaceTier(BlobMetadata meta) { + return new BlobMetadataImpl(meta.getProviderId(), meta.getName(), + meta.getLocation(), meta.getUri(), meta.getETag(), + meta.getCreationDate(), meta.getLastModified(), + meta.getUserMetadata(), meta.getPublicUri(), + meta.getContainer(), meta.getContentMetadata(), meta.getSize(), + tier); + } + + // TODO: copyBlob +} diff --git a/src/test/java/org/gaul/s3proxy/TierBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/TierBlobStoreTest.java new file mode 100644 index 00000000..f0251605 --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/TierBlobStoreTest.java @@ -0,0 +1,132 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; + +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Tier; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.io.Payloads; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.jclouds.s3.domain.ObjectMetadata.StorageClass; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("UnstableApiUsage") +public final class TierBlobStoreTest { + private static final Logger logger = + LoggerFactory.getLogger(TierBlobStoreTest.class); + + private BlobStoreContext context; + private BlobStore blobStore; + private String containerName; + private BlobStore tierBlobStore; + + @Before + public void setUp() throws Exception { + containerName = TestUtils.createRandomContainerName(); + + //noinspection UnstableApiUsage + context = ContextBuilder + .newBuilder("transient") + .credentials("identity", "credential") + .modules(ImmutableList.of(new SLF4JLoggingModule())) + .build(BlobStoreContext.class); + blobStore = context.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + + tierBlobStore = StorageClassBlobStore.newStorageClassBlobStore( + blobStore, StorageClass.DEEP_ARCHIVE.toString()); + } + + @After + public void tearDown() throws Exception { + if (context != null) { + blobStore.deleteContainer(containerName); + context.close(); + } + } + + @Test + public void testPutNewBlob() { + var blobName = TestUtils.createRandomBlobName(); + var content = TestUtils.randomByteSource().slice(0, 1024); + var blob = tierBlobStore.blobBuilder(blobName).payload(content).build(); + tierBlobStore.putBlob(containerName, blob); + + var blobMetadata = tierBlobStore.blobMetadata(containerName, blobName); + assertThat(blobMetadata.getTier()).isEqualTo(Tier.ARCHIVE); + } + + @Test + public void testGetExistingBlob() { + var blobName = TestUtils.createRandomBlobName(); + var content = TestUtils.randomByteSource().slice(0, 1024); + var blob = blobStore.blobBuilder(blobName).payload(content).build(); + blobStore.putBlob(containerName, blob); + + var blobMetadata = tierBlobStore.blobMetadata(containerName, blobName); + assertThat(blobMetadata.getTier()).isEqualTo(Tier.STANDARD); + } + + @Test + public void testPutNewMpu() { + var blobName = TestUtils.createRandomBlobName(); + var content = TestUtils.randomByteSource().slice(0, 1024); + var blob = tierBlobStore.blobBuilder(blobName).payload(content).build(); + + var mpu = tierBlobStore.initiateMultipartUpload( + containerName, blob.getMetadata(), new PutOptions()); + + var payload = Payloads.newByteSourcePayload(content); + tierBlobStore.uploadMultipartPart(mpu, 1, payload); + + var parts = tierBlobStore.listMultipartUpload(mpu); + tierBlobStore.completeMultipartUpload(mpu, parts); + + var blobMetadata = tierBlobStore.blobMetadata(containerName, blobName); + assertThat(blobMetadata.getTier()).isEqualTo(Tier.ARCHIVE); + } + + @Test + public void testGetExistingMpu() { + var blobName = TestUtils.createRandomBlobName(); + var content = TestUtils.randomByteSource().slice(0, 1024); + var blob = blobStore.blobBuilder(blobName).payload(content).build(); + + var mpu = blobStore.initiateMultipartUpload( + containerName, blob.getMetadata(), new PutOptions()); + + var payload = Payloads.newByteSourcePayload(content); + blobStore.uploadMultipartPart(mpu, 1, payload); + + var parts = blobStore.listMultipartUpload(mpu); + blobStore.completeMultipartUpload(mpu, parts); + + var blobMetadata = tierBlobStore.blobMetadata(containerName, blobName); + assertThat(blobMetadata.getTier()).isEqualTo(Tier.STANDARD); + } +}