Local s3 support added

README.md

@@ -223,10 +223,15 @@ The following configuration values are available:
 | rejectStorageWriteOnLowMemory           | redis  | false                    | When set to _true_, PUT requests with the x-importance-level header can be rejected when memory gets low                              |
 | freeMemoryCheckIntervalMs               | redis  | 60000                    | The interval in milliseconds to calculate the actual memory usage                                                                     |
 | redisReadyCheckIntervalMs               | redis  | -1                       | The interval in milliseconds to calculate the "ready state" of redis. When value < 1, no "ready state" will be calculated             |
-| awsS3Region                             | aws-s3 |                          | The region of S3 server                                                                                                               |
-| awsS3BucketName                         | aws-s3 |                          | The S3 bucket name                                                                                                                    |
-| awsS3AccessKeyId                        | aws-s3 |                          | The AWS access key Id                                                                                                                 |
-| awsS3SecretAccessKey                    | aws-s3 |                          | The AWS secret access key                                                                                                             |
+| awsS3Region                             | s3     |                          | The region of AWS S3 server, with local service such localstack, also need set a valid region                                         |
+| s3BucketName                            | s3     |                          | The S3 bucket name                                                                                                                    |
+| s3AccessKeyId                           | s3     |                          | The s3 access key Id                                                                                                                  |
+| s3SecretAccessKey                       | s3     |                          | The s3 secret access key                                                                                                              |
+| localS3                                 | s3     |                          | Set to true in order to use a local S3 instance instead of AWS                                                                        |
+| localS3Endpoint                         | s3     |                          | The endpoint/host to use in case that localS3 is set to true, e.g. 127.0.0.1 (in my case it had to be an IP)                          |
+| localS3Port                             | s3     |                          | The port to use in case that localS3 is set to true, e.g. 4566                                                                        |
+| createBucketIfNotExist                  | s3     |                          | create bucket if bucket not exist, related permission required                                                                        |
+
 
 ### Configuration util
 
@@ -260,8 +265,13 @@ The data is stored hierarchically on the file system. This is the default storag
 
 ### S3 storage
 The data is stored in a S3 instance.
-Before use, need sign up in AWS service at https://docs.aws.amazon.com/SetUp/latest/UserGuide/setup-AWSsignup.html
-you also need create bucket from S3 web console first
+
+#### AWS S3
+See https://aws.amazon.com/s3 it is also possible to use a local instance using https://docs.localstack.cloud/user-guide/aws/s3/
+
+
+docker run --rm -p 4566:4566 -v ./s3:/var/lib/localstack localstack/localstack:s3-latest
+
 
 ### Redis Storage
 The data is stored in a redis database.

src/main/java/org/swisspush/reststorage/RestStorageMod.java

@@ -80,8 +80,11 @@ private Future<Storage> createStorage(ModuleConfiguration moduleConfiguration) {
                 break;
             case s3:
                 promise.complete(new S3FileSystemStorage(vertx, exceptionFactory, moduleConfiguration.getRoot(),
-                        moduleConfiguration.getAwsS3Region(), moduleConfiguration.getAwsS3BucketName(),
-                        moduleConfiguration.getAwsS3AccessKeyId(), moduleConfiguration.getAwsS3SecretAccessKey()));
+                        moduleConfiguration.getAwsS3Region(), moduleConfiguration.getS3BucketName(),
+                        moduleConfiguration.getS3AccessKeyId(), moduleConfiguration.getS3SecretAccessKey(),
+                        moduleConfiguration.getS3UseTlsConnection(), moduleConfiguration.isLocalS3(),
+                        moduleConfiguration.getLocalS3Endpoint(), moduleConfiguration.getLocalS3Port(),
+                        moduleConfiguration.getCreateBucketIfNotPresentYet()));
                 break;
             case redis:
                 createRedisStorage(vertx, moduleConfiguration).onComplete(event -> {

src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java

@@ -79,14 +79,11 @@ private void listDirBlocking(Path path, int offset, int count, Promise<Collectio
                 log.trace("Processing entry '{}'", entryName);
                 // Create resource representing currently processed directory entry.
                 final Resource resource;
-                if (Files.isDirectory(entry)) {
+                if (entryName.endsWith(S3_PATH_SEPARATOR)) {
                     resource = new CollectionResource();
                     entryName = entryName.replace(S3_PATH_SEPARATOR, "");
-                } else if (Files.isRegularFile(entry)) {
-                    resource = new DocumentResource();
                 } else {
-                    resource = new Resource();
-                    resource.exists = false;
+                    resource = new DocumentResource();
                 }
                 resource.name = entryName;
                 collection.items.add(resource);

src/main/java/org/swisspush/reststorage/s3/S3FileSystemStorage.java

@@ -2,6 +2,7 @@
 
 import io.vertx.core.Handler;
 import io.vertx.core.Vertx;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.swisspush.reststorage.CollectionResource;
@@ -21,6 +22,7 @@
 import java.net.URI;
 import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystemAlreadyExistsException;
 import java.nio.file.FileSystemException;
 import java.nio.file.FileSystems;
 import java.nio.file.FileVisitResult;
@@ -31,6 +33,7 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -62,23 +65,54 @@ private static S3FileSystem getFileSystem(URI uri) {
     }
 
     public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFactory, String rootPath,
-                               String awsS3Region, String awsS3BucketName, String awsS3AccessKeyId, String awsS3SecretAccessKey) {
+                               String awsS3Region, String s3BucketName, String s3AccessKeyId, String s3SecretAccessKey,
+                               boolean useTlsConnection, boolean isLocalS3, String localS3Endpoint, int localS3Port, boolean createBucketIfNotPresentYet) {
         this.vertx = vertx;
         this.exceptionFactory = exceptionFactory;
+        Objects.requireNonNull(s3BucketName, "BucketName must not be null");
         Objects.requireNonNull(awsS3Region, "Region must not be null");
-        Objects.requireNonNull(awsS3BucketName, "BucketName must not be null");
-        Objects.requireNonNull(awsS3AccessKeyId, "AccessKeyId must not be null");
-        Objects.requireNonNull(awsS3SecretAccessKey, "SecretAccessKey must not be null");
+        System.setProperty("aws.region", awsS3Region);
 
-        System.setProperty("aws.region",awsS3Region);
-        System.setProperty("aws.accessKeyId", awsS3AccessKeyId);
-        System.setProperty("aws.secretAccessKey", awsS3SecretAccessKey);
+        if (isLocalS3) {
+            // local S3, AWS SDK requires these two properties to be set
+            System.setProperty("aws.accessKeyId", "local");
+            System.setProperty("aws.secretAccessKey", "local");
 
-        if (!awsS3BucketName.startsWith("s3:") && !awsS3BucketName.startsWith("s3x:")) {
-            awsS3BucketName = "s3://" + awsS3BucketName;
+            String credentials = "";
+            if (StringUtils.isNotEmpty(s3AccessKeyId) && StringUtils.isNotEmpty(s3SecretAccessKey)) {
+                credentials = s3AccessKeyId + ":" + s3SecretAccessKey + "@";
+            }
+            String port = "";
+            if (localS3Port > 0) {
+                port = ":" + localS3Port;
+            }
+            // s3x://[key:secret@]endpoint[:port]/bucket
+            s3BucketName = "s3x://" + credentials + localS3Endpoint + port + "/" + s3BucketName;
+            if (!useTlsConnection) {
+                System.setProperty("s3.spi.endpoint-protocol", "http");
+            }
+        } else {
+            // AWS S3
+            Objects.requireNonNull(s3AccessKeyId, "AccessKeyId must not be null");
+            Objects.requireNonNull(s3SecretAccessKey, "SecretAccessKey must not be null");
+            System.setProperty("aws.accessKeyId", s3AccessKeyId);
+            System.setProperty("aws.secretAccessKey", s3SecretAccessKey);
+            s3BucketName = "s3://" + s3BucketName;
+        }
+
+        var uri = URI.create(s3BucketName);
+
+        if (createBucketIfNotPresentYet) {
+            try (var fs = FileSystems.newFileSystem(uri,
+                    Map.of("locationConstraint", awsS3Region))) {
+                log.info("Bucket created: " + fs.toString());
+            } catch (FileSystemAlreadyExistsException e) {
+                log.info("Bucket " + s3BucketName + " already exists: ", e);
+            } catch (IOException e) {
+                log.error("Failed to create bucket " + s3BucketName, e);
+            }
         }
 
-        var uri = URI.create(awsS3BucketName);
         fileSystem = getFileSystem(uri);
         root = fileSystem.getPath(rootPath);
 
@@ -95,7 +129,7 @@ public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFac
 
         // Cache string length of root without trailing slashes
         int rootLen;
-        for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen) ;
+        for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen);
         this.rootLen = rootLen;
     }
 

src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java

@@ -70,10 +70,15 @@ public enum StorageType {
     private int maxRedisWaitingHandlers = 2048;
     private int maxStorageExpandSubresources = 1000;
 
-    private String awsS3BucketName = null;
+    private String s3BucketName = null;
     private String awsS3Region = null;
-    private String awsS3AccessKeyId = null;
-    private String awsS3SecretAccessKey = null;
+    private String s3AccessKeyId = null;
+    private String s3SecretAccessKey = null;
+    private boolean s3UseTlsConnection = true;
+    private boolean createBucketIfNotPresentYet = false;
+    private boolean localS3 = false;
+    private String localS3Endpoint = null;
+    private int localS3Port = 0;
 
     public ModuleConfiguration root(String root) {
         this.root = root;
@@ -301,18 +306,43 @@ public ModuleConfiguration awsS3Region(String awsS3Region) {
         return this;
     }
 
-    public ModuleConfiguration awsS3BucketName(String awsS3BucketName) {
-        this.awsS3BucketName = awsS3BucketName;
+    public ModuleConfiguration s3BucketName(String awsS3BucketName) {
+        this.s3BucketName = awsS3BucketName;
         return this;
     }
 
-    public ModuleConfiguration awsS3AccessKeyId(String awsS3AccessKeyId) {
-        this.awsS3AccessKeyId = awsS3AccessKeyId;
+    public ModuleConfiguration s3AccessKeyId(String awsS3AccessKeyId) {
+        this.s3AccessKeyId = awsS3AccessKeyId;
         return this;
     }
 
-    public ModuleConfiguration awsS3SecretAccessKey(String awsS3SecretAccessKey) {
-        this.awsS3SecretAccessKey = awsS3SecretAccessKey;
+    public ModuleConfiguration s3SecretAccessKey(String awsS3SecretAccessKey) {
+        this.s3SecretAccessKey = awsS3SecretAccessKey;
+        return this;
+    }
+
+    public ModuleConfiguration s3UseTlsConnection(boolean s3UseTlsConnection) {
+        this.s3UseTlsConnection = s3UseTlsConnection;
+        return this;
+    }
+
+    public ModuleConfiguration localS3Endpoint(String s3Endpoint) {
+        this.localS3Endpoint = s3Endpoint;
+        return this;
+    }
+
+    public ModuleConfiguration localS3Port(int s3Port) {
+        this.localS3Port = s3Port;
+        return this;
+    }
+
+    public ModuleConfiguration createBucketIfNotPresentYet(boolean createBucketIfNotExist) {
+        this.createBucketIfNotPresentYet = createBucketIfNotExist;
+        return this;
+    }
+
+    public ModuleConfiguration localS3(boolean localS3) {
+        this.localS3 = localS3;
         return this;
     }
 
@@ -493,16 +523,36 @@ public String getAwsS3Region() {
         return awsS3Region;
     }
 
-    public String getAwsS3BucketName() {
-        return awsS3BucketName;
+    public String getS3BucketName() {
+        return s3BucketName;
+    }
+
+    public String getS3AccessKeyId() {
+        return s3AccessKeyId;
+    }
+
+    public String getS3SecretAccessKey() {
+        return s3SecretAccessKey;
+    }
+
+    public boolean getS3UseTlsConnection() {
+        return s3UseTlsConnection;
+    }
+
+    public String getLocalS3Endpoint() {
+        return localS3Endpoint;
+    }
+
+    public int getLocalS3Port() {
+        return localS3Port;
     }
 
-    public String getAwsS3AccessKeyId() {
-        return awsS3AccessKeyId;
+    public boolean getCreateBucketIfNotPresentYet() {
+        return createBucketIfNotPresentYet;
     }
 
-    public String getAwsS3SecretAccessKey() {
-        return awsS3SecretAccessKey;
+    public boolean isLocalS3() {
+        return localS3;
     }
 
     public JsonObject asJsonObject() {