diff --git a/README.md b/README.md index 5d2178d..ddb9731 100644 --- a/README.md +++ b/README.md @@ -223,10 +223,15 @@ The following configuration values are available: | rejectStorageWriteOnLowMemory | redis | false | When set to _true_, PUT requests with the x-importance-level header can be rejected when memory gets low | | freeMemoryCheckIntervalMs | redis | 60000 | The interval in milliseconds to calculate the actual memory usage | | redisReadyCheckIntervalMs | redis | -1 | The interval in milliseconds to calculate the "ready state" of redis. When value < 1, no "ready state" will be calculated | -| awsS3Region | aws-s3 | | The region of S3 server | -| awsS3BucketName | aws-s3 | | The S3 bucket name | -| awsS3AccessKeyId | aws-s3 | | The AWS access key Id | -| awsS3SecretAccessKey | aws-s3 | | The AWS secret access key | +| awsS3Region | s3 | | The region of AWS S3 server, with local service such localstack, also need set a valid region | +| s3BucketName | s3 | | The S3 bucket name | +| s3AccessKeyId | s3 | | The s3 access key Id | +| s3SecretAccessKey | s3 | | The s3 secret access key | +| localS3 | s3 | | Set to true in order to use a local S3 instance instead of AWS | +| localS3Endpoint | s3 | | The endpoint/host to use in case that localS3 is set to true, e.g. 127.0.0.1 (in my case it had to be an IP) | +| localS3Port | s3 | | The port to use in case that localS3 is set to true, e.g. 4566 | +| createBucketIfNotExist | s3 | | create bucket if bucket not exist, related permission required | + ### Configuration util @@ -260,8 +265,13 @@ The data is stored hierarchically on the file system. This is the default storag ### S3 storage The data is stored in a S3 instance. -Before use, need sign up in AWS service at https://docs.aws.amazon.com/SetUp/latest/UserGuide/setup-AWSsignup.html -you also need create bucket from S3 web console first + +#### AWS S3 +See https://aws.amazon.com/s3 it is also possible to use a local instance using https://docs.localstack.cloud/user-guide/aws/s3/ + + +docker run --rm -p 4566:4566 -v ./s3:/var/lib/localstack localstack/localstack:s3-latest + ### Redis Storage The data is stored in a redis database. diff --git a/src/main/java/org/swisspush/reststorage/RestStorageMod.java b/src/main/java/org/swisspush/reststorage/RestStorageMod.java index d038e82..6347095 100644 --- a/src/main/java/org/swisspush/reststorage/RestStorageMod.java +++ b/src/main/java/org/swisspush/reststorage/RestStorageMod.java @@ -80,8 +80,11 @@ private Future createStorage(ModuleConfiguration moduleConfiguration) { break; case s3: promise.complete(new S3FileSystemStorage(vertx, exceptionFactory, moduleConfiguration.getRoot(), - moduleConfiguration.getAwsS3Region(), moduleConfiguration.getAwsS3BucketName(), - moduleConfiguration.getAwsS3AccessKeyId(), moduleConfiguration.getAwsS3SecretAccessKey())); + moduleConfiguration.getAwsS3Region(), moduleConfiguration.getS3BucketName(), + moduleConfiguration.getS3AccessKeyId(), moduleConfiguration.getS3SecretAccessKey(), + moduleConfiguration.getS3UseTlsConnection(), moduleConfiguration.isLocalS3(), + moduleConfiguration.getLocalS3Endpoint(), moduleConfiguration.getLocalS3Port(), + moduleConfiguration.getCreateBucketIfNotPresentYet())); break; case redis: createRedisStorage(vertx, moduleConfiguration).onComplete(event -> { diff --git a/src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java b/src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java index b4c4033..a3d2c7d 100644 --- a/src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java +++ b/src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java @@ -79,14 +79,11 @@ private void listDirBlocking(Path path, int offset, int count, Promise 0) { + port = ":" + localS3Port; + } + // s3x://[key:secret@]endpoint[:port]/bucket + s3BucketName = "s3x://" + credentials + localS3Endpoint + port + "/" + s3BucketName; + if (!useTlsConnection) { + System.setProperty("s3.spi.endpoint-protocol", "http"); + } + } else { + // AWS S3 + Objects.requireNonNull(s3AccessKeyId, "AccessKeyId must not be null"); + Objects.requireNonNull(s3SecretAccessKey, "SecretAccessKey must not be null"); + System.setProperty("aws.accessKeyId", s3AccessKeyId); + System.setProperty("aws.secretAccessKey", s3SecretAccessKey); + s3BucketName = "s3://" + s3BucketName; + } + + var uri = URI.create(s3BucketName); + + if (createBucketIfNotPresentYet) { + try (var fs = FileSystems.newFileSystem(uri, + Map.of("locationConstraint", awsS3Region))) { + log.info("Bucket created: " + fs.toString()); + } catch (FileSystemAlreadyExistsException e) { + log.info("Bucket " + s3BucketName + " already exists: ", e); + } catch (IOException e) { + log.error("Failed to create bucket " + s3BucketName, e); + } } - var uri = URI.create(awsS3BucketName); fileSystem = getFileSystem(uri); root = fileSystem.getPath(rootPath); @@ -95,7 +129,7 @@ public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFac // Cache string length of root without trailing slashes int rootLen; - for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen) ; + for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen); this.rootLen = rootLen; } diff --git a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java index e6f0910..ee3b0b3 100644 --- a/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java +++ b/src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java @@ -70,10 +70,15 @@ public enum StorageType { private int maxRedisWaitingHandlers = 2048; private int maxStorageExpandSubresources = 1000; - private String awsS3BucketName = null; + private String s3BucketName = null; private String awsS3Region = null; - private String awsS3AccessKeyId = null; - private String awsS3SecretAccessKey = null; + private String s3AccessKeyId = null; + private String s3SecretAccessKey = null; + private boolean s3UseTlsConnection = true; + private boolean createBucketIfNotPresentYet = false; + private boolean localS3 = false; + private String localS3Endpoint = null; + private int localS3Port = 0; public ModuleConfiguration root(String root) { this.root = root; @@ -301,18 +306,43 @@ public ModuleConfiguration awsS3Region(String awsS3Region) { return this; } - public ModuleConfiguration awsS3BucketName(String awsS3BucketName) { - this.awsS3BucketName = awsS3BucketName; + public ModuleConfiguration s3BucketName(String awsS3BucketName) { + this.s3BucketName = awsS3BucketName; return this; } - public ModuleConfiguration awsS3AccessKeyId(String awsS3AccessKeyId) { - this.awsS3AccessKeyId = awsS3AccessKeyId; + public ModuleConfiguration s3AccessKeyId(String awsS3AccessKeyId) { + this.s3AccessKeyId = awsS3AccessKeyId; return this; } - public ModuleConfiguration awsS3SecretAccessKey(String awsS3SecretAccessKey) { - this.awsS3SecretAccessKey = awsS3SecretAccessKey; + public ModuleConfiguration s3SecretAccessKey(String awsS3SecretAccessKey) { + this.s3SecretAccessKey = awsS3SecretAccessKey; + return this; + } + + public ModuleConfiguration s3UseTlsConnection(boolean s3UseTlsConnection) { + this.s3UseTlsConnection = s3UseTlsConnection; + return this; + } + + public ModuleConfiguration localS3Endpoint(String s3Endpoint) { + this.localS3Endpoint = s3Endpoint; + return this; + } + + public ModuleConfiguration localS3Port(int s3Port) { + this.localS3Port = s3Port; + return this; + } + + public ModuleConfiguration createBucketIfNotPresentYet(boolean createBucketIfNotExist) { + this.createBucketIfNotPresentYet = createBucketIfNotExist; + return this; + } + + public ModuleConfiguration localS3(boolean localS3) { + this.localS3 = localS3; return this; } @@ -493,16 +523,36 @@ public String getAwsS3Region() { return awsS3Region; } - public String getAwsS3BucketName() { - return awsS3BucketName; + public String getS3BucketName() { + return s3BucketName; + } + + public String getS3AccessKeyId() { + return s3AccessKeyId; + } + + public String getS3SecretAccessKey() { + return s3SecretAccessKey; + } + + public boolean getS3UseTlsConnection() { + return s3UseTlsConnection; + } + + public String getLocalS3Endpoint() { + return localS3Endpoint; + } + + public int getLocalS3Port() { + return localS3Port; } - public String getAwsS3AccessKeyId() { - return awsS3AccessKeyId; + public boolean getCreateBucketIfNotPresentYet() { + return createBucketIfNotPresentYet; } - public String getAwsS3SecretAccessKey() { - return awsS3SecretAccessKey; + public boolean isLocalS3() { + return localS3; } public JsonObject asJsonObject() {