Skip to content

Commit

Permalink
doc improved
Browse files Browse the repository at this point in the history
  • Loading branch information
@dominik-cnx
dominik-cnx committed Nov 12, 2024
1 parent 41b0bca commit c62b28c
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 52 deletions.
12 changes: 8 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,9 +227,9 @@ The following configuration values are available:
| s3BucketName | s3 | | The S3 bucket name |
| s3AccessKeyId | s3 | | The s3 access key Id |
| s3SecretAccessKey | s3 | | The s3 secret access key |
| localS3 | s3 | | is 3rd party S3-like services |
| s3Endpoint | s3 | | 3rd party S3-like services endpoint |
| s3Port | s3 | | 3rd party S3-like services port |
| localS3 | s3 | | Set to true in order to use a local S3 instance instead of AWS |
| localS3Endpoint | s3 | | The endpoint/host to use in case that localS3 is set to true, e.g. localhost |
| localS3Port | s3 | | The port to use in case that localS3 is set to true, e.g. 4566 |
| createBucketIfNotExist | s3 | | create bucket if bucket not exist, related permission required |


Expand Down Expand Up @@ -267,7 +267,11 @@ The data is stored hierarchically on the file system. This is the default storag
The data is stored in a S3 instance.

#### AWS S3
Before use, need sign up in AWS service at https://docs.aws.amazon.com/SetUp/latest/UserGuide/setup-AWSsignup.html
See https://aws.amazon.com/s3 it is also possible to use a local instance using https://docs.localstack.cloud/user-guide/aws/s3/


docker run --rm -p 4566:4566 -v ./s3:/var/lib/localstack localstack/localstack:s3-latest


### Redis Storage
The data is stored in a redis database.
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/swisspush/reststorage/RestStorageMod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ private Future<Storage> createStorage(ModuleConfiguration moduleConfiguration) {
moduleConfiguration.getAwsS3Region(), moduleConfiguration.getS3BucketName(),
moduleConfiguration.getS3AccessKeyId(), moduleConfiguration.getS3SecretAccessKey(),
moduleConfiguration.getS3UseTlsConnection(), moduleConfiguration.isLocalS3(),
moduleConfiguration.getS3Endpoint(), moduleConfiguration.getS3Port(),
moduleConfiguration.getCreateBucketIfNotExist()));
moduleConfiguration.getLocalS3Endpoint(), moduleConfiguration.getLocalS3Port(),
moduleConfiguration.getCreateBucketIfNotPresentYet()));
break;
case redis:
createRedisStorage(vertx, moduleConfiguration).onComplete(event -> {
Expand Down
60 changes: 29 additions & 31 deletions src/main/java/org/swisspush/reststorage/s3/S3FileSystemStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,52 +66,50 @@ private static S3FileSystem getFileSystem(URI uri) {

public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFactory, String rootPath,
String awsS3Region, String s3BucketName, String s3AccessKeyId, String s3SecretAccessKey,
boolean useTlsConnection, boolean isLocalS3, String s3Endpoint, int s3Port, boolean createBucketIfNotExist) {
boolean useTlsConnection, boolean isLocalS3, String localS3Endpoint, int localS3Port, boolean createBucketIfNotExist) {
this.vertx = vertx;
this.exceptionFactory = exceptionFactory;
Objects.requireNonNull(s3BucketName, "BucketName must not be null");
Objects.requireNonNull(awsS3Region, "Region must not be null");
System.setProperty("aws.region", awsS3Region);

if (!s3BucketName.startsWith("s3:") && !s3BucketName.startsWith("s3x:")) {
if (isLocalS3) {
// AWS SDK required those properties to be set
System.setProperty("aws.accessKeyId", "local");
System.setProperty("aws.secretAccessKey", "local");
// for nom-AWS
// s3x://[key:secret@]endpoint[:port]/bucket
String credentials = "";
if (StringUtils.isNotEmpty(s3AccessKeyId) && StringUtils.isNotEmpty(s3SecretAccessKey)) {
credentials = s3AccessKeyId + ":" + s3SecretAccessKey + "@";
}
String port = "";
if (s3Port > 0) {
port = ":" + s3Port;
}
s3BucketName = "s3x://" + credentials + s3Endpoint + port + "/" + s3BucketName;
if (!useTlsConnection) {
System.setProperty("s3.spi.endpoint-protocol", "http");
}
} else {
// for AWS S3
Objects.requireNonNull(s3AccessKeyId, "AccessKeyId must not be null");
Objects.requireNonNull(s3SecretAccessKey, "SecretAccessKey must not be null");
System.setProperty("aws.accessKeyId", s3AccessKeyId);
System.setProperty("aws.secretAccessKey", s3SecretAccessKey);
s3BucketName = "s3://" + s3BucketName;
if (isLocalS3) {
// local S3, AWS SDK requires these two properties to be set
System.setProperty("aws.accessKeyId", "local");
System.setProperty("aws.secretAccessKey", "local");

String credentials = "";
if (StringUtils.isNotEmpty(s3AccessKeyId) && StringUtils.isNotEmpty(s3SecretAccessKey)) {
credentials = s3AccessKeyId + ":" + s3SecretAccessKey + "@";
}
String port = "";
if (localS3Port > 0) {
port = ":" + localS3Port;
}
// s3x://[key:secret@]endpoint[:port]/bucket
s3BucketName = "s3x://" + credentials + localS3Endpoint + port + "/" + s3BucketName;
if (!useTlsConnection) {
System.setProperty("s3.spi.endpoint-protocol", "http");
}
} else {
// AWS S3
Objects.requireNonNull(s3AccessKeyId, "AccessKeyId must not be null");
Objects.requireNonNull(s3SecretAccessKey, "SecretAccessKey must not be null");
System.setProperty("aws.accessKeyId", s3AccessKeyId);
System.setProperty("aws.secretAccessKey", s3SecretAccessKey);
s3BucketName = "s3://" + s3BucketName;
}

var uri = URI.create(s3BucketName);

if (createBucketIfNotExist) {
try (var fs = FileSystems.newFileSystem(uri,
Map.of("locationConstraint", awsS3Region))) {
System.out.println(fs.toString());
log.info("Bucket created: " + fs.toString());
} catch (FileSystemAlreadyExistsException e) {
log.info("Bucket already exists: ", e);
log.info("Bucket " + s3BucketName + " already exists: ", e);
} catch (IOException e) {
log.error("Failed to create bucket", e);
log.error("Failed to create bucket " + s3BucketName, e);
}
}

Expand All @@ -131,7 +129,7 @@ public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFac

// Cache string length of root without trailing slashes
int rootLen;
for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen) ;
for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen);
this.rootLen = rootLen;
}

Expand Down
30 changes: 15 additions & 15 deletions src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ public enum StorageType {
private String s3AccessKeyId = null;
private String s3SecretAccessKey = null;
private boolean s3UseTlsConnection = true;
private boolean createBucketIfNotExist = false;
private boolean createBucketIfNotPresentYet = false;
private boolean localS3 = false;
private String s3Endpoint = null;
private int s3Port = 0;
private String localS3Endpoint = null;
private int localS3Port = 0;

public ModuleConfiguration root(String root) {
this.root = root;
Expand Down Expand Up @@ -326,18 +326,18 @@ public ModuleConfiguration s3UseTlsConnection(boolean s3UseTlsConnection) {
return this;
}

public ModuleConfiguration s3Endpoint(String s3Endpoint) {
this.s3Endpoint = s3Endpoint;
public ModuleConfiguration localS3Endpoint(String s3Endpoint) {
this.localS3Endpoint = s3Endpoint;
return this;
}

public ModuleConfiguration s3Port(int s3Port) {
this.s3Port = s3Port;
public ModuleConfiguration localS3Port(int s3Port) {
this.localS3Port = s3Port;
return this;
}

public ModuleConfiguration createBucketIfNotExist(boolean createBucketIfNotExist) {
this.createBucketIfNotExist = createBucketIfNotExist;
public ModuleConfiguration createBucketIfNotPresentYet(boolean createBucketIfNotExist) {
this.createBucketIfNotPresentYet = createBucketIfNotExist;
return this;
}

Expand Down Expand Up @@ -539,16 +539,16 @@ public boolean getS3UseTlsConnection() {
return s3UseTlsConnection;
}

public String getS3Endpoint() {
return s3Endpoint;
public String getLocalS3Endpoint() {
return localS3Endpoint;
}

public int getS3Port() {
return s3Port;
public int getLocalS3Port() {
return localS3Port;
}

public boolean getCreateBucketIfNotExist() {
return createBucketIfNotExist;
public boolean getCreateBucketIfNotPresentYet() {
return createBucketIfNotPresentYet;
}

public boolean isLocalS3() {
Expand Down

0 comments on commit c62b28c

Please sign in to comment.