From 4bcd368f4d6c9dbe74691719f3d9d453ca662403 Mon Sep 17 00:00:00 2001 From: Esko Suomi Date: Mon, 16 Dec 2024 11:04:03 +0200 Subject: [PATCH] add AWS S3 compatability through rutebanken-helpers storage implementation --- README.md | 61 ++++++++++++++++-- docker-compose.yml | 22 +++++++ pom.xml | 5 ++ scripts/init-localstack.sh | 4 ++ .../RutebankenBlobStoreConfiguration.java | 64 +++++++++++++++++++ .../resources/application-local.properties | 8 ++- 6 files changed, 156 insertions(+), 8 deletions(-) create mode 100755 scripts/init-localstack.sh diff --git a/README.md b/README.md index 8482b5b1d..f9c5d9a25 100644 --- a/README.md +++ b/README.md @@ -106,11 +106,31 @@ docker compose down docker compose --profile aws down ``` +#### Supported Docker Compose profiles + +Docker Compose has its own profiles which start up additional supporting services to e.g. make specific feature +development easier. You may include any number of additional profiles when working with Docker Compose by listing +them in the commands with the `--profile {profile name}` argument. Multiple profiles are activated by providing the +same attribute multiple times, for example starting Compose environment with profiles a and b would be +```shell +docker compose --profile a --profile b up +``` + +The provided profiles for Tiamat development are + + +| profile | description | +|:--------|---------------------------------------------------------------------------------------------------| +| `aws` | Starts up [LocalStack](https://www.localstack.cloud/) meant for developing AWS specific features. | + + See [Docker Compose reference](https://docs.docker.com/compose/reference/) for more details. +See [Supported Docker Compose Profiles](#supported-docker-compose-profiles) for more information on provided profiles. + ### 2. Run the Service -#### Available Profiles +#### Available Spring Boot Profiles > **Note!** You must choose at least one of the options from each category below! @@ -118,11 +138,13 @@ See [Docker Compose reference](https://docs.docker.com/compose/reference/) for m ##### Storage -| profile | description | -|:-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `gcs-blobstore` | GCP GCS implementation of tiamat's blob storage | -| `local-blobstore` | Use local directory as backing storage location. | -| `rutebanken-blobstore` | Use [`rutebanken-helpers/storage`](https://github.














com/entur/rutebanken-helpers/tree/master/storage) based implementation for storage. Must be combined with one of the supported extra profiles (see below). | +| profile | description | +|:-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `gcs-blobstore` | GCP GCS implementation of tiamat's blob storage | +| `local-blobstore` | Use local directory as backing storage location. | +| `rutebanken-blobstore` | Use [`rutebanken-helpers/storage`][rutebanken-storage] based implementation for storage. Must be combined with one of the supported extra profiles (see below). | + +[rutebanken-storage]: https://github.com/entur/rutebanken-helpers/tree/master/storage ###### Supported `rutebanken-blobstore` extra profiles @@ -133,12 +155,16 @@ Supported extra profiles are |:-----------------------|------------------------------------------| | `local-disk-blobstore` | Similar to `local-blobstore`. | | `in-memory-blobstore` | Entirely in-memory based implementation. | +| `s3-blobstore` | AWS S3 implementation. | **Example: Activating `in-memory-blobstore` for local development** ```properties spring.profiles.active=local,rutebanken-blobstore,in-memory-blobstore,local-changelog ``` +See the [`RutebankenBlobStoreServiceConfiguration`](./src/main/java/org/rutebanken/tiamat/config/RutebankenBlobStoreConfiguration.java) +class for configuration keys and additional information. + ##### Changelog | profile | description | @@ -147,6 +173,28 @@ spring.profiles.active=local,rutebanken-blobstore,in-memory-blobstore,local-chan | `activemq` | JMS based ActiveMQ implementation. | | `google-pubsub` | GCP PubSub implementation for publishing tiamat entity changes. | +#### Supported Docker Compose Profiles + +Tiamat's [`docker-compose.yml`](./docker-compose.yml) comes with built-in profiles for various use cases. The profiles +are mostly optional, default profile contains all mandatory configuration while the named profiles add features on +top of that. You can always activate zero or more profiles at the same time, e.g. + +```shell +docker compose --profile first --profile second up +# or +COMPOSE_PROFILES=first,second docker compose up +``` + +### Default profile (no activation key) + +Starts up PostGIS server with settings matching the ones in [`application-local.properties`](./src/main/resources/application-local.properties). + +### `aws` profile + +Starts up [LocalStack](https://www.localstack.cloud/) meant for developing AWS specific features. + +See also [Disable AWS S3 Autoconfiguration](#disable-aws-s3-autoconfiguration), [NeTEx Export](#netex-export). + #### Run It! **IntelliJ**: Right-click on `TiamatApplication.java` and choose Run (or Cmd+Shift+F10). Open Run -> Edit @@ -483,4 +531,3 @@ https://github.com/entur/tiamat-scripts ## CircleCI Tiamat is built using CircleCI. See the .circleci folder. - diff --git a/docker-compose.yml b/docker-compose.yml index f18686c53..d413a7cf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,28 @@ services: networks: - tiamat-net + localstack: + container_name: "${COMPOSE_PROJECT_NAME}_localstack" + profiles: ["aws"] + image: localstack/localstack:4.0 + ports: + - "37566:4566" # LocalStack Gateway + - "37510-37559:4510-4559" # external services port range + environment: + - DEBUG=${DEBUG-} + - DOCKER_HOST=unix:///var/run/docker.sock + - DISABLE_EVENTS=1 + - SERVICES=s3 + - AWS_ACCESS_KEY_ID=localstack + - AWS_SECRET_ACCESS_KEY=localstack + - AWS_DEFAULT_REGION=eu-north-1 + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "./scripts/init-localstack.sh:/etc/localstack/init/ready.d/init-localstack.sh" + networks: + - tiamat-net + volumes: postgres-data: diff --git a/pom.xml b/pom.xml index bcd88e154..bc887695c 100644 --- a/pom.xml +++ b/pom.xml @@ -135,6 +135,11 @@ storage ${rutebanken-storage.version} + + org.entur.ror.helpers + storage-aws-s3 + ${rutebanken-storage.version} + io.micrometer diff --git a/scripts/init-localstack.sh b/scripts/init-localstack.sh new file mode 100755 index 000000000..a65109e4f --- /dev/null +++ b/scripts/init-localstack.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# -- > Create S3 bucket for blob storage +awslocal s3api create-bucket --bucket 'tiamat-test' --create-bucket-configuration LocationConstraint=eu-north-1 diff --git a/src/main/java/org/rutebanken/tiamat/config/RutebankenBlobStoreConfiguration.java b/src/main/java/org/rutebanken/tiamat/config/RutebankenBlobStoreConfiguration.java index 65f77846c..dffec2c01 100644 --- a/src/main/java/org/rutebanken/tiamat/config/RutebankenBlobStoreConfiguration.java +++ b/src/main/java/org/rutebanken/tiamat/config/RutebankenBlobStoreConfiguration.java @@ -1,5 +1,6 @@ package org.rutebanken.tiamat.config; +import org.rutebanken.helper.aws.repository.S3BlobStoreRepository; import org.rutebanken.helper.storage.repository.BlobStoreRepository; import org.rutebanken.helper.storage.repository.InMemoryBlobStoreRepository; import org.rutebanken.helper.storage.repository.LocalDiskBlobStoreRepository; @@ -8,8 +9,19 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import java.net.URI; +import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; /** @@ -33,6 +45,7 @@ * {@link org.rutebanken.tiamat.service.GcsBlobStoreService} first needs to be adapted directly, and additional feature * gap fixing will probably need to be done in rutebanken-helpers as well. */ +@Lazy @Configuration @Profile("rutebanken-blobstore") public class RutebankenBlobStoreConfiguration { @@ -62,4 +75,55 @@ BlobStoreRepository inMemoryBlobStoreRepository( inMemoryBlobStoreRepository.setContainerName(containerName); return inMemoryBlobStoreRepository; } + + @Bean + BlobStoreRepository blobStoreRepository( + @Value("${blobstore.s3.bucket}") String containerName, + S3Client s3Client + ) { + S3BlobStoreRepository s3BlobStoreRepository = new S3BlobStoreRepository(s3Client); + s3BlobStoreRepository.setContainerName(containerName); + return s3BlobStoreRepository; + } + + @Profile("local | test") + @Bean + public AwsCredentialsProvider localCredentials( + @Value("blobstore.s3.access-key-id") String accessKeyId, + @Value("blobstore.s3.secret-key") String secretKey + ) { + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(accessKeyId, secretKey) + ); + } + + @Profile("!local & !test") + @Bean + public AwsCredentialsProvider cloudCredentials() { + return DefaultCredentialsProvider.create(); + } + + @Bean + public S3Client s3Client( + @Value("${blobstore.s3.region}") String region, + @Value("${blobstore.s3.endpoint-override:#{null}}") String endpointOverride, + AwsCredentialsProvider credentialsProvider + ) { + S3ClientBuilder builder = S3Client + .builder() + .region(Region.of(region)) + .credentialsProvider(credentialsProvider) + .overrideConfiguration( + ClientOverrideConfiguration + .builder() + .apiCallAttemptTimeout(Duration.ofSeconds(15)) + .apiCallTimeout(Duration.ofSeconds(15)) + .retryPolicy(retryPolicy -> retryPolicy.numRetries(5)) + .build() + ); + if (endpointOverride != null) { + builder = builder.endpointOverride(URI.create(endpointOverride)); + } + return builder.build(); + } } diff --git a/src/main/resources/application-local.properties b/src/main/resources/application-local.properties index 57707ff83..fb6f464d2 100644 --- a/src/main/resources/application-local.properties +++ b/src/main/resources/application-local.properties @@ -83,8 +83,14 @@ blobstore.gcs.bucket.name=tiamat-test blobstore.gcs.credential.path=gcloud-storage.json blobstore.gcs.project.id=carbon-1287 -# rutebanken-storage configurations +### rutebanken-storage configurations +# local-disk-blobstore blobstore.local.container.name=${blobstore.gcs.bucket.name} +# s3-blobstore, for local development with Localstack (see docker-compose.yml) +blobstore.s3.region=eu-north-1 +blobstore.s3.access-key-id=dev-access-key-id +blobstore.s3.secret-key=dev-secret-key +blobstore.s3.endpoint-override=http://localhost:37566 security.basic.enabled=false management.security.enabled=false