From 9d87a28c62242eb38a13d782e31c172e9baf40ea Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 29 May 2024 14:13:46 +0200 Subject: [PATCH 1/4] Endpoint nuevo storageasset --- extensions/store-asset-api/README.md | 19 +++ extensions/store-asset-api/build.gradle.kts | 33 +++++ .../StorageAssetApiExtension.java | 99 ++++++++++++++ .../storageasset/config/AppConfig.java | 18 +++ .../controller/StorageAssetApi.java | 50 +++++++ .../controller/StorageAssetApiController.java | 125 ++++++++++++++++++ .../storageasset/service/S3Service.java | 102 ++++++++++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + gradle/libs.versions.toml | 10 +- launchers/connector/build.gradle.kts | 3 + settings.gradle.kts | 1 + 11 files changed, 460 insertions(+), 1 deletion(-) create mode 100644 extensions/store-asset-api/README.md create mode 100644 extensions/store-asset-api/build.gradle.kts create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java create mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java create mode 100644 extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/store-asset-api/README.md b/extensions/store-asset-api/README.md new file mode 100644 index 0000000..6fbc14d --- /dev/null +++ b/extensions/store-asset-api/README.md @@ -0,0 +1,19 @@ +# Vocabulary API + +Provides a management API for work with vocabularies. This API expands the functionality of the control-plane management API to be able to handle Vocabulary entities. + + +## Vocabulary entity + +An example of a Vocabulary entity is shown below. + +``` +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@id": "vocabularyId", + "name": "Vocabulary name", + "jsonSchema": "{ \"title\": \"vocabulary\", \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\", \"title\": \"Name\" }, \"keyword\": { \"type\": \"array\", \"title\": \"Keywords\", \"items\": { \"type\": \"string\" } } }, \"required\": [ \"name\" ] }" +} +``` diff --git a/extensions/store-asset-api/build.gradle.kts b/extensions/store-asset-api/build.gradle.kts new file mode 100644 index 0000000..0d222d9 --- /dev/null +++ b/extensions/store-asset-api/build.gradle.kts @@ -0,0 +1,33 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + api(libs.edc.spi.asset) + api(libs.edc.spi.core) + implementation(libs.edc.spi.transform) + implementation(libs.edc.web.spi) + + implementation(libs.edc.connector.core) + implementation(libs.edc.api.core) + implementation(libs.edc.lib.util) + implementation(libs.edc.lib.transform) + implementation(libs.edc.dsp.api.configuration) + implementation(libs.edc.api.management.config) + implementation(libs.edc.transaction.spi) + implementation(libs.edc.lib.validator) + implementation(libs.edc.validator.spi) + implementation(libs.swagger.annotations.jakarta) + implementation(libs.jakarta.rsApi) + implementation(libs.jakarta.eeApi) + implementation(libs.parsson) + implementation(libs.jersey) + implementation(libs.edc.json.ld.lib) + implementation(libs.aws.s3) + implementation(libs.aws.s3.transfer) + implementation(libs.edc.api.asset) + implementation(libs.edc.control.plane.transform) + runtimeOnly(libs.edc.spi.jsonld) + runtimeOnly(libs.edc.json.ld.lib) +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java new file mode 100644 index 0000000..c604b2f --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -0,0 +1,99 @@ +package org.upm.inesdata.storageasset; + +import jakarta.json.Json; +import org.eclipse.edc.api.validation.DataAddressValidator; +import org.eclipse.edc.connector.api.management.configuration.ManagementApiConfiguration; +import org.eclipse.edc.connector.controlplane.api.management.asset.validation.AssetValidator; +import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; +import org.eclipse.edc.connector.controlplane.transform.edc.from.JsonObjectFromAssetTransformer; +import org.eclipse.edc.connector.controlplane.transform.edc.to.JsonObjectToAssetTransformer; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; +import org.eclipse.edc.transaction.spi.TransactionContext; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.WebService; +import org.upm.inesdata.storageasset.controller.StorageAssetApiController; +import org.upm.inesdata.storageasset.service.S3Service; +import software.amazon.awssdk.regions.Region; + +import java.util.Map; + +import static org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset.EDC_ASSET_TYPE; +import static org.eclipse.edc.spi.constants.CoreConstants.JSON_LD; +import static org.eclipse.edc.spi.types.domain.DataAddress.EDC_DATA_ADDRESS_TYPE; +/** + * Extension that provides an API for managing vocabularies + */ +@Extension(value = StorageAssetApiExtension.NAME) +public class StorageAssetApiExtension implements ServiceExtension { + + public static final String NAME = "StorageAsset API Extension"; + + @Inject + private AssetService assetService; + + @Inject + private WebService webService; + + @Inject + private ManagementApiConfiguration config; + + @Inject + private TypeManager typeManager; + + @Inject + private TransactionContext transactionContext; + + @Inject + private TypeTransformerRegistry transformerRegistry; + + @Inject + private JsonObjectValidatorRegistry validator; + @Inject + private JsonLd jsonLd; + + + @Override + public String name() { + return NAME; + } + + /** + * Initializes the service + */ + @Override + public void initialize(ServiceExtensionContext context) { + var monitor = context.getMonitor(); + + var managementApiTransformerRegistry = transformerRegistry.forContext("management-api"); + + var factory = Json.createBuilderFactory(Map.of()); + var jsonLdMapper = typeManager.getMapper(JSON_LD); + managementApiTransformerRegistry.register(new JsonObjectFromAssetTransformer(factory, jsonLdMapper)); + managementApiTransformerRegistry.register(new JsonObjectToAssetTransformer()); + + validator.register(EDC_ASSET_TYPE, AssetValidator.instance()); + validator.register(EDC_DATA_ADDRESS_TYPE, DataAddressValidator.instance()); + + // Leer las variables de entorno + String accessKey = context.getSetting("edc.aws.access.key",""); + String secretKey = context.getSetting("edc.aws.secret.access.key",""); + String endpointOverride = context.getSetting("edc.aws.endpoint.override",""); + String regionName = context.getSetting("edc.aws.region",""); + String bucketName = context.getSetting("edc.aws.bucket.name",""); + + Region region = Region.of(regionName); + + // Crear una instancia de S3Service + S3Service s3Service = new S3Service(accessKey, secretKey, endpointOverride, region, bucketName); + + var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, monitor, validator,s3Service, + jsonLd, context); + webService.registerResource(config.getContextAlias(), storageAssetApiController); + } +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java new file mode 100644 index 0000000..d73e79e --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java @@ -0,0 +1,18 @@ +package org.upm.inesdata.storageasset.config; + +import jakarta.ws.rs.ApplicationPath; +import jakarta.ws.rs.core.Application; +import org.upm.inesdata.storageasset.controller.StorageAssetApiController; + +import java.util.HashSet; +import java.util.Set; + +@ApplicationPath("/api") +public class AppConfig extends Application { + @Override + public Set> getClasses() { + Set> classes = new HashSet<>(); + classes.add(StorageAssetApiController.class); + return classes; + } +} \ No newline at end of file diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java new file mode 100644 index 0000000..9392365 --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApi.java @@ -0,0 +1,50 @@ +package org.upm.inesdata.storageasset.controller; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.info.Info; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonObject; +import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.api.model.ApiCoreSchema; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + +import java.io.InputStream; + +@OpenAPIDefinition( + info = @Info(description = "Manages the connector s3 assets.", + title = "S3 Asset API", version = "1")) +@Tag(name = "S3Asset") +public interface StorageAssetApi { + + /** + * Creates a new storage asset + * + * @param fileInputStream the input stream of the file to be uploaded + * @param fileDetail the details of the file to be uploaded + * @param assetJson the input stream of the asset metadata in JSON format + * @return JsonObject with the created asset + */ + @Operation(description = "Creates a new S3 asset", + requestBody = @RequestBody(content = @Content(mediaType = MediaType.MULTIPART_FORM_DATA, schema = @Schema( + type = "object", requiredProperties = {"file", "json"} + ))), + responses = { + @ApiResponse(responseCode = "200", description = "S3 asset was created successfully", + content = @Content(schema = @Schema(implementation = ApiCoreSchema.IdResponseSchema.class))), + @ApiResponse(responseCode = "400", description = "Request body was malformed", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))), + @ApiResponse(responseCode = "409", description = "Could not create asset, because an asset with that ID already exists", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) + } + ) + JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, + @FormDataParam("json") JsonObject assetJson); +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java new file mode 100644 index 0000000..4cc64a8 --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java @@ -0,0 +1,125 @@ +package org.upm.inesdata.storageasset.controller; + +import jakarta.json.JsonObject; +import jakarta.servlet.annotation.MultipartConfig; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.api.model.IdResponse; +import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; +import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; +import org.eclipse.edc.web.spi.exception.ValidationFailureException; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; +import org.upm.inesdata.storageasset.service.S3Service; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; + +import static org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset.EDC_ASSET_TYPE; +import static org.eclipse.edc.web.spi.exception.ServiceResultHandler.exceptionMapper; + +@MultipartConfig +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Produces(MediaType.APPLICATION_JSON) +@Path("/s3assets") +public class StorageAssetApiController implements StorageAssetApi { + private final TypeTransformerRegistry transformerRegistry; + private final AssetService service; + private final Monitor monitor; + private final JsonObjectValidatorRegistry validator; + private final S3Service s3Service; + + private final JsonLd jsonLd; + + private final ServiceExtensionContext context; + + public StorageAssetApiController(AssetService service, TypeTransformerRegistry transformerRegistry, Monitor monitor, + JsonObjectValidatorRegistry validator, S3Service s3Service, JsonLd jsonLd, ServiceExtensionContext context) { + this.transformerRegistry = transformerRegistry; + this.service = service; + this.monitor = monitor; + this.validator = validator; + this.s3Service = s3Service; + this.jsonLd = jsonLd; + this.context = context; + } + + @POST + @Override + public JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, @FormDataParam("json") JsonObject assetJson) { + + String fileName = fileDetail.getFileName(); + + InputStream bufferedInputStream = new BufferedInputStream(fileInputStream); + + JsonObject expand = jsonLd.expand(assetJson).orElseThrow((f) -> new EdcException("Failed to expand request")); + // Validación + validator.validate(EDC_ASSET_TYPE, expand).orElseThrow(ValidationFailureException::new); + + // Transformación + var asset = transformerRegistry.transform(expand, Asset.class).orElseThrow(InvalidRequestException::new); + + // Guardar fichero en MinIO + // Calcular el tamaño del fichero manualmente + long contentLength = 0; + try { + contentLength = getFileSize(bufferedInputStream); + } catch (IOException e) { + throw new EdcException("Failed to process file size", e); + } + s3Service.uploadFile(fileName, bufferedInputStream, contentLength); + try { + setStorageProperties(asset, fileName); + + // Creación de asset + var idResponse = service.create(asset) + .map(a -> IdResponse.Builder.newInstance().id(a.getId()).createdAt(a.getCreatedAt()).build()) + .orElseThrow(exceptionMapper(Asset.class, asset.getId())); + + return transformerRegistry.transform(idResponse, JsonObject.class) + .orElseThrow(f -> new EdcException(f.getFailureDetail())); + } catch (Exception e) { + // Eliminar el archivo en caso de fallo + s3Service.deleteFile(fileName); + throw new EdcException("Failed to process multipart data", e); + } + } + + private long getFileSize(InputStream inputStream) throws IOException { + byte[] buffer = new byte[8192]; + int bytesRead; + long size = 0; + + inputStream.mark(Integer.MAX_VALUE); + + while ((bytesRead = inputStream.read(buffer)) != -1) { + size += bytesRead; + } + + inputStream.reset(); + + return size; + } + + private void setStorageProperties(Asset asset, String fileName) { + String regionName = context.getSetting("edc.aws.region", ""); + String bucketName = context.getSetting("edc.aws.bucket.name", ""); + asset.getPrivateProperties().put("storageAssetFile", fileName); + asset.getDataAddress().setKeyName(fileName); + asset.getDataAddress().setType("InesDataStore"); + asset.getDataAddress().getProperties().put("bucketName", bucketName); + asset.getDataAddress().getProperties().put("region", regionName); + } +} diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java new file mode 100644 index 0000000..c6036cf --- /dev/null +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java @@ -0,0 +1,102 @@ +package org.upm.inesdata.storageasset.service; + +import org.eclipse.edc.spi.EdcException; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedUpload; +import software.amazon.awssdk.transfer.s3.model.UploadRequest; +import software.amazon.awssdk.core.async.AsyncRequestBody; + +import java.io.InputStream; +import java.net.URI; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Servicio para manejar operaciones de almacenamiento en S3. + */ +public class S3Service { + private final S3AsyncClient s3AsyncClient; + private final S3TransferManager transferManager; + private final String bucketName; + private final ExecutorService executorService; + + public S3Service(String accessKey, String secretKey, String endpointOverride, Region region, String bucketName) { + this.s3AsyncClient = S3AsyncClient.builder() + .region(region) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey))) + .endpointOverride(URI.create(endpointOverride)) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + this.transferManager = S3TransferManager.builder().s3Client(s3AsyncClient).build(); + this.bucketName = bucketName; + this.executorService = Executors.newFixedThreadPool(10); // Crear un pool de hilos fijo + } + + public String uploadFile(String key, InputStream inputStream, long contentLength) { + // Verificar si el archivo ya existe + boolean exists = doesObjectExist(bucketName, key).join(); + if (exists) { + throw new EdcException("File with key " + key + " already exists."); + } + + PutObjectRequest objectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + AsyncRequestBody requestBody = AsyncRequestBody.fromInputStream(inputStream, contentLength, executorService); + + UploadRequest uploadRequest = UploadRequest.builder() + .putObjectRequest(objectRequest) + .requestBody(requestBody) + .build(); + + CompletableFuture upload = transferManager.upload(uploadRequest).completionFuture(); + upload.join(); // Esperar a que la carga se complete + + return key; + } + + public void deleteFile(String key) { + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + s3AsyncClient.deleteObject(deleteObjectRequest).join(); // Esperar a que se complete la eliminación + } + + public void close() { + transferManager.close(); + s3AsyncClient.close(); + executorService.shutdown(); + } + + public CompletableFuture doesObjectExist(String bucketName, String key) { + HeadObjectRequest headObjectRequest = HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + + return s3AsyncClient.headObject(headObjectRequest) + .thenApply(response -> true) + .exceptionally(ex -> { + if (ex.getCause() instanceof NoSuchKeyException || (ex.getCause() instanceof S3Exception && ((S3Exception) ex.getCause()).statusCode() == 404)) { + return false; + } else { + throw new RuntimeException("Error checking if object exists", ex); + } + }); + } + +} diff --git a/extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..6bdc8db --- /dev/null +++ b/extensions/store-asset-api/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.storageasset.StorageAssetApiExtension \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7c9ef3f..c9eb809 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,9 @@ jupiter = "5.10.2" mockito = "5.2.0" postgres = "42.7.3" rsApi = "4.0.0" +rseeApi = "10.0.0" +parsson = "1.1.4" +jersey = "3.0.0" swagger-annotations-jakarta = "2.2.21" [libraries] @@ -24,6 +27,7 @@ edc-control-plane-api = { module = "org.eclipse.edc:control-plane-api", version. edc-control-plane-api-client = { module = "org.eclipse.edc:control-plane-api-client", version.ref = "edc" } edc-control-plane-core = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } edc-control-plane-spi = { module = "org.eclipse.edc:control-plane-spi", version.ref = "edc" } +edc-control-plane-transform = { module = "org.eclipse.edc:control-plane-transform", version.ref = "edc" } edc-data-plane-api = { module = "org.eclipse.edc:data-plane-api", version.ref = "edc" } edc-data-plane-control-api = { module = "org.eclipse.edc:data-plane-control-api", version.ref = "edc" } edc-data-plane-core = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" } @@ -43,6 +47,8 @@ edc-policy-engine-spi = { module = "org.eclipse.edc:policy-engine-spi", version. edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } edc-spi-transform = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } +edc-spi-asset = { module = "org.eclipse.edc:asset-spi", version.ref = "edc" } +edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-transaction-datasource-spi = { module = "org.eclipse.edc:transaction-datasource-spi", version.ref = "edc" } edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" } @@ -52,7 +58,9 @@ edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.re edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } - +jakarta-eeApi = { module = "jakarta.platform:jakarta.jakartaee-api", version.ref = "rseeApi" } +parsson = { module = "org.eclipse.parsson:jakarta.json", version.ref = "parsson" } +jersey = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } edc-lib-validator = { module = "org.eclipse.edc:validator-lib", version.ref = "edc" } diff --git a/launchers/connector/build.gradle.kts b/launchers/connector/build.gradle.kts index 9c2ab18..5a131a6 100644 --- a/launchers/connector/build.gradle.kts +++ b/launchers/connector/build.gradle.kts @@ -74,6 +74,9 @@ dependencies { implementation(libs.edc.federated.catalog.spi) implementation(libs.edc.federated.catalog.core) implementation(libs.edc.federated.catalog.api) + + // Storage assets + implementation(project(":extensions:store-asset-api")) runtimeOnly(libs.edc.transaction.local) runtimeOnly(libs.postgres) diff --git a/settings.gradle.kts b/settings.gradle.kts index e5c3707..d127ea0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,6 +15,7 @@ include(":extensions:policy-always-true") include(":extensions:policy-time-interval") include(":extensions:vocabulary-api") include(":extensions:vocabulary-index-sql") +include(":extensions:store-asset-api") // Connector include(":launchers:connector") From 06dd1579b4f2f745f1e504744fe525c2b463fa19 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 29 May 2024 16:49:49 +0200 Subject: [PATCH 2/4] Correcciones --- extensions/store-asset-api/build.gradle.kts | 1 + .../StorageAssetApiExtension.java | 5 ++-- .../controller/StorageAssetApiController.java | 30 +++++++++---------- .../storageasset/service/S3Service.java | 8 +++-- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/extensions/store-asset-api/build.gradle.kts b/extensions/store-asset-api/build.gradle.kts index 0d222d9..300da81 100644 --- a/extensions/store-asset-api/build.gradle.kts +++ b/extensions/store-asset-api/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation(libs.aws.s3.transfer) implementation(libs.edc.api.asset) implementation(libs.edc.control.plane.transform) + implementation(libs.edc.aws.s3.core) runtimeOnly(libs.edc.spi.jsonld) runtimeOnly(libs.edc.json.ld.lib) } diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java index c604b2f..a6f5ef8 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/StorageAssetApiExtension.java @@ -92,8 +92,9 @@ public void initialize(ServiceExtensionContext context) { // Crear una instancia de S3Service S3Service s3Service = new S3Service(accessKey, secretKey, endpointOverride, region, bucketName); - var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, monitor, validator,s3Service, - jsonLd, context); + var storageAssetApiController = new StorageAssetApiController(assetService, managementApiTransformerRegistry, + validator,s3Service, + jsonLd, bucketName, regionName); webService.registerResource(config.getContextAlias(), storageAssetApiController); } } diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java index 4cc64a8..8e208fa 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/controller/StorageAssetApiController.java @@ -12,9 +12,9 @@ import org.eclipse.edc.connector.controlplane.services.spi.asset.AssetService; import org.eclipse.edc.jsonld.spi.JsonLd; import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.constants.CoreConstants; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.util.string.StringUtils; import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.exception.InvalidRequestException; import org.eclipse.edc.web.spi.exception.ValidationFailureException; @@ -36,23 +36,23 @@ public class StorageAssetApiController implements StorageAssetApi { private final TypeTransformerRegistry transformerRegistry; private final AssetService service; - private final Monitor monitor; private final JsonObjectValidatorRegistry validator; private final S3Service s3Service; private final JsonLd jsonLd; - private final ServiceExtensionContext context; + private final String bucketName; + private final String region; - public StorageAssetApiController(AssetService service, TypeTransformerRegistry transformerRegistry, Monitor monitor, - JsonObjectValidatorRegistry validator, S3Service s3Service, JsonLd jsonLd, ServiceExtensionContext context) { + public StorageAssetApiController(AssetService service, TypeTransformerRegistry transformerRegistry, + JsonObjectValidatorRegistry validator, S3Service s3Service, JsonLd jsonLd, String bucketName, String region) { this.transformerRegistry = transformerRegistry; this.service = service; - this.monitor = monitor; this.validator = validator; this.s3Service = s3Service; this.jsonLd = jsonLd; - this.context = context; + this.bucketName = bucketName; + this.region = region; } @POST @@ -79,9 +79,11 @@ public JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInpu } catch (IOException e) { throw new EdcException("Failed to process file size", e); } - s3Service.uploadFile(fileName, bufferedInputStream, contentLength); + String folder = String.valueOf(asset.getDataAddress().getProperties().get(CoreConstants.EDC_NAMESPACE+"folder")); + String fullKey = StringUtils.isNullOrBlank(folder) || "null".equals(folder)?fileName:(folder.endsWith("/") ? folder + fileName : folder + "/" + fileName); + s3Service.uploadFile(fullKey,bufferedInputStream, contentLength); try { - setStorageProperties(asset, fileName); + setStorageProperties(asset, fullKey); // Creación de asset var idResponse = service.create(asset) @@ -92,7 +94,7 @@ public JsonObject createStorageAsset(@FormDataParam("file") InputStream fileInpu .orElseThrow(f -> new EdcException(f.getFailureDetail())); } catch (Exception e) { // Eliminar el archivo en caso de fallo - s3Service.deleteFile(fileName); + s3Service.deleteFile(fullKey); throw new EdcException("Failed to process multipart data", e); } } @@ -114,12 +116,10 @@ private long getFileSize(InputStream inputStream) throws IOException { } private void setStorageProperties(Asset asset, String fileName) { - String regionName = context.getSetting("edc.aws.region", ""); - String bucketName = context.getSetting("edc.aws.bucket.name", ""); asset.getPrivateProperties().put("storageAssetFile", fileName); asset.getDataAddress().setKeyName(fileName); asset.getDataAddress().setType("InesDataStore"); - asset.getDataAddress().getProperties().put("bucketName", bucketName); - asset.getDataAddress().getProperties().put("region", regionName); + asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+ "bucketName", bucketName); + asset.getDataAddress().getProperties().put(CoreConstants.EDC_NAMESPACE+"region", region); } } diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java index c6036cf..20a1527 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java @@ -1,6 +1,7 @@ package org.upm.inesdata.storageasset.service; import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.web.spi.exception.ObjectConflictException; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -44,10 +45,11 @@ public S3Service(String accessKey, String secretKey, String endpointOverride, Re } public String uploadFile(String key, InputStream inputStream, long contentLength) { + // Verificar si el archivo ya existe boolean exists = doesObjectExist(bucketName, key).join(); if (exists) { - throw new EdcException("File with key " + key + " already exists."); + throw new ObjectConflictException("File with key " + key + " already exists."); } PutObjectRequest objectRequest = PutObjectRequest.builder() @@ -69,9 +71,10 @@ public String uploadFile(String key, InputStream inputStream, long contentLength } public void deleteFile(String key) { + // Ajustar la clave para incluir la carpeta DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() .bucket(bucketName) - .key(key) + .key(fullKey) .build(); s3AsyncClient.deleteObject(deleteObjectRequest).join(); // Esperar a que se complete la eliminación } @@ -98,5 +101,4 @@ public CompletableFuture doesObjectExist(String bucketName, String key) } }); } - } From f579747807fda7b3efd3dc3ea9eceacfbc72428a Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Wed, 29 May 2024 16:53:43 +0200 Subject: [PATCH 3/4] =?UTF-8?q?Error=20de=20compilaci=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/upm/inesdata/storageasset/service/S3Service.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java index 20a1527..204e79f 100644 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java +++ b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/service/S3Service.java @@ -74,7 +74,7 @@ public void deleteFile(String key) { // Ajustar la clave para incluir la carpeta DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() .bucket(bucketName) - .key(fullKey) + .key(key) .build(); s3AsyncClient.deleteObject(deleteObjectRequest).join(); // Esperar a que se complete la eliminación } From ffc168194407de29fe26dfa8695d6898ffe970d5 Mon Sep 17 00:00:00 2001 From: "GRUPOGMV\\ssis" Date: Mon, 3 Jun 2024 17:23:36 +0200 Subject: [PATCH 4/4] Resueltos comentarios MR. --- extensions/store-asset-api/README.md | 20 ++----------------- .../storageasset/config/AppConfig.java | 18 ----------------- gradle/libs.versions.toml | 10 +++++----- 3 files changed, 7 insertions(+), 41 deletions(-) delete mode 100644 extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java diff --git a/extensions/store-asset-api/README.md b/extensions/store-asset-api/README.md index 6fbc14d..022c913 100644 --- a/extensions/store-asset-api/README.md +++ b/extensions/store-asset-api/README.md @@ -1,19 +1,3 @@ -# Vocabulary API +# S3 Asset API -Provides a management API for work with vocabularies. This API expands the functionality of the control-plane management API to be able to handle Vocabulary entities. - - -## Vocabulary entity - -An example of a Vocabulary entity is shown below. - -``` -{ - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@id": "vocabularyId", - "name": "Vocabulary name", - "jsonSchema": "{ \"title\": \"vocabulary\", \"type\": \"object\", \"properties\": { \"name\": { \"type\": \"string\", \"title\": \"Name\" }, \"keyword\": { \"type\": \"array\", \"title\": \"Keywords\", \"items\": { \"type\": \"string\" } } }, \"required\": [ \"name\" ] }" -} -``` +Provides a management API for uploading files to S3 and creating assets. diff --git a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java b/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java deleted file mode 100644 index d73e79e..0000000 --- a/extensions/store-asset-api/src/main/java/org/upm/inesdata/storageasset/config/AppConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.upm.inesdata.storageasset.config; - -import jakarta.ws.rs.ApplicationPath; -import jakarta.ws.rs.core.Application; -import org.upm.inesdata.storageasset.controller.StorageAssetApiController; - -import java.util.HashSet; -import java.util.Set; - -@ApplicationPath("/api") -public class AppConfig extends Application { - @Override - public Set> getClasses() { - Set> classes = new HashSet<>(); - classes.add(StorageAssetApiController.class); - return classes; - } -} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c9eb809..f13176c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,6 +16,7 @@ jersey = "3.0.0" swagger-annotations-jakarta = "2.2.21" [libraries] +edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-api-core = { module = "org.eclipse.edc:api-core", version.ref = "edc" } edc-api-management-config = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" } edc-auth-spi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" } @@ -48,7 +49,6 @@ edc-spi-core = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } edc-spi-jsonld = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } edc-spi-transform = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } edc-spi-asset = { module = "org.eclipse.edc:asset-spi", version.ref = "edc" } -edc-api-asset = { module = "org.eclipse.edc:asset-api", version.ref = "edc" } edc-transaction-local = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } edc-transaction-datasource-spi = { module = "org.eclipse.edc:transaction-datasource-spi", version.ref = "edc" } edc-transaction-spi = { module = "org.eclipse.edc:transaction-spi", version.ref = "edc" } @@ -57,10 +57,6 @@ edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http edc-vault-filesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } edc-validator-spi = { module = "org.eclipse.edc:validator-spi", version.ref = "edc" } edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } -jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } -jakarta-eeApi = { module = "jakarta.platform:jakarta.jakartaee-api", version.ref = "rseeApi" } -parsson = { module = "org.eclipse.parsson:jakarta.json", version.ref = "parsson" } -jersey = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } edc-lib-transform = { module = "org.eclipse.edc:transform-lib", version.ref = "edc" } edc-lib-util = { module = "org.eclipse.edc:util-lib", version.ref = "edc" } edc-lib-validator = { module = "org.eclipse.edc:validator-lib", version.ref = "edc" } @@ -95,6 +91,10 @@ edc-core-junit = { module = "org.eclipse.edc:junit", version.ref = "edc" } # No EDC references aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } aws-s3-transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } +jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } +jakarta-eeApi = { module = "jakarta.platform:jakarta.jakartaee-api", version.ref = "rseeApi" } +jersey = { module = "org.glassfish.jersey.media:jersey-media-multipart", version.ref = "jersey" } +parsson = { module = "org.eclipse.parsson:jakarta.json", version.ref = "parsson" } postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } swagger-annotations-jakarta = { module = "io.swagger.core.v3:swagger-annotations-jakarta", version.ref = "swagger-annotations-jakarta" }