diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/dataimport/DataImportController.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/dataimport/DataImportController.java index dfb00e6adf..3c5b4d0f4e 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/dataimport/DataImportController.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/dataimport/DataImportController.java @@ -6,7 +6,7 @@ import bio.terra.pearl.api.admin.service.enrollee.EnrolleeImportExtService; import bio.terra.pearl.core.model.EnvironmentName; import bio.terra.pearl.core.model.admin.AdminUser; -import bio.terra.pearl.core.service.dataimport.ImportFileFormat; +import bio.terra.pearl.core.service.export.dataimport.ImportFileFormat; import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.UUID; diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/datarepo/DataRepoExportController.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/datarepo/DataRepoExportController.java index 8fea39cb35..be6041abc7 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/datarepo/DataRepoExportController.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/datarepo/DataRepoExportController.java @@ -2,13 +2,13 @@ import bio.terra.pearl.api.admin.api.DatarepoApi; import bio.terra.pearl.api.admin.model.CreateDataset; -import bio.terra.pearl.api.admin.service.DataRepoExportExtService; import bio.terra.pearl.api.admin.service.auth.AuthUtilService; import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext; +import bio.terra.pearl.api.admin.service.export.DataRepoExportExtService; import bio.terra.pearl.core.model.EnvironmentName; import bio.terra.pearl.core.model.admin.AdminUser; -import bio.terra.pearl.core.model.datarepo.DataRepoJob; -import bio.terra.pearl.core.model.datarepo.Dataset; +import bio.terra.pearl.core.model.export.datarepo.DataRepoJob; +import bio.terra.pearl.core.model.export.datarepo.Dataset; import jakarta.servlet.http.HttpServletRequest; import java.util.List; import org.springframework.http.ResponseEntity; diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportController.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportController.java index 270b0543fc..59048b67c4 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportController.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportController.java @@ -1,13 +1,14 @@ package bio.terra.pearl.api.admin.controller.export; import bio.terra.pearl.api.admin.api.ExportApi; -import bio.terra.pearl.api.admin.service.EnrolleeExportExtService; import bio.terra.pearl.api.admin.service.auth.AuthUtilService; import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext; +import bio.terra.pearl.api.admin.service.export.EnrolleeExportExtService; import bio.terra.pearl.core.model.EnvironmentName; import bio.terra.pearl.core.model.admin.AdminUser; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.ExportFileFormat; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; import bio.terra.pearl.core.service.search.EnrolleeSearchExpression; import bio.terra.pearl.core.service.search.EnrolleeSearchExpressionParser; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,7 +16,7 @@ import jakarta.servlet.http.HttpServletResponse; import java.io.ByteArrayOutputStream; import java.util.List; -import java.util.Objects; +import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; @@ -58,15 +59,15 @@ public ResponseEntity exportData( List excludeModules, String searchExpression, String fileFormat, - Integer limit) { + Integer rowLimit) { EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); AdminUser user = authUtilService.requireAdminUser(request); - ExportOptions exportOptions = + ExportOptionsWithExpression exportOptions = optionsFromParams( searchExpression, fileFormat, - limit, + rowLimit, splitOptionsIntoColumns, stableIdsForOptions, includeOnlyMostRecent, @@ -114,8 +115,8 @@ public ResponseEntity exportDictionary( return ResponseEntity.ok().body(new ByteArrayResource(baos.toByteArray())); } - private ExportOptions optionsFromParams( - String searchExpression, + private ExportOptionsWithExpression optionsFromParams( + String filter, String fileFormat, Integer limit, Boolean splitOptionsIntoColumns, @@ -123,21 +124,20 @@ private ExportOptions optionsFromParams( Boolean includeOnlyMostRecent, Boolean includeSubHeaders, List excludeModules) { - EnrolleeSearchExpression filter = - Objects.nonNull(searchExpression) && !searchExpression.isEmpty() - ? enrolleeSearchExpressionParser.parseRule(searchExpression) - : null; + EnrolleeSearchExpression searchExp = + !StringUtils.isBlank(filter) ? enrolleeSearchExpressionParser.parseRule(filter) : null; - ExportOptions exportOptions = - ExportOptions.builder() + ExportOptionsWithExpression exportOptions = + ExportOptionsWithExpression.builder() .splitOptionsIntoColumns( splitOptionsIntoColumns != null ? splitOptionsIntoColumns : false) .stableIdsForOptions(stableIdsForOptions != null ? stableIdsForOptions : false) .onlyIncludeMostRecent(includeOnlyMostRecent != null ? includeOnlyMostRecent : false) - .filter(filter) + .filterString(filter) + .filterExpression(searchExp) .fileFormat( fileFormat != null ? ExportFileFormat.valueOf(fileFormat) : ExportFileFormat.TSV) - .limit(limit) + .rowLimit(limit) .includeSubHeaders(includeSubHeaders) .excludeModules(excludeModules != null ? excludeModules : List.of()) .build(); diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportIntegrationController.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportIntegrationController.java new file mode 100644 index 0000000000..5fb12d5d7b --- /dev/null +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/export/ExportIntegrationController.java @@ -0,0 +1,99 @@ +package bio.terra.pearl.api.admin.controller.export; + +import bio.terra.pearl.api.admin.api.ExportIntegrationApi; +import bio.terra.pearl.api.admin.service.auth.AuthUtilService; +import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext; +import bio.terra.pearl.api.admin.service.export.ExportIntegrationExtService; +import bio.terra.pearl.core.model.EnvironmentName; +import bio.terra.pearl.core.model.admin.AdminUser; +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.http.HttpServletRequest; +import java.util.List; +import java.util.UUID; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; + +@Controller +public class ExportIntegrationController implements ExportIntegrationApi { + private final AuthUtilService authUtilService; + private final HttpServletRequest request; + private final ExportIntegrationExtService exportIntegrationExtService; + private final ObjectMapper objectMapper; + + public ExportIntegrationController( + AuthUtilService authUtilService, + HttpServletRequest request, + ExportIntegrationExtService exportIntegrationExtService, + ObjectMapper objectMapper) { + this.authUtilService = authUtilService; + this.request = request; + this.exportIntegrationExtService = exportIntegrationExtService; + this.objectMapper = objectMapper; + } + + @Override + public ResponseEntity findByStudy( + String portalShortcode, String studyShortcode, String envName) { + AdminUser operator = authUtilService.requireAdminUser(request); + EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); + List integrations = + exportIntegrationExtService.list( + PortalStudyEnvAuthContext.of( + operator, portalShortcode, studyShortcode, environmentName)); + return ResponseEntity.ok(integrations); + } + + @Override + public ResponseEntity get( + String portalShortcode, String studyShortcode, String envName, UUID id) { + AdminUser operator = authUtilService.requireAdminUser(request); + EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); + ExportIntegration integration = + exportIntegrationExtService.find( + PortalStudyEnvAuthContext.of( + operator, portalShortcode, studyShortcode, environmentName), + id); + return ResponseEntity.ok(integration); + } + + @Override + public ResponseEntity run( + String portalShortcode, String studyShortcode, String envName, UUID id) { + AdminUser operator = authUtilService.requireAdminUser(request); + EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); + ExportIntegrationJob job = + exportIntegrationExtService.run( + PortalStudyEnvAuthContext.of( + operator, portalShortcode, studyShortcode, environmentName), + id); + return ResponseEntity.ok(job); + } + + @Override + public ResponseEntity create( + String portalShortcode, String studyShortcode, String envName, Object body) { + AdminUser adminUser = authUtilService.requireAdminUser(request); + EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); + ExportIntegration integration = objectMapper.convertValue(body, ExportIntegration.class); + ExportIntegration newIntegration = + exportIntegrationExtService.create( + PortalStudyEnvAuthContext.of( + adminUser, portalShortcode, studyShortcode, environmentName), + integration); + return ResponseEntity.ok(newIntegration); + } + + @Override + public ResponseEntity findJobsByStudy( + String portalShortcode, String studyShortcode, String envName) { + AdminUser operator = authUtilService.requireAdminUser(request); + EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); + List integrations = + exportIntegrationExtService.listJobs( + PortalStudyEnvAuthContext.of( + operator, portalShortcode, studyShortcode, environmentName)); + return ResponseEntity.ok(integrations); + } +} diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/notifications/TriggerController.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/notifications/TriggerController.java index 3b93d9ecca..5cd6c19755 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/notifications/TriggerController.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/controller/notifications/TriggerController.java @@ -43,12 +43,12 @@ public TriggerController( @Override public ResponseEntity findByStudy( String portalShortcode, String studyShortcode, String envName) { - AdminUser adminUser = authUtilService.requireAdminUser(request); + AdminUser operator = authUtilService.requireAdminUser(request); EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName); List configs = triggerExtService.findForStudy( PortalStudyEnvAuthContext.of( - adminUser, portalShortcode, studyShortcode, environmentName)); + operator, portalShortcode, studyShortcode, environmentName)); return ResponseEntity.ok(configs); } diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/ConfigExtService.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/ConfigExtService.java index bceb8f1504..cc2687af89 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/ConfigExtService.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/ConfigExtService.java @@ -4,6 +4,7 @@ import bio.terra.pearl.core.model.admin.AdminUser; import bio.terra.pearl.core.service.address.AddressValidationConfig; import bio.terra.pearl.core.service.exception.PermissionDeniedException; +import bio.terra.pearl.core.service.export.integration.AirtableExporter; import bio.terra.pearl.core.service.kit.pepper.LivePepperDSMClient; import bio.terra.pearl.core.shared.ApplicationRoutingPaths; import java.util.Map; @@ -18,16 +19,20 @@ public class ConfigExtService { private Map configMap; private final LivePepperDSMClient.PepperDSMConfig pepperDSMConfig; private final AddressValidationConfig addressValidationConfig; + private final AirtableExporter.AirtableConfig airtableConfig; public ConfigExtService( B2CConfiguration b2CConfiguration, ApplicationRoutingPaths applicationRoutingPaths, LivePepperDSMClient.PepperDSMConfig pepperDSMConfig, - AddressValidationConfig addressValidationConfig) { + AddressValidationConfig addressValidationConfig, + AirtableExporter.AirtableConfig airtableConfig) { this.b2CConfiguration = b2CConfiguration; this.pepperDSMConfig = pepperDSMConfig; this.applicationRoutingPaths = applicationRoutingPaths; this.addressValidationConfig = addressValidationConfig; + this.airtableConfig = airtableConfig; + configMap = buildConfigMap(); } @@ -64,7 +69,7 @@ private Map buildConfigMap() { if (!user.isSuperuser()) { throw new PermissionDeniedException("You do not have permission to view this config"); } - var configMap = + Map> internalConfigMap = Map.of( "pepperDsmConfig", Map.of( @@ -78,8 +83,10 @@ private Map buildConfigMap() { Map.of( "addrValidationServiceClass", addressValidationConfig.getAddressValidationClass(), "smartyAuthId", addressValidationConfig.getAuthId(), - "smartyAuthToken", maskSecret(addressValidationConfig.getAuthToken()))); - return configMap; + "smartyAuthToken", maskSecret(addressValidationConfig.getAuthToken())), + "airtable", + Map.of("authToken", maskSecret(airtableConfig.getAuthToken()))); + return internalConfigMap; } public static String maskSecret(String secret) { diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/enrollee/EnrolleeImportExtService.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/enrollee/EnrolleeImportExtService.java index 60ac73c758..fd5fc1be69 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/enrollee/EnrolleeImportExtService.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/enrollee/EnrolleeImportExtService.java @@ -6,12 +6,12 @@ import bio.terra.pearl.core.model.dataimport.Import; import bio.terra.pearl.core.model.dataimport.ImportItemStatus; import bio.terra.pearl.core.model.dataimport.ImportStatus; -import bio.terra.pearl.core.service.dataimport.ImportFileFormat; -import bio.terra.pearl.core.service.dataimport.ImportItemService; -import bio.terra.pearl.core.service.dataimport.ImportService; import bio.terra.pearl.core.service.exception.NotFoundException; import bio.terra.pearl.core.service.exception.PermissionDeniedException; import bio.terra.pearl.core.service.export.EnrolleeImportService; +import bio.terra.pearl.core.service.export.dataimport.ImportFileFormat; +import bio.terra.pearl.core.service.export.dataimport.ImportItemService; +import bio.terra.pearl.core.service.export.dataimport.ImportService; import bio.terra.pearl.core.service.study.StudyEnvironmentService; import java.io.InputStream; import java.util.List; diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/DataRepoExportExtService.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/DataRepoExportExtService.java similarity index 94% rename from api-admin/src/main/java/bio/terra/pearl/api/admin/service/DataRepoExportExtService.java rename to api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/DataRepoExportExtService.java index e063690e20..27862d022b 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/DataRepoExportExtService.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/DataRepoExportExtService.java @@ -1,10 +1,10 @@ -package bio.terra.pearl.api.admin.service; +package bio.terra.pearl.api.admin.service.export; import bio.terra.pearl.api.admin.model.CreateDataset; import bio.terra.pearl.api.admin.service.auth.*; import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext; -import bio.terra.pearl.core.model.datarepo.DataRepoJob; -import bio.terra.pearl.core.model.datarepo.Dataset; +import bio.terra.pearl.core.model.export.datarepo.DataRepoJob; +import bio.terra.pearl.core.model.export.datarepo.Dataset; import bio.terra.pearl.core.service.datarepo.DataRepoExportService; import bio.terra.pearl.core.service.exception.PermissionDeniedException; import bio.terra.pearl.core.service.study.StudyEnvironmentService; diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/EnrolleeExportExtService.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/EnrolleeExportExtService.java similarity index 68% rename from api-admin/src/main/java/bio/terra/pearl/api/admin/service/EnrolleeExportExtService.java rename to api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/EnrolleeExportExtService.java index 72b73ae44f..6a8ec98cd9 100644 --- a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/EnrolleeExportExtService.java +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/EnrolleeExportExtService.java @@ -1,36 +1,29 @@ -package bio.terra.pearl.api.admin.service; +package bio.terra.pearl.api.admin.service.export; -import bio.terra.pearl.api.admin.service.auth.AuthUtilService; import bio.terra.pearl.api.admin.service.auth.EnforcePortalStudyEnvPermission; import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.DictionaryExportService; import bio.terra.pearl.core.service.export.EnrolleeExportService; -import bio.terra.pearl.core.service.export.ExportOptions; -import bio.terra.pearl.core.service.study.StudyEnvironmentService; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; import java.io.OutputStream; import org.springframework.stereotype.Service; @Service public class EnrolleeExportExtService { - private AuthUtilService authUtilService; - private StudyEnvironmentService studyEnvironmentService; private EnrolleeExportService enrolleeExportService; private DictionaryExportService dictionaryExportService; public EnrolleeExportExtService( - AuthUtilService authUtilService, - StudyEnvironmentService studyEnvironmentService, EnrolleeExportService enrolleeExportService, DictionaryExportService dictionaryExportService) { - this.authUtilService = authUtilService; - this.studyEnvironmentService = studyEnvironmentService; this.enrolleeExportService = enrolleeExportService; this.dictionaryExportService = dictionaryExportService; } @EnforcePortalStudyEnvPermission(permission = "participant_data_view") public void export( - PortalStudyEnvAuthContext authContext, ExportOptions options, OutputStream os) { + PortalStudyEnvAuthContext authContext, ExportOptionsWithExpression options, OutputStream os) { enrolleeExportService.export(options, authContext.getStudyEnvironment().getId(), os); } diff --git a/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/ExportIntegrationExtService.java b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/ExportIntegrationExtService.java new file mode 100644 index 0000000000..5fe7a4146d --- /dev/null +++ b/api-admin/src/main/java/bio/terra/pearl/api/admin/service/export/ExportIntegrationExtService.java @@ -0,0 +1,73 @@ +package bio.terra.pearl.api.admin.service.export; + +import bio.terra.pearl.api.admin.service.auth.EnforcePortalStudyEnvPermission; +import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext; +import bio.terra.pearl.core.model.audit.ResponsibleEntity; +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import bio.terra.pearl.core.service.exception.NotFoundException; +import bio.terra.pearl.core.service.export.integration.ExportIntegrationJobService; +import bio.terra.pearl.core.service.export.integration.ExportIntegrationService; +import java.util.List; +import java.util.UUID; +import org.springframework.stereotype.Service; + +@Service +public class ExportIntegrationExtService { + private final ExportIntegrationService exportIntegrationService; + + private final ExportIntegrationJobService exportIntegrationJobService; + + public ExportIntegrationExtService( + ExportIntegrationService exportIntegrationService, + ExportIntegrationJobService exportIntegrationJobService) { + this.exportIntegrationService = exportIntegrationService; + this.exportIntegrationJobService = exportIntegrationJobService; + } + + @EnforcePortalStudyEnvPermission(permission = "BASE") + public List list(PortalStudyEnvAuthContext authContext) { + return exportIntegrationService.findByStudyEnvironmentId( + authContext.getStudyEnvironment().getId()); + } + + @EnforcePortalStudyEnvPermission(permission = "BASE") + public ExportIntegration find(PortalStudyEnvAuthContext authContext, UUID id) { + ExportIntegration integration = + exportIntegrationService + .findWithOptions(id) + .orElseThrow(() -> new NotFoundException("Export Integration not found")); + if (!integration.getStudyEnvironmentId().equals(authContext.getStudyEnvironment().getId())) { + throw new NotFoundException("Export Integration not found"); + } + return integration; + } + + @EnforcePortalStudyEnvPermission(permission = "participant_data_view") + public ExportIntegrationJob run(PortalStudyEnvAuthContext authContext, UUID id) { + ExportIntegration integration = + exportIntegrationService + .findWithOptions(id) + .orElseThrow(() -> new NotFoundException("Export Integration not found")); + if (!integration.getStudyEnvironmentId().equals(authContext.getStudyEnvironment().getId())) { + throw new NotFoundException("Export Integration not found"); + } + ExportIntegrationJob job = + exportIntegrationService.doExport( + integration, new ResponsibleEntity(authContext.getOperator())); + return job; + } + + @EnforcePortalStudyEnvPermission(permission = "export_integration") + public ExportIntegration create( + PortalStudyEnvAuthContext authContext, ExportIntegration exportIntegration) { + exportIntegration.setStudyEnvironmentId(authContext.getStudyEnvironment().getId()); + return exportIntegrationService.create(exportIntegration); + } + + @EnforcePortalStudyEnvPermission(permission = "BASE") + public List listJobs(PortalStudyEnvAuthContext authContext) { + return exportIntegrationJobService.findByStudyEnvironment( + authContext.getStudyEnvironment().getId()); + } +} diff --git a/api-admin/src/main/resources/api/openapi.yml b/api-admin/src/main/resources/api/openapi.yml index 251bddf80c..21d45d2a38 100644 --- a/api-admin/src/main/resources/api/openapi.yml +++ b/api-admin/src/main/resources/api/openapi.yml @@ -1173,7 +1173,7 @@ paths: - { name: excludeModules, in: query, required: false, schema: { type: array, items: { type: string } } } - { name: filter, in: query, required: false, schema: { type: string, default: '' } } - { name: fileFormat, in: query, required: false, schema: { type: string, default: "TSV" } } - - { name: limit, in: query, required: false, schema: { type: integer } } + - { name: rowLimit, in: query, required: false, schema: { type: integer } } responses: '200': description: export data @@ -1200,6 +1200,85 @@ paths: content: { text/plain: { schema: { type: string, format: binary } } } '500': $ref: '#/components/responses/ServerError' + /api/portals/v1/{portalShortcode}/studies/{studyShortcode}/env/{envName}/exportIntegrations: + get: + summary: gets all the active exportIntegrations for the environment + tags: [ exportIntegration ] + operationId: findByStudy + parameters: + - *portalShortcodeParam + - *studyShortcodeParam + - *envNameParam + responses: + '200': + description: The exportIntegrations + content: *jsonContent + '500': + $ref: '#/components/responses/ServerError' + post: + summary: create a new exportIntegration for the environment + tags: [ exportIntegration ] + operationId: create + parameters: + - *portalShortcodeParam + - *studyShortcodeParam + - *envNameParam + requestBody: + required: true + content: *jsonContent + responses: + '200': + description: The exportIntegration + content: *jsonContent + '500': + $ref: '#/components/responses/ServerError' + /api/portals/v1/{portalShortcode}/studies/{studyShortcode}/env/{envName}/exportIntegrations/{id}: + get: + summary: gets the exportIntegration by id + tags: [ exportIntegration ] + operationId: get + parameters: + - *portalShortcodeParam + - *studyShortcodeParam + - *envNameParam + - { name: id, in: path, required: true, schema: { type: string, format: uuid } } + responses: + '200': + description: The exportIntegration + content: *jsonContent + '500': + $ref: '#/components/responses/ServerError' + /api/portals/v1/{portalShortcode}/studies/{studyShortcode}/env/{envName}/exportIntegrations/{id}/run: + post: + summary: runs the exportIntegration by id + tags: [ exportIntegration ] + operationId: run + parameters: + - *portalShortcodeParam + - *studyShortcodeParam + - *envNameParam + - { name: id, in: path, required: true, schema: { type: string, format: uuid } } + responses: + '200': + description: The exportIntegration result + content: *jsonContent + '500': + $ref: '#/components/responses/ServerError' + /api/portals/v1/{portalShortcode}/studies/{studyShortcode}/env/{envName}/exportIntegrationJobs: + get: + summary: gets all the exportIntegrationJobs for the environment + tags: [ exportIntegration ] + operationId: findJobsByStudy + parameters: + - *portalShortcodeParam + - *studyShortcodeParam + - *envNameParam + responses: + '200': + description: The exportIntegrationJobs + content: *jsonContent + '500': + $ref: '#/components/responses/ServerError' /api/portals/v1/{portalShortcode}/env/{envName}/mailingList: post: summary: Adds a list of contacts to the mailing list for the environment diff --git a/api-admin/src/main/resources/application.yml b/api-admin/src/main/resources/application.yml index 9125a80709..96e4df91ef 100644 --- a/api-admin/src/main/resources/application.yml +++ b/api-admin/src/main/resources/application.yml @@ -43,6 +43,8 @@ env: mixpanel: enabled: ${MIXPANEL_ENABLED:false} token: ${MIXPANEL_TOKEN:} + airtable: + authToken: ${AIRTABLE_AUTH_TOKEN:foobar} # Below here is non-deployment-specific diff --git a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/ConfigExtServiceTests.java b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/ConfigExtServiceTests.java index eef8141ac6..8ff487647c 100644 --- a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/ConfigExtServiceTests.java +++ b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/ConfigExtServiceTests.java @@ -8,6 +8,7 @@ import bio.terra.pearl.core.model.admin.AdminUser; import bio.terra.pearl.core.service.address.AddressValidationConfig; import bio.terra.pearl.core.service.exception.PermissionDeniedException; +import bio.terra.pearl.core.service.export.integration.AirtableExporter; import bio.terra.pearl.core.service.kit.pepper.LivePepperDSMClient; import bio.terra.pearl.core.shared.ApplicationRoutingPaths; import java.util.Map; @@ -26,6 +27,7 @@ public class ConfigExtServiceTests { @MockBean private B2CConfiguration b2CConfiguration; @MockBean private LivePepperDSMClient.PepperDSMConfig pepperDSMConfig; @MockBean private AddressValidationConfig addressValidationConfig; + @MockBean private AirtableExporter.AirtableConfig airtableConfig; @Test public void testConfigMap() { @@ -38,7 +40,11 @@ public void testConfigMap() { when(b2CConfiguration.policyName()).thenReturn("policy123"); ConfigExtService configExtService = new ConfigExtService( - b2CConfiguration, applicationRoutingPaths, pepperDSMConfig, addressValidationConfig); + b2CConfiguration, + applicationRoutingPaths, + pepperDSMConfig, + addressValidationConfig, + airtableConfig); Map configMap = configExtService.getConfigMap(); Assertions.assertEquals("something.org", configMap.get("participantUiHostname")); } @@ -48,7 +54,11 @@ public void testInternalConfigRequiresSuperuser() { AdminUser user = AdminUser.builder().superuser(false).build(); ConfigExtService configExtService = new ConfigExtService( - b2CConfiguration, applicationRoutingPaths, pepperDSMConfig, addressValidationConfig); + b2CConfiguration, + applicationRoutingPaths, + pepperDSMConfig, + addressValidationConfig, + airtableConfig); Assertions.assertThrows( PermissionDeniedException.class, () -> { @@ -74,7 +84,11 @@ public void testInternalConfigMap() { AddressValidationConfig testAddrConfig = new AddressValidationConfig(mockEnvironment); ConfigExtService configExtService = new ConfigExtService( - b2CConfiguration, applicationRoutingPaths, testPepperConfig, testAddrConfig); + b2CConfiguration, + applicationRoutingPaths, + testPepperConfig, + testAddrConfig, + airtableConfig); Map dsmConfigMap = (Map) configExtService.getInternalConfigMap(user).get("pepperDsmConfig"); Map addressValidationConfigMap = diff --git a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/DataRepoExportExtServiceTests.java b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/DataRepoExportExtServiceTests.java similarity index 98% rename from api-admin/src/test/java/bio/terra/pearl/api/admin/service/DataRepoExportExtServiceTests.java rename to api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/DataRepoExportExtServiceTests.java index ea43083385..274395290c 100644 --- a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/DataRepoExportExtServiceTests.java +++ b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/DataRepoExportExtServiceTests.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.api.admin.service; +package bio.terra.pearl.api.admin.service.export; import bio.terra.pearl.api.admin.BaseSpringBootTest; import bio.terra.pearl.api.admin.model.CreateDataset; diff --git a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/EnrolleeExportExtServiceTests.java b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/EnrolleeExportExtServiceTests.java similarity index 93% rename from api-admin/src/test/java/bio/terra/pearl/api/admin/service/EnrolleeExportExtServiceTests.java rename to api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/EnrolleeExportExtServiceTests.java index b65bb5f0b8..b8a2b76b06 100644 --- a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/EnrolleeExportExtServiceTests.java +++ b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/EnrolleeExportExtServiceTests.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.api.admin.service; +package bio.terra.pearl.api.admin.service.export; import bio.terra.pearl.api.admin.AuthAnnotationSpec; import bio.terra.pearl.api.admin.AuthTestUtils; diff --git a/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/ExportIntegrationExtServiceTests.java b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/ExportIntegrationExtServiceTests.java new file mode 100644 index 0000000000..3cb0b6aadf --- /dev/null +++ b/api-admin/src/test/java/bio/terra/pearl/api/admin/service/export/ExportIntegrationExtServiceTests.java @@ -0,0 +1,30 @@ +package bio.terra.pearl.api.admin.service.export; + +import bio.terra.pearl.api.admin.AuthAnnotationSpec; +import bio.terra.pearl.api.admin.AuthTestUtils; +import bio.terra.pearl.api.admin.BaseSpringBootTest; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +public class ExportIntegrationExtServiceTests extends BaseSpringBootTest { + + @Autowired private ExportIntegrationExtService exportIntegrationExtService; + + @Test + public void testAuthentication() { + AuthTestUtils.assertAllMethodsAnnotated( + exportIntegrationExtService, + Map.of( + "list", + AuthAnnotationSpec.withPortalStudyEnvPerm("BASE"), + "find", + AuthAnnotationSpec.withPortalStudyEnvPerm("BASE"), + "run", + AuthAnnotationSpec.withPortalStudyEnvPerm("participant_data_view"), + "create", + AuthAnnotationSpec.withPortalStudyEnvPerm("export_integration"), + "listJobs", + AuthAnnotationSpec.withPortalStudyEnvPerm("BASE"))); + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/dao/BaseJdbiDao.java b/core/src/main/java/bio/terra/pearl/core/dao/BaseJdbiDao.java index c4a19282a2..b8a78195fd 100644 --- a/core/src/main/java/bio/terra/pearl/core/dao/BaseJdbiDao.java +++ b/core/src/main/java/bio/terra/pearl/core/dao/BaseJdbiDao.java @@ -540,6 +540,19 @@ protected void deleteByProperty(String columnName, Object columnValue) { ); } + protected void deleteByProperty(String columnName, Collection columnValues) { + if (columnValues.isEmpty()) { + return; + } + jdbi.withHandle(handle -> + handle.createUpdate(""" + delete from %s where %s in () + """.formatted(tableName, columnName)) + .bindList("columnValues", columnValues) + .execute() + ); + } + protected void deleteByTwoProperties(String columnName, Object columnValue, String column2Name, Object column2Value) { jdbi.withHandle(handle -> handle.createUpdate(""" diff --git a/core/src/main/java/bio/terra/pearl/core/dao/export/ExportIntegrationDao.java b/core/src/main/java/bio/terra/pearl/core/dao/export/ExportIntegrationDao.java new file mode 100644 index 0000000000..9c7cc8a891 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/dao/export/ExportIntegrationDao.java @@ -0,0 +1,29 @@ +package bio.terra.pearl.core.dao.export; + +import bio.terra.pearl.core.dao.BaseMutableJdbiDao; +import bio.terra.pearl.core.model.export.ExportIntegration; +import org.jdbi.v3.core.Jdbi; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.UUID; + +@Component +public class ExportIntegrationDao extends BaseMutableJdbiDao { + public ExportIntegrationDao(Jdbi jdbi) { + super(jdbi); + } + + @Override + protected Class getClazz() { + return ExportIntegration.class; + } + + public List findByStudyEnvironmentId(UUID studyEnvId) { + return findAllByProperty("study_environment_id", studyEnvId); + } + + public void deleteByStudyEnvironmentId(UUID studyEnvId) { + deleteByProperty("study_environment_id", studyEnvId); + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/dao/export/ExportIntegrationJobDao.java b/core/src/main/java/bio/terra/pearl/core/dao/export/ExportIntegrationJobDao.java new file mode 100644 index 0000000000..38ea6e1995 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/dao/export/ExportIntegrationJobDao.java @@ -0,0 +1,47 @@ +package bio.terra.pearl.core.dao.export; + +import bio.terra.pearl.core.dao.BaseMutableJdbiDao; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import org.jdbi.v3.core.Jdbi; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.UUID; + +@Component +public class ExportIntegrationJobDao extends BaseMutableJdbiDao { + + public ExportIntegrationJobDao(Jdbi jdbi) { + super(jdbi); + } + + @Override + protected Class getClazz() { + return ExportIntegrationJob.class; + } + + public List findByExportIntegrationId(UUID integrationId) { + return findAllByProperty("export_integration_id", integrationId); + } + + public List findByStudyEnvironmentId(UUID studyEnvironmentId) { + return jdbi.withHandle(handle -> + handle.createQuery(""" + select %s from %s eij join export_integration + on eij.export_integration_id = export_integration.id + where export_integration.study_environment_id = :studyEnvironmentId + """.formatted(prefixedGetQueryColumns("eij"), tableName)) + .bind("studyEnvironmentId", studyEnvironmentId) + .mapTo(clazz) + .list() + ); + } + + public void deleteByExportIntegrationId(UUID integrationId) { + deleteByProperty("export_integration_id", integrationId); + } + + public void deleteByExportIntegrationIds(List integrationIds) { + deleteByProperty("export_integration_id", integrationIds); + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/dao/export/ExportOptionsDao.java b/core/src/main/java/bio/terra/pearl/core/dao/export/ExportOptionsDao.java new file mode 100644 index 0000000000..928a119442 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/dao/export/ExportOptionsDao.java @@ -0,0 +1,27 @@ +package bio.terra.pearl.core.dao.export; + +import bio.terra.pearl.core.dao.BaseMutableJdbiDao; +import bio.terra.pearl.core.model.export.ExportOptions; +import org.jdbi.v3.core.Jdbi; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class ExportOptionsDao extends BaseMutableJdbiDao { + public ExportOptionsDao(Jdbi jdbi) { + super(jdbi); + } + + @Override + protected Class getClazz() { + return ExportOptions.class; + } + + @Override + protected List generateGetFields(Class clazz) { + List getFields = super.generateGetFields(clazz); + getFields.add("excludeModules"); + return getFields; + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/dao/datarepo/DataRepoJobDao.java b/core/src/main/java/bio/terra/pearl/core/dao/export/datarepo/DataRepoJobDao.java similarity index 87% rename from core/src/main/java/bio/terra/pearl/core/dao/datarepo/DataRepoJobDao.java rename to core/src/main/java/bio/terra/pearl/core/dao/export/datarepo/DataRepoJobDao.java index 89a6ba6879..4ebdd815a2 100644 --- a/core/src/main/java/bio/terra/pearl/core/dao/datarepo/DataRepoJobDao.java +++ b/core/src/main/java/bio/terra/pearl/core/dao/export/datarepo/DataRepoJobDao.java @@ -1,8 +1,7 @@ -package bio.terra.pearl.core.dao.datarepo; +package bio.terra.pearl.core.dao.export.datarepo; import bio.terra.pearl.core.dao.BaseMutableJdbiDao; -import bio.terra.pearl.core.model.datarepo.DataRepoJob; -import bio.terra.pearl.core.model.datarepo.Dataset; +import bio.terra.pearl.core.model.export.datarepo.DataRepoJob; import org.jdbi.v3.core.Jdbi; import org.springframework.stereotype.Component; diff --git a/core/src/main/java/bio/terra/pearl/core/dao/datarepo/DatasetDao.java b/core/src/main/java/bio/terra/pearl/core/dao/export/datarepo/DatasetDao.java similarity index 86% rename from core/src/main/java/bio/terra/pearl/core/dao/datarepo/DatasetDao.java rename to core/src/main/java/bio/terra/pearl/core/dao/export/datarepo/DatasetDao.java index 70063a9bc2..7cbc6ce463 100644 --- a/core/src/main/java/bio/terra/pearl/core/dao/datarepo/DatasetDao.java +++ b/core/src/main/java/bio/terra/pearl/core/dao/export/datarepo/DatasetDao.java @@ -1,14 +1,14 @@ -package bio.terra.pearl.core.dao.datarepo; +package bio.terra.pearl.core.dao.export.datarepo; import bio.terra.pearl.core.dao.BaseMutableJdbiDao; -import bio.terra.pearl.core.model.datarepo.Dataset; +import bio.terra.pearl.core.model.export.datarepo.Dataset; import java.time.Instant; import java.util.List; import java.util.Optional; import java.util.UUID; -import bio.terra.pearl.core.model.datarepo.DatasetStatus; +import bio.terra.pearl.core.model.export.datarepo.DatasetStatus; import org.jdbi.v3.core.Jdbi; import org.springframework.stereotype.Component; diff --git a/core/src/main/java/bio/terra/pearl/core/model/export/ExportDestinationType.java b/core/src/main/java/bio/terra/pearl/core/model/export/ExportDestinationType.java new file mode 100644 index 0000000000..91a86948da --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/model/export/ExportDestinationType.java @@ -0,0 +1,5 @@ +package bio.terra.pearl.core.model.export; + +public enum ExportDestinationType { + AIRTABLE +} diff --git a/core/src/main/java/bio/terra/pearl/core/model/export/ExportIntegration.java b/core/src/main/java/bio/terra/pearl/core/model/export/ExportIntegration.java new file mode 100644 index 0000000000..a807330a86 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/model/export/ExportIntegration.java @@ -0,0 +1,23 @@ +package bio.terra.pearl.core.model.export; + +import bio.terra.pearl.core.model.BaseEntity; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class ExportIntegration extends BaseEntity { + private String name; + private UUID studyEnvironmentId; + private boolean enabled = true; + private ExportDestinationType destinationType; + private String destinationUrl; + private UUID exportOptionsId; + private ExportOptions exportOptions; +} diff --git a/core/src/main/java/bio/terra/pearl/core/model/export/ExportIntegrationJob.java b/core/src/main/java/bio/terra/pearl/core/model/export/ExportIntegrationJob.java new file mode 100644 index 0000000000..ebefce1a98 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/model/export/ExportIntegrationJob.java @@ -0,0 +1,32 @@ +package bio.terra.pearl.core.model.export; + +import bio.terra.pearl.core.model.BaseEntity; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.time.Instant; +import java.util.UUID; + +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class ExportIntegrationJob extends BaseEntity { + private UUID exportIntegrationId; + private UUID creatingAdminUserId; + private String systemProcess; + private ExportIntegrationJob.Status status; + private Instant startedAt; + private Instant completedAt; + private String result; + + public enum Status { + NEW, + GENERATING, + SENDING, + COMPLETE, + FAILED + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/model/export/ExportOptions.java b/core/src/main/java/bio/terra/pearl/core/model/export/ExportOptions.java new file mode 100644 index 0000000000..3393c821e8 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/model/export/ExportOptions.java @@ -0,0 +1,33 @@ +package bio.terra.pearl.core.model.export; + +import bio.terra.pearl.core.model.BaseEntity; +import bio.terra.pearl.core.service.export.ExportFileFormat; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.util.ArrayList; +import java.util.List; + +@SuperBuilder +@Getter @Setter +@NoArgsConstructor +public class ExportOptions extends BaseEntity { + @Builder.Default + private boolean splitOptionsIntoColumns = false; + @Builder.Default + private boolean stableIdsForOptions = false; + @Builder.Default + private boolean onlyIncludeMostRecent = true; + private String filterString; + @Builder.Default + private ExportFileFormat fileFormat = ExportFileFormat.TSV; + + private Integer rowLimit; + @Builder.Default + private boolean includeSubHeaders = true; + @Builder.Default + private List excludeModules = new ArrayList<>(); +} diff --git a/core/src/main/java/bio/terra/pearl/core/model/datarepo/DataRepoJob.java b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/DataRepoJob.java similarity index 90% rename from core/src/main/java/bio/terra/pearl/core/model/datarepo/DataRepoJob.java rename to core/src/main/java/bio/terra/pearl/core/model/export/datarepo/DataRepoJob.java index 2e22a97fc1..8c5633ec45 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/datarepo/DataRepoJob.java +++ b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/DataRepoJob.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.model.datarepo; +package bio.terra.pearl.core.model.export.datarepo; import bio.terra.pearl.core.model.BaseEntity; import lombok.Getter; diff --git a/core/src/main/java/bio/terra/pearl/core/model/datarepo/Dataset.java b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/Dataset.java similarity index 91% rename from core/src/main/java/bio/terra/pearl/core/model/datarepo/Dataset.java rename to core/src/main/java/bio/terra/pearl/core/model/export/datarepo/Dataset.java index 7fdadc9198..dfa7cd9a86 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/datarepo/Dataset.java +++ b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/Dataset.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.model.datarepo; +package bio.terra.pearl.core.model.export.datarepo; import bio.terra.pearl.core.model.BaseEntity; import lombok.Getter; diff --git a/core/src/main/java/bio/terra/pearl/core/model/datarepo/DatasetStatus.java b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/DatasetStatus.java similarity index 61% rename from core/src/main/java/bio/terra/pearl/core/model/datarepo/DatasetStatus.java rename to core/src/main/java/bio/terra/pearl/core/model/export/datarepo/DatasetStatus.java index 063e66c39c..ac63736fa1 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/datarepo/DatasetStatus.java +++ b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/DatasetStatus.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.model.datarepo; +package bio.terra.pearl.core.model.export.datarepo; public enum DatasetStatus { CREATED, diff --git a/core/src/main/java/bio/terra/pearl/core/model/datarepo/JobType.java b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/JobType.java similarity index 61% rename from core/src/main/java/bio/terra/pearl/core/model/datarepo/JobType.java rename to core/src/main/java/bio/terra/pearl/core/model/export/datarepo/JobType.java index 226939ef76..e2128d404b 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/datarepo/JobType.java +++ b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/JobType.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.model.datarepo; +package bio.terra.pearl.core.model.export.datarepo; public enum JobType { CREATE_DATASET, diff --git a/core/src/main/java/bio/terra/pearl/core/model/datarepo/TdrColumn.java b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/TdrColumn.java similarity index 69% rename from core/src/main/java/bio/terra/pearl/core/model/datarepo/TdrColumn.java rename to core/src/main/java/bio/terra/pearl/core/model/export/datarepo/TdrColumn.java index 8c43b544f8..51a64fcb76 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/datarepo/TdrColumn.java +++ b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/TdrColumn.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.model.datarepo; +package bio.terra.pearl.core.model.export.datarepo; import bio.terra.datarepo.model.TableDataType; diff --git a/core/src/main/java/bio/terra/pearl/core/model/datarepo/TdrTable.java b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/TdrTable.java similarity index 68% rename from core/src/main/java/bio/terra/pearl/core/model/datarepo/TdrTable.java rename to core/src/main/java/bio/terra/pearl/core/model/export/datarepo/TdrTable.java index 9eb3a50bb4..a5a742c7a9 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/datarepo/TdrTable.java +++ b/core/src/main/java/bio/terra/pearl/core/model/export/datarepo/TdrTable.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.model.datarepo; +package bio.terra.pearl.core.model.export.datarepo; import java.util.Set; diff --git a/core/src/main/java/bio/terra/pearl/core/model/study/StudyEnvironment.java b/core/src/main/java/bio/terra/pearl/core/model/study/StudyEnvironment.java index e0e3e7173f..8fd188e16b 100644 --- a/core/src/main/java/bio/terra/pearl/core/model/study/StudyEnvironment.java +++ b/core/src/main/java/bio/terra/pearl/core/model/study/StudyEnvironment.java @@ -2,6 +2,7 @@ import bio.terra.pearl.core.model.BaseEntity; import bio.terra.pearl.core.model.EnvironmentName; +import bio.terra.pearl.core.model.export.ExportIntegration; import bio.terra.pearl.core.model.kit.KitType; import bio.terra.pearl.core.model.kit.StudyEnvironmentKitType; import bio.terra.pearl.core.model.notification.Trigger; @@ -33,4 +34,7 @@ public class StudyEnvironment extends BaseEntity { private List triggers = new ArrayList<>(); @Builder.Default private List kitTypes = new ArrayList<>(); + @Builder.Default + private List exportIntegrations = new ArrayList<>(); + } diff --git a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportFileFormat.java b/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportFileFormat.java deleted file mode 100644 index a5fb026534..0000000000 --- a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportFileFormat.java +++ /dev/null @@ -1,6 +0,0 @@ -package bio.terra.pearl.core.service.dataimport; - -public enum ImportFileFormat { - TSV, - CSV -} diff --git a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoClient.java b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoClient.java index 9e2c47d867..8331440bdb 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoClient.java +++ b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoClient.java @@ -5,7 +5,7 @@ import bio.terra.datarepo.api.UnauthenticatedApi; import bio.terra.datarepo.client.ApiException; import bio.terra.datarepo.model.*; -import bio.terra.pearl.core.model.datarepo.TdrTable; +import bio.terra.pearl.core.model.export.datarepo.TdrTable; import bio.terra.pearl.core.shared.GoogleServiceAccountUtils; import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; diff --git a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoExportService.java b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoExportService.java index 23fec129a1..4d821db3d1 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoExportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoExportService.java @@ -3,15 +3,16 @@ import bio.terra.datarepo.client.ApiException; import bio.terra.datarepo.model.JobModel; import bio.terra.datarepo.model.JobModel.JobStatusEnum; -import bio.terra.pearl.core.dao.datarepo.DataRepoJobDao; -import bio.terra.pearl.core.dao.datarepo.DatasetDao; +import bio.terra.pearl.core.dao.export.datarepo.DataRepoJobDao; +import bio.terra.pearl.core.dao.export.datarepo.DatasetDao; import bio.terra.pearl.core.dao.participant.EnrolleeDao; import bio.terra.pearl.core.dao.study.PortalStudyDao; import bio.terra.pearl.core.dao.study.StudyDao; import bio.terra.pearl.core.dao.study.StudyEnvironmentDao; import bio.terra.pearl.core.dao.survey.AnswerDao; import bio.terra.pearl.core.model.admin.AdminUser; -import bio.terra.pearl.core.model.datarepo.*; +import bio.terra.pearl.core.model.export.ExportOptions; +import bio.terra.pearl.core.model.export.datarepo.*; import bio.terra.pearl.core.model.study.PortalStudy; import bio.terra.pearl.core.model.study.StudyEnvironment; import bio.terra.pearl.core.service.azure.AzureBlobStorageClient; @@ -19,10 +20,7 @@ import bio.terra.pearl.core.service.exception.datarepo.DatasetCreationException; import bio.terra.pearl.core.service.exception.datarepo.DatasetDeletionException; import bio.terra.pearl.core.service.exception.datarepo.DatasetNotFoundException; -import bio.terra.pearl.core.service.export.EnrolleeExportService; -import bio.terra.pearl.core.service.export.ExportFileFormat; -import bio.terra.pearl.core.service.export.ExportOptions; -import bio.terra.pearl.core.service.export.TsvExporter; +import bio.terra.pearl.core.service.export.*; import bio.terra.pearl.core.service.export.formatters.module.ModuleFormatter; import lombok.extern.slf4j.Slf4j; import org.springframework.core.env.Environment; @@ -157,11 +155,11 @@ public void deleteDataset(StudyEnvironment studyEnv, String datasetName) { } public String uploadCsvToAzureStorage(UUID studyEnvironmentId, UUID datasetId) { - ExportOptions exportOptions = ExportOptions + ExportOptionsWithExpression exportOptions = ExportOptionsWithExpression .builder() .onlyIncludeMostRecent(false) .fileFormat(ExportFileFormat.TSV) - .limit(null) + .rowLimit(null) .build(); //Even though this is actually formatted as a TSV, TDR only accepts files ending in .csv or .json. @@ -214,7 +212,7 @@ public Set generateDatasetSchema(UUID studyEnvironmentId) { .builder() .onlyIncludeMostRecent(false) .fileFormat(ExportFileFormat.TSV) - .limit(null) + .rowLimit(null) .build(); //Backtrack from studyEnvironmentId to get the portalId, so we can export the study environment data diff --git a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoJobService.java b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoJobService.java index f541ef80d6..2d63d939bd 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoJobService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DataRepoJobService.java @@ -1,7 +1,7 @@ package bio.terra.pearl.core.service.datarepo; -import bio.terra.pearl.core.dao.datarepo.DataRepoJobDao; -import bio.terra.pearl.core.model.datarepo.DataRepoJob; +import bio.terra.pearl.core.dao.export.datarepo.DataRepoJobDao; +import bio.terra.pearl.core.model.export.datarepo.DataRepoJob; import bio.terra.pearl.core.service.CrudService; import org.springframework.stereotype.Service; diff --git a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DatasetService.java b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DatasetService.java index 24bb2cd847..ecee50f27b 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/datarepo/DatasetService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/datarepo/DatasetService.java @@ -2,9 +2,9 @@ import bio.terra.datarepo.client.ApiException; import bio.terra.datarepo.model.JobModel; -import bio.terra.pearl.core.dao.datarepo.DatasetDao; -import bio.terra.pearl.core.model.datarepo.Dataset; -import bio.terra.pearl.core.model.datarepo.DatasetStatus; +import bio.terra.pearl.core.dao.export.datarepo.DatasetDao; +import bio.terra.pearl.core.model.export.datarepo.Dataset; +import bio.terra.pearl.core.model.export.datarepo.DatasetStatus; import bio.terra.pearl.core.service.CrudService; import java.time.Instant; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/DictionaryExportService.java b/core/src/main/java/bio/terra/pearl/core/service/export/DictionaryExportService.java index 49612dee13..df25b406f5 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/DictionaryExportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/DictionaryExportService.java @@ -1,5 +1,6 @@ package bio.terra.pearl.core.service.export; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.module.ModuleFormatter; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.stereotype.Service; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeExportService.java b/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeExportService.java index fd73d9342b..6243e7614b 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeExportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeExportService.java @@ -3,6 +3,7 @@ import bio.terra.pearl.core.dao.search.EnrolleeSearchExpressionDao; import bio.terra.pearl.core.dao.survey.AnswerDao; import bio.terra.pearl.core.dao.survey.SurveyQuestionDefinitionDao; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.model.participant.Enrollee; import bio.terra.pearl.core.model.participant.EnrolleeRelation; import bio.terra.pearl.core.model.search.EnrolleeSearchExpressionResult; @@ -81,7 +82,7 @@ public EnrolleeExportService(ProfileService profileService, * exports the specified number of enrollees from the given environment * The enrollees will be returned most-recently-created first * */ - public void export(ExportOptions exportOptions, UUID studyEnvironmentId, OutputStream os) { + public void export(ExportOptionsWithExpression exportOptions, UUID studyEnvironmentId, OutputStream os) { List enrolleeExportData = loadEnrolleeExportData(studyEnvironmentId, exportOptions); @@ -91,10 +92,10 @@ public void export(ExportOptions exportOptions, UUID studyEnvironmentId, OutputS exporter.export(os, exportOptions.isIncludeSubHeaders()); } - public List loadEnrolleeExportData(UUID studyEnvironmentId, ExportOptions exportOptions) { + public List loadEnrolleeExportData(UUID studyEnvironmentId, ExportOptionsWithExpression exportOptions) { return loadEnrolleesForExport( studyEnvironmentConfigService.findByStudyEnvironmentId(studyEnvironmentId), - loadEnrollees(studyEnvironmentId, exportOptions.getFilter(), exportOptions.getLimit())); + loadEnrollees(studyEnvironmentId, exportOptions.getFilterExpression(), exportOptions.getRowLimit())); } private List loadEnrollees(UUID studyEnvironmentId, EnrolleeSearchExpression filter, Integer limit) { diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java b/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java index 982bafbbd0..997528e6ea 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeImportService.java @@ -4,6 +4,7 @@ import bio.terra.pearl.core.model.audit.DataAuditInfo; import bio.terra.pearl.core.model.audit.ResponsibleEntity; import bio.terra.pearl.core.model.dataimport.*; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.model.kit.KitRequest; import bio.terra.pearl.core.model.kit.KitType; import bio.terra.pearl.core.model.participant.Enrollee; @@ -15,9 +16,9 @@ import bio.terra.pearl.core.model.workflow.HubResponse; import bio.terra.pearl.core.model.workflow.ParticipantTask; import bio.terra.pearl.core.model.workflow.TaskType; -import bio.terra.pearl.core.service.dataimport.ImportFileFormat; -import bio.terra.pearl.core.service.dataimport.ImportItemService; -import bio.terra.pearl.core.service.dataimport.ImportService; +import bio.terra.pearl.core.service.export.dataimport.ImportFileFormat; +import bio.terra.pearl.core.service.export.dataimport.ImportItemService; +import bio.terra.pearl.core.service.export.dataimport.ImportService; import bio.terra.pearl.core.service.export.formatters.module.*; import bio.terra.pearl.core.service.kit.KitRequestDto; import bio.terra.pearl.core.service.kit.KitRequestService; @@ -42,7 +43,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import java.beans.FeatureDescriptor; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -59,7 +59,7 @@ public class EnrolleeImportService { .stableIdsForOptions(true) .onlyIncludeMostRecent(true) .fileFormat(ExportFileFormat.TSV) - .limit(null) + .rowLimit(null) .build(); ExportOptions IMPORT_OPTIONS_CSV = ExportOptions @@ -67,7 +67,7 @@ public class EnrolleeImportService { .stableIdsForOptions(true) .onlyIncludeMostRecent(true) .fileFormat(ExportFileFormat.CSV) - .limit(null) + .rowLimit(null) .build(); private final RegistrationService registrationService; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/ExportOptions.java b/core/src/main/java/bio/terra/pearl/core/service/export/ExportOptions.java deleted file mode 100644 index e824e3a27b..0000000000 --- a/core/src/main/java/bio/terra/pearl/core/service/export/ExportOptions.java +++ /dev/null @@ -1,35 +0,0 @@ -package bio.terra.pearl.core.service.export; - -import bio.terra.pearl.core.service.search.EnrolleeSearchExpression; -import lombok.Builder; -import lombok.Getter; -import lombok.experimental.SuperBuilder; - -import java.util.ArrayList; -import java.util.List; - - -@SuperBuilder -@Getter -public final class ExportOptions { - private final boolean splitOptionsIntoColumns; - private final boolean stableIdsForOptions; - private final boolean onlyIncludeMostRecent; - private final EnrolleeSearchExpression filter; - private final ExportFileFormat fileFormat; - private final Integer limit; - private final boolean includeSubHeaders; - @Builder.Default - private List excludeModules = new ArrayList<>(); - - public ExportOptions() { - this.splitOptionsIntoColumns = false; - this.stableIdsForOptions = false; - this.onlyIncludeMostRecent = true; - this.filter = null; - this.fileFormat = ExportFileFormat.TSV; - this.limit = null; - this.includeSubHeaders = false; - this.excludeModules = new ArrayList<>(); - } -} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/ExportOptionsWithExpression.java b/core/src/main/java/bio/terra/pearl/core/service/export/ExportOptionsWithExpression.java new file mode 100644 index 0000000000..dafb1bb028 --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/ExportOptionsWithExpression.java @@ -0,0 +1,16 @@ +package bio.terra.pearl.core.service.export; + +import bio.terra.pearl.core.model.export.ExportOptions; +import bio.terra.pearl.core.service.search.EnrolleeSearchExpression; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +@Getter +@Setter +@SuperBuilder +@NoArgsConstructor +public class ExportOptionsWithExpression extends ExportOptions { + private EnrolleeSearchExpression filterExpression; +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportFileFormat.java b/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportFileFormat.java new file mode 100644 index 0000000000..5f68d1233d --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportFileFormat.java @@ -0,0 +1,6 @@ +package bio.terra.pearl.core.service.export.dataimport; + +public enum ImportFileFormat { + TSV, + CSV +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java b/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportItemService.java similarity index 97% rename from core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java rename to core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportItemService.java index 529e1b8494..bf12321bf5 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportItemService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportItemService.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.service.dataimport; +package bio.terra.pearl.core.service.export.dataimport; import bio.terra.pearl.core.dao.dataimport.ImportItemDao; import bio.terra.pearl.core.model.dataimport.Import; diff --git a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java b/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportService.java similarity index 97% rename from core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java rename to core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportService.java index edb8136ae5..58f8c12739 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/dataimport/ImportService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/dataimport/ImportService.java @@ -1,4 +1,4 @@ -package bio.terra.pearl.core.service.dataimport; +package bio.terra.pearl.core.service.export.dataimport; import bio.terra.pearl.core.dao.dataimport.ImportDao; import bio.terra.pearl.core.model.dataimport.Import; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/AnswerItemFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/AnswerItemFormatter.java index cb4800e697..87822a855e 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/AnswerItemFormatter.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/item/AnswerItemFormatter.java @@ -3,7 +3,7 @@ import bio.terra.pearl.core.model.survey.*; import bio.terra.pearl.core.service.export.BaseExporter; import bio.terra.pearl.core.service.export.DataValueExportType; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.ExportFormatUtils; import bio.terra.pearl.core.service.export.formatters.module.ModuleFormatter; import bio.terra.pearl.core.service.export.formatters.module.SurveyFormatter; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/EnrolleeFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/EnrolleeFormatter.java index 350d23cedd..5bf0ddac56 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/EnrolleeFormatter.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/EnrolleeFormatter.java @@ -2,7 +2,7 @@ import bio.terra.pearl.core.model.participant.Enrollee; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.item.PropertyItemFormatter; import java.util.List; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ParticipantUserFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ParticipantUserFormatter.java index f15891bac5..0ec52f1b01 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ParticipantUserFormatter.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ParticipantUserFormatter.java @@ -2,7 +2,7 @@ import bio.terra.pearl.core.model.participant.ParticipantUser; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.item.PropertyItemFormatter; import java.util.List; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ProfileFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ProfileFormatter.java index db06cf6f5d..d52bab2788 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ProfileFormatter.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/ProfileFormatter.java @@ -1,10 +1,9 @@ package bio.terra.pearl.core.service.export.formatters.module; import bio.terra.pearl.core.model.address.MailingAddress; -import bio.terra.pearl.core.model.participant.ParticipantUser; import bio.terra.pearl.core.model.participant.Profile; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.ExportFormatUtils; import bio.terra.pearl.core.service.export.formatters.item.PropertyItemFormatter; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatter.java b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatter.java index 23d053ef32..93549aa1e9 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatter.java +++ b/core/src/main/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatter.java @@ -2,7 +2,7 @@ import bio.terra.pearl.core.model.survey.*; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.ExportFormatUtils; import bio.terra.pearl.core.service.export.formatters.item.AnswerItemFormatter; import bio.terra.pearl.core.service.export.formatters.item.ItemFormatter; diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/integration/AirtableExporter.java b/core/src/main/java/bio/terra/pearl/core/service/export/integration/AirtableExporter.java new file mode 100644 index 0000000000..4a51746f8c --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/integration/AirtableExporter.java @@ -0,0 +1,65 @@ +package bio.terra.pearl.core.service.export.integration; + +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.service.export.EnrolleeExportService; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.client.WebClient; + +import java.io.ByteArrayOutputStream; +import java.util.function.Consumer; + +@Service +@Slf4j +public class AirtableExporter extends ExternalExporter { + private static final String AIRTABLE_API_KEY_NAME = "env.airtable.authToken"; + private static final String AIRTABLE_BASE_URL = "https://api.airtable.com/"; + private final WebClient webClient; + private final AirtableConfig config; + + public AirtableExporter(WebClient.Builder webClientBuilder, AirtableConfig config, + ExportIntegrationJobService exportIntegrationJobService, + EnrolleeExportService enrolleeExportService) { + super(exportIntegrationJobService, enrolleeExportService); + this.webClient = webClientBuilder.build(); + this.config = config; + } + + protected void send(ExportIntegration integration, ByteArrayOutputStream baos, + Consumer handleComplete, Consumer handleError) { + var postRequest = buildAuthedPostRequest(buildPath(integration), baos); + postRequest.retrieve() + .toBodilessEntity() + .subscribe( + responseEntity -> { + handleComplete.accept("success"); // we can put detailed parsing of the response here if needed + }, + error -> { + handleError.accept(new RuntimeException(error)); + }); + } + + private String buildPath(ExportIntegration integration) { + return AIRTABLE_BASE_URL + integration.getDestinationUrl(); + } + + private WebClient.RequestHeadersSpec buildAuthedPostRequest(String path, ByteArrayOutputStream baos) { + return webClient.post() + .uri(path) + .header("Authorization", "Bearer " + config.getAuthToken()) + .bodyValue(baos.toString()); // there'smight be a way to have the webClient take the stream directly, but I couldn't find it easily + } + + @Component @Getter @Setter + public static class AirtableConfig { + public AirtableConfig(Environment environment) { + this.authToken = environment.getProperty(AIRTABLE_API_KEY_NAME, ""); + } + private String authToken; + } + +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationJobService.java b/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationJobService.java new file mode 100644 index 0000000000..7372e3e55e --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationJobService.java @@ -0,0 +1,28 @@ +package bio.terra.pearl.core.service.export.integration; + +import bio.terra.pearl.core.dao.export.ExportIntegrationJobDao; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import bio.terra.pearl.core.service.CrudService; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.UUID; + +@Service +public class ExportIntegrationJobService extends CrudService { + public ExportIntegrationJobService(ExportIntegrationJobDao dao) { + super(dao); + } + + public List findByStudyEnvironment(UUID studyEnvironmentId) { + return dao.findByStudyEnvironmentId(studyEnvironmentId); + } + + public void deleteByExportIntegrationId(UUID integrationId) { + dao.deleteByExportIntegrationId(integrationId); + } + + public void deleteByExportIntegrationIds(List integrationIds) { + dao.deleteByExportIntegrationIds(integrationIds); + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationService.java b/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationService.java new file mode 100644 index 0000000000..037d88679f --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationService.java @@ -0,0 +1,94 @@ +package bio.terra.pearl.core.service.export.integration; + +import bio.terra.pearl.core.dao.export.ExportIntegrationDao; +import bio.terra.pearl.core.dao.export.ExportOptionsDao; +import bio.terra.pearl.core.model.audit.ResponsibleEntity; +import bio.terra.pearl.core.model.export.ExportDestinationType; +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import bio.terra.pearl.core.model.export.ExportOptions; +import bio.terra.pearl.core.service.CrudService; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; +import bio.terra.pearl.core.service.search.EnrolleeSearchExpressionParser; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +@Service +public class ExportIntegrationService extends CrudService { + private final ExportIntegrationJobService exportIntegrationJobService; + private final ExportOptionsDao exportOptionsDao; + private final EnrolleeSearchExpressionParser enrolleeSearchExpressionParser; + private final Map externalExporters; + + + public ExportIntegrationService(ExportIntegrationDao dao, + ExportIntegrationJobService exportIntegrationJobService, + ExportOptionsDao exportOptionsDao, + EnrolleeSearchExpressionParser enrolleeSearchExpressionParser, + AirtableExporter airtableExporter) { + super(dao); + this.exportIntegrationJobService = exportIntegrationJobService; + this.exportOptionsDao = exportOptionsDao; + this.enrolleeSearchExpressionParser = enrolleeSearchExpressionParser; + this.externalExporters = Map.of(ExportDestinationType.AIRTABLE, airtableExporter); + } + + public ExportIntegration create(ExportIntegration integration) { + ExportOptions newOptions = integration.getExportOptions() != null ? integration.getExportOptions() : new ExportOptions(); + newOptions = exportOptionsDao.create(newOptions); + integration.setExportOptionsId(newOptions.getId()); + ExportIntegration newIntegration = super.create(integration); + newIntegration.setExportOptions(newOptions); + return newIntegration; + } + + public ExportIntegrationJob doExport(ExportIntegration integration, ResponsibleEntity operator) { + ExternalExporter exporter = externalExporters.get(integration.getDestinationType()); + return doExport(exporter, integration, operator); + } + + protected ExportIntegrationJob doExport(ExternalExporter exporter, ExportIntegration integration, ResponsibleEntity operator) { + if (integration.getExportOptions() == null) { + throw new IllegalArgumentException("Export options must be set to run an export integration"); + } + ExportOptionsWithExpression parsedOpts = enrolleeSearchExpressionParser.parseExportOptions(integration.getExportOptions()); + ExportIntegrationJob job = ExportIntegrationJob.builder() + .exportIntegrationId(integration.getId()) + .status(ExportIntegrationJob.Status.GENERATING) + .creatingAdminUserId(operator.getAdminUser() != null ? operator.getAdminUser().getId() : null) + .systemProcess(operator.getSystemProcess()) + .startedAt(Instant.now()) + .build(); + job = exportIntegrationJobService.create(job); + exporter.export(integration, parsedOpts, job); + return job; + } + + + + public List findByStudyEnvironmentId(UUID studyEnvId) { + return dao.findByStudyEnvironmentId(studyEnvId); + } + + public Optional findWithOptions(UUID id) { + Optional integrationOpt = dao.find(id); + integrationOpt.ifPresent(integration -> { + integration.setExportOptions(exportOptionsDao.find(integration.getExportOptionsId()).orElse(null)); + }); + return integrationOpt; + } + + public void deleteByStudyEnvironmentId(UUID studyEnvId) { + List integrations = dao.findByStudyEnvironmentId(studyEnvId); + List integrationIds = integrations.stream().map(ExportIntegration::getId).toList(); + exportIntegrationJobService.deleteByExportIntegrationIds(integrationIds); + dao.deleteByStudyEnvironmentId(studyEnvId); + exportOptionsDao.deleteAll(integrations.stream().map(ExportIntegration::getExportOptionsId).toList()); + + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExternalExporter.java b/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExternalExporter.java new file mode 100644 index 0000000000..faf02ff0ef --- /dev/null +++ b/core/src/main/java/bio/terra/pearl/core/service/export/integration/ExternalExporter.java @@ -0,0 +1,65 @@ +package bio.terra.pearl.core.service.export.integration; + +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import bio.terra.pearl.core.service.export.EnrolleeExportService; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; + +import java.io.ByteArrayOutputStream; +import java.util.function.Consumer; + +@Slf4j +public abstract class ExternalExporter { + + private final ExportIntegrationJobService exportIntegrationJobService; + + private final EnrolleeExportService enrolleeExportService; + + protected abstract void send(ExportIntegration integration, ByteArrayOutputStream baos, Consumer handleComplete, Consumer handleError); + + public ExternalExporter(ExportIntegrationJobService exportIntegrationJobService, EnrolleeExportService enrolleeExportService) { + this.exportIntegrationJobService = exportIntegrationJobService; + this.enrolleeExportService = enrolleeExportService; + } + + protected ByteArrayOutputStream createExportStream(ExportIntegration integration, ExportOptionsWithExpression parsedOpts) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + enrolleeExportService.export(parsedOpts, integration.getStudyEnvironmentId(), baos); + return baos; + } + + @Async + public void export(ExportIntegration integration, ExportOptionsWithExpression parsedOpts, ExportIntegrationJob job) { + try { + job.setStatus(ExportIntegrationJob.Status.GENERATING); + job = exportIntegrationJobService.update(job); + + ByteArrayOutputStream baos = createExportStream(integration, parsedOpts); + + job.setStatus(ExportIntegrationJob.Status.SENDING); + final ExportIntegrationJob updatedJob = exportIntegrationJobService.update(job); + + send(integration, baos, + (String msg) -> handleComplete(updatedJob, integration, msg), + (Exception e) -> handleError(updatedJob, integration, e)); + } catch (Exception e) { + handleError(job, integration, e); + } + } + + public void handleComplete(ExportIntegrationJob job, ExportIntegration integration, String message) { + log.info("Export job complete: integration id: %s, job id: %s, message: %s".formatted(integration.getId(), job.getId(), message)); + job.setStatus(ExportIntegrationJob.Status.COMPLETE); + job.setResult(message); + exportIntegrationJobService.update(job); + } + + public void handleError(ExportIntegrationJob job, ExportIntegration integration, Exception e) { + log.error("Export job failed: integration id: %s, job id: %s".formatted(integration.getId(), job.getId()), e); + job.setStatus(ExportIntegrationJob.Status.FAILED); + job.setResult(e.getMessage()); + exportIntegrationJobService.update(job); + } +} diff --git a/core/src/main/java/bio/terra/pearl/core/service/kit/pepper/LivePepperDSMClient.java b/core/src/main/java/bio/terra/pearl/core/service/kit/pepper/LivePepperDSMClient.java index 5309ade802..ebafc4f259 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/kit/pepper/LivePepperDSMClient.java +++ b/core/src/main/java/bio/terra/pearl/core/service/kit/pepper/LivePepperDSMClient.java @@ -59,7 +59,8 @@ public PepperKit sendKitRequest(String studyShortcode, StudyEnvironmentConfig st kitRequestBody = makeKitRequestBody(studyShortcode, studyEnvironmentConfig, enrollee, kitRequest, address); } - WebClient.RequestHeadersSpec> request = buildAuthedPostRequest("shipKit", kitRequestBody); PepperKitStatusResponse response = retrieveAndDeserializeResponse(request, PepperKitStatusResponse.class); + WebClient.RequestHeadersSpec> request = buildAuthedPostRequest("shipKit", kitRequestBody); + PepperKitStatusResponse response = retrieveAndDeserializeResponse(request, PepperKitStatusResponse.class); if (response.getKits().length != 1) { throw new PepperParseException("Expected a single result from shipKit by ID (%s), got %d".formatted( kitRequest.getId(), response.getKits().length), Arrays.toString(response.getKits()), response); diff --git a/core/src/main/java/bio/terra/pearl/core/service/search/EnrolleeSearchExpressionParser.java b/core/src/main/java/bio/terra/pearl/core/service/search/EnrolleeSearchExpressionParser.java index fdb643bbf3..05a76a319d 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/search/EnrolleeSearchExpressionParser.java +++ b/core/src/main/java/bio/terra/pearl/core/service/search/EnrolleeSearchExpressionParser.java @@ -6,6 +6,8 @@ import bio.terra.pearl.core.dao.participant.*; import bio.terra.pearl.core.dao.survey.AnswerDao; import bio.terra.pearl.core.dao.workflow.ParticipantTaskDao; +import bio.terra.pearl.core.model.export.ExportOptions; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; import bio.terra.pearl.core.service.rule.RuleParsingErrorListener; import bio.terra.pearl.core.service.rule.RuleParsingException; import bio.terra.pearl.core.service.search.expressions.*; @@ -19,6 +21,7 @@ import org.antlr.v4.runtime.misc.ParseCancellationException; import org.apache.commons.lang3.StringUtils; import org.jooq.Operator; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Component; import java.util.List; @@ -251,4 +254,13 @@ private String[] parseFields(String variable) { } throw new IllegalArgumentException("No field in variable"); } + + public ExportOptionsWithExpression parseExportOptions(ExportOptions exportOptions) { + ExportOptionsWithExpression exportOptionsWithExpression = new ExportOptionsWithExpression(); + BeanUtils.copyProperties(exportOptions, exportOptionsWithExpression); + if (!StringUtils.isBlank(exportOptions.getFilterString())) { + exportOptionsWithExpression.setFilterExpression(parseRule(exportOptions.getFilterString())); + } + return exportOptionsWithExpression; + } } diff --git a/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java b/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java index 0da4313497..d90fe07eee 100644 --- a/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java +++ b/core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentService.java @@ -11,10 +11,11 @@ import bio.terra.pearl.core.model.survey.StudyEnvironmentSurvey; import bio.terra.pearl.core.service.CascadeProperty; import bio.terra.pearl.core.service.CrudService; -import bio.terra.pearl.core.service.dataimport.ImportService; +import bio.terra.pearl.core.service.export.dataimport.ImportService; import bio.terra.pearl.core.service.datarepo.DataRepoJobService; import bio.terra.pearl.core.service.datarepo.DatasetService; import bio.terra.pearl.core.service.exception.NotFoundException; +import bio.terra.pearl.core.service.export.integration.ExportIntegrationService; import bio.terra.pearl.core.service.kit.StudyEnvironmentKitTypeService; import bio.terra.pearl.core.service.notification.TriggerService; import bio.terra.pearl.core.service.participant.EnrolleeRelationService; @@ -46,6 +47,7 @@ public class StudyEnvironmentService extends CrudService findByStudy(UUID studyId) { @@ -132,6 +139,7 @@ public void delete(UUID studyEnvironmentId, Set cascade) { withdrawnEnrolleeDao.deleteByStudyEnvironmentId(studyEnvironmentId); studyEnvironmentKitTypeService.deleteByStudyEnvironmentId(studyEnvironmentId, cascade); importService.deleteByStudyEnvId(studyEnvironmentId); + exportIntegrationService.deleteByStudyEnvironmentId(studyEnvironmentId); dao.delete(studyEnvironmentId); if (studyEnv.getStudyEnvironmentConfigId() != null) { studyEnvironmentConfigService.delete(studyEnv.getStudyEnvironmentConfigId()); diff --git a/core/src/main/resources/db/changelog/changesets/2024_09_20_export_integration.yaml b/core/src/main/resources/db/changelog/changesets/2024_09_20_export_integration.yaml new file mode 100644 index 0000000000..2968f56ea0 --- /dev/null +++ b/core/src/main/resources/db/changelog/changesets/2024_09_20_export_integration.yaml @@ -0,0 +1,50 @@ + +databaseChangeLog: + - changeSet: + id: "export_integration" + author: dbush + changes: + + - createTable: + tableName: export_options + columns: + - column: { name: id, type: uuid, defaultValueComputed: gen_random_uuid(), constraints: { nullable: false, primaryKey: true } } + - column: { name: created_at, type: datetime, constraints: { nullable: false } } + - column: { name: last_updated_at, type: datetime, constraints: { nullable: false } } + - column: { name: split_options_into_columns, type: boolean, defaultValueBoolean: false, constraints: { nullable: false } } + - column: { name: stable_ids_for_options, type: boolean, defaultValueBoolean: false, constraints: { nullable: false } } + - column: { name: only_include_most_recent, type: boolean, defaultValueBoolean: true, constraints: { nullable: false } } + - column: { name: filter_string, type: text } + - column: { name: file_format, type: text, constraints: { nullable: false } } + - column: { name: row_limit, type: int } + - column: { name: include_sub_headers, type: boolean, defaultValueBoolean: true, constraints: { nullable: false } } + - sql: + sql: "ALTER TABLE export_options ADD COLUMN exclude_modules text[];" + + - createTable: + tableName: export_integration + columns: + - column: { name: id, type: uuid, defaultValueComputed: gen_random_uuid(), constraints: { nullable: false, primaryKey: true } } + - column: { name: name, type: text, constraints: { nullable: false } } + - column: { name: created_at, type: datetime, constraints: { nullable: false } } + - column: { name: last_updated_at, type: datetime, constraints: { nullable: false } } + - column: { name: study_environment_id, type: uuid, constraints: { nullable: false, foreignKeyName: fk_export_int_study_environment_id, references: study_environment(id) } } + - column: { name: destination_url, type: text } + - column: { name: destination_type, type: text } + - column: { name: export_options_id, type: uuid, constraints: { nullable: false, foreignKeyName: fk_export_int_export_options_id, references: export_options(id) } } + - column: { name: enabled, type: boolean, defaultValueBoolean: true, constraints: { nullable: false } } + + - createTable: + tableName: export_integration_job + columns: + - column: { name: id, type: uuid, defaultValueComputed: gen_random_uuid(), constraints: { nullable: false, primaryKey: true } } + - column: { name: created_at, type: datetime, constraints: { nullable: false } } + - column: { name: last_updated_at, type: datetime, constraints: { nullable: false } } + - column: { name: export_integration_id, type: uuid, constraints: { nullable: false , foreignKeyName: fk_export_job_to_config_id, references: export_integration(id) } } + - column: { name: creating_admin_user_id, type: uuid, constraints: { foreignKeyName: fk_export_job_to_admin_user_id, references: admin_user(id) } } + - column: { name: system_process, type: text } + - column: { name: started_at, type: datetime } + - column: { name: completed_at, type: datetime } + - column: { name: status, type: text } + - column: { name: result, type: text } + diff --git a/core/src/main/resources/db/changelog/db.changelog-master.yaml b/core/src/main/resources/db/changelog/db.changelog-master.yaml index 59fa0459f2..4b9480e957 100644 --- a/core/src/main/resources/db/changelog/db.changelog-master.yaml +++ b/core/src/main/resources/db/changelog/db.changelog-master.yaml @@ -326,6 +326,9 @@ databaseChangeLog: - include: file: changesets/2024_09_18_survey_assign_rename.yaml relativeToChangelogFile: true + - include: + file: changesets/2024_09_20_export_integration.yaml + relativeToChangelogFile: true # README: it is a best practice to put each DDL statement in its own change set. DDL statements diff --git a/core/src/test/java/bio/terra/pearl/core/service/dataimport/ImportServiceTest.java b/core/src/test/java/bio/terra/pearl/core/service/dataimport/ImportServiceTest.java index 3293030426..92af9aba63 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/dataimport/ImportServiceTest.java +++ b/core/src/test/java/bio/terra/pearl/core/service/dataimport/ImportServiceTest.java @@ -10,7 +10,7 @@ import bio.terra.pearl.core.model.dataimport.ImportStatus; import bio.terra.pearl.core.model.dataimport.ImportType; import bio.terra.pearl.core.service.CascadeProperty; -import bio.terra.pearl.core.service.admin.AdminUserService; +import bio.terra.pearl.core.service.export.dataimport.ImportService; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.springframework.beans.factory.annotation.Autowired; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeExportServiceTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeExportServiceTests.java index 61ff766bdf..1ce1ad6016 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeExportServiceTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeExportServiceTests.java @@ -7,6 +7,7 @@ import bio.terra.pearl.core.factory.survey.SurveyFactory; import bio.terra.pearl.core.factory.survey.SurveyResponseFactory; import bio.terra.pearl.core.model.EnvironmentName; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.model.participant.*; import bio.terra.pearl.core.model.portal.PortalEnvironment; import bio.terra.pearl.core.model.study.StudyEnvironment; @@ -82,7 +83,7 @@ public void testExportNumberLimit(TestInfo testInfo) { Enrollee enrollee2 = enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); Enrollee enrollee3 = enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); - ExportOptions opts = ExportOptions.builder().limit(2).build(); + ExportOptionsWithExpression opts = ExportOptionsWithExpression.builder().rowLimit(2).build(); List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), opts); List exportModuleInfo = enrolleeExportService.generateModuleInfos(opts, studyEnv.getId(), exportData); @@ -103,13 +104,13 @@ public void testExportWithProxies(TestInfo testInfo) { EnrolleeFactory.EnrolleeAndProxy enrolleeWithProxy = enrolleeFactory.buildProxyAndGovernedEnrollee(testName, studyEnvBundle.getPortalEnv(), studyEnvBundle.getStudyEnv()); Enrollee regularEnrollee = enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); - List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptions()); + List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptionsWithExpression()); List exportModuleInfoWithProxies = enrolleeExportService.generateModuleInfos(ExportOptions .builder() - .filter(null) // no filter means proxies will be included + .filterString(null) // no filter means proxies will be included .onlyIncludeMostRecent(true) .fileFormat(ExportFileFormat.TSV) - .limit(null) + .rowLimit(null) .build(), studyEnv.getId(), exportData); @@ -129,9 +130,9 @@ public void testExportWithProxies(TestInfo testInfo) { List exportDataNoProxies = enrolleeExportService.loadEnrolleeExportData( studyEnv.getId(), - ExportOptions + ExportOptionsWithExpression .builder() - .filter(enrolleeSearchExpressionParser.parseRule("{enrollee.subject} = true")) + .filterExpression(enrolleeSearchExpressionParser.parseRule("{enrollee.subject} = true")) .build()); List exportModuleInfoNoProxies = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportDataNoProxies); List> exportMapsNoProxies = enrolleeExportService.generateExportMaps(exportDataNoProxies, exportModuleInfoNoProxies); @@ -155,7 +156,7 @@ public void testExportChecksStudyEnvConfigProxy(TestInfo testInfo) { enrolleeFactory.buildProxyAndGovernedEnrollee(testName, portalEnv, studyEnv); - List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptions()); + List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptionsWithExpression()); List exportModuleInfo = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportData); List> exportMaps = enrolleeExportService.generateExportMaps(exportData, exportModuleInfo); @@ -171,7 +172,7 @@ public void testExportChecksStudyEnvConfigProxy(TestInfo testInfo) { .acceptingProxyEnrollment(true) .build()); - exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptions()); + exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptionsWithExpression()); exportModuleInfo = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportData); exportMaps = enrolleeExportService.generateExportMaps(exportData, exportModuleInfo); @@ -215,7 +216,7 @@ public void testExportChecksStudyEnvConfigFamily(TestInfo testInfo) { getAuditInfo(testInfo) ); - List data = enrolleeExportService.loadEnrolleeExportData(studyEnvId, new ExportOptions()); + List data = enrolleeExportService.loadEnrolleeExportData(studyEnvId, new ExportOptionsWithExpression()); List exportModuleInfo = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnvId, data); List> exportMaps = enrolleeExportService.generateExportMaps(data, exportModuleInfo); @@ -232,7 +233,7 @@ public void testExportChecksStudyEnvConfigFamily(TestInfo testInfo) { .enableFamilyLinkage(true) .build()); - data = enrolleeExportService.loadEnrolleeExportData(studyEnvId, new ExportOptions()); + data = enrolleeExportService.loadEnrolleeExportData(studyEnvId, new ExportOptionsWithExpression()); exportModuleInfo = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnvId, data); exportMaps = enrolleeExportService.generateExportMaps(data, exportModuleInfo); @@ -495,7 +496,7 @@ public void testDynamicPanelExport(TestInfo testInfo) { ); - List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptions()); + List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptionsWithExpression()); List moduleFormatters = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportData); List> exportMaps = enrolleeExportService.generateExportMaps(exportData, moduleFormatters); @@ -538,7 +539,7 @@ public void testDynamicPanelExportNoResponses(TestInfo testInfo) { surveyFactory.attachToEnv(survey, studyEnv.getId(), true); - List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptions()); + List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptionsWithExpression()); List moduleFormatters = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportData); List> exportMaps = enrolleeExportService.generateExportMaps(exportData, moduleFormatters); @@ -597,7 +598,7 @@ public void testMultiVersionDynamicPanelExport(TestInfo testInfo) { ) ); - List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptions()); + List exportData = enrolleeExportService.loadEnrolleeExportData(studyEnv.getId(), new ExportOptionsWithExpression()); List moduleFormatters = enrolleeExportService.generateModuleInfos(new ExportOptions(), studyEnv.getId(), exportData); List> exportMaps = enrolleeExportService.generateExportMaps(exportData, moduleFormatters); @@ -618,7 +619,7 @@ public void testExportSubheadersOption(TestInfo testInfo) throws Exception { StudyEnvironment studyEnv = studyEnvironmentFactory.buildPersisted(testName); Enrollee enrollee1 = enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); - ExportOptions opts = ExportOptions.builder() + ExportOptionsWithExpression opts = ExportOptionsWithExpression.builder() .fileFormat(ExportFileFormat.CSV) .includeSubHeaders(false).build(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -629,7 +630,7 @@ public void testExportSubheadersOption(TestInfo testInfo) throws Exception { assertThat(export, not(containsString(",Created at"))); // now check it includes subheaders if asked - opts = ExportOptions.builder() + opts = ExportOptionsWithExpression.builder() .includeSubHeaders(true) .fileFormat(ExportFileFormat.CSV).build(); baos = new ByteArrayOutputStream(); diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java index f8f5e9c594..5f81d1f60e 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/EnrolleeImportServiceTests.java @@ -11,6 +11,7 @@ import bio.terra.pearl.core.model.dataimport.Import; import bio.terra.pearl.core.model.dataimport.ImportItem; import bio.terra.pearl.core.model.dataimport.ImportStatus; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.model.kit.KitRequestStatus; import bio.terra.pearl.core.model.kit.KitType; import bio.terra.pearl.core.model.participant.Enrollee; @@ -20,9 +21,9 @@ import bio.terra.pearl.core.model.workflow.ParticipantTask; import bio.terra.pearl.core.model.workflow.TaskStatus; import bio.terra.pearl.core.service.admin.AdminUserService; -import bio.terra.pearl.core.service.dataimport.ImportFileFormat; -import bio.terra.pearl.core.service.dataimport.ImportItemService; -import bio.terra.pearl.core.service.dataimport.ImportService; +import bio.terra.pearl.core.service.export.dataimport.ImportFileFormat; +import bio.terra.pearl.core.service.export.dataimport.ImportItemService; +import bio.terra.pearl.core.service.export.dataimport.ImportService; import bio.terra.pearl.core.service.kit.KitRequestDto; import bio.terra.pearl.core.service.kit.KitRequestService; import bio.terra.pearl.core.service.participant.EnrolleeService; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/TsvExporterTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/TsvExporterTests.java index 81f1332156..aaf73d3901 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/TsvExporterTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/TsvExporterTests.java @@ -1,6 +1,7 @@ package bio.terra.pearl.core.service.export; import bio.terra.pearl.core.BaseSpringBootTest; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.model.participant.Enrollee; import bio.terra.pearl.core.model.survey.Survey; import bio.terra.pearl.core.model.survey.SurveyQuestionDefinition; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/EnrolleeFormatterTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/EnrolleeFormatterTests.java index 32b5519f07..32a975267d 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/EnrolleeFormatterTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/EnrolleeFormatterTests.java @@ -2,7 +2,7 @@ import bio.terra.pearl.core.model.participant.Enrollee; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.module.EnrolleeFormatter; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ParticipantUserFormatterTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ParticipantUserFormatterTests.java index 588be9227a..020e45e5f7 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ParticipantUserFormatterTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ParticipantUserFormatterTests.java @@ -2,7 +2,7 @@ import bio.terra.pearl.core.model.participant.ParticipantUser; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.module.ParticipantUserFormatter; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ProfileFormatterTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ProfileFormatterTests.java index 18b5ed4b56..5eb8d2c721 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ProfileFormatterTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/ProfileFormatterTests.java @@ -3,7 +3,7 @@ import bio.terra.pearl.core.model.address.MailingAddress; import bio.terra.pearl.core.model.participant.Profile; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.module.ProfileFormatter; import org.junit.jupiter.api.Test; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatterTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatterTests.java index 7a65e5caae..f1556e759e 100644 --- a/core/src/test/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatterTests.java +++ b/core/src/test/java/bio/terra/pearl/core/service/export/formatters/module/SurveyFormatterTests.java @@ -6,7 +6,7 @@ import bio.terra.pearl.core.model.survey.SurveyQuestionDefinition; import bio.terra.pearl.core.model.survey.SurveyResponse; import bio.terra.pearl.core.service.export.EnrolleeExportData; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; import bio.terra.pearl.core.service.export.formatters.item.AnswerItemFormatter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/core/src/test/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationServiceTests.java b/core/src/test/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationServiceTests.java new file mode 100644 index 0000000000..5db0cf4919 --- /dev/null +++ b/core/src/test/java/bio/terra/pearl/core/service/export/integration/ExportIntegrationServiceTests.java @@ -0,0 +1,119 @@ +package bio.terra.pearl.core.service.export.integration; + +import bio.terra.pearl.core.BaseSpringBootTest; +import bio.terra.pearl.core.factory.StudyEnvironmentFactory; +import bio.terra.pearl.core.factory.participant.EnrolleeFactory; +import bio.terra.pearl.core.model.audit.ResponsibleEntity; +import bio.terra.pearl.core.model.export.ExportDestinationType; +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.model.export.ExportIntegrationJob; +import bio.terra.pearl.core.model.participant.Profile; +import bio.terra.pearl.core.model.study.StudyEnvironment; +import bio.terra.pearl.core.service.export.EnrolleeExportService; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; + +import java.io.ByteArrayOutputStream; +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +public class ExportIntegrationServiceTests extends BaseSpringBootTest { + @Autowired + private StudyEnvironmentFactory studyEnvironmentFactory; + @Autowired + private EnrolleeFactory enrolleeFactory; + @Autowired + private ExportIntegrationService exportIntegrationService; + @Autowired + private ExportIntegrationJobService exportIntegrationJobService; + @Autowired + private EnrolleeExportService enrolleeExportService; + + + @Test + @Transactional + public void testExternalExportJobCreation(TestInfo testInfo) throws InterruptedException { + String testName = getTestName(testInfo); + StudyEnvironment studyEnv = studyEnvironmentFactory.buildPersisted(testName); + enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); + enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); + + ExportOptionsWithExpression opts = ExportOptionsWithExpression.builder().rowLimit(2).build(); + + ExportIntegration exportIntegration = exportIntegrationService.create(ExportIntegration.builder() + .name(getTestName(testInfo)) + .studyEnvironmentId(studyEnv.getId()) + .enabled(true) + .exportOptions(opts) + .destinationType(ExportDestinationType.AIRTABLE) + .destinationUrl("badURL") + .build()); + ExportIntegration loadedIntegration = exportIntegrationService.findWithOptions(exportIntegration.getId()).get(); + + assertThat(exportIntegrationJobService.findByStudyEnvironment(studyEnv.getId()), hasSize(0)); + exportIntegrationService.doExport(new MockExporter(), loadedIntegration, new ResponsibleEntity("testExternalExportJobCreation")); + Thread.sleep(200); // quick-and-dirty way to wait until the async op completes + List jobs = exportIntegrationJobService.findByStudyEnvironment(studyEnv.getId()); + assertThat(jobs, hasSize(1)); + assertThat(jobs.get(0).getExportIntegrationId(), equalTo(exportIntegration.getId())); + assertThat(jobs.get(0).getStatus(), equalTo(ExportIntegrationJob.Status.COMPLETE)); + assertThat(jobs.get(0).getResult(), containsString("mock done")); + } + + @Test + @Transactional + public void testExternalExportJobError(TestInfo testInfo) throws InterruptedException { + String testName = getTestName(testInfo); + StudyEnvironment studyEnv = studyEnvironmentFactory.buildPersisted(testName); + enrolleeFactory.buildPersisted(testName, studyEnv, new Profile()); + + ExportOptionsWithExpression opts = ExportOptionsWithExpression.builder().rowLimit(2).build(); + + ExportIntegration exportIntegration = exportIntegrationService.create(ExportIntegration.builder() + .name(getTestName(testInfo)) + .studyEnvironmentId(studyEnv.getId()) + .enabled(true) + .exportOptions(opts) + .destinationType(ExportDestinationType.AIRTABLE) + .destinationUrl("badURL") + .build()); + ExportIntegration loadedIntegration = exportIntegrationService.findWithOptions(exportIntegration.getId()).get(); + + assertThat(exportIntegrationJobService.findByStudyEnvironment(studyEnv.getId()), hasSize(0)); + exportIntegrationService.doExport(new MockErrorExporter(), loadedIntegration, new ResponsibleEntity("testExternalExportJobError")); + Thread.sleep(200); // quick-and-dirty way to wait until the async op completes + List jobs = exportIntegrationJobService.findByStudyEnvironment(studyEnv.getId()); + assertThat(jobs, hasSize(1)); + assertThat(jobs.get(0).getExportIntegrationId(), equalTo(exportIntegration.getId())); + assertThat(jobs.get(0).getStatus(), equalTo(ExportIntegrationJob.Status.FAILED)); + assertThat(jobs.get(0).getResult(), containsString("mock error")); + } + + protected class MockExporter extends ExternalExporter { + public MockExporter() { + super(exportIntegrationJobService, enrolleeExportService); + } + @Override + public void send(ExportIntegration integration, ByteArrayOutputStream baos, Consumer handleComplete, Consumer handleError) { + // confirm the output stream can be stringified, otherwise do nothing + baos.toString(); + handleComplete.accept("mock done"); + } + } + + protected class MockErrorExporter extends ExternalExporter { + public MockErrorExporter() { + super(exportIntegrationJobService, enrolleeExportService); + } + @Override + public void send(ExportIntegration integration, ByteArrayOutputStream baos, Consumer handleComplete, Consumer handleError) { + handleError.accept(new RuntimeException("mock error")); + } + } +} diff --git a/populate/src/main/java/bio/terra/pearl/populate/service/ExportIntegrationPopulator.java b/populate/src/main/java/bio/terra/pearl/populate/service/ExportIntegrationPopulator.java new file mode 100644 index 0000000000..5a21ddb1ac --- /dev/null +++ b/populate/src/main/java/bio/terra/pearl/populate/service/ExportIntegrationPopulator.java @@ -0,0 +1,59 @@ +package bio.terra.pearl.populate.service; + +import bio.terra.pearl.core.model.export.ExportIntegration; +import bio.terra.pearl.core.model.study.StudyEnvironment; +import bio.terra.pearl.core.service.CascadeProperty; +import bio.terra.pearl.core.service.export.integration.ExportIntegrationService; +import bio.terra.pearl.core.service.search.EnrolleeSearchExpressionParser; +import bio.terra.pearl.core.service.study.StudyEnvironmentService; +import bio.terra.pearl.populate.service.contexts.StudyPopulateContext; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.Optional; + +@Service +public class ExportIntegrationPopulator extends BasePopulator { + + private final ExportIntegrationService exportIntegrationService; + private final StudyEnvironmentService studyEnvironmentService; + + public ExportIntegrationPopulator(ExportIntegrationService exportIntegrationService, StudyEnvironmentService studyEnvironmentService, EnrolleeSearchExpressionParser enrolleeSearchExpressionParser) { + this.exportIntegrationService = exportIntegrationService; + this.studyEnvironmentService = studyEnvironmentService; + } + + @Override + protected void preProcessDto(ExportIntegration popDto, StudyPopulateContext context) throws IOException { + StudyEnvironment studyEnvironment = studyEnvironmentService.findByStudy(context.getStudyShortcode(), context.getEnvironmentName()) + .orElseThrow(() -> new IllegalArgumentException("Study environment not found")); + popDto.setStudyEnvironmentId(studyEnvironment.getId()); + } + + @Override + protected Class getDtoClazz() { + return ExportIntegration.class; + } + + @Override + public Optional findFromDto(ExportIntegration popDto, StudyPopulateContext context) { + // export integrations are not uniquely identified + return Optional.empty(); + } + + @Override + public ExportIntegration overwriteExisting(ExportIntegration existingObj, ExportIntegration popDto, StudyPopulateContext context) throws IOException { + exportIntegrationService.delete(existingObj.getId(), CascadeProperty.EMPTY_SET); + return createNew(popDto, context, true); + } + + @Override + public ExportIntegration createPreserveExisting(ExportIntegration existingObj, ExportIntegration popDto, StudyPopulateContext context) throws IOException { + return createNew(popDto, context, true); + } + + @Override + public ExportIntegration createNew(ExportIntegration popDto, StudyPopulateContext context, boolean overwrite) throws IOException { + return exportIntegrationService.create(popDto); + } +} diff --git a/populate/src/main/java/bio/terra/pearl/populate/service/StudyPopulator.java b/populate/src/main/java/bio/terra/pearl/populate/service/StudyPopulator.java index 26234463e0..459106fc2b 100644 --- a/populate/src/main/java/bio/terra/pearl/populate/service/StudyPopulator.java +++ b/populate/src/main/java/bio/terra/pearl/populate/service/StudyPopulator.java @@ -4,6 +4,7 @@ import bio.terra.pearl.core.dao.kit.StudyEnvironmentKitTypeDao; import bio.terra.pearl.core.dao.survey.PreEnrollmentResponseDao; import bio.terra.pearl.core.model.EnvironmentName; +import bio.terra.pearl.core.model.export.ExportIntegration; import bio.terra.pearl.core.model.kit.KitType; import bio.terra.pearl.core.model.kit.StudyEnvironmentKitType; import bio.terra.pearl.core.model.notification.Trigger; @@ -48,6 +49,7 @@ public class StudyPopulator extends BasePopulator sandboxEnrollees) } private void checkExportContent(UUID sandboxEnvironmentId) throws Exception { - ExportOptions options = ExportOptions + ExportOptionsWithExpression options = ExportOptionsWithExpression .builder() .onlyIncludeMostRecent(true) .fileFormat(ExportFileFormat.TSV) - .filter(enrolleeSearchExpressionParser.parseRule("{enrollee.subject} = true")) - .limit(null) + .filterExpression(enrolleeSearchExpressionParser.parseRule("{enrollee.subject} = true")) + .rowLimit(null) .build(); List enrolleeExportData = enrolleeExportService.loadEnrolleeExportData(sandboxEnvironmentId, options); diff --git a/populate/src/test/java/bio/terra/pearl/populate/PopulateOurhealthTest.java b/populate/src/test/java/bio/terra/pearl/populate/PopulateOurhealthTest.java index 5c086b8240..e97157c254 100644 --- a/populate/src/test/java/bio/terra/pearl/populate/PopulateOurhealthTest.java +++ b/populate/src/test/java/bio/terra/pearl/populate/PopulateOurhealthTest.java @@ -15,7 +15,8 @@ import bio.terra.pearl.core.model.survey.SurveyResponse; import bio.terra.pearl.core.service.export.EnrolleeExportData; import bio.terra.pearl.core.service.export.ExportFileFormat; -import bio.terra.pearl.core.service.export.ExportOptions; +import bio.terra.pearl.core.model.export.ExportOptions; +import bio.terra.pearl.core.service.export.ExportOptionsWithExpression; import bio.terra.pearl.core.service.export.formatters.module.ModuleFormatter; import bio.terra.pearl.core.service.workflow.ParticipantTaskService; import bio.terra.pearl.populate.service.contexts.FilePopulateContext; @@ -113,13 +114,13 @@ private void checkAdminTasks(UUID sandboxStudyEnvId) { private void checkExportContent(UUID sandboxEnvironmentId) { // test the analysis-friendly export as that is the most important for data integrity, and the least visible via admin tool - ExportOptions options = ExportOptions + ExportOptionsWithExpression options = ExportOptionsWithExpression .builder() .splitOptionsIntoColumns(true) .stableIdsForOptions(true) .onlyIncludeMostRecent(true) .fileFormat(ExportFileFormat.TSV) - .limit(null) + .rowLimit(null) .build(); List enrolleeExportData = enrolleeExportService.loadEnrolleeExportData(sandboxEnvironmentId, options); List moduleInfos = enrolleeExportService.generateModuleInfos(options, sandboxEnvironmentId, enrolleeExportData); @@ -141,7 +142,7 @@ private void checkDataDictionary(UUID portalId, UUID sandboxEnvironmentId) throw .builder() .onlyIncludeMostRecent(true) .fileFormat(ExportFileFormat.TSV) - .limit(null) + .rowLimit(null) .build(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); dictionaryExportService.exportDictionary(options, portalId, sandboxEnvironmentId, baos); diff --git a/scripts/render_environment_vars.sh b/scripts/render_environment_vars.sh old mode 100644 new mode 100755 index 8e4e52a2f3..4bff809312 --- a/scripts/render_environment_vars.sh +++ b/scripts/render_environment_vars.sh @@ -21,6 +21,7 @@ ADMIN_API_ENV_VARS=( "TDR_EXPORT_STORAGE_ACCOUNT_KEY:vault:vault read -field=storage_account_key secret/dsp/ddp/d2p/dev/tdr-export-storage-account" "TDR_EXPORT_STORAGE_CONTAINER_NAME:static:juniper-dataset-ingest" "DEPLOYMENT_ZONE:static:local" + "AIRTABLE_AUTH_TOKEN:vault:vault read -field=authToken secret/dsp/ddp/d2p/dev/airtable" ) PARTICIPANT_API_ENV_VARS=( diff --git a/ui-admin/src/api/api.tsx b/ui-admin/src/api/api.tsx index b74366e33e..24d280c578 100644 --- a/ui-admin/src/api/api.tsx +++ b/ui-admin/src/api/api.tsx @@ -268,8 +268,8 @@ export type ExportOptions = { onlyIncludeMostRecent?: boolean, includeSubheaders?: boolean, excludeModules?: string[], - filter?: string, - limit?: number + filterString?: string, + rowLimit?: number } export type ExportData = { @@ -318,6 +318,7 @@ export type KitRequestListResponse = { export type InternalConfig = { pepperDsmConfig: Record addrValidationConfig: Record + airtable: Record } export type ParticipantTaskUpdateDto = { @@ -365,6 +366,28 @@ export type WithdrawnEnrollee = { userData: string } +export type ExportIntegration = { + id: string, + name: string, + createdAt: number, + lastUpdatedAt: number, + destinationType: string, + enabled: boolean, + exportOptions: ExportOptions, + destinationUrl: string +} + +export type ExportIntegrationJob = { + id: string, + status: string, + exportIntegrationId: string, + startedAt: number, + completedAt?: number, + result: string, + creatingAdminUserId?: string, + systemProcess?: string +} + let bearerToken: string | null = null export const API_ROOT = '/api' @@ -1094,6 +1117,44 @@ export default { return fetch(url, this.getGetInit()) }, + async fetchExportIntegrations(studyEnvParams: StudyEnvParams): Promise { + const url = `${baseStudyEnvUrlFromParams(studyEnvParams)}/exportIntegrations` + const response = await fetch(url, this.getGetInit()) + return await this.processJsonResponse(response) + }, + + async fetchExportIntegration(studyEnvParams: StudyEnvParams, id: string): Promise { + const url = `${baseStudyEnvUrlFromParams(studyEnvParams)}/exportIntegrations/${id}` + const response = await fetch(url, this.getGetInit()) + return await this.processJsonResponse(response) + }, + + async createExportIntegration(studyEnvParams: StudyEnvParams, integration: ExportIntegration): + Promise { + const url = `${baseStudyEnvUrlFromParams(studyEnvParams)}/exportIntegrations` + const response = await fetch(url, { + method: 'POST', + headers: this.getInitHeaders(), + body: JSON.stringify(integration) + }) + return await this.processJsonResponse(response) + }, + + async runExportIntegration(studyEnvParams: StudyEnvParams, id: string): Promise { + const url = `${baseStudyEnvUrlFromParams(studyEnvParams)}/exportIntegrations/${id}/run` + const response = await fetch(url, { + method: 'POST', + headers: this.getInitHeaders() + }) + return await this.processJsonResponse(response) + }, + + async fetchExportIntegrationJobs(studyEnvParams: StudyEnvParams): Promise { + const url = `${baseStudyEnvUrlFromParams(studyEnvParams)}/exportIntegrationJobs` + const response = await fetch(url, this.getGetInit()) + return await this.processJsonResponse(response) + }, + async findTrigger(portalShortcode: string, studyShortcode: string, envName: string, id: string): Promise { const url = `${baseStudyEnvUrl(portalShortcode, studyShortcode, envName)}/triggers/${id}` diff --git a/ui-admin/src/integration/AirtableIntegrationDashboard.tsx b/ui-admin/src/integration/AirtableIntegrationDashboard.tsx new file mode 100644 index 0000000000..d20b1819b7 --- /dev/null +++ b/ui-admin/src/integration/AirtableIntegrationDashboard.tsx @@ -0,0 +1,26 @@ +import React, { useState } from 'react' +import { useLoadingEffect } from 'api/api-utils' +import Api, { InternalConfig } from 'api/api' +import LoadingSpinner from 'util/LoadingSpinner' + + +/** shows basic config for address validation service */ +export default function AirtableIntegrationDashboard() { + const [config, setConfig] = useState() + + const { isLoading } = useLoadingEffect(async () => { + const response = await Api.fetchInternalConfig() + setConfig(response) + }) + + return
+

Airtable integration

+
+ {!isLoading &&
+
apiKey
+
{config?.airtable.authToken}
+
} + {isLoading && } +
+
+} diff --git a/ui-admin/src/integration/IntegrationDashboard.tsx b/ui-admin/src/integration/IntegrationDashboard.tsx index b61a125694..c807b2f331 100644 --- a/ui-admin/src/integration/IntegrationDashboard.tsx +++ b/ui-admin/src/integration/IntegrationDashboard.tsx @@ -4,6 +4,7 @@ import KitIntegrationDashboard from './KitIntegrationDashboard' import AddressValidationIntegrationDashboard from './AddressValidationIntegrationDashboard' import { navDivStyle, navLinkStyleFunc, navListItemStyle } from 'util/subNavStyles' import { renderPageHeader } from 'util/pageUtils' +import AirtableIntegrationDashboard from './AirtableIntegrationDashboard' /** shows links to the populate control panels, and handles the routing for them */ export default function IntegrationDashboard() { @@ -15,12 +16,14 @@ export default function IntegrationDashboard() {
  • Kits
  • Address Validation
    • +
  • Airtable
    • }/> }/> + }/>
    diff --git a/ui-admin/src/navbar/StudySidebar.tsx b/ui-admin/src/navbar/StudySidebar.tsx index 7a5b01a623..61f5123b63 100644 --- a/ui-admin/src/navbar/StudySidebar.tsx +++ b/ui-admin/src/navbar/StudySidebar.tsx @@ -6,7 +6,7 @@ import React from 'react' import { adminTasksPath, studyEnvDataBrowserPath, - studyEnvDatasetListViewPath, + studyEnvDatasetListViewPath, studyEnvExportIntegrationsPath, studyEnvFormsPath, studyEnvImportPath, studyEnvMailingListPath, @@ -36,6 +36,11 @@ export const StudySidebar = ({ study, portalList, portalShortcode }: return isActive ? { background: 'rgba(255, 255, 255, 0.3)' } : {} } + const studyParams = { + portalShortcode, + studyShortcode: study.shortcode + } + return
    @@ -70,6 +75,11 @@ export const StudySidebar = ({ study, portalList, portalShortcode }: Data Export + { portalId && userHasPermission(user.user, portalId, 'export_integration') &&
  • + Export Integrations +
    • + } { portalId && userHasPermission(user.user, portalId, 'tdr_export') &&
  • Terra Data Repo diff --git a/ui-admin/src/study/CreateNewCohortModal.tsx b/ui-admin/src/study/CreateNewCohortModal.tsx index c815b01b97..86a45ec6d2 100644 --- a/ui-admin/src/study/CreateNewCohortModal.tsx +++ b/ui-admin/src/study/CreateNewCohortModal.tsx @@ -29,7 +29,7 @@ export default function CreateNewCohortModal({ onDismiss }: {onDismiss: () => vo const response = await Api.exportEnrollees( selectedPortal.shortcode, selectedStudy.study.shortcode, - 'live', { fileFormat: 'JSON', limit: 0 }) + 'live', { fileFormat: 'JSON', rowLimit: 0 }) const result = await response.json() setParticipantFields(result) }, [selectedStudy], 'Failed to load cohort criteria options') diff --git a/ui-admin/src/study/StudyEnvironmentRouter.tsx b/ui-admin/src/study/StudyEnvironmentRouter.tsx index f225455d4d..c86f3f1148 100644 --- a/ui-admin/src/study/StudyEnvironmentRouter.tsx +++ b/ui-admin/src/study/StudyEnvironmentRouter.tsx @@ -26,10 +26,10 @@ import StudyContent from './StudyContent' import KitsRouter from './kits/KitsRouter' import ParticipantsRouter from './participants/ParticipantsRouter' import QuestionScratchbox from './surveys/editor/QuestionScratchbox' -import ExportDataBrowser from './participants/export/ExportDataBrowser' +import ExportDataBrowser from './export/ExportDataBrowser' import StudyEnvMetricsView from './metrics/StudyEnvMetricsView' -import DatasetDashboard from './participants/datarepo/DatasetDashboard' -import DatasetList from './participants/datarepo/DatasetList' +import DatasetDashboard from './export/datarepo/DatasetDashboard' +import DatasetList from './export/datarepo/DatasetList' import Select from 'react-select' import MailingListView from '../portal/MailingListView' import { ENVIRONMENT_ICON_MAP } from './publishing/PortalPublishingView' @@ -50,6 +50,9 @@ import DataImportList from '../portal/DataImportList' import FamilyRouter from './families/FamilyRouter' import { KitScanner } from './kits/kitcollection/KitScanner' import { LoadedSettingsView } from 'study/settings/SettingsView' +import ExportIntegrationList from './export/integrations/ExportIntegrationList' +import ExportIntegrationView from './export/integrations/ExportIntegrationView' +import ExportIntegrationJobList from './export/integrations/ExportIntegrationJobList' export type StudyEnvContextT = { study: Study, currentEnv: StudyEnvironment, currentEnvPath: string, portal: Portal } @@ -127,6 +130,11 @@ function StudyEnvironmentRouter({ study }: { study: Study }) { studyEnvContext={studyEnvContext} portalContext={portalContext}/>} /> + }/> + }/> + }/> + }/> }/> { + return `${baseStudyEnvPath(studyEnvParams)}/export/integrations` +} + +/** path to the export integration configs */ +export const studyEnvExportIntegrationPath = (studyEnvParams: StudyEnvParams, id: string) => { + return `${studyEnvExportIntegrationsPath(studyEnvParams)}/${id}` +} + +export const studyEnvExportIntegrationJobsPath = (studyEnvParams: StudyEnvParams) => { + return `${studyEnvExportIntegrationsPath(studyEnvParams)}/jobs` +} + + /** helper function for metrics route */ export const studyEnvMetricsPath = (portalShortcode: string, studyShortcode: string, envName: string) => { return `${studyEnvPath(portalShortcode, studyShortcode, envName)}/metrics` diff --git a/ui-admin/src/study/participants/export/ExportDataBrowser.tsx b/ui-admin/src/study/export/ExportDataBrowser.tsx similarity index 82% rename from ui-admin/src/study/participants/export/ExportDataBrowser.tsx rename to ui-admin/src/study/export/ExportDataBrowser.tsx index 4255ff7356..e95922b2ce 100644 --- a/ui-admin/src/study/participants/export/ExportDataBrowser.tsx +++ b/ui-admin/src/study/export/ExportDataBrowser.tsx @@ -2,9 +2,9 @@ import React, { useMemo, useState } from 'react' -import { StudyEnvContextT } from 'study/StudyEnvironmentRouter' -import Api, { ExportData } from 'api/api' -import LoadingSpinner from 'util/LoadingSpinner' +import { StudyEnvContextT } from '../StudyEnvironmentRouter' +import Api, { ExportData } from '../../api/api' +import LoadingSpinner from '../../util/LoadingSpinner' import { CellContext, ColumnDef, @@ -14,19 +14,19 @@ import { useReactTable, VisibilityState } from '@tanstack/react-table' -import { basicTableLayout } from 'util/tableUtils' -import ExportDataControl from './ExportDataControl' +import { basicTableLayout } from '../../util/tableUtils' +import ExportDataModal from './ExportDataModal' import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' import { faDownload } from '@fortawesome/free-solid-svg-icons' -import { useLoadingEffect } from 'api/api-utils' -import { Button } from 'components/forms/Button' +import { useLoadingEffect } from '../../api/api-utils' +import { Button } from '../../components/forms/Button' import { renderPageHeader, renderTruncatedText -} from 'util/pageUtils' -import { failureNotification } from 'util/notifications' +} from '../../util/pageUtils' +import { failureNotification } from '../../util/notifications' import { Store } from 'react-notifications-component' -import { buildFilter } from 'util/exportUtils' +import { buildFilter } from '../../util/exportUtils' const ExportDataBrowser = ({ studyEnvContext }: {studyEnvContext: StudyEnvContextT}) => { const [data, setData] = useState(null) @@ -80,7 +80,7 @@ const ExportDataBrowser = ({ studyEnvContext }: {studyEnvContext: StudyEnvContex studyEnvContext.portal.shortcode, studyEnvContext.study.shortcode, studyEnvContext.currentEnv.environmentName, { - fileFormat: 'JSON', limit: 10, filter: buildFilter() + fileFormat: 'JSON', rowLimit: 10, filterString: buildFilter() }) const result = await response.json() if (!response.ok) { @@ -106,7 +106,7 @@ const ExportDataBrowser = ({ studyEnvContext }: {studyEnvContext: StudyEnvContex
    • - + {!isLoading && basicTableLayout(table)} diff --git a/ui-admin/src/study/participants/export/ExportDataControl.test.tsx b/ui-admin/src/study/export/ExportDataControl.test.tsx similarity index 73% rename from ui-admin/src/study/participants/export/ExportDataControl.test.tsx rename to ui-admin/src/study/export/ExportDataControl.test.tsx index 50db460365..186880954b 100644 --- a/ui-admin/src/study/participants/export/ExportDataControl.test.tsx +++ b/ui-admin/src/study/export/ExportDataControl.test.tsx @@ -1,15 +1,15 @@ import React from 'react' -import { mockStudyEnvContext } from 'test-utils/mocking-utils' +import { mockStudyEnvContext } from '../../test-utils/mocking-utils' import { render, screen, waitFor } from '@testing-library/react' -import ExportDataControl from './ExportDataControl' +import ExportDataModal from './ExportDataModal' import { userEvent } from '@testing-library/user-event' import { setupRouterTest } from '@juniper/ui-core' test('renders the file types', async () => { const { RoutedComponent } = setupRouterTest( // eslint-disable-next-line @typescript-eslint/no-empty-function - {}}/>) + {}}/>) render(RoutedComponent) expect(screen.getByText('Tab-delimited (.tsv)')).toBeInTheDocument() expect(screen.getByText('Excel (.xlsx)')).toBeInTheDocument() @@ -18,7 +18,7 @@ test('renders the file types', async () => { test('help page loads', async () => { const { RoutedComponent } = setupRouterTest( // eslint-disable-next-line @typescript-eslint/no-empty-function - {}}/>) + {}}/>) render(RoutedComponent) userEvent.click(screen.getByText('help page')) waitFor(() => expect(screen.getByText('Participant List Export Info')).toBeInTheDocument()) diff --git a/ui-admin/src/study/export/ExportDataModal.tsx b/ui-admin/src/study/export/ExportDataModal.tsx new file mode 100644 index 0000000000..fc8019e71b --- /dev/null +++ b/ui-admin/src/study/export/ExportDataModal.tsx @@ -0,0 +1,233 @@ +import React, { useState } from 'react' +import { StudyEnvContextT } from '../StudyEnvironmentRouter' +import Modal from 'react-bootstrap/Modal' +import LoadingSpinner from 'util/LoadingSpinner' +import Api, { ExportOptions } from 'api/api' +import { currentIsoDate } from '@juniper/ui-core' +import { Link } from 'react-router-dom' +import { saveBlobAsDownload } from 'util/downloadUtils' +import { doApiLoad } from 'api/api-utils' +import { buildFilter } from 'util/exportUtils' +import { Button } from 'components/forms/Button' +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' +import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons' +import Select from 'react-select' +import { useReactMultiSelect } from 'util/react-select-utils' +import InfoPopup from '../../components/forms/InfoPopup' + +const FILE_FORMATS = [{ + label: 'Tab-delimited (.tsv)', + value: 'TSV', + fileSuffix: 'tsv' +}, { + label: 'Comma-delimited (.csv)', + value: 'CSV', + fileSuffix: 'csv' +}, { + label: 'Excel (.xlsx)', + value: 'EXCEL', + fileSuffix: 'xlsx' +}] + +const DEFAULT_EXPORT_OPTS: ExportOptions = { + splitOptionsIntoColumns: false, + stableIdsForOptions: false, + fileFormat: 'TSV', + includeSubheaders: true, + onlyIncludeMostRecent: true, + filterString: undefined, + excludeModules: [] +} + +const MODULE_EXCLUDE_OPTIONS: Record = { surveys: 'Surveys' } + +/** form for configuring and downloading enrollee data */ +const ExportDataModal = ({ studyEnvContext, show, setShow }: {studyEnvContext: StudyEnvContextT, show: boolean, + setShow: React.Dispatch>}) => { + const [isLoading, setIsLoading] = useState(false) + const [exportOptions, setExportOptions] = useState(DEFAULT_EXPORT_OPTS) + + const doExport = () => { + doApiLoad(async () => { + const response = await Api.exportEnrollees(studyEnvContext.portal.shortcode, studyEnvContext.study.shortcode, + studyEnvContext.currentEnv.environmentName, exportOptions) + const fileSuffix = FILE_FORMATS.find(format => + exportOptions.fileFormat === format.value)?.fileSuffix + const fileName = `${currentIsoDate()}-enrollees.${fileSuffix}` + const blob = await response.blob() + saveBlobAsDownload(blob, fileName) + }, { setIsLoading }) + } + + const doDictionaryExport = () => { + doApiLoad(async () => { + const response = await Api.exportDictionary(studyEnvContext.portal.shortcode, studyEnvContext.study.shortcode, + studyEnvContext.currentEnv.environmentName, exportOptions) + const fileName = `${currentIsoDate()}-DataDictionary.xlsx` + const blob = await response.blob() + saveBlobAsDownload(blob, fileName) + }, { setIsLoading }) + } + + return setShow(false)} size="lg"> + + + Download + + + + + + + + + + + + + +} + +export function ExportOptionsForm({ exportOptions, setExportOptions }: + { exportOptions: ExportOptions, setExportOptions: (opts: ExportOptions) => void }) { + const [showAdvancedOptions, setShowAdvancedOptions] = useState(false) + + + const { selectInputId, selectedOptions, options, onChange } = useReactMultiSelect( + Object.keys(MODULE_EXCLUDE_OPTIONS), + key => ({ label: MODULE_EXCLUDE_OPTIONS[key], value: key }), + (excludeModules: string[]) => setExportOptions({ ...exportOptions, excludeModules }), + exportOptions.excludeModules + ) + + const includeUnconsented = + !exportOptions.filterString?.includes('{enrollee.consented} = true') + + const includeProxiesAsRows = + !exportOptions.filterString?.includes('{enrollee.subject} = true') + + + return
    e.preventDefault()}> +
    +

    + Data format +

    + + +
    +
    + File format
    + {FILE_FORMATS.map(format => )} +
    +
    + +
    + { showAdvancedOptions &&
    +
    +

    + Completions included of a survey (for recurring surveys) +

    + + +
    +
    +

    + Include subheaders for columns +

    + + +
    +
    +

    + Filter Options +

    + + + + + setIntegration({ + ...integration, + enabled: e.target.checked + })} + className="me-1"/> + + +
    Url:
    + https://api.airtable.com/ + setIntegration({ ...integration, destinationUrl: val })} /> +
    + +
    + +
    + { (showOptions && integration?.exportOptions) &&
    + setIntegration({ + ...integration, + exportOptions: opts + })}/> +
    } + +} diff --git a/ui-admin/src/study/participants/export/ExportDataControl.tsx b/ui-admin/src/study/participants/export/ExportDataControl.tsx deleted file mode 100644 index be0574f35e..0000000000 --- a/ui-admin/src/study/participants/export/ExportDataControl.tsx +++ /dev/null @@ -1,202 +0,0 @@ -import React, { useState } from 'react' -import { StudyEnvContextT } from 'study/StudyEnvironmentRouter' -import Modal from 'react-bootstrap/Modal' -import LoadingSpinner from 'util/LoadingSpinner' -import Api, { ExportOptions } from 'api/api' -import { currentIsoDate } from '@juniper/ui-core' -import { Link } from 'react-router-dom' -import { saveBlobAsDownload } from 'util/downloadUtils' -import { doApiLoad } from 'api/api-utils' -import { buildFilter } from 'util/exportUtils' -import { Button } from '../../../components/forms/Button' -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' -import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons' -import Select from 'react-select' -import { useReactMultiSelect } from '../../../util/react-select-utils' - -const FILE_FORMATS = [{ - label: 'Tab-delimited (.tsv)', - value: 'TSV', - fileSuffix: 'tsv' -}, { - label: 'Comma-delimited (.csv)', - value: 'CSV', - fileSuffix: 'csv' -}, { - label: 'Excel (.xlsx)', - value: 'EXCEL', - fileSuffix: 'xlsx' -}] - -const MODULE_EXCLUDE_OPTIONS: Record = { surveys: 'Surveys' } - -/** form for configuring and downloading enrollee data */ -const ExportDataControl = ({ studyEnvContext, show, setShow }: {studyEnvContext: StudyEnvContextT, show: boolean, - setShow: React.Dispatch>}) => { - const [humanReadable, setHumanReadable] = useState(true) - const [onlyIncludeMostRecent, setOnlyIncludeMostRecent] = useState(true) - const [fileFormat, setFileFormat] = useState(FILE_FORMATS[0]) - const [includeProxiesAsRows, setIncludeProxiesAsRows] = useState(false) - const [includeUnconsented, setIncludeUnconsented] = useState(false) - const [includeSubheaders, setIncludeSubheaders] = useState(true) - const [showAdvancedOptions, setShowAdvancedOptions] = useState(false) - const [excludeModules, setExcludeModules] = useState([]) - const [isLoading, setIsLoading] = useState(false) - - const { selectInputId, selectedOptions, options, onChange } = useReactMultiSelect( - Object.keys(MODULE_EXCLUDE_OPTIONS), - key => ({ label: MODULE_EXCLUDE_OPTIONS[key], value: key }), - setExcludeModules, - excludeModules - ) - - const optionsFromState = (): ExportOptions => { - return { - onlyIncludeMostRecent, - splitOptionsIntoColumns: !humanReadable, - stableIdsForOptions: !humanReadable, - includeSubheaders, - excludeModules, - filter: buildFilter({ includeProxiesAsRows, includeUnconsented }), - fileFormat: fileFormat.value - } - } - - const doExport = () => { - doApiLoad(async () => { - const response = await Api.exportEnrollees(studyEnvContext.portal.shortcode, studyEnvContext.study.shortcode, - studyEnvContext.currentEnv.environmentName, optionsFromState()) - const fileName = `${currentIsoDate()}-enrollees.${fileFormat.fileSuffix}` - const blob = await response.blob() - saveBlobAsDownload(blob, fileName) - }, { setIsLoading }) - } - - const doDictionaryExport = () => { - doApiLoad(async () => { - const response = await Api.exportDictionary(studyEnvContext.portal.shortcode, studyEnvContext.study.shortcode, - studyEnvContext.currentEnv.environmentName, optionsFromState()) - const fileName = `${currentIsoDate()}-DataDictionary.xlsx` - const blob = await response.blob() - saveBlobAsDownload(blob, fileName) - }, { setIsLoading }) - } - - const humanReadableChanged = (e: React.ChangeEvent) => { - setHumanReadable(e.target.value === 'true') - } - const includeRecentChanged = (e: React.ChangeEvent) => { - setOnlyIncludeMostRecent(e.target.value === 'true') - } - - const inlcudeSubheadersChanged = (e: React.ChangeEvent) => { - setIncludeSubheaders(e.target.value === 'true') - } - - return setShow(false)}> - - - Download - - - -
    e.preventDefault()}> -
    -

    - Data format -

    - - -
    -
    - File format
    - {FILE_FORMATS.map(format => )} -
    -
    - -
    - { showAdvancedOptions &&
    -
    -

    - Completions included of a survey (for recurring surveys) -

    - - -
    -
    -

    - Include subheaders for columns -

    - - -
    -
    -

    - Filter Options -

    - - - -