Skip to content

Commit

Permalink
[JN-1365] export integrations for AirTable (#1118)
Browse files Browse the repository at this point in the history
  • Loading branch information
devonbush authored Oct 1, 2024
1 parent 7b28532 commit accf59d
Show file tree
Hide file tree
Showing 96 changed files with 1,955 additions and 420 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import bio.terra.pearl.api.admin.service.enrollee.EnrolleeImportExtService;
import bio.terra.pearl.core.model.EnvironmentName;
import bio.terra.pearl.core.model.admin.AdminUser;
import bio.terra.pearl.core.service.dataimport.ImportFileFormat;
import bio.terra.pearl.core.service.export.dataimport.ImportFileFormat;
import jakarta.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.util.UUID;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

import bio.terra.pearl.api.admin.api.DatarepoApi;
import bio.terra.pearl.api.admin.model.CreateDataset;
import bio.terra.pearl.api.admin.service.DataRepoExportExtService;
import bio.terra.pearl.api.admin.service.auth.AuthUtilService;
import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext;
import bio.terra.pearl.api.admin.service.export.DataRepoExportExtService;
import bio.terra.pearl.core.model.EnvironmentName;
import bio.terra.pearl.core.model.admin.AdminUser;
import bio.terra.pearl.core.model.datarepo.DataRepoJob;
import bio.terra.pearl.core.model.datarepo.Dataset;
import bio.terra.pearl.core.model.export.datarepo.DataRepoJob;
import bio.terra.pearl.core.model.export.datarepo.Dataset;
import jakarta.servlet.http.HttpServletRequest;
import java.util.List;
import org.springframework.http.ResponseEntity;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
package bio.terra.pearl.api.admin.controller.export;

import bio.terra.pearl.api.admin.api.ExportApi;
import bio.terra.pearl.api.admin.service.EnrolleeExportExtService;
import bio.terra.pearl.api.admin.service.auth.AuthUtilService;
import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext;
import bio.terra.pearl.api.admin.service.export.EnrolleeExportExtService;
import bio.terra.pearl.core.model.EnvironmentName;
import bio.terra.pearl.core.model.admin.AdminUser;
import bio.terra.pearl.core.model.export.ExportOptions;
import bio.terra.pearl.core.service.export.ExportFileFormat;
import bio.terra.pearl.core.service.export.ExportOptions;
import bio.terra.pearl.core.service.export.ExportOptionsWithExpression;
import bio.terra.pearl.core.service.search.EnrolleeSearchExpression;
import bio.terra.pearl.core.service.search.EnrolleeSearchExpressionParser;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import java.io.ByteArrayOutputStream;
import java.util.List;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.core.io.Resource;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -58,15 +59,15 @@ public ResponseEntity<Resource> exportData(
List<String> excludeModules,
String searchExpression,
String fileFormat,
Integer limit) {
Integer rowLimit) {
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
AdminUser user = authUtilService.requireAdminUser(request);

ExportOptions exportOptions =
ExportOptionsWithExpression exportOptions =
optionsFromParams(
searchExpression,
fileFormat,
limit,
rowLimit,
splitOptionsIntoColumns,
stableIdsForOptions,
includeOnlyMostRecent,
Expand Down Expand Up @@ -114,30 +115,29 @@ public ResponseEntity<Resource> exportDictionary(
return ResponseEntity.ok().body(new ByteArrayResource(baos.toByteArray()));
}

private ExportOptions optionsFromParams(
String searchExpression,
private ExportOptionsWithExpression optionsFromParams(
String filter,
String fileFormat,
Integer limit,
Boolean splitOptionsIntoColumns,
Boolean stableIdsForOptions,
Boolean includeOnlyMostRecent,
Boolean includeSubHeaders,
List<String> excludeModules) {
EnrolleeSearchExpression filter =
Objects.nonNull(searchExpression) && !searchExpression.isEmpty()
? enrolleeSearchExpressionParser.parseRule(searchExpression)
: null;
EnrolleeSearchExpression searchExp =
!StringUtils.isBlank(filter) ? enrolleeSearchExpressionParser.parseRule(filter) : null;

ExportOptions exportOptions =
ExportOptions.builder()
ExportOptionsWithExpression exportOptions =
ExportOptionsWithExpression.builder()
.splitOptionsIntoColumns(
splitOptionsIntoColumns != null ? splitOptionsIntoColumns : false)
.stableIdsForOptions(stableIdsForOptions != null ? stableIdsForOptions : false)
.onlyIncludeMostRecent(includeOnlyMostRecent != null ? includeOnlyMostRecent : false)
.filter(filter)
.filterString(filter)
.filterExpression(searchExp)
.fileFormat(
fileFormat != null ? ExportFileFormat.valueOf(fileFormat) : ExportFileFormat.TSV)
.limit(limit)
.rowLimit(limit)
.includeSubHeaders(includeSubHeaders)
.excludeModules(excludeModules != null ? excludeModules : List.of())
.build();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package bio.terra.pearl.api.admin.controller.export;

import bio.terra.pearl.api.admin.api.ExportIntegrationApi;
import bio.terra.pearl.api.admin.service.auth.AuthUtilService;
import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext;
import bio.terra.pearl.api.admin.service.export.ExportIntegrationExtService;
import bio.terra.pearl.core.model.EnvironmentName;
import bio.terra.pearl.core.model.admin.AdminUser;
import bio.terra.pearl.core.model.export.ExportIntegration;
import bio.terra.pearl.core.model.export.ExportIntegrationJob;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletRequest;
import java.util.List;
import java.util.UUID;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;

@Controller
public class ExportIntegrationController implements ExportIntegrationApi {
private final AuthUtilService authUtilService;
private final HttpServletRequest request;
private final ExportIntegrationExtService exportIntegrationExtService;
private final ObjectMapper objectMapper;

public ExportIntegrationController(
AuthUtilService authUtilService,
HttpServletRequest request,
ExportIntegrationExtService exportIntegrationExtService,
ObjectMapper objectMapper) {
this.authUtilService = authUtilService;
this.request = request;
this.exportIntegrationExtService = exportIntegrationExtService;
this.objectMapper = objectMapper;
}

@Override
public ResponseEntity<Object> findByStudy(
String portalShortcode, String studyShortcode, String envName) {
AdminUser operator = authUtilService.requireAdminUser(request);
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
List<ExportIntegration> integrations =
exportIntegrationExtService.list(
PortalStudyEnvAuthContext.of(
operator, portalShortcode, studyShortcode, environmentName));
return ResponseEntity.ok(integrations);
}

@Override
public ResponseEntity<Object> get(
String portalShortcode, String studyShortcode, String envName, UUID id) {
AdminUser operator = authUtilService.requireAdminUser(request);
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
ExportIntegration integration =
exportIntegrationExtService.find(
PortalStudyEnvAuthContext.of(
operator, portalShortcode, studyShortcode, environmentName),
id);
return ResponseEntity.ok(integration);
}

@Override
public ResponseEntity<Object> run(
String portalShortcode, String studyShortcode, String envName, UUID id) {
AdminUser operator = authUtilService.requireAdminUser(request);
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
ExportIntegrationJob job =
exportIntegrationExtService.run(
PortalStudyEnvAuthContext.of(
operator, portalShortcode, studyShortcode, environmentName),
id);
return ResponseEntity.ok(job);
}

@Override
public ResponseEntity<Object> create(
String portalShortcode, String studyShortcode, String envName, Object body) {
AdminUser adminUser = authUtilService.requireAdminUser(request);
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
ExportIntegration integration = objectMapper.convertValue(body, ExportIntegration.class);
ExportIntegration newIntegration =
exportIntegrationExtService.create(
PortalStudyEnvAuthContext.of(
adminUser, portalShortcode, studyShortcode, environmentName),
integration);
return ResponseEntity.ok(newIntegration);
}

@Override
public ResponseEntity<Object> findJobsByStudy(
String portalShortcode, String studyShortcode, String envName) {
AdminUser operator = authUtilService.requireAdminUser(request);
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
List<ExportIntegrationJob> integrations =
exportIntegrationExtService.listJobs(
PortalStudyEnvAuthContext.of(
operator, portalShortcode, studyShortcode, environmentName));
return ResponseEntity.ok(integrations);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,12 @@ public TriggerController(
@Override
public ResponseEntity<Object> findByStudy(
String portalShortcode, String studyShortcode, String envName) {
AdminUser adminUser = authUtilService.requireAdminUser(request);
AdminUser operator = authUtilService.requireAdminUser(request);
EnvironmentName environmentName = EnvironmentName.valueOfCaseInsensitive(envName);
List<Trigger> configs =
triggerExtService.findForStudy(
PortalStudyEnvAuthContext.of(
adminUser, portalShortcode, studyShortcode, environmentName));
operator, portalShortcode, studyShortcode, environmentName));
return ResponseEntity.ok(configs);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import bio.terra.pearl.core.model.admin.AdminUser;
import bio.terra.pearl.core.service.address.AddressValidationConfig;
import bio.terra.pearl.core.service.exception.PermissionDeniedException;
import bio.terra.pearl.core.service.export.integration.AirtableExporter;
import bio.terra.pearl.core.service.kit.pepper.LivePepperDSMClient;
import bio.terra.pearl.core.shared.ApplicationRoutingPaths;
import java.util.Map;
Expand All @@ -18,16 +19,20 @@ public class ConfigExtService {
private Map<String, String> configMap;
private final LivePepperDSMClient.PepperDSMConfig pepperDSMConfig;
private final AddressValidationConfig addressValidationConfig;
private final AirtableExporter.AirtableConfig airtableConfig;

public ConfigExtService(
B2CConfiguration b2CConfiguration,
ApplicationRoutingPaths applicationRoutingPaths,
LivePepperDSMClient.PepperDSMConfig pepperDSMConfig,
AddressValidationConfig addressValidationConfig) {
AddressValidationConfig addressValidationConfig,
AirtableExporter.AirtableConfig airtableConfig) {
this.b2CConfiguration = b2CConfiguration;
this.pepperDSMConfig = pepperDSMConfig;
this.applicationRoutingPaths = applicationRoutingPaths;
this.addressValidationConfig = addressValidationConfig;
this.airtableConfig = airtableConfig;

configMap = buildConfigMap();
}

Expand Down Expand Up @@ -64,7 +69,7 @@ private Map<String, String> buildConfigMap() {
if (!user.isSuperuser()) {
throw new PermissionDeniedException("You do not have permission to view this config");
}
var configMap =
Map<String, Map<String, String>> internalConfigMap =
Map.of(
"pepperDsmConfig",
Map.of(
Expand All @@ -78,8 +83,10 @@ private Map<String, String> buildConfigMap() {
Map.of(
"addrValidationServiceClass", addressValidationConfig.getAddressValidationClass(),
"smartyAuthId", addressValidationConfig.getAuthId(),
"smartyAuthToken", maskSecret(addressValidationConfig.getAuthToken())));
return configMap;
"smartyAuthToken", maskSecret(addressValidationConfig.getAuthToken())),
"airtable",
Map.of("authToken", maskSecret(airtableConfig.getAuthToken())));
return internalConfigMap;
}

public static String maskSecret(String secret) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
import bio.terra.pearl.core.model.dataimport.Import;
import bio.terra.pearl.core.model.dataimport.ImportItemStatus;
import bio.terra.pearl.core.model.dataimport.ImportStatus;
import bio.terra.pearl.core.service.dataimport.ImportFileFormat;
import bio.terra.pearl.core.service.dataimport.ImportItemService;
import bio.terra.pearl.core.service.dataimport.ImportService;
import bio.terra.pearl.core.service.exception.NotFoundException;
import bio.terra.pearl.core.service.exception.PermissionDeniedException;
import bio.terra.pearl.core.service.export.EnrolleeImportService;
import bio.terra.pearl.core.service.export.dataimport.ImportFileFormat;
import bio.terra.pearl.core.service.export.dataimport.ImportItemService;
import bio.terra.pearl.core.service.export.dataimport.ImportService;
import bio.terra.pearl.core.service.study.StudyEnvironmentService;
import java.io.InputStream;
import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package bio.terra.pearl.api.admin.service;
package bio.terra.pearl.api.admin.service.export;

import bio.terra.pearl.api.admin.model.CreateDataset;
import bio.terra.pearl.api.admin.service.auth.*;
import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext;
import bio.terra.pearl.core.model.datarepo.DataRepoJob;
import bio.terra.pearl.core.model.datarepo.Dataset;
import bio.terra.pearl.core.model.export.datarepo.DataRepoJob;
import bio.terra.pearl.core.model.export.datarepo.Dataset;
import bio.terra.pearl.core.service.datarepo.DataRepoExportService;
import bio.terra.pearl.core.service.exception.PermissionDeniedException;
import bio.terra.pearl.core.service.study.StudyEnvironmentService;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,36 +1,29 @@
package bio.terra.pearl.api.admin.service;
package bio.terra.pearl.api.admin.service.export;

import bio.terra.pearl.api.admin.service.auth.AuthUtilService;
import bio.terra.pearl.api.admin.service.auth.EnforcePortalStudyEnvPermission;
import bio.terra.pearl.api.admin.service.auth.context.PortalStudyEnvAuthContext;
import bio.terra.pearl.core.model.export.ExportOptions;
import bio.terra.pearl.core.service.export.DictionaryExportService;
import bio.terra.pearl.core.service.export.EnrolleeExportService;
import bio.terra.pearl.core.service.export.ExportOptions;
import bio.terra.pearl.core.service.study.StudyEnvironmentService;
import bio.terra.pearl.core.service.export.ExportOptionsWithExpression;
import java.io.OutputStream;
import org.springframework.stereotype.Service;

@Service
public class EnrolleeExportExtService {
private AuthUtilService authUtilService;
private StudyEnvironmentService studyEnvironmentService;
private EnrolleeExportService enrolleeExportService;
private DictionaryExportService dictionaryExportService;

public EnrolleeExportExtService(
AuthUtilService authUtilService,
StudyEnvironmentService studyEnvironmentService,
EnrolleeExportService enrolleeExportService,
DictionaryExportService dictionaryExportService) {
this.authUtilService = authUtilService;
this.studyEnvironmentService = studyEnvironmentService;
this.enrolleeExportService = enrolleeExportService;
this.dictionaryExportService = dictionaryExportService;
}

@EnforcePortalStudyEnvPermission(permission = "participant_data_view")
public void export(
PortalStudyEnvAuthContext authContext, ExportOptions options, OutputStream os) {
PortalStudyEnvAuthContext authContext, ExportOptionsWithExpression options, OutputStream os) {
enrolleeExportService.export(options, authContext.getStudyEnvironment().getId(), os);
}

Expand Down
Loading

0 comments on commit accf59d

Please sign in to comment.