From f09b3ff8abbf5e0a80b829dbdfc68777eafd6bdc Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Tue, 11 Jul 2023 16:29:14 -0400 Subject: [PATCH] Handle timezones in bulk uploader and Fhir converter (#5985) * Handle timezones in bulk uploader and Fhir converter * Handle timezones in bulk uploader and Fhir converter * Fix intellij auto conversion to * import * Check testResultTimeZoneInfo for null * Use ZonedDateTime for test result date * Use ZonedDateTime in DiagnosticReport Use Z for system timestamps with setTimeZoneZulu * Fix intellij wildcard import * Handle null date tested on DiagnosticReport * Update FhirConverter JSON test date format * Update issued as UTC instant from zoned time * Use UTC Date instant for DiagnosticReport * Remove ZonedDateTime helper method * Revert intellij wildcard import * Test invalid address exception * Use timezone with ET fallback for date tested * Nullable zonedDateTime for conversion helper * Validate and cache timezones in datetime fields * Backend lint fixes * Fix lints from missing lefthook run * Remove address service from FhirConverter * Update timezone regex for coverage and ICANN options * Fix timezone suffix regex * Add tests for date time utils * Fix test mock and code smells * Fix DateTimeUtils line separators * Update FHIR tests with timezones * Fill specimen received time with order test date * Add caching test and rename service * Fill date result released with test result date * Fill blank ordering facility with testing lab * Update caching mock in DateTimeUtilsTest * Remove flaky test for not caching unique addresses * Add sonarlint fixes * Transform zoned datetimes for covid csv pipeline * Use offset datetime string format --- .../api/converter/ConvertToSpecimenProps.java | 18 ++ .../api/converter/FhirConverter.java | 227 ++++++++++++------ .../api/model/filerow/TestResultRow.java | 15 +- .../simplereport/config/CachingConfig.java | 6 +- .../config/FileValidatorConfiguration.java | 7 +- .../service/AddressValidationService.java | 60 ++++- ...ava => ResultsUploaderCachingService.java} | 47 ++-- .../service/TestResultUploadService.java | 123 ++++++++-- .../errors/InvalidAddressException.java | 11 + .../service/model/TimezoneInfo.java | 10 + .../utils/BulkUploadResultsToFhir.java | 151 +++++++----- .../simplereport/utils/DateTimeUtils.java | 123 ++++++++++ .../simplereport/utils/FhirDateTimeUtil.java | 24 -- .../simplereport/utils/ZoneIdGenerator.java | 28 --- .../validators/CsvValidatorUtils.java | 40 ++- .../api/converter/FhirConverterTest.java | 124 ++++++---- .../service/AddressValidationServiceTest.java | 50 ++++ ...=> ResultsUploaderCachingServiceTest.java} | 22 +- .../service/TestResultUploadServiceTest.java | 4 +- .../simplereport/test_util/JsonTestUtils.java | 5 +- .../utils/BulkUploadResultsToFhirTest.java | 120 ++++++--- .../simplereport/utils/DateTimeUtilsTest.java | 60 +++++ .../validators/CsvValidatorUtilsTest.java | 41 ++++ .../validators/FileValidatorTest.java | 10 +- .../validators/TestResultRowTest.java | 12 +- backend/src/test/resources/fhir/bundle.json | 16 +- .../test/resources/fhir/diagnosticReport.json | 4 +- .../resources/fhir/observationCorrection.json | 2 +- .../test/resources/fhir/observationCovid.json | 2 +- .../test/resources/fhir/observationFlu.json | 2 +- .../test/resources/fhir/serviceRequest.json | 2 +- backend/src/test/resources/fhir/specimen.json | 3 +- .../test-results-upload-valid-as-fhir.json | 14 +- ...results-upload-valid-with-blank-fields.csv | 2 + .../test-results-upload-valid.csv | 2 +- 35 files changed, 1026 insertions(+), 361 deletions(-) create mode 100644 backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConvertToSpecimenProps.java rename backend/src/main/java/gov/cdc/usds/simplereport/service/{ResultsUploaderDeviceValidationService.java => ResultsUploaderCachingService.java} (74%) create mode 100644 backend/src/main/java/gov/cdc/usds/simplereport/service/errors/InvalidAddressException.java create mode 100644 backend/src/main/java/gov/cdc/usds/simplereport/service/model/TimezoneInfo.java create mode 100644 backend/src/main/java/gov/cdc/usds/simplereport/utils/DateTimeUtils.java delete mode 100644 backend/src/main/java/gov/cdc/usds/simplereport/utils/FhirDateTimeUtil.java delete mode 100644 backend/src/main/java/gov/cdc/usds/simplereport/utils/ZoneIdGenerator.java rename backend/src/test/java/gov/cdc/usds/simplereport/service/{ResultsUploaderDeviceValidationServiceTest.java => ResultsUploaderCachingServiceTest.java} (78%) create mode 100644 backend/src/test/java/gov/cdc/usds/simplereport/utils/DateTimeUtilsTest.java create mode 100644 backend/src/test/resources/testResultUpload/test-results-upload-valid-with-blank-fields.csv diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConvertToSpecimenProps.java b/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConvertToSpecimenProps.java new file mode 100644 index 0000000000..c00d73e782 --- /dev/null +++ b/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConvertToSpecimenProps.java @@ -0,0 +1,18 @@ +package gov.cdc.usds.simplereport.api.converter; + +import java.time.ZonedDateTime; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ConvertToSpecimenProps { + private String specimenCode; + private String specimenName; + private String collectionCode; + private String collectionName; + private String id; + private String identifier; + private ZonedDateTime collectionDate; + private ZonedDateTime receivedTime; +} diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/FhirConverter.java b/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/FhirConverter.java index eea960d63e..d98b6ac744 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/FhirConverter.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/api/converter/FhirConverter.java @@ -56,11 +56,13 @@ import gov.cdc.usds.simplereport.db.model.auxiliary.PhoneType; import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; import gov.cdc.usds.simplereport.db.model.auxiliary.TestCorrectionStatus; -import gov.cdc.usds.simplereport.utils.FhirDateTimeUtil; +import gov.cdc.usds.simplereport.service.TestOrderService; import gov.cdc.usds.simplereport.utils.MultiplexUtils; import gov.cdc.usds.simplereport.utils.UUIDGenerator; import java.time.LocalDate; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; @@ -68,9 +70,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TimeZone; import java.util.UUID; import java.util.function.Function; -import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -125,7 +127,6 @@ public class FhirConverter { private final UUIDGenerator uuidGenerator; - private final FhirDateTimeUtil fhirDateTimeUtil; private static final String SIMPLE_REPORT_ORG_ID = "07640c5d-87cd-488b-9343-a226c5166539"; @@ -156,7 +157,7 @@ public HumanName convertToHumanName(String first, String middle, String last, St public List convertPhoneNumbersToContactPoint( @NotNull List phoneNumber) { - return phoneNumber.stream().map(this::convertToContactPoint).collect(Collectors.toList()); + return phoneNumber.stream().map(this::convertToContactPoint).toList(); } public ContactPoint convertToContactPoint(@NotNull PhoneNumber phoneNumber) { @@ -185,9 +186,7 @@ public ContactPoint convertToContactPoint(ContactPointUse contactPointUse, Strin } public List convertEmailsToContactPoint(ContactPointUse use, List emails) { - return emails.stream() - .map(email -> convertEmailToContactPoint(use, email)) - .collect(Collectors.toList()); + return emails.stream().map(email -> convertEmailToContactPoint(use, email)).toList(); } public ContactPoint convertEmailToContactPoint(ContactPointUse use, @NotNull String email) { @@ -201,11 +200,9 @@ public ContactPoint convertToContactPoint( public AdministrativeGender convertToAdministrativeGender(@NotNull String gender) { switch (gender.toLowerCase()) { - case "male": - case "m": + case "male", "m": return AdministrativeGender.MALE; - case "female": - case "f": + case "female", "f": return AdministrativeGender.FEMALE; default: return AdministrativeGender.UNKNOWN; @@ -216,6 +213,52 @@ public Date convertToDate(@NotNull LocalDate date) { return Date.from(date.atStartOfDay(ZoneId.systemDefault()).toInstant()); } + /** + * @param zonedDateTime the date time with a time zone + * @return the DateTimeType object created from the ZonedDateTime, it's timezone, and second + * temporal precision + */ + public DateTimeType convertToDateTimeType(ZonedDateTime zonedDateTime) { + return convertToDateTimeType(zonedDateTime, TemporalPrecisionEnum.SECOND); + } + + /** + * @param zonedDateTime the date time with a time zone + * @param temporalPrecisionEnum precision of the date time, defaults to {@code + * TemporalPrecisionEnum.SECOND} + * @return the DateTimeType object created from the ZonedDateTime and it's timezone + */ + public DateTimeType convertToDateTimeType( + ZonedDateTime zonedDateTime, TemporalPrecisionEnum temporalPrecisionEnum) { + if (zonedDateTime == null) { + return null; + } + if (temporalPrecisionEnum == null) { + temporalPrecisionEnum = TemporalPrecisionEnum.SECOND; + } + return new DateTimeType( + Date.from(zonedDateTime.toInstant()), + temporalPrecisionEnum, + TimeZone.getTimeZone(zonedDateTime.getZone())); + } + + /** + * @param zonedDateTime the date time with a time zone + * @param temporalPrecisionEnum precision of the date time, defaults to {@code + * TemporalPrecisionEnum.MILLI} + * @return the InstantType object created from the Instant of the ZonedDateTime and its time zone + * offset + */ + public InstantType convertToInstantType( + ZonedDateTime zonedDateTime, TemporalPrecisionEnum temporalPrecisionEnum) { + if (zonedDateTime == null) return null; + if (temporalPrecisionEnum == null) temporalPrecisionEnum = TemporalPrecisionEnum.MILLI; + return new InstantType( + Date.from(zonedDateTime.toInstant()), + temporalPrecisionEnum, + TimeZone.getTimeZone(zonedDateTime.getZone())); + } + public Address convertToAddress(@NotNull StreetAddress address, String country) { return convertToAddress( address.getStreet(), @@ -448,51 +491,54 @@ public Device convertToDevice(String manufacturer, @NotNull String model, String return device; } - public Specimen convertToSpecimen( - String specimenCode, - String specimenName, - String collectionCode, - String collectionName, - String id, - String identifier, - Date collectionDate) { + public Specimen convertToSpecimen(ConvertToSpecimenProps props) { var specimen = new Specimen(); - specimen.setId(id); - specimen.addIdentifier().setValue(identifier); - if (StringUtils.isNotBlank(specimenCode)) { + specimen.setId(props.getId()); + specimen.addIdentifier().setValue(props.getIdentifier()); + if (StringUtils.isNotBlank(props.getSpecimenCode())) { var codeableConcept = specimen.getType(); var coding = codeableConcept.addCoding(); coding.setSystem(SNOMED_CODE_SYSTEM); - coding.setCode(specimenCode); - codeableConcept.setText(specimenName); + coding.setCode(props.getSpecimenCode()); + codeableConcept.setText(props.getSpecimenName()); } - if (StringUtils.isNotBlank(collectionCode)) { + if (StringUtils.isNotBlank(props.getCollectionCode())) { var collection = specimen.getCollection(); var codeableConcept = collection.getBodySite(); var coding = codeableConcept.addCoding(); coding.setSystem(SNOMED_CODE_SYSTEM); - coding.setCode(collectionCode); - codeableConcept.setText(collectionName); + coding.setCode(props.getCollectionCode()); + codeableConcept.setText(props.getCollectionName()); } - if (collectionDate != null) { + if (props.getCollectionDate() != null) { var collection = specimen.getCollection(); - collection.setCollected(new DateTimeType(collectionDate).setTimeZoneZulu(true)); + collection.setCollected(convertToDateTimeType(props.getCollectionDate())); + } + + if (props.getReceivedTime() != null) { + specimen.setReceivedTimeElement(convertToDateTimeType(props.getReceivedTime())); } return specimen; } public Specimen convertToSpecimen( - @NotNull SpecimenType specimenType, UUID specimenIdentifier, Date collectionDate) { + @NotNull SpecimenType specimenType, + UUID specimenIdentifier, + ZonedDateTime collectionDate, + ZonedDateTime receivedTime) { return convertToSpecimen( - specimenType.getTypeCode(), - specimenType.getName(), - specimenType.getCollectionLocationCode(), - specimenType.getCollectionLocationName(), - specimenType.getInternalId().toString(), - specimenIdentifier.toString(), - collectionDate); + ConvertToSpecimenProps.builder() + .specimenCode(specimenType.getTypeCode()) + .specimenName(specimenType.getName()) + .collectionCode(specimenType.getCollectionLocationCode()) + .collectionName(specimenType.getCollectionLocationName()) + .id(specimenType.getInternalId().toString()) + .identifier(specimenIdentifier.toString()) + .collectionDate(collectionDate) + .receivedTime(receivedTime) + .build()); } public List convertToObservation( @@ -529,7 +575,7 @@ public List convertToObservation( deviceType.getModel(), resultDate); }) - .collect(Collectors.toList()); + .toList(); } public String getCommonDiseaseValue( @@ -597,7 +643,7 @@ public Observation convertToObservation(ConvertToObservationProps props) { .addCoding(convertToAbnormalFlagInterpretation(props.getResultCode())); observation.setIssued(props.getIssued()); - observation.getIssuedElement().setTimeZoneZulu(true).setPrecision(TemporalPrecisionEnum.SECOND); + observation.getIssuedElement().setTimeZoneZulu(true); return observation; } @@ -671,7 +717,7 @@ public Set convertToAOEObservation( public Set convertToAOEObservations(String eventId, AskOnEntrySurvey surveyData) { Boolean symptomatic = null; - if (surveyData.getNoSymptoms()) { + if (Boolean.TRUE.equals(surveyData.getNoSymptoms())) { symptomatic = false; } else if (surveyData.getSymptoms().containsValue(Boolean.TRUE)) { symptomatic = true; @@ -728,7 +774,8 @@ private void addSNOMEDValue(String resultCode, Observation observation, String r observation.setValue(valueCodeableConcept); } - public ServiceRequest convertToServiceRequest(@NotNull TestOrder order, Date orderTestDate) { + public ServiceRequest convertToServiceRequest( + @NotNull TestOrder order, ZonedDateTime orderTestDate) { ServiceRequestStatus serviceRequestStatus = null; switch (order.getOrderStatus()) { case PENDING: @@ -757,7 +804,10 @@ public ServiceRequest convertToServiceRequest(@NotNull TestOrder order, Date ord } public ServiceRequest convertToServiceRequest( - ServiceRequestStatus status, String requestedCode, String id, Date orderEffectiveDate) { + ServiceRequestStatus status, + String requestedCode, + String id, + ZonedDateTime orderEffectiveDate) { var serviceRequest = new ServiceRequest(); serviceRequest.setId(id); serviceRequest.setIntent(ServiceRequestIntent.ORDER); @@ -780,11 +830,19 @@ public ServiceRequest convertToServiceRequest( serviceRequest .addExtension() .setUrl(ORDER_EFFECTIVE_DATE_EXTENSION_URL) - .setValue(new DateTimeType(orderEffectiveDate).setTimeZoneZulu(true)); + .setValue(convertToDateTimeType(orderEffectiveDate, TemporalPrecisionEnum.SECOND)); return serviceRequest; } + /** + * Used during single entry FHIR conversion + * + * @param testEvent Single entry test event. + * @param currentDate Used to set {@code DiagnosticReport.issued}, the instant this version was * + * made. + * @return DiagnosticReport + */ public DiagnosticReport convertToDiagnosticReport(TestEvent testEvent, Date currentDate) { DiagnosticReportStatus status = null; switch (testEvent.getCorrectionStatus()) { @@ -806,37 +864,42 @@ public DiagnosticReport convertToDiagnosticReport(TestEvent testEvent, Date curr testEvent.getDeviceType().getSupportedDiseaseTestPerformed()); } + ZonedDateTime dateTested = null; + if (testEvent.getDateTested() != null) { + // getDateTested returns a Date representing an exact moment of time so + // finding a specific timezone for the TestEvent is not required to ensure it is accurate + dateTested = ZonedDateTime.ofInstant(testEvent.getDateTested().toInstant(), ZoneOffset.UTC); + } + ZonedDateTime dateIssued = null; + if (currentDate != null) + dateIssued = ZonedDateTime.ofInstant(currentDate.toInstant(), ZoneOffset.UTC); + return convertToDiagnosticReport( - status, - code, - Objects.toString(testEvent.getInternalId(), ""), - testEvent.getDateTested(), - currentDate); + status, code, Objects.toString(testEvent.getInternalId(), ""), dateTested, dateIssued); } + /** + * @param status Diagnostic report status + * @param code LOINC code + * @param id Diagnostic report id + * @param dateTested Used to set {@code DiagnosticReport.effective}, the clinically relevant + * time/time-period for report. + * @param dateIssued Used to set {@code DiagnosticReport.issued}, the date and time that this + * version of the report was made available to providers, typically after the report was + * reviewed and verified. + * @return DiagnosticReport + */ public DiagnosticReport convertToDiagnosticReport( - DiagnosticReportStatus status, String code, String id, Date dateTested, Date currentDate) { + DiagnosticReportStatus status, + String code, + String id, + ZonedDateTime dateTested, + ZonedDateTime dateIssued) { var diagnosticReport = new DiagnosticReport() .setStatus(status) - .setEffective(new DateTimeType(dateTested).setTimeZoneZulu(true)) - .setIssued(currentDate); - - diagnosticReport - .getIssuedElement() - .setTimeZoneZulu(true) - .setPrecision(TemporalPrecisionEnum.SECOND); - - // Allows EffectiveDateTimeType to be mocked during tests - var localEffectiveDateTimeType = diagnosticReport.getEffectiveDateTimeType(); - var unchangedEffectiveDateTimeType = - fhirDateTimeUtil.getBaseDateTimeType(localEffectiveDateTimeType); - diagnosticReport.setEffective(unchangedEffectiveDateTimeType); - - // Allows issued element to be mocked during tests - var localIssued = diagnosticReport.getIssuedElement(); - var unchangedIssued = (InstantType) fhirDateTimeUtil.getBaseDateTimeType(localIssued); - diagnosticReport.setIssuedElement(unchangedIssued); + .setEffective(convertToDateTimeType(dateTested)) + .setIssuedElement(convertToInstantType(dateIssued, TemporalPrecisionEnum.SECOND)); diagnosticReport.setId(id); if (StringUtils.isNotBlank(code)) { @@ -846,12 +909,25 @@ public DiagnosticReport convertToDiagnosticReport( return diagnosticReport; } + /** + * @param testEvent The single entry test event created in {@code TestOrderService} + * @param gitProperties Git properties + * @param currentDate Current date + * @param processingId Processing id + * @return FHIR bundle + * @see TestOrderService + */ public Bundle createFhirBundle( @NotNull TestEvent testEvent, GitProperties gitProperties, Date currentDate, String processingId) { + ZonedDateTime dateTested = + testEvent.getDateTested() != null + ? ZonedDateTime.ofInstant(testEvent.getDateTested().toInstant(), ZoneOffset.UTC) + : null; + return createFhirBundle( CreateFhirBundleProps.builder() .patient(convertToPatient(testEvent.getPatient())) @@ -860,7 +936,8 @@ public Bundle createFhirBundle( .practitioner(convertToPractitioner(testEvent.getProviderData())) .device(convertToDevice(testEvent.getDeviceType())) .specimen( - convertToSpecimen(testEvent.getSpecimenType(), uuidGenerator.randomUUID(), null)) + convertToSpecimen( + testEvent.getSpecimenType(), uuidGenerator.randomUUID(), null, null)) .resultObservations( convertToObservation( testEvent.getResults(), @@ -871,8 +948,7 @@ public Bundle createFhirBundle( .aoeObservations( convertToAOEObservations( testEvent.getInternalId().toString(), testEvent.getSurveyData())) - .serviceRequest( - convertToServiceRequest(testEvent.getOrder(), testEvent.getDateTested())) + .serviceRequest(convertToServiceRequest(testEvent.getOrder(), dateTested)) .diagnosticReport(convertToDiagnosticReport(testEvent, currentDate)) .currentDate(currentDate) .gitProperties(gitProperties) @@ -982,10 +1058,7 @@ public Bundle createFhirBundle(CreateFhirBundleProps props) { .setTimestamp(props.getCurrentDate()) .setIdentifier(new Identifier().setValue(props.getDiagnosticReport().getId())); - // Allows timestamp element to be mocked during tests - var localTimestamp = bundle.getTimestampElement(); - var unchangedTimestamp = (InstantType) fhirDateTimeUtil.getBaseDateTimeType(localTimestamp); - bundle.setTimestampElement(unchangedTimestamp); + bundle.getTimestampElement().setTimeZoneZulu(true); entryList.forEach( pair -> @@ -1001,11 +1074,6 @@ public Provenance createProvenance( String organizationFullUrl, Date dateTested, UUID provenanceId) { var provenance = new Provenance(); - // Allows recorded element to be mocked during tests - var localRecorded = provenance.getRecordedElement(); - var unchangedRecorded = fhirDateTimeUtil.getBaseDateTimeType(localRecorded); - provenance.setRecordedElement((InstantType) unchangedRecorded); - provenance.setId(provenanceId.toString()); provenance .getActivity() @@ -1015,6 +1083,7 @@ public Provenance createProvenance( .setDisplay(EVENT_TYPE_DISPLAY); provenance.addAgent().setWho(new Reference().setReference(organizationFullUrl)); provenance.setRecorded(dateTested); + provenance.getRecordedElement().setTimeZoneZulu(true); return provenance; } diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/TestResultRow.java b/backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/TestResultRow.java index 85fa904d71..88e1d56cdf 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/TestResultRow.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/TestResultRow.java @@ -19,7 +19,7 @@ import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateZipCode; import static java.util.Collections.emptyList; -import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage; import gov.cdc.usds.simplereport.validators.CsvValidatorUtils.ValueOrError; import java.util.ArrayList; @@ -211,7 +211,7 @@ public class TestResultRow implements FileRow { "92977-8", "9531-5", "9534-9"); - private ResultsUploaderDeviceValidationService resultsUploaderDeviceValidationService; + private ResultsUploaderCachingService resultsUploaderCachingService; private static final List requiredFields = List.of( @@ -250,10 +250,9 @@ public class TestResultRow implements FileRow { TESTING_LAB_ZIP_CODE_FIELD); public TestResultRow( - Map rawRow, - ResultsUploaderDeviceValidationService resultsUploaderDeviceValidationService) { + Map rawRow, ResultsUploaderCachingService resultsUploaderCachingService) { this(rawRow); - this.resultsUploaderDeviceValidationService = resultsUploaderDeviceValidationService; + this.resultsUploaderCachingService = resultsUploaderCachingService; } public TestResultRow(Map rawRow) { @@ -380,10 +379,10 @@ private boolean validModelTestPerformedCombination( String equipmentModelName, String testPerformedCode) { return equipmentModelName != null && testPerformedCode != null - && resultsUploaderDeviceValidationService + && resultsUploaderCachingService .getModelAndTestPerformedCodeToDeviceMap() .containsKey( - ResultsUploaderDeviceValidationService.getMapKey( + ResultsUploaderCachingService.getMapKey( removeTrailingAsterisk(equipmentModelName), testPerformedCode)); } @@ -453,7 +452,7 @@ public List validateIndividualValues() { errors.addAll(validateTestResultStatus(testResultStatus)); errors.addAll( validateSpecimenType( - specimenType, resultsUploaderDeviceValidationService.getSpecimenTypeNameToSNOMEDMap())); + specimenType, resultsUploaderCachingService.getSpecimenTypeNameToSNOMEDMap())); errors.addAll(validateClia(testingLabClia)); diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/config/CachingConfig.java b/backend/src/main/java/gov/cdc/usds/simplereport/config/CachingConfig.java index a09f335495..7a55d2e766 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/config/CachingConfig.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/config/CachingConfig.java @@ -12,12 +12,14 @@ public class CachingConfig { public static final String DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP = "deviceModelAndTestPerformedCodeMap"; - public static final String SPECIMEN_NAME_AND_SNOMED_MAP = "specimenTypeNameSNOMEDMap"; + public static final String ADDRESS_TIMEZONE_LOOKUP_MAP = "addressTimezoneLookupMap"; @Bean public CacheManager cacheManager() { return new ConcurrentMapCacheManager( - DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP, SPECIMEN_NAME_AND_SNOMED_MAP); + DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP, + SPECIMEN_NAME_AND_SNOMED_MAP, + ADDRESS_TIMEZONE_LOOKUP_MAP); } } diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/config/FileValidatorConfiguration.java b/backend/src/main/java/gov/cdc/usds/simplereport/config/FileValidatorConfiguration.java index 7e1cd7988b..ae72f07a52 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/config/FileValidatorConfiguration.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/config/FileValidatorConfiguration.java @@ -2,7 +2,7 @@ import gov.cdc.usds.simplereport.api.model.filerow.PatientUploadRow; import gov.cdc.usds.simplereport.api.model.filerow.TestResultRow; -import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; import gov.cdc.usds.simplereport.validators.FileValidator; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -11,9 +11,8 @@ public class FileValidatorConfiguration { @Bean public FileValidator testResultRowFileValidator( - ResultsUploaderDeviceValidationService resultsUploaderDeviceValidationService) { - return new FileValidator<>( - row -> new TestResultRow(row, resultsUploaderDeviceValidationService)); + ResultsUploaderCachingService resultsUploaderCachingService) { + return new FileValidator<>(row -> new TestResultRow(row, resultsUploaderCachingService)); } @Bean diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/service/AddressValidationService.java b/backend/src/main/java/gov/cdc/usds/simplereport/service/AddressValidationService.java index 8fb3441727..e92e83e5c4 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/service/AddressValidationService.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/service/AddressValidationService.java @@ -12,7 +12,11 @@ import gov.cdc.usds.simplereport.api.model.errors.IllegalGraphqlArgumentException; import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; import gov.cdc.usds.simplereport.properties.SmartyStreetsProperties; +import gov.cdc.usds.simplereport.service.errors.InvalidAddressException; +import gov.cdc.usds.simplereport.service.model.TimezoneInfo; import java.io.IOException; +import java.time.ZoneId; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -48,7 +52,7 @@ private Lookup getStrictLookup( return lookup; } - public StreetAddress getValidatedAddress(Lookup lookup, String fieldName) { + private List getLookupResults(Lookup lookup) { try { _client.send(lookup); } catch (SmartyException | IOException ex) { @@ -60,7 +64,11 @@ public StreetAddress getValidatedAddress(Lookup lookup, String fieldName) { Thread.currentThread().interrupt(); } - var results = lookup.getResult(); + return lookup.getResult(); + } + + public StreetAddress getValidatedAddress(Lookup lookup, String fieldName) { + var results = getLookupResults(lookup); if (results.isEmpty()) { return new StreetAddress( @@ -96,4 +104,52 @@ public StreetAddress getValidatedAddress( Lookup lookup = getStrictLookup(street1, street2, city, state, postalCode); return getValidatedAddress(lookup, fieldName); } + + public TimezoneInfo getTimezoneInfoByLookup(Lookup lookup) { + var results = getLookupResults(lookup); + + if (results.isEmpty()) { + throw new InvalidAddressException("The server is unable to verify the address you entered."); + } + + Candidate addressMatch = results.get(0); + + return TimezoneInfo.builder() + .timezoneCommonName(addressMatch.getMetadata().getTimeZone()) + .utcOffset((int) addressMatch.getMetadata().getUtcOffset()) + .obeysDaylightSavings(addressMatch.getMetadata().obeysDst()) + .build(); + } + + /** + * @param address the StreetAddress to find the timezone id of + * @return ZoneId of the address or null if not found + */ + public ZoneId getZoneIdByAddress(StreetAddress address) { + if (address == null) { + return null; + } + Lookup lookup = + getStrictLookup( + address.getStreetOne(), + address.getStreetTwo(), + address.getCity(), + address.getState(), + address.getPostalCode()); + return getZoneIdByLookup(lookup); + } + + public ZoneId getZoneIdByLookup(Lookup lookup) { + TimezoneInfo timezoneInfo = null; + try { + timezoneInfo = getTimezoneInfoByLookup(lookup); + } catch (InvalidAddressException | IllegalGraphqlArgumentException exception) { + log.error("Unable to find timezone by testing lab address", exception); + } + if (timezoneInfo == null) { + return null; + } + + return ZoneId.of("US/" + timezoneInfo.timezoneCommonName); + } } diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderDeviceValidationService.java b/backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingService.java similarity index 74% rename from backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderDeviceValidationService.java rename to backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingService.java index d0d60fe2ee..11fab763ce 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderDeviceValidationService.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingService.java @@ -1,12 +1,15 @@ package gov.cdc.usds.simplereport.service; +import static gov.cdc.usds.simplereport.config.CachingConfig.ADDRESS_TIMEZONE_LOOKUP_MAP; import static gov.cdc.usds.simplereport.config.CachingConfig.DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP; import static gov.cdc.usds.simplereport.config.CachingConfig.SPECIMEN_NAME_AND_SNOMED_MAP; import gov.cdc.usds.simplereport.db.model.DeviceType; import gov.cdc.usds.simplereport.db.model.SpecimenType; +import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; import gov.cdc.usds.simplereport.db.repository.DeviceTypeRepository; import gov.cdc.usds.simplereport.db.repository.SpecimenTypeRepository; +import java.time.ZoneId; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -23,16 +26,22 @@ @Slf4j @Service @RequiredArgsConstructor -public class ResultsUploaderDeviceValidationService { +public class ResultsUploaderCachingService { private final DeviceTypeRepository deviceTypeRepository; private final SpecimenTypeRepository specimenTypeRepository; + private final AddressValidationService addressValidationService; + + private static final String NASAL_SWAB_SNOMED = "445297001"; + private static final String NASAL_THROAT_SWAB_SNOMED = "433801000124107"; + private static final String BRONCHOALVEOLAR_LAVAGE = "258607008"; + private static final String DRIED_BLOOD_SPOT = "440500007"; private static final Map specimenSNOMEDMap = Map.ofEntries( - Map.entry("swab of internal nose", "445297001"), - Map.entry("nasal swab", "445297001"), - Map.entry("nasal", "445297001"), - Map.entry("varied", "445297001"), + Map.entry("swab of internal nose", NASAL_SWAB_SNOMED), + Map.entry("nasal swab", NASAL_SWAB_SNOMED), + Map.entry("nasal", NASAL_SWAB_SNOMED), + Map.entry("varied", NASAL_SWAB_SNOMED), Map.entry("nasopharyngeal swab", "258500001"), Map.entry("mid-turbinate nasal swab", "871810001"), Map.entry("anterior nares swab", "697989009"), @@ -60,17 +69,17 @@ public class ResultsUploaderDeviceValidationService { Map.entry("blood specimen", "119297000"), Map.entry("capillary blood specimen", "122554006"), Map.entry("fingerstick whole blood", "122554006"), - Map.entry("dried blood spot specimen", "440500007"), - Map.entry("dried blood spot", "440500007"), - Map.entry("fingerstick blood dried blood spot", "440500007"), - Map.entry("nasopharyngeal and oropharyngeal swab", "433801000124107"), - Map.entry("nasal and throat swab combination", "433801000124107"), - Map.entry("nasal and throat swab", "433801000124107"), + Map.entry("dried blood spot specimen", DRIED_BLOOD_SPOT), + Map.entry("dried blood spot", DRIED_BLOOD_SPOT), + Map.entry("fingerstick blood dried blood spot", DRIED_BLOOD_SPOT), + Map.entry("nasopharyngeal and oropharyngeal swab", NASAL_THROAT_SWAB_SNOMED), + Map.entry("nasal and throat swab combination", NASAL_THROAT_SWAB_SNOMED), + Map.entry("nasal and throat swab", NASAL_THROAT_SWAB_SNOMED), Map.entry("lower respiratory fluid sample", "309171007"), Map.entry("lower respiratory tract aspirates", "309171007"), - Map.entry("bronchoalveolar lavage fluid sample", "258607008"), - Map.entry("bronchoalveolar lavage fluid", "258607008"), - Map.entry("bronchoalveolar lavage", "258607008")); + Map.entry("bronchoalveolar lavage fluid sample", BRONCHOALVEOLAR_LAVAGE), + Map.entry("bronchoalveolar lavage fluid", BRONCHOALVEOLAR_LAVAGE), + Map.entry("bronchoalveolar lavage", BRONCHOALVEOLAR_LAVAGE)); @Cacheable(DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP) public Map getModelAndTestPerformedCodeToDeviceMap() { @@ -131,4 +140,14 @@ public void cacheSpecimenTypeNameToSNOMEDMap() { public static String getMapKey(String model, String testPerformedCode) { return model.toLowerCase() + "|" + testPerformedCode.toLowerCase(); } + + @Cacheable(ADDRESS_TIMEZONE_LOOKUP_MAP) + public ZoneId getZoneIdByAddress(StreetAddress address) { + return addressValidationService.getZoneIdByAddress(address); + } + + @CacheEvict(cacheNames = ADDRESS_TIMEZONE_LOOKUP_MAP, allEntries = true) + public void clearAddressTimezoneLookupCache() { + log.info("clear address timezone lookup cache"); + } } diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/service/TestResultUploadService.java b/backend/src/main/java/gov/cdc/usds/simplereport/service/TestResultUploadService.java index 8fde6af88d..4101a91176 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/service/TestResultUploadService.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/service/TestResultUploadService.java @@ -1,6 +1,7 @@ package gov.cdc.usds.simplereport.service; import static gov.cdc.usds.simplereport.utils.AsyncLoggingUtils.withMDC; +import static gov.cdc.usds.simplereport.utils.DateTimeUtils.convertToZonedDateTime; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -12,6 +13,7 @@ import gov.cdc.usds.simplereport.config.AuthorizationConfiguration; import gov.cdc.usds.simplereport.db.model.Organization; import gov.cdc.usds.simplereport.db.model.TestResultUpload; +import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; import gov.cdc.usds.simplereport.db.model.auxiliary.UploadStatus; import gov.cdc.usds.simplereport.db.repository.TestResultUploadRepository; import gov.cdc.usds.simplereport.service.errors.InvalidBulkTestResultUploadException; @@ -39,6 +41,7 @@ import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -52,7 +55,7 @@ public class TestResultUploadService { private final TestResultUploadRepository _repo; private final DataHubClient _client; private final OrganizationService _orgService; - private final ResultsUploaderDeviceValidationService resultsUploaderDeviceValidationService; + private final ResultsUploaderCachingService resultsUploaderCachingService; private final TokenAuthentication _tokenAuth; private final FileValidator testResultFileValidator; private final BulkUploadResultsToFhir fhirConverter; @@ -77,6 +80,13 @@ public class TestResultUploadService { private static final int FIVE_MINUTES_MS = 300 * 1000; public static final String PROCESSING_MODE_CODE_COLUMN_NAME = "processing_mode_code"; + private static final String ORDER_TEST_DATE_COLUMN_NAME = "order_test_date"; + private static final String SPECIMEN_COLLECTION_DATE_COLUMN_NAME = "specimen_collection_date"; + private static final String TESTING_LAB_SPECIMEN_RECEIVED_DATE_COLUMN_NAME = + "testing_lab_specimen_received_date"; + private static final String TEST_RESULT_DATE_COLUMN_NAME = "test_result_date"; + private static final String DATE_RESULT_RELEASED_COLUMN_NAME = "date_result_released"; + public static final String SPECIMEN_TYPE_COLUMN_NAME = "specimen_type"; private static final String ALPHABET_REGEX = "^[a-zA-Z\\s]+$"; @@ -144,29 +154,111 @@ public TestResultUpload processResultCSV(InputStream csvStream) { return csvResult; } - private byte[] translateSpecimenNameToSNOMED(byte[] content, Map snomedMap) { + private byte[] transformCsvContent(byte[] content) { String[] rows = new String(content, StandardCharsets.UTF_8).split("\n"); - String headers = rows[0]; - - int specimenTypeIndex = - Arrays.stream(headers.split(",")).toList().indexOf(SPECIMEN_TYPE_COLUMN_NAME); + List headers = Arrays.stream(rows[0].split(",")).toList(); for (int i = 1; i < rows.length; i++) { var row = rows[i].split(",", -1); - var specimenTypeName = Arrays.stream(row).toList().get(specimenTypeIndex).toLowerCase(); - if (!specimenTypeName.matches(ALPHABET_REGEX)) { - continue; - } - - row[specimenTypeIndex] = snomedMap.get(specimenTypeName); + // row is passed by object reference here + modifyRowSpecimenNameToSNOMED(row, headers); + modifyRowDatetimeStrings(row, headers); rows[i] = String.join(",", row); } - return String.join("\n", rows).getBytes(); } + private void modifyRowSpecimenNameToSNOMED(String[] row, List headers) { + var snomedMap = resultsUploaderCachingService.getSpecimenTypeNameToSNOMEDMap(); + int specimenTypeIndex = headers.indexOf(SPECIMEN_TYPE_COLUMN_NAME); + var specimenTypeName = Arrays.stream(row).toList().get(specimenTypeIndex).toLowerCase(); + if (specimenTypeName.matches(ALPHABET_REGEX)) { + row[specimenTypeIndex] = snomedMap.get(specimenTypeName); + } + } + + private String valueAtRowIndex(int index, String[] row) { + return Arrays.stream(row).toList().get(index).toLowerCase(); + } + + private void modifyRowDatetimeStrings(String[] row, List headers) { + var testResultDateIndex = headers.indexOf(TEST_RESULT_DATE_COLUMN_NAME); + var orderTestDateIndex = headers.indexOf(ORDER_TEST_DATE_COLUMN_NAME); + var specimenCollectionDateIndex = headers.indexOf(SPECIMEN_COLLECTION_DATE_COLUMN_NAME); + var specimenReceivedDateIndex = headers.indexOf(TESTING_LAB_SPECIMEN_RECEIVED_DATE_COLUMN_NAME); + var dateResultReleasedIndex = headers.indexOf(DATE_RESULT_RELEASED_COLUMN_NAME); + + var testResultDate = valueAtRowIndex(testResultDateIndex, row); + var orderTestDate = valueAtRowIndex(orderTestDateIndex, row); + var specimenCollectionDate = valueAtRowIndex(specimenCollectionDateIndex, row); + var specimenReceivedDate = valueAtRowIndex(specimenReceivedDateIndex, row); + var dateResultReleased = valueAtRowIndex(dateResultReleasedIndex, row); + + var testingLabAddr = getTestingLabAddress(row, headers); + var providerAddr = getOrderingFacilityAddress(row, headers); + + testResultDate = + convertToZonedDateTime(testResultDate, resultsUploaderCachingService, testingLabAddr) + .toOffsetDateTime() + .toString(); + orderTestDate = + convertToZonedDateTime(orderTestDate, resultsUploaderCachingService, providerAddr) + .toOffsetDateTime() + .toString(); + + specimenCollectionDate = + StringUtils.isNotBlank(specimenCollectionDate) + ? convertToZonedDateTime( + specimenCollectionDate, resultsUploaderCachingService, providerAddr) + .toOffsetDateTime() + .toString() + : orderTestDate; + + specimenReceivedDate = + StringUtils.isNotBlank(specimenReceivedDate) + ? convertToZonedDateTime( + specimenReceivedDate, resultsUploaderCachingService, providerAddr) + .toOffsetDateTime() + .toString() + : orderTestDate; + + dateResultReleased = + StringUtils.isNotBlank(dateResultReleased) + ? convertToZonedDateTime( + dateResultReleased, resultsUploaderCachingService, providerAddr) + .toOffsetDateTime() + .toString() + : testResultDate; + + row[testResultDateIndex] = testResultDate; + row[orderTestDateIndex] = orderTestDate; + row[specimenCollectionDateIndex] = specimenCollectionDate; + row[specimenReceivedDateIndex] = specimenReceivedDate; + row[dateResultReleasedIndex] = dateResultReleased; + } + + private StreetAddress getTestingLabAddress(String[] row, List headers) { + return new StreetAddress( + valueAtRowIndex(headers.indexOf("testing_lab_street"), row), + valueAtRowIndex(headers.indexOf("testing_lab_street2"), row), + valueAtRowIndex(headers.indexOf("testing_lab_city"), row), + valueAtRowIndex(headers.indexOf("testing_lab_state"), row), + valueAtRowIndex(headers.indexOf("testing_lab_zip_code"), row), + null); + } + + private StreetAddress getOrderingFacilityAddress(String[] row, List headers) { + return new StreetAddress( + valueAtRowIndex(headers.indexOf("ordering_facility_street"), row), + valueAtRowIndex(headers.indexOf("ordering_facility_street2"), row), + valueAtRowIndex(headers.indexOf("ordering_facility_city"), row), + valueAtRowIndex(headers.indexOf("ordering_facility_state"), row), + valueAtRowIndex(headers.indexOf("ordering_facility_zip_code"), row), + null); + } + private byte[] attachProcessingModeCode(byte[] content) { String[] row = new String(content, StandardCharsets.UTF_8).split("\n"); String headers = row[0]; @@ -250,10 +342,7 @@ private Future submitResultsAsCsv(byte[] content) { long start = System.currentTimeMillis(); UploadResponse response; try { - var csvContent = - translateSpecimenNameToSNOMED( - content, - resultsUploaderDeviceValidationService.getSpecimenTypeNameToSNOMEDMap()); + var csvContent = transformCsvContent(content); response = _client.uploadCSV(csvContent); } catch (FeignException e) { diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/service/errors/InvalidAddressException.java b/backend/src/main/java/gov/cdc/usds/simplereport/service/errors/InvalidAddressException.java new file mode 100644 index 0000000000..132fd0328d --- /dev/null +++ b/backend/src/main/java/gov/cdc/usds/simplereport/service/errors/InvalidAddressException.java @@ -0,0 +1,11 @@ +package gov.cdc.usds.simplereport.service.errors; + +import java.io.Serial; + +public class InvalidAddressException extends RuntimeException { + @Serial private static final long serialVersionUID = 1L; + + public InvalidAddressException(String message) { + super(message); + } +} diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/service/model/TimezoneInfo.java b/backend/src/main/java/gov/cdc/usds/simplereport/service/model/TimezoneInfo.java new file mode 100644 index 0000000000..48522ab0cb --- /dev/null +++ b/backend/src/main/java/gov/cdc/usds/simplereport/service/model/TimezoneInfo.java @@ -0,0 +1,10 @@ +package gov.cdc.usds.simplereport.service.model; + +import lombok.Builder; + +@Builder +public class TimezoneInfo { + public final String timezoneCommonName; + public final int utcOffset; + public final boolean obeysDaylightSavings; +} diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhir.java b/backend/src/main/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhir.java index 5343a0d6d8..e835b499e9 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhir.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhir.java @@ -1,6 +1,8 @@ package gov.cdc.usds.simplereport.utils; import static gov.cdc.usds.simplereport.api.converter.FhirConstants.DEFAULT_COUNTRY; +import static gov.cdc.usds.simplereport.utils.DateTimeUtils.DATE_TIME_FORMATTER; +import static gov.cdc.usds.simplereport.utils.DateTimeUtils.convertToZonedDateTime; import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.getIteratorForCsv; import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.getNextRow; import static java.util.Collections.emptyList; @@ -11,6 +13,7 @@ import gov.cdc.usds.simplereport.api.Translators; import gov.cdc.usds.simplereport.api.converter.ConvertToObservationProps; import gov.cdc.usds.simplereport.api.converter.ConvertToPatientProps; +import gov.cdc.usds.simplereport.api.converter.ConvertToSpecimenProps; import gov.cdc.usds.simplereport.api.converter.CreateFhirBundleProps; import gov.cdc.usds.simplereport.api.converter.FhirConverter; import gov.cdc.usds.simplereport.api.model.errors.CsvProcessingException; @@ -22,13 +25,10 @@ import gov.cdc.usds.simplereport.db.model.auxiliary.PhoneType; import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; import gov.cdc.usds.simplereport.db.model.auxiliary.TestCorrectionStatus; -import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; import java.io.InputStream; import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -38,7 +38,6 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -56,12 +55,11 @@ public class BulkUploadResultsToFhir { private static final String ALPHABET_REGEX = "^[a-zA-Z\\s]+$"; - private static final String SNOMED_REGEX = "(^[0-9]{9}$)|(^[0-9]{15}$)"; - private final ResultsUploaderDeviceValidationService resultsUploaderDeviceValidationService; + private static final String SNOMED_REGEX = "(^\\d{9}$)|(^\\d{15}$)"; + private final ResultsUploaderCachingService resultsUploaderCachingService; private final GitProperties gitProperties; private final UUIDGenerator uuidGenerator; private final DateGenerator dateGenerator; - private final ZoneIdGenerator zoneIdGenerator; private final FhirConverter fhirConverter; @Value("${simple-report.processing-mode-code:P}") @@ -111,28 +109,29 @@ public List convertToFhirBundles(InputStream csvStream, UUID orgId) { futureTestEvents.add(future); } - return futureTestEvents.stream() - .map( - future -> { - try { - return future.get(); - } catch (InterruptedException | ExecutionException e) { - log.error("Bulk upload failure to convert to fhir.", e); - Thread.currentThread().interrupt(); - throw new CsvProcessingException("Unable to process file."); - } - }) - .collect(Collectors.toList()); + List bundles = + futureTestEvents.stream() + .map( + future -> { + try { + return future.get(); + } catch (InterruptedException | ExecutionException e) { + log.error("Bulk upload failure to convert to fhir.", e); + Thread.currentThread().interrupt(); + throw new CsvProcessingException("Unable to process file."); + } + }) + .toList(); + + // Clear cache to free memory + resultsUploaderCachingService.clearAddressTimezoneLookupCache(); + + return bundles; } private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { - DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M/d/yyyy[ HH:mm]"); - var testEventId = row.getAccessionNumber().getValue(); - Date testResultDate = - parseRowDateTimeValue(row.getTestResultDate().getValue(), dateTimeFormatter); - var patientAddr = new StreetAddress( row.getPatientStreet().getValue(), @@ -158,6 +157,43 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { row.getOrderingProviderZipCode().getValue(), null); + // Must be zoned because DateTimeType fields on FHIR bundle objects require + // a Date as a specific moment of time. Otherwise, parsing the string to a + // LocalDateTime cannot accurately place it on a timeline because there is + // no way to know if 00:00 refers to 12am ET or 12am PT or 12am UTC, each of + // which is a different moment of time and potentially even a different day + var testResultDate = + convertToZonedDateTime( + row.getTestResultDate().getValue(), resultsUploaderCachingService, testingLabAddr); + + var orderTestDate = + convertToZonedDateTime( + row.getOrderTestDate().getValue(), resultsUploaderCachingService, providerAddr); + + var specimenCollectionDate = + StringUtils.isNotBlank(row.getSpecimenCollectionDate().getValue()) + ? convertToZonedDateTime( + row.getSpecimenCollectionDate().getValue(), + resultsUploaderCachingService, + providerAddr) + : orderTestDate; + + var testingLabSpecimenReceivedDate = + StringUtils.isNotBlank(row.getSpecimenCollectionDate().getValue()) + ? convertToZonedDateTime( + row.getTestingLabSpecimenReceivedDate().getValue(), + resultsUploaderCachingService, + testingLabAddr) + : orderTestDate; + + var dateResultReleased = + StringUtils.isNotBlank(row.getDateResultReleased().getValue()) + ? convertToZonedDateTime( + row.getDateResultReleased().getValue(), + resultsUploaderCachingService, + testingLabAddr) + : testResultDate; + List patientPhoneNumbers = StringUtils.isNotBlank(row.getPatientPhoneNumber().getValue()) ? List.of(new PhoneNumber(PhoneType.MOBILE, row.getPatientPhoneNumber().getValue())) @@ -180,7 +216,7 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { .phoneNumbers(patientPhoneNumbers) .emails(patientEmails) .gender(row.getPatientGender().getValue()) - .dob(LocalDate.parse(row.getPatientDob().getValue(), dateTimeFormatter)) + .dob(LocalDate.parse(row.getPatientDob().getValue(), DATE_TIME_FORMATTER)) .address(patientAddr) .country(DEFAULT_COUNTRY) .race(row.getPatientRace().getValue()) @@ -199,13 +235,13 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { DEFAULT_COUNTRY); Organization orderingFacility = null; - if (row.getOrderingFacilityStreet().getValue() != null - || row.getOrderingFacilityStreet2().getValue() != null - || row.getOrderingFacilityCity().getValue() != null - || row.getOrderingFacilityState().getValue() != null - || row.getOrderingFacilityZipCode().getValue() != null - || row.getOrderingFacilityName().getValue() != null - || row.getOrderingFacilityPhoneNumber().getValue() != null) { + if (StringUtils.isNotEmpty(row.getOrderingFacilityStreet().getValue()) + || StringUtils.isNotEmpty(row.getOrderingFacilityStreet2().getValue()) + || StringUtils.isNotEmpty(row.getOrderingFacilityCity().getValue()) + || StringUtils.isNotEmpty(row.getOrderingFacilityState().getValue()) + || StringUtils.isNotEmpty(row.getOrderingFacilityZipCode().getValue()) + || StringUtils.isNotEmpty(row.getOrderingFacilityName().getValue()) + || StringUtils.isNotEmpty(row.getOrderingFacilityPhoneNumber().getValue())) { var orderingFacilityAddr = new StreetAddress( row.getOrderingFacilityStreet().getValue(), @@ -225,6 +261,10 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { DEFAULT_COUNTRY); } + if (orderingFacility == null) { + orderingFacility = testingLabOrg; + } + var practitioner = fhirConverter.convertToPractitioner( row.getOrderingProviderId().getValue(), @@ -248,9 +288,9 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { var testPerformedCode = row.getTestPerformedCode().getValue(); var modelName = row.getEquipmentModelName().getValue(); var matchingDevice = - resultsUploaderDeviceValidationService + resultsUploaderCachingService .getModelAndTestPerformedCodeToDeviceMap() - .get(ResultsUploaderDeviceValidationService.getMapKey(modelName, testPerformedCode)); + .get(ResultsUploaderCachingService.getMapKey(modelName, testPerformedCode)); if (matchingDevice != null) { List deviceTypeDiseaseEntries = @@ -291,13 +331,16 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { var specimen = fhirConverter.convertToSpecimen( - getSpecimenTypeSnomed(row.getSpecimenType().getValue()), - getDescriptionValue(row.getSpecimenType().getValue()), - null, - null, - uuidGenerator.randomUUID().toString(), - uuidGenerator.randomUUID().toString(), - parseRowDateTimeValue(row.getSpecimenCollectionDate().getValue(), dateTimeFormatter)); + ConvertToSpecimenProps.builder() + .specimenCode(getSpecimenTypeSnomed(row.getSpecimenType().getValue())) + .specimenName(getDescriptionValue(row.getSpecimenType().getValue())) + .collectionCode(null) + .collectionName(null) + .id(uuidGenerator.randomUUID().toString()) + .identifier(uuidGenerator.randomUUID().toString()) + .collectionDate(specimenCollectionDate) + .receivedTime(testingLabSpecimenReceivedDate) + .build()); var observation = List.of( @@ -316,14 +359,15 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { .testkitNameId(testKitNameId) .equipmentUid(equipmentUid) .deviceModel(row.getEquipmentModelName().getValue()) - .issued(testResultDate) + .issued(Date.from(testResultDate.toInstant())) .build())); LocalDate symptomOnsetDate = null; if (row.getIllnessOnsetDate().getValue() != null && !row.getIllnessOnsetDate().getValue().trim().isBlank()) { try { - symptomOnsetDate = LocalDate.parse(row.getIllnessOnsetDate().getValue(), dateTimeFormatter); + symptomOnsetDate = + LocalDate.parse(row.getIllnessOnsetDate().getValue(), DATE_TIME_FORMATTER); } catch (DateTimeParseException e) { // empty values for optional fields come through as empty strings, not null log.error("Unable to parse date from CSV."); @@ -343,7 +387,7 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { ServiceRequest.ServiceRequestStatus.COMPLETED, testOrderLoinc, uuidGenerator.randomUUID().toString(), - parseRowDateTimeValue(row.getOrderTestDate().getValue(), dateTimeFormatter)); + orderTestDate); var diagnosticReport = fhirConverter.convertToDiagnosticReport( @@ -351,7 +395,7 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) { testPerformedCode, testEventId, testResultDate, - dateGenerator.newDate()); + dateResultReleased); return fhirConverter.createFhirBundle( CreateFhirBundleProps.builder() @@ -380,7 +424,7 @@ private String getTestResultSnomed(String input) { private String getSpecimenTypeSnomed(String input) { if (input.matches(ALPHABET_REGEX)) { - return resultsUploaderDeviceValidationService + return resultsUploaderCachingService .getSpecimenTypeNameToSNOMEDMap() .get(input.toLowerCase()); } else if (input.matches(SNOMED_REGEX)) { @@ -415,17 +459,4 @@ private TestCorrectionStatus mapTestResultStatusToSRValue(String input) { return TestCorrectionStatus.ORIGINAL; } } - - private Date parseRowDateTimeValue(String val, DateTimeFormatter dateTimeFormatter) { - if (val == null || val.trim().isBlank()) return null; - LocalDateTime ldt; - TemporalAccessor temporalAccessor = - dateTimeFormatter.parseBest(val, LocalDateTime::from, LocalDate::from); - if (temporalAccessor instanceof LocalDateTime) { - ldt = (LocalDateTime) temporalAccessor; - } else { - ldt = ((LocalDate) temporalAccessor).atStartOfDay(); - } - return Date.from(ldt.atZone(zoneIdGenerator.getSystemZoneId()).toInstant()); - } } diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/utils/DateTimeUtils.java b/backend/src/main/java/gov/cdc/usds/simplereport/utils/DateTimeUtils.java new file mode 100644 index 0000000000..db58ac19e4 --- /dev/null +++ b/backend/src/main/java/gov/cdc/usds/simplereport/utils/DateTimeUtils.java @@ -0,0 +1,123 @@ +package gov.cdc.usds.simplereport.utils; + +import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; +import java.time.DateTimeException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Map; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class DateTimeUtils { + /** + * Noon instead of midnight so that if fallback timezone of US/Eastern is used, value is still + * within the same calendar date (otherwise 6/27 00:00 ET is 6/26 21:00 PT) + */ + private static final int DEFAULT_HOUR = 12; + + public static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormatter.ofPattern("M/d/yyyy[ HH:mm]"); + + public static final String TIMEZONE_SUFFIX_REGEX = + "^(0?[1-9]|1[0-2])/(0?[1-9]|1\\d|2\\d|3[01])/\\d{4}( ([0-1]?\\d|2[0-3]):[0-5]\\d)( \\S+)$"; + + public static final ZoneId FALLBACK_TIMEZONE_ID = ZoneId.of("US/Eastern"); + + private static final ZoneId easternTimeZoneId = ZoneId.of("US/Eastern"); + private static final ZoneId centralTimeZoneId = ZoneId.of("US/Central"); + private static final ZoneId mountainTimeZoneId = ZoneId.of("US/Mountain"); + private static final ZoneId pacificTimeZoneId = ZoneId.of("US/Pacific"); + private static final ZoneId alaskaTimeZoneId = ZoneId.of("US/Alaska"); + private static final ZoneId hawaiiTimeZoneId = ZoneId.of("US/Hawaii"); + private static final ZoneId aleutianTimeZoneId = ZoneId.of("US/Aleutian"); + private static final ZoneId samoaTimeZoneId = ZoneId.of("US/Samoa"); + + public static final Map validTimeZoneIdMap = + Map.ofEntries( + Map.entry("UTC", ZoneOffset.UTC), + Map.entry("UT", ZoneOffset.UTC), + Map.entry("GMT", ZoneOffset.UTC), + Map.entry("Z", ZoneOffset.UTC), + Map.entry("ET", easternTimeZoneId), + Map.entry("EST", easternTimeZoneId), + Map.entry("EDT", easternTimeZoneId), + Map.entry("CT", centralTimeZoneId), + Map.entry("CST", centralTimeZoneId), + Map.entry("CDT", centralTimeZoneId), + Map.entry("MT", mountainTimeZoneId), + Map.entry("MST", mountainTimeZoneId), + Map.entry("MDT", mountainTimeZoneId), + Map.entry("PT", pacificTimeZoneId), + Map.entry("PST", pacificTimeZoneId), + Map.entry("PDT", pacificTimeZoneId), + Map.entry("AK", alaskaTimeZoneId), + Map.entry("AKDT", alaskaTimeZoneId), + Map.entry("AKST", alaskaTimeZoneId), + Map.entry("HI", hawaiiTimeZoneId), + Map.entry("HST", hawaiiTimeZoneId), + Map.entry("HDT", aleutianTimeZoneId), + Map.entry("AS", samoaTimeZoneId), + Map.entry("ASM", samoaTimeZoneId), + Map.entry("SST", samoaTimeZoneId)); + + public static ZonedDateTime convertToZonedDateTime( + String dateString, + ResultsUploaderCachingService resultsUploaderCachingService, + StreetAddress addressForTimezone) { + ZoneId zoneId; + LocalDateTime localDateTime; + + // If user provided timezone code in datetime field + if (hasTimezoneSubstring(dateString)) { + var timezoneCode = dateString.substring(dateString.lastIndexOf(" ")).trim(); + try { + zoneId = parseZoneId(timezoneCode); + } catch (DateTimeException e) { + zoneId = FALLBACK_TIMEZONE_ID; + } + } else { // Otherwise try to get timezone by address + zoneId = resultsUploaderCachingService.getZoneIdByAddress(addressForTimezone); + // If that fails, use fallback + if (zoneId == null) { + zoneId = FALLBACK_TIMEZONE_ID; + } + } + + localDateTime = parseLocalDateTime(dateString, DATE_TIME_FORMATTER); + return ZonedDateTime.of(localDateTime, zoneId); + } + + public static boolean hasTimezoneSubstring(String value) { + return value.matches(TIMEZONE_SUFFIX_REGEX); + } + + public static ZoneId parseZoneId(String timezoneCode) { + if (validTimeZoneIdMap.containsKey(timezoneCode.toUpperCase())) { + return validTimeZoneIdMap.get(timezoneCode.toUpperCase()); + } + return ZoneId.of(timezoneCode); + } + + public static LocalDateTime parseLocalDateTime( + String value, DateTimeFormatter dateTimeFormatter) { + String dateTimeString = value; + if (hasTimezoneSubstring(value)) { + dateTimeString = dateTimeString.substring(0, value.lastIndexOf(' ')).trim(); + } + LocalDateTime localDateTime; + var temporalAccessor = + dateTimeFormatter.parseBest(dateTimeString, LocalDateTime::from, LocalDate::from); + if (temporalAccessor instanceof LocalDateTime) { + localDateTime = (LocalDateTime) temporalAccessor; + } else { // example "6/28/2023" + localDateTime = ((LocalDate) temporalAccessor).atTime(DEFAULT_HOUR, 0, 0); + } + return localDateTime; + } +} diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/utils/FhirDateTimeUtil.java b/backend/src/main/java/gov/cdc/usds/simplereport/utils/FhirDateTimeUtil.java deleted file mode 100644 index cdb5586a67..0000000000 --- a/backend/src/main/java/gov/cdc/usds/simplereport/utils/FhirDateTimeUtil.java +++ /dev/null @@ -1,24 +0,0 @@ -package gov.cdc.usds.simplereport.utils; - -import org.hl7.fhir.r4.model.BaseDateTimeType; -import org.springframework.stereotype.Component; - -/** A utility class for handling data and time objects in FHIR. */ -@Component -public class FhirDateTimeUtil { - - /** - * Returns an unchanged value of the {@code BaseDateTimeType} argument. Used for mocking during - * tests to convert the value from the system default timezone to UTC so that the value can be - * properly compared with UTC values in the expected test data. - * - *

By default, any object that inherits from {@code org.hl7.fhir.r4.model.BaseDateTimeType} - * will use the system's default timezone when setting its value. This value can be converted to - * UTC by using {@code BaseDateTimeType.setTimeZoneZulu}. - * - * @see BaseDateTimeType - */ - public BaseDateTimeType getBaseDateTimeType(BaseDateTimeType baseDateTimeType) { - return baseDateTimeType; - } -} diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/utils/ZoneIdGenerator.java b/backend/src/main/java/gov/cdc/usds/simplereport/utils/ZoneIdGenerator.java deleted file mode 100644 index dd00eb8100..0000000000 --- a/backend/src/main/java/gov/cdc/usds/simplereport/utils/ZoneIdGenerator.java +++ /dev/null @@ -1,28 +0,0 @@ -package gov.cdc.usds.simplereport.utils; - -import java.time.ZoneId; -import org.springframework.stereotype.Component; - -/** - * A utility class for generating {@code ZoneId} objects. It is primarily used to enable mocking of - * {@code ZoneId} objects to allow for predictable timezones during testing. You can inject {@code - * ZoneIdGenerator} in Spring classes that require the system's default {@code ZoneId}. - * - *

This generator avoids the limitations of mocking static methods since Mockito explicitly - * advises against mocking static methods from the standard library such as {@code - * ZoneId.systemDefault}. - * - * @see ZoneId - */ -@Component -public class ZoneIdGenerator { - - /** - * Generates a new ZoneId object representing the system's default zone id. - * - * @return a new Date object representing the current date and time - */ - public ZoneId getSystemZoneId() { - return ZoneId.systemDefault(); - } -} diff --git a/backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java b/backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java index ca3aff6aed..62a57506d6 100644 --- a/backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java +++ b/backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java @@ -4,6 +4,8 @@ import static gov.cdc.usds.simplereport.api.Translators.COUNTRY_CODES; import static gov.cdc.usds.simplereport.api.Translators.PAST_DATE_FLEXIBLE_FORMATTER; import static gov.cdc.usds.simplereport.api.Translators.STATE_CODES; +import static gov.cdc.usds.simplereport.utils.DateTimeUtils.TIMEZONE_SUFFIX_REGEX; +import static gov.cdc.usds.simplereport.utils.DateTimeUtils.validTimeZoneIdMap; import com.fasterxml.jackson.databind.MappingIterator; import com.fasterxml.jackson.databind.RuntimeJsonMappingException; @@ -18,6 +20,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.time.ZoneId; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collections; @@ -42,9 +45,16 @@ public class CsvValidatorUtils { private static final String DATE_REGEX = "^(0{0,1}[1-9]|1[0-2])\\/(0{0,1}[1-9]|1\\d|2\\d|3[01])\\/\\d{4}$"; - // MM/DD/YYYY HH:mm, MM/DD/YYYY H:mm, M/D/YYYY HH:mm OR M/D/YYYY H:mm + /** + * Validates MM/DD/YYYY HH:mm, MM/DD/YYYY H:mm, M/D/YYYY HH:mm OR M/D/YYYY H:mm + * + *

Optional timezone code suffix which is checked as a valid timezone separately + * + * @see gov.cdc.usds.simplereport.utils.DateTimeUtils + */ private static final String DATE_TIME_REGEX = - "^(0{0,1}[1-9]|1[0-2])\\/(0{0,1}[1-9]|1\\d|2\\d|3[01])\\/\\d{4}( ([0-1]?[0-9]|2[0-3]):[0-5][0-9])?$"; + "^(0{0,1}[1-9]|1[0-2])\\/(0{0,1}[1-9]|1\\d|2\\d|3[01])\\/\\d{4}( ([0-1]?[0-9]|2[0-3]):[0-5][0-9]( \\S+)?)?$"; + private static final String EMAIL_REGEX = "^[\\w-\\.]+@([\\w-]+\\.)+[\\w-]{2,4}$"; private static final String SNOMED_REGEX = "(^[0-9]{9}$)|(^[0-9]{15}$)"; private static final String CLIA_REGEX = "^[A-Za-z0-9]{2}[Dd][A-Za-z0-9]{7}$"; @@ -284,7 +294,31 @@ public static List validateDateFormat(ValueOrError input) { } public static List validateDateTime(ValueOrError input) { - return validateRegex(input, DATE_TIME_REGEX); + List errors = new ArrayList<>(validateRegex(input, DATE_TIME_REGEX)); + if (input.getValue() != null + && errors.isEmpty() + && input.getValue().matches(TIMEZONE_SUFFIX_REGEX)) { + errors.addAll(validateDateTimeZoneCode(input)); + } + return errors; + } + + public static List validateDateTimeZoneCode(ValueOrError input) { + List errors = new ArrayList<>(); + String value = input.getValue(); + String timezoneCode = value.substring(value.lastIndexOf(' ')).trim(); + if (!ZoneId.getAvailableZoneIds().contains(timezoneCode) + && !validTimeZoneIdMap.containsKey(timezoneCode.toUpperCase())) { + errors.add( + FeedbackMessage.builder() + .scope(ITEM_SCOPE) + .fieldHeader(input.getHeader()) + .message(getInValidValueErrorMessage(input.getValue(), input.getHeader())) + .errorType(FeedbackMessage.ErrorType.INVALID_DATA) + .fieldRequired(false) + .build()); + } + return errors; } public static List validateEmail(ValueOrError input) { diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/api/converter/FhirConverterTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/api/converter/FhirConverterTest.java index f4d93058c0..f0477ea71f 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/api/converter/FhirConverterTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/api/converter/FhirConverterTest.java @@ -7,6 +7,7 @@ import static org.mockito.Mockito.when; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.model.api.TemporalPrecisionEnum; import ca.uhn.fhir.parser.IParser; import gov.cdc.usds.simplereport.db.model.DeviceType; import gov.cdc.usds.simplereport.db.model.DeviceTypeDisease; @@ -34,9 +35,9 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDate; -import java.time.OffsetDateTime; import java.time.ZoneId; -import java.time.format.DateTimeFormatter; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; @@ -45,6 +46,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TimeZone; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -60,6 +62,7 @@ import org.hl7.fhir.r4.model.DiagnosticReport; import org.hl7.fhir.r4.model.DiagnosticReport.DiagnosticReportStatus; import org.hl7.fhir.r4.model.Enumerations.AdministrativeGender; +import org.hl7.fhir.r4.model.InstantType; import org.hl7.fhir.r4.model.Observation; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.Practitioner; @@ -93,6 +96,7 @@ class FhirConverterTest { private static final String tribalSystemUrl = "http://terminology.hl7.org/CodeSystem/v3-TribalEntityUS"; public static final String snomedCode = "http://snomed.info/sct"; + public static final ZoneId DEFAULT_TIME_ZONE_ID = ZoneId.of("US/Eastern"); final FhirContext ctx = FhirContext.forR4(); final IParser parser = ctx.newJsonParser(); @@ -509,15 +513,22 @@ void convertToDevice_DeviceType_matchesJson() throws IOException { @Test void convertToSpecimen_Strings_valid() { + var collectionDate = + ZonedDateTime.ofInstant(Instant.parse("2023-06-22T16:38:00.000Z"), DEFAULT_TIME_ZONE_ID); + var receivedTime = ZonedDateTime.of(2023, 6, 23, 12, 0, 0, 0, ZoneId.of("US/Eastern")); + var actual = fhirConverter.convertToSpecimen( - "258500001", - "Nasopharyngeal swab", - "53342003", - "Internal nose structure (body structure)", - "id-123", - "uuid-123", - Date.from(Instant.parse("2023-06-22T16:38:00Z"))); + ConvertToSpecimenProps.builder() + .specimenCode("258500001") + .specimenName("Nasopharyngeal swab") + .collectionCode("53342003") + .collectionName("Internal nose structure (body structure)") + .id("id-123") + .identifier("uuid-123") + .collectionDate(collectionDate) + .receivedTime(receivedTime) + .build()); assertThat(actual.getId()).isEqualTo("id-123"); assertThat(actual.getIdentifierFirstRep().getValue()).isEqualTo("uuid-123"); @@ -539,7 +550,7 @@ void convertToSpecimen_Strings_valid() { @Test void convertToSpecimen_Strings_null() { - var actual = fhirConverter.convertToSpecimen(null, null, null, null, null, null, null); + var actual = fhirConverter.convertToSpecimen(ConvertToSpecimenProps.builder().build()); assertThat(actual.getId()).isNull(); assertThat(actual.getType().getText()).isNull(); @@ -547,6 +558,7 @@ void convertToSpecimen_Strings_null() { assertThat(actual.getCollection().getBodySite().getText()).isNull(); assertThat(actual.getCollection().getBodySite().getCoding()).isEmpty(); assertThat(actual.getCollection().getCollected()).isNull(); + assertThat(actual.getReceivedTime()).isNull(); } @Test @@ -560,9 +572,13 @@ void convertToSpecimen_SpecimenType_valid() { var internalId = UUID.randomUUID(); ReflectionTestUtils.setField(specimenType, "internalId", internalId); + var collectionDate = + ZonedDateTime.ofInstant(Instant.parse("2023-06-22T13:16:00.000Z"), DEFAULT_TIME_ZONE_ID); + var receivedTime = ZonedDateTime.of(2023, 6, 23, 12, 0, 0, 0, ZoneId.of("US/Eastern")); + var actual = fhirConverter.convertToSpecimen( - specimenType, UUID.randomUUID(), Date.from(Instant.parse("2023-06-22T13:16:00.00Z"))); + specimenType, UUID.randomUUID(), collectionDate, receivedTime); assertThat(actual.getId()).isEqualTo(internalId.toString()); assertThat(actual.getType().getCoding()).hasSize(1); @@ -579,6 +595,8 @@ void convertToSpecimen_SpecimenType_valid() { .isEqualTo("Internal nose structure (body structure)"); assertThat(((DateTimeType) actual.getCollection().getCollected()).getValue()) .isEqualTo("2023-06-22T13:16:00.00Z"); + assertThat(actual.getReceivedTimeElement().getValueAsString()) + .isEqualTo("2023-06-23T12:00:00-04:00"); } @Test @@ -589,7 +607,11 @@ void convertToSpecimen_SpecimenType_matchesJson() throws IOException { var actual = fhirConverter.convertToSpecimen( - specimenType, UUID.randomUUID(), Date.from(Instant.parse("2023-06-22T15:35:00.00Z"))); + specimenType, + UUID.randomUUID(), + ZonedDateTime.ofInstant( + Instant.parse("2023-06-22T15:35:00.000Z"), DEFAULT_TIME_ZONE_ID), + ZonedDateTime.of(2023, 6, 23, 12, 0, 0, 0, ZoneId.of("US/Eastern"))); String actualSerialized = parser.encodeResourceToString(actual); var expectedSerialized = @@ -816,8 +838,8 @@ void convertToObservation_Result_matchesJson() throws IOException { Objects.requireNonNull( getClass().getClassLoader().getResourceAsStream("fhir/observationFlu.json")), StandardCharsets.UTF_8); - JSONAssert.assertEquals(covidSerialized, expectedSerialized1, true); - JSONAssert.assertEquals(fluSerialized, expectedSerialized2, true); + JSONAssert.assertEquals(expectedSerialized1, covidSerialized, true); + JSONAssert.assertEquals(expectedSerialized2, fluSerialized, true); } @Test @@ -952,9 +974,7 @@ void convertToDiagnosticReport_TestEvent_matchesJson() throws IOException { ReflectionTestUtils.setField( testEvent, "deviceType", TestDataBuilder.createDeviceTypeForMultiplex()); - var actual = - fhirConverter.convertToDiagnosticReport( - testEvent, Date.from(Instant.parse("2023-06-22T11:46:00.00Z"))); + var actual = fhirConverter.convertToDiagnosticReport(testEvent, date); String actualSerialized = parser.encodeResourceToString(actual); var expectedSerialized = @@ -964,8 +984,10 @@ void convertToDiagnosticReport_TestEvent_matchesJson() throws IOException { StandardCharsets.UTF_8); expectedSerialized = expectedSerialized.replace( - "$EFFECTIVE_DATE_TIME_TESTED", - new DateTimeType(date).setTimeZoneZulu(true).getValueAsString()); + "$CURRENT_DATE_TIMEZONE", + new DateTimeType( + date, TemporalPrecisionEnum.SECOND, TimeZone.getTimeZone(ZoneOffset.UTC)) + .getValueAsString()); JSONAssert.assertEquals(expectedSerialized, actualSerialized, true); } @@ -973,18 +995,25 @@ void convertToDiagnosticReport_TestEvent_matchesJson() throws IOException { @Test void convertToDiagnosticReport_Strings_valid() { var date = new Date(); + var zonedDateTime = ZonedDateTime.ofInstant(date.toInstant(), DEFAULT_TIME_ZONE_ID); + var expectedDateTimeType = + (DateTimeType) + new DateTimeType( + Date.from(zonedDateTime.toInstant()), + TemporalPrecisionEnum.SECOND, + TimeZone.getTimeZone(DEFAULT_TIME_ZONE_ID)); var actual = fhirConverter.convertToDiagnosticReport( - DiagnosticReportStatus.FINAL, "95422-2", "id-123", date, date); + DiagnosticReportStatus.FINAL, "95422-2", "id-123", zonedDateTime, zonedDateTime); assertThat(actual.getId()).isEqualTo("id-123"); assertThat(actual.getStatus()).isEqualTo(DiagnosticReportStatus.FINAL); assertThat(actual.getCode().getCoding()).hasSize(1); assertThat(actual.getCode().getCodingFirstRep().getSystem()).isEqualTo("http://loinc.org"); assertThat(actual.getCode().getCodingFirstRep().getCode()).isEqualTo("95422-2"); - assertThat(((DateTimeType) actual.getEffective()).getValueAsString()) - .isEqualTo(new DateTimeType(date).setTimeZoneZulu(true).getValueAsString()); - assertThat((actual.getIssued())).isEqualTo(date); + assertThat(((DateTimeType) actual.getEffective()).getAsV3()) + .isEqualTo(expectedDateTimeType.getAsV3()); + assertThat(actual.getIssued()).isEqualTo(date); } @Test @@ -1001,7 +1030,9 @@ void convertToServiceRequest_TestOrder_valid() { var testOrder = TestDataBuilder.createTestOrderWithMultiplexDevice(); var actual = fhirConverter.convertToServiceRequest( - testOrder, Date.from(Instant.parse("2023-06-22T10:30:00.00Z"))); + testOrder, + ZonedDateTime.ofInstant( + Instant.parse("2023-06-22T10:30:00.000Z"), DEFAULT_TIME_ZONE_ID)); assertThat(actual.getStatus()).isEqualTo(ServiceRequestStatus.ACTIVE); assertThat(actual.getIntent()).isEqualTo(ServiceRequestIntent.ORDER); @@ -1048,7 +1079,8 @@ void convertToServiceRequest_TestOrder_complete() { new TestOrder( TestDataBuilder.createEmptyPerson(true), TestDataBuilder.createEmptyFacility(true)); testOrder.markComplete(); - var actual = fhirConverter.convertToServiceRequest(testOrder, new Date()); + var actual = + fhirConverter.convertToServiceRequest(testOrder, ZonedDateTime.now(DEFAULT_TIME_ZONE_ID)); assertThat(actual.getStatus()).isEqualTo(ServiceRequestStatus.COMPLETED); } @@ -1059,7 +1091,8 @@ void convertToServiceRequest_TestOrder_cancelled() { new TestOrder( TestDataBuilder.createEmptyPerson(true), TestDataBuilder.createEmptyFacility(true)); testOrder.cancelOrder(); - var actual = fhirConverter.convertToServiceRequest(testOrder, new Date()); + var actual = + fhirConverter.convertToServiceRequest(testOrder, ZonedDateTime.now(DEFAULT_TIME_ZONE_ID)); assertThat(actual.getStatus()).isEqualTo(ServiceRequestStatus.REVOKED); } @@ -1070,7 +1103,8 @@ void convertToServiceRequest_TestOrder_nullDeviceType() { new TestOrder( TestDataBuilder.createEmptyPerson(true), TestDataBuilder.createEmptyFacility(false)); testOrder.cancelOrder(); - var actual = fhirConverter.convertToServiceRequest(testOrder, new Date()); + var actual = + fhirConverter.convertToServiceRequest(testOrder, ZonedDateTime.now(DEFAULT_TIME_ZONE_ID)); assertThat(actual.getCode().getCoding()).isEmpty(); } @@ -1082,7 +1116,8 @@ void convertToServiceRequest_Strings_valid() { ServiceRequestStatus.COMPLETED, "94533-7", "id-123", - Date.from(Instant.parse("2023-06-22T10:35:00.00Z"))); + ZonedDateTime.ofInstant( + Instant.parse("2023-06-22T10:35:00.000Z"), DEFAULT_TIME_ZONE_ID)); assertThat(actual.getId()).isEqualTo("id-123"); assertThat(actual.getStatus()).isEqualTo(ServiceRequestStatus.COMPLETED); assertThat(actual.getCode().getCoding()).hasSize(1); @@ -1101,7 +1136,9 @@ void convertToServiceRequest_Strings_valid() { @Test void convertToServiceRequest_Strings_null() { - var actual = fhirConverter.convertToServiceRequest(null, null, null, new Date()); + var actual = + fhirConverter.convertToServiceRequest( + null, null, null, ZonedDateTime.now(DEFAULT_TIME_ZONE_ID)); assertThat(actual.getId()).isNull(); assertThat(actual.getStatus()).isNull(); assertThat(actual.getCode().getCoding()).isEmpty(); @@ -1116,7 +1153,9 @@ void convertToServiceRequest_TestOrder_matchesJson() throws IOException { var actual = fhirConverter.convertToServiceRequest( - testOrder, Date.from(Instant.parse("2023-06-22T10:35:00.00Z"))); + testOrder, + ZonedDateTime.ofInstant( + Instant.parse("2023-06-22T10:35:00.000Z"), DEFAULT_TIME_ZONE_ID)); String actualSerialized = parser.encodeResourceToString(actual); var expectedSerialized = @@ -1354,8 +1393,7 @@ void createFhirBundle_TestEvent_valid() { .getSupportingInfo().stream() .allMatch(r -> r.getReference().contains("Observation/"))) .isTrue(); - assertThat(((ServiceRequest) serviceRequestEntry.getResource()).getSupportingInfo().size()) - .isEqualTo(2); + assertThat(((ServiceRequest) serviceRequestEntry.getResource()).getSupportingInfo()).hasSize(2); var diagnosticReportEntry = actual.getEntry().stream() @@ -1476,7 +1514,6 @@ void createFhirBundle_TestEvent_matchesJson() throws IOException { "testkitNameId3", "95422-2")); var date = new Date(); - var dateTested = new Date(); ReflectionTestUtils.setField(provider, "internalId", providerId); ReflectionTestUtils.setField(facility, "internalId", facilityId); ReflectionTestUtils.setField(person, "internalId", personId); @@ -1489,7 +1526,7 @@ void createFhirBundle_TestEvent_matchesJson() throws IOException { ReflectionTestUtils.setField(fluBResult, "internalId", fluBResultId); ReflectionTestUtils.setField(testOrder, "internalId", testOrderId); ReflectionTestUtils.setField(testEvent, "internalId", testEventId); - ReflectionTestUtils.setField(testEvent, "createdAt", dateTested); + ReflectionTestUtils.setField(testEvent, "createdAt", date); ReflectionTestUtils.setField( person, "phoneNumbers", List.of(new PhoneNumber(PhoneType.LANDLINE, "7735551234"))); @@ -1511,22 +1548,17 @@ void createFhirBundle_TestEvent_matchesJson() throws IOException { getClass().getClassLoader().getResourceAsStream("fhir/bundle.json")), StandardCharsets.UTF_8); + var expectedCurrentDateTimezone = + new DateTimeType(date, TemporalPrecisionEnum.SECOND, TimeZone.getTimeZone("UTC")) + .getValueAsString(); + var expectedCurrentDateZulu = new InstantType(date).setTimeZoneZulu(true).getValueAsString(); + expectedSerialized = expectedSerialized.replace("$MESSAGE_HEADER_ID", messageHeaderId); expectedSerialized = expectedSerialized.replace("$PRACTITIONER_ROLE_ID", practitionerRoleId); expectedSerialized = expectedSerialized.replace("$PROVENANCE_ID", provenanceId); expectedSerialized = - expectedSerialized.replace( - "$EFFECTIVE_DATE_TIME_TESTED", - new DateTimeType(dateTested).setTimeZoneZulu(true).getValueAsString()); - expectedSerialized = - expectedSerialized.replace( - "$CURRENT_DATE_TZ", - OffsetDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()) - .format(DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSxxx"))); - expectedSerialized = - expectedSerialized.replace( - "$CURRENT_DATE_ZULU", - new DateTimeType(dateTested).setTimeZoneZulu(true).getValueAsString()); + expectedSerialized.replace("$CURRENT_DATE_TIMEZONE", expectedCurrentDateTimezone); + expectedSerialized = expectedSerialized.replace("$CURRENT_DATE_ZULU", expectedCurrentDateZulu); expectedSerialized = expectedSerialized.replace( "$SPECIMEN_IDENTIFIER", diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/service/AddressValidationServiceTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/service/AddressValidationServiceTest.java index a6b28ffac8..69c4192849 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/service/AddressValidationServiceTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/service/AddressValidationServiceTest.java @@ -1,6 +1,7 @@ package gov.cdc.usds.simplereport.service; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; @@ -12,7 +13,10 @@ import com.smartystreets.api.us_street.Lookup; import com.smartystreets.api.us_street.Metadata; import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; +import gov.cdc.usds.simplereport.service.errors.InvalidAddressException; +import gov.cdc.usds.simplereport.service.model.TimezoneInfo; import java.io.IOException; +import java.time.ZoneId; import java.util.ArrayList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -64,6 +68,52 @@ void doesNotCorrectStreet() throws SmartyException, IOException { assertEquals("User entered street", address.getStreetOne()); } + @Test + void returnsCorrectZoneIdByLookup() { + ArrayList results = new ArrayList(); + results.add(getMockTimeZoneInfoResult()); + Lookup lookup = mock(Lookup.class); + when(lookup.getResult()).thenReturn(results); + + ZoneId zoneId = s.getZoneIdByLookup(lookup); + + assertEquals(zoneId, ZoneId.of("US/Central")); + } + + @Test + void returnsCorrectTimeZoneInfoByLookup() { + ArrayList results = new ArrayList<>(); + results.add(getMockResult()); + Lookup lookup = mock(Lookup.class); + when(lookup.getResult()).thenReturn(results); + + TimezoneInfo timeZoneInfo = s.getTimezoneInfoByLookup(lookup); + + assertEquals(results.get(0).getMetadata().getTimeZone(), timeZoneInfo.timezoneCommonName); + assertEquals(results.get(0).getMetadata().getUtcOffset(), timeZoneInfo.utcOffset); + assertEquals(results.get(0).getMetadata().obeysDst(), timeZoneInfo.obeysDaylightSavings); + } + + @Test + void throwsInvalidAddressExceptionOnEmptyResults() { + ArrayList results = new ArrayList<>(); + Lookup lookup = mock(Lookup.class); + when(lookup.getResult()).thenReturn(results); + + assertThrows(InvalidAddressException.class, () -> s.getTimezoneInfoByLookup(lookup)); + } + + private Candidate getMockTimeZoneInfoResult() { + Metadata metadata = mock(Metadata.class); + when(metadata.getTimeZone()).thenReturn("Central"); + when(metadata.getUtcOffset()).thenReturn(-5.0); + when(metadata.obeysDst()).thenReturn(true); + + Candidate result = mock(Candidate.class); + when(result.getMetadata()).thenReturn(metadata); + return result; + } + private Candidate getMockResult() { Metadata metadata = mock(Metadata.class); when(metadata.getCountyName()).thenReturn("District of Columbia"); diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/service/ResultsUploaderDeviceValidationServiceTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingServiceTest.java similarity index 78% rename from backend/src/test/java/gov/cdc/usds/simplereport/service/ResultsUploaderDeviceValidationServiceTest.java rename to backend/src/test/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingServiceTest.java index b08c1bd8b3..f788365eb5 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/service/ResultsUploaderDeviceValidationServiceTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingServiceTest.java @@ -1,13 +1,17 @@ package gov.cdc.usds.simplereport.service; -import static gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService.getMapKey; +import static gov.cdc.usds.simplereport.service.ResultsUploaderCachingService.getMapKey; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import gov.cdc.usds.simplereport.api.model.CreateDeviceType; import gov.cdc.usds.simplereport.api.model.CreateSpecimenType; import gov.cdc.usds.simplereport.api.model.SupportedDiseaseTestPerformedInput; import gov.cdc.usds.simplereport.db.model.DeviceType; import gov.cdc.usds.simplereport.db.model.SpecimenType; +import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; import gov.cdc.usds.simplereport.test_util.SliceTestConfiguration; import java.util.Arrays; import java.util.List; @@ -17,14 +21,15 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; @SliceTestConfiguration.WithSimpleReportSiteAdminUser -class ResultsUploaderDeviceValidationServiceTest - extends BaseServiceTest { +class ResultsUploaderCachingServiceTest extends BaseServiceTest { + @MockBean private AddressValidationService addressValidationService; @Autowired private SpecimenTypeService specimenTypeService; @Autowired private DeviceTypeService deviceTypeService; @Autowired private DiseaseService diseaseService; - @Autowired private ResultsUploaderDeviceValidationService sut; + @Autowired private ResultsUploaderCachingService sut; private final Random random = new Random(); @Test @@ -47,6 +52,15 @@ void builds_deviceModel_testPerformedCode_set_test() { .contains(getMapKey("GenBody COVID-19 Ag", "97097-0")); } + @Test + void addressValidation_cachesIdenticalAddresses() { + var address = new StreetAddress("123 Main St", null, "Buffalo", "New York", "14202", "Erie"); + sut.getZoneIdByAddress(address); + sut.getZoneIdByAddress(address); + sut.getZoneIdByAddress(address); + verify(addressValidationService, times(1)).getZoneIdByAddress(any()); + } + protected void createDeviceType(String model, String... testPerformedCodes) { SpecimenType specimenType = specimenTypeService.createSpecimenType( diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/service/TestResultUploadServiceTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/service/TestResultUploadServiceTest.java index 7e307d27f9..8415c9e5ae 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/service/TestResultUploadServiceTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/service/TestResultUploadServiceTest.java @@ -74,7 +74,7 @@ class TestResultUploadServiceTest extends BaseServiceTest csvFileValidatorMock; @Mock private BulkUploadResultsToFhir bulkUploadFhirConverterMock; @@ -399,7 +399,7 @@ void uploadService_processCsv_translatesSpecimenNameToSNOMED() { response.setErrors(new FeedbackMessage[] {}); response.setWarnings(new FeedbackMessage[] {}); when(dataHubMock.uploadCSV(any())).thenReturn(response); - when(resultsUploaderDeviceValidationServiceMock.getSpecimenTypeNameToSNOMEDMap()) + when(resultsUploaderCachingServiceMock.getSpecimenTypeNameToSNOMEDMap()) .thenReturn(Map.of("nasal swab", "000111222")); // WHEN diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/test_util/JsonTestUtils.java b/backend/src/test/java/gov/cdc/usds/simplereport/test_util/JsonTestUtils.java index 73697694a8..245462d94b 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/test_util/JsonTestUtils.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/test_util/JsonTestUtils.java @@ -5,7 +5,9 @@ import com.fasterxml.jackson.databind.JsonNode; import java.util.Iterator; import java.util.Set; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class JsonTestUtils { /** @@ -75,7 +77,8 @@ public static void assertJsonNodesEqual( fieldPath + "[" + i + "]"); } } else { // Otherwise assert values are equal - assertThat(actualFieldValue.toString()).isEqualTo(expectedFieldValue.toString()); + log.debug("Comparing " + fieldPath); + assertThat(actualFieldValue.toString()).hasToString(expectedFieldValue.toString()); } } } diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhirTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhirTest.java index c6d532904f..bac091b7d3 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhirTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhirTest.java @@ -3,6 +3,7 @@ import static gov.cdc.usds.simplereport.test_util.JsonTestUtils.assertJsonNodesEqual; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -11,8 +12,9 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import com.fasterxml.jackson.databind.ObjectMapper; +import com.smartystreets.api.exceptions.SmartyException; import gov.cdc.usds.simplereport.api.converter.FhirConverter; -import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; import gov.cdc.usds.simplereport.test_util.TestDataBuilder; import java.io.BufferedReader; import java.io.IOException; @@ -24,14 +26,16 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; -import org.hl7.fhir.r4.model.BaseDateTimeType; import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.DiagnosticReport; import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Organization; +import org.hl7.fhir.r4.model.Specimen; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.stubbing.Answer; @@ -40,46 +44,33 @@ @ExtendWith(MockitoExtension.class) public class BulkUploadResultsToFhirTest { private static GitProperties gitProperties; - private static ResultsUploaderDeviceValidationService resultsUploaderDeviceValidationService; + private static ResultsUploaderCachingService resultsUploaderCachingService; private static final Instant commitTime = (new Date(1675891986000L)).toInstant(); final FhirContext ctx = FhirContext.forR4(); final IParser parser = ctx.newJsonParser(); private final UUIDGenerator uuidGenerator = new UUIDGenerator(); private final DateGenerator dateGenerator = new DateGenerator(); - private static ZoneIdGenerator zoneIdGenerator; - private static FhirDateTimeUtil fhirDateTimeUtil; BulkUploadResultsToFhir sut; @BeforeAll - public static void init() { + public static void init() throws SmartyException, IOException, InterruptedException { gitProperties = mock(GitProperties.class); - zoneIdGenerator = mock(ZoneIdGenerator.class); - fhirDateTimeUtil = mock(FhirDateTimeUtil.class); when(gitProperties.getCommitTime()).thenReturn(commitTime); when(gitProperties.getShortCommitId()).thenReturn("short-commit-id"); - when(zoneIdGenerator.getSystemZoneId()).thenReturn(ZoneId.of("UTC")); - when(fhirDateTimeUtil.getBaseDateTimeType(Mockito.any(BaseDateTimeType.class))) - .thenAnswer( - (Answer) - invocation -> { - BaseDateTimeType localBaseDateTimeType = invocation.getArgument(0); - return localBaseDateTimeType.setTimeZoneZulu(true); - }); } @BeforeEach public void beforeEach() { - resultsUploaderDeviceValidationService = mock(ResultsUploaderDeviceValidationService.class); - FhirConverter fhirConverter = new FhirConverter(uuidGenerator, fhirDateTimeUtil); + resultsUploaderCachingService = mock(ResultsUploaderCachingService.class); + FhirConverter fhirConverter = new FhirConverter(uuidGenerator); sut = new BulkUploadResultsToFhir( - resultsUploaderDeviceValidationService, + resultsUploaderCachingService, gitProperties, uuidGenerator, dateGenerator, - zoneIdGenerator, fhirConverter); } @@ -95,8 +86,7 @@ void convertExistingCsv_success() { .map(Bundle.BundleEntryComponent::getFullUrl) .collect(Collectors.toList()); - verify(resultsUploaderDeviceValidationService, times(1)) - .getModelAndTestPerformedCodeToDeviceMap(); + verify(resultsUploaderCachingService, times(1)).getModelAndTestPerformedCodeToDeviceMap(); assertThat(serializedBundles).hasSize(1); assertThat(deserializedBundle.getEntry()).hasSize(14); assertThat(resourceUrls).hasSize(14); @@ -140,7 +130,7 @@ private InputStream loadCsv(String csvFile) { @Test void convertExistingCsv_observationValuesPresent() { InputStream input = loadCsv("testResultUpload/test-results-upload-valid.csv"); - when(resultsUploaderDeviceValidationService.getModelAndTestPerformedCodeToDeviceMap()) + when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap()) .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceTypeForBulkUpload())); var serializedBundles = sut.convertToFhirBundles(input, UUID.randomUUID()); @@ -180,7 +170,7 @@ private String readJsonStream(InputStream is) throws IOException { @Test void convertExistingCsv_matchesFhirJson() throws IOException { - // Configure mocks for random UUIDs and Date timestamp + // Mock random UUIDs var mockedUUIDGenerator = mock(UUIDGenerator.class); when(mockedUUIDGenerator.randomUUID()) .thenAnswer( @@ -205,7 +195,7 @@ public UUID answer(InvocationOnMock invocation) { } }); - // Construct UTC date object + // Mock constructed UTC date object String dateString = "2023-05-24T19:33:06.472Z"; Instant instant = Instant.parse(dateString); Date date = Date.from(instant); @@ -213,17 +203,20 @@ public UUID answer(InvocationOnMock invocation) { var mockedDateGenerator = mock(DateGenerator.class); when(mockedDateGenerator.newDate()).thenReturn(date); + // Mock timezone retrieval from address + var mockedZoneId = ZoneId.of("US/Central"); + when(resultsUploaderCachingService.getZoneIdByAddress(any())).thenReturn(mockedZoneId); + + when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap()) + .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceTypeForBulkUpload())); + sut = new BulkUploadResultsToFhir( - resultsUploaderDeviceValidationService, + resultsUploaderCachingService, gitProperties, mockedUUIDGenerator, mockedDateGenerator, - zoneIdGenerator, - new FhirConverter(mockedUUIDGenerator, fhirDateTimeUtil)); - - when(resultsUploaderDeviceValidationService.getModelAndTestPerformedCodeToDeviceMap()) - .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceTypeForBulkUpload())); + new FhirConverter(mockedUUIDGenerator)); InputStream csvStream = loadCsv("testResultUpload/test-results-upload-valid.csv"); @@ -248,7 +241,7 @@ public UUID answer(InvocationOnMock invocation) { void convertExistingCsv_meetsProcessingSpeed() { InputStream input = loadCsv("testResultUpload/test-results-upload-valid-5000-rows.csv"); - when(resultsUploaderDeviceValidationService.getModelAndTestPerformedCodeToDeviceMap()) + when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap()) .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceTypeForBulkUpload())); var startTime = System.currentTimeMillis(); @@ -260,4 +253,65 @@ void convertExistingCsv_meetsProcessingSpeed() { assertTrue(elapsedTime < 20000, "Bundle processing took more than 20 seconds for 5000 rows"); } + + @Test + void convertExistingCsv_populatesBlankFields() { + InputStream input = loadCsv("testResultUpload/test-results-upload-valid-with-blank-fields.csv"); + var orderTestDate = Instant.parse("2021-12-20T14:00:00-06:00"); + var testResultDate = Instant.parse("2021-12-23T14:00:00-06:00"); + + when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap()) + .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceTypeForBulkUpload())); + when(resultsUploaderCachingService.getZoneIdByAddress(any())) + .thenReturn(ZoneId.of("US/Central")); + + var serializedBundles = sut.convertToFhirBundles(input, UUID.randomUUID()); + var first = serializedBundles.get(0); + var deserializedBundle = (Bundle) parser.parseResource(first); + + var specimen = + (Specimen) + deserializedBundle.getEntry().stream() + .filter(entry -> entry.getFullUrl().contains("Specimen/")) + .findFirst() + .get() + .getResource(); + + var diagnosticReport = + (DiagnosticReport) + deserializedBundle.getEntry().stream() + .filter(entry -> entry.getFullUrl().contains("DiagnosticReport/")) + .findFirst() + .get() + .getResource(); + + var organizations = + new java.util.ArrayList<>( + deserializedBundle.getEntry().stream() + .filter(entry -> entry.getFullUrl().contains("Organization/")) + .map(org -> (Organization) org.getResource()) + .toList()); + + organizations.removeIf(org -> org.hasName() && org.getName().equals("SimpleReport")); + + // Order test date should populate specimen collection date (aka collected) + assertThat(((DateTimeType) specimen.getCollection().getCollected()).getValue()) + .isEqualTo(orderTestDate); + // Order test date should populate testing lab specimen received date (aka received time) + assertThat(specimen.getReceivedTime()).isEqualTo(orderTestDate); + // Test result date should populate date result released (aka issued) + assertThat(diagnosticReport.getIssued()).isEqualTo(testResultDate); + // Testing lab should populate ordering facility + assertThat(organizations).hasSize(2); + assertThat(organizations.get(0).getAddress()).hasSize(1); + assertThat(organizations.get(1).getAddress()).hasSize(1); + assertThat(organizations.get(0).getAddress().get(0).getLine().get(0)) + .hasToString(organizations.get(1).getAddress().get(0).getLine().get(0).toString()); + assertThat(organizations.get(0).getAddress().get(0).getCity()) + .isEqualTo(organizations.get(1).getAddress().get(0).getCity()); + assertThat(organizations.get(0).getAddress().get(0).getState()) + .isEqualTo(organizations.get(1).getAddress().get(0).getState()); + assertThat(organizations.get(0).getAddress().get(0).getPostalCode()) + .isEqualTo(organizations.get(1).getAddress().get(0).getPostalCode()); + } } diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/utils/DateTimeUtilsTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/utils/DateTimeUtilsTest.java new file mode 100644 index 0000000000..bda9168bb6 --- /dev/null +++ b/backend/src/test/java/gov/cdc/usds/simplereport/utils/DateTimeUtilsTest.java @@ -0,0 +1,60 @@ +package gov.cdc.usds.simplereport.utils; + +import static gov.cdc.usds.simplereport.utils.DateTimeUtils.convertToZonedDateTime; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import gov.cdc.usds.simplereport.db.model.auxiliary.StreetAddress; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class DateTimeUtilsTest { + private static ResultsUploaderCachingService resultsUploaderCachingService; + + @BeforeAll + public static void init() { + resultsUploaderCachingService = mock(ResultsUploaderCachingService.class); + } + + @BeforeEach + public void beforeEach() { + when(resultsUploaderCachingService.getZoneIdByAddress(any())) + .thenReturn(ZoneId.of("US/Eastern")); + } + + void testConvertToZonedDateTime(String dateString, ZoneId expectedZoneId) { + var address = new StreetAddress(null, null, null, null, null); + var actualZonedDateTime = + convertToZonedDateTime(dateString, resultsUploaderCachingService, address); + var expectedZonedDateTime = ZonedDateTime.of(2023, 6, 28, 14, 0, 0, 0, expectedZoneId); + assertThat(actualZonedDateTime).hasToString(expectedZonedDateTime.toString()); + } + + @Test + void testConvertToZonedDateTime_withValidTimezoneSuffix() { + testConvertToZonedDateTime("6/28/2023 14:00 MST", ZoneId.of("US/Mountain")); + } + + @Test + void testConvertToZonedDateTime_withAddress() { + when(resultsUploaderCachingService.getZoneIdByAddress(any())) + .thenReturn(ZoneId.of("US/Pacific")); + testConvertToZonedDateTime("6/28/2023 14:00", ZoneId.of("US/Pacific")); + } + + @Test + void testConvertToZonedDateTime_withFallback() { + testConvertToZonedDateTime("6/28/2023 14:00", ZoneId.of("US/Eastern")); + } + + @Test + void testConvertToZonedDateTime_withICANNTzIdentifier() { + testConvertToZonedDateTime("6/28/2023 14:00 US/Samoa", ZoneId.of("US/Samoa")); + } +} diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtilsTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtilsTest.java index 35e0e4105e..902b8b51e6 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtilsTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtilsTest.java @@ -200,4 +200,45 @@ void invalidDateTime() { assertThat(validateDateTime(datetime)).hasSize(1); } } + + @Test + void validDateTimeWithTimeZone() { + var validDateTimes = new ArrayList(); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 ET", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 EST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 EDT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 CT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 CST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 CDT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 MT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 MST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 MDT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 PT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 PST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 PDT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 AKDT", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 AKST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 HST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 SST", "datetime")); + validDateTimes.add(new ValueOrError("01/01/2023 11:11 ", "datetime")); + for (var datetime : validDateTimes) { + assertThat(validateDateTime(datetime)).isEmpty(); + } + } + + @Test + void invalidDateTimeWithTimeZone() { + var invalidDateTimes = new ArrayList(); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 New York", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 Eastern", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 central", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 pacific time", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 123", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 ABC", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 ET2", "datetime")); + invalidDateTimes.add(new ValueOrError("01/01/2023 11:11 denver", "datetime")); + for (var datetime : invalidDateTimes) { + assertThat(validateDateTime(datetime)).hasSize(1); + } + } } diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/validators/FileValidatorTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/validators/FileValidatorTest.java index bab1159235..a74e09d3b1 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/validators/FileValidatorTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/validators/FileValidatorTest.java @@ -6,7 +6,7 @@ import gov.cdc.usds.simplereport.api.model.filerow.PatientUploadRow; import gov.cdc.usds.simplereport.api.model.filerow.TestResultRow; -import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage; import gov.cdc.usds.simplereport.test_util.TestDataBuilder; import java.io.ByteArrayInputStream; @@ -79,13 +79,13 @@ class FileValidatorTest { @BeforeEach public void setup() { - var resultsUploaderDeviceValidationService = mock(ResultsUploaderDeviceValidationService.class); - when(resultsUploaderDeviceValidationService.getModelAndTestPerformedCodeToDeviceMap()) + var resultsUploaderCachingService = mock(ResultsUploaderCachingService.class); + when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap()) .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceType())); - when(resultsUploaderDeviceValidationService.getSpecimenTypeNameToSNOMEDMap()) + when(resultsUploaderCachingService.getSpecimenTypeNameToSNOMEDMap()) .thenReturn(Map.of("nasal swab", "000111222")); testResultFileValidator = - new FileValidator<>(row -> new TestResultRow(row, resultsUploaderDeviceValidationService)); + new FileValidator<>(row -> new TestResultRow(row, resultsUploaderCachingService)); } @Test diff --git a/backend/src/test/java/gov/cdc/usds/simplereport/validators/TestResultRowTest.java b/backend/src/test/java/gov/cdc/usds/simplereport/validators/TestResultRowTest.java index a9a85c3fb1..f606d67bdb 100644 --- a/backend/src/test/java/gov/cdc/usds/simplereport/validators/TestResultRowTest.java +++ b/backend/src/test/java/gov/cdc/usds/simplereport/validators/TestResultRowTest.java @@ -6,7 +6,7 @@ import static org.mockito.Mockito.when; import gov.cdc.usds.simplereport.api.model.filerow.TestResultRow; -import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService; +import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService; import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage; import gov.cdc.usds.simplereport.test_util.TestDataBuilder; import gov.cdc.usds.simplereport.test_util.TestErrorMessageUtil; @@ -405,7 +405,7 @@ void validateIndividualFields() { invalidIndividualFields.put("specimen_type", "100"); invalidIndividualFields.put("testing_lab_clia", "à"); var testResultRow = - new TestResultRow(invalidIndividualFields, mockResultsUploaderDeviceValidationService()); + new TestResultRow(invalidIndividualFields, mockResultsUploaderCachingService()); var actual = testResultRow.validateIndividualValues(); @@ -419,10 +419,10 @@ void validateIndividualFields() { individualFields.forEach(fieldName -> assertThat(messages).contains(fieldName)); } - private ResultsUploaderDeviceValidationService mockResultsUploaderDeviceValidationService() { - var resultsUploaderDeviceValidationService = mock(ResultsUploaderDeviceValidationService.class); - when(resultsUploaderDeviceValidationService.getModelAndTestPerformedCodeToDeviceMap()) + private ResultsUploaderCachingService mockResultsUploaderCachingService() { + var resultsUploaderCachingService = mock(ResultsUploaderCachingService.class); + when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap()) .thenReturn(Map.of("id now|94534-5", TestDataBuilder.createDeviceType())); - return resultsUploaderDeviceValidationService; + return resultsUploaderCachingService; } } diff --git a/backend/src/test/resources/fhir/bundle.json b/backend/src/test/resources/fhir/bundle.json index 3e115ab319..89253e0dca 100644 --- a/backend/src/test/resources/fhir/bundle.json +++ b/backend/src/test/resources/fhir/bundle.json @@ -4,7 +4,7 @@ "identifier": { "value": "45e9539f-c9a4-4c86-b79d-4ba2c43f9ee0" }, - "timestamp": "$CURRENT_DATE_TZ", + "timestamp": "$CURRENT_DATE_ZULU", "entry":[ { "fullUrl":"MessageHeader/$MESSAGE_HEADER_ID", @@ -82,7 +82,7 @@ } } ], - "recorded": "$CURRENT_DATE_TZ" + "recorded": "$CURRENT_DATE_ZULU" } }, { @@ -96,7 +96,7 @@ } ], "status":"final", - "effectiveDateTime": "$EFFECTIVE_DATE_TIME_TESTED", + "effectiveDateTime": "$CURRENT_DATE_TIMEZONE", "code":{ "coding":[ { @@ -105,7 +105,7 @@ } ] }, - "issued" : "$CURRENT_DATE_ZULU", + "issued" : "$CURRENT_DATE_TIMEZONE", "subject":{ "reference":"Patient/55c53ed2-add5-47fb-884e-b4542ee64589" }, @@ -328,7 +328,7 @@ }, { "url":"https://reportstream.cdc.gov/fhir/StructureDefinition/order-effective-date", - "valueDateTime":"$EFFECTIVE_DATE_TIME_TESTED" + "valueDateTime":"$CURRENT_DATE_TIMEZONE" } ], "status":"active", @@ -456,7 +456,7 @@ } ] }, - "issued": "$EFFECTIVE_DATE_TIME_TESTED" + "issued": "$CURRENT_DATE_ZULU" } }, { @@ -529,7 +529,7 @@ } ] }, - "issued": "$EFFECTIVE_DATE_TIME_TESTED" + "issued": "$CURRENT_DATE_ZULU" } }, { @@ -602,7 +602,7 @@ } ] }, - "issued": "$EFFECTIVE_DATE_TIME_TESTED" + "issued": "$CURRENT_DATE_ZULU" } }, { diff --git a/backend/src/test/resources/fhir/diagnosticReport.json b/backend/src/test/resources/fhir/diagnosticReport.json index 0681bcd113..62d02cd16e 100644 --- a/backend/src/test/resources/fhir/diagnosticReport.json +++ b/backend/src/test/resources/fhir/diagnosticReport.json @@ -2,7 +2,7 @@ "resourceType": "DiagnosticReport", "id": "3c9c7370-e2e3-49ad-bb7a-f6005f41cf29", "status": "final", - "effectiveDateTime": "$EFFECTIVE_DATE_TIME_TESTED", + "effectiveDateTime": "$CURRENT_DATE_TIMEZONE", "code": { "coding": [ { @@ -16,5 +16,5 @@ "value": "3c9c7370-e2e3-49ad-bb7a-f6005f41cf29" } ], - "issued":"2023-06-22T11:46:00Z" + "issued":"$CURRENT_DATE_TIMEZONE" } diff --git a/backend/src/test/resources/fhir/observationCorrection.json b/backend/src/test/resources/fhir/observationCorrection.json index 45ed6bebed..f8e285fc96 100644 --- a/backend/src/test/resources/fhir/observationCorrection.json +++ b/backend/src/test/resources/fhir/observationCorrection.json @@ -52,5 +52,5 @@ "text": "Corrected Result: woops" } ], - "issued": "2023-06-10T23:15:00Z" + "issued": "2023-06-10T23:15:00.000Z" } diff --git a/backend/src/test/resources/fhir/observationCovid.json b/backend/src/test/resources/fhir/observationCovid.json index 0a38414f6c..0d98d00361 100644 --- a/backend/src/test/resources/fhir/observationCovid.json +++ b/backend/src/test/resources/fhir/observationCovid.json @@ -52,5 +52,5 @@ } ] }, - "issued": "2023-06-10T23:15:00Z" + "issued": "2023-06-10T23:15:00.000Z" } diff --git a/backend/src/test/resources/fhir/observationFlu.json b/backend/src/test/resources/fhir/observationFlu.json index 81c1ce0942..23299c00aa 100644 --- a/backend/src/test/resources/fhir/observationFlu.json +++ b/backend/src/test/resources/fhir/observationFlu.json @@ -52,5 +52,5 @@ } ] }, - "issued": "2023-06-10T23:15:00Z" + "issued": "2023-06-10T23:15:00.000Z" } diff --git a/backend/src/test/resources/fhir/serviceRequest.json b/backend/src/test/resources/fhir/serviceRequest.json index 8075aa821a..0cc83246ad 100644 --- a/backend/src/test/resources/fhir/serviceRequest.json +++ b/backend/src/test/resources/fhir/serviceRequest.json @@ -16,7 +16,7 @@ }, { "url":"https://reportstream.cdc.gov/fhir/StructureDefinition/order-effective-date", - "valueDateTime":"2023-06-22T10:35:00Z" + "valueDateTime":"2023-06-22T06:35:00-04:00" } ], "intent": "order", diff --git a/backend/src/test/resources/fhir/specimen.json b/backend/src/test/resources/fhir/specimen.json index 7a3f53b455..095b26277a 100644 --- a/backend/src/test/resources/fhir/specimen.json +++ b/backend/src/test/resources/fhir/specimen.json @@ -20,8 +20,9 @@ ], "text": "nose" }, - "collectedDateTime":"2023-06-22T15:35:00Z" + "collectedDateTime":"2023-06-22T11:35:00-04:00" }, + "receivedTime":"2023-06-23T12:00:00-04:00", "identifier": [ { "value": "$SPECIMEN_IDENTIFIER" diff --git a/backend/src/test/resources/testResultUpload/test-results-upload-valid-as-fhir.json b/backend/src/test/resources/testResultUpload/test-results-upload-valid-as-fhir.json index 9e1ced1fd1..205cc0e541 100644 --- a/backend/src/test/resources/testResultUpload/test-results-upload-valid-as-fhir.json +++ b/backend/src/test/resources/testResultUpload/test-results-upload-valid-as-fhir.json @@ -116,8 +116,8 @@ "subject": { "reference": "Patient/1234" }, - "effectiveDateTime": "2021-12-20T14:00:00Z", - "issued": "2023-05-24T19:33:06Z", + "effectiveDateTime": "2021-12-23T14:00:00-06:00", + "issued": "2021-12-24T14:00:00-06:00", "specimen": [ { "reference": "Specimen/00000000-0000-0000-0000-000000000004" @@ -220,7 +220,7 @@ "value": "01D1058442" } ], - "name": "My Urgent Care", + "name": "My Testing Lab", "telecom": [ { "system": "phone", @@ -231,7 +231,7 @@ "address": [ { "line": [ - "400 Main Street" + "300 North Street" ], "city": "Birmingham", "state": "AL", @@ -335,7 +335,7 @@ "reference": "Patient/1234" }, "collection" : { - "collectedDateTime":"2021-12-20T14:00:00Z" + "collectedDateTime":"2021-12-21T14:00:00-06:00" } } }, @@ -358,7 +358,7 @@ }, { "url":"https://reportstream.cdc.gov/fhir/StructureDefinition/order-effective-date", - "valueDateTime":"2021-12-20T14:00:00Z" + "valueDateTime":"2021-12-20T14:00:00-06:00" } ], "status": "completed", @@ -494,7 +494,7 @@ "device": { "reference": "Device/00000000-0000-0000-0000-000000000003" }, - "issued": "2021-12-20T14:00:00Z" + "issued": "2021-12-23T20:00:00.000Z" } }, { diff --git a/backend/src/test/resources/testResultUpload/test-results-upload-valid-with-blank-fields.csv b/backend/src/test/resources/testResultUpload/test-results-upload-valid-with-blank-fields.csv new file mode 100644 index 0000000000..8d61012e80 --- /dev/null +++ b/backend/src/test/resources/testResultUpload/test-results-upload-valid-with-blank-fields.csv @@ -0,0 +1,2 @@ +patient_id,patient_last_name,patient_first_name,patient_middle_name,patient_street,patient_street2,patient_city,patient_state,patient_zip_code,patient_county,patient_phone_number,patient_dob,patient_gender,patient_race,patient_ethnicity,patient_preferred_language,patient_email,accession_number,equipment_model_name,test_performed_code,test_result,order_test_date,specimen_collection_date,testing_lab_specimen_received_date,test_result_date,date_result_released,specimen_type,ordering_provider_id,ordering_provider_last_name,ordering_provider_first_name,ordering_provider_middle_name,ordering_provider_street,ordering_provider_street2,ordering_provider_city,ordering_provider_state,ordering_provider_zip_code,ordering_provider_phone_number,testing_lab_clia,testing_lab_name,testing_lab_street,testing_lab_street2,testing_lab_city,testing_lab_state,testing_lab_zip_code,testing_lab_phone_number,pregnant,employed_in_healthcare,symptomatic_for_disease,illness_onset_date,resident_congregate_setting,residence_type,hospitalized,icu,ordering_facility_name,ordering_facility_street,ordering_facility_street2,ordering_facility_city,ordering_facility_state,ordering_facility_zip_code,ordering_facility_phone_number,comment,test_result_status +1234,Doe,Jane,,123 Main St,,Birmingham,AL,35226,Jefferson,205-999-2800,1/20/1962,F,White,Not Hispanic or Latino,,,123,ID NOW,94534-5,Detected,12/20/2021 14:00,,,12/23/2021 14:00,,Nasal Swab,,,,,,,,,,,01D1058442,My Testing Lab,300 North Street,,Birmingham,AL,35228,205-888-2000,N,N,N,,N,,N,N,,,,,,,,Test Comment,F diff --git a/backend/src/test/resources/testResultUpload/test-results-upload-valid.csv b/backend/src/test/resources/testResultUpload/test-results-upload-valid.csv index ebb6125781..a661d385e6 100644 --- a/backend/src/test/resources/testResultUpload/test-results-upload-valid.csv +++ b/backend/src/test/resources/testResultUpload/test-results-upload-valid.csv @@ -1,2 +1,2 @@ patient_id,patient_last_name,patient_first_name,patient_middle_name,patient_street,patient_street2,patient_city,patient_state,patient_zip_code,patient_county,patient_phone_number,patient_dob,patient_gender,patient_race,patient_ethnicity,patient_preferred_language,patient_email,accession_number,equipment_model_name,test_performed_code,test_result,order_test_date,specimen_collection_date,testing_lab_specimen_received_date,test_result_date,date_result_released,specimen_type,ordering_provider_id,ordering_provider_last_name,ordering_provider_first_name,ordering_provider_middle_name,ordering_provider_street,ordering_provider_street2,ordering_provider_city,ordering_provider_state,ordering_provider_zip_code,ordering_provider_phone_number,testing_lab_clia,testing_lab_name,testing_lab_street,testing_lab_street2,testing_lab_city,testing_lab_state,testing_lab_zip_code,testing_lab_phone_number,pregnant,employed_in_healthcare,symptomatic_for_disease,illness_onset_date,resident_congregate_setting,residence_type,hospitalized,icu,ordering_facility_name,ordering_facility_street,ordering_facility_street2,ordering_facility_city,ordering_facility_state,ordering_facility_zip_code,ordering_facility_phone_number,comment,test_result_status -1234,Doe,Jane,,123 Main St,,Birmingham,AL,35226,Jefferson,205-999-2800,1/20/1962,F,White,Not Hispanic or Latino,,,123,ID NOW,94534-5,Detected,12/20/2021 14:00,12/20/2021 14:00,12/20/2021 14:00,12/20/2021 14:00,12/20/2021 14:00,Nasal Swab,1013012657,Smith MD,John,,400 Main Street,,Birmingham,AL,35228,205-888-2000,01D1058442,My Urgent Care,400 Main Street,,Birmingham,AL,35228,205-888-2000,N,N,N,,N,,N,N,My Urgent Care,400 Main Street,Suite 100,Birmingham,AL,35228,205-888-2000,Test Comment,F +1234,Doe,Jane,,123 Main St,,Birmingham,AL,35226,Jefferson,205-999-2800,1/20/1962,F,White,Not Hispanic or Latino,,,123,ID NOW,94534-5,Detected,12/20/2021 14:00,12/21/2021 14:00,12/22/2021 14:00,12/23/2021 14:00,12/24/2021 14:00,Nasal Swab,1013012657,Smith MD,John,,400 Main Street,,Birmingham,AL,35228,205-888-2000,01D1058442,My Testing Lab,300 North Street,,Birmingham,AL,35228,205-888-2000,N,N,N,,N,,N,N,My Urgent Care,400 Main Street,Suite 100,Birmingham,AL,35228,205-888-2000,Test Comment,F