Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add test date to Observation.issued for single-entry and bulk upload #5982

Merged
merged 10 commits into from
Jun 16, 2023
+77 −26
Merged

Add test date to Observation.issued for single-entry and bulk upload #5982

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3f7544f
Add test date to Observation.issued for single-entry and bulk upload
mehansen Jun 13, 2023
2d1840a
Handle input dates with no time (oops)
mehansen Jun 14, 2023
490466c
clean up todo
mehansen Jun 14, 2023
fbda45f
Merge branch 'main' into merethe/move-fhir-message-dates
mehansen Jun 14, 2023
50b80da
update test to reflect not rounding to start of day
mehansen Jun 14, 2023
ba322f7
use z format for datetimes instead of +0:00
mehansen Jun 14, 2023
54fe8bf
code review feedback
mehansen Jun 14, 2023
e1d5396
Merge branch 'main' into merethe/move-fhir-message-dates
mehansen Jun 14, 2023
ed58e16
Merge branch 'main' into merethe/move-fhir-message-dates
mehansen Jun 15, 2023
45f49ad
Merge branch 'main' into merethe/move-fhir-message-dates
mehansen Jun 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/src/main/java/gov/cdc/usds/simplereport/api/converter/ConvertToObservationProps.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package gov.cdc.usds.simplereport.api.converter;

import gov.cdc.usds.simplereport.db.model.auxiliary.TestCorrectionStatus;
import java.util.Date;
import lombok.Builder;
import lombok.Getter;

Expand All @@ -17,4 +18,5 @@ public class ConvertToObservationProps {
private String testkitNameId;
private String equipmentUid;
private String deviceModel;
private Date issued;
}
22 changes: 17 additions & 5 deletions backend/src/main/java/gov/cdc/usds/simplereport/api/converter/FhirConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import static gov.cdc.usds.simplereport.api.converter.FhirConstants.UNIVERSAL_ID_SYSTEM;
import static gov.cdc.usds.simplereport.api.converter.FhirConstants.YESNO_CODE_SYSTEM;

import ca.uhn.fhir.model.api.TemporalPrecisionEnum;
import com.google.i18n.phonenumbers.NumberParseException;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.i18n.phonenumbers.PhoneNumberUtil.PhoneNumberFormat;
Expand Down Expand Up @@ -66,6 +67,7 @@
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -489,7 +491,8 @@ public List<Observation> convertToObservation(
Set<Result> results,
DeviceType deviceType,
TestCorrectionStatus correctionStatus,
String correctionReason) {
String correctionReason,
Date resultDate) {
return results.stream()
.map(
result -> {
Expand All @@ -515,7 +518,8 @@ public List<Observation> convertToObservation(
correctionReason,
testkitNameId,
equipmentUid,
deviceType.getModel());
deviceType.getModel(),
resultDate);
})
.collect(Collectors.toList());
}
Expand All @@ -534,7 +538,8 @@ public Observation convertToObservation(
String correctionReason,
String testkitNameId,
String equipmentUid,
String deviceModel) {
String deviceModel,
Date resultDate) {
if (result != null && result.getDisease() != null) {

return convertToObservation(
Expand All @@ -550,6 +555,7 @@ public Observation convertToObservation(
.testkitNameId(testkitNameId)
.equipmentUid(equipmentUid)
.deviceModel(deviceModel)
.issued(resultDate)
.build());
}
return null;
Expand Down Expand Up @@ -582,6 +588,9 @@ public Observation convertToObservation(ConvertToObservationProps props) {
.addInterpretation()
.addCoding(convertToAbnormalFlagInterpretation(props.getResultCode()));

observation.setIssued(props.getIssued());
observation.getIssuedElement().setTimeZone(TimeZone.getTimeZone("utc"));
mehansen marked this conversation as resolved.
Show resolved Hide resolved

return observation;
}

Expand Down Expand Up @@ -793,7 +802,9 @@ public DiagnosticReport convertToDiagnosticReport(
var diagnosticReport =
new DiagnosticReport()
.setStatus(status)
.setEffective(new DateTimeType(dateTested))
.setEffective(
new DateTimeType(
dateTested, TemporalPrecisionEnum.MILLI, TimeZone.getTimeZone("utc")))
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see above comment - this just forces the messages to look the same when run on systems with different default precision/time zone

.setIssued(dateUpdated);

// Allows EffectiveDateTimeType to be mocked during tests
Expand Down Expand Up @@ -834,7 +845,8 @@ public Bundle createFhirBundle(
testEvent.getResults(),
testEvent.getDeviceType(),
testEvent.getCorrectionStatus(),
testEvent.getReasonForCorrection()))
testEvent.getReasonForCorrection(),
testEvent.getDateTested()))
.aoeObservations(
convertToAOEObservations(
testEvent.getInternalId().toString(), testEvent.getSurveyData()))
Expand Down
18 changes: 16 additions & 2 deletions backend/src/main/java/gov/cdc/usds/simplereport/utils/BulkUploadResultsToFhir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,10 @@
import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceValidationService;
import java.io.InputStream;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
Expand Down Expand Up @@ -292,6 +294,16 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) {
uuidGenerator.randomUUID().toString(),
uuidGenerator.randomUUID().toString());

LocalDateTime testResultDate;
TemporalAccessor temporalAccessor =
dateTimeFormatter.parseBest(
row.getTestResultDate().getValue(), LocalDateTime::from, LocalDate::from);
if (temporalAccessor instanceof LocalDateTime) {
testResultDate = (LocalDateTime) temporalAccessor;
} else {
testResultDate = ((LocalDate) temporalAccessor).atStartOfDay();
}

Copy link
Collaborator

@mpbrown mpbrown Jun 14, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is an awesome way to handle the date parsing! 💯💯💯 I'll update #5985 with this.

Do you think we should move this parsing towards the top of convertRowToFhirBundle to group it near testEventId? That value would now be used by at least observation and diagnosticReport (I think we might soon be using it for specimen.collected too)

var observation =
List.of(
fhirConverter.convertToObservation(
Expand All @@ -309,6 +321,9 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) {
.testkitNameId(testKitNameId)
.equipmentUid(equipmentUid)
.deviceModel(row.getEquipmentModelName().getValue())
.issued(
Date.from(
testResultDate.atZone(zoneIdGenerator.getSystemZoneId()).toInstant()))
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this on the other hand uses the system default time zone to infer the time zone for text date time in the CSV (e.g. 06/14/23 12:36, which has no time zone)
we have to do this any time there's a date time in the CSV since the format doesn't specify time zone. we should figure out a better way to determine what time zone to use

.build()));

LocalDate symptomOnsetDate = null;
Expand Down Expand Up @@ -336,13 +351,12 @@ private Bundle convertRowToFhirBundle(TestResultRow row, UUID orgId) {
testOrderLoinc,
uuidGenerator.randomUUID().toString());

var testDate = LocalDate.parse(row.getTestResultDate().getValue(), dateTimeFormatter);
var diagnosticReport =
fhirConverter.convertToDiagnosticReport(
mapTestResultStatusToFhirValue(row.getTestResultStatus().getValue()),
testPerformedCode,
testEventId,
Date.from(testDate.atStartOfDay(zoneIdGenerator.getSystemZoneId()).toInstant()),
Date.from(testResultDate.atZone(zoneIdGenerator.getSystemZoneId()).toInstant()),
dateGenerator.newDate());

return fhirConverter.createFhirBundle(
Expand Down
44 changes: 34 additions & 10 deletions backend/src/test/java/gov/cdc/usds/simplereport/api/converter/FhirConverterTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import static org.mockito.Mockito.when;

import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.model.api.TemporalPrecisionEnum;
import ca.uhn.fhir.parser.IParser;
import gov.cdc.usds.simplereport.db.model.DeviceType;
import gov.cdc.usds.simplereport.db.model.DeviceTypeDisease;
Expand Down Expand Up @@ -45,6 +46,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TimeZone;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -609,6 +611,7 @@ void convertToObservation_Strings_valid() {
.testkitNameId("testKitName")
.equipmentUid("equipmentUid")
.deviceModel("modelName")
.issued(Date.from(Instant.parse("2023-06-10T23:15:00.00Z")))
.build());

assertThat(actual.getId()).isEqualTo("id-123");
Expand All @@ -625,6 +628,7 @@ void convertToObservation_Strings_valid() {
.isEqualTo("resultCode");
assertThat(actual.getNote()).isEmpty();
assertThat(actual.getMethod().getCodingFirstRep().getDisplay()).isEqualTo("modelName");
assertThat(actual.getIssued()).isEqualTo("2023-06-10T23:15:00.00Z");
}

@Test
Expand All @@ -642,7 +646,8 @@ void convertToObservation_Result_valid() {
null,
"testkitName",
"equipmentUid",
"modelName");
"modelName",
Date.from(Instant.parse("2023-06-10T23:15:00.00Z")));

assertThat(actual.getId()).isEqualTo(internalId.toString());
assertThat(actual.getStatus().getDisplay()).isEqualTo(ObservationStatus.FINAL.getDisplay());
Expand All @@ -657,6 +662,7 @@ void convertToObservation_Result_valid() {
.isEqualTo("260373001");
assertThat(actual.getNote()).isEmpty();
assertThat(actual.getMethod().getCodingFirstRep().getDisplay()).isEqualTo("modelName");
assertThat(actual.getIssued()).isEqualTo("2023-06-10T23:15:00.00Z");
}

@Test
Expand All @@ -674,7 +680,8 @@ void convertToObservation_Result_correction() {
"Oopsy Daisy",
"testkitName",
"equipmentUid",
"modelName");
"modelName",
new Date());

assertThat(actual.getId()).isEqualTo(internalId.toString());
assertThat(actual.getStatus().getDisplay()).isEqualTo(ObservationStatus.CORRECTED.getDisplay());
Expand All @@ -697,7 +704,8 @@ void convertToObservation_Result_removeWithoutReason() {
null,
"testkitName",
"equipmentUid",
"modelName");
"modelName",
new Date());

assertThat(actual.getId()).isEqualTo(internalId.toString());
assertThat(actual.getStatus().getDisplay())
Expand All @@ -716,7 +724,8 @@ void convertToObservation_Result_null() {
null,
"testkitName",
"equipmentUid",
"modelName");
"modelName",
new Date());

assertThat(actual).isNull();
}
Expand All @@ -733,7 +742,8 @@ void convertToObservation_Result_nullDisease() {
null,
"testkitName",
"equipmentUid",
"modelName");
"modelName",
new Date());

assertThat(actual).isNull();
}
Expand Down Expand Up @@ -765,7 +775,11 @@ void convertToObservation_Result_matchesJson() throws IOException {

var actual =
fhirConverter.convertToObservation(
Set.of(covidResult, fluResult), device, TestCorrectionStatus.ORIGINAL, null);
Set.of(covidResult, fluResult),
device,
TestCorrectionStatus.ORIGINAL,
null,
Date.from(Instant.parse("2023-06-10T23:15:00.00Z")));

assertThat(actual).hasSize(2);
String covidSerialized =
Expand Down Expand Up @@ -815,7 +829,11 @@ void convertToObservation_Result_correctionMatchesJson() throws IOException {

var actual =
fhirConverter.convertToObservation(
Set.of(result), device, TestCorrectionStatus.CORRECTED, "woops");
Set.of(result),
device,
TestCorrectionStatus.CORRECTED,
"woops",
Date.from(Instant.parse("2023-06-10T23:15:00.00Z")));

String actualSerialized = parser.encodeResourceToString(actual.get(0));
var expectedSerialized1 =
Expand Down Expand Up @@ -936,7 +954,9 @@ void convertToDiagnosticReport_TestEvent_matchesJson() throws IOException {
StandardCharsets.UTF_8);
expectedSerialized =
expectedSerialized.replace(
"$EFFECTIVE_DATE_TIME_TESTED", new DateTimeType(date).getValueAsString());
"$EFFECTIVE_DATE_TIME_TESTED",
new DateTimeType(date, TemporalPrecisionEnum.MILLI, TimeZone.getTimeZone("utc"))
.getValueAsString());

JSONAssert.assertEquals(expectedSerialized, actualSerialized, true);
}
Expand All @@ -954,7 +974,9 @@ void convertToDiagnosticReport_Strings_valid() {
assertThat(actual.getCode().getCodingFirstRep().getSystem()).isEqualTo("http://loinc.org");
assertThat(actual.getCode().getCodingFirstRep().getCode()).isEqualTo("95422-2");
assertThat(((DateTimeType) actual.getEffective()).getAsV3())
.isEqualTo(new DateTimeType(date).getAsV3());
.isEqualTo(
new DateTimeType(date, TemporalPrecisionEnum.MILLI, TimeZone.getTimeZone("utc"))
.getAsV3());
assertThat((actual.getIssued())).isEqualTo(date);
}

Expand Down Expand Up @@ -1461,7 +1483,9 @@ void createFhirBundle_TestEvent_matchesJson() throws IOException {
expectedSerialized = expectedSerialized.replace("$PROVENANCE_ID", provenanceId);
expectedSerialized =
expectedSerialized.replace(
"$EFFECTIVE_DATE_TIME_TESTED", new DateTimeType(dateTested).getValueAsString());
"$EFFECTIVE_DATE_TIME_TESTED",
new DateTimeType(dateTested, TemporalPrecisionEnum.MILLI, TimeZone.getTimeZone("utc"))
.getValueAsString());
expectedSerialized =
expectedSerialized.replace(
"$PROVENANCE_RECORDED_DATE",
Expand Down
9 changes: 6 additions & 3 deletions backend/src/test/resources/fhir/bundle.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,8 @@
}
}
]
}
},
"issued": "$EFFECTIVE_DATE_TIME_TESTED"
}
},
{
Expand Down Expand Up @@ -522,7 +523,8 @@
}
}
]
}
},
"issued": "$EFFECTIVE_DATE_TIME_TESTED"
}
},
{
Expand Down Expand Up @@ -594,7 +596,8 @@
}
}
]
}
},
"issued": "$EFFECTIVE_DATE_TIME_TESTED"
}
},
{
Expand Down
3 changes: 2 additions & 1 deletion backend/src/test/resources/fhir/observationCorrection.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,6 @@
{
"text": "Corrected Result: woops"
}
]
],
"issued": "2023-06-10T23:15:00.000+00:00"
}
3 changes: 2 additions & 1 deletion backend/src/test/resources/fhir/observationCovid.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,6 @@
}
}
]
}
},
"issued": "2023-06-10T23:15:00.000+00:00"
}
3 changes: 2 additions & 1 deletion backend/src/test/resources/fhir/observationFlu.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,6 @@
}
}
]
}
},
"issued": "2023-06-10T23:15:00.000+00:00"
}
2 changes: 1 addition & 1 deletion backend/src/test/resources/testResultUpload/test-results-upload-valid-as-fhir.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
"subject": {
"reference": "Patient/1234"
},
"effectiveDateTime": "2021-12-20T00:00:00Z",
"effectiveDateTime": "2021-12-20T14:00:00.000Z",
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reflects the date in the CSV since we don't round to start of day in bulk upload code anymore

"issued": "2023-05-24T19:33:06.472Z",
"specimen": [
{
Expand Down