Skip to content

Commit

Permalink
Normalize times before comparison
Browse files Browse the repository at this point in the history
  • Loading branch information
johngrimes committed Jun 19, 2024
1 parent 3bb87c1 commit 6359423
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ protected void registerUDFs(final UDFRegistrar udfRegistrar) {
.register(new DateTimeGreaterThanFunction())
.register(new DateTimeGreaterThanOrEqualToFunction())
.register(new DateTimeLessThanFunction())
.register(new DateTimeLessThanOrEqualToFunction())
.register(new NormalizeDateTimeFunction());
.register(new DateTimeLessThanOrEqualToFunction());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package au.csiro.pathling.sql.dates.time;

import au.csiro.pathling.sql.udf.SqlFunction1;
import java.time.LocalTime;
import java.time.temporal.ChronoUnit;
import java.util.Objects;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.DataTypes;

/**
* A function for normalizing a time string to a long (nanoseconds) for comparison purposes.
* <p>
* Truncates the precision of the time to milliseconds, as this is the highest precision supported
* by FHIR.
*/
public class NormalizeTimeFunction implements SqlFunction1<String, Long> {

private static final long serialVersionUID = 4117774962772392910L;

public static final String FUNCTION_NAME = "normalize_time";

@Override
public String getName() {
return FUNCTION_NAME;
}

@Override
public DataType getReturnType() {
return DataTypes.LongType;
}

@Override
public Long call(final String timeString) throws Exception {
final LocalTime time = Objects.requireNonNull(
LocalTime.parse(timeString));
return time.truncatedTo(ChronoUnit.MILLIS).toNanoOfDay();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package au.csiro.pathling.sql.misc;

import au.csiro.pathling.sql.dates.datetime.NormalizeDateTimeFunction;
import au.csiro.pathling.sql.dates.time.NormalizeTimeFunction;
import au.csiro.pathling.sql.udf.AbstractUDFRegistrar;

/**
Expand All @@ -27,6 +29,8 @@ public class MiscUDFRegistrar extends AbstractUDFRegistrar {
protected void registerUDFs(final UDFRegistrar udfRegistrar) {
udfRegistrar
.register(new CodingToLiteral())
.register(new TemporalDifferenceFunction());
.register(new TemporalDifferenceFunction())
.register(new NormalizeDateTimeFunction())
.register(new NormalizeTimeFunction());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import au.csiro.pathling.encoders.datatypes.DecimalCustomCoder;
import au.csiro.pathling.io.source.DataSource;
import au.csiro.pathling.sql.dates.datetime.NormalizeDateTimeFunction;
import au.csiro.pathling.sql.dates.time.NormalizeTimeFunction;
import au.csiro.pathling.terminology.TerminologyServiceFactory;
import au.csiro.pathling.test.SpringBootUnitTest;
import ca.uhn.fhir.context.FhirContext;
Expand Down Expand Up @@ -142,6 +143,8 @@ class Expect implements ResultExpectation {
public static final String FHIR_DATE_TIME_PATTERN = "^([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|"
+ "[1-9]000)(-(0[1-9]|1[0-2])(-(0[1-9]|[1-2][0-9]|3[0-1])(T([01][0-9]|2[0-3]):[0-5][0-9]:"
+ "([0-5][0-9]|60)(\\.[0-9]{1,9})?)?)?(Z|(\\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00)?)?)?$";
public static final String FHIR_TIME_PATTERN = "^([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)"
+ "(\\.[0-9]+)?$";
Path expectedJson;
List<String> expectedColumns;

Expand All @@ -166,6 +169,9 @@ public void expectResult(@Nonnull final Dataset<Row> rowDataset) {
return when(
col(field.name()).rlike(FHIR_DATE_TIME_PATTERN),
callUDF(NormalizeDateTimeFunction.FUNCTION_NAME, col(field.name()))
).when(
col(field.name()).rlike(FHIR_TIME_PATTERN),
callUDF(NormalizeTimeFunction.FUNCTION_NAME, col(field.name()))
).otherwise(col(field.name())).alias(field.name());
} else {
// Add the field to the selection without alteration.
Expand Down

0 comments on commit 6359423

Please sign in to comment.