From d73e938ec937eb480b789d19e8a2b38f0ecb1c93 Mon Sep 17 00:00:00 2001 From: Arnav Balyan Date: Mon, 2 Oct 2023 18:00:44 +0530 Subject: [PATCH 1/4] update --- .../core/query/reduce/BaseGapfillProcessor.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java index 25106858ea4..7680b2671b9 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java @@ -23,6 +23,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.FunctionContext; import org.apache.pinot.common.response.broker.BrokerResponseNative; @@ -134,6 +137,8 @@ protected void replaceColumnNameWithAlias(DataSchema dataSchema) { for (int i = 0; i < dataSchema.getColumnNames().length; i++) { if (columnNameToAliasMap.containsKey(dataSchema.getColumnNames()[i])) { dataSchema.getColumnNames()[i] = columnNameToAliasMap.get(dataSchema.getColumnNames()[i]); + } else if (columnNameToAliasMap.containsKey(caseInsensitiveTypeString(dataSchema.getColumnNames()[i]))) { + dataSchema.getColumnNames()[i] = columnNameToAliasMap.get(caseInsensitiveTypeString(dataSchema.getColumnNames()[i])); } } } @@ -226,4 +231,16 @@ protected List gapFillAndAggregate(List rows, DataSchema dat DataSchema resultTableSchema) { throw new UnsupportedOperationException("Not supported"); } + + protected String caseInsensitiveTypeString(String columnName) { + String dataTypePattern = "(BOOLEAN|INT|LONG|FLOAT|DOUBLE|STRING)"; + Matcher matcher = Pattern.compile(dataTypePattern, Pattern.CASE_INSENSITIVE).matcher(columnName); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + String dataType = matcher.group().toLowerCase(); + matcher.appendReplacement(result, dataType); + } + matcher.appendTail(result); + return result.toString(); + } } From 27bf07358ca0cc556a1df9efb31bf20d4395eb59 Mon Sep 17 00:00:00 2001 From: Arnav Balyan Date: Mon, 2 Oct 2023 18:06:32 +0530 Subject: [PATCH 2/4] rebase --- .../pinot/queries/GapfillQueriesTest.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java index 91cce686616..3980d2752da 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java @@ -4071,6 +4071,89 @@ public void datetimeconvertGapfillTestAggregateAggregateOutOfBoundary() { } } + @Test + public void GapfillTestAggregateUpperCaseDataType() { + DateTimeFormatSpec dateTimeFormatter = + new DateTimeFormatSpec("1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS"); + DateTimeGranularitySpec dateTimeGranularity = new DateTimeGranularitySpec("1:HOURS"); + long start; + + String gapfillQuery1 = "SELECT " + + "time_col, SUM(occupied) as occupied_slots_count, time_col " + + "FROM (" + + " SELECT GapFill(time_col, " + + " '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', " + + " '2021-11-07 8:00:00.000', '2021-11-07 10:00:00.000', '1:HOURS'," + + " FILL(occupied, 'FILL_PREVIOUS_VALUE'), TIMESERIESON(levelId, lotId)) AS time_col," + + " occupied, lotId, levelId" + + " FROM (" + + " SELECT DATETIMECONVERT(eventTime, '1:MILLISECONDS:EPOCH', " + + " '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:HOURS') AS time_col," + + " lastWithTime(isOccupied, eventTime, 'int') as occupied, lotId, levelId" + + " FROM parkingData " + + " WHERE eventTime >= 1636257600000 AND eventTime <= 1636286400000 " + + " GROUP BY time_col, levelId, lotId " + + " LIMIT 200 " + + " ) " + + " LIMIT 200 " + + ") " + + " GROUP BY time_col " + + " LIMIT 200 "; + + BrokerResponseNative gapfillBrokerResponse1 = getBrokerResponse(gapfillQuery1); + + double[] expectedOccupiedSlotsCounts1 = new double[]{6, 4}; + ResultTable gapFillResultTable1 = gapfillBrokerResponse1.getResultTable(); + List gapFillRows1 = gapFillResultTable1.getRows(); + Assert.assertEquals(gapFillRows1.size(), expectedOccupiedSlotsCounts1.length); + start = dateTimeFormatter.fromFormatToMillis("2021-11-07 08:00:00.000"); + for (int i = 0; i < expectedOccupiedSlotsCounts1.length; i++) { + String firstTimeCol = (String) gapFillRows1.get(i)[0]; + long timeStamp = dateTimeFormatter.fromFormatToMillis(firstTimeCol); + Assert.assertEquals(timeStamp, start); + Assert.assertEquals(expectedOccupiedSlotsCounts1[i], gapFillRows1.get(i)[1]); + start += dateTimeGranularity.granularityToMillis(); + } + + String gapfillQuery2 = "SELECT " + + "time_col, SUM(occupied) as occupied_slots_count, time_col " + + "FROM (" + + " SELECT GapFill(time_col, " + + " '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', " + + " '2021-11-07 8:00:00.000', '2021-11-07 10:00:00.000', '1:HOURS'," + + " FILL(occupied, 'FILL_PREVIOUS_VALUE'), TIMESERIESON(levelId, lotId)) AS time_col," + + " occupied, lotId, levelId" + + " FROM (" + + " SELECT DATETIMECONVERT(eventTime, '1:MILLISECONDS:EPOCH', " + + " '1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS', '1:HOURS') AS time_col," + + " lastWithTime(isOccupied, eventTime, 'int') as occupied, lotId, levelId" + + " FROM parkingData " + + " WHERE eventTime >= 1636257600000 AND eventTime <= 1636286400000 " + + " GROUP BY time_col, levelId, lotId " + + " LIMIT 200 " + + " ) " + + " LIMIT 200 " + + ") " + + " WHERE occupied = 1 " + + " GROUP BY time_col " + + " LIMIT 200 "; + + BrokerResponseNative gapfillBrokerResponse2 = getBrokerResponse(gapfillQuery2); + + double[] expectedOccupiedSlotsCounts2 = new double[]{6, 4}; + ResultTable gapFillResultTable2 = gapfillBrokerResponse2.getResultTable(); + List gapFillRows2 = gapFillResultTable2.getRows(); + Assert.assertEquals(gapFillRows2.size(), expectedOccupiedSlotsCounts2.length); + start = dateTimeFormatter.fromFormatToMillis("2021-11-07 08:00:00.000"); + for (int i = 0; i < expectedOccupiedSlotsCounts2.length; i++) { + String firstTimeCol = (String) gapFillRows2.get(i)[0]; + long timeStamp = dateTimeFormatter.fromFormatToMillis(firstTimeCol); + Assert.assertEquals(timeStamp, start); + Assert.assertEquals(expectedOccupiedSlotsCounts2[i], gapFillRows2.get(i)[1]); + start += dateTimeGranularity.granularityToMillis(); + } + } + @AfterClass public void tearDown() throws IOException { From cadc17ce9a6f266bfccd6c9861cc089c5e17e229 Mon Sep 17 00:00:00 2001 From: Arnav Balyan Date: Mon, 2 Oct 2023 18:18:59 +0530 Subject: [PATCH 3/4] lint fix --- .../apache/pinot/core/query/reduce/BaseGapfillProcessor.java | 3 ++- .../test/java/org/apache/pinot/queries/GapfillQueriesTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java index 7680b2671b9..e81daea8e48 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java @@ -138,7 +138,8 @@ protected void replaceColumnNameWithAlias(DataSchema dataSchema) { if (columnNameToAliasMap.containsKey(dataSchema.getColumnNames()[i])) { dataSchema.getColumnNames()[i] = columnNameToAliasMap.get(dataSchema.getColumnNames()[i]); } else if (columnNameToAliasMap.containsKey(caseInsensitiveTypeString(dataSchema.getColumnNames()[i]))) { - dataSchema.getColumnNames()[i] = columnNameToAliasMap.get(caseInsensitiveTypeString(dataSchema.getColumnNames()[i])); + dataSchema.getColumnNames()[i] = + columnNameToAliasMap.get(caseInsensitiveTypeString(dataSchema.getColumnNames()[i])); } } } diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java index 3980d2752da..500a76d478f 100644 --- a/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/queries/GapfillQueriesTest.java @@ -4072,7 +4072,7 @@ public void datetimeconvertGapfillTestAggregateAggregateOutOfBoundary() { } @Test - public void GapfillTestAggregateUpperCaseDataType() { + public void gapfillTestAggregateUpperCaseDataType() { DateTimeFormatSpec dateTimeFormatter = new DateTimeFormatSpec("1:MILLISECONDS:SIMPLE_DATE_FORMAT:yyyy-MM-dd HH:mm:ss.SSS"); DateTimeGranularitySpec dateTimeGranularity = new DateTimeGranularitySpec("1:HOURS"); From 31ef6b78c3d890623cb14993820728da8f30de71 Mon Sep 17 00:00:00 2001 From: Arnav Balyan Date: Mon, 2 Oct 2023 18:22:51 +0530 Subject: [PATCH 4/4] lint-fix --- .../org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java index e81daea8e48..62b01e4105a 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/query/reduce/BaseGapfillProcessor.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; - import org.apache.pinot.common.request.context.ExpressionContext; import org.apache.pinot.common.request.context.FunctionContext; import org.apache.pinot.common.response.broker.BrokerResponseNative;