Skip to content

Commit

Permalink
[improvement][headless]Introduce new time mode CURRENT. #1692
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerryjzhang
jerryjzhang committed Sep 21, 2024
1 parent 26ca530 commit b676538
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 227 deletions.
7 changes: 5 additions & 2 deletions common/src/main/java/com/tencent/supersonic/common/pojo/enums/TimeMode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

public enum TimeMode {

/** date mode LAST - a certain time RECENT - a period time */
// a specific date at N days ago
LAST,
RECENT
// a period of time from N days ago to today
RECENT,
// a period of time from the first day of current month/year to today
CURRENT
}
61 changes: 34 additions & 27 deletions common/src/main/java/com/tencent/supersonic/common/util/DateUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import lombok.extern.slf4j.Slf4j;

import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
Expand All @@ -16,7 +17,6 @@
import java.time.temporal.TemporalAdjusters;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Objects;
Expand All @@ -27,14 +27,8 @@ public class DateUtils {
public static final String DATE_FORMAT = "yyyy-MM-dd";
public static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static final String FORMAT = "yyyyMMddHHmmss";

public static Integer currentYear() {
Date date = new Date();
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
String time = dateFormat.format(date).replaceAll("-", "");
int year = Integer.parseInt(time.substring(0, 4));
return year;
}
private static final DateTimeFormatter dateTimeFormatter =
DateTimeFormatter.ofPattern(DATE_FORMAT);

public static DateTimeFormatter getDateFormatter(String date, String[] formats) {
for (int i = 0; i < formats.length; i++) {
Expand All @@ -43,8 +37,8 @@ public static DateTimeFormatter getDateFormatter(String date, String[] formats)
try {
dateFormat.parse(date);
return DateTimeFormatter.ofPattern(format);
} catch (Exception e) {
log.info("date parse has a exception:{}", e.toString());
} catch (ParseException e) {
log.warn("date parse has a exception:{}", e.toString());
}
}
return DateTimeFormatter.ofPattern(formats[0]);
Expand All @@ -57,55 +51,67 @@ public static DateTimeFormatter getTimeFormatter(String date, String[] formats)
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(format);
LocalDateTime.parse(date, dateTimeFormatter);
return dateTimeFormatter;
} catch (Exception e) {
log.info("date parse has a exception:{}", e.toString());
} catch (DateTimeParseException e) {
log.warn("date parse has a exception:{}", e.toString());
}
}
return DateTimeFormatter.ofPattern(formats[0]);
}

public static String getBeforeDate(int intervalDay) {
Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date());
calendar.add(Calendar.DAY_OF_MONTH, -intervalDay);
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
return dateFormat.format(calendar.getTime());
return getBeforeDate(intervalDay, DatePeriodEnum.DAY);
}

public static String getBeforeDate(int intervalDay, DatePeriodEnum datePeriodEnum) {
if (Objects.isNull(datePeriodEnum)) {
return getBeforeDate(intervalDay);
datePeriodEnum = DatePeriodEnum.DAY;
}
SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
String currentDate = dateFormat.format(new Date());
return getBeforeDate(currentDate, intervalDay, datePeriodEnum);
}

public static String getBeforeDate(String currentDate, DatePeriodEnum datePeriodEnum) {
LocalDate specifiedDate = LocalDate.parse(currentDate, dateTimeFormatter);
LocalDate startDate;
switch (datePeriodEnum) {
case MONTH:
startDate = specifiedDate.withDayOfMonth(1);
break;
case YEAR:
startDate = specifiedDate.withDayOfYear(1);
break;
default:
startDate = specifiedDate;
}

return startDate.format(dateTimeFormatter);
}

public static String getBeforeDate(
String date, int intervalDay, DatePeriodEnum datePeriodEnum) {
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern(DATE_FORMAT);
LocalDate currentDate = LocalDate.parse(date, dateTimeFormatter);
String currentDate, int intervalDay, DatePeriodEnum datePeriodEnum) {
LocalDate specifiedDate = LocalDate.parse(currentDate, dateTimeFormatter);
LocalDate result = null;
switch (datePeriodEnum) {
case DAY:
result = currentDate.minusDays(intervalDay);
result = specifiedDate.minusDays(intervalDay);
break;
case WEEK:
result = currentDate.minusWeeks(intervalDay);
result = specifiedDate.minusWeeks(intervalDay);
if (intervalDay == 0) {
result =
result.with(
TemporalAdjusters.previousOrSame(java.time.DayOfWeek.MONDAY));
}
break;
case MONTH:
result = currentDate.minusMonths(intervalDay);
result = specifiedDate.minusMonths(intervalDay);
if (intervalDay == 0) {
result = result.with(TemporalAdjusters.firstDayOfMonth());
}
break;
case QUARTER:
result = currentDate.minusMonths(intervalDay * 3L);
result = specifiedDate.minusMonths(intervalDay * 3L);
if (intervalDay == 0) {
TemporalAdjuster firstDayOfQuarter =
temporal -> {
Expand All @@ -119,7 +125,7 @@ public static String getBeforeDate(
}
break;
case YEAR:
result = currentDate.minusYears(intervalDay);
result = specifiedDate.minusYears(intervalDay);
if (intervalDay == 0) {
result = result.with(TemporalAdjusters.firstDayOfYear());
}
Expand All @@ -129,6 +135,7 @@ public static String getBeforeDate(
if (Objects.nonNull(result)) {
return result.format(DateTimeFormatter.ofPattern(DATE_FORMAT));
}

return null;
}

Expand Down
10 changes: 0 additions & 10 deletions headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SemanticSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,16 +118,6 @@ public List<SchemaElement> getTags() {
return tags;
}

public List<SchemaElement> getTags(Long dataSetId) {
List<SchemaElement> tags = new ArrayList<>();
dataSetSchemaList.stream()
.filter(
schemaElement ->
dataSetId.equals(schemaElement.getDataSet().getDataSetId()))
.forEach(d -> tags.addAll(d.getTags()));
return tags;
}

public List<SchemaElement> getTerms() {
List<SchemaElement> terms = new ArrayList<>();
dataSetSchemaList.stream().forEach(d -> terms.addAll(d.getTerms()));
Expand Down
92 changes: 31 additions & 61 deletions ...ss/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/S2SqlDateHelper.java
View file
Original file line number Diff line number Diff line change
@@ -1,95 +1,65 @@
package com.tencent.supersonic.headless.chat.corrector;

import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.common.util.DateUtils;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Objects;

public class S2SqlDateHelper {

public static String getReferenceDate(ChatQueryContext chatQueryContext, Long dataSetId) {
String defaultDate = DateUtils.getBeforeDate(0);
if (Objects.isNull(dataSetId)) {
return defaultDate;
}
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (dataSetSchema == null || dataSetSchema.getTagTypeTimeDefaultConfig() == null) {
return defaultDate;
}
TimeDefaultConfig tagTypeTimeDefaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
String partitionTimeFormat = dataSetSchema.getPartitionTimeFormat();
return getDefaultDate(defaultDate, tagTypeTimeDefaultConfig, partitionTimeFormat).getLeft();
}

public static Pair<String, String> getStartEndDate(
ChatQueryContext chatQueryContext, Long dataSetId, QueryType queryType) {
String defaultDate = DateUtils.getBeforeDate(0);
if (Objects.isNull(dataSetId)) {
return Pair.of(defaultDate, defaultDate);
}
DataSetSchema dataSetSchema =
chatQueryContext.getSemanticSchema().getDataSetSchemaMap().get(dataSetId);
if (Objects.isNull(dataSetSchema)) {
return Pair.of(defaultDate, defaultDate);
}
TimeDefaultConfig defaultConfig = dataSetSchema.getMetricTypeTimeDefaultConfig();
if (QueryType.DETAIL.equals(queryType) && defaultConfig.getUnit() >= 0) {
defaultConfig = dataSetSchema.getTagTypeTimeDefaultConfig();
}
String partitionTimeFormat = dataSetSchema.getPartitionTimeFormat();
return getDefaultDate(defaultDate, defaultConfig, partitionTimeFormat);
public static Pair<String, String> calculateDateRange(
TimeDefaultConfig timeConfig, String timeFormat) {
return calculateDateRange(DateUtils.getBeforeDate(0), timeConfig, timeFormat);
}

private static Pair<String, String> getDefaultDate(
String defaultDate, TimeDefaultConfig defaultConfig, String partitionTimeFormat) {
if (defaultConfig == null) {
public static Pair<String, String> calculateDateRange(
String currentDate, TimeDefaultConfig timeConfig, String timeFormat) {
Integer unit = timeConfig.getUnit();
if (timeConfig == null || unit == null || unit < 0) {
return Pair.of(null, null);
}
Integer unit = defaultConfig.getUnit();
if (unit == null) {
return Pair.of(defaultDate, defaultDate);
}

// If the unit is set to less than 0, then do not add relative date.
if (unit < 0) {
return Pair.of(null, null);
TimeMode timeMode = timeConfig.getTimeMode();
DatePeriodEnum datePeriod = DatePeriodEnum.get(timeConfig.getPeriod());
String startDate;
String endDate;
switch (timeMode) {
case CURRENT:
startDate = DateUtils.getBeforeDate(currentDate, datePeriod);
endDate = currentDate;
break;
case RECENT:
startDate = DateUtils.getBeforeDate(currentDate, unit, datePeriod);
endDate = currentDate;
break;
case LAST:
default:
startDate = DateUtils.getBeforeDate(currentDate, unit, datePeriod);
endDate = DateUtils.getBeforeDate(currentDate, unit, datePeriod);
break;
}

String period = defaultConfig.getPeriod();
TimeMode timeMode = defaultConfig.getTimeMode();
DatePeriodEnum datePeriodEnum = DatePeriodEnum.get(period);

String startDate = DateUtils.getBeforeDate(unit, datePeriodEnum);
String endDate = DateUtils.getBeforeDate(0, DatePeriodEnum.DAY);

if (unit == 0 || TimeMode.LAST.equals(timeMode)) {
endDate = startDate;
}
if (StringUtils.isNotBlank(partitionTimeFormat)) {
startDate = formatDate(startDate, partitionTimeFormat);
endDate = formatDate(endDate, partitionTimeFormat);
if (StringUtils.isNotBlank(timeFormat)) {
startDate = reformatDate(startDate, timeFormat);
endDate = reformatDate(endDate, timeFormat);
}
return Pair.of(startDate, endDate);
}

private static String formatDate(String dateStr, String format) {
private static String reformatDate(String dateStr, String format) {
try {
// Assuming the input date format is "yyyy-MM-dd"
SimpleDateFormat inputFormat = new SimpleDateFormat(DateUtils.DATE_FORMAT);
Date date = inputFormat.parse(dateStr);
SimpleDateFormat outputFormat = new SimpleDateFormat(format);
return outputFormat.format(date);
} catch (Exception e) {
} catch (ParseException e) {
// Handle the exception, maybe log it and return the original dateStr
return dateStr;
}
Expand Down
25 changes: 17 additions & 8 deletions ...less/chat/src/main/java/com/tencent/supersonic/headless/chat/corrector/TimeCorrector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,12 @@
import com.tencent.supersonic.common.jsqlparser.SqlAddHelper;
import com.tencent.supersonic.common.jsqlparser.SqlDateSelectHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.DataSetSchema;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.chat.ChatQueryContext;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
Expand All @@ -27,10 +30,10 @@ public class TimeCorrector extends BaseSemanticCorrector {
public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo semanticParseInfo) {
if (containsPartitionDimensions(chatQueryContext, semanticParseInfo)) {
addDateIfNotExist(chatQueryContext, semanticParseInfo);
addLowerBoundDate(semanticParseInfo);
} else {
removeDateIfExist(chatQueryContext, semanticParseInfo);
}
addLowerBoundDate(semanticParseInfo);
}

private void addDateIfNotExist(
Expand All @@ -48,15 +51,21 @@ private void addDateIfNotExist(
}
String partitionDimension = dataSetSchema.getPartitionDimension().getName();
if (CollectionUtils.isEmpty(whereFields) || !whereFields.contains(partitionDimension)) {
Pair<String, String> startEndDate =
S2SqlDateHelper.getStartEndDate(
chatQueryContext, dataSetId, semanticParseInfo.getQueryType());
TimeDefaultConfig timeConfig;
QueryConfig queryConfig = dataSetSchema.getQueryConfig();
if (QueryType.METRIC.equals(semanticParseInfo.getQueryType())) {
timeConfig = queryConfig.getMetricTypeDefaultConfig().getTimeDefaultConfig();
} else {
timeConfig = queryConfig.getTagTypeDefaultConfig().getTimeDefaultConfig();
}

if (isValidDateRange(startEndDate)) {
String timeFormat = dataSetSchema.getPartitionTimeFormat();
Pair<String, String> dateRange =
S2SqlDateHelper.calculateDateRange(timeConfig, timeFormat);
if (isValidDateRange(dateRange)) {
correctS2SQL = SqlAddHelper.addParenthesisToWhere(correctS2SQL);
String startDateLeft = startEndDate.getLeft();
String endDateRight = startEndDate.getRight();

String startDateLeft = dateRange.getLeft();
String endDateRight = dateRange.getRight();
String condExpr =
String.format(
" ( %s >= '%s' and %s <= '%s' )",
Expand Down
Loading

0 comments on commit b676538

Please sign in to comment.