diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java index 4e79ecea244..8fc421f79e8 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentGenerationTaskRunner.java @@ -65,7 +65,7 @@ public class SegmentGenerationTaskRunner implements Serializable { public static final String DEPRECATED_USE_LOCAL_DIRECTORY_SEQUENCE_ID = "local.directory.sequence.id"; public static final String USE_GLOBAL_DIRECTORY_SEQUENCE_ID = "use.global.directory.sequence.id"; public static final String APPEND_UUID_TO_SEGMENT_NAME = "append.uuid.to.segment.name"; - public static final String EXCLUDE_TIMESTAMPS_IN_SEGMENT_NAME = "exclude.timestamps.in.segment.name"; + public static final String EXCLUDE_TIME_IN_SEGMENT_NAME = BatchConfigProperties.EXCLUDE_TIME_IN_SEGMENT_NAME; private final SegmentGenerationTaskSpec _taskSpec; @@ -135,15 +135,15 @@ private SegmentNameGenerator getSegmentNameGenerator(SegmentGeneratorConfig segm boolean appendUUIDToSegmentName = Boolean.parseBoolean(segmentNameGeneratorConfigs.get(APPEND_UUID_TO_SEGMENT_NAME)); - boolean excludeTimestampsInSegmentName = - Boolean.parseBoolean(segmentNameGeneratorConfigs.get(EXCLUDE_TIMESTAMPS_IN_SEGMENT_NAME)); + boolean excludeTimeInSegmentName = + Boolean.parseBoolean(segmentNameGeneratorConfigs.get(EXCLUDE_TIME_IN_SEGMENT_NAME)); switch (segmentNameGeneratorType) { case BatchConfigProperties.SegmentNameGeneratorType.FIXED: return new FixedSegmentNameGenerator(segmentNameGeneratorConfigs.get(SEGMENT_NAME)); case BatchConfigProperties.SegmentNameGeneratorType.SIMPLE: return new SimpleSegmentNameGenerator(tableName, segmentNameGeneratorConfigs.get(SEGMENT_NAME_POSTFIX), - appendUUIDToSegmentName, excludeTimestampsInSegmentName); + appendUUIDToSegmentName, excludeTimeInSegmentName); case BatchConfigProperties.SegmentNameGeneratorType.NORMALIZED_DATE: SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); DateTimeFormatSpec dateTimeFormatSpec = null; diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java index 64309ff6ccb..2ae4c349a3e 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/IngestionUtils.java @@ -167,7 +167,7 @@ private static SegmentNameGenerator getSegmentNameGenerator(BatchConfig batchCon case BatchConfigProperties.SegmentNameGeneratorType.SIMPLE: return new SimpleSegmentNameGenerator(rawTableName, batchConfig.getSegmentNamePostfix(), - batchConfig.isAppendUUIDToSegmentName(), batchConfig.isExcludeTimestampsFromSegmentName()); + batchConfig.isAppendUUIDToSegmentName(), batchConfig.isExcludeTimeFromSegmentName()); default: throw new IllegalStateException(String .format("Unsupported segmentNameGeneratorType: %s for table: %s", segmentNameGeneratorType, diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGenerator.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGenerator.java index bf342c4f71c..a1b28320edd 100644 --- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGenerator.java +++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGenerator.java @@ -40,14 +40,14 @@ public class SimpleSegmentNameGenerator implements SegmentNameGenerator { private final String _segmentNamePrefix; private final String _segmentNamePostfix; private final boolean _appendUUIDToSegmentName; - private final boolean _excludeTimestampsInSegmentName; + private final boolean _excludeTimeInSegmentName; public SimpleSegmentNameGenerator(String segmentNamePrefix, @Nullable String segmentNamePostfix) { this(segmentNamePrefix, segmentNamePostfix, false, false); } public SimpleSegmentNameGenerator(String segmentNamePrefix, @Nullable String segmentNamePostfix, - boolean appendUUIDToSegmentName, boolean excludeTimestampsInSegmentName) { + boolean appendUUIDToSegmentName, boolean excludeTimeInSegmentName) { Preconditions.checkArgument(segmentNamePrefix != null, "Missing segmentNamePrefix for SimpleSegmentNameGenerator"); SegmentNameUtils.validatePartialOrFullSegmentName(segmentNamePrefix); if (segmentNamePostfix != null) { @@ -56,21 +56,25 @@ public SimpleSegmentNameGenerator(String segmentNamePrefix, @Nullable String seg _segmentNamePrefix = segmentNamePrefix; _segmentNamePostfix = segmentNamePostfix; _appendUUIDToSegmentName = appendUUIDToSegmentName; - _excludeTimestampsInSegmentName = excludeTimestampsInSegmentName; + _excludeTimeInSegmentName = excludeTimeInSegmentName; } @Override public String generateSegmentName(int sequenceId, @Nullable Object minTimeValue, @Nullable Object maxTimeValue) { - if (minTimeValue != null) { - SegmentNameUtils.validatePartialOrFullSegmentName(minTimeValue.toString()); - } - if (maxTimeValue != null) { - SegmentNameUtils.validatePartialOrFullSegmentName(maxTimeValue.toString()); - } + if (_excludeTimeInSegmentName) { + return JOINER.join(_segmentNamePrefix, _segmentNamePostfix, sequenceId >= 0 ? sequenceId : null, + _appendUUIDToSegmentName ? UUID.randomUUID().toString() : null); + } else { + if (minTimeValue != null) { + SegmentNameUtils.validatePartialOrFullSegmentName(minTimeValue.toString()); + } + if (maxTimeValue != null) { + SegmentNameUtils.validatePartialOrFullSegmentName(maxTimeValue.toString()); + } - return JOINER.join(_segmentNamePrefix, _excludeTimestampsInSegmentName ? null : minTimeValue, - _excludeTimestampsInSegmentName ? null : maxTimeValue, _segmentNamePostfix, sequenceId >= 0 ? sequenceId : null, - _appendUUIDToSegmentName ? UUID.randomUUID() : null); + return JOINER.join(_segmentNamePrefix, minTimeValue, maxTimeValue, _segmentNamePostfix, + sequenceId >= 0 ? sequenceId : null, _appendUUIDToSegmentName ? UUID.randomUUID().toString() : null); + } } @Override @@ -81,7 +85,7 @@ public String toString() { stringBuilder.append(", segmentNamePostfix=").append(_segmentNamePostfix); } stringBuilder.append(", appendUUIDToSegmentName=").append(_appendUUIDToSegmentName); - stringBuilder.append(", excludeTimestampsInSegmentName=").append(_excludeTimestampsInSegmentName); + stringBuilder.append(", excludeTimeInSegmentName=").append(_excludeTimeInSegmentName); return stringBuilder.toString(); } } diff --git a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGeneratorTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGeneratorTest.java index 833ecab62d6..1edb9dcdef4 100644 --- a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGeneratorTest.java +++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/creator/name/SimpleSegmentNameGeneratorTest.java @@ -40,7 +40,7 @@ public void testWithoutSegmentNamePostfix() { SegmentNameGenerator segmentNameGenerator = new SimpleSegmentNameGenerator(TABLE_NAME, null); assertEquals(segmentNameGenerator.toString(), "SimpleSegmentNameGenerator: tableName=testTable, appendUUIDToSegmentName=false, " - + "excludeTimestampsInSegmentName=false"); + + "excludeTimeInSegmentName=false"); assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID, null, null), "testTable"); assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID, MIN_TIME_VALUE, MAX_TIME_VALUE), "testTable_1234_5678"); @@ -54,7 +54,7 @@ public void testWithSegmentNamePostfix() { SegmentNameGenerator segmentNameGenerator = new SimpleSegmentNameGenerator(TABLE_NAME, SEGMENT_NAME_POSTFIX); assertEquals(segmentNameGenerator.toString(), "SimpleSegmentNameGenerator: tableName=testTable, segmentNamePostfix=postfix, appendUUIDToSegmentName=false, " - + "excludeTimestampsInSegmentName=false"); + + "excludeTimeInSegmentName=false"); assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID, null, null), "testTable_postfix"); assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID, MIN_TIME_VALUE, MAX_TIME_VALUE), "testTable_1234_5678_postfix"); @@ -88,12 +88,12 @@ public void testWithMalFormedTableNameSegmentNamePostfixTimeValue() { } @Test - public void testWithExcludeTimestampsInSegmentName() { + public void testWithexcludeTimeInSegmentName() { SegmentNameGenerator segmentNameGenerator = new SimpleSegmentNameGenerator(TABLE_NAME, null, false, true); segmentNameGenerator.generateSegmentName(VALID_SEQUENCE_ID, MIN_TIME_VALUE, MAX_TIME_VALUE); assertEquals(segmentNameGenerator.toString(), "SimpleSegmentNameGenerator: tableName=testTable, appendUUIDToSegmentName=false, " - + "excludeTimestampsInSegmentName=true"); + + "excludeTimeInSegmentName=true"); assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID, null, null), "testTable"); assertEquals(segmentNameGenerator.generateSegmentName(INVALID_SEQUENCE_ID, MIN_TIME_VALUE, MAX_TIME_VALUE), "testTable"); diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfig.java index a4b5b3c770b..7c77b9b0ef2 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfig.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfig.java @@ -51,7 +51,7 @@ public class BatchConfig { private final String _segmentNamePostfix; private final boolean _excludeSequenceId; private final boolean _appendUUIDToSegmentName; - private final boolean _excludeTimestampsFromSegmentName; + private final boolean _excludeTimeFromSegmentName; private final String _sequenceId; private final String _pushMode; @@ -100,8 +100,8 @@ public BatchConfig(String tableNameWithType, Map batchConfigsMap _sequenceId = batchConfigsMap.get(BatchConfigProperties.SEQUENCE_ID); _appendUUIDToSegmentName = Boolean.parseBoolean(segmentNameGeneratorProps.get(BatchConfigProperties.APPEND_UUID_TO_SEGMENT_NAME)); - _excludeTimestampsFromSegmentName = - Boolean.parseBoolean(segmentNameGeneratorProps.get(BatchConfigProperties.EXCLUDE_TIMESTAMPS_IN_SEGMENT_NAME)); + _excludeTimeFromSegmentName = + Boolean.parseBoolean(segmentNameGeneratorProps.get(BatchConfigProperties.EXCLUDE_TIME_IN_SEGMENT_NAME)); _pushMode = IngestionConfigUtils.getPushMode(batchConfigsMap); _pushAttempts = IngestionConfigUtils.getPushAttempts(batchConfigsMap); @@ -193,8 +193,8 @@ public boolean isAppendUUIDToSegmentName() { return _appendUUIDToSegmentName; } - public boolean isExcludeTimestampsFromSegmentName() { - return _excludeTimestampsFromSegmentName; + public boolean isExcludeTimeFromSegmentName() { + return _excludeTimeFromSegmentName; } public String getPushMode() { diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java index da18c3c9bc2..0782b7a8ecc 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java @@ -62,7 +62,7 @@ private BatchConfigProperties() { public static final String FAIL_ON_EMPTY_SEGMENT = "fail.on.empty.segment"; public static final String AUTH_TOKEN = "authToken"; public static final String APPEND_UUID_TO_SEGMENT_NAME = "append.uuid.to.segment.name"; - public static final String EXCLUDE_TIMESTAMPS_IN_SEGMENT_NAME = "exclude.timestamps.in.segment.name"; + public static final String EXCLUDE_TIME_IN_SEGMENT_NAME = "exclude.time.in.segment.name"; public static final String OUTPUT_SEGMENT_DIR_URI = "output.segment.dir.uri";