diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml index 9fedfd444a4..570d83dff21 100644 --- a/pinot-common/pom.xml +++ b/pinot-common/pom.xml @@ -225,22 +225,10 @@ org.apache.calcite calcite-core - - - com.google.protobuf - protobuf-java - - org.apache.calcite calcite-babel - - - com.google.protobuf - protobuf-java - - org.testng @@ -405,12 +393,6 @@ io.grpc grpc-protobuf - - - com.google.protobuf - protobuf-java - - io.grpc diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml index 87d5d8d21d0..529bd4912b9 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml @@ -77,10 +77,6 @@ org.slf4j slf4j-log4j12 - - com.google.protobuf - protobuf-java - com.zaxxer HikariCP-java7 diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml index fdb8906a2da..0d11fe655eb 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3.2/pom.xml @@ -78,10 +78,6 @@ org.slf4j slf4j-log4j12 - - com.google.protobuf - protobuf-java - com.zaxxer HikariCP-java7 diff --git a/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml b/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml index 4300c09d899..49eac2ea89e 100644 --- a/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml +++ b/pinot-plugins/pinot-file-system/pinot-gcs/pom.xml @@ -52,10 +52,6 @@ com.google.guava guava - - com.google.protobuf - protobuf-java - com.google.j2objc j2objc-annotations @@ -87,22 +83,12 @@ com.google.api.grpc proto-google-common-protos 1.18.1 - - - com.google.protobuf - protobuf-java - - com.google.api.grpc proto-google-iam-v1 1.0.0 - - com.google.protobuf - protobuf-java - com.google.api.grpc proto-google-common-protos diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml b/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml index f56d8493a93..7783fb63283 100644 --- a/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml +++ b/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml @@ -58,12 +58,6 @@ com.github.os72 protobuf-dynamic 1.0.1 - - - com.google.protobuf - protobuf-java - - diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java b/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java index 5215ff94d0d..83c6c7fc5a1 100644 --- a/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java +++ b/pinot-plugins/pinot-input-format/pinot-protobuf/src/main/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractor.java @@ -62,7 +62,7 @@ private Object getFieldValue(Descriptors.FieldDescriptor fieldDescriptor, Messag // Note w.r.t proto3 - If a field is not declared with optional keyword, there's no way to distinguish // if its explicitly set to a proto default or not been set at all i.e hasField() returns false // and we would use null. - if (fieldDescriptor.isRepeated() || !fieldDescriptor.isOptional() || message.hasField(fieldDescriptor)) { + if (fieldDescriptor.isRepeated() || !fieldDescriptor.hasPresence() || message.hasField(fieldDescriptor)) { return message.getField(fieldDescriptor); } else { return null; diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractorLowLevelTest.java b/pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractorLowLevelTest.java new file mode 100644 index 00000000000..b57f61ac631 --- /dev/null +++ b/pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufRecordExtractorLowLevelTest.java @@ -0,0 +1,262 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.plugin.inputformat.protobuf; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import org.apache.commons.io.FileUtils; +import org.apache.pinot.spi.data.readers.GenericRow; +import org.apache.pinot.spi.data.readers.RecordExtractorConfig; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + + +/** + * Tests for the {@link ProtoBufRecordExtractor} + */ +public class ProtoBufRecordExtractorLowLevelTest { + protected final File _tempDir = Files.createTempDirectory("ProtoBufRecordExtractorLowLevelTest").toFile(); + + private final File _dataFile = new File(_tempDir, "test_complex_proto.data"); + private ProtoBufRecordExtractor _extractor; + + private static final String DESCRIPTOR_FILE = "complex_types.desc"; + + private static final String STRING_FIELD = "string_field"; + private static final String INT_FIELD = "int_field"; + private static final String LONG_FIELD = "long_field"; + private static final String DOUBLE_FIELD = "double_field"; + private static final String FLOAT_FIELD = "float_field"; + private static final String BOOL_FIELD = "bool_field"; + private static final String BYTES_FIELD = "bytes_field"; + private static final String NULLABLE_STRING_FIELD = "nullable_string_field"; + private static final String NULLABLE_INT_FIELD = "nullable_int_field"; + private static final String NULLABLE_LONG_FIELD = "nullable_long_field"; + private static final String NULLABLE_DOUBLE_FIELD = "nullable_double_field"; + private static final String NULLABLE_FLOAT_FIELD = "nullable_float_field"; + private static final String NULLABLE_BOOL_FIELD = "nullable_bool_field"; + private static final String NULLABLE_BYTES_FIELD = "nullable_bytes_field"; + private static final String REPEATED_STRINGS = "repeated_strings"; + private static final String NESTED_MESSAGE = "nested_message"; + private static final String REPEATED_NESTED_MESSAGES = "repeated_nested_messages"; + private static final String COMPLEX_MAP = "complex_map"; + private static final String SIMPLE_MAP = "simple_map"; + private static final String ENUM_FIELD = "enum_field"; + private static final String NESTED_INT_FIELD = "nested_int_field"; + private static final String NESTED_STRING_FIELD = "nested_string_field"; + + public ProtoBufRecordExtractorLowLevelTest() + throws IOException { + } + + @AfterClass + public void cleanUp() + throws IOException { + FileUtils.forceDelete(_tempDir); + } + + @BeforeMethod + public void beforeMethod() { + _extractor = new ProtoBufRecordExtractor(); + _extractor.init(null, new RecordExtractorConfig() { + }); + } + + /** + * For each case, we have: + *
    + *
  1. The name of the field to change
    2. + *
  2. A valid protobuf value which is not the default field value
    3. + *
  3. The expected pinot value
    4. + *
+ */ + @DataProvider(name = "normalCases") + public Object[][] normalCases() { + return new Object[][]{ + new Object[] {STRING_FIELD, "some text", "some text"}, + new Object[] {NULLABLE_STRING_FIELD, "some text", "some text"}, + + new Object[] {INT_FIELD, 123, 123}, + new Object[] {NULLABLE_INT_FIELD, 123, 123}, + + new Object[] {LONG_FIELD, 123L, 123L}, + new Object[] {NULLABLE_LONG_FIELD, 123L, 123L}, + + new Object[] {DOUBLE_FIELD, 0.5d, 0.5d}, + new Object[] {NULLABLE_DOUBLE_FIELD, 0.5d, 0.5d}, + + new Object[] {FLOAT_FIELD, 0.5f, 0.5f}, + new Object[] {NULLABLE_FLOAT_FIELD, 0.5f, 0.5f}, + + new Object[] {BYTES_FIELD, ByteString.copyFrom(new byte[] {0, 1, 2, 3}), new byte[] {0, 1, 2, 3}}, + new Object[] {NULLABLE_BYTES_FIELD, ByteString.copyFrom(new byte[] {0, 1, 2, 3}), new byte[] {0, 1, 2, 3}}, + + new Object[] {BOOL_FIELD, true, "true"}, + new Object[] {NULLABLE_BOOL_FIELD, true, "true"} + }; + } + + /** + * For each case, we have: + *
    + *
  1. The name of the field to change
    2. + *
  2. A valid protobuf value which is the default field value
    3. + *
  3. The expected pinot value
    4. + *
+ */ + @Test(dataProvider = "normalCases") + public void whenNormalCases(String fieldName, Object protoVal, Object pinotVal) { + Descriptors.FieldDescriptor fd = ComplexTypes.TestMessage.getDescriptor().findFieldByName(fieldName); + ComplexTypes.TestMessage.Builder messageBuilder = ComplexTypes.TestMessage.newBuilder(); + messageBuilder.setField(fd, protoVal); + + GenericRow row = new GenericRow(); + _extractor.extract(messageBuilder.build(), row); + + Assert.assertEquals(row.getValue(fd.getName()), pinotVal); + } + + /** + * For each case, we have: + *
    + *
  1. The name of the field read
    2. + *
  2. The expected pinot value when the value is not set
    3. + *
+ */ + @DataProvider(name = "defaultCases") + public Object[][] defaultCases() { + return new Object[][]{ + new Object[] {STRING_FIELD, "", ""}, + new Object[] {NULLABLE_STRING_FIELD, "", ""}, + + new Object[] {INT_FIELD, 0, 0}, + new Object[] {NULLABLE_INT_FIELD, 0, 0}, + + new Object[] {LONG_FIELD, 0L, 0L}, + new Object[] {NULLABLE_LONG_FIELD, 0L, 0L}, + + new Object[] {DOUBLE_FIELD, 0d, 0d}, + new Object[] {NULLABLE_DOUBLE_FIELD, 0d, 0d}, + + new Object[] {FLOAT_FIELD, 0f, 0f}, + new Object[] {NULLABLE_FLOAT_FIELD, 0f, 0f}, + + new Object[] {BYTES_FIELD, ByteString.empty(), new byte[] {}}, + new Object[] {NULLABLE_BYTES_FIELD, ByteString.empty(), new byte[] {}}, + + new Object[] {BOOL_FIELD, false, "false"}, + new Object[] {NULLABLE_BOOL_FIELD, false, "false"} + }; + } + + @Test(dataProvider = "defaultCases") + public void whenDefaultCases(String fieldName, Object protoValue, Object pinotVal) { + Descriptors.FieldDescriptor fd = ComplexTypes.TestMessage.getDescriptor().findFieldByName(fieldName); + ComplexTypes.TestMessage.Builder messageBuilder = ComplexTypes.TestMessage.newBuilder(); + + Assert.assertEquals(protoValue, fd.getDefaultValue()); + messageBuilder.setField(fd, protoValue); + + GenericRow row = new GenericRow(); + _extractor.extract(messageBuilder.build(), row); + + Assert.assertEquals(row.getValue(fd.getName()), pinotVal); + } + + @DataProvider(name = "unsetCases") + public Object[][] unsetCases() { + return new Object[][]{ + new Object[] {STRING_FIELD, ""}, + new Object[] {NULLABLE_STRING_FIELD, null}, + + new Object[] {INT_FIELD, 0}, + new Object[] {NULLABLE_INT_FIELD, null}, + + new Object[] {LONG_FIELD, 0L}, + new Object[] {NULLABLE_LONG_FIELD, null}, + + new Object[] {DOUBLE_FIELD, 0d}, + new Object[] {NULLABLE_DOUBLE_FIELD, null}, + + new Object[] {FLOAT_FIELD, 0f}, + new Object[] {NULLABLE_FLOAT_FIELD, null}, + + new Object[] {BYTES_FIELD, new byte[] {}}, + new Object[] {NULLABLE_BYTES_FIELD, null}, + + new Object[] {BOOL_FIELD, "false"}, + new Object[] {NULLABLE_BOOL_FIELD, null} + }; + } + + @Test(dataProvider = "unsetCases") + public void whenUnset(String fieldName, Object pinotVal) { + Descriptors.FieldDescriptor fd = ComplexTypes.TestMessage.getDescriptor().findFieldByName(fieldName); + ComplexTypes.TestMessage.Builder messageBuilder = ComplexTypes.TestMessage.newBuilder(); + + GenericRow row = new GenericRow(); + _extractor.extract(messageBuilder.build(), row); + + Assert.assertEquals(row.getValue(fd.getName()), pinotVal); + } + + @DataProvider(name = "clearCases") + public Object[][] clearCases() { + return new Object[][]{ + new Object[] {STRING_FIELD, ""}, + new Object[] {NULLABLE_STRING_FIELD, null}, + + new Object[] {INT_FIELD, 0}, + new Object[] {NULLABLE_INT_FIELD, null}, + + new Object[] {LONG_FIELD, 0L}, + new Object[] {NULLABLE_LONG_FIELD, null}, + + new Object[] {DOUBLE_FIELD, 0d}, + new Object[] {NULLABLE_DOUBLE_FIELD, null}, + + new Object[] {FLOAT_FIELD, 0f}, + new Object[] {NULLABLE_FLOAT_FIELD, null}, + + new Object[] {BYTES_FIELD, new byte[] {}}, + new Object[] {NULLABLE_BYTES_FIELD, null}, + + new Object[] {BOOL_FIELD, "false"}, + new Object[] {NULLABLE_BOOL_FIELD, null} + }; + } + + @Test(dataProvider = "clearCases") + public void whenClear(String fieldName, Object pinotVal) { + Descriptors.FieldDescriptor fd = ComplexTypes.TestMessage.getDescriptor().findFieldByName(fieldName); + ComplexTypes.TestMessage.Builder messageBuilder = ComplexTypes.TestMessage.newBuilder(); + messageBuilder.clearField(fd); + + GenericRow row = new GenericRow(); + _extractor.extract(messageBuilder.build(), row); + + Assert.assertEquals(row.getValue(fd.getName()), pinotVal); + } +} diff --git a/pinot-server/pom.xml b/pinot-server/pom.xml index 68ee0f586de..a0254362c91 100644 --- a/pinot-server/pom.xml +++ b/pinot-server/pom.xml @@ -198,12 +198,6 @@ io.grpc grpc-protobuf - - - com.google.protobuf - protobuf-java - - io.grpc diff --git a/pom.xml b/pom.xml index 8d540dc4b4b..b8845ecf954 100644 --- a/pom.xml +++ b/pom.xml @@ -166,7 +166,7 @@ -Xms4g -Xmx4g - 3.22.0 + 3.24.3 1.53.0 5.5.3 @@ -714,10 +714,6 @@ org.slf4j slf4j-log4j12 - - com.google.protobuf - protobuf-java - org.apache.yetus audience-annotations @@ -806,10 +802,6 @@ org.slf4j slf4j-log4j12 - - com.google.protobuf - protobuf-java - @@ -890,12 +882,6 @@ org.apache.orc orc-core 1.5.9 - - - com.google.protobuf - protobuf-java - - org.apache.avro @@ -947,18 +933,10 @@ calcite-core ${calcite.version} - - com.google.protobuf - protobuf-java - org.apache.calcite.avatica avatica-metrics - - com.google.protobuf - protobuf-java - org.apache.httpcomponents httpclient