Upgrade dependencies to fix vulnerabitilites (apache#13892)

vvivekiyer · Aug 27, 2024 · 64eb323 · 64eb323
1 parent 7d82474
commit 64eb323
Show file tree

Hide file tree

Showing 16 changed files with 424 additions and 304 deletions.
diff --git a/pinot-compatibility-verifier/pom.xml b/pinot-compatibility-verifier/pom.xml
@@ -34,7 +34,6 @@
 
   <properties>
     <pinot.root>${basedir}/..</pinot.root>
-    <kafka.lib.version>2.8.2</kafka.lib.version>
   </properties>
 
   <build>

diff --git a/pinot-connectors/pinot-spark-2-connector/pom.xml b/pinot-connectors/pinot-spark-2-connector/pom.xml
@@ -32,9 +32,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../..</pinot.root>
-    <spark.version>2.4.8</spark.version>
-    <scalaxml.version>2.3.0</scalaxml.version>
-    <scalatest.version>3.2.18</scalatest.version>
     <shadeBase>org.apache.pinot.\$internal</shadeBase>
   </properties>
 
@@ -48,14 +45,18 @@
         <dependency>
           <groupId>org.scala-lang.modules</groupId>
           <artifactId>scala-xml_${scala.compat.version}</artifactId>
-          <version>${scalaxml.version}</version>
         </dependency>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-sql_${scala.compat.version}</artifactId>
-          <version>${spark.version}</version>
+          <version>${spark2.version}</version>
           <scope>provided</scope>
           <exclusions>
+            <!-- Exclude it here and include explicitly because it has "hadoop2" classifier -->
+            <exclusion>
+              <groupId>org.apache.avro</groupId>
+              <artifactId>avro-mapred</artifactId>
+            </exclusion>
             <exclusion>
               <groupId>log4j</groupId>
               <artifactId>log4j</artifactId>
@@ -66,6 +67,16 @@
             </exclusion>
           </exclusions>
         </dependency>
+        <dependency>
+          <groupId>org.apache.avro</groupId>
+          <artifactId>avro-mapred</artifactId>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j-impl</artifactId>
+          <scope>provided</scope>
+        </dependency>
         <dependency>
           <groupId>org.scala-lang</groupId>
           <artifactId>scala-library</artifactId>
@@ -75,7 +86,6 @@
         <dependency>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest_${scala.compat.version}</artifactId>
-          <version>${scalatest.version}</version>
           <scope>test</scope>
         </dependency>
       </dependencies>

diff --git a/pinot-connectors/pinot-spark-3-connector/pom.xml b/pinot-connectors/pinot-spark-3-connector/pom.xml
@@ -32,8 +32,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../..</pinot.root>
-    <spark.version>3.5.2</spark.version>
-    <scalatest.version>3.2.18</scalatest.version>
     <shadeBase>org.apache.pinot.\$internal</shadeBase>
   </properties>
 
@@ -47,7 +45,7 @@
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-sql_${scala.compat.version}</artifactId>
-          <version>${spark.version}</version>
+          <version>${spark3.version}</version>
           <scope>provided</scope>
         </dependency>
         <dependency>
@@ -59,7 +57,6 @@
         <dependency>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest_${scala.compat.version}</artifactId>
-          <version>${scalatest.version}</version>
           <scope>test</scope>
         </dependency>
       </dependencies>

diff --git a/pinot-connectors/pinot-spark-common/pom.xml b/pinot-connectors/pinot-spark-common/pom.xml
@@ -32,9 +32,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../..</pinot.root>
-    <circe.version>0.14.9</circe.version>
-    <scalaxml.version>2.3.0</scalaxml.version>
-    <scalatest.version>3.2.18</scalatest.version>
   </properties>
 
   <profiles>
@@ -51,17 +48,14 @@
         <dependency>
           <groupId>org.scala-lang.modules</groupId>
           <artifactId>scala-xml_${scala.compat.version}</artifactId>
-          <version>${scalaxml.version}</version>
         </dependency>
         <dependency>
           <groupId>io.circe</groupId>
           <artifactId>circe-parser_${scala.compat.version}</artifactId>
-          <version>${circe.version}</version>
         </dependency>
         <dependency>
           <groupId>io.circe</groupId>
           <artifactId>circe-generic_${scala.compat.version}</artifactId>
-          <version>${circe.version}</version>
         </dependency>
         <dependency>
           <groupId>org.scala-lang</groupId>
@@ -72,7 +66,6 @@
         <dependency>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest_${scala.compat.version}</artifactId>
-          <version>${scalatest.version}</version>
           <scope>test</scope>
         </dependency>
       </dependencies>

diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml
@@ -34,9 +34,6 @@
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
     <shade.phase.prop>package</shade.phase.prop>
-    <scala.major.version>2.11</scala.major.version>
-    <spark.version>2.4.6</spark.version>
-    <scala.minor.version>2.11.12</scala.minor.version>
   </properties>
 
   <dependencies>
@@ -46,25 +43,14 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-core_${scala.major.version}</artifactId>
-      <version>${spark.version}</version>
+      <artifactId>spark-core_${scala.compat.version}</artifactId>
+      <version>${spark2.version}</version>
       <scope>provided</scope>
       <exclusions>
+        <!-- Exclude it here and include explicitly because it has "hadoop2" classifier -->
         <exclusion>
-          <groupId>com.zaxxer</groupId>
-          <artifactId>HikariCP-java7</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.twitter</groupId>
-          <artifactId>chill_2.11</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.twitter</groupId>
-          <artifactId>chill-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.curator</groupId>
-          <artifactId>curator-recipes</artifactId>
+          <groupId>org.apache.avro</groupId>
+          <artifactId>avro-mapred</artifactId>
         </exclusion>
         <exclusion>
           <groupId>log4j</groupId>
@@ -76,10 +62,19 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro-mapred</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
-      <version>${scala.minor.version}</version>
       <scope>provided</scope>
     </dependency>
 
@@ -93,13 +88,11 @@
     <dependency>
       <groupId>com.esotericsoftware.kryo</groupId>
       <artifactId>kryo</artifactId>
-      <version>2.24.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>chill_2.11</artifactId>
-      <version>0.10.0</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>

diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml
@@ -34,7 +34,6 @@
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
     <shade.phase.prop>package</shade.phase.prop>
-    <spark.version>3.5.2</spark.version>
   </properties>
 
   <dependencies>
@@ -45,24 +44,12 @@
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.compat.version}</artifactId>
-      <version>${spark.version}</version>
+      <version>${spark3.version}</version>
       <scope>provided</scope>
       <exclusions>
         <exclusion>
-          <groupId>com.zaxxer</groupId>
-          <artifactId>HikariCP-java7</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.twitter</groupId>
-          <artifactId>chill_2.11</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.twitter</groupId>
-          <artifactId>chill-java</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.curator</groupId>
-          <artifactId>curator-recipes</artifactId>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
         </exclusion>
         <exclusion>
           <groupId>log4j</groupId>
@@ -72,16 +59,11 @@
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-log4j12</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>commons-logging</groupId>
-          <artifactId>commons-logging</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
       <artifactId>scala-library</artifactId>
-      <version>${scala.version}</version>
       <scope>provided</scope>
     </dependency>
 

diff --git a/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml b/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml
@@ -39,6 +39,11 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
     </dependency>
+    <!-- Replace bcprov-jdk15on which is excluded from hadoop-common -->
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk18on</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.codehaus.woodstox</groupId>
       <artifactId>stax2-api</artifactId>

diff --git a/pinot-plugins/pinot-input-format/pinot-confluent-avro/pom.xml b/pinot-plugins/pinot-input-format/pinot-confluent-avro/pom.xml
@@ -33,7 +33,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
-    <kafka.lib.version>2.8.2</kafka.lib.version>
     <shade.phase.prop>package</shade.phase.prop>
   </properties>
   <repositories>
@@ -50,23 +49,14 @@
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
-      <version>${kafka.lib.version}</version>
     </dependency>
     <dependency>
       <groupId>io.confluent</groupId>
       <artifactId>kafka-schema-registry-client</artifactId>
-      <version>${confluent.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.kafka</groupId>
-          <artifactId>kafka-clients</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.confluent</groupId>
       <artifactId>kafka-avro-serializer</artifactId>
-      <version>${confluent.version}</version>
     </dependency>
   </dependencies>
 </project>
diff --git a/pinot-plugins/pinot-input-format/pinot-orc/pom.xml b/pinot-plugins/pinot-input-format/pinot-orc/pom.xml
@@ -41,6 +41,11 @@
       <artifactId>hadoop-common</artifactId>
       <scope>${hadoop.dependencies.scope}</scope>
     </dependency>
+    <!-- Replace bcprov-jdk15on which is excluded from hadoop-common -->
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk18on</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>

diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml b/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml
@@ -49,6 +49,10 @@
       <artifactId>hadoop-common</artifactId>
       <scope>${hadoop.dependencies.scope}</scope>
     </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk18on</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client-runtime</artifactId>

diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml b/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml
@@ -34,7 +34,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
-    <kafka.lib.version>2.8.2</kafka.lib.version>
     <shade.phase.prop>package</shade.phase.prop>
   </properties>
   <repositories>
@@ -60,23 +59,14 @@
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
-      <version>${kafka.lib.version}</version>
     </dependency>
     <dependency>
       <groupId>io.confluent</groupId>
       <artifactId>kafka-schema-registry-client</artifactId>
-      <version>${confluent.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.kafka</groupId>
-          <artifactId>kafka-clients</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.confluent</groupId>
       <artifactId>kafka-protobuf-serializer</artifactId>
-      <version>${confluent.version}</version>
     </dependency>
 
     <dependency>

diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml
@@ -33,7 +33,6 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
-    <kafka.lib.version>2.8.2</kafka.lib.version>
     <shade.phase.prop>package</shade.phase.prop>
   </properties>
 
@@ -46,12 +45,10 @@
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
-      <version>${kafka.lib.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka_${scala.compat.version}</artifactId>
-      <version>${kafka.lib.version}</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.module</groupId>

diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/pom.xml
@@ -33,22 +33,9 @@
   <url>https://pinot.apache.org/</url>
   <properties>
     <pinot.root>${basedir}/../../..</pinot.root>
-    <reactive.version>1.0.2</reactive.version>
     <localstack-utils.version>0.2.23</localstack-utils.version>
   </properties>
 
-  <dependencyManagement>
-    <dependencies>
-      <dependency>
-        <groupId>software.amazon.awssdk</groupId>
-        <artifactId>bom</artifactId>
-        <version>${aws.sdk.version}</version>
-        <type>pom</type>
-        <scope>import</scope>
-      </dependency>
-    </dependencies>
-  </dependencyManagement>
-
   <dependencies>
     <dependency>
       <groupId>software.amazon.awssdk</groupId>