This is an automated email from the ASF dual-hosted git repository. jackie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push: new 64eb3237d7 Upgrade dependencies to fix vulnerabitilites (#13892) 64eb3237d7 is described below commit 64eb3237d782a2b5dd092edae2415b8c1d12bf37 Author: Xiaotian (Jackie) Jiang <17555551+jackie-ji...@users.noreply.github.com> AuthorDate: Tue Aug 27 14:48:02 2024 -0700 Upgrade dependencies to fix vulnerabitilites (#13892) --- pinot-compatibility-verifier/pom.xml | 1 - pinot-connectors/pinot-spark-2-connector/pom.xml | 22 +- pinot-connectors/pinot-spark-3-connector/pom.xml | 5 +- pinot-connectors/pinot-spark-common/pom.xml | 7 - .../pinot-batch-ingestion-spark-2.4/pom.xml | 37 +- .../pinot-batch-ingestion-spark-3/pom.xml | 24 +- pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml | 5 + .../pinot-confluent-avro/pom.xml | 10 - pinot-plugins/pinot-input-format/pinot-orc/pom.xml | 5 + .../pinot-input-format/pinot-parquet/pom.xml | 4 + .../pinot-input-format/pinot-protobuf/pom.xml | 10 - .../pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 3 - .../pinot-stream-ingestion/pinot-kinesis/pom.xml | 13 - .../pinot-stream-ingestion/pinot-pulsar/pom.xml | 2 - pinot-tools/pom.xml | 9 +- pom.xml | 571 +++++++++++++-------- 16 files changed, 424 insertions(+), 304 deletions(-) diff --git a/pinot-compatibility-verifier/pom.xml b/pinot-compatibility-verifier/pom.xml index 3a024f4802..38cc3374de 100644 --- a/pinot-compatibility-verifier/pom.xml +++ b/pinot-compatibility-verifier/pom.xml @@ -34,7 +34,6 @@ <properties> <pinot.root>${basedir}/..</pinot.root> - <kafka.lib.version>2.8.2</kafka.lib.version> </properties> <build> diff --git a/pinot-connectors/pinot-spark-2-connector/pom.xml b/pinot-connectors/pinot-spark-2-connector/pom.xml index fde5cf1af2..92a9d4142c 100644 --- a/pinot-connectors/pinot-spark-2-connector/pom.xml +++ b/pinot-connectors/pinot-spark-2-connector/pom.xml @@ -32,9 +32,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../..</pinot.root> - <spark.version>2.4.8</spark.version> - <scalaxml.version>2.3.0</scalaxml.version> - <scalatest.version>3.2.18</scalatest.version> <shadeBase>org.apache.pinot.\$internal</shadeBase> </properties> @@ -48,14 +45,18 @@ <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-xml_${scala.compat.version}</artifactId> - <version>${scalaxml.version}</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.compat.version}</artifactId> - <version>${spark.version}</version> + <version>${spark2.version}</version> <scope>provided</scope> <exclusions> + <!-- Exclude it here and include explicitly because it has "hadoop2" classifier --> + <exclusion> + <groupId>org.apache.avro</groupId> + <artifactId>avro-mapred</artifactId> + </exclusion> <exclusion> <groupId>log4j</groupId> <artifactId>log4j</artifactId> @@ -66,6 +67,16 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-mapred</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> @@ -75,7 +86,6 @@ <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.compat.version}</artifactId> - <version>${scalatest.version}</version> <scope>test</scope> </dependency> </dependencies> diff --git a/pinot-connectors/pinot-spark-3-connector/pom.xml b/pinot-connectors/pinot-spark-3-connector/pom.xml index 1c3661c723..c4ae0528b6 100644 --- a/pinot-connectors/pinot-spark-3-connector/pom.xml +++ b/pinot-connectors/pinot-spark-3-connector/pom.xml @@ -32,8 +32,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../..</pinot.root> - <spark.version>3.5.2</spark.version> - <scalatest.version>3.2.18</scalatest.version> <shadeBase>org.apache.pinot.\$internal</shadeBase> </properties> @@ -47,7 +45,7 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_${scala.compat.version}</artifactId> - <version>${spark.version}</version> + <version>${spark3.version}</version> <scope>provided</scope> </dependency> <dependency> @@ -59,7 +57,6 @@ <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.compat.version}</artifactId> - <version>${scalatest.version}</version> <scope>test</scope> </dependency> </dependencies> diff --git a/pinot-connectors/pinot-spark-common/pom.xml b/pinot-connectors/pinot-spark-common/pom.xml index 942f8c97e4..353216f5eb 100644 --- a/pinot-connectors/pinot-spark-common/pom.xml +++ b/pinot-connectors/pinot-spark-common/pom.xml @@ -32,9 +32,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../..</pinot.root> - <circe.version>0.14.9</circe.version> - <scalaxml.version>2.3.0</scalaxml.version> - <scalatest.version>3.2.18</scalatest.version> </properties> <profiles> @@ -51,17 +48,14 @@ <dependency> <groupId>org.scala-lang.modules</groupId> <artifactId>scala-xml_${scala.compat.version}</artifactId> - <version>${scalaxml.version}</version> </dependency> <dependency> <groupId>io.circe</groupId> <artifactId>circe-parser_${scala.compat.version}</artifactId> - <version>${circe.version}</version> </dependency> <dependency> <groupId>io.circe</groupId> <artifactId>circe-generic_${scala.compat.version}</artifactId> - <version>${circe.version}</version> </dependency> <dependency> <groupId>org.scala-lang</groupId> @@ -72,7 +66,6 @@ <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_${scala.compat.version}</artifactId> - <version>${scalatest.version}</version> <scope>test</scope> </dependency> </dependencies> diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml index 7dfbb2a643..f8f7c07639 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pom.xml @@ -34,9 +34,6 @@ <properties> <pinot.root>${basedir}/../../..</pinot.root> <shade.phase.prop>package</shade.phase.prop> - <scala.major.version>2.11</scala.major.version> - <spark.version>2.4.6</spark.version> - <scala.minor.version>2.11.12</scala.minor.version> </properties> <dependencies> @@ -46,25 +43,14 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-core_${scala.major.version}</artifactId> - <version>${spark.version}</version> + <artifactId>spark-core_${scala.compat.version}</artifactId> + <version>${spark2.version}</version> <scope>provided</scope> <exclusions> + <!-- Exclude it here and include explicitly because it has "hadoop2" classifier --> <exclusion> - <groupId>com.zaxxer</groupId> - <artifactId>HikariCP-java7</artifactId> - </exclusion> - <exclusion> - <groupId>com.twitter</groupId> - <artifactId>chill_2.11</artifactId> - </exclusion> - <exclusion> - <groupId>com.twitter</groupId> - <artifactId>chill-java</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> + <groupId>org.apache.avro</groupId> + <artifactId>avro-mapred</artifactId> </exclusion> <exclusion> <groupId>log4j</groupId> @@ -76,10 +62,19 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-mapred</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> - <version>${scala.minor.version}</version> <scope>provided</scope> </dependency> @@ -93,13 +88,11 @@ <dependency> <groupId>com.esotericsoftware.kryo</groupId> <artifactId>kryo</artifactId> - <version>2.24.0</version> <scope>test</scope> </dependency> <dependency> <groupId>com.twitter</groupId> <artifactId>chill_2.11</artifactId> - <version>0.10.0</version> <scope>test</scope> <exclusions> <exclusion> diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml index 30c463d563..f19057a87e 100644 --- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml +++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/pom.xml @@ -34,7 +34,6 @@ <properties> <pinot.root>${basedir}/../../..</pinot.root> <shade.phase.prop>package</shade.phase.prop> - <spark.version>3.5.2</spark.version> </properties> <dependencies> @@ -45,24 +44,12 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_${scala.compat.version}</artifactId> - <version>${spark.version}</version> + <version>${spark3.version}</version> <scope>provided</scope> <exclusions> <exclusion> - <groupId>com.zaxxer</groupId> - <artifactId>HikariCP-java7</artifactId> - </exclusion> - <exclusion> - <groupId>com.twitter</groupId> - <artifactId>chill_2.11</artifactId> - </exclusion> - <exclusion> - <groupId>com.twitter</groupId> - <artifactId>chill-java</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> </exclusion> <exclusion> <groupId>log4j</groupId> @@ -72,16 +59,11 @@ <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> </exclusion> - <exclusion> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </exclusion> </exclusions> </dependency> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> - <version>${scala.version}</version> <scope>provided</scope> </dependency> diff --git a/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml b/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml index 6986ae37e0..32074000b5 100644 --- a/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml +++ b/pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml @@ -39,6 +39,11 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> + <!-- Replace bcprov-jdk15on which is excluded from hadoop-common --> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk18on</artifactId> + </dependency> <dependency> <groupId>org.codehaus.woodstox</groupId> <artifactId>stax2-api</artifactId> diff --git a/pinot-plugins/pinot-input-format/pinot-confluent-avro/pom.xml b/pinot-plugins/pinot-input-format/pinot-confluent-avro/pom.xml index 642d84cf2f..5ea6510f58 100644 --- a/pinot-plugins/pinot-input-format/pinot-confluent-avro/pom.xml +++ b/pinot-plugins/pinot-input-format/pinot-confluent-avro/pom.xml @@ -33,7 +33,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../../..</pinot.root> - <kafka.lib.version>2.8.2</kafka.lib.version> <shade.phase.prop>package</shade.phase.prop> </properties> <repositories> @@ -50,23 +49,14 @@ <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> - <version>${kafka.lib.version}</version> </dependency> <dependency> <groupId>io.confluent</groupId> <artifactId>kafka-schema-registry-client</artifactId> - <version>${confluent.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>io.confluent</groupId> <artifactId>kafka-avro-serializer</artifactId> - <version>${confluent.version}</version> </dependency> </dependencies> </project> diff --git a/pinot-plugins/pinot-input-format/pinot-orc/pom.xml b/pinot-plugins/pinot-input-format/pinot-orc/pom.xml index 658d07f00a..1936710fee 100644 --- a/pinot-plugins/pinot-input-format/pinot-orc/pom.xml +++ b/pinot-plugins/pinot-input-format/pinot-orc/pom.xml @@ -41,6 +41,11 @@ <artifactId>hadoop-common</artifactId> <scope>${hadoop.dependencies.scope}</scope> </dependency> + <!-- Replace bcprov-jdk15on which is excluded from hadoop-common --> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk18on</artifactId> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> diff --git a/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml b/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml index 1ec75e569c..aed20f33ab 100644 --- a/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml +++ b/pinot-plugins/pinot-input-format/pinot-parquet/pom.xml @@ -49,6 +49,10 @@ <artifactId>hadoop-common</artifactId> <scope>${hadoop.dependencies.scope}</scope> </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk18on</artifactId> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client-runtime</artifactId> diff --git a/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml b/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml index 19afe9ff35..44a97a7458 100644 --- a/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml +++ b/pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml @@ -34,7 +34,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../../..</pinot.root> - <kafka.lib.version>2.8.2</kafka.lib.version> <shade.phase.prop>package</shade.phase.prop> </properties> <repositories> @@ -60,23 +59,14 @@ <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> - <version>${kafka.lib.version}</version> </dependency> <dependency> <groupId>io.confluent</groupId> <artifactId>kafka-schema-registry-client</artifactId> - <version>${confluent.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.kafka</groupId> - <artifactId>kafka-clients</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>io.confluent</groupId> <artifactId>kafka-protobuf-serializer</artifactId> - <version>${confluent.version}</version> </dependency> <dependency> diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml index 0d4b974ad5..f2c9391078 100644 --- a/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml +++ b/pinot-plugins/pinot-stream-ingestion/pinot-kafka-2.0/pom.xml @@ -33,7 +33,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../../..</pinot.root> - <kafka.lib.version>2.8.2</kafka.lib.version> <shade.phase.prop>package</shade.phase.prop> </properties> @@ -46,12 +45,10 @@ <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka-clients</artifactId> - <version>${kafka.lib.version}</version> </dependency> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_${scala.compat.version}</artifactId> - <version>${kafka.lib.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.module</groupId> diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/pom.xml index 24b4a50965..7bae0df3e8 100644 --- a/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/pom.xml +++ b/pinot-plugins/pinot-stream-ingestion/pinot-kinesis/pom.xml @@ -33,22 +33,9 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/../../..</pinot.root> - <reactive.version>1.0.2</reactive.version> <localstack-utils.version>0.2.23</localstack-utils.version> </properties> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>software.amazon.awssdk</groupId> - <artifactId>bom</artifactId> - <version>${aws.sdk.version}</version> - <type>pom</type> - <scope>import</scope> - </dependency> - </dependencies> - </dependencyManagement> - <dependencies> <dependency> <groupId>software.amazon.awssdk</groupId> diff --git a/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml b/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml index 700110e4dc..b07237a8df 100644 --- a/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml +++ b/pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml @@ -41,13 +41,11 @@ <dependency> <groupId>org.apache.pulsar</groupId> <artifactId>pulsar-client</artifactId> - <version>${pulsar.version}</version> </dependency> <dependency> <groupId>org.apache.pulsar</groupId> <artifactId>pulsar-client-admin</artifactId> - <version>${pulsar.version}</version> <scope>test</scope> </dependency> <dependency> diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml index b40fac3108..0186d9a4b2 100644 --- a/pinot-tools/pom.xml +++ b/pinot-tools/pom.xml @@ -32,7 +32,6 @@ <url>https://pinot.apache.org/</url> <properties> <pinot.root>${basedir}/..</pinot.root> - <spark.version>3.5.2</spark.version> </properties> <dependencies> <dependency> @@ -127,6 +126,11 @@ <artifactId>hadoop-common</artifactId> <scope>compile</scope> </dependency> + <!-- Replace bcprov-jdk15on which is excluded from hadoop-common --> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk18on</artifactId> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> @@ -135,7 +139,6 @@ <dependency> <groupId>xml-apis</groupId> <artifactId>xml-apis</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>software.amazon.awssdk</groupId> @@ -167,7 +170,7 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-launcher_${scala.compat.version}</artifactId> - <version>${spark.version}</version> + <version>${spark3.version}</version> </dependency> <dependency> diff --git a/pom.xml b/pom.xml index 4f127a7e27..2cb4257381 100644 --- a/pom.xml +++ b/pom.xml @@ -146,7 +146,6 @@ <async-http-client.version>3.0.0</async-http-client.version> <jersey.version>2.42</jersey.version> <hk2.version>2.6.1</hk2.version> - <javassist.version>3.30.2-GA</javassist.version> <swagger.version>1.6.14</swagger.version> <swagger-ui.version>5.17.14</swagger-ui.version> <hadoop.version>3.4.0</hadoop.version> @@ -180,10 +179,14 @@ <azure.msal4j.version>1.17.0</azure.msal4j.version> <joda-time.version>2.12.7</joda-time.version> <janino.version>3.1.12</janino.version> - <woodstox.version>7.0.0</woodstox.version> <sslcontext.kickstart.version>8.3.6</sslcontext.kickstart.version> <jbcrypt.version>0.4</jbcrypt.version> + <scala-xml.version>2.3.0</scala-xml.version> + <circe.version>0.14.9</circe.version> + <spark2.version>2.4.8</spark2.version> + <spark3.version>3.5.2</spark3.version> + <kafka2.version>2.8.2</kafka2.version> <confluent.version>7.7.0</confluent.version> <pulsar.version>3.3.1</pulsar.version> <flink.version>1.20.0</flink.version> @@ -237,26 +240,43 @@ <scala.version>2.12.19</scala.version> <scala.compat.version>2.12</scala.compat.version> - <!-- Solve conflict across dependencies --> - <jetbrains.annotations.version>24.1.0</jetbrains.annotations.version> - <kotlin.stdlib.version>2.0.10</kotlin.stdlib.version> - <okio.version>3.9.0</okio.version> + <!-- Solve conflicts and vulnerabilities --> <kerby.version>2.1.0</kerby.version> <jline.version>3.26.3</jline.version> <wildfly.version>2.0.0</wildfly.version> <jettison.version>1.5.4</jettison.version> - <eclipse.jetty.version>9.4.55.v20240627</eclipse.jetty.version> <nimbus-jose-jwt.version>9.40</nimbus-jose-jwt.version> + <dnsjava.version>3.6.1</dnsjava.version> + <eclipse.jetty.version>9.4.55.v20240627</eclipse.jetty.version> + <woodstox.version>7.0.0</woodstox.version> + <curator.version>5.7.0</curator.version> + <javassist.version>3.30.2-GA</javassist.version> <bouncycastle.version>1.78.1</bouncycastle.version> <aircompressor.version>0.27</aircompressor.version> <jna.version>5.14.0</jna.version> + <jnr-ffi.version>2.2.16</jnr-ffi.version> + <jnr-constants.version>0.10.4</jnr-constants.version> + <asm.version>9.7</asm.version> <paranamer.version>2.8</paranamer.version> + <kotlin.stdlib.version>2.0.20</kotlin.stdlib.version> + <jetbrains.annotations.version>24.1.0</jetbrains.annotations.version> + <okio.version>3.9.0</okio.version> + <kryo.version>2.24.0</kryo.version> + <objenesis.version>3.4</objenesis.version> + <chill.version>0.10.0</chill.version> + <HikariCP-java7.version>2.4.13</HikariCP-java7.version> + <ivy.version>2.5.2</ivy.version> + <c3p0.version>0.10.1</c3p0.version> + <mchange-commons-java.version>0.3.1</mchange-commons-java.version> <!-- Test Libraries --> <testng.version>7.10.2</testng.version> <mockito-core.version>5.12.0</mockito-core.version> <equalsverifier.version>3.16.2</equalsverifier.version> <testcontainers.version>1.20.1</testcontainers.version> + <h2.version>2.3.232</h2.version> + <jnr-posix.version>3.1.19</jnr-posix.version> + <scalatest.version>3.2.19</scalatest.version> </properties> <profiles> @@ -768,6 +788,16 @@ <artifactId>parquet-avro</artifactId> <version>${parquet.version}</version> </dependency> + <dependency> + <groupId>org.apache.parquet</groupId> + <artifactId>parquet-column</artifactId> + <version>${parquet.version}</version> + </dependency> + <dependency> + <groupId>org.apache.parquet</groupId> + <artifactId>parquet-hadoop</artifactId> + <version>${parquet.version}</version> + </dependency> <dependency> <groupId>org.apache.orc</groupId> <artifactId>orc-core</artifactId> @@ -812,12 +842,6 @@ <groupId>org.apache.helix</groupId> <artifactId>helix-core</artifactId> <version>${helix.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-slf4j-impl</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.openjdk.jmh</groupId> @@ -827,21 +851,6 @@ </dependency> <!-- log4j2 related dependencies --> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - <version>${slf4j.version}</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jul-to-slf4j</artifactId> - <version>${slf4j.version}</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <version>${slf4j.version}</version> - </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> <artifactId>log4j-api</artifactId> @@ -862,6 +871,27 @@ <artifactId>log4j-1.2-api</artifactId> <version>${log4j.version}</version> </dependency> + <!-- We don't use slf4j but slf4j2. Including it here to support old libraries such as Spark 2.4 --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <version>${slf4j.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + <version>${slf4j.version}</version> + </dependency> <dependency> <groupId>com.lmax</groupId> <artifactId>disruptor</artifactId> @@ -883,11 +913,20 @@ <artifactId>larray-mmap</artifactId> <version>0.4.1</version> </dependency> - <!-- Transitive dependencies with inconsistent version numbers --> <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.version}</version> + <exclusions> + <exclusion> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-core</artifactId> + </exclusion> + <exclusion> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + </exclusion> + </exclusions> </dependency> <!-- Jackson --> @@ -1114,6 +1153,10 @@ <version>${hadoop.version}</version> <scope>provided</scope> <exclusions> + <exclusion> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + </exclusion> <exclusion> <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> @@ -1241,69 +1284,6 @@ <artifactId>hadoop-shaded-protobuf_3_21</artifactId> <version>1.2.0</version> </dependency> - <!-- The following dependencies are added to solve the vulnerabilities of the old version --> - <dependency> - <groupId>org.apache.kerby</groupId> - <artifactId>kerb-core</artifactId> - <version>${kerby.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kerby</groupId> - <artifactId>kerb-simplekdc</artifactId> - <version>${kerby.version}</version> - </dependency> - <dependency> - <groupId>org.jline</groupId> - <artifactId>jline</artifactId> - <version>${jline.version}</version> - </dependency> - <dependency> - <groupId>org.wildfly.common</groupId> - <artifactId>wildfly-common</artifactId> - <version>${wildfly.version}</version> - </dependency> - <dependency> - <groupId>org.codehaus.jettison</groupId> - <artifactId>jettison</artifactId> - <version>${jettison.version}</version> - </dependency> - <dependency> - <groupId>com.nimbusds</groupId> - <artifactId>nimbus-jose-jwt</artifactId> - <version>${nimbus-jose-jwt.version}</version> - </dependency> - - <!-- Consolidate eclipse jetty dependencies for hadoop/spark/pulsar --> - <dependency> - <groupId>org.eclipse.jetty.websocket</groupId> - <artifactId>websocket-client</artifactId> - <version>${eclipse.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <version>${eclipse.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - <version>${eclipse.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - <version>${eclipse.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util-ajax</artifactId> - <version>${eclipse.jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-webapp</artifactId> - <version>${eclipse.jetty.version}</version> - </dependency> <!-- Metrics --> <dependency> @@ -1465,12 +1445,6 @@ <artifactId>hk2-metadata-generator</artifactId> <version>${hk2.version}</version> </dependency> - <!-- Used by HK2 and reflections --> - <dependency> - <groupId>org.javassist</groupId> - <artifactId>javassist</artifactId> - <version>${javassist.version}</version> - </dependency> <dependency> <groupId>io.swagger</groupId> <artifactId>swagger-jersey2-jaxrs</artifactId> @@ -1488,36 +1462,6 @@ <version>${swagger-ui.version}</version> </dependency> - <dependency> - <groupId>org.apache.maven.surefire</groupId> - <artifactId>surefire-testng</artifactId> - <version>${surefire.version}</version> - </dependency> - <dependency> - <groupId>org.jetbrains</groupId> - <artifactId>annotations</artifactId> - <version>24.1.0</version> - </dependency> - <dependency> - <groupId>com.h2database</groupId> - <artifactId>h2</artifactId> - <version>2.3.232</version> - </dependency> - <dependency> - <groupId>com.github.jnr</groupId> - <artifactId>jnr-posix</artifactId> - <version>3.1.19</version> - </dependency> - <dependency> - <groupId>com.github.jnr</groupId> - <artifactId>jnr-ffi</artifactId> - <version>2.2.16</version> - </dependency> - <dependency> - <groupId>com.github.jnr</groupId> - <artifactId>jnr-constants</artifactId> - <version>0.10.4</version> - </dependency> <dependency> <groupId>info.picocli</groupId> <artifactId>picocli</artifactId> @@ -1528,7 +1472,6 @@ <artifactId>tyrus-standalone-client</artifactId> <version>2.2.0</version> </dependency> - <!-- kafka_2.10 & jmh-core use jopt-simple --> <dependency> <groupId>net.sf.jopt-simple</groupId> <artifactId>jopt-simple</artifactId> @@ -1566,37 +1509,41 @@ <artifactId>chronicle-core</artifactId> <version>2.26ea1</version> </dependency> + <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - <version>9.7</version> + <groupId>com.yscope.clp</groupId> + <artifactId>clp-ffi</artifactId> + <version>${clp-ffi.version}</version> </dependency> <dependency> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna-platform</artifactId> - <version>${jna.version}</version> + <groupId>org.codehaus.woodstox</groupId> + <artifactId>stax2-api</artifactId> + <version>${stax2-api.version}</version> </dependency> <dependency> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - <version>${jna.version}</version> + <groupId>io.github.hakky54</groupId> + <artifactId>sslcontext-kickstart-for-netty</artifactId> + <version>${sslcontext.kickstart.version}</version> </dependency> <dependency> - <groupId>com.thoughtworks.paranamer</groupId> - <artifactId>paranamer</artifactId> - <version>${paranamer.version}</version> + <groupId>org.mindrot</groupId> + <artifactId>jbcrypt</artifactId> + <version>${jbcrypt.version}</version> </dependency> - <dependency> - <groupId>com.yscope.clp</groupId> - <artifactId>clp-ffi</artifactId> - <version>${clp-ffi.version}</version> + <groupId>org.scala-lang.modules</groupId> + <artifactId>scala-xml_${scala.compat.version}</artifactId> + <version>${scala-xml.version}</version> </dependency> - <dependency> - <groupId>org.codehaus.woodstox</groupId> - <artifactId>stax2-api</artifactId> - <version>${stax2-api.version}</version> + <groupId>io.circe</groupId> + <artifactId>circe-parser_${scala.compat.version}</artifactId> + <version>${circe.version}</version> + </dependency> + <dependency> + <groupId>io.circe</groupId> + <artifactId>circe-generic_${scala.compat.version}</artifactId> + <version>${circe.version}</version> </dependency> <!-- AWS SDK --> @@ -1634,74 +1581,60 @@ </dependency> <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-clients</artifactId> - <version>${flink.version}</version> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>${kafka2.version}</version> </dependency> <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-streaming-java</artifactId> - <version>${flink.version}</version> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka_${scala.compat.version}</artifactId> + <version>${kafka2.version}</version> </dependency> + <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-java</artifactId> - <version>${flink.version}</version> - <exclusions> - <!-- Resolve conflict with flink-streaming-java --> - <exclusion> - <groupId>com.esotericsoftware.kryo</groupId> - <artifactId>kryo</artifactId> - </exclusion> - </exclusions> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>${confluent.version}</version> </dependency> <dependency> - <groupId>org.jetbrains.kotlin</groupId> - <artifactId>kotlin-stdlib-jdk8</artifactId> - <version>${kotlin.stdlib.version}</version> + <groupId>io.confluent</groupId> + <artifactId>kafka-avro-serializer</artifactId> + <version>${confluent.version}</version> </dependency> <dependency> - <groupId>org.jetbrains.kotlin</groupId> - <artifactId>kotlin-stdlib</artifactId> - <version>${kotlin.stdlib.version}</version> - </dependency> - <dependency> - <groupId>org.jetbrains.kotlin</groupId> - <artifactId>kotlin-stdlib-common</artifactId> - <version>${kotlin.stdlib.version}</version> - </dependency> - <dependency> - <groupId>org.jetbrains</groupId> - <artifactId>annotations</artifactId> - <version>${jetbrains.annotations.version}</version> + <groupId>io.confluent</groupId> + <artifactId>kafka-protobuf-serializer</artifactId> + <version>${confluent.version}</version> </dependency> + <dependency> - <groupId>com.squareup.okio</groupId> - <artifactId>okio</artifactId> - <version>${okio.version}</version> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar-client</artifactId> + <version>${pulsar.version}</version> </dependency> <dependency> - <groupId>com.squareup.okio</groupId> - <artifactId>okio-jvm</artifactId> - <version>${okio.version}</version> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar-client-admin</artifactId> + <version>${pulsar.version}</version> </dependency> + <dependency> - <groupId>com.fasterxml.woodstox</groupId> - <artifactId>woodstox-core</artifactId> - <version>${woodstox.version}</version> + <groupId>org.apache.flink</groupId> + <artifactId>flink-clients</artifactId> + <version>${flink.version}</version> </dependency> <dependency> - <groupId>io.github.hakky54</groupId> - <artifactId>sslcontext-kickstart-for-netty</artifactId> - <version>${sslcontext.kickstart.version}</version> + <groupId>org.apache.flink</groupId> + <artifactId>flink-streaming-java</artifactId> + <version>${flink.version}</version> </dependency> <dependency> - <groupId>org.mindrot</groupId> - <artifactId>jbcrypt</artifactId> - <version>${jbcrypt.version}</version> + <groupId>org.apache.flink</groupId> + <artifactId>flink-java</artifactId> + <version>${flink.version}</version> </dependency> - <!-- Lucene dependencies --> + <!-- Lucene --> <dependency> <groupId>org.apache.lucene</groupId> <artifactId>lucene-backward-codecs</artifactId> @@ -1722,9 +1655,104 @@ <artifactId>lucene-analysis-common</artifactId> <version>${lucene.version}</version> </dependency> - <!-- Lucene dependencies end --> - <!-- Bouncy Castle libraries are used by Kafka and Pulsar plugins --> + <!-- Solve conflicts and vulnerabilities --> + <!-- Dependencies in Hadoop libraries --> + <dependency> + <groupId>org.apache.kerby</groupId> + <artifactId>kerb-core</artifactId> + <version>${kerby.version}</version> + </dependency> + <dependency> + <groupId>org.apache.kerby</groupId> + <artifactId>kerb-simplekdc</artifactId> + <version>${kerby.version}</version> + </dependency> + <dependency> + <groupId>org.jline</groupId> + <artifactId>jline</artifactId> + <version>${jline.version}</version> + </dependency> + <dependency> + <groupId>org.wildfly.common</groupId> + <artifactId>wildfly-common</artifactId> + <version>${wildfly.version}</version> + </dependency> + <dependency> + <groupId>org.codehaus.jettison</groupId> + <artifactId>jettison</artifactId> + <version>${jettison.version}</version> + </dependency> + <dependency> + <groupId>com.nimbusds</groupId> + <artifactId>nimbus-jose-jwt</artifactId> + <version>${nimbus-jose-jwt.version}</version> + </dependency> + <dependency> + <groupId>dnsjava</groupId> + <artifactId>dnsjava</artifactId> + <version>${dnsjava.version}</version> + </dependency> + <!-- Eclipse jetty dependencies in Hadoop/Spark/Pulsar --> + <dependency> + <groupId>org.eclipse.jetty.websocket</groupId> + <artifactId>websocket-client</artifactId> + <version>${eclipse.jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <version>${eclipse.jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <version>${eclipse.jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + <version>${eclipse.jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util-ajax</artifactId> + <version>${eclipse.jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + <version>${eclipse.jetty.version}</version> + </dependency> + <!-- Used by Hadoop and Jackson --> + <dependency> + <groupId>com.fasterxml.woodstox</groupId> + <artifactId>woodstox-core</artifactId> + <version>${woodstox.version}</version> + </dependency> + <!-- Used by Hadoop and Spark --> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-client</artifactId> + <version>${curator.version}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>${curator.version}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-recipes</artifactId> + <version>${curator.version}</version> + </dependency> + <!-- Used by reflections and jersey-hk2 --> + <dependency> + <groupId>org.javassist</groupId> + <artifactId>javassist</artifactId> + <version>${javassist.version}</version> + </dependency> + <!-- Bouncy Castle libraries are used by Hadoop/Kafka/Pulsar --> <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk18on</artifactId> @@ -1745,13 +1773,131 @@ <artifactId>bcprov-ext-jdk18on</artifactId> <version>${bouncycastle.version}</version> </dependency> - <!-- Used by ORC, Parquet and Pulsar --> <dependency> <groupId>io.airlift</groupId> <artifactId>aircompressor</artifactId> <version>${aircompressor.version}</version> </dependency> + <!-- Native access libraries --> + <dependency> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + <version>${jna.version}</version> + </dependency> + <dependency> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna-platform</artifactId> + <version>${jna.version}</version> + </dependency> + <dependency> + <groupId>com.github.jnr</groupId> + <artifactId>jnr-ffi</artifactId> + <version>${jnr-ffi.version}</version> + </dependency> + <dependency> + <groupId>com.github.jnr</groupId> + <artifactId>jnr-constants</artifactId> + <version>${jnr-constants.version}</version> + </dependency> + <!-- Used by jnr-ffi and json-smart --> + <dependency> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + <version>${asm.version}</version> + </dependency> + <!-- Used by Jackson and Spark --> + <dependency> + <groupId>com.thoughtworks.paranamer</groupId> + <artifactId>paranamer</artifactId> + <version>${paranamer.version}</version> + </dependency> + <!-- Dependencies in Confluent Kafka libraries --> + <dependency> + <groupId>org.jetbrains.kotlin</groupId> + <artifactId>kotlin-stdlib-jdk8</artifactId> + <version>${kotlin.stdlib.version}</version> + </dependency> + <dependency> + <groupId>org.jetbrains.kotlin</groupId> + <artifactId>kotlin-stdlib</artifactId> + <version>${kotlin.stdlib.version}</version> + </dependency> + <dependency> + <groupId>org.jetbrains.kotlin</groupId> + <artifactId>kotlin-stdlib-common</artifactId> + <version>${kotlin.stdlib.version}</version> + </dependency> + <dependency> + <groupId>org.jetbrains.kotlin</groupId> + <artifactId>kotlin-reflect</artifactId> + <version>${kotlin.stdlib.version}</version> + </dependency> + <dependency> + <groupId>com.squareup.okio</groupId> + <artifactId>okio</artifactId> + <version>${okio.version}</version> + </dependency> + <dependency> + <groupId>com.squareup.okio</groupId> + <artifactId>okio-jvm</artifactId> + <version>${okio.version}</version> + </dependency> + <!-- Used in async-http-client and orc-core --> + <dependency> + <groupId>org.jetbrains</groupId> + <artifactId>annotations</artifactId> + <version>${jetbrains.annotations.version}</version> + </dependency> + <!-- Used in Spark and Flink --> + <dependency> + <groupId>com.esotericsoftware.kryo</groupId> + <artifactId>kryo</artifactId> + <version>${kryo.version}</version> + </dependency> + <dependency> + <groupId>org.objenesis</groupId> + <artifactId>objenesis</artifactId> + <version>${objenesis.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill-java</artifactId> + <version>${chill.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_2.11</artifactId> + <version>${chill.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_2.12</artifactId> + <version>${chill.version}</version> + </dependency> + <!-- Used by Spark and quartz --> + <dependency> + <groupId>com.zaxxer</groupId> + <artifactId>HikariCP-java7</artifactId> + <version>${HikariCP-java7.version}</version> + </dependency> + <!-- Used by Spark --> + <dependency> + <groupId>org.apache.ivy</groupId> + <artifactId>ivy</artifactId> + <version>${ivy.version}</version> + </dependency> + <!-- Used by quartz --> + <dependency> + <groupId>com.mchange</groupId> + <artifactId>c3p0</artifactId> + <version>${c3p0.version}</version> + </dependency> + <dependency> + <groupId>com.mchange</groupId> + <artifactId>mchange-commons-java</artifactId> + <version>${mchange-commons-java.version}</version> + </dependency> <!-- Test Libraries --> <dependency> @@ -1790,7 +1936,24 @@ <version>${testcontainers.version}</version> <scope>test</scope> </dependency> - + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <version>${h2.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.github.jnr</groupId> + <artifactId>jnr-posix</artifactId> + <version>${jnr-posix.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.scalatest</groupId> + <artifactId>scalatest_${scala.compat.version}</artifactId> + <version>${scalatest.version}</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement> @@ -1973,14 +2136,18 @@ <!-- Use org.apache.logging.log4j:log4j-core --> <exclude>log4j:log4j</exclude> <!-- Use org.apache.logging.log4j:log4j-slf4j2-impl --> - <exclude>org.apache.logging.log4j:log4j-slf4j-impl</exclude> <exclude>org.slf4j:slf4j-log4j12</exclude> <exclude>org.slf4j:slf4j-reload4j</exclude> <exclude>ch.qos.reload4j:reload4j</exclude> + <exclude>ch.qos.logback</exclude> + <!-- Use com.fasterxml.jackson --> + <exclude>org.codehaus.jackson</exclude> <!-- Use org.glassfish.jersey --> <exclude>com.sun.jersey</exclude> <!-- Use hadoop-shaded-protobuf_3_21 --> <exclude>org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7</exclude> + <!-- Use org.bouncycastle:bcprov-jdk18on --> + <exclude>org.bouncycastle:bcprov-jdk15on</exclude> </excludes> </bannedDependencies> </rules> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org