This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 81e4a7297f [#10788] improvement(deps): upgrade Hadoop from 2.10.2 to
3.3.6 and remove hadoop2 dependency line (#10789)
81e4a7297f is described below
commit 81e4a7297f8284558fb7fdfc79b6966f96aa7a17
Author: geyanggang <[email protected]>
AuthorDate: Wed May 6 17:28:18 2026 +0800
[#10788] improvement(deps): upgrade Hadoop from 2.10.2 to 3.3.6 and remove
hadoop2 dependency line (#10789)
### What changes were proposed in this pull request?
- Remove `hadoop2` version (2.10.2) and its 6 library definitions from
libs.versions.toml
- Bump `hadoop3` from 3.3.1 to 3.3.6 (including `hadoop3-abs`,
`hadoop3-aliyun`, `hadoop-minikdc`)
- Replace all `libs.hadoop2.*` references with `libs.hadoop3.*` in 8
build files
- Add Hadoop 3.x runtime dependencies stripped by `exclude("*")`:
`hadoop-shaded-guava`, `hadoop-shaded-protobuf`,
`commons-configuration2`, `re2j`,`hadoop-hdfs-client`
- Remove outdated hadoop2/hadoop3 conflict comment in `catalog-hive`
### Why are the changes needed?
- The `hadoop2` (2.10.2) dependency line brings in outdated transitive
dependencies (Jackson 2.6.x, Netty 3.x, Jetty 9.3.x) that are no longer
maintained
- `hive-metastore3-libs` was incorrectly configured with `hadoop2`
despite Hive 3.1.3 natively depending on Hadoop 3.1.0
- Unifying on a single Hadoop version eliminates classpath conflicts and
simplifies dependency management
Fix: #10788
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Full compilation (`./gradlew build -x test`) — passed
- Verified `hadoop-common-3.3.6` resolves in all affected modules, no
Hadoop 2.x JARs in distribution
- All unit tests passed across 12 key modules including `catalog-hive`,
`hive-metastore-common`, `catalog-lakehouse-hudi`, `catalog-fileset`,
`filesystem-hadoop3`, `hadoop-common`, `authorization-ranger`,
`spark-connector`, `flink-connector`, `core`
- Integration tests pending CI validation
---
.../authorization-chain/build.gradle.kts | 1 +
.../authorization-ranger/build.gradle.kts | 1 +
catalogs/catalog-hive/build.gradle.kts | 31 ++++++++++++----------
catalogs/catalog-lakehouse-hudi/build.gradle.kts | 21 +++++++++++----
.../hudi/integration/test/HudiCatalogHMSIT.java | 4 +++
catalogs/catalog-lakehouse-paimon/build.gradle.kts | 17 ++++++------
catalogs/hive-metastore-common/build.gradle.kts | 11 +++++---
catalogs/hive-metastore2-libs/build.gradle.kts | 12 +++++++--
catalogs/hive-metastore3-libs/build.gradle.kts | 5 +++-
.../filesystem-hadoop3-runtime/build.gradle.kts | 6 ++++-
clients/filesystem-hadoop3/build.gradle.kts | 1 +
.../GravitinoVirtualFileSystemABSCredentialIT.java | 2 +-
flink-connector/flink/build.gradle.kts | 12 ++++++---
gradle/libs.versions.toml | 25 +++++++++--------
trino-connector/integration-test/build.gradle.kts | 12 ++++++---
15 files changed, 108 insertions(+), 53 deletions(-)
diff --git a/authorizations/authorization-chain/build.gradle.kts
b/authorizations/authorization-chain/build.gradle.kts
index cf19b23834..8ab42a9e4d 100644
--- a/authorizations/authorization-chain/build.gradle.kts
+++ b/authorizations/authorization-chain/build.gradle.kts
@@ -104,6 +104,7 @@ dependencies {
exclude("javax.servlet", "servlet-api")
exclude("io.netty")
}
+ testImplementation(libs.hadoop3.hdfs.client)
}
tasks {
diff --git a/authorizations/authorization-ranger/build.gradle.kts
b/authorizations/authorization-ranger/build.gradle.kts
index 4b6ee3b7c4..506466fa38 100644
--- a/authorizations/authorization-ranger/build.gradle.kts
+++ b/authorizations/authorization-ranger/build.gradle.kts
@@ -105,6 +105,7 @@ dependencies {
exclude("javax.servlet", "servlet-api")
exclude("io.netty")
}
+ testImplementation(libs.hadoop3.hdfs.client)
testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
testImplementation("org.apache.paimon:paimon-spark-$sparkMajorVersion:$paimonVersion")
}
diff --git a/catalogs/catalog-hive/build.gradle.kts
b/catalogs/catalog-hive/build.gradle.kts
index dba710c587..ed40edb1a8 100644
--- a/catalogs/catalog-hive/build.gradle.kts
+++ b/catalogs/catalog-hive/build.gradle.kts
@@ -49,12 +49,17 @@ dependencies {
implementation(libs.commons.io)
implementation(libs.commons.lang3)
implementation(libs.guava)
- implementation(libs.hadoop2.auth) {
+ implementation(libs.hadoop3.auth) {
exclude("*")
}
- implementation(libs.hadoop2.common) {
+ implementation(libs.hadoop3.common) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ implementation(libs.hadoop3.shaded.guava)
+ implementation(libs.hadoop3.shaded.protobuf)
+ implementation(libs.commons.configuration2)
+ implementation(libs.re2j)
implementation(libs.htrace.core4)
implementation(libs.slf4j.api)
implementation(libs.woodstox.core)
@@ -89,14 +94,20 @@ dependencies {
testImplementation(libs.bundles.jersey)
testImplementation(libs.bundles.jetty)
testImplementation(libs.bundles.log4j)
- testImplementation(libs.hadoop2.aws)
- testImplementation(libs.hadoop2.common) {
+ testImplementation(libs.hadoop3.aws)
+ testImplementation(libs.hadoop3.common) {
exclude("*")
}
- testImplementation(libs.hadoop2.hdfs)
- testImplementation(libs.hadoop2.mapreduce.client.core) {
+ testImplementation(libs.hadoop3.hdfs)
+ testImplementation(libs.hadoop3.hdfs.client)
+ testImplementation(libs.hadoop3.mapreduce.client.core) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ testImplementation(libs.hadoop3.shaded.guava)
+ testImplementation(libs.hadoop3.shaded.protobuf)
+ testImplementation(libs.commons.configuration2)
+ testImplementation(libs.re2j)
testImplementation(libs.hadoop3.abs)
testImplementation(libs.hadoop3.gcs)
testImplementation(libs.hive2.common) {
@@ -112,14 +123,6 @@ dependencies {
testImplementation(libs.testcontainers.localstack)
testImplementation(libs.testcontainers.mysql)
- // You need this to run test CatalogHiveABSIT as it required hadoop3
environment introduced by hadoop3.abs
- // (The protocol `abfss` was first introduced in Hadoop 3.2.0), However, as
the there already exists
- // hadoop2.common in the test classpath, If we added the following
dependencies directly, it will
- // cause the conflict between hadoop2 and hadoop3, resulting test failures,
so we comment the
- // following line temporarily, if you want to run the test, please uncomment
it.
- // In the future, we may need to refactor the test to avoid the conflict.
- // testImplementation(libs.hadoop3.common)
-
testRuntimeOnly(libs.junit.jupiter.engine)
}
diff --git a/catalogs/catalog-lakehouse-hudi/build.gradle.kts
b/catalogs/catalog-lakehouse-hudi/build.gradle.kts
index e50fed0778..5fd87a9c7a 100644
--- a/catalogs/catalog-lakehouse-hudi/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-hudi/build.gradle.kts
@@ -48,12 +48,17 @@ dependencies {
implementation(libs.commons.io)
implementation(libs.commons.lang3)
implementation(libs.guava)
- implementation(libs.hadoop2.auth) {
+ implementation(libs.hadoop3.auth) {
exclude("*")
}
- implementation(libs.hadoop2.common) {
+ implementation(libs.hadoop3.common) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ implementation(libs.hadoop3.shaded.guava)
+ implementation(libs.hadoop3.shaded.protobuf)
+ implementation(libs.commons.configuration2)
+ implementation(libs.re2j)
implementation(libs.htrace.core4)
implementation(libs.slf4j.api)
implementation(libs.woodstox.core)
@@ -113,13 +118,19 @@ dependencies {
testImplementation(libs.datanucleus.jdo)
testImplementation(libs.datanucleus.rdbms)
testImplementation(libs.derby)
- testImplementation(libs.hadoop2.auth) {
+ testImplementation(libs.hadoop3.auth) {
exclude("*")
}
- testImplementation(libs.hadoop2.hdfs)
- testImplementation(libs.hadoop2.mapreduce.client.core) {
+ testImplementation(libs.hadoop3.hdfs)
+ testImplementation(libs.hadoop3.hdfs.client)
+ testImplementation(libs.hadoop3.mapreduce.client.core) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ testImplementation(libs.hadoop3.shaded.guava)
+ testImplementation(libs.hadoop3.shaded.protobuf)
+ testImplementation(libs.commons.configuration2)
+ testImplementation(libs.re2j)
testImplementation(libs.htrace.core4)
testImplementation(libs.junit.jupiter.api)
testImplementation(libs.mysql.driver)
diff --git
a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
index c893997b4c..dca8dbbc40 100644
---
a/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
+++
b/catalogs/catalog-lakehouse-hudi/src/test/java/org/apache/gravitino/catalog/lakehouse/hudi/integration/test/HudiCatalogHMSIT.java
@@ -469,6 +469,10 @@ public class HudiCatalogHMSIT extends BaseIT {
"org.apache.spark.sql.hudi.catalog.HoodieCatalog")
.config("spark.kryo.registrator",
"org.apache.spark.HoodieSparkKryoRegistrar")
.config("dfs.replication", "1")
+ // Disable Hudi metadata table to avoid NoSuchMethodError caused
by Hudi 0.15.0's
+ // shaded HBase code (HoodieHFileReader) being incompatible with
hadoop-hdfs-client
+ // 3.3.6. See: https://github.com/apache/hudi/issues/5765
+ .config("hoodie.metadata.enable", "false")
.enableHiveSupport()
.getOrCreate();
diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts
b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
index 903814bd1c..ab9eb42986 100644
--- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
@@ -103,26 +103,25 @@ dependencies {
}
implementation(libs.commons.lang3)
implementation(libs.guava)
- implementation(libs.hadoop2.common) {
+ implementation(libs.hadoop3.common) {
exclude("com.github.spotbugs")
exclude("com.sun.jersey")
+ exclude("com.github.pjfanning", "jersey-json")
exclude("javax.servlet")
exclude("org.apache.avro")
exclude("net.java.dev.jets3t")
exclude("org.apache.curator")
exclude("org.apache.zookeeper")
exclude("org.mortbay.jetty")
+ exclude("org.eclipse.jetty")
+ exclude("ch.qos.reload4j")
+ exclude("org.slf4j", "slf4j-reload4j")
}
- implementation(libs.hadoop2.hdfs) {
+ implementation(libs.hadoop3.hdfs) {
exclude("*")
}
- implementation(libs.hadoop2.hdfs.client) {
- exclude("com.sun.jersey")
- exclude("javax.servlet")
- exclude("org.fusesource.leveldbjni")
- exclude("org.mortbay.jetty")
- }
- implementation(libs.hadoop2.mapreduce.client.core) {
+ implementation(libs.hadoop3.hdfs.client)
+ implementation(libs.hadoop3.mapreduce.client.core) {
exclude("*")
}
diff --git a/catalogs/hive-metastore-common/build.gradle.kts
b/catalogs/hive-metastore-common/build.gradle.kts
index aee7e7b812..10db17f962 100644
--- a/catalogs/hive-metastore-common/build.gradle.kts
+++ b/catalogs/hive-metastore-common/build.gradle.kts
@@ -55,15 +55,20 @@ dependencies {
testImplementation(libs.datanucleus.jdo)
testImplementation(libs.datanucleus.rdbms)
testImplementation(libs.derby)
- testImplementation(libs.hadoop2.auth) {
+ testImplementation(libs.hadoop3.auth) {
exclude("*")
}
- testImplementation(libs.hadoop2.common) {
+ testImplementation(libs.hadoop3.common) {
exclude("*")
}
- testImplementation(libs.hadoop2.mapreduce.client.core) {
+ testImplementation(libs.hadoop3.mapreduce.client.core) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ testImplementation(libs.hadoop3.shaded.guava)
+ testImplementation(libs.hadoop3.shaded.protobuf)
+ testImplementation(libs.commons.configuration2)
+ testImplementation(libs.re2j)
testImplementation(libs.hive2.exec) {
artifact {
classifier = "core"
diff --git a/catalogs/hive-metastore2-libs/build.gradle.kts
b/catalogs/hive-metastore2-libs/build.gradle.kts
index c16395e77d..5a308de7f0 100644
--- a/catalogs/hive-metastore2-libs/build.gradle.kts
+++ b/catalogs/hive-metastore2-libs/build.gradle.kts
@@ -29,27 +29,35 @@ plugins {
// Guava and Logback are excluded because they are provided by the Gravitino
runtime classpath.
dependencies {
- implementation(libs.hadoop2.common) {
+ implementation(libs.hadoop3.common) {
exclude(group = "ch.qos.logback")
+ exclude(group = "ch.qos.reload4j")
exclude(group = "com.fasterxml.jackson.core")
+ exclude(group = "com.github.pjfanning", module = "jersey-json")
exclude(group = "com.github.spotbugs")
exclude(group = "com.google.code.findbugs")
exclude(group = "com.google.guava")
+ exclude(group = "com.sun.jersey")
exclude(group = "log4j")
exclude(group = "net.java.dev.jets3t")
exclude(group = "org.apache.avro")
exclude(group = "org.apache.logging.log4j")
+ exclude(group = "org.eclipse.jetty")
exclude(group = "org.eclipse.jetty.aggregate", module = "jetty-all")
exclude(group = "org.eclipse.jetty.orbit", module = "javax.servlet")
exclude(group = "org.slf4j")
}
- implementation(libs.hadoop2.mapreduce.client.core) {
+ implementation(libs.hadoop3.mapreduce.client.core) {
+ exclude(group = "ch.qos.reload4j")
+ exclude(group = "com.github.pjfanning", module = "jersey-json")
exclude(group = "com.github.spotbugs")
exclude(group = "com.google.code.findbugs")
exclude(group = "com.google.guava")
+ exclude(group = "com.sun.jersey")
exclude(group = "log4j")
exclude(group = "org.apache.avro")
exclude(group = "org.apache.logging.log4j")
+ exclude(group = "org.eclipse.jetty")
exclude(group = "org.slf4j")
}
implementation(libs.hive2.metastore) {
diff --git a/catalogs/hive-metastore3-libs/build.gradle.kts
b/catalogs/hive-metastore3-libs/build.gradle.kts
index bfb06c7d8d..55669a493b 100644
--- a/catalogs/hive-metastore3-libs/build.gradle.kts
+++ b/catalogs/hive-metastore3-libs/build.gradle.kts
@@ -29,12 +29,15 @@ plugins {
// Guava and Logback are excluded because they are provided by the Gravitino
runtime classpath.
dependencies {
- implementation(libs.hadoop2.common) {
+ implementation(libs.hadoop3.common) {
exclude(group = "ch.qos.logback")
+ exclude(group = "ch.qos.reload4j")
exclude(group = "com.fasterxml.jackson.core")
+ exclude(group = "com.github.pjfanning", module = "jersey-json")
exclude(group = "com.github.spotbugs")
exclude(group = "com.google.code.findbugs")
exclude(group = "com.google.guava")
+ exclude(group = "com.sun.jersey")
exclude(group = "log4j")
exclude(group = "net.java.dev.jets3t")
exclude(group = "org.apache.avro")
diff --git a/clients/filesystem-hadoop3-runtime/build.gradle.kts
b/clients/filesystem-hadoop3-runtime/build.gradle.kts
index 0916f94a70..2a5b36b79f 100644
--- a/clients/filesystem-hadoop3-runtime/build.gradle.kts
+++ b/clients/filesystem-hadoop3-runtime/build.gradle.kts
@@ -43,7 +43,11 @@ tasks.withType<ShadowJar>(ShadowJar::class.java) {
archiveClassifier.set("")
// Relocate dependencies to avoid conflicts
- relocate("com.google", "org.apache.gravitino.shaded.com.google")
+ relocate("com.google", "org.apache.gravitino.shaded.com.google") {
+ // Do not relocate com.google.cloud.hadoop classes — they come from the
external
+ // gcs-connector jar on the user's Hadoop classpath and must keep their
original package name.
+ exclude("com.google.cloud.hadoop.**")
+ }
relocate("com.github.benmanes.caffeine",
"org.apache.gravitino.shaded.com.github.benmanes.caffeine")
// relocate common lang3 package
relocate("org.apache.commons.lang3",
"org.apache.gravitino.shaded.org.apache.commons.lang3")
diff --git a/clients/filesystem-hadoop3/build.gradle.kts
b/clients/filesystem-hadoop3/build.gradle.kts
index 48c2d9b2c6..76000c6abe 100644
--- a/clients/filesystem-hadoop3/build.gradle.kts
+++ b/clients/filesystem-hadoop3/build.gradle.kts
@@ -68,6 +68,7 @@ dependencies {
exclude("javax.servlet", "servlet-api")
exclude("io.netty")
}
+ testImplementation(libs.hadoop3.hdfs.client)
testImplementation(libs.httpclient5)
testImplementation(libs.javax.jaxb.api) {
exclude("*")
diff --git
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java
index 9f30257253..99f21812bb 100644
---
a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java
+++
b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java
@@ -111,7 +111,7 @@ public class GravitinoVirtualFileSystemABSCredentialIT
extends GravitinoVirtualF
conf.set(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
ABS_ACCOUNT_KEY);
conf.set("fs.abfss.impl",
"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem");
- conf.set("fs.gravitino.client.useCloudStoreCredential", "true");
+ conf.set("fs.gravitino.enableCredentialVending", "true");
}
@AfterAll
diff --git a/flink-connector/flink/build.gradle.kts
b/flink-connector/flink/build.gradle.kts
index da01614f51..0e3b595366 100644
--- a/flink-connector/flink/build.gradle.kts
+++ b/flink-connector/flink/build.gradle.kts
@@ -129,10 +129,10 @@ dependencies {
exclude("org.slf4j")
}
- testImplementation(libs.hadoop2.common) {
+ testImplementation(libs.hadoop3.common) {
exclude("*")
}
- testImplementation(libs.hadoop2.hdfs) {
+ testImplementation(libs.hadoop3.hdfs) {
exclude("com.sun.jersey")
exclude("commons-cli", "commons-cli")
exclude("commons-io", "commons-io")
@@ -141,9 +141,15 @@ dependencies {
exclude("javax.servlet", "servlet-api")
exclude("org.mortbay.jetty")
}
- testImplementation(libs.hadoop2.mapreduce.client.core) {
+ testImplementation(libs.hadoop3.hdfs.client)
+ testImplementation(libs.hadoop3.mapreduce.client.core) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ testImplementation(libs.hadoop3.shaded.guava)
+ testImplementation(libs.hadoop3.shaded.protobuf)
+ testImplementation(libs.commons.configuration2)
+ testImplementation(libs.re2j)
testImplementation(libs.hive2.common) {
exclude("org.eclipse.jetty.aggregate", "jetty-all")
exclude("org.eclipse.jetty.orbit", "javax.servlet")
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 5fa2a69779..685be80560 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -38,18 +38,20 @@ airlift-json = "237"
airlift-resolver = "1.6"
hive2 = "2.3.9"
hive3 = "3.1.3"
-hadoop2 = "2.10.2"
-hadoop3 = "3.3.1"
+hadoop3 = "3.3.6"
hadoop3-gcs = "1.9.4-hadoop3"
-hadoop3-abs = "3.3.1"
-hadoop3-aliyun = "3.3.1"
-hadoop-minikdc = "3.3.1"
+hadoop3-abs = "3.3.6"
+hadoop3-aliyun = "3.3.6"
+hadoop3-thirdparty = "1.1.1"
+hadoop-minikdc = "3.3.6"
+re2j = "1.7"
htrace-core4 = "4.1.0-incubating"
httpclient = "4.4.1"
httpclient5 = "5.4.4"
httpcore5 = "5.3.4"
micrometer-core = "1.12.2"
mockserver = "5.15.0"
+commons-configuration2 = "2.8.0"
commons-csv = "1.12.0"
commons-lang3 = "3.14.0"
commons-lang = "2.6"
@@ -197,22 +199,22 @@ hive2-common = { group = "org.apache.hive", name =
"hive-common", version.ref =
hive2-jdbc = { group = "org.apache.hive", name = "hive-jdbc", version.ref =
"hive2"}
hive3-metastore = { group = "org.apache.hive", name = "hive-metastore",
version.ref = "hive3"}
hive3-common = { group = "org.apache.hive", name = "hive-common", version.ref
= "hive3"}
-hadoop2-auth = { group = "org.apache.hadoop", name = "hadoop-auth",
version.ref = "hadoop2" }
-hadoop2-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs",
version.ref = "hadoop2" }
-hadoop2-hdfs-client = { group = "org.apache.hadoop", name =
"hadoop-hdfs-client", version.ref = "hadoop2" }
-hadoop2-common = { group = "org.apache.hadoop", name = "hadoop-common",
version.ref = "hadoop2"}
-hadoop2-mapreduce-client-core = { group = "org.apache.hadoop", name =
"hadoop-mapreduce-client-core", version.ref = "hadoop2"}
-hadoop2-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref
= "hadoop2"}
hadoop3-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref
= "hadoop3"}
+hadoop3-auth = { group = "org.apache.hadoop", name = "hadoop-auth",
version.ref = "hadoop3" }
hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs",
version.ref = "hadoop3" }
+hadoop3-hdfs-client = { group = "org.apache.hadoop", name =
"hadoop-hdfs-client", version.ref = "hadoop3" }
hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common",
version.ref = "hadoop3"}
hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client",
version.ref = "hadoop3"}
hadoop3-client-api = { group = "org.apache.hadoop", name =
"hadoop-client-api", version.ref = "hadoop3"}
hadoop3-client-runtime = { group = "org.apache.hadoop", name =
"hadoop-client-runtime", version.ref = "hadoop3"}
+hadoop3-mapreduce-client-core = { group = "org.apache.hadoop", name =
"hadoop-mapreduce-client-core", version.ref = "hadoop3"}
hadoop3-minicluster = { group = "org.apache.hadoop", name =
"hadoop-minicluster", version.ref = "hadoop-minikdc"}
hadoop3-gcs = { group = "com.google.cloud.bigdataoss", name = "gcs-connector",
version.ref = "hadoop3-gcs"}
hadoop3-oss = { group = "org.apache.hadoop", name = "hadoop-aliyun",
version.ref = "hadoop3-aliyun"}
hadoop3-abs = { group = "org.apache.hadoop", name = "hadoop-azure",
version.ref = "hadoop3-abs"}
+hadoop3-shaded-guava = { group = "org.apache.hadoop.thirdparty", name =
"hadoop-shaded-guava", version.ref = "hadoop3-thirdparty" }
+hadoop3-shaded-protobuf = { group = "org.apache.hadoop.thirdparty", name =
"hadoop-shaded-protobuf_3_7", version.ref = "hadoop3-thirdparty" }
+re2j = { group = "com.google.re2j", name = "re2j", version.ref = "re2j" }
htrace-core4 = { group = "org.apache.htrace", name = "htrace-core4",
version.ref = "htrace-core4" }
airlift-json = { group = "io.airlift", name = "json", version.ref =
"airlift-json"}
airlift-resolver = { group = "io.airlift.resolver", name = "resolver",
version.ref = "airlift-resolver"}
@@ -232,6 +234,7 @@ caffeine = { group = "com.github.ben-manes.caffeine", name
= "caffeine", version
commons-collections4 = { group = "org.apache.commons", name =
"commons-collections4", version.ref = "commons-collections4" }
commons-collections3 = { group = "commons-collections", name =
"commons-collections", version.ref = "commons-collections3" }
commons-configuration1 = { group = "commons-configuration", name =
"commons-configuration", version.ref = "commons-configuration1" }
+commons-configuration2 = { group = "org.apache.commons", name =
"commons-configuration2", version.ref = "commons-configuration2" }
iceberg-aliyun = { group = "org.apache.iceberg", name = "iceberg-aliyun",
version.ref = "iceberg" }
iceberg-aws = { group = "org.apache.iceberg", name = "iceberg-aws",
version.ref = "iceberg" }
iceberg-aws-bundle = { group = "org.apache.iceberg", name =
"iceberg-aws-bundle", version.ref = "iceberg" }
diff --git a/trino-connector/integration-test/build.gradle.kts
b/trino-connector/integration-test/build.gradle.kts
index bb15ef356e..5110e8e949 100644
--- a/trino-connector/integration-test/build.gradle.kts
+++ b/trino-connector/integration-test/build.gradle.kts
@@ -41,13 +41,19 @@ dependencies {
testImplementation(libs.bundles.jetty)
testImplementation(libs.bundles.log4j)
testImplementation(libs.commons.cli)
- testImplementation(libs.hadoop2.common) {
+ testImplementation(libs.hadoop3.common) {
exclude("*")
}
- testImplementation(libs.hadoop2.hdfs)
- testImplementation(libs.hadoop2.mapreduce.client.core) {
+ testImplementation(libs.hadoop3.hdfs)
+ testImplementation(libs.hadoop3.hdfs.client)
+ testImplementation(libs.hadoop3.mapreduce.client.core) {
exclude("*")
}
+ // Hadoop 3.x runtime requirements (stripped by exclude("*") above)
+ testImplementation(libs.hadoop3.shaded.guava)
+ testImplementation(libs.hadoop3.shaded.protobuf)
+ testImplementation(libs.commons.configuration2)
+ testImplementation(libs.re2j)
testImplementation(libs.hive2.common) {
exclude("org.eclipse.jetty.aggregate", "jetty-all")
exclude("org.eclipse.jetty.orbit", "javax.servlet")
