This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 49e2c9ecb975 [SPARK-53849][BUILD] Upgrade Netty to 4.2.6.Final
49e2c9ecb975 is described below
commit 49e2c9ecb97576b0210444b3ef75e235d539088f
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Oct 8 23:57:50 2025 -0700
[SPARK-53849][BUILD] Upgrade Netty to 4.2.6.Final
### What changes were proposed in this pull request?
This PR aims to upgrade `Netty` to `4.2.6.Final` as a part of `Apache Spark
4.1.0` preparation.
This PR follows the official community migration guide.
- https://github.com/netty/netty/wiki/Netty-4.2-Migration-Guide
### Why are the changes needed?
`Netty` community recommends `4.2`.
> After a lot of work and many contributions from our OSS community we are
thrilled to finally announce the release of netty 4.2.0.Final! Everyone using
netty 4.1.x should be able to upgrade to 4.2.0.Final without any API breakage.
The only new requirement is JDK8 or later (and JDK9+ for io_uring) and a more
recent GLIBC version when using our native transports.
- https://netty.io/news/2025/09/08/4-2-6.html
- https://netty.io/news/2025/09/03/4-2-5.html
- https://netty.io/news/2025/08/13/4-2-4.html
- https://netty.io/news/2025/07/15/4-2-3.html
- https://netty.io/news/2025/06/05/4-2-2.html
- https://netty.io/news/2025/05/06/4-2-1.html
- https://netty.io/news/2025/04/03/4-2-0.html
### Does this PR introduce _any_ user-facing change?
No behavior change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52552 from dongjoon-hyun/SPARK-53849.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../client/TransportClientFactorySuite.java | 4 +-
dev/deps/spark-deps-hadoop-3-hive-2.3 | 57 ++++++++++++++--------
.../apache/spark/launcher/JavaModuleOptions.java | 2 +
pom.xml | 4 +-
project/SparkBuild.scala | 2 +
5 files changed, 46 insertions(+), 23 deletions(-)
diff --git
a/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
b/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
index b57f0be920c7..58faea2cf2dd 100644
---
a/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
+++
b/common/network-common/src/test/java/org/apache/spark/network/client/TransportClientFactorySuite.java
@@ -225,11 +225,12 @@ public class TransportClientFactorySuite {
}
@Test
- public void fastFailConnectionInTimeWindow() {
+ public void fastFailConnectionInTimeWindow() throws IOException,
InterruptedException {
TransportClientFactory factory = context.createClientFactory();
TransportServer server = context.createServer();
int unreachablePort = server.getPort();
server.close();
+ Thread.sleep(1000);
Assertions.assertThrows(IOException.class,
() -> factory.createClient(TestUtils.getLocalHost(), unreachablePort,
true));
Assertions.assertThrows(IOException.class,
@@ -258,6 +259,7 @@ public class TransportClientFactorySuite {
TransportServer server = ctx.createServer();
int unreachablePort = server.getPort();
JavaUtils.closeQuietly(server);
+ Thread.sleep(1000);
IOException exception = Assertions.assertThrows(IOException.class,
() -> factory.createClient(TestUtils.getLocalHost(),
unreachablePort, true));
assertNotEquals(exception.getCause(), null);
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 4fbeadacd045..08dd444efd87 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -198,33 +198,48 @@ metrics-jmx/4.2.33//metrics-jmx-4.2.33.jar
metrics-json/4.2.33//metrics-json-4.2.33.jar
metrics-jvm/4.2.33//metrics-jvm-4.2.33.jar
minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.127.Final//netty-all-4.1.127.Final.jar
-netty-buffer/4.1.127.Final//netty-buffer-4.1.127.Final.jar
-netty-codec-dns/4.1.127.Final//netty-codec-dns-4.1.127.Final.jar
-netty-codec-http/4.1.127.Final//netty-codec-http-4.1.127.Final.jar
-netty-codec-http2/4.1.127.Final//netty-codec-http2-4.1.127.Final.jar
-netty-codec-socks/4.1.127.Final//netty-codec-socks-4.1.127.Final.jar
-netty-codec/4.1.127.Final//netty-codec-4.1.127.Final.jar
-netty-common/4.1.127.Final//netty-common-4.1.127.Final.jar
-netty-handler-proxy/4.1.127.Final//netty-handler-proxy-4.1.127.Final.jar
-netty-handler/4.1.127.Final//netty-handler-4.1.127.Final.jar
-netty-resolver-dns/4.1.127.Final//netty-resolver-dns-4.1.127.Final.jar
-netty-resolver/4.1.127.Final//netty-resolver-4.1.127.Final.jar
+netty-all/4.2.6.Final//netty-all-4.2.6.Final.jar
+netty-buffer/4.2.6.Final//netty-buffer-4.2.6.Final.jar
+netty-codec-base/4.2.6.Final//netty-codec-base-4.2.6.Final.jar
+netty-codec-classes-quic/4.2.6.Final//netty-codec-classes-quic-4.2.6.Final.jar
+netty-codec-compression/4.2.6.Final//netty-codec-compression-4.2.6.Final.jar
+netty-codec-dns/4.2.6.Final//netty-codec-dns-4.2.6.Final.jar
+netty-codec-http/4.2.6.Final//netty-codec-http-4.2.6.Final.jar
+netty-codec-http2/4.2.6.Final//netty-codec-http2-4.2.6.Final.jar
+netty-codec-http3/4.2.6.Final//netty-codec-http3-4.2.6.Final.jar
+netty-codec-marshalling/4.2.6.Final//netty-codec-marshalling-4.2.6.Final.jar
+netty-codec-native-quic/4.2.6.Final/linux-aarch_64/netty-codec-native-quic-4.2.6.Final-linux-aarch_64.jar
+netty-codec-native-quic/4.2.6.Final/linux-x86_64/netty-codec-native-quic-4.2.6.Final-linux-x86_64.jar
+netty-codec-native-quic/4.2.6.Final/osx-aarch_64/netty-codec-native-quic-4.2.6.Final-osx-aarch_64.jar
+netty-codec-native-quic/4.2.6.Final/osx-x86_64/netty-codec-native-quic-4.2.6.Final-osx-x86_64.jar
+netty-codec-native-quic/4.2.6.Final/windows-x86_64/netty-codec-native-quic-4.2.6.Final-windows-x86_64.jar
+netty-codec-protobuf/4.2.6.Final//netty-codec-protobuf-4.2.6.Final.jar
+netty-codec-socks/4.2.6.Final//netty-codec-socks-4.2.6.Final.jar
+netty-codec/4.2.6.Final//netty-codec-4.2.6.Final.jar
+netty-common/4.2.6.Final//netty-common-4.2.6.Final.jar
+netty-handler-proxy/4.2.6.Final//netty-handler-proxy-4.2.6.Final.jar
+netty-handler/4.2.6.Final//netty-handler-4.2.6.Final.jar
+netty-resolver-dns/4.2.6.Final//netty-resolver-dns-4.2.6.Final.jar
+netty-resolver/4.2.6.Final//netty-resolver-4.2.6.Final.jar
netty-tcnative-boringssl-static/2.0.74.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.74.Final-linux-aarch_64.jar
netty-tcnative-boringssl-static/2.0.74.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-linux-x86_64.jar
netty-tcnative-boringssl-static/2.0.74.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.74.Final-osx-aarch_64.jar
netty-tcnative-boringssl-static/2.0.74.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-osx-x86_64.jar
netty-tcnative-boringssl-static/2.0.74.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.74.Final-windows-x86_64.jar
netty-tcnative-classes/2.0.74.Final//netty-tcnative-classes-2.0.74.Final.jar
-netty-transport-classes-epoll/4.1.127.Final//netty-transport-classes-epoll-4.1.127.Final.jar
-netty-transport-classes-kqueue/4.1.127.Final//netty-transport-classes-kqueue-4.1.127.Final.jar
-netty-transport-native-epoll/4.1.127.Final/linux-aarch_64/netty-transport-native-epoll-4.1.127.Final-linux-aarch_64.jar
-netty-transport-native-epoll/4.1.127.Final/linux-riscv64/netty-transport-native-epoll-4.1.127.Final-linux-riscv64.jar
-netty-transport-native-epoll/4.1.127.Final/linux-x86_64/netty-transport-native-epoll-4.1.127.Final-linux-x86_64.jar
-netty-transport-native-kqueue/4.1.127.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.127.Final-osx-aarch_64.jar
-netty-transport-native-kqueue/4.1.127.Final/osx-x86_64/netty-transport-native-kqueue-4.1.127.Final-osx-x86_64.jar
-netty-transport-native-unix-common/4.1.127.Final//netty-transport-native-unix-common-4.1.127.Final.jar
-netty-transport/4.1.127.Final//netty-transport-4.1.127.Final.jar
+netty-transport-classes-epoll/4.2.6.Final//netty-transport-classes-epoll-4.2.6.Final.jar
+netty-transport-classes-io_uring/4.2.6.Final//netty-transport-classes-io_uring-4.2.6.Final.jar
+netty-transport-classes-kqueue/4.2.6.Final//netty-transport-classes-kqueue-4.2.6.Final.jar
+netty-transport-native-epoll/4.2.6.Final/linux-aarch_64/netty-transport-native-epoll-4.2.6.Final-linux-aarch_64.jar
+netty-transport-native-epoll/4.2.6.Final/linux-riscv64/netty-transport-native-epoll-4.2.6.Final-linux-riscv64.jar
+netty-transport-native-epoll/4.2.6.Final/linux-x86_64/netty-transport-native-epoll-4.2.6.Final-linux-x86_64.jar
+netty-transport-native-io_uring/4.2.6.Final/linux-aarch_64/netty-transport-native-io_uring-4.2.6.Final-linux-aarch_64.jar
+netty-transport-native-io_uring/4.2.6.Final/linux-riscv64/netty-transport-native-io_uring-4.2.6.Final-linux-riscv64.jar
+netty-transport-native-io_uring/4.2.6.Final/linux-x86_64/netty-transport-native-io_uring-4.2.6.Final-linux-x86_64.jar
+netty-transport-native-kqueue/4.2.6.Final/osx-aarch_64/netty-transport-native-kqueue-4.2.6.Final-osx-aarch_64.jar
+netty-transport-native-kqueue/4.2.6.Final/osx-x86_64/netty-transport-native-kqueue-4.2.6.Final-osx-x86_64.jar
+netty-transport-native-unix-common/4.2.6.Final//netty-transport-native-unix-common-4.2.6.Final.jar
+netty-transport/4.2.6.Final//netty-transport-4.2.6.Final.jar
objenesis/3.4//objenesis-3.4.jar
okhttp/3.12.12//okhttp-3.12.12.jar
okio/1.17.6//okio-1.17.6.jar
diff --git
a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
index cef426035e63..b42c7906fe18 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
@@ -45,6 +45,8 @@ public class JavaModuleOptions {
"--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED",
"-Djdk.reflect.useDirectMethodHandle=false",
"-Dio.netty.tryReflectionSetAccessible=true",
+ "-Dio.netty.allocator.type=pooled",
+ "-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE",
"--enable-native-access=ALL-UNNAMED"};
/**
diff --git a/pom.xml b/pom.xml
index 3f2039dbd730..8ab199988fd3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -217,7 +217,7 @@
<bouncycastle.version>1.82</bouncycastle.version>
<tink.version>1.16.0</tink.version>
<datasketches.version>6.2.0</datasketches.version>
- <netty.version>4.1.127.Final</netty.version>
+ <netty.version>4.2.6.Final</netty.version>
<netty-tcnative.version>2.0.74.Final</netty-tcnative.version>
<icu4j.version>77.1</icu4j.version>
<junit-jupiter.version>5.13.4</junit-jupiter.version>
@@ -337,6 +337,8 @@
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-Djdk.reflect.useDirectMethodHandle=false
-Dio.netty.tryReflectionSetAccessible=true
+ -Dio.netty.allocator.type=pooled
+ -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
--enable-native-access=ALL-UNNAMED
</extraJavaTestArgs>
<mariadb.java.client.version>2.7.12</mariadb.java.client.version>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 09459671b3cc..14ce2608d6fb 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1704,6 +1704,8 @@ object TestSettings {
"--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
"-Djdk.reflect.useDirectMethodHandle=false",
"-Dio.netty.tryReflectionSetAccessible=true",
+ "-Dio.netty.allocator.type=pooled",
+ "-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE",
"--enable-native-access=ALL-UNNAMED").mkString(" ")
s"-Xmx$heapSize -Xss4m -XX:MaxMetaspaceSize=$metaspaceSize
-XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs"
.split(" ").toSeq
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
