This is an automated email from the ASF dual-hosted git repository.
mridulm80 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4f09b6cc101 [SPARK-45376][CORE] Add netty-tcnative-boringssl-static
dependency
4f09b6cc101 is described below
commit 4f09b6cc101bd17f283fe1f9013579c6529795ba
Author: Hasnain Lakhani <[email protected]>
AuthorDate: Tue Oct 3 18:59:18 2023 -0500
[SPARK-45376][CORE] Add netty-tcnative-boringssl-static dependency
### What changes were proposed in this pull request?
This PR adds a new dependency on this library so that we can use it for SSL
functionality.
We also include the network-common test helper library in a few places,
which will be needed for reusing some test configuration files in the future.
### Why are the changes needed?
These dependency changes are needed so we can use this library to provide
SSL functionality, and for test helpers.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI. This is used in unit tests and functionality as part of
https://github.com/apache/spark/pull/42685, from which this is being split out
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #43164 from hasnain-db/spark-tls-deps.
Authored-by: Hasnain Lakhani <[email protected]>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
common/network-common/pom.xml | 34 +++++++++++++++++++++++++++++++++-
core/pom.xml | 7 +++++++
dev/deps/spark-deps-hadoop-3-hive-2.3 | 6 ++++++
pom.xml | 25 +++++++++++++++++++++++++
resource-managers/yarn/pom.xml | 7 +++++++
5 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index c8bacf96aed..bec9531fcbe 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -61,6 +61,26 @@
<artifactId>netty-transport-native-kqueue</artifactId>
<classifier>osx-x86_64</classifier>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <classifier>linux-x86_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <classifier>linux-aarch_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <classifier>osx-aarch_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <classifier>osx-x86_64</classifier>
+ </dependency>
<!-- Netty End -->
<dependency>
@@ -141,12 +161,24 @@
<artifactId>log4j-slf4j2-impl</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>${bouncycastle.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-common-utils_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- </dependency>
+ </dependency>
<!--
This spark-tags test-dep is needed even though it isn't used in this
module, otherwise testing-cmds that exclude
diff --git a/core/pom.xml b/core/pom.xml
index bbfac34117b..5ac3d5bb4de 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -446,6 +446,13 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-common_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-network-shuffle_${scala.binary.version}</artifactId>
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 38f05e39dcd..f4e3e2006f8 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -197,6 +197,12 @@ netty-common/4.1.99.Final//netty-common-4.1.99.Final.jar
netty-handler-proxy/4.1.99.Final//netty-handler-proxy-4.1.99.Final.jar
netty-handler/4.1.99.Final//netty-handler-4.1.99.Final.jar
netty-resolver/4.1.99.Final//netty-resolver-4.1.99.Final.jar
+netty-tcnative-boringssl-static/2.0.61.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
+netty-tcnative-boringssl-static/2.0.61.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
+netty-tcnative-boringssl-static/2.0.61.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
+netty-tcnative-boringssl-static/2.0.61.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
+netty-tcnative-boringssl-static/2.0.61.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
+netty-tcnative-classes/2.0.61.Final//netty-tcnative-classes-2.0.61.Final.jar
netty-transport-classes-epoll/4.1.99.Final//netty-transport-classes-epoll-4.1.99.Final.jar
netty-transport-classes-kqueue/4.1.99.Final//netty-transport-classes-kqueue-4.1.99.Final.jar
netty-transport-native-epoll/4.1.99.Final/linux-aarch_64/netty-transport-native-epoll-4.1.99.Final-linux-aarch_64.jar
diff --git a/pom.xml b/pom.xml
index fc3297c96bd..ef98ecf1291 100644
--- a/pom.xml
+++ b/pom.xml
@@ -223,6 +223,7 @@
<bouncycastle.version>1.70</bouncycastle.version>
<tink.version>1.9.0</tink.version>
<netty.version>4.1.99.Final</netty.version>
+ <netty-tcnative.version>2.0.61.Final</netty-tcnative.version>
<!--
If you are changing Arrow version specification, please check
./python/pyspark/sql/pandas/utils.py, and ./python/setup.py too.
@@ -928,6 +929,30 @@
<version>${netty.version}</version>
<classifier>osx-x86_64</classifier>
</dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>${netty-tcnative.version}</version>
+ <classifier>linux-x86_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>${netty-tcnative.version}</version>
+ <classifier>linux-aarch_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>${netty-tcnative.version}</version>
+ <classifier>osx-aarch_64</classifier>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>${netty-tcnative.version}</version>
+ <classifier>osx-x86_64</classifier>
+ </dependency>
<!-- Netty End -->
<dependency>
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 073661e9ac6..75ab7bc6e2f 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -91,6 +91,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-common_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
