This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 24a6abf34d25 [SPARK-53935][BUILD] SBT assembly should handle META-INF
correctly
24a6abf34d25 is described below
commit 24a6abf34d253162055c8b9bd0030bf9a2ca75b1
Author: Cheng Pan <[email protected]>
AuthorDate: Thu Oct 16 18:33:59 2025 +0900
[SPARK-53935][BUILD] SBT assembly should handle META-INF correctly
### What changes were proposed in this pull request?
Fix SBT assembly for all sub-projects to handle META-INF correctly
- `META-INF/*.SF` (signature files, should be dropped for assembly jars)
- `META-INF/services/*` (service loader implementation list, should be
merged for assembly jars)
Similiar thing was already done in the `assembly` project.
https://github.com/apache/spark/blob/83be7e7cfea4eadf2da990cdee4651a4f8b5da65/project/SparkBuild.scala#L1300-L1310
### Why are the changes needed?
Issue was detected when I'm trying to integrate Connect JDBC driver with
BeeLine in dev mode.
### Does this PR introduce _any_ user-facing change?
No, official release uses Maven, which does not get affected.
### How was this patch tested?
Pass GHA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52636 from pan3793/SPARK-53935.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
---
project/SparkBuild.scala | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index b602edebe8fe..a57aae28a699 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -719,6 +719,8 @@ object SparkConnectCommon {
(assembly / assemblyMergeStrategy) := {
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).matches("meta-inf.*\\.sf$") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).startsWith("meta-inf/services/") =>
MergeStrategy.filterDistinctLines
// Drop all proto files that are not needed as artifacts of the build.
case m if m.toLowerCase(Locale.ROOT).endsWith(".proto") =>
MergeStrategy.discard
case _ => MergeStrategy.first
@@ -826,6 +828,8 @@ object SparkConnect {
(assembly / assemblyMergeStrategy) := {
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).matches("meta-inf.*\\.sf$") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).startsWith("meta-inf/services/") =>
MergeStrategy.filterDistinctLines
// Drop all proto files that are not needed as artifacts of the build.
case m if m.toLowerCase(Locale.ROOT).endsWith(".proto") =>
MergeStrategy.discard
case _ => MergeStrategy.first
@@ -904,6 +908,8 @@ object SparkConnectClient {
(assembly / assemblyMergeStrategy) := {
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).matches("meta-inf.*\\.sf$") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).startsWith("meta-inf/services/") =>
MergeStrategy.filterDistinctLines
// Drop all proto files that are not needed as artifacts of the build.
case m if m.toLowerCase(Locale.ROOT).endsWith(".proto") =>
MergeStrategy.discard
case _ => MergeStrategy.first
@@ -959,6 +965,8 @@ object SparkProtobuf {
(assembly / assemblyMergeStrategy) := {
case m if m.toLowerCase(Locale.ROOT).endsWith("manifest.mf") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).matches("meta-inf.*\\.sf$") =>
MergeStrategy.discard
+ case m if m.toLowerCase(Locale.ROOT).startsWith("meta-inf/services/") =>
MergeStrategy.filterDistinctLines
// Drop all proto files that are not needed as artifacts of the build.
case m if m.toLowerCase(Locale.ROOT).endsWith(".proto") =>
MergeStrategy.discard
case _ => MergeStrategy.first
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
