This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new fb13ecd27944 [SPARK-54847][BUILD] unify the proto/antlr output folder
between sbt and maven
fb13ecd27944 is described below
commit fb13ecd279440cc08fca664ddcd224b8927d796e
Author: Wenchen Fan <[email protected]>
AuthorDate: Sat Dec 27 09:22:05 2025 +0800
[SPARK-54847][BUILD] unify the proto/antlr output folder between sbt and
maven
### What changes were proposed in this pull request?
People may use sbt and maven together: local test with sbt, IDE with Maven.
As of today this will trigger duplicated class issue during compilation,
because sbt and Maven use different output folder for generated proto/antlr
java classes. This PR updates the sbt build to use the same folder as Maven for
proto/antlr output.
### Why are the changes needed?
Improve dev experience.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manual test
### Was this patch authored or co-authored using generative AI tooling?
cursor 2.2.43
Closes #53616 from cloud-fan/shim.
Lead-authored-by: Wenchen Fan <[email protected]>
Co-authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 174b62d01212c8032c9ef63cd2beddcc80d0d8d6)
Signed-off-by: Wenchen Fan <[email protected]>
---
.sbtopts | 20 ++++++++++++++++++++
project/SparkBuild.scala | 21 ++++++++++++++-------
2 files changed, 34 insertions(+), 7 deletions(-)
diff --git a/.sbtopts b/.sbtopts
new file mode 100644
index 000000000000..3516fc4bd7eb
--- /dev/null
+++ b/.sbtopts
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+-J-Xmx8g
+-J-Xms8g
+-J-XX:MaxMetaspaceSize=1g
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index fec6a3346f7d..988d5c2e6439 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -646,8 +646,10 @@ object Core {
"com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf"
)
},
+ // Use Maven's output directory so sbt and Maven can share generated
sources.
+ // Core uses protoc-jar-maven-plugin which outputs to
target/generated-sources.
(Compile / PB.targets) := Seq(
- PB.gens.java -> (Compile / sourceManaged).value
+ PB.gens.java -> target.value / "generated-sources"
)
) ++ {
val sparkProtocExecPath = sys.props.get("spark.protoc.executable.path")
@@ -732,19 +734,20 @@ object SparkConnectCommon {
) ++ {
val sparkProtocExecPath = sys.props.get("spark.protoc.executable.path")
val connectPluginExecPath = sys.props.get("connect.plugin.executable.path")
+ // Use Maven's output directory so sbt and Maven can share generated
sources
if (sparkProtocExecPath.isDefined && connectPluginExecPath.isDefined) {
Seq(
(Compile / PB.targets) := Seq(
- PB.gens.java -> (Compile / sourceManaged).value,
- PB.gens.plugin(name = "grpc-java", path = connectPluginExecPath.get)
-> (Compile / sourceManaged).value
+ PB.gens.java -> target.value / "generated-sources" / "protobuf" /
"java",
+ PB.gens.plugin(name = "grpc-java", path = connectPluginExecPath.get)
-> target.value / "generated-sources" / "protobuf" / "grpc-java"
),
PB.protocExecutable := file(sparkProtocExecPath.get)
)
} else {
Seq(
(Compile / PB.targets) := Seq(
- PB.gens.java -> (Compile / sourceManaged).value,
- PB.gens.plugin("grpc-java") -> (Compile / sourceManaged).value
+ PB.gens.java -> target.value / "generated-sources" / "protobuf" /
"java",
+ PB.gens.plugin("grpc-java") -> target.value / "generated-sources" /
"protobuf" / "grpc-java"
)
)
}
@@ -1283,7 +1286,9 @@ object SqlApi {
(Antlr4 / antlr4PackageName) :=
Some("org.apache.spark.sql.catalyst.parser"),
(Antlr4 / antlr4GenListener) := true,
(Antlr4 / antlr4GenVisitor) := true,
- (Antlr4 / antlr4TreatWarningsAsErrors) := true
+ (Antlr4 / antlr4TreatWarningsAsErrors) := true,
+ // Use Maven's output directory so sbt and Maven can share generated
sources
+ (Antlr4 / javaSource) := target.value / "generated-sources" / "antlr4"
)
}
@@ -1300,8 +1305,10 @@ object SQL {
"com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf"
)
},
+ // Use Maven's output directory so sbt and Maven can share generated
sources.
+ // sql/core uses protoc-jar-maven-plugin which outputs to
target/generated-sources.
(Compile / PB.targets) := Seq(
- PB.gens.java -> (Compile / sourceManaged).value
+ PB.gens.java -> target.value / "generated-sources"
)
) ++ {
val sparkProtocExecPath = sys.props.get("spark.protoc.executable.path")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
