This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 15645d8d20c7 [SPARK-55831][YARN] Support
`spark.yarn.am.defaultJavaOptions`
15645d8d20c7 is described below
commit 15645d8d20c7ecf9147d64818d7d9363182aa3bd
Author: Cheng Pan <[email protected]>
AuthorDate: Thu Mar 5 10:35:15 2026 +0800
[SPARK-55831][YARN] Support `spark.yarn.am.defaultJavaOptions`
### What changes were proposed in this pull request?
Introduce a new config `spark.yarn.am.defaultJavaOptions`, if set, it will
be prepended to the `spark.yarn.am.extraJavaOptions`. Obviously, this only
takes effect on YARN client mode.
### Why are the changes needed?
Currently, we have both `defaultJavaOptions` and `extraJavaOptions` for
driver and executor, but only have `extraJavaOptions` for YARN AM, this is not
intuitive. I also found AI sometimes suggests such a non-existent config.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New UT is added.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #54618 from pan3793/SPARK-55831.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
docs/running-on-yarn.md | 17 +++++-
.../apache/spark/deploy/yarn/config/package.scala | 11 +++-
.../org/apache/spark/deploy/yarn/ClientSuite.scala | 65 +++++++++++++++++++++-
3 files changed, 90 insertions(+), 3 deletions(-)
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 465f3a9d075a..d42e5eb4257e 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -406,6 +406,19 @@ To use a custom metrics.properties for the application
master and executors, upd
</td>
<td>1.2.0</td>
</tr>
+<tr>
+ <td><code>spark.yarn.am.defaultJavaOptions</code></td>
+ <td>(none)</td>
+ <td>
+ A string of default JVM options to prepend to
<code>spark.yarn.am.extraJavaOptions</code>
+ for the YARN Application Master in client mode. Note that it is illegal to
set maximum
+ heap size (-Xmx) settings with this option. Maximum heap size settings can
be set with
+ <code>spark.yarn.am.memory</code>.
+
+ This is intended to be set by administrators.
+ </td>
+ <td>4.2.0</td>
+</tr>
<tr>
<td><code>spark.yarn.am.extraJavaOptions</code></td>
<td>(none)</td>
@@ -413,7 +426,9 @@ To use a custom metrics.properties for the application
master and executors, upd
A string of extra JVM options to pass to the YARN Application Master in
client mode.
In cluster mode, use <code>spark.driver.extraJavaOptions</code> instead.
Note that it is illegal
to set maximum heap size (-Xmx) settings with this option. Maximum heap size
settings can be set
- with <code>spark.yarn.am.memory</code>
+ with <code>spark.yarn.am.memory</code>.
+
+ <code>spark.yarn.am.defaultJavaOptions</code> will be prepended to this
configuration.
</td>
<td>1.3.0</td>
</tr>
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
index f719ca4677af..8b63d117e7fa 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config/package.scala
@@ -301,8 +301,17 @@ package object config extends Logging {
.intConf
.createWithDefault(1)
+ private[spark] val AM_DEFAULT_JAVA_OPTIONS =
ConfigBuilder("spark.yarn.am.defaultJavaOptions")
+ .doc("Default Java options for the client-mode AM to prepend to " +
+ "`spark.yarn.am.extraJavaOptions`. This is intended to be set by
administrators.")
+ .version("4.2.0")
+ .stringConf
+ .createOptional
+
private[spark] val AM_JAVA_OPTIONS =
ConfigBuilder("spark.yarn.am.extraJavaOptions")
- .doc("Extra Java options for the client-mode AM.")
+ .withPrepended(AM_DEFAULT_JAVA_OPTIONS.key)
+ .doc("Extra Java options for the client-mode AM. " +
+ s"`${AM_DEFAULT_JAVA_OPTIONS.key}` will be prepended to this
configuration.")
.version("1.3.0")
.stringConf
.createOptional
diff --git
a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index 93d6cc474d20..918cb790bdc9 100644
---
a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++
b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -46,19 +46,22 @@ import org.apache.hadoop.yarn.util.Records
import org.mockito.ArgumentMatchers.{any, anyBoolean, eq => meq}
import org.mockito.Mockito._
import org.mockito.invocation.InvocationOnMock
+import org.scalatest.PrivateMethodTester
import org.scalatest.matchers.must.Matchers
import org.scalatest.matchers.should.Matchers._
import org.apache.spark.{SparkConf, SparkException, SparkFunSuite, TestUtils}
import org.apache.spark.deploy.yarn.config._
import org.apache.spark.internal.config._
+import org.apache.spark.launcher.SparkLauncher
import org.apache.spark.resource.ResourceID
import org.apache.spark.resource.ResourceUtils.AMOUNT
import org.apache.spark.util.{SparkConfWithEnv, Utils}
class ClientSuite extends SparkFunSuite
with Matchers
- with ResourceRequestTestHelper {
+ with ResourceRequestTestHelper
+ with PrivateMethodTester {
private def doReturn(value: Any) = org.mockito.Mockito.doReturn(value,
Seq.empty: _*)
import Client._
@@ -755,6 +758,66 @@ class ClientSuite extends SparkFunSuite
env("SPARK_USER") should be ("overrideuser")
}
+ test("YARN AM JavaOptions") {
+ Seq("client", "cluster").foreach { deployMode =>
+ withTempDir { stagingDir =>
+ val sparkConf = new SparkConfWithEnv(
+ Map("SPARK_HOME" -> System.getProperty("spark.test.home")))
+ .set(SUBMIT_DEPLOY_MODE, deployMode)
+ .set(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS, "-Dx=1 -Dy=2")
+ .set(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Dz=3")
+ .set(AM_DEFAULT_JAVA_OPTIONS, "-Da=1 -Db=2")
+ .set(AM_JAVA_OPTIONS, "-Dc=3")
+
+ val client = createClient(sparkConf)
+ val appIdField = classOf[Client]
+ .getDeclaredField("org$apache$spark$deploy$yarn$Client$$appId")
+ appIdField.setAccessible(true)
+ // A dummy ApplicationId impl, only `toString` method will be called
+ // in Client.createContainerLaunchContext
+ appIdField.set(client, new ApplicationId {
+ override def getId: Int = 1
+ override def setId(i: Int): Unit = {}
+ override def getClusterTimestamp: Long = 1770077136288L
+ override def setClusterTimestamp(l: Long): Unit = {}
+ override def build(): Unit = {}
+ override def toString: String = "application_1770077136288_0001"
+ })
+ val stagingDirPathField = classOf[Client]
+
.getDeclaredField("org$apache$spark$deploy$yarn$Client$$stagingDirPath")
+ stagingDirPathField.setAccessible(true)
+ stagingDirPathField.set(client, new Path(stagingDir.getAbsolutePath))
+ val _createContainerLaunchContext =
+
PrivateMethod[ContainerLaunchContext](Symbol("createContainerLaunchContext"))
+ val containerLaunchContext = client invokePrivate
_createContainerLaunchContext()
+
+ val commands = containerLaunchContext.getCommands.asScala
+ deployMode match {
+ case "client" =>
+ // In client mode, spark.yarn.am.defaultJavaOptions and
spark.yarn.am.extraJavaOptions
+ // should be set in AM container command JAVA_OPTIONS
+ commands should contain("'-Da=1'")
+ commands should contain("'-Db=2'")
+ commands should contain("'-Dc=3'")
+ commands should not contain "'-Dx=1'"
+ commands should not contain "'-Dy=2'"
+ commands should not contain "'-Dz=3'"
+ case "cluster" =>
+ // In cluster mode, spark.driver.defaultJavaOptions and
spark.driver.extraJavaOptions
+ // should be set in AM container command JAVA_OPTIONS
+ commands should not contain "'-Da=1'"
+ commands should not contain "'-Db=2'"
+ commands should not contain "'-Dc=3'"
+ commands should contain ("'-Dx=1'")
+ commands should contain ("'-Dy=2'")
+ commands should contain ("'-Dz=3'")
+ case m =>
+ fail(s"Unexpected deploy mode: $m")
+ }
+ }
+ }
+ }
+
private val matching = Seq(
("files URI match test1", "file:///file1", "file:///file2"),
("files URI match test2", "file:///c:file1", "file://c:file2"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
