This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 145be5fc6ff5 [SPARK-54313][CORE] Add --extra-properties-file option
for configuration layering
145be5fc6ff5 is described below
commit 145be5fc6ff52fa10ec47bf3cf06d087ac383a65
Author: John Zhuge <[email protected]>
AuthorDate: Tue Jan 6 14:37:36 2026 +0900
[SPARK-54313][CORE] Add --extra-properties-file option for configuration
layering
### What changes were proposed in this pull request?
This PR adds support for multiple properties files via the
`--extra-properties-file` option in spark-submit, enabling configuration
layering and composition.
**Changes:**
- Added `extraPropertiesFiles: Seq[String]` field to `SparkSubmitArguments`
- Added `EXTRA_PROPERTIES_FILE` constant to `SparkSubmitOptionParser`
- Modified `AbstractCommandBuilder` to load and merge extra properties files
- Updated `SparkSubmitCommandBuilder` to parse and rebuild args with extra
files
- Added 4 unit tests in `SparkSubmitSuite`
### Why are the changes needed?
Large-scale Spark deployments often need to compose configurations from
multiple sources:
- Base/shared defaults - Common settings reused across applications
- Environment-specific configs - Different settings for dev/staging/prod
- Cluster-specific settings - YARN queues, resource limits per cluster
- Application-specific overrides - Per-app tuning parameters
### Does this PR introduce any user-facing changes?
Yes. Adds new `--extra-properties-file` option to spark-submit.
**Usage:**
```bash
spark-submit \
--properties-file base.conf \
--extra-properties-file cluster.conf \
--extra-properties-file app.conf \
--class com.example.App app.jar
```
**Property Precedence** (highest to lowest):
1. `--conf` flags (command-line overrides)
2. `--extra-properties-file` files (later files override earlier ones)
3. `--properties-file` (base file)
4. `conf/spark-defaults.conf` (with `--load-spark-defaults`)
**Key Features:**
- Repeatable flag - can be specified multiple times
- Later files override earlier files (left-to-right precedence)
- No breaking changes - new flag, existing behavior unchanged
- Shell-friendly - no comma escaping or string manipulation needed
### How was this patch tested?
Added 4 unit tests in `SparkSubmitSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Unit tests and descriptions were generated by Claude Code.
Closes #53012 from jzhuge/SPARK-54313.
Authored-by: John Zhuge <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
---
.../apache/spark/deploy/SparkSubmitArguments.scala | 32 ++++++++-
.../org/apache/spark/deploy/SparkSubmitSuite.scala | 81 ++++++++++++++++++++++
.../spark/launcher/AbstractCommandBuilder.java | 11 +++
.../spark/launcher/SparkSubmitCommandBuilder.java | 8 +++
.../spark/launcher/SparkSubmitOptionParser.java | 2 +
5 files changed, 133 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 3eb2fd2a0e4d..14f7973e9ea7 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -51,6 +51,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String],
env: Map[String, S
var executorCores: String = null
var totalExecutorCores: String = null
var propertiesFile: String = null
+ var extraPropertiesFiles: Seq[String] = Nil
private var loadSparkDefaults: Boolean = false
var driverMemory: String = null
var driverExtraClassPath: String = null
@@ -132,9 +133,34 @@ private[deploy] class SparkSubmitArguments(args:
Seq[String], env: Map[String, S
* When this is called, `sparkProperties` is already filled with configs
from the latter.
*/
private def mergeDefaultSparkProperties(): Unit = {
+ // Save properties from --conf (these have the highest priority)
+ val confProperties = sparkProperties.clone()
+
// Honor --conf before the specified properties file and defaults file
loadPropertiesFromFile(propertiesFile)
+ // Extra properties files should override base properties file
+ // Later files override earlier files
+ extraPropertiesFiles.foreach { filePath =>
+ if (filePath != null) {
+ if (verbose) {
+ logInfo(log"Using properties file: ${MDC(PATH, filePath)}")
+ }
+ val properties = Utils.getPropertiesFromFile(filePath)
+ properties.foreach { case (k, v) =>
+ // Override any existing property except those from --conf
+ if (!confProperties.contains(k)) {
+ sparkProperties(k) = v
+ }
+ }
+ if (verbose) {
+ Utils.redact(properties).foreach { case (k, v) =>
+ logInfo(log"Adding default property: ${MDC(KEY, k)}=${MDC(VALUE,
v)}")
+ }
+ }
+ }
+ }
+
// Also load properties from `spark-defaults.conf` if they do not exist in
the properties file
// and --conf list when:
// - no input properties file is specified
@@ -319,6 +345,7 @@ private[deploy] class SparkSubmitArguments(args:
Seq[String], env: Map[String, S
| executorCores $executorCores
| totalExecutorCores $totalExecutorCores
| propertiesFile $propertiesFile
+ | extraPropertiesFiles [${extraPropertiesFiles.mkString(", ")}]
| driverMemory $driverMemory
| driverCores $driverCores
| driverExtraClassPath $driverExtraClassPath
@@ -341,7 +368,7 @@ private[deploy] class SparkSubmitArguments(args:
Seq[String], env: Map[String, S
| verbose $verbose
|
|Spark properties used, including those specified through
- | --conf and those from the properties file $propertiesFile:
+ | --conf and those from the properties files:
|${Utils.redact(sparkProperties).sorted.mkString(" ", "\n ", "\n")}
""".stripMargin
}
@@ -397,6 +424,9 @@ private[deploy] class SparkSubmitArguments(args:
Seq[String], env: Map[String, S
case PROPERTIES_FILE =>
propertiesFile = value
+ case EXTRA_PROPERTIES_FILE =>
+ extraPropertiesFiles :+= value
+
case LOAD_SPARK_DEFAULTS =>
loadSparkDefaults = true
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 18d3c35ea94f..eda86fdeb035 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -1816,6 +1816,87 @@ class SparkSubmitSuite
}
}
+ test("SPARK-54313: handle multiple --extra-properties-file options") {
+ withPropertyFile("base.properties", Map.empty) { baseFile =>
+ withPropertyFile("extra1.properties", Map.empty) { extra1File =>
+ withPropertyFile("extra2.properties", Map.empty) { extra2File =>
+ val clArgs = Seq(
+ "--class", "org.SomeClass",
+ "--properties-file", baseFile,
+ "--extra-properties-file", extra1File,
+ "--extra-properties-file", extra2File,
+ "--master", "yarn",
+ "thejar.jar")
+
+ val appArgs = new SparkSubmitArguments(clArgs)
+ appArgs.propertiesFile should be (baseFile)
+ appArgs.extraPropertiesFiles should be (Seq(extra1File, extra2File))
+ }
+ }
+ }
+ }
+
+ test("SPARK-54313: extra properties files override base properties") {
+ val baseProps = Map("spark.executor.memory" -> "1g", "spark.driver.memory"
-> "512m")
+ val extraProps = Map("spark.executor.memory" -> "2g")
+
+ withPropertyFile("base.properties", baseProps) { baseFile =>
+ withPropertyFile("extra.properties", extraProps) { extraFile =>
+ val clArgs = Seq(
+ "--class", "org.SomeClass",
+ "--properties-file", baseFile,
+ "--extra-properties-file", extraFile,
+ "--master", "local",
+ "thejar.jar")
+
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (_, _, conf, _) = submit.prepareSubmitEnvironment(appArgs)
+
+ conf.get("spark.executor.memory") should be ("2g") // Overridden
+ conf.get("spark.driver.memory") should be ("512m") // From base
+ }
+ }
+ }
+
+ test("SPARK-54313: later extra properties files override earlier ones") {
+ val extra1Props = Map("spark.executor.memory" -> "1g")
+ val extra2Props = Map("spark.executor.memory" -> "3g")
+
+ withPropertyFile("extra1.properties", extra1Props) { extra1File =>
+ withPropertyFile("extra2.properties", extra2Props) { extra2File =>
+ val clArgs = Seq(
+ "--class", "org.SomeClass",
+ "--extra-properties-file", extra1File,
+ "--extra-properties-file", extra2File,
+ "--master", "local",
+ "thejar.jar")
+
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (_, _, conf, _) = submit.prepareSubmitEnvironment(appArgs)
+
+ conf.get("spark.executor.memory") should be ("3g") // Last wins
+ }
+ }
+ }
+
+ test("SPARK-54313: --conf overrides extra properties files") {
+ val extraProps = Map("spark.executor.memory" -> "2g")
+
+ withPropertyFile("extra.properties", extraProps) { extraFile =>
+ val clArgs = Seq(
+ "--class", "org.SomeClass",
+ "--extra-properties-file", extraFile,
+ "--conf", "spark.executor.memory=4g",
+ "--master", "local",
+ "thejar.jar")
+
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (_, _, conf, _) = submit.prepareSubmitEnvironment(appArgs)
+
+ conf.get("spark.executor.memory") should be ("4g") // --conf wins
+ }
+ }
+
test("get a Spark configuration from arguments") {
val testConf = "spark.test.hello" -> "world"
val masterConf = "spark.master" -> "yarn"
diff --git
a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 0214b1102381..f32501c83aa1 100644
---
a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++
b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -49,6 +49,7 @@ abstract class AbstractCommandBuilder {
String master;
String remote;
protected String propertiesFile;
+ protected List<String> extraPropertiesFiles = new ArrayList<>();
protected boolean loadSparkDefaults;
final List<String> appArgs;
final List<String> jars;
@@ -381,6 +382,16 @@ abstract class AbstractCommandBuilder {
props = loadPropertiesFile(propsFile);
}
+ // Load extra properties files, later files override earlier ones
+ for (String extraFile : extraPropertiesFiles) {
+ File extraPropsFile = new File(extraFile);
+ checkArgument(extraPropsFile.isFile(), "Invalid properties file '%s'.",
extraFile);
+ Properties extraProps = loadPropertiesFile(extraPropsFile);
+ for (Map.Entry<Object, Object> entry : extraProps.entrySet()) {
+ props.put(entry.getKey(), entry.getValue());
+ }
+ }
+
Properties defaultsProps = new Properties();
if (propertiesFile == null || loadSparkDefaults) {
defaultsProps = loadPropertiesFile(new File(getConfDir(),
DEFAULT_PROPERTIES_FILE));
diff --git
a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
index 7b9f90ac7b7a..73336b55b89d 100644
---
a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
+++
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -251,6 +251,13 @@ class SparkSubmitCommandBuilder extends
AbstractCommandBuilder {
args.add(propertiesFile);
}
+ if (extraPropertiesFiles != null) {
+ for (String file : extraPropertiesFiles) {
+ args.add(parser.EXTRA_PROPERTIES_FILE);
+ args.add(file);
+ }
+ }
+
if (loadSparkDefaults) {
args.add(parser.LOAD_SPARK_DEFAULTS);
}
@@ -554,6 +561,7 @@ class SparkSubmitCommandBuilder extends
AbstractCommandBuilder {
}
case DEPLOY_MODE -> deployMode = value;
case PROPERTIES_FILE -> propertiesFile = value;
+ case EXTRA_PROPERTIES_FILE -> extraPropertiesFiles.add(value);
case LOAD_SPARK_DEFAULTS -> loadSparkDefaults = true;
case DRIVER_MEMORY -> conf.put(SparkLauncher.DRIVER_MEMORY, value);
case DRIVER_JAVA_OPTIONS ->
conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, value);
diff --git
a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
index e4511421cd13..2902b162a60e 100644
---
a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
+++
b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
@@ -55,6 +55,7 @@ class SparkSubmitOptionParser {
protected final String PACKAGES = "--packages";
protected final String PACKAGES_EXCLUDE = "--exclude-packages";
protected final String PROPERTIES_FILE = "--properties-file";
+ protected final String EXTRA_PROPERTIES_FILE = "--extra-properties-file";
protected final String LOAD_SPARK_DEFAULTS = "--load-spark-defaults";
protected final String PROXY_USER = "--proxy-user";
protected final String PY_FILES = "--py-files";
@@ -114,6 +115,7 @@ class SparkSubmitOptionParser {
{ PACKAGES_EXCLUDE },
{ PRINCIPAL },
{ PROPERTIES_FILE },
+ { EXTRA_PROPERTIES_FILE },
{ PROXY_USER },
{ PY_FILES },
{ QUEUE },
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
