This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 3a06de476703 [SPARK-57098][3.5][UI] Worker UI JSON endpoint redaction
3a06de476703 is described below
commit 3a06de476703571f08a2bd06616146b61846d014
Author: Peter Toth <[email protected]>
AuthorDate: Wed May 27 14:23:26 2026 -0700
[SPARK-57098][3.5][UI] Worker UI JSON endpoint redaction
### What changes were proposed in this pull request?
The standalone Worker UI serves `GET /json/`, which returns
`JsonProtocol.writeWorkerState(...)` and includes each executor's
`ApplicationDescription.command` rendered as `Command.toString`. Since
`Command` is a case class, that string contains the full `environment` map and
`javaOpts` sequence.
This PR redacts `Command.environment` via `Utils.redact(conf, ...)` and
`Command.javaOpts` via `Utils.redactCommandLineArgs(conf, ...)` before calling
`toString`, reusing the same redaction APIs already applied to the launch
command in `ExecutorRunner` logging (`ExecutorRunner.scala:162-164`). The
`command` field stays a string rendered from a redacted `Command.copy(...)`, so
the JSON schema is unchanged.
`ExecutorRunner.conf` is promoted to `val` so
`JsonProtocol.writeExecutorRunner` can plumb the worker's `SparkConf` into
`writeApplicationDescription` for `spark.redaction.regex` lookups.
### Why are the changes needed?
`environment` and `javaOpts` routinely carry secrets: JDBC passwords, AWS
credentials, SSL keystore passwords, Hadoop credential store passwords,
`spark.executorEnv.*` values, etc. `ExecutorRunner` already redacts the same
content when writing the launch command to logs, but `JsonProtocol` emits it
unredacted over the Worker UI HTTP endpoint. The Worker UI listens on port 8081
with no authentication by default, so any caller with network access to the
worker can read the secrets with [...]
### Does this PR introduce _any_ user-facing change?
Yes. The `command` field returned by `GET /json/` on the standalone Worker
UI now has secret-bearing values in `environment` and `javaOpts` replaced with
`*********(redacted)` when they match `spark.redaction.regex` (the default
pattern matches keys like `secret`, `password`, `token`, etc.). The JSON schema
is unchanged -- `command` remains a single string in `Command.toString` format
-- so existing tooling that parses this endpoint continues to work; only the
sensitive values that we [...]
### How was this patch tested?
- Added `SPARK-57098: secrets in executor command are redacted in worker
JSON endpoint` to `JsonProtocolSuite`, covering both environment-variable and
`-D` java-opt secret carriers, asserting that redacted values are scrubbed
while non-sensitive values (`JAVA_HOME`, `-Xmx2g`) pass through.
- `build/sbt 'core/testOnly org.apache.spark.deploy.JsonProtocolSuite'` --
11/11 tests pass.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.7
Closes #56148 from peter-toth/SPARK-57098-worker-ui-json-redaction-3.5.
Authored-by: Peter Toth <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/deploy/JsonProtocol.scala | 16 +++++++---
.../spark/deploy/worker/ExecutorRunner.scala | 2 +-
.../apache/spark/deploy/JsonProtocolSuite.scala | 37 ++++++++++++++++++++--
3 files changed, 48 insertions(+), 7 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
index f697892aacc8..0d22ebefa57c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
@@ -20,10 +20,12 @@ package org.apache.spark.deploy
import org.json4s.JsonAST._
import org.json4s.JsonDSL._
+import org.apache.spark.SparkConf
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse,
WorkerStateResponse}
import org.apache.spark.deploy.master._
import org.apache.spark.deploy.worker.ExecutorRunner
import org.apache.spark.resource.{ResourceInformation, ResourceRequirement}
+import org.apache.spark.util.Utils
private[deploy] object JsonProtocol {
@@ -123,10 +125,16 @@ private[deploy] object JsonProtocol {
* `memoryperexecutor` minimal memory in MB required to each executor
* `resourcesperexecutor` minimal resources required to each executor
* `user` name of the user who submitted the application
- * `command` the command string used to submit the application
+ * `command` the command string used to submit the application, with
secret-bearing
+ * fields (`environment`, `javaOpts`) redacted using
`spark.redaction.regex`
* For compatibility also returns the deprecated `memoryperslave` &
`resourcesperslave` fields.
*/
- def writeApplicationDescription(obj: ApplicationDescription): JObject = {
+ def writeApplicationDescription(obj: ApplicationDescription, conf:
SparkConf): JObject = {
+ val redactedEnvironment = Utils.redact(conf,
obj.command.environment.toSeq).toMap
+ val redactedJavaOpts = Utils.redactCommandLineArgs(conf,
obj.command.javaOpts)
+ val redactedCommand = obj.command.copy(
+ environment = redactedEnvironment,
+ javaOpts = redactedJavaOpts)
("name" -> obj.name) ~
("cores" -> obj.maxCores.getOrElse(0)) ~
("memoryperexecutor" -> obj.memoryPerExecutorMB) ~
@@ -134,7 +142,7 @@ private[deploy] object JsonProtocol {
("memoryperslave" -> obj.memoryPerExecutorMB) ~
("resourcesperslave" ->
obj.resourceReqsPerExecutor.toList.map(writeResourceRequirement)) ~
("user" -> obj.user) ~
- ("command" -> obj.command.toString)
+ ("command" -> redactedCommand.toString)
}
/**
@@ -154,7 +162,7 @@ private[deploy] object JsonProtocol {
("memory" -> obj.memory) ~
("resources" -> writeResourcesInfo(obj.resources)) ~
("appid" -> obj.appId) ~
- ("appdesc" -> writeApplicationDescription(obj.appDesc))
+ ("appdesc" -> writeApplicationDescription(obj.appDesc, obj.conf))
}
/**
diff --git
a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index bf5d889a87cb..3cb3863e9b56 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -55,7 +55,7 @@ private[deploy] class ExecutorRunner(
val sparkHome: File,
val executorDir: File,
val workerUrl: String,
- conf: SparkConf,
+ val conf: SparkConf,
val appLocalDirs: Seq[String],
@volatile var state: ExecutorState.Value,
val rpId: Int,
diff --git
a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
index 5e62323770e1..9aeed65ee8c5 100644
--- a/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/JsonProtocolSuite.scala
@@ -23,10 +23,11 @@ import com.fasterxml.jackson.core.JsonParseException
import org.json4s._
import org.json4s.jackson.JsonMethods
-import org.apache.spark.{JsonTestUtils, SparkFunSuite}
+import org.apache.spark.{JsonTestUtils, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse,
WorkerStateResponse}
import org.apache.spark.deploy.master.{ApplicationInfo, RecoveryState}
import org.apache.spark.deploy.worker.ExecutorRunner
+import org.apache.spark.util.Utils
class JsonProtocolSuite extends SparkFunSuite with JsonTestUtils {
@@ -45,7 +46,7 @@ class JsonProtocolSuite extends SparkFunSuite with
JsonTestUtils {
}
test("writeApplicationDescription") {
- val output = JsonProtocol.writeApplicationDescription(createAppDesc())
+ val output = JsonProtocol.writeApplicationDescription(createAppDesc(), new
SparkConf())
assertValidJson(output)
assertValidDataInJson(output,
JsonMethods.parse(JsonConstants.appDescJsonStr))
}
@@ -89,6 +90,38 @@ class JsonProtocolSuite extends SparkFunSuite with
JsonTestUtils {
assertValidDataInJson(output,
JsonMethods.parse(JsonConstants.workerStateJsonStr))
}
+ test("SPARK-57098: secrets in executor command are redacted in worker JSON
endpoint") {
+ val conf = new SparkConf()
+ val secretEnv = Map(
+ "HADOOP_CREDSTORE_PASSWORD" -> "topsecret",
+ "JAVA_HOME" -> "/usr/lib/jvm/default",
+ "AWS_SECRET_ACCESS_KEY" -> "aws-secret-value")
+ val secretJavaOpts = Seq(
+ "-Dspark.ssl.keyStorePassword=ssl-secret",
+ "-Dspark.executorEnv.PASSWORD=env-secret",
+ "-Xmx2g")
+ val cmd = new Command(
+ "mainClass", List("arg1"), secretEnv, Seq(), Seq(), secretJavaOpts)
+ val appDesc = new ApplicationDescription(
+ "name", Some(4), cmd, "appUiUrl", defaultResourceProfile)
+
+ val output = JsonProtocol.writeApplicationDescription(appDesc, conf)
+ val commandStr = (output \ "command") match {
+ case JString(s) => s
+ case other => fail(s"Expected JString for 'command', got: $other")
+ }
+
+ // Sensitive values are scrubbed.
+ assert(!commandStr.contains("topsecret"))
+ assert(!commandStr.contains("ssl-secret"))
+ assert(!commandStr.contains("env-secret"))
+ assert(!commandStr.contains("aws-secret-value"))
+ assert(commandStr.contains(Utils.REDACTION_REPLACEMENT_TEXT))
+ // Non-sensitive values pass through.
+ assert(commandStr.contains("/usr/lib/jvm/default"))
+ assert(commandStr.contains("-Xmx2g"))
+ }
+
def assertValidJson(json: JValue): Unit = {
try {
JsonMethods.parse(JsonMethods.compact(json))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
