This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 4497f07 [SPARK-53993] Support `spark.logConf` configuration
4497f07 is described below
commit 4497f07621bebd202bfe313b0781d24f054e0535
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Oct 22 15:12:01 2025 -0700
[SPARK-53993] Support `spark.logConf` configuration
### What changes were proposed in this pull request?
This PR aims to support `spark.logConf` configuration.
### Why are the changes needed?
Last Apache Spark, we had better support `spark.logConf` to help the users.
### Does this PR introduce _any_ user-facing change?
No behavior change because this is a new configuration which is disabled by
default.
### How was this patch tested?
Manually.
**BEFORE**
```
$ kubectl logs spark-kubernetes-operator-cdb4b547d-5vhgs
Starting Operator...
25/10/22 22:04:05 INFO o.a.s.k.o.SparkOperator Configuring operator with
50 reconciliation threads.
25/10/22 22:04:05 INFO o.a.s.k.o.SparkOperator Adding Operator
JosdkMetrics to metrics system.
25/10/22 22:04:05 INFO o.a.s.k.o.SparkOperator Configuring operator with
50 reconciliation threads.
25/10/22 22:04:05 INFO o.a.s.k.o.SparkOperator Adding Operator
JosdkMetrics to metrics system.
25/10/22 22:04:05 INFO i.j.o.a.c.BaseConfigurationService Created
configuration for reconciler
org.apache.spark.k8s.operator.reconciler.SparkAppReconciler with name
sparkappreconciler
25/10/22 22:04:05 INFO o.a.s.k.o.SparkOperator Initializing with watched
namespaces [default]
25/10/22 22:04:05 INFO o.a.s.k.o.c.SparkOperatorConf Reconciler retry
policy is configured with unlimited max attempts
25/10/22 22:04:05 INFO i.j.o.Operator Registered reconciler:
'sparkappreconciler' for resource: 'class
org.apache.spark.k8s.operator.SparkApplication' for namespace(s): [default]
25/10/22 22:04:05 INFO i.j.o.a.c.BaseConfigurationService Created
configuration for reconciler
org.apache.spark.k8s.operator.reconciler.SparkClusterReconciler with name
sparkclusterreconciler
25/10/22 22:04:05 INFO o.a.s.k.o.SparkOperator Initializing with watched
namespaces [default]
25/10/22 22:04:05 INFO o.a.s.k.o.c.SparkOperatorConf Reconciler retry
policy is configured with unlimited max attempts
25/10/22 22:04:05 INFO i.j.o.Operator Registered reconciler:
'sparkclusterreconciler' for resource: 'class
org.apache.spark.k8s.operator.SparkCluster' for namespace(s): [default]
25/10/22 22:04:05 INFO i.j.o.Operator Operator SDK 5.1.4 (commit:
87ec587) built on 2025-10-13T15:37:47.000+0000 starting...
...
```
**AFTER**
```
$ git diff
diff --git a/build-tools/helm/spark-kubernetes-operator/values.yaml
b/build-tools/helm/spark-kubernetes-operator/values.yaml
index af6504b..1b693a1 100644
--- a/build-tools/helm/spark-kubernetes-operator/values.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/values.yaml
-192,6 +192,7 operatorConfiguration:
spark-operator.properties: |+
# Property Overrides. e.g.
# spark.kubernetes.operator.reconciler.intervalSeconds=60
+ spark.logConf=true
metrics.properties: |+
# Metrics Properties Overrides
dynamicConfig:
```
```
$ helm install spark -f
build-tools/helm/spark-kubernetes-operator/values.yaml
build-tools/helm/spark-kubernetes-operator/
NAME: spark
LAST DEPLOYED: Wed Oct 22 15:03:14 2025
NAMESPACE: default
STATUS: deployed
REVISION: 1
```
```
$ kubectl logs spark-kubernetes-operator-cdb4b547d-pcbfj
Starting Operator...
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator Configuring operator with
50 reconciliation threads.
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator Adding Operator
JosdkMetrics to metrics system.
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator Configuring operator with
50 reconciliation threads.
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator Adding Operator
JosdkMetrics to metrics system.
25/10/22 22:03:16 INFO i.j.o.a.c.BaseConfigurationService Created
configuration for reconciler
org.apache.spark.k8s.operator.reconciler.SparkAppReconciler with name
sparkappreconciler
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator Initializing with watched
namespaces [default]
25/10/22 22:03:16 INFO o.a.s.k.o.c.SparkOperatorConf Reconciler retry
policy is configured with unlimited max attempts
25/10/22 22:03:16 INFO i.j.o.Operator Registered reconciler:
'sparkappreconciler' for resource: 'class
org.apache.spark.k8s.operator.SparkApplication' for namespace(s): [default]
25/10/22 22:03:16 INFO i.j.o.a.c.BaseConfigurationService Created
configuration for reconciler
org.apache.spark.k8s.operator.reconciler.SparkClusterReconciler with name
sparkclusterreconciler
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator Initializing with watched
namespaces [default]
25/10/22 22:03:16 INFO o.a.s.k.o.c.SparkOperatorConf Reconciler retry
policy is configured with unlimited max attempts
25/10/22 22:03:16 INFO i.j.o.Operator Registered reconciler:
'sparkclusterreconciler' for resource: 'class
org.apache.spark.k8s.operator.SparkCluster' for namespace(s): [default]
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.specification.version
= 25
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
spark.kubernetes.operator.watchedNamespaces = default
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.jnu.encoding = UTF-8
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.arch.data.model = 64
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vendor.url =
http://www.azul.com/
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.boot.library.path =
/usr/lib/jvm/zulu25-ca-arm64/lib
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator spark.logConf = true
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.java.command =
org.apache.spark.k8s.operator.SparkOperator
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator jdk.debug = release
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.specification.vendor
= Oracle Corporation
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.version.date =
2025-10-21
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.home =
/usr/lib/jvm/zulu25-ca-arm64
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator file.separator = /
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vm.compressedOopsMode
= 32-bit
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator line.separator =
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.specification.name =
Java Platform API Specification
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
java.vm.specification.vendor = Oracle Corporation
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.management.compiler =
HotSpot 64-Bit Tiered Compilers
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator io.netty.noUnsafe = true
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.runtime.version =
25.0.1+8-LTS
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator user.name = spark
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
spark.kubernetes.operator.dynamicConfig.enabled = false
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator file.encoding = UTF8
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vendor.version =
Zulu25.30+17-CA
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.io.tmpdir = /tmp
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.version = 25.0.1
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vm.specification.name
= Java Virtual Machine Specification
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator native.encoding = UTF-8
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.library.path =
/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vendor = Azul
Systems, Inc.
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator stderr.encoding = UTF-8
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.io.unicode.encoding =
UnicodeLittle
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator log4j.configurationFile =
/opt/spark-operator/conf/log4j2.properties
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.class.path =
./spark-kubernetes-operator.jar
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vm.vendor = Azul
Systems, Inc.
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator user.timezone = UTC
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator os.name = Linux
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
java.vm.specification.version = 25
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.java.launcher =
SUN_STANDARD
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator user.country = US
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
spark.kubernetes.operator.namespace = default
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
spark.kubernetes.operator.name = spark-kubernetes-operator
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator sun.cpu.endian = little
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator user.home =
/opt/spark-operator
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator user.language = en
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
spark.kubernetes.operator.health.probePort = 19091
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator stdout.encoding = UTF-8
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator path.separator = :
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator os.version =
6.10.14-linuxkit
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.runtime.name =
OpenJDK Runtime Environment
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vm.name = OpenJDK
64-Bit Server VM
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vendor.url.bug =
http://www.azul.com/support/
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator user.dir =
/opt/spark-operator/operator
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator os.arch = aarch64
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vm.info = mixed mode,
sharing
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.vm.version =
25.0.1+8-LTS
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator stdin.encoding = UTF-8
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator
spark.kubernetes.operator.metrics.port = 19090
25/10/22 22:03:16 INFO o.a.s.k.o.SparkOperator java.class.version = 69.0
25/10/22 22:03:16 INFO i.j.o.Operator Operator SDK 5.1.4 (commit:
87ec587) built on 2025-10-13T15:37:47.000+0000 starting...
...
```
### Was this patch authored or co-authored using generative AI tooling?
Closes #403 from dongjoon-hyun/SPARK-53993.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/main/java/org/apache/spark/k8s/operator/SparkOperator.java | 6 ++++++
.../org/apache/spark/k8s/operator/config/SparkOperatorConf.java | 7 +++++++
2 files changed, 13 insertions(+)
diff --git
a/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java
b/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java
index e4b6a5b..8804f57 100644
---
a/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java
+++
b/spark-operator/src/main/java/org/apache/spark/k8s/operator/SparkOperator.java
@@ -42,6 +42,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.spark.k8s.operator.client.KubernetesClientFactory;
import org.apache.spark.k8s.operator.config.SparkOperatorConf;
+import org.apache.spark.k8s.operator.config.SparkOperatorConfManager;
import org.apache.spark.k8s.operator.config.SparkOperatorConfigMapReconciler;
import org.apache.spark.k8s.operator.metrics.MetricsService;
import org.apache.spark.k8s.operator.metrics.MetricsSystem;
@@ -96,6 +97,11 @@ public class SparkOperator {
this.sparkClusterSentinelManager = new SentinelManager<>();
this.registeredOperators = new ArrayList<>();
this.registeredOperators.add(registerSparkOperator());
+ if (SparkOperatorConf.LOG_CONF.getValue()) {
+ for (var entry : SparkOperatorConfManager.INSTANCE.getAll().entrySet()) {
+ log.info("{} = {}", entry.getKey(), entry.getValue());
+ }
+ }
if (SparkOperatorConf.DYNAMIC_CONFIG_ENABLED.getValue()) {
this.registeredOperators.add(registerSparkOperatorConfMonitor());
}
diff --git
a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java
b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java
index 18cd4cd..6870918 100644
---
a/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java
+++
b/spark-operator/src/main/java/org/apache/spark/k8s/operator/config/SparkOperatorConf.java
@@ -32,6 +32,13 @@ import org.apache.spark.k8s.operator.utils.Utils;
/** Spark Operator Configuration options. */
@Slf4j
public final class SparkOperatorConf {
+ public static final ConfigOption<Boolean> LOG_CONF =
+ ConfigOption.<Boolean>builder()
+ .key("spark.logConf")
+ .description("When enabled, operator will print configurations")
+ .typeParameterClass(Boolean.class)
+ .defaultValue(false)
+ .build();
/** Name of the operator. */
public static final ConfigOption<String> OPERATOR_APP_NAME =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
