This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 7915164 [SPARK-52997] Fixes wrong worker assignment if multiple
clusters are deployed to the same namespace
7915164 is described below
commit 79151646db48ddb13aaaf57579efc4d66337367b
Author: Schmöller Maximilian <[email protected]>
AuthorDate: Fri Sep 19 14:55:56 2025 -0700
[SPARK-52997] Fixes wrong worker assignment if multiple clusters are
deployed to the same namespace
### What changes were proposed in this pull request?
Updated the podSelector for master and worker services to include both
clusterRole and name labels.
### Why are the changes needed?
Using only clusterRole caused service misrouting when multiple Spark
clusters were deployed in the same namespace. Adding name ensures correct pod
targeting.
### Does this PR introduce any user-facing change?
No, this is an internal fix to service selectors.
### How was this patch tested?
Tested with multiple clusters in the same namespace. Verified each service
only matched its own pods via kubectl describe service.
Also adapted unit tests to reflect new behaviour
### Was this patch authored or co-authored using generative AI tooling?
Yes, PR metadata was assisted by AI, but code changes were made manually.
Closes #291 from schmaxXximilian/main.
Authored-by: Schmöller Maximilian <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../k8s/operator/SparkClusterResourceSpec.java | 8 +++--
.../k8s/operator/SparkClusterResourceSpecTest.java | 39 +++++++++++++++++++++-
2 files changed, 44 insertions(+), 3 deletions(-)
diff --git
a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java
b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java
index 1c98273..efa5f45 100644
---
a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java
+++
b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkClusterResourceSpec.java
@@ -133,7 +133,8 @@ public class SparkClusterResourceSpec {
.endMetadata()
.withNewSpecLike(serviceSpec)
.withClusterIP("None")
- .withSelector(
+ .addToSelector(Collections.singletonMap(LABEL_SPARK_CLUSTER_NAME,
name))
+ .addToSelector(
Collections.singletonMap(LABEL_SPARK_ROLE_NAME,
LABEL_SPARK_ROLE_MASTER_VALUE))
.addNewPort()
.withName("web")
@@ -175,7 +176,8 @@ public class SparkClusterResourceSpec {
.endMetadata()
.withNewSpecLike(serviceSpec)
.withClusterIP("None")
- .withSelector(
+ .addToSelector(Collections.singletonMap(LABEL_SPARK_CLUSTER_NAME,
name))
+ .addToSelector(
Collections.singletonMap(LABEL_SPARK_ROLE_NAME,
LABEL_SPARK_ROLE_WORKER_VALUE))
.addNewPort()
.withName("web")
@@ -225,6 +227,7 @@ public class SparkClusterResourceSpec {
.editOrNewTemplate()
.editOrNewMetadata()
.addToLabels(LABEL_SPARK_ROLE_NAME, LABEL_SPARK_ROLE_MASTER_VALUE)
+ .addToLabels(LABEL_SPARK_CLUSTER_NAME, name)
.addToLabels(LABEL_SPARK_VERSION_NAME, version)
.endMetadata()
.editOrNewSpec()
@@ -306,6 +309,7 @@ public class SparkClusterResourceSpec {
.editOrNewTemplate()
.editOrNewMetadata()
.addToLabels(LABEL_SPARK_ROLE_NAME, LABEL_SPARK_ROLE_WORKER_VALUE)
+ .addToLabels(LABEL_SPARK_CLUSTER_NAME, name)
.addToLabels(LABEL_SPARK_VERSION_NAME, version)
.endMetadata()
.editOrNewSpec()
diff --git
a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
index ace3138..7335ff1 100644
---
a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
+++
b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
@@ -19,12 +19,13 @@
package org.apache.spark.k8s.operator;
-import static org.apache.spark.k8s.operator.Constants.LABEL_SPARK_VERSION_NAME;
+import static org.apache.spark.k8s.operator.Constants.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.Map;
import java.util.Optional;
import io.fabric8.kubernetes.api.model.ObjectMeta;
@@ -129,6 +130,13 @@ class SparkClusterResourceSpecTest {
assertEquals("bar", service1.getMetadata().getLabels().get("foo"));
assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals("foo", service1.getSpec().getExternalName());
+ assertEquals(
+ Map.of(
+ LABEL_SPARK_CLUSTER_NAME,
+ "cluster-name",
+ LABEL_SPARK_ROLE_NAME,
+ LABEL_SPARK_ROLE_WORKER_VALUE),
+ service1.getSpec().getSelector());
}
@Test
@@ -151,6 +159,13 @@ class SparkClusterResourceSpecTest {
assertEquals("bar", service1.getMetadata().getLabels().get("foo"));
assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals("foo", service1.getSpec().getExternalName());
+ assertEquals(
+ Map.of(
+ LABEL_SPARK_CLUSTER_NAME,
+ "cluster-name",
+ LABEL_SPARK_ROLE_NAME,
+ LABEL_SPARK_ROLE_MASTER_VALUE),
+ service1.getSpec().getSelector());
}
@Test
@@ -172,6 +187,17 @@ class SparkClusterResourceSpecTest {
SparkClusterResourceSpec spec2 = new SparkClusterResourceSpec(cluster,
sparkConf);
StatefulSet statefulSet2 = spec2.getMasterStatefulSet();
assertEquals("other-namespace", statefulSet2.getMetadata().getNamespace());
+ assertEquals(
+ "cluster-name",
+ statefulSet2
+ .getSpec()
+ .getTemplate()
+ .getMetadata()
+ .getLabels()
+ .get(LABEL_SPARK_CLUSTER_NAME));
+ assertEquals(
+ LABEL_SPARK_ROLE_MASTER_VALUE,
+
statefulSet2.getSpec().getTemplate().getMetadata().getLabels().get(LABEL_SPARK_ROLE_NAME));
}
@Test
@@ -236,6 +262,17 @@ class SparkClusterResourceSpecTest {
SparkClusterResourceSpec spec2 = new SparkClusterResourceSpec(cluster,
sparkConf);
StatefulSet statefulSet2 = spec2.getWorkerStatefulSet();
assertEquals("other-namespace", statefulSet2.getMetadata().getNamespace());
+ assertEquals(
+ "cluster-name",
+ statefulSet2
+ .getSpec()
+ .getTemplate()
+ .getMetadata()
+ .getLabels()
+ .get(LABEL_SPARK_CLUSTER_NAME));
+ assertEquals(
+ LABEL_SPARK_ROLE_WORKER_VALUE,
+
statefulSet2.getSpec().getTemplate().getMetadata().getLabels().get(LABEL_SPARK_ROLE_NAME));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
