This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new cc1f475 [SPARK-53703] Improve unit tests to be neutral from Spark
versions
cc1f475 is described below
commit cc1f475220fdca40b39a82877969590fd737c775
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Sep 24 16:18:57 2025 -0700
[SPARK-53703] Improve unit tests to be neutral from Spark versions
### What changes were proposed in this pull request?
This PR aims to improve unit tests to be neutral from Spark versions.
### Why are the changes needed?
It's important to become robust for (1) the future development and (2) the
downstream fork's version changes.
### Does this PR introduce _any_ user-facing change?
No. This is a test-only change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #339 from dongjoon-hyun/SPARK-53703.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark-job-with-driver-service-ingress.json | 13 ++-------
.../k8s/operator/SparkClusterResourceSpecTest.java | 31 +++++++++++-----------
2 files changed, 18 insertions(+), 26 deletions(-)
diff --git
a/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json
b/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json
index 1f44cbd..6222fb4 100644
---
a/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json
+++
b/spark-operator-api/src/test/resources/spark-job-with-driver-service-ingress.json
@@ -1,15 +1,6 @@
{
- "mainClass": "org.apache.spark.examples.SparkPi",
- "jars": "local:///opt/spark/examples/jars/spark-examples.jar",
- "sparkConf": {
- "spark.kubernetes.driver.master": "local[10]",
- "spark.kubernetes.driver.request.cores": "5",
- "spark.kubernetes.driver.limit.cores": "5",
- "spark.kubernetes.authenticate.driver.serviceAccountName": "spark",
- "spark.kubernetes.container.image": "apache/spark:4.0.0"
- },
"runtimeVersions": {
- "sparkVersion": "4.0.0"
+ "sparkVersion": "test"
},
"driverServiceIngressList": [
{
@@ -56,4 +47,4 @@
}
}
]
-}
\ No newline at end of file
+}
diff --git
a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
index 7335ff1..9717bf5 100644
---
a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
+++
b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkClusterResourceSpecTest.java
@@ -58,6 +58,7 @@ class SparkClusterResourceSpecTest {
RuntimeVersions runtimeVersions = new RuntimeVersions();
SparkConf sparkConf = new SparkConf().set("spark.kubernetes.namespace",
"other-namespace");
ClusterTolerations clusterTolerations = new ClusterTolerations();
+ static final String VERSION = "dev";
@BeforeEach
void setUp() {
@@ -76,7 +77,7 @@ class SparkClusterResourceSpecTest {
when(clusterSpec.getMasterSpec()).thenReturn(masterSpec);
when(clusterSpec.getWorkerSpec()).thenReturn(workerSpec);
when(clusterSpec.getRuntimeVersions()).thenReturn(runtimeVersions);
- runtimeVersions.setSparkVersion("4.0.0");
+ runtimeVersions.setSparkVersion(VERSION);
when(masterSpec.getStatefulSetSpec()).thenReturn(statefulSetSpec);
when(masterSpec.getStatefulSetMetadata()).thenReturn(objectMeta);
when(masterSpec.getServiceSpec()).thenReturn(serviceSpec);
@@ -92,11 +93,11 @@ class SparkClusterResourceSpecTest {
Service service1 = new SparkClusterResourceSpec(cluster, new
SparkConf()).getMasterService();
assertEquals("my-namespace", service1.getMetadata().getNamespace());
assertEquals("cluster-name-master-svc", service1.getMetadata().getName());
- assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
Service service2 = new SparkClusterResourceSpec(cluster,
sparkConf).getMasterService();
assertEquals("other-namespace", service2.getMetadata().getNamespace());
- assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
}
@Test
@@ -104,7 +105,7 @@ class SparkClusterResourceSpecTest {
Service service1 = new SparkClusterResourceSpec(cluster, new
SparkConf()).getWorkerService();
assertEquals("my-namespace", service1.getMetadata().getNamespace());
assertEquals("cluster-name-worker-svc", service1.getMetadata().getName());
- assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
Service service2 = new SparkClusterResourceSpec(cluster,
sparkConf).getMasterService();
assertEquals("other-namespace", service2.getMetadata().getNamespace());
@@ -128,7 +129,7 @@ class SparkClusterResourceSpecTest {
assertEquals("my-namespace", service1.getMetadata().getNamespace());
assertEquals("cluster-name-worker-svc", service1.getMetadata().getName());
assertEquals("bar", service1.getMetadata().getLabels().get("foo"));
- assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals("foo", service1.getSpec().getExternalName());
assertEquals(
Map.of(
@@ -157,7 +158,7 @@ class SparkClusterResourceSpecTest {
assertEquals("my-namespace", service1.getMetadata().getNamespace());
assertEquals("cluster-name-master-svc", service1.getMetadata().getName());
assertEquals("bar", service1.getMetadata().getLabels().get("foo"));
- assertEquals("4.0.0",
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
service1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals("foo", service1.getSpec().getExternalName());
assertEquals(
Map.of(
@@ -174,9 +175,9 @@ class SparkClusterResourceSpecTest {
StatefulSet statefulSet1 = spec1.getMasterStatefulSet();
assertEquals("my-namespace", statefulSet1.getMetadata().getNamespace());
assertEquals("cluster-name-master", statefulSet1.getMetadata().getName());
- assertEquals("4.0.0",
statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals(
- "4.0.0",
+ VERSION,
statefulSet1
.getSpec()
.getTemplate()
@@ -230,11 +231,11 @@ class SparkClusterResourceSpecTest {
assertEquals("my-namespace", statefulSet1.getMetadata().getNamespace());
assertEquals("cluster-name-master", statefulSet1.getMetadata().getName());
assertEquals("bar", statefulSet1.getMetadata().getLabels().get("foo"));
- assertEquals("4.0.0",
statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
statefulSet1.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals(1,
statefulSet1.getSpec().getTemplate().getSpec().getInitContainers().size());
assertEquals(2,
statefulSet1.getSpec().getTemplate().getSpec().getContainers().size());
assertEquals(
- "4.0.0",
+ VERSION,
statefulSet1
.getSpec()
.getTemplate()
@@ -249,9 +250,9 @@ class SparkClusterResourceSpecTest {
StatefulSet statefulSet = spec.getWorkerStatefulSet();
assertEquals("my-namespace", statefulSet.getMetadata().getNamespace());
assertEquals("cluster-name-worker", statefulSet.getMetadata().getName());
- assertEquals("4.0.0",
statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals(
- "4.0.0",
+ VERSION,
statefulSet
.getSpec()
.getTemplate()
@@ -304,9 +305,9 @@ class SparkClusterResourceSpecTest {
StatefulSet statefulSet = spec.getWorkerStatefulSet();
assertEquals("my-namespace", statefulSet.getMetadata().getNamespace());
assertEquals("cluster-name-worker", statefulSet.getMetadata().getName());
- assertEquals("4.0.0",
statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
statefulSet.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals(
- "4.0.0",
+ VERSION,
statefulSet
.getSpec()
.getTemplate()
@@ -338,7 +339,7 @@ class SparkClusterResourceSpecTest {
assertEquals("HorizontalPodAutoscaler", hpa.getKind());
assertEquals("my-namespace", hpa.getMetadata().getNamespace());
assertEquals("cluster-name-worker-hpa", hpa.getMetadata().getName());
- assertEquals("4.0.0",
hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
+ assertEquals(VERSION,
hpa.getMetadata().getLabels().get(LABEL_SPARK_VERSION_NAME));
assertEquals(1, hpa.getSpec().getMinReplicas());
assertEquals(3, hpa.getSpec().getMaxReplicas());
assertEquals(1, hpa.getSpec().getMetrics().size());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
