This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new e00f795 [SPARK-49214] Add e2e test in operator workflow
e00f795 is described below
commit e00f795904c248cc870dd8f03e4f9d681dc0068c
Author: Qi Tan <qi_...@apple.com>
AuthorDate: Wed Aug 21 07:58:47 2024 -0700
[SPARK-49214] Add e2e test in operator workflow
### What changes were proposed in this pull request?
Provide a test framework base for Spark Operator E2E
### Why are the changes needed?
make sure the operator behave as expected.
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
Test from personal GitHub workflow
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #77 from TQJADE/main.
Lead-authored-by: Qi Tan <qi_...@apple.com>
Co-authored-by: Dongjoon Hyun <dongj...@apache.org>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
.github/workflows/build_and_test.yml | 21 +++++
tests/e2e/assertions/spark-driver-config-map.yaml | 21 +++++
tests/e2e/assertions/spark-driver-pod.yaml | 30 +++++++
tests/e2e/assertions/spark-exec-pod.yaml | 24 ++++++
tests/e2e/python/chainsaw-test.yaml | 69 ++++++++++++++++
tests/e2e/python/pyspark-example.yaml | 32 ++++++++
tests/e2e/spark-versions/chainsaw-test.yaml | 98 +++++++++++++++++++++++
tests/e2e/spark-versions/spark-example.yaml | 34 ++++++++
8 files changed, 329 insertions(+)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index 17f4deb..edfca8a 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -63,6 +63,15 @@ jobs:
k8s-integration-tests:
name: "K8s Integration Tests"
runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ kubernetes_versions:
+ - "1.27.0"
+ - "1.31.0"
+ test_group:
+ - spark-versions
+ - python
steps:
- name: Checkout repository
uses: actions/checkout@v4
@@ -75,8 +84,15 @@ jobs:
- name: Set up Minikube
uses: medyagh/setup-minikube@v0.0.18
with:
+ cache: true
+ kubernetes-version: ${{ matrix.kube-version }}
cpu: 2
memory: 6144m
+ - name: Set Up Go
+ uses: actions/setup-go@v5
+ - name: Set Up Chainsaw
+ run: |
+ go install github.com/kyverno/chainsaw@latest
- name: Print K8S pods and nodes info
run: |
kubectl get pods -A
@@ -89,6 +105,11 @@ jobs:
./gradlew spark-operator-api:relocateGeneratedCRD
helm install spark-kubernetes-operator --create-namespace -f
build-tools/helm/spark-kubernetes-operator/values.yaml
build-tools/helm/spark-kubernetes-operator/
helm test spark-kubernetes-operator
+ # Use remote host' s docker image
+ minikube docker-env --unset
+ - name: Run E2E Test
+ run: |
+ chainsaw test --test-dir ./tests/e2e/${{ matrix.test_group }}
--parallel 2
lint:
name: "Linter and documentation"
runs-on: ubuntu-latest
diff --git a/tests/e2e/assertions/spark-driver-config-map.yaml
b/tests/e2e/assertions/spark-driver-config-map.yaml
new file mode 100644
index 0000000..d0e0bc0
--- /dev/null
+++ b/tests/e2e/assertions/spark-driver-config-map.yaml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: ($SPARK_DRIVER_CONFIG_MAP_NAME)
+ namespace: default
\ No newline at end of file
diff --git a/tests/e2e/assertions/spark-driver-pod.yaml
b/tests/e2e/assertions/spark-driver-pod.yaml
new file mode 100644
index 0000000..0ab9657
--- /dev/null
+++ b/tests/e2e/assertions/spark-driver-pod.yaml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ($SPARK_DRIVER_POD_NAME)
+ namespace: default
+ labels:
+ spark-app-name: spark
+ spark-role: driver
+spec:
+ (volumes[?configMap.name=='spark-conf-volume-driver'] != []): true
+status:
+ phase: Running
+
diff --git a/tests/e2e/assertions/spark-exec-pod.yaml
b/tests/e2e/assertions/spark-exec-pod.yaml
new file mode 100644
index 0000000..430b6f5
--- /dev/null
+++ b/tests/e2e/assertions/spark-exec-pod.yaml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: default
+ labels:
+ spark-role: executor
+ spark-version: ($SPARK_VERSION)
diff --git a/tests/e2e/python/chainsaw-test.yaml
b/tests/e2e/python/chainsaw-test.yaml
new file mode 100644
index 0000000..1edbc65
--- /dev/null
+++ b/tests/e2e/python/chainsaw-test.yaml
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+ name: spark-operator-python-validation
+spec:
+ scenarios:
+ - bindings:
+ - name: "SPARK_VERSION"
+ value: "3.5.2"
+ - name: "SCALA_VERSION"
+ value: "2.12"
+ - name: "IMAGE"
+ value: 'spark:3.5.2-scala2.12-java17-python3-r-ubuntu'
+ steps:
+ - name: install-spark-application
+ try:
+ - apply:
+ bindings:
+ - name: V_SPARK_VERSION
+ value: (concat('v', replace_all(($SPARK_VERSION), '.', '_')))
+ - name: SPARK_APPLICATION_NAME
+ value: (join('-', ['pyspark', replace_all(($SPARK_VERSION),
'.', '-')]))
+ file: pyspark-example.yaml
+ - name: verify-common-created-resources
+ bindings:
+ - name: SPARK_DRIVER_POD_NAME
+ value: (join('-', ['pyspark', replace_all(($SPARK_VERSION), '.',
'-'), '0-driver']))
+ - name: SPARK_EXECUTOR_POD_NAME
+ value: (join('-', ['pyspark', replace_all(($SPARK_VERSION), '.',
'-'), '0-executor']))
+ - name: SPARK_DRIVER_CONFIG_MAP_NAME
+ value: (join('-', ['pyspark', replace_all(($SPARK_VERSION), '.',
'-'), '0-spark-drv-conf-map']))
+ try:
+ - assert:
+ timeout: 2m
+ file: "../assertions/*.yaml"
+ catch:
+ - events:
+ namespace: default
+ - describe:
+ apiVersion: v1
+ kind: SparkApplication
+ namespace: default
+ - describe:
+ apiVersion: v1
+ kind: Pod
+ namespace: default
+ - podLogs:
+ selector: app.kubernetes.io/name=spark-kubernetes-operator
+ namespace: default
+ - podLogs:
+ selector: spark-app-name=spark
+ namespace: default
\ No newline at end of file
diff --git a/tests/e2e/python/pyspark-example.yaml
b/tests/e2e/python/pyspark-example.yaml
new file mode 100644
index 0000000..ec5ec3c
--- /dev/null
+++ b/tests/e2e/python/pyspark-example.yaml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: spark.apache.org/v1alpha1
+kind: SparkApplication
+metadata:
+ name: ($SPARK_APPLICATION_NAME)
+ namespace: default
+spec:
+ driverArgs: ["localhost", "9999"]
+ pyFiles:
"local:///opt/spark/examples/src/main/python/streaming/network_wordcount.py"
+ sparkConf:
+ spark.executor.instances: "1"
+ # see also https://hub.docker.com/_/spark
+ spark.kubernetes.container.image: ($IMAGE)
+ spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
+ runtimeVersions:
+ sparkVersion: ($V_SPARK_VERSION)
\ No newline at end of file
diff --git a/tests/e2e/spark-versions/chainsaw-test.yaml
b/tests/e2e/spark-versions/chainsaw-test.yaml
new file mode 100644
index 0000000..7d55a76
--- /dev/null
+++ b/tests/e2e/spark-versions/chainsaw-test.yaml
@@ -0,0 +1,98 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+ name: spark-operator-spark-version-validation
+spec:
+ scenarios:
+ - bindings:
+ - name: "SPARK_VERSION"
+ value: "4.0.0-preview1"
+ - name: "SCALA_VERSION"
+ value: "2.13"
+ - name: "JAVA_VERSION"
+ value: "17"
+ - name: "IMAGE"
+ value: 'spark:4.0.0-preview1-scala2.13-java17-ubuntu'
+ - bindings:
+ - name: "SPARK_VERSION"
+ value: "3.5.2"
+ - name: "SCALA_VERSION"
+ value: "2.12"
+ - name: "JAVA_VERSION"
+ value: "17"
+ - name: "IMAGE"
+ value: 'spark:3.5.2-scala2.12-java17-ubuntu'
+ - bindings:
+ - name: "SPARK_VERSION"
+ value: "3.4.3"
+ - name: "SCALA_VERSION"
+ value: "2.12"
+ - name: "JAVA_VERSION"
+ value: "11"
+ - name: "IMAGE"
+ value: 'spark:3.4.3-scala2.12-java11-ubuntu'
+ steps:
+ - name: install-spark-application
+ try:
+ - apply:
+ bindings:
+ - name: V_SPARK_VERSION
+ value: (concat('v', replace_all(($SPARK_VERSION), '.', '_')))
+ - name: V_SCALA_VERSION
+ value: (concat('v', replace_all(($SCALA_VERSION), '.', '_')))
+ - name: SPARK_APPLICATION_NAME
+ value: (join('-', ['spark-streaming',
replace_all(($SPARK_VERSION), '.', '-'), replace_all(($SCALA_VERSION), '.',
'-')]))
+ - name: V_JAVA_VERSION
+ value: (concat('Java', ($JAVA_VERSION)))
+ - name: JARS
+ value: (join('',
['local:///opt/spark/examples/jars/spark-examples_', join('-',
[($SCALA_VERSION), ($SPARK_VERSION)]), '.jar']))
+ file: spark-example.yaml
+ - name: verify-common-created-resources
+ bindings:
+ - name: SPARK_DRIVER_POD_NAME
+ value: (join('-', ['spark-streaming', replace_all(($SPARK_VERSION),
'.', '-'),
+ replace_all(($SCALA_VERSION), '.', '-'), '0-driver']))
+ - name: SPARK_EXECUTOR_POD_NAME
+ value: (join('-', ['spark-streaming', replace_all(($SPARK_VERSION),
'.', '-'),
+ replace_all(($SCALA_VERSION), '.', '-'), '0-executor']))
+ - name: SPARK_DRIVER_CONFIG_MAP_NAME
+ value: (join('-', ['spark-streaming', replace_all(($SPARK_VERSION),
'.', '-'),
+ replace_all(($SCALA_VERSION), '.', '-'), '0-spark-drv-conf-map']))
+ try:
+ - assert:
+ timeout: 2m
+ file: "../assertions/*.yaml"
+ catch:
+ - events:
+ namespace: default
+ - describe:
+ apiVersion: v1
+ kind: SparkApplication
+ namespace: default
+ - describe:
+ apiVersion: v1
+ kind: Pod
+ namespace: default
+ - podLogs:
+ selector: app.kubernetes.io/name=spark-kubernetes-operator
+ namespace: default
+ - podLogs:
+ selector: spark-app-name=spark
+ namespace: default
diff --git a/tests/e2e/spark-versions/spark-example.yaml
b/tests/e2e/spark-versions/spark-example.yaml
new file mode 100644
index 0000000..7d9f466
--- /dev/null
+++ b/tests/e2e/spark-versions/spark-example.yaml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: spark.apache.org/v1alpha1
+kind: SparkApplication
+metadata:
+ name: ($SPARK_APPLICATION_NAME)
+ namespace: default
+spec:
+ mainClass: "org.apache.spark.examples.streaming.CustomReceiver"
+ driverArgs: ["localhost", "9999"]
+ jars: ($JARS)
+ sparkConf:
+ spark.executor.instances: "1"
+ # see also https://hub.docker.com/_/spark
+ spark.kubernetes.container.image: ($IMAGE)
+ spark.kubernetes.authenticate.driver.serviceAccountName: "spark"
+ runtimeVersions:
+ sparkVersion: ($V_SPARK_VERSION)
+ javaVersion: ($V_JAVA_VERSION)
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
