This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 1d1a207 [SPARK-31787][K8S][TESTS][2.4] Fix
Minikube.getIfNewMinikubeStatus to understand 1.5+
1d1a207 is described below
commit 1d1a2079a56de25b0d14dd57f458a0f3830a6e14
Author: Marcelo Vanzin <[email protected]>
AuthorDate: Thu May 21 14:09:51 2020 -0700
[SPARK-31787][K8S][TESTS][2.4] Fix Minikube.getIfNewMinikubeStatus to
understand 1.5+
### What changes were proposed in this pull request?
This PR aims to fix the testing infra to support Minikube 1.5+ in K8s IT.
Also, note that this is a subset of #26488 with the same ownership.
### Why are the changes needed?
This helps us testing `master/3.0/2.4` in the same Minikube version.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Pass the Jenkins K8s IT with Minikube v0.34.1.
- Manually, test with Minikube 1.5.x.
```
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
Run completed in 4 minutes, 37 seconds.
Total number of tests run: 14
Suites: completed 2, aborted 0
Tests: succeeded 14, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```
Closes #28599 from dongjoon-hyun/SPARK-31787.
Authored-by: Marcelo Vanzin <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../integrationtest/backend/minikube/Minikube.scala | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
index 78ef44b..5968d3a 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
@@ -30,6 +30,7 @@ private[spark] object Minikube extends Logging {
private val KUBELET_PREFIX = "kubelet:"
private val APISERVER_PREFIX = "apiserver:"
private val KUBECTL_PREFIX = "kubectl:"
+ private val KUBECONFIG_PREFIX = "kubeconfig:"
private val MINIKUBE_VM_PREFIX = "minikubeVM: "
private val MINIKUBE_PREFIX = "minikube: "
private val MINIKUBE_PATH = ".minikube"
@@ -86,18 +87,23 @@ private[spark] object Minikube extends Logging {
val kubeletString = statusString.find(_.contains(s"$KUBELET_PREFIX "))
val apiserverString = statusString.find(_.contains(s"$APISERVER_PREFIX "))
val kubectlString = statusString.find(_.contains(s"$KUBECTL_PREFIX "))
+ val kubeconfigString = statusString.find(_.contains(s"$KUBECONFIG_PREFIX
"))
+ val hasConfigStatus = kubectlString.isDefined || kubeconfigString.isDefined
- if (hostString.isEmpty || kubeletString.isEmpty
- || apiserverString.isEmpty || kubectlString.isEmpty) {
+ if (hostString.isEmpty || kubeletString.isEmpty || apiserverString.isEmpty
||
+ !hasConfigStatus) {
MinikubeStatus.NONE
} else {
val status1 = hostString.get.replaceFirst(s"$HOST_PREFIX ", "")
val status2 = kubeletString.get.replaceFirst(s"$KUBELET_PREFIX ", "")
val status3 = apiserverString.get.replaceFirst(s"$APISERVER_PREFIX ", "")
- val status4 = kubectlString.get.replaceFirst(s"$KUBECTL_PREFIX ", "")
- if (!status4.contains("Correctly Configured:")) {
- MinikubeStatus.NONE
+ val isConfigured = if (kubectlString.isDefined) {
+ val cfgStatus = kubectlString.get.replaceFirst(s"$KUBECTL_PREFIX ", "")
+ cfgStatus.contains("Correctly Configured:")
} else {
+ kubeconfigString.get.replaceFirst(s"$KUBECONFIG_PREFIX ", "") ==
"Configured"
+ }
+ if (isConfigured) {
val stats = List(status1, status2, status3)
.map(MinikubeStatus.unapply)
.map(_.getOrElse(throw new IllegalStateException(s"Unknown status
$statusString")))
@@ -106,6 +112,8 @@ private[spark] object Minikube extends Logging {
} else {
MinikubeStatus.RUNNING
}
+ } else {
+ MinikubeStatus.NONE
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
