This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1254c88 [SPARK-31118][K8S][DOC] Add version information to the
configuration of K8S
1254c88 is described below
commit 1254c8803454ec855bd66de5f763188f81e7ea66
Author: beliefer <[email protected]>
AuthorDate: Thu Mar 12 09:54:08 2020 +0900
[SPARK-31118][K8S][DOC] Add version information to the configuration of K8S
### What changes were proposed in this pull request?
Add version information to the configuration of `K8S`.
I sorted out some information show below.
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.kubernetes.context | 3.0.0 | SPARK-25887 |
c542c247bbfe1214c0bf81076451718a9e8931dc#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.master | 3.0.0 | SPARK-30371 |
f14061c6a4729ad419902193aa23575d8f17f597#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.namespace | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.container.image | 2.3.0 | SPARK-22994 |
b94debd2b01b87ef1d2a34d48877e38ade0969e6#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.container.image | 2.3.0 | SPARK-22807 |
fb3636b482be3d0940345b1528c1d5090bbc25e6#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.container.image | 2.3.0 | SPARK-22807 |
fb3636b482be3d0940345b1528c1d5090bbc25e6#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.container.image.pullPolicy | 2.3.0 | SPARK-22807 |
fb3636b482be3d0940345b1528c1d5090bbc25e6#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.container.image.pullSecrets | 2.4.0 | SPARK-23668 |
cccaaa14ad775fb981e501452ba2cc06ff5c0f0a#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.submission.requestTimeout | 3.0.0 | SPARK-27023 |
e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.submission.connectionTimeout | 3.0.0 | SPARK-27023 |
e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.requestTimeout | 3.0.0 | SPARK-27023 |
e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.connectionTimeout | 3.0.0 | SPARK-27023 |
e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8
|
KUBERNETES_AUTH_DRIVER_CONF_PREFIX.serviceAccountName | 2.3.0 | SPARK-18278
|
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
| spark.kubernetes.authenticate.driver
KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX.serviceAccountName | 3.1.0 |
SPARK-30122 |
f9f06eee9853ad4b6458ac9d31233e729a1ca226#diff-6e882d5561424e7e6651eb46f10104b8
| spark.kubernetes.authenticate.executor
spark.kubernetes.driver.limit.cores | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.request.cores | 3.0.0 | SPARK-27754 |
1a8c09334db87b0e938c38cd6b59d326bdcab3c3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.submitInDriver | 2.4.0 | SPARK-22839 |
f15906da153f139b698e192ec6f82f078f896f1e#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.limit.cores | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.scheduler.name | 3.0.0 | SPARK-29436 |
f800fa383131559c4e841bf062c9775d09190935#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.request.cores | 2.4.0 | SPARK-23285 |
fe2b7a4568d65a62da6e6eb00fff05f248b4332c#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.pod.name | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.resourceNamePrefix | 3.0.0 | SPARK-25876 |
6be272b75b4ae3149869e19df193675cc4117763#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.podNamePrefix | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.allocation.batch.size | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.allocation.batch.delay | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.lostCheck.maxAttempts | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.submission.waitAppCompletion | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.report.interval | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.apiPollingInterval | 2.4.0 | SPARK-24248 |
270a9a3cac25f3e799460320d0fc94ccd7ecfaea#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.eventProcessingInterval | 2.4.0 | SPARK-24248 |
270a9a3cac25f3e799460320d0fc94ccd7ecfaea#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.memoryOverheadFactor | 2.4.0 | SPARK-23984 |
1a644afbac35c204f9ad55f86999319a9ab458c6#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.pyspark.pythonVersion | 2.4.0 | SPARK-23984 |
a791c29bd824adadfb2d85594bc8dad4424df936#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.kerberos.krb5.path | 3.0.0 | SPARK-23257 |
6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.kerberos.krb5.configMapName | 3.0.0 | SPARK-23257 |
6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.hadoop.configMapName | 3.0.0 | SPARK-23257 |
6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.kerberos.tokenSecret.name | 3.0.0 | SPARK-23257 |
6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.kerberos.tokenSecret.itemKey | 3.0.0 | SPARK-23257 |
6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.resource.type | 2.4.1 | SPARK-25021 |
9031c784847353051bc0978f63ef4146ae9095ff#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.local.dirs.tmpfs | 3.0.0 | SPARK-25262 |
da6fa3828bb824b65f50122a8a0a0d4741551257#diff-6e882d5561424e7e6651eb46f10104b8
| It exists in branch-3.0, but in pom.xml it is 2.4.0-snapshot
spark.kubernetes.driver.podTemplateFile | 3.0.0 | SPARK-24434 |
f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.podTemplateFile | 3.0.0 | SPARK-24434 |
f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.podTemplateContainerName | 3.0.0 | SPARK-24434 |
f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.podTemplateContainerName | 3.0.0 | SPARK-24434 |
f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.deleteOnTermination | 3.0.0 | SPARK-25515 |
0c2935b01def8a5f631851999d9c2d57b63763e6#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.dynamicAllocation.deleteGracePeriod | 3.0.0 | SPARK-28487
|
0343854f54b48b206ca434accec99355011560c2#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.appKillPodDeletionGracePeriod | 3.0.0 | SPARK-24793 |
05168e725d2a17c4164ee5f9aa068801ec2454f4#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.file.upload.path | 3.0.0 | SPARK-23153 |
5e74570c8f5e7dfc1ca1c53c177827c5cea57bf1#diff-6e882d5561424e7e6651eb46f10104b8
|
The following appears in the document | | | |
spark.kubernetes.authenticate.submission.caCertFile | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.submission.clientKeyFile | 2.3.0 |
SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.submission.clientCertFile | 2.3.0 |
SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.submission.oauthToken | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.submission.oauthTokenFile | 2.3.0 |
SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.caCertFile | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.clientKeyFile | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.clientCertFile | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.oauthToken | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.oauthTokenFile | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.mounted.caCertFile | 2.3.0 |
SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.mounted.clientKeyFile | 2.3.0 |
SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.mounted.clientCertFile | 2.3.0 |
SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.driver.mounted.oauthTokenFile | 2.3.0 |
SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.caCertFile | 2.4.0 | SPARK-23146 |
571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.clientKeyFile | 2.4.0 | SPARK-23146 |
571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.clientCertFile | 2.4.0 | SPARK-23146 |
571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.oauthToken | 2.4.0 | SPARK-23146 |
571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.authenticate.oauthTokenFile | 2.4.0 | SPARK-23146 |
571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.label.[LabelName] | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.annotation.[AnnotationName] | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.label.[LabelName] | 2.3.0 | SPARK-22646 |
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.annotation.[AnnotationName] | 2.3.0 | SPARK-22646
|
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.node.selector.[labelKey] | 2.3.0 | SPARK-18278 |
e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driverEnv.[EnvironmentVariableName] | 2.3.0 | SPARK-22646
|
3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.secrets.[SecretName] | 2.3.0 | SPARK-22757 |
171f6ddadc6185ffcc6ad82e5f48952fb49095b2#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.secrets.[SecretName] | 2.3.0 | SPARK-22757 |
171f6ddadc6185ffcc6ad82e5f48952fb49095b2#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.secretKeyRef.[EnvName] | 2.4.0 | SPARK-24232 |
21e1fc7d4aed688d7b685be6ce93f76752159c98#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.secretKeyRef.[EnvName] | 2.4.0 | SPARK-24232 |
21e1fc7d4aed688d7b685be6ce93f76752159c98#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.path |
2.4.0 | SPARK-23529 |
5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.subPath |
3.0.0 | SPARK-25960 |
3df307aa515b3564686e75d1b71754bbcaaf2dec#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.readOnly |
2.4.0 | SPARK-23529 |
5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].options.[OptionName]
| 2.4.0 | SPARK-23529 |
5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-b5527f236b253e0d9f5db5164bdb43e9
|
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.path |
2.4.0 | SPARK-23529 |
5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.subPath |
3.0.0 | SPARK-25960 |
3df307aa515b3564686e75d1b71754bbcaaf2dec#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.readOnly
| 2.4.0 | SPARK-23529 |
5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8
|
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].options.[OptionName]
| 2.4.0 | SPARK-23529 |
5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-b5527f236b253e0d9f5db5164bdb43e9
|
### Why are the changes needed?
Supplemental configuration version information.
### Does this PR introduce any user-facing change?
'No'
### How was this patch tested?
Exists UT
Closes #27875 from beliefer/add-version-to-k8s-config.
Authored-by: beliefer <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>
---
docs/running-on-kubernetes.md | 77 +++++++++++++++++++++-
.../scala/org/apache/spark/deploy/k8s/Config.scala | 49 +++++++++++++-
2 files changed, 124 insertions(+), 2 deletions(-)
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index a941b8c..4f228a5 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -494,7 +494,7 @@ See the [configuration page](configuration.html) for
information on Spark config
#### Spark Properties
<table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since
Version</th></tr>
<tr>
<td><code>spark.kubernetes.context</code></td>
<td><code>(none)</code></td>
@@ -505,6 +505,7 @@ See the [configuration page](configuration.html) for
information on Spark config
auto-configured settings can be overridden by the use of other Spark
configuration properties e.g. <code>spark.kubernetes.namespace</code>.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.master</code></td>
@@ -512,6 +513,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
The internal Kubernetes master (API server) address to be used for driver
to request executors.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.namespace</code></td>
@@ -519,6 +521,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
The namespace that will be used for running the driver and executor pods.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.container.image</code></td>
@@ -529,6 +532,7 @@ See the [configuration page](configuration.html) for
information on Spark config
This configuration is required and must be provided by the user, unless
explicit
images are provided for each different container type.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.container.image</code></td>
@@ -536,6 +540,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Custom container image to use for the driver.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.container.image</code></td>
@@ -543,6 +548,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Custom container image to use for executors.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.container.image.pullPolicy</code></td>
@@ -550,6 +556,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Container image pull policy used when pulling images within Kubernetes.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.container.image.pullSecrets</code></td>
@@ -557,6 +564,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Comma separated list of Kubernetes secrets used to pull images from
private image registries.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.allocation.batch.size</code></td>
@@ -564,6 +572,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Number of pods to launch at once in each round of executor pod allocation.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.allocation.batch.delay</code></td>
@@ -572,6 +581,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Time to wait between each round of executor pod allocation. Specifying
values less than 1 second may lead to
excessive CPU usage on the spark driver.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.submission.caCertFile</code></td>
@@ -581,6 +591,7 @@ See the [configuration page](configuration.html) for
information on Spark config
must be located on the submitting machine's disk. Specify this as a path
as opposed to a URI (i.e. do not provide
a scheme). In client mode, use
<code>spark.kubernetes.authenticate.caCertFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.submission.clientKeyFile</code></td>
@@ -590,6 +601,7 @@ See the [configuration page](configuration.html) for
information on Spark config
must be located on the submitting machine's disk. Specify this as a path
as opposed to a URI (i.e. do not provide
a scheme). In client mode, use
<code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.submission.clientCertFile</code></td>
@@ -599,6 +611,7 @@ See the [configuration page](configuration.html) for
information on Spark config
file must be located on the submitting machine's disk. Specify this as a
path as opposed to a URI (i.e. do not
provide a scheme). In client mode, use
<code>spark.kubernetes.authenticate.clientCertFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.submission.oauthToken</code></td>
@@ -608,6 +621,7 @@ See the [configuration page](configuration.html) for
information on Spark config
that unlike the other authentication options, this is expected to be the
exact string value of the token to use for
the authentication. In client mode, use
<code>spark.kubernetes.authenticate.oauthToken</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.submission.oauthTokenFile</code></td>
@@ -617,6 +631,7 @@ See the [configuration page](configuration.html) for
information on Spark config
This file must be located on the submitting machine's disk. Specify this
as a path as opposed to a URI (i.e. do not
provide a scheme). In client mode, use
<code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.caCertFile</code></td>
@@ -627,6 +642,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
In client mode, use
<code>spark.kubernetes.authenticate.caCertFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.clientKeyFile</code></td>
@@ -637,6 +653,7 @@ See the [configuration page](configuration.html) for
information on Spark config
a Kubernetes secret. Specify this as a path as opposed to a URI (i.e. do
not provide a scheme).
In client mode, use
<code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.clientCertFile</code></td>
@@ -647,6 +664,7 @@ See the [configuration page](configuration.html) for
information on Spark config
driver pod as a Kubernetes secret. Specify this as a path as opposed to a
URI (i.e. do not provide a scheme).
In client mode, use
<code>spark.kubernetes.authenticate.clientCertFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.oauthToken</code></td>
@@ -657,6 +675,7 @@ See the [configuration page](configuration.html) for
information on Spark config
the token to use for the authentication. This token value is uploaded to
the driver pod as a Kubernetes secret.
In client mode, use <code>spark.kubernetes.authenticate.oauthToken</code>
instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.oauthTokenFile</code></td>
@@ -667,6 +686,7 @@ See the [configuration page](configuration.html) for
information on Spark config
the token to use for the authentication. This token value is uploaded to
the driver pod as a secret. In client mode, use
<code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.mounted.caCertFile</code></td>
@@ -677,6 +697,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
In client mode, use
<code>spark.kubernetes.authenticate.caCertFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.mounted.clientKeyFile</code></td>
@@ -687,6 +708,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
In client mode, use
<code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.mounted.clientCertFile</code></td>
@@ -697,6 +719,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
In client mode, use
<code>spark.kubernetes.authenticate.clientCertFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.mounted.oauthTokenFile</code></td>
@@ -707,6 +730,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Note that unlike the other authentication options, this file must contain
the exact string value of the token to use
for the authentication. In client mode, use
<code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.driver.serviceAccountName</code></td>
@@ -716,6 +740,7 @@ See the [configuration page](configuration.html) for
information on Spark config
executor pods from the API server. Note that this cannot be specified
alongside a CA cert file, client key file,
client cert file, and/or OAuth token. In client mode, use
<code>spark.kubernetes.authenticate.serviceAccountName</code> instead.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.caCertFile</code></td>
@@ -724,6 +749,7 @@ See the [configuration page](configuration.html) for
information on Spark config
In client mode, path to the CA cert file for connecting to the Kubernetes
API server over TLS when
requesting executors. Specify this as a path as opposed to a URI (i.e. do
not provide a scheme).
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.clientKeyFile</code></td>
@@ -732,6 +758,7 @@ See the [configuration page](configuration.html) for
information on Spark config
In client mode, path to the client key file for authenticating against the
Kubernetes API server
when requesting executors. Specify this as a path as opposed to a URI
(i.e. do not provide a scheme).
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.clientCertFile</code></td>
@@ -740,6 +767,7 @@ See the [configuration page](configuration.html) for
information on Spark config
In client mode, path to the client cert file for authenticating against
the Kubernetes API server
when requesting executors. Specify this as a path as opposed to a URI
(i.e. do not provide a scheme).
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.oauthToken</code></td>
@@ -749,6 +777,7 @@ See the [configuration page](configuration.html) for
information on Spark config
requesting executors. Note that unlike the other authentication options,
this must be the exact string value of
the token to use for the authentication.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.authenticate.oauthTokenFile</code></td>
@@ -757,6 +786,7 @@ See the [configuration page](configuration.html) for
information on Spark config
In client mode, path to the file containing the OAuth token to use when
authenticating against the Kubernetes API
server when requesting executors.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.label.[LabelName]</code></td>
@@ -767,6 +797,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Note that Spark also adds its own labels to the driver pod
for bookkeeping purposes.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.annotation.[AnnotationName]</code></td>
@@ -775,6 +806,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the annotation specified by <code>AnnotationName</code> to the driver
pod.
For example,
<code>spark.kubernetes.driver.annotation.something=true</code>.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.label.[LabelName]</code></td>
@@ -785,6 +817,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Note that Spark also adds its own labels to the executor pod
for bookkeeping purposes.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.annotation.[AnnotationName]</code></td>
@@ -793,6 +826,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the annotation specified by <code>AnnotationName</code> to the
executor pods.
For example,
<code>spark.kubernetes.executor.annotation.something=true</code>.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.pod.name</code></td>
@@ -804,6 +838,7 @@ See the [configuration page](configuration.html) for
information on Spark config
value in client mode allows the driver to become the owner of its executor
pods, which in turn allows the executor
pods to be garbage collected by the cluster.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.lostCheck.maxAttempts</code></td>
@@ -813,6 +848,7 @@ See the [configuration page](configuration.html) for
information on Spark config
The loss reason is used to ascertain whether the executor failure is due
to a framework or an application error
which in turn decides whether the executor is removed and replaced, or
placed into a failed state for debugging.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.submission.waitAppCompletion</code></td>
@@ -821,6 +857,7 @@ See the [configuration page](configuration.html) for
information on Spark config
In cluster mode, whether to wait for the application to finish before
exiting the launcher process. When changed to
false, the launcher has a "fire-and-forget" behavior when launching the
Spark job.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.report.interval</code></td>
@@ -828,6 +865,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Interval between reports of the current Spark job status in cluster mode.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.request.cores</code></td>
@@ -837,6 +875,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Example values include 0.1, 500m, 1.5, 5, etc., with the definition of cpu
units documented in <a
href="https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units";>CPU
units</a>.
This takes precedence over <code>spark.driver.cores</code> for specifying
the driver pod cpu request if set.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.limit.cores</code></td>
@@ -844,6 +883,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Specify a hard cpu <a
href="https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container";>limit</a>
for the driver pod.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.request.cores</code></td>
@@ -854,6 +894,7 @@ See the [configuration page](configuration.html) for
information on Spark config
This is distinct from <code>spark.executor.cores</code>: it is only used
and takes precedence over <code>spark.executor.cores</code> for specifying the
executor pod cpu request if set. Task
parallelism, e.g., number of tasks an executor can run concurrently is not
affected by this.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.limit.cores</code></td>
@@ -861,6 +902,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Specify a hard cpu <a
href="https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container";>limit</a>
for each executor pod launched for the Spark Application.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.node.selector.[labelKey]</code></td>
@@ -871,6 +913,7 @@ See the [configuration page](configuration.html) for
information on Spark config
will result in the driver pod and executors having a node selector with
key <code>identifier</code> and value
<code>myIdentifier</code>. Multiple node selector keys can be added by
setting multiple configurations with this prefix.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driverEnv.[EnvironmentVariableName]</code></td>
@@ -879,6 +922,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the environment variable specified by
<code>EnvironmentVariableName</code> to
the Driver process. The user can specify multiple of these to set multiple
environment variables.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.secrets.[SecretName]</code></td>
@@ -887,6 +931,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the <a
href="https://kubernetes.io/docs/concepts/configuration/secret/";>Kubernetes
Secret</a> named <code>SecretName</code> to the driver pod on the path
specified in the value. For example,
<code>spark.kubernetes.driver.secrets.spark-secret=/etc/secrets</code>.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.secrets.[SecretName]</code></td>
@@ -895,6 +940,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the <a
href="https://kubernetes.io/docs/concepts/configuration/secret/";>Kubernetes
Secret</a> named <code>SecretName</code> to the executor pod on the path
specified in the value. For example,
<code>spark.kubernetes.executor.secrets.spark-secret=/etc/secrets</code>.
</td>
+ <td>2.3.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.secretKeyRef.[EnvName]</code></td>
@@ -903,6 +949,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add as an environment variable to the driver container with name EnvName
(case sensitive), the value referenced by key <code> key </code> in the data of
the referenced <a
href="https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables";>Kubernetes
Secret</a>. For example,
<code>spark.kubernetes.driver.secretKeyRef.ENV_VAR=spark-secret:key</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.secretKeyRef.[EnvName]</code></td>
@@ -911,6 +958,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add as an environment variable to the executor container with name EnvName
(case sensitive), the value referenced by key <code> key </code> in the data of
the referenced <a
href="https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables";>Kubernetes
Secret</a>. For example,
<code>spark.kubernetes.executor.secrets.ENV_VAR=spark-secret:key</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.path</code></td>
@@ -919,6 +967,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the <a
href="https://kubernetes.io/docs/concepts/storage/volumes/";>Kubernetes
Volume</a> named <code>VolumeName</code> of the <code>VolumeType</code> type to
the driver pod on the path specified in the value. For example,
<code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/checkpoint</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.subPath</code></td>
@@ -927,6 +976,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specifies a <a
href="https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath";>subpath</a>
to be mounted from the volume into the driver pod.
<code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.subPath=checkpoint</code>.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.readOnly</code></td>
@@ -935,6 +985,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify if the mounted volume is read only or not. For example,
<code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.readOnly=false</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].options.[OptionName]</code></td>
@@ -943,6 +994,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Configure <a
href="https://kubernetes.io/docs/concepts/storage/volumes/";>Kubernetes
Volume</a> options passed to the Kubernetes with <code>OptionName</code> as key
having specified value, must conform with Kubernetes option format. For example,
<code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=spark-pvc-claim</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.path</code></td>
@@ -951,6 +1003,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Add the <a
href="https://kubernetes.io/docs/concepts/storage/volumes/";>Kubernetes
Volume</a> named <code>VolumeName</code> of the <code>VolumeType</code> type to
the executor pod on the path specified in the value. For example,
<code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/checkpoint</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.subPath</code></td>
@@ -959,6 +1012,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specifies a <a
href="https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath";>subpath</a>
to be mounted from the volume into the executor pod.
<code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.subPath=checkpoint</code>.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.readOnly</code></td>
@@ -967,6 +1021,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify if the mounted volume is read only or not. For example,
<code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.readOnly=false</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].options.[OptionName]</code></td>
@@ -975,6 +1030,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Configure <a
href="https://kubernetes.io/docs/concepts/storage/volumes/";>Kubernetes
Volume</a> options passed to the Kubernetes with <code>OptionName</code> as key
having specified value. For example,
<code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=spark-pvc-claim</code>.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.local.dirs.tmpfs</code></td>
@@ -983,6 +1039,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Configure the <code>emptyDir</code> volumes used to back
<code>SPARK_LOCAL_DIRS</code> within the Spark driver and executor pods to use
<code>tmpfs</code> backing i.e. RAM. See <a href="#local-storage">Local
Storage</a> earlier on this page
for more discussion of this.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.memoryOverheadFactor</code></td>
@@ -991,6 +1048,7 @@ See the [configuration page](configuration.html) for
information on Spark config
This sets the Memory Overhead Factor that will allocate memory to non-JVM
memory, which includes off-heap memory allocations, non-JVM tasks, and various
systems processes. For JVM-based jobs this value will default to 0.10 and 0.40
for non-JVM jobs.
This is done as non-JVM tasks need more non-JVM heap space and such tasks
commonly fail with "Memory Overhead Exceeded" errors. This prempts this error
with a higher default.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.pyspark.pythonVersion</code></td>
@@ -998,6 +1056,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
This sets the major Python version of the docker image used to run the
driver and executor containers. Can either be 2 or 3.
</td>
+ <td>2.4.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.kerberos.krb5.path</code></td>
@@ -1006,6 +1065,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the local location of the krb5.conf file to be mounted on the
driver and executors for Kerberos interaction.
It is important to note that the KDC defined needs to be visible from
inside the containers.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.kerberos.krb5.configMapName</code></td>
@@ -1015,6 +1075,7 @@ See the [configuration page](configuration.html) for
information on Spark config
for Kerberos interaction. The KDC defined needs to be visible from inside
the containers. The ConfigMap must also
be in the same namespace of the driver and executor pods.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.hadoop.configMapName</code></td>
@@ -1023,6 +1084,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the name of the ConfigMap, containing the HADOOP_CONF_DIR files,
to be mounted on the driver
and executors for custom Hadoop configuration.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.kerberos.tokenSecret.name</code></td>
@@ -1031,6 +1093,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the name of the secret where your existing delegation tokens are
stored. This removes the need for the job user
to provide any kerberos credentials for launching a job.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.kerberos.tokenSecret.itemKey</code></td>
@@ -1039,6 +1102,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the item key of the data where your existing delegation tokens are
stored. This removes the need for the job user
to provide any kerberos credentials for launching a job.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.podTemplateFile</code></td>
@@ -1047,6 +1111,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the local file that contains the driver <a href="#pod-template">pod
template</a>. For example
<code>spark.kubernetes.driver.podTemplateFile=/path/to/driver-pod-template.yaml</code>
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.podTemplateContainerName</code></td>
@@ -1055,6 +1120,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the container name to be used as a basis for the driver in the
given <a href="#pod-template">pod template</a>.
For example
<code>spark.kubernetes.driver.podTemplateContainerName=spark-driver</code>
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.podTemplateFile</code></td>
@@ -1063,6 +1129,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the local file that contains the executor <a
href="#pod-template">pod template</a>. For example
<code>spark.kubernetes.executor.podTemplateFile=/path/to/executor-pod-template.yaml</code>
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.podTemplateContainerName</code></td>
@@ -1071,6 +1138,7 @@ See the [configuration page](configuration.html) for
information on Spark config
Specify the container name to be used as a basis for the executor in the
given <a href="#pod-template">pod template</a>.
For example
<code>spark.kubernetes.executor.podTemplateContainerName=spark-executor</code>
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.executor.deleteOnTermination</code></td>
@@ -1078,6 +1146,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Specify whether executor pods should be deleted in case of failure or normal
termination.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.submission.connectionTimeout</code></td>
@@ -1085,6 +1154,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Connection timeout in milliseconds for the kubernetes client to use for
starting the driver.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.submission.requestTimeout</code></td>
@@ -1092,6 +1162,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Request timeout in milliseconds for the kubernetes client to use for
starting the driver.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.connectionTimeout</code></td>
@@ -1099,6 +1170,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Connection timeout in milliseconds for the kubernetes client in driver to
use when requesting executors.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.requestTimeout</code></td>
@@ -1106,6 +1178,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Request timeout in milliseconds for the kubernetes client in driver to use
when requesting executors.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.appKillPodDeletionGracePeriod</code></td>
@@ -1113,6 +1186,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<td>
Specify the grace period in seconds when deleting a Spark application using
spark-submit.
</td>
+ <td>3.0.0</td>
</tr>
<tr>
<td><code>spark.kubernetes.file.upload.path</code></td>
@@ -1122,6 +1196,7 @@ See the [configuration page](configuration.html) for
information on Spark config
<code>spark.kubernetes.file.upload.path=s3a://<s3-bucket>/path</code>
File should specified as <code>file://path/to/file </code> or absolute
path.
</td>
+ <td>3.0.0</td>
</tr>
</table>
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 46fd8e7..6eb2ca2 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -33,6 +33,7 @@ private[spark] object Config extends Logging {
"to .kube/config under your home directory. If not specified then
your current " +
"context is used. You can always override specific aspects of the
config file " +
"provided configuration using other Spark on K8S configuration
options.")
+ .version("3.0.0")
.stringConf
.createOptional
@@ -40,12 +41,14 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.driver.master")
.doc("The internal Kubernetes master (API server) address " +
"to be used for driver to request executors.")
+ .version("3.0.0")
.stringConf
.createWithDefault(KUBERNETES_MASTER_INTERNAL_URL)
val KUBERNETES_NAMESPACE =
ConfigBuilder("spark.kubernetes.namespace")
.doc("The namespace that will be used for running the driver and
executor pods.")
+ .version("2.3.0")
.stringConf
.createWithDefault("default")
@@ -54,22 +57,26 @@ private[spark] object Config extends Logging {
.doc("Container image to use for Spark containers. Individual container
types " +
"(e.g. driver or executor) can also be configured to use different
images if desired, " +
"by setting the container type-specific image name.")
+ .version("2.3.0")
.stringConf
.createOptional
val DRIVER_CONTAINER_IMAGE =
ConfigBuilder("spark.kubernetes.driver.container.image")
.doc("Container image to use for the driver.")
+ .version("2.3.0")
.fallbackConf(CONTAINER_IMAGE)
val EXECUTOR_CONTAINER_IMAGE =
ConfigBuilder("spark.kubernetes.executor.container.image")
.doc("Container image to use for the executors.")
+ .version("2.3.0")
.fallbackConf(CONTAINER_IMAGE)
val CONTAINER_IMAGE_PULL_POLICY =
ConfigBuilder("spark.kubernetes.container.image.pullPolicy")
.doc("Kubernetes image pull policy. Valid values are Always, Never, and
IfNotPresent.")
+ .version("2.3.0")
.stringConf
.checkValues(Set("Always", "Never", "IfNotPresent"))
.createWithDefault("IfNotPresent")
@@ -78,6 +85,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.container.image.pullSecrets")
.doc("Comma separated list of the Kubernetes secrets used " +
"to access private image registries.")
+ .version("2.4.0")
.stringConf
.toSequence
.createWithDefault(Nil)
@@ -95,24 +103,28 @@ private[spark] object Config extends Logging {
val SUBMISSION_CLIENT_REQUEST_TIMEOUT =
ConfigBuilder("spark.kubernetes.submission.requestTimeout")
.doc("request timeout to be used in milliseconds for starting the
driver")
+ .version("3.0.0")
.intConf
.createWithDefault(10000)
val SUBMISSION_CLIENT_CONNECTION_TIMEOUT =
ConfigBuilder("spark.kubernetes.submission.connectionTimeout")
.doc("connection timeout to be used in milliseconds for starting the
driver")
+ .version("3.0.0")
.intConf
.createWithDefault(10000)
val DRIVER_CLIENT_REQUEST_TIMEOUT =
ConfigBuilder("spark.kubernetes.driver.requestTimeout")
.doc("request timeout to be used in milliseconds for driver to request
executors")
+ .version("3.0.0")
.intConf
.createWithDefault(10000)
val DRIVER_CLIENT_CONNECTION_TIMEOUT =
ConfigBuilder("spark.kubernetes.driver.connectionTimeout")
.doc("connection timeout to be used in milliseconds for driver to
request executors")
+ .version("3.0.0")
.intConf
.createWithDefault(10000)
@@ -122,6 +134,7 @@ private[spark] object Config extends Logging {
"this service account when requesting executor pods from the API
server. If specific " +
"credentials are given for the driver pod to use, the driver will
favor " +
"using those credentials instead.")
+ .version("2.3.0")
.stringConf
.createOptional
@@ -129,48 +142,56 @@ private[spark] object Config extends Logging {
ConfigBuilder(s"$KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX.serviceAccountName")
.doc("Service account that is used when running the executor pod." +
"If this parameter is not setup, the fallback logic will use the
driver's service account.")
+ .version("3.1.0")
.stringConf
.createOptional
val KUBERNETES_DRIVER_LIMIT_CORES =
ConfigBuilder("spark.kubernetes.driver.limit.cores")
.doc("Specify the hard cpu limit for the driver pod")
+ .version("2.3.0")
.stringConf
.createOptional
val KUBERNETES_DRIVER_REQUEST_CORES =
ConfigBuilder("spark.kubernetes.driver.request.cores")
.doc("Specify the cpu request for the driver pod")
+ .version("3.0.0")
.stringConf
.createOptional
val KUBERNETES_DRIVER_SUBMIT_CHECK =
ConfigBuilder("spark.kubernetes.submitInDriver")
.internal()
+ .version("2.4.0")
.booleanConf
.createWithDefault(false)
val KUBERNETES_EXECUTOR_LIMIT_CORES =
ConfigBuilder("spark.kubernetes.executor.limit.cores")
.doc("Specify the hard cpu limit for each executor pod")
+ .version("2.3.0")
.stringConf
.createOptional
val KUBERNETES_EXECUTOR_SCHEDULER_NAME =
ConfigBuilder("spark.kubernetes.executor.scheduler.name")
.doc("Specify the scheduler name for each executor pod")
+ .version("3.0.0")
.stringConf
.createOptional
val KUBERNETES_EXECUTOR_REQUEST_CORES =
ConfigBuilder("spark.kubernetes.executor.request.cores")
.doc("Specify the cpu request for each executor pod")
+ .version("2.4.0")
.stringConf
.createOptional
val KUBERNETES_DRIVER_POD_NAME =
ConfigBuilder("spark.kubernetes.driver.pod.name")
.doc("Name of the driver pod.")
+ .version("2.3.0")
.stringConf
.createOptional
@@ -178,12 +199,14 @@ private[spark] object Config extends Logging {
val KUBERNETES_DRIVER_POD_NAME_PREFIX =
ConfigBuilder("spark.kubernetes.driver.resourceNamePrefix")
.internal()
+ .version("3.0.0")
.stringConf
.createOptional
val KUBERNETES_EXECUTOR_POD_NAME_PREFIX =
ConfigBuilder("spark.kubernetes.executor.podNamePrefix")
.doc("Prefix to use in front of the executor pod names.")
+ .version("2.3.0")
.internal()
.stringConf
.createOptional
@@ -191,6 +214,7 @@ private[spark] object Config extends Logging {
val KUBERNETES_ALLOCATION_BATCH_SIZE =
ConfigBuilder("spark.kubernetes.allocation.batch.size")
.doc("Number of pods to launch at once in each round of executor
allocation.")
+ .version("2.3.0")
.intConf
.checkValue(value => value > 0, "Allocation batch size should be a
positive integer")
.createWithDefault(5)
@@ -198,6 +222,7 @@ private[spark] object Config extends Logging {
val KUBERNETES_ALLOCATION_BATCH_DELAY =
ConfigBuilder("spark.kubernetes.allocation.batch.delay")
.doc("Time to wait between each round of executor allocation.")
+ .version("2.3.0")
.timeConf(TimeUnit.MILLISECONDS)
.checkValue(value => value > 0, "Allocation batch delay must be a
positive time value.")
.createWithDefaultString("1s")
@@ -206,6 +231,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.executor.lostCheck.maxAttempts")
.doc("Maximum number of attempts allowed for checking the reason of an
executor loss " +
"before it is assumed that the executor failed.")
+ .version("2.3.0")
.intConf
.checkValue(value => value > 0, "Maximum attempts of checks of executor
lost reason " +
"must be a positive integer")
@@ -215,12 +241,14 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.submission.waitAppCompletion")
.doc("In cluster mode, whether to wait for the application to finish
before exiting the " +
"launcher process.")
+ .version("2.3.0")
.booleanConf
.createWithDefault(true)
val REPORT_INTERVAL =
ConfigBuilder("spark.kubernetes.report.interval")
.doc("Interval between reports of the current app status in cluster
mode.")
+ .version("2.3.0")
.timeConf(TimeUnit.MILLISECONDS)
.checkValue(interval => interval > 0, s"Logging interval must be a
positive time value.")
.createWithDefaultString("1s")
@@ -229,6 +257,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.executor.apiPollingInterval")
.doc("Interval between polls against the Kubernetes API server to
inspect the " +
"state of executors.")
+ .version("2.4.0")
.timeConf(TimeUnit.MILLISECONDS)
.checkValue(interval => interval > 0, s"API server polling interval must
be a" +
" positive time value.")
@@ -238,6 +267,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.executor.eventProcessingInterval")
.doc("Interval between successive inspection of executor events sent
from the" +
" Kubernetes API.")
+ .version("2.4.0")
.timeConf(TimeUnit.MILLISECONDS)
.checkValue(interval => interval > 0, s"Event processing interval must
be a positive" +
" time value.")
@@ -247,6 +277,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.memoryOverheadFactor")
.doc("This sets the Memory Overhead Factor that will allocate memory to
non-JVM jobs " +
"which in the case of JVM tasks will default to 0.10 and 0.40 for
non-JVM jobs")
+ .version("2.4.0")
.doubleConf
.checkValue(mem_overhead => mem_overhead >= 0 && mem_overhead < 1,
"Ensure that memory overhead is a double between 0 --> 1.0")
@@ -255,6 +286,7 @@ private[spark] object Config extends Logging {
val PYSPARK_MAJOR_PYTHON_VERSION =
ConfigBuilder("spark.kubernetes.pyspark.pythonVersion")
.doc("This sets the major Python version. Either 2 or 3. (Python2 or
Python3)")
+ .version("2.4.0")
.stringConf
.checkValue(pv => List("2", "3").contains(pv),
"Ensure that major Python version is either Python2 or Python3")
@@ -265,6 +297,7 @@ private[spark] object Config extends Logging {
.doc("Specify the local location of the krb5.conf file to be mounted on
the driver " +
"and executors for Kerberos. Note: The KDC defined needs to be " +
"visible from inside the containers ")
+ .version("3.0.0")
.stringConf
.createOptional
@@ -273,6 +306,7 @@ private[spark] object Config extends Logging {
.doc("Specify the name of the ConfigMap, containing the krb5.conf file,
to be mounted " +
"on the driver and executors for Kerberos. Note: The KDC defined" +
"needs to be visible from inside the containers ")
+ .version("3.0.0")
.stringConf
.createOptional
@@ -280,6 +314,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.hadoop.configMapName")
.doc("Specify the name of the ConfigMap, containing the HADOOP_CONF_DIR
files, " +
"to be mounted on the driver and executors for custom Hadoop
configuration.")
+ .version("3.0.0")
.stringConf
.createOptional
@@ -287,6 +322,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.kerberos.tokenSecret.name")
.doc("Specify the name of the secret where your existing delegation
tokens are stored. " +
"This removes the need for the job user to provide any keytab for
launching a job")
+ .version("3.0.0")
.stringConf
.createOptional
@@ -294,13 +330,15 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.kerberos.tokenSecret.itemKey")
.doc("Specify the item key of the data where your existing delegation
tokens are stored. " +
"This removes the need for the job user to provide any keytab for
launching a job")
+ .version("3.0.0")
.stringConf
.createOptional
val APP_RESOURCE_TYPE =
ConfigBuilder("spark.kubernetes.resource.type")
- .doc("This sets the resource type internally")
.internal()
+ .doc("This sets the resource type internally")
+ .version("2.4.1")
.stringConf
.checkValues(Set(APP_RESOURCE_TYPE_JAVA, APP_RESOURCE_TYPE_PYTHON,
APP_RESOURCE_TYPE_R))
.createOptional
@@ -311,30 +349,35 @@ private[spark] object Config extends Logging {
"their medium set to Memory so that they will be created as tmpfs
(i.e. RAM) backed " +
"volumes. This may improve performance but scratch space usage will
count towards " +
"your pods memory limit so you may wish to request more memory.")
+ .version("3.0.0")
.booleanConf
.createWithDefault(false)
val KUBERNETES_DRIVER_PODTEMPLATE_FILE =
ConfigBuilder("spark.kubernetes.driver.podTemplateFile")
.doc("File containing a template pod spec for the driver")
+ .version("3.0.0")
.stringConf
.createOptional
val KUBERNETES_EXECUTOR_PODTEMPLATE_FILE =
ConfigBuilder("spark.kubernetes.executor.podTemplateFile")
.doc("File containing a template pod spec for executors")
+ .version("3.0.0")
.stringConf
.createOptional
val KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME =
ConfigBuilder("spark.kubernetes.driver.podTemplateContainerName")
.doc("container name to be used as a basis for the driver in the given
pod template")
+ .version("3.0.0")
.stringConf
.createOptional
val KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME =
ConfigBuilder("spark.kubernetes.executor.podTemplateContainerName")
.doc("container name to be used as a basis for executors in the given
pod template")
+ .version("3.0.0")
.stringConf
.createOptional
@@ -347,12 +390,14 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.executor.deleteOnTermination")
.doc("If set to false then executor pods will not be deleted in case " +
"of failure or normal termination.")
+ .version("3.0.0")
.booleanConf
.createWithDefault(true)
val KUBERNETES_DYN_ALLOC_KILL_GRACE_PERIOD =
ConfigBuilder("spark.kubernetes.dynamicAllocation.deleteGracePeriod")
.doc("How long to wait for executors to shut down gracefully before a
forceful kill.")
+ .version("3.0.0")
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("5s")
@@ -360,6 +405,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.appKillPodDeletionGracePeriod")
.doc("Time to wait for graceful deletion of Spark pods when
spark-submit" +
" is used for killing an application.")
+ .version("3.0.0")
.timeConf(TimeUnit.SECONDS)
.createOptional
@@ -367,6 +413,7 @@ private[spark] object Config extends Logging {
ConfigBuilder("spark.kubernetes.file.upload.path")
.doc("Hadoop compatible file system path where files from the local file
system " +
"will be uploded to in cluster mode.")
+ .version("3.0.0")
.stringConf
.createOptional
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
