This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f3d5fe2b76958ce780a644be58c278fd0d6c8798 Author: Daniel Imberman <[email protected]> AuthorDate: Mon Oct 12 11:32:05 2020 -0700 Mount volumes and volumemounts into scheduler and workers (#11426) * Mount arbitrary volumes and volumeMounts to scheduler and worker Allows users to mount volumes to scheduler and workers * tested (cherry picked from commit 045d68da7fccccdf5b8209a4fd3bf138b9ca9f96) --- .../templates/scheduler/scheduler-deployment.yaml | 6 ++++ chart/templates/workers/worker-deployment.yaml | 6 ++++ chart/tests/scheduler_test.yaml | 38 ++++++++++++++++++++++ chart/tests/worker_test.yaml | 38 ++++++++++++++++++++++ chart/values.schema.json | 20 ++++++++++-- chart/values.yaml | 7 ++++ 6 files changed, 113 insertions(+), 2 deletions(-) diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index 794a473..b6878a7 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -152,6 +152,9 @@ spec: mountPath: {{ template "airflow_dags_mount_path" . }} {{- include "git_sync_container" . | indent 8 }} {{- end }} +{{- if .Values.scheduler.extraVolumeMounts }} +{{ toYaml .Values.scheduler.extraVolumeMounts | indent 12 }} +{{- end }} # Always start the garbage collector sidecar. - name: scheduler-gc image: {{ template "airflow_image" . }} @@ -195,6 +198,9 @@ spec: {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret }} {{- include "git_sync_ssh_key_volume" . | indent 8 }} {{- end }} +{{- if .Values.scheduler.extraVolumes }} +{{ toYaml .Values.scheduler.extraVolumes | indent 8 }} +{{- end }} {{- if not $stateful }} - name: logs emptyDir: {} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 38b3a9a..47aa18e 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -118,6 +118,9 @@ spec: - name: worker-logs containerPort: {{ .Values.ports.workerLogs }} volumeMounts: +{{- if .Values.workers.extraVolumeMounts }} +{{ toYaml .Values.workers.extraVolumeMounts | indent 12 }} +{{- end }} - name: logs mountPath: {{ template "airflow_logs" . }} - name: config @@ -200,6 +203,9 @@ spec: {{- include "standard_airflow_environment" . | indent 10 }} {{- end }} volumes: +{{- if .Values.workers.extraVolumes }} +{{ toYaml .Values.workers.extraVolumes | indent 8 }} +{{- end }} - name: kerberos-keytab secret: secretName: {{ include "kerberos_keytab_secret" . | quote }} diff --git a/chart/tests/scheduler_test.yaml b/chart/tests/scheduler_test.yaml new file mode 100644 index 0000000..327b6e3 --- /dev/null +++ b/chart/tests/scheduler_test.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +templates: + - scheduler/scheduler-deployment.yaml +tests: + - it: should add extraVolume and extraVolumeMount + set: + executor: CeleryExecutor + scheduler: + extraVolumes: + - name: test-volume + emptyDir: {} + extraVolumeMounts: + - name: test-volume + mountPath: /opt/test + asserts: + - equal: + path: spec.template.spec.volumes[1].name + value: test-volume + - equal: + path: spec.template.spec.containers[0].volumeMounts[3].name + value: test-volume diff --git a/chart/tests/worker_test.yaml b/chart/tests/worker_test.yaml new file mode 100644 index 0000000..29c4b97 --- /dev/null +++ b/chart/tests/worker_test.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +templates: + - workers/worker-deployment.yaml +tests: + - it: should add extraVolume and extraVolumeMount + set: + executor: CeleryExecutor + workers: + extraVolumes: + - name: test-volume + emptyDir: {} + extraVolumeMounts: + - name: test-volume + mountPath: /opt/test + asserts: + - equal: + path: spec.template.spec.volumes[0].name + value: test-volume + - equal: + path: spec.template.spec.containers[0].volumeMounts[0].name + value: test-volume diff --git a/chart/values.schema.json b/chart/values.schema.json index 2fc01a9..9370c80 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -472,7 +472,15 @@ "description": "Annotations to add to the worker kubernetes service account.", "type": "object" } - } + }, + "extraVolumes": { + "description": "Mount additional volumes into workers.", + "type": "array" + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into workers.", + "type": "array" + } }, "scheduler": { "description": "Airflow scheduler settings.", @@ -516,7 +524,15 @@ "description": "Annotations to add to the scheduler kubernetes service account.", "type": "object" } - } + }, + "extraVolumes": { + "description": "Mount additional volumes into scheduler.", + "type": "array" + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into scheduler.", + "type": "array" + } }, "webserver": { "description": "Airflow webserver settings.", diff --git a/chart/values.yaml b/chart/values.yaml index 707a34b..1370088 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -286,6 +286,9 @@ workers: safeToEvict: true # Annotations to add to worker kubernetes service account. serviceAccountAnnotations: {} + # Mount additional volumes into worker. + extraVolumes: [] + extraVolumeMounts: [] # Airflow scheduler settings scheduler: @@ -316,6 +319,10 @@ scheduler: # Annotations to add to scheduler kubernetes service account. serviceAccountAnnotations: {} + # Mount additional volumes into scheduler. + extraVolumes: [] + extraVolumeMounts: [] + # Airflow webserver settings webserver: livenessProbe:
