This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4c2cb60e6294475f252979832e3842a7f3500667 Author: Daniel Imberman <[email protected]> AuthorDate: Mon Oct 5 15:46:10 2020 -0700 Add CeleryKubernetesExecutor to helm chart (#11288) Users of the CeleryKubernetesExecutor will require both Celery and Kubernetes features to launch tasks. This PR will also serve as the basis for integration tests for this executor Co-authored-by: Daniel Imberman <[email protected]> (cherry picked from commit 48474600ccd7c586eb5dd8ea9aa2d818db7de505) --- chart/templates/NOTES.txt | 16 ++++++-- chart/templates/_helpers.yaml | 6 +-- chart/templates/configmap.yaml | 2 +- chart/templates/flower/flower-deployment.yaml | 2 +- .../flower-ingress.yaml} | 43 ++++++++++++---------- chart/templates/flower/flower-service.yaml | 2 +- chart/templates/rbac/pod-launcher-rolebinding.yaml | 4 +- chart/templates/redis/redis-networkpolicy.yaml | 2 +- chart/templates/redis/redis-service.yaml | 2 +- chart/templates/redis/redis-statefulset.yaml | 2 +- .../templates/scheduler/scheduler-deployment.yaml | 2 - chart/templates/workers/worker-deployment.yaml | 2 +- chart/templates/workers/worker-kedaautoscaler.yaml | 2 +- chart/templates/workers/worker-networkpolicy.yaml | 2 +- chart/templates/workers/worker-service.yaml | 2 +- 15 files changed, 51 insertions(+), 40 deletions(-) diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 3df1e3b..afca72a 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -20,10 +20,18 @@ under the License. Your release is named {{ .Release.Name }}. -You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser: +{{- if .Values.ingress.enabled }} +You can now access your dashboard(s) by following defined Ingress urls: -Airflow dashboard: kubectl port-forward svc/{{ .Release.Name }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} -{{- if eq .Values.executor "CeleryExecutor"}} -Flower dashboard: kubectl port-forward svc/{{ .Release.Name }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} +Airflow dashboard: http{{ if .Values.ingress.web.tls.enabled }}s{{ end }}://{{ .Values.ingress.web.host }}{{ .Values.ingress.web.path }}/ +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} +Flower dashboard: http{{ if .Values.ingress.flower.tls.enabled }}s{{ end }}://{{ .Values.ingress.flower.host }}{{ .Values.ingress.flower.path }}/ +{{- end }} +{{- else }} +You can now access your dashboard(s) by executing the following command(s) and visiting the corresponding port at localhost in your browser: +Airflow dashboard: kubectl port-forward svc/{{ .Release.Name }}-webserver {{ .Values.ports.airflowUI }}:{{ .Values.ports.airflowUI }} --namespace {{ .Release.Namespace }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")}} +Flower dashboard: kubectl port-forward svc/{{ .Release.Name }}-flower {{ .Values.ports.flowerUI }}:{{ .Values.ports.flowerUI }} --namespace {{ .Release.Namespace }} +{{- end }} {{- end }} diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 5d3ae73..02b255a 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -33,7 +33,7 @@ secretKeyRef: name: {{ template "airflow_metadata_secret" . }} key: connection - {{- if eq .Values.executor "CeleryExecutor" }} + {{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} - name: AIRFLOW__CELERY__CELERY_RESULT_BACKEND valueFrom: secretKeyRef: @@ -72,7 +72,7 @@ {{- range $i, $config := .Values.env }} - name: {{ $config.name }} value: {{ $config.value | quote }} - {{- if eq $.Values.executor "KubernetesExecutor" }} + {{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }} - name: AIRFLOW__KUBERNETES_ENVIRONMENT_VARIABLES__{{ $config.name }} value: {{ $config.value | quote }} {{- end }} @@ -85,7 +85,7 @@ name: {{ $config.secretName }} key: {{ default "value" $config.secretKey }} {{- end }} - {{- if eq .Values.executor "KubernetesExecutor" }} + {{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }} {{- range $i, $config := .Values.secret }} - name: AIRFLOW__KUBERNETES_SECRETS__{{ $config.envName }} value: {{ printf "%s=%s" $config.secretName $config.secretKey }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index b5bc656..2cf4502 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -55,7 +55,7 @@ data: known_hosts: | {{ .Values.dags.gitSync.knownHosts | nindent 4 }} {{- end }} -{{- if eq .Values.executor "KubernetesExecutor" }} +{{- if or (eq $.Values.executor "KubernetesExecutor") (eq $.Values.executor "CeleryKubernetesExecutor") }} pod_template_file.yaml: |- {{- if .Values.podTemplate }} {{ .Values.podTemplate | nindent 4 }} diff --git a/chart/templates/flower/flower-deployment.yaml b/chart/templates/flower/flower-deployment.yaml index 145c80b..887b70d 100644 --- a/chart/templates/flower/flower-deployment.yaml +++ b/chart/templates/flower/flower-deployment.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Flower Deployment ################################# -{{- if eq .Values.executor "CeleryExecutor" }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} kind: Deployment apiVersion: apps/v1 metadata: diff --git a/chart/templates/workers/worker-service.yaml b/chart/templates/flower/flower-ingress.yaml similarity index 54% copy from chart/templates/workers/worker-service.yaml copy to chart/templates/flower/flower-ingress.yaml index 2c7768f..228631b 100644 --- a/chart/templates/workers/worker-service.yaml +++ b/chart/templates/flower/flower-ingress.yaml @@ -16,31 +16,36 @@ # under the License. ################################ -## Airflow Worker Service +## Airflow Flower Ingress ################################# -{{- if eq .Values.executor "CeleryExecutor" }} -kind: Service -apiVersion: v1 +{{- if and .Values.ingress.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor")) }} +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress metadata: - name: {{ .Release.Name }}-worker + name: {{ .Release.Name }}-flower-ingress labels: tier: airflow - component: worker + component: flower-ingress release: {{ .Release.Name }} chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service }} -{{- with .Values.labels }} -{{ toYaml . | indent 4 }} -{{- end }} + annotations: + {{- range $key, $value := .Values.ingress.flower.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} spec: - clusterIP: None - selector: - tier: airflow - component: worker - release: {{ .Release.Name }} - ports: - - name: worker-logs - protocol: TCP - port: {{ .Values.ports.workerLogs }} - targetPort: {{ .Values.ports.workerLogs }} + {{- if .Values.ingress.flower.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.flower.host }} + secretName: {{ .Values.ingress.flower.tls.secretName }} + {{- end }} + rules: + - http: + paths: + - path: {{ .Values.ingress.flower.path }} + backend: + serviceName: {{ .Release.Name }}-flower + servicePort: flower-ui + host: {{ .Values.ingress.flower.host }} {{- end }} diff --git a/chart/templates/flower/flower-service.yaml b/chart/templates/flower/flower-service.yaml index 187046b8f..b723a12 100644 --- a/chart/templates/flower/flower-service.yaml +++ b/chart/templates/flower/flower-service.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Flower Service Component ################################# -{{- if eq .Values.executor "CeleryExecutor" }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} kind: Service apiVersion: v1 metadata: diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml index 6a1a5be..a3eab66 100644 --- a/chart/templates/rbac/pod-launcher-rolebinding.yaml +++ b/chart/templates/rbac/pod-launcher-rolebinding.yaml @@ -19,8 +19,8 @@ ## Airflow Pod Launcher Role Binding ################################# {{- if and .Values.rbacEnabled .Values.allowPodLaunching }} -{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") }} -{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") }} +{{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "CeleryKubernetesExecutor")}} +{{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} {{- if .Values.multiNamespaceMode }} kind: ClusterRoleBinding {{- else }} diff --git a/chart/templates/redis/redis-networkpolicy.yaml b/chart/templates/redis/redis-networkpolicy.yaml index b0a9e49..a7a7950 100644 --- a/chart/templates/redis/redis-networkpolicy.yaml +++ b/chart/templates/redis/redis-networkpolicy.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Redis NetworkPolicy ################################# -{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }} +{{- if (and .Values.networkPolicies.enabled (or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor"))) }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/chart/templates/redis/redis-service.yaml b/chart/templates/redis/redis-service.yaml index ac4bb21..b6e79db 100644 --- a/chart/templates/redis/redis-service.yaml +++ b/chart/templates/redis/redis-service.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Redis Service ################################# -{{- if eq .Values.executor "CeleryExecutor" }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} kind: Service apiVersion: v1 metadata: diff --git a/chart/templates/redis/redis-statefulset.yaml b/chart/templates/redis/redis-statefulset.yaml index e1eff0c..6cd7361 100644 --- a/chart/templates/redis/redis-statefulset.yaml +++ b/chart/templates/redis/redis-statefulset.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Redis StatefulSet ################################# -{{- if eq .Values.executor "CeleryExecutor" }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} kind: StatefulSet apiVersion: apps/v1 metadata: diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index f2b4a99..794a473 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -21,8 +21,6 @@ # Are we using a local/sequenial executor? {{- $local := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") }} -# Are we using the kubernetes executor? -{{- $kube := eq .Values.executor "KubernetesExecutor" }} # Is persistence enabled on the _workers_? # This is important because in $local mode, the scheduler assumes the role of the worker {{- $persistence := .Values.workers.persistence.enabled }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index fe07e20..38b3a9a 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -19,7 +19,7 @@ ## Airflow Worker Deployment ################################# {{- $persistence := .Values.workers.persistence.enabled }} -{{- if eq .Values.executor "CeleryExecutor" }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} kind: {{ if $persistence }}StatefulSet{{ else }}Deployment{{ end }} apiVersion: apps/v1 metadata: diff --git a/chart/templates/workers/worker-kedaautoscaler.yaml b/chart/templates/workers/worker-kedaautoscaler.yaml index 891afbf..c733618 100644 --- a/chart/templates/workers/worker-kedaautoscaler.yaml +++ b/chart/templates/workers/worker-kedaautoscaler.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Worker KEDA Scaler ################################# -{{- if (and .Values.workers.keda.enabled (eq .Values.executor "CeleryExecutor")) }} +{{- if (and .Values.workers.keda.enabled ( or (eq .Values.executor "CeleryExecutor")) (eq .Values.executor "CeleryKubernetesExecutor")) }} apiVersion: keda.k8s.io/v1alpha1 kind: ScaledObject metadata: diff --git a/chart/templates/workers/worker-networkpolicy.yaml b/chart/templates/workers/worker-networkpolicy.yaml index f6420e3..2c5e123 100644 --- a/chart/templates/workers/worker-networkpolicy.yaml +++ b/chart/templates/workers/worker-networkpolicy.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Worker NetworkPolicy ################################# -{{- if (and .Values.networkPolicies.enabled (eq .Values.executor "CeleryExecutor")) }} +{{- if (and .Values.networkPolicies.enabled ( or (eq .Values.executor "CeleryExecutor")) (eq .Values.executor "CeleryKubernetesExecutor")) }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/chart/templates/workers/worker-service.yaml b/chart/templates/workers/worker-service.yaml index 2c7768f..8923f53 100644 --- a/chart/templates/workers/worker-service.yaml +++ b/chart/templates/workers/worker-service.yaml @@ -18,7 +18,7 @@ ################################ ## Airflow Worker Service ################################# -{{- if eq .Values.executor "CeleryExecutor" }} +{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} kind: Service apiVersion: v1 metadata:
