This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2458461314d97b7759db9a887e4fd1528bf6d94f Author: Daniel Imberman <[email protected]> AuthorDate: Mon Oct 12 13:48:59 2020 -0700 Allow multiple schedulers in helm chart (#11330) * Allow multiple schedulers in helm chart * schema * add docs * add to readme Co-authored-by: Daniel Imberman <[email protected]> (cherry picked from commit 7b3a00d4f97ada5e09bfc33c0e48b3b1833bfe56) --- chart/README.md | 1 + chart/templates/create-user-job.yaml | 8 ++++++++ chart/templates/scheduler/scheduler-deployment.yaml | 8 ++++---- chart/values.schema.json | 4 ++++ chart/values.yaml | 3 +++ 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/chart/README.md b/chart/README.md index 266ef58..41a53ad 100644 --- a/chart/README.md +++ b/chart/README.md @@ -187,6 +187,7 @@ The following tables lists the configurable parameters of the Airflow chart and | `workers.safeToEvict` | Allow Kubernetes to evict worker pods if needed (node downscaling) | `true` | | `scheduler.podDisruptionBudget.enabled` | Enable PDB on Airflow scheduler | `false` | | `scheduler.podDisruptionBudget.config.maxUnavailable` | MaxUnavailable pods for scheduler | `1` | +| `scheduler.replicas` | # of parallel schedulers (Airflow 2.0 using Mysql 8+ or Postgres only) | `1` | | `scheduler.resources.limits.cpu` | CPU Limit of scheduler | `~` | | `scheduler.resources.limits.memory` | Memory Limit of scheduler | `~` | | `scheduler.resources.requests.cpu` | CPU Request of scheduler | `~` | diff --git a/chart/templates/create-user-job.yaml b/chart/templates/create-user-job.yaml index f21d4b4..642e36f 100644 --- a/chart/templates/create-user-job.yaml +++ b/chart/templates/create-user-job.yaml @@ -51,6 +51,14 @@ spec: tolerations: {{ toYaml .Values.tolerations | indent 8 }} containers: + - name: run-airflow-migrations + image: {{ template "airflow_image" . }} + imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} + # Support running against 1.10.x and 2.0.0dev/master + args: ["bash", "-c", "airflow upgradedb || airflow db upgrade"] + env: + {{- include "custom_airflow_environment" . | indent 10 }} + {{- include "standard_airflow_environment" . | indent 10 }} - name: create-user image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml index b6878a7..2cf3dce 100644 --- a/chart/templates/scheduler/scheduler-deployment.yaml +++ b/chart/templates/scheduler/scheduler-deployment.yaml @@ -46,7 +46,7 @@ spec: {{- if $stateful }} serviceName: {{ .Release.Name }}-scheduler {{- end }} - replicas: 1 + replicas: {{ .Values.scheduler.replicas }} selector: matchLabels: tier: airflow @@ -90,11 +90,11 @@ spec: - name: {{ template "registry_secret" . }} {{- end }} initContainers: - - name: run-airflow-migrations + - name: wait-for-airflow-migrations image: {{ template "airflow_image" . }} imagePullPolicy: {{ .Values.images.airflow.pullPolicy }} - # Support running against 1.10.x and 2.0.0dev/master - args: ["bash", "-c", "airflow upgradedb || airflow db upgrade"] + args: + {{- include "wait-for-migrations-command" . | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" . | indent 10 }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 9370c80..b2f7288 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -486,6 +486,10 @@ "description": "Airflow scheduler settings.", "type": "object", "properties": { + "replicas": { + "description": "Airflow 2.0 allows users to run multiple schedulers. This feature is only recommended for Mysql 8+ and postgres", + "type": "integer" + }, "podDisruptionBudget": { "description": "Scheduler pod disruption budget.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 1370088..7b609a6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -292,6 +292,9 @@ workers: # Airflow scheduler settings scheduler: + # Airflow 2.0 allows users to run multiple schedulers, + # However this feature is only recommended for MySQL 8+ and Postgres + replicas: 1 # Scheduler pod disruption budget podDisruptionBudget: enabled: false
