This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ae681ba4b0c2c83c4d4533f500b539ee64ba0af2 Author: Daniel Imberman <[email protected]> AuthorDate: Mon Oct 5 10:38:52 2020 -0700 Single/Multi-Namespace mode for helm chart (#11034) * Multi-Namespace mode for helm chart Users should not REQUIRE a ClusterRole/ClusterRolebinding to run airflow via helm. This change will allow "single" and "multi" namespace modes so users can add airflow to managed kubernetes clusters * add namespace to role * add rolebinding too * add docs * add values.schema.json change (cherry picked from commit 93475e9f4dc92d5cb5efea3310e7ddc468c0b991) --- chart/README.md | 2 +- chart/templates/rbac/pod-launcher-role.yaml | 7 + chart/templates/rbac/pod-launcher-rolebinding.yaml | 7 + chart/values.schema.json | 1059 ++++++++++++++++++++ chart/values.yaml | 145 ++- 5 files changed, 1169 insertions(+), 51 deletions(-) diff --git a/chart/README.md b/chart/README.md index 8372bb4..ef1caa3 100644 --- a/chart/README.md +++ b/chart/README.md @@ -206,7 +206,7 @@ The following tables lists the configurable parameters of the Airflow chart and | `webserver.defaultUser` | Optional default airflow user information | `{}` | | `dags.persistence.*` | Dag persistence configuration | Please refer to `values.yaml` | | `dags.gitSync.*` | Git sync configuration | Please refer to `values.yaml` | - +| `multiNamespaceMode` | Whether the KubernetesExecutor can launch pods in multiple namespaces | `False` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, diff --git a/chart/templates/rbac/pod-launcher-role.yaml b/chart/templates/rbac/pod-launcher-role.yaml index 7e10122..dc56c92 100644 --- a/chart/templates/rbac/pod-launcher-role.yaml +++ b/chart/templates/rbac/pod-launcher-role.yaml @@ -19,10 +19,17 @@ ## Airflow Pod Launcher Role ################################# {{- if and .Values.rbacEnabled .Values.allowPodLaunching }} +{{- if .Values.multiNamespaceMode }} kind: ClusterRole +{{- else }} +kind: Role +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Release.Name }}-pod-launcher-role +{{- if not .Values.multiNamespaceMode }} + namespace: {{ .Release.Namespace }} +{{- end }} labels: tier: airflow release: {{ .Release.Name }} diff --git a/chart/templates/rbac/pod-launcher-rolebinding.yaml b/chart/templates/rbac/pod-launcher-rolebinding.yaml index 4dba494..6a1a5be 100644 --- a/chart/templates/rbac/pod-launcher-rolebinding.yaml +++ b/chart/templates/rbac/pod-launcher-rolebinding.yaml @@ -21,9 +21,16 @@ {{- if and .Values.rbacEnabled .Values.allowPodLaunching }} {{- $grantScheduler := or (eq .Values.executor "LocalExecutor") (eq .Values.executor "SequentialExecutor") (eq .Values.executor "KubernetesExecutor") }} {{- $grantWorker := or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "KubernetesExecutor") }} +{{- if .Values.multiNamespaceMode }} kind: ClusterRoleBinding +{{- else }} +kind: RoleBinding +{{- end }} apiVersion: rbac.authorization.k8s.io/v1 metadata: +{{- if not .Values.multiNamespaceMode }} + namespace: {{ .Release.Namespace }} +{{- end }} name: {{ .Release.Name }}-pod-launcher-rolebinding labels: tier: airflow diff --git a/chart/values.schema.json b/chart/values.schema.json new file mode 100644 index 0000000..9776116 --- /dev/null +++ b/chart/values.schema.json @@ -0,0 +1,1059 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema";, + "description": "Default values for airflow. Declare variables to be passed into your templates.", + "type": "object", + "properties": { + "uid": { + "description": "User of airflow user.", + "type": "integer" + }, + "gid": { + "description": "Group of airflow user.", + "type": "integer" + }, + "airflowHome": { + "description": "Airflow home directory. Used for mount paths.", + "type": "string" + }, + "defaultAirflowRepository": { + "description": "Default airflow repository. Overrides all the specific images below.", + "type": "string" + }, + "defaultAirflowTag": { + "description": "Default airflow tag to deploy.", + "type": "string" + }, + "multi_namespaceMode": { + "description": "Whether the KubernetesExecutor can launch workers in multiple namespaces", + "type": "boolean" + }, + "nodeSelector": { + "description": "Select certain nodes for airflow pods.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "affinity": { + "description": "Select certain nodes for airflow pods.", + "type": "object" + }, + "tolerations": { + "description": "Select certain nodes for airflow pods.", + "type": "array" + }, + "labels": { + "description": "Add common labels to all objects and pods defined in this chart.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "ingress": { + "description": "Ingress configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable ingress resource.", + "type": "boolean" + }, + "web": { + "description": "Configuration for the Ingress of the web Service.", + "type": "object", + "properties": { + "annotations": { + "description": "Annotations for the web Ingress.", + "type": "object" + }, + "path": { + "description": "The path for the web Ingress.", + "type": "string" + }, + "host": { + "description": "The hostname for the web Ingress.", + "type": "string" + }, + "tls": { + "description": "Configuration for web Ingress TLS.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable TLS termination for the web Ingress.", + "type": "boolean" + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string" + } + } + }, + "precedingPaths": { + "description": "HTTP paths to add to the web Ingress before the default path.", + "type": "array" + }, + "succeedingPaths": { + "description": "HTTP paths to add to the web Ingress after the default path.", + "type": "array" + } + } + }, + "flower": { + "description": "Configuration for the Ingress of the flower Service.", + "type": "object", + "properties": { + "annotations": { + "description": "Annotations for the flower Ingress.", + "type": "object" + }, + "path": { + "description": "The path for the flower Ingress.", + "type": "string" + }, + "host": { + "description": "The hostname for the flower Ingress.", + "type": "string" + }, + "tls": { + "description": "Configuration for flower Ingress TLS.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable TLS termination for the flower Ingress.", + "type": "boolean" + }, + "secretName": { + "description": "The name of a pre-created Secret containing a TLS private key and certificate.", + "type": "string" + } + } + }, + "precedingPaths": { + "description": "HTTP paths to add to the flower Ingress before the default path.", + "type": "array" + }, + "succeedingPaths": { + "description": "HTTP paths to add to the flower Ingress after the default path.", + "type": "array" + } + } + } + } + }, + "networkPolicies": { + "description": "Network policy configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Enabled network policies.", + "type": "boolean" + } + } + }, + "airflowPodAnnotations": { + "description": "Extra annotations to apply to all Airflow pods.", + "type": "object" + }, + "rbacEnabled": { + "description": "Enable RBAC (default on most clusters these days).", + "type": "boolean" + }, + "executor": { + "description": "Airflow executor.", + "type": "string" + }, + "allowPodLaunching": { + "description": "If this is true and using LocalExecutor/SequentialExecutor/KubernetesExecutor, the scheduler's service account will have access to communicate with the api-server and launch pods. If this is true and using the CeleryExecutor, the workers will be able to launch pods.", + "type": "boolean" + }, + "images": { + "description": "Images.", + "type": "object", + "properties": { + "airflow": { + "description": "Configuration of the airflow image.", + "type": "object", + "properties": { + "repository": { + "description": "The airflow image repository.", + "type": [ + "string", + "null" + ] + }, + "tag": { + "description": "The airflow image tag.", + "type": [ + "string", + "null" + ] + }, + "pullPolicy": { + "description": "The airflow image pull policy.", + "type": "string" + } + } + }, + "flower": { + "description": "Configuration of the flower image.", + "type": "object", + "properties": { + "repository": { + "description": "The flower image repository.", + "type": [ + "string", + "null" + ] + }, + "tag": { + "description": "The flower image tag.", + "type": [ + "string", + "null" + ] + }, + "pullPolicy": { + "description": "The flower image pull policy.", + "type": "string" + } + } + }, + "statsd": { + "description": "Configuration of the statsd image.", + "type": "object", + "properties": { + "repository": { + "description": "The statsd image repository.", + "type": "string" + }, + "tag": { + "description": "The statsd image tag.", + "type": "string" + }, + "pullPolicy": { + "description": "The statsd image pull policy.", + "type": "string" + } + } + }, + "redis": { + "description": "Configuration of the redis image.", + "type": "object", + "properties": { + "repository": { + "description": "The redis image repository.", + "type": "string" + }, + "tag": { + "description": "The redis image tag.", + "type": "string" + }, + "pullPolicy": { + "description": "The redis image pull policy.", + "type": "string" + } + } + }, + "pgbouncer": { + "description": "Configuration of the pgbouncer image.", + "type": "object", + "properties": { + "repository": { + "description": "The pgbouncer image repository.", + "type": "string" + }, + "tag": { + "description": "The pgbouncer image tag.", + "type": "string" + }, + "pullPolicy": { + "description": "The pgbouncer image pull policy.", + "type": "string" + } + } + }, + "pgbouncerExporter": { + "description": "Configuration of the pgbouncerExporter image.", + "type": "object", + "properties": { + "repository": { + "description": "The pgbouncerExporter image repository.", + "type": "string" + }, + "tag": { + "description": "The pgbouncerExporter image tag.", + "type": "string" + }, + "pullPolicy": { + "description": "The pgbouncerExporter image pull policy.", + "type": "string" + } + } + } + } + }, + "env": { + "description": "Environment variables for all airflow containers.", + "type": "array" + }, + "secret": { + "description": "Secrets for all airflow containers.", + "type": "array" + }, + "data": { + "description": "Airflow database configuration.", + "type": "object", + "properties": { + "metadataSecretName": { + "description": "Metadata connection string secret.", + "type": [ + "string", + "null" + ] + }, + "resultBackendSecretName": { + "description": "Result backend connection string secret.", + "type": [ + "string", + "null" + ] + }, + "metadataConnection": { + "description": "Metadata connection configuration.", + "type": "object", + "properties": { + "user": { + "description": "The database user.", + "type": "string" + }, + "pass": { + "description": "The user's password.", + "type": "string" + }, + "host": { + "description": "The database host.", + "type": [ + "string", + "null" + ] + }, + "port": { + "description": "The database port.", + "type": "integer" + }, + "db": { + "description": "The name of the database.", + "type": "string" + }, + "sslmode": { + "description": "The database SSL parameter.", + "type": "string" + } + } + }, + "resultBackendConnection": { + "description": "Result backend connection configuration.", + "type": "object", + "properties": { + "user": { + "description": "The database user.", + "type": "string" + }, + "pass": { + "description": "The database password.", + "type": "string" + }, + "host": { + "description": "The database host.", + "type": [ + "string", + "null" + ] + }, + "port": { + "description": "The database port.", + "type": "integer" + }, + "db": { + "description": "The name of the database.", + "type": "string" + }, + "sslmode": { + "description": "The database SSL parameter.", + "type": "string" + } + } + } + } + }, + "fernetKey": { + "description": "The Fernet key used to encrypt passwords.", + "type": [ + "string", + "null" + ] + }, + "fernetKeySecretName": { + "description": "The Fernet key secret name.", + "type": [ + "string", + "null" + ] + }, + "workers": { + "description": "Airflow Worker configuration.", + "type": "object", + "properties": { + "replicas": { + "description": "Number of airflow celery workers in StatefulSet.", + "type": "integer" + }, + "keda": { + "description": "KEDA configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Allow KEDA autoscaling. `Persistence.enabled` must be set to false to use KEDA.", + "type": "boolean" + }, + "namespaceLabels": { + "type": "object" + }, + "pollingInterval": { + "description": "How often KEDA polls the airflow DB to report new scale requests to the HPA.", + "type": "integer" + }, + "cooldownPeriod": { + "description": "How many seconds KEDA will wait before scaling to zero.", + "type": "integer" + }, + "maxReplicaCount": { + "description": "Maximum number of workers created by KEDA.", + "type": "integer" + } + } + }, + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": "boolean" + }, + "size": { + "description": "Volume size for worker StatefulSet.", + "type": "string" + }, + "storageClassName": { + "description": "If using a custom storageClass, pass name ref to all StatefulSets here.", + "type": [ + "string", + "null" + ] + }, + "fixPermissions": { + "description": "Execute init container to chown log directory. This is currently only needed in KinD, due to usage of local-path provisioner.", + "type": "boolean" + } + } + }, + "resources": { + "type": "object" + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": "integer" + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", + "type": "boolean" + } + } + }, + "scheduler": { + "description": "Airflow scheduler settings.", + "type": "object", + "properties": { + "podDisruptionBudget": { + "description": "Scheduler pod disruption budget.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable pod disruption budget.", + "type": "boolean" + }, + "config": { + "description": "Disruption budget configuration.", + "type": "object", + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for scheduler.", + "type": "integer" + } + } + } + } + }, + "resources": { + "type": "object" + }, + "airflowLocalSettings": { + "description": "This setting can overwrite podMutation setting.", + "type": [ + "string", + "null" + ] + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.", + "type": "boolean" + } + } + }, + "webserver": { + "description": "Airflow webserver settings.", + "type": "object", + "properties": { + "livenessProbe": { + "description": "Liveness probe configuration.", + "type": "object", + "properties": { + "initialDelaySeconds": { + "description": "Webserver Liveness probe initial delay.", + "type": "integer" + }, + "timeoutSeconds": { + "description": "Webserver Liveness probe timeout seconds.", + "type": "integer" + }, + "failureThreshold": { + "description": "Webserver Liveness probe failure threshold.", + "type": "integer" + }, + "periodSeconds": { + "description": "Webserver Liveness probe period seconds.", + "type": "integer" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration.", + "type": "object", + "properties": { + "initialDelaySeconds": { + "description": "Webserver Readiness probe initial delay.", + "type": "integer" + }, + "timeoutSeconds": { + "description": "Webserver Readiness probe timeout seconds.", + "type": "integer" + }, + "failureThreshold": { + "description": "Webserver Readiness probe failure threshold.", + "type": "integer" + }, + "periodSeconds": { + "description": "Webserver Readiness probe period seconds.", + "type": "integer" + } + } + }, + "replicas": { + "description": "How many Airflow webserver replicas should run.", + "type": "integer" + }, + "extraNetworkPolicies": { + "description": "Additional network policies as needed.", + "type": "array" + }, + "resources": { + "type": "object" + }, + "defaultUser": { + "description": "Optional default airflow user information", + "type": "object", + "properties": { + "enabled": { + "description": "Enable default user creation.", + "type": "boolean" + }, + "role": { + "description": "Default user role.", + "type": "string" + }, + "username": { + "description": "Default user username.", + "type": "string" + }, + "email": { + "description": "Default user email address.", + "type": "string" + }, + "firstName": { + "description": "Default user firstname.", + "type": "string" + }, + "lastName": { + "description": "Default user lastname.", + "type": "string" + }, + "password": { + "description": "Default user password.", + "type": "string" + } + } + }, + "extraVolumes": { + "description": "Mount additional volumes into webserver.", + "type": "array" + }, + "extraVolumeMounts": { + "description": "Mount additional volumes into webserver.", + "type": "array" + }, + "webserverConfig": { + "description": "This will be mounted into the Airflow Webserver as a custom `webserver_config.py`. You can bake a `webserver_config.py` in to your image instead.", + "type": [ + "string", + "null" + ] + }, + "service": { + "description": "Webserver service configuration.", + "type": "object", + "properties": { + "type": { + "description": "Webserver service type.", + "type": "string" + }, + "annotations": { + "description": "Annotations for the webserver service.", + "type": "object" + } + } + } + } + }, + "flower": { + "description": "Flower settings.", + "type": "object", + "properties": { + "extraNetworkPolicies": { + "description": "Additional network policies as needed.", + "type": "array" + }, + "resources": { + "type": "object" + }, + "service": { + "description": "Flower service configuration.", + "type": "object", + "properties": { + "type": { + "description": "Flower service type.", + "type": "string" + } + } + } + } + }, + "statsd": { + "description": "Statsd settings.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable statsd.", + "type": "boolean" + }, + "extraNetworkPolicies": { + "description": "Additional network policies as needed.", + "type": "array" + }, + "resources": { + "type": "object" + }, + "service": { + "description": "Statsd service configuration.", + "type": "object", + "properties": { + "extraAnnotations": { + "description": "Extra annotations for the statsd service.", + "type": "object" + } + } + } + } + }, + "pgbouncer": { + "description": "Pgbouncer settings.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable pgbouncer.", + "type": "boolean" + }, + "extraNetworkPolicies": { + "description": "Additional network policies as needed.", + "type": "array" + }, + "metadataPoolSize": { + "description": "Metadata pool size.", + "type": "integer" + }, + "resultBackendPoolSize": { + "description": "Result backend pool size.", + "type": "integer" + }, + "maxClientConn": { + "description": "Maximum clients that can connect to pgbouncer (higher = more file descriptors).", + "type": "integer" + }, + "podDisruptionBudget": { + "description": "Pgbouner pod disruption budget.", + "type": "object", + "properties": { + "enabled": { + "description": "Enabled pod distribution budget.", + "type": "boolean" + }, + "config": { + "description": "Pod distribution configuration.", + "type": "object", + "properties": { + "maxUnavailable": { + "description": "Max unavailable pods for pgbouncer.", + "type": "integer" + } + } + } + } + }, + "resources": { + "type": "object" + }, + "service": { + "description": "Pgbouncer service configuration.", + "type": "object", + "properties": { + "extraAnnotations": { + "description": "Extra annotations for the pgbouncer service.", + "type": "object" + } + } + }, + "verbose": { + "description": "Increase pgbouncer verbosity.", + "type": "integer" + }, + "logDisconnections": { + "description": "Log disconnections with reasons.", + "type": "integer" + }, + "logConnections": { + "description": "Log successful logins.", + "type": "integer" + } + } + }, + "redis": { + "description": "", + "type": "object", + "properties": { + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": "integer" + }, + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable persistent volumes.", + "type": "boolean" + }, + "size": { + "description": "Volume size for worker StatefulSet.", + "type": "string" + }, + "storageClassName": { + "description": "If using a custom storageClass, pass name ref to all StatefulSets here.", + "type": [ + "string", + "null" + ] + } + } + }, + "resources": { + "type": "object" + }, + "passwordSecretName": { + "description": "Redis password secret.", + "type": [ + "string", + "null" + ] + }, + "brokerURLSecretName": { + "description": "Redis broker URL secret.", + "type": [ + "string", + "null" + ] + }, + "password": { + "description": "If password is set, create secret with it, else generate a new one on install.", + "type": [ + "string", + "null" + ] + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that its ok to evict when it wants to scale a node down.", + "type": "boolean" + } + } + }, + "registry": { + "description": "Auth secret for a private registry. This is used if pulling airflow images from a private registry.", + "type": "object", + "properties": { + "secretName": { + "description": "Registry connection string secret.", + "type": [ + "string", + "null" + ] + }, + "connection": { + "description": "Registry connection configuration.", + "type": "object" + } + } + }, + "elasticsearch": { + "description": "Elasticsearch logging configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable elasticsearch task logging.", + "type": "boolean" + }, + "secretName": { + "description": "A secret containing the connection string.", + "type": [ + "string", + "null" + ] + }, + "connection": { + "description": "Elasticsearch connection configuration.", + "type": "object" + } + } + }, + "ports": { + "description": "All ports used by chart.", + "type": "object", + "properties": { + "flowerUI": { + "description": "Flower UI port.", + "type": "integer" + }, + "airflowUI": { + "description": "Airflow UI port.", + "type": "integer" + }, + "workerLogs": { + "description": "Worker logs port.", + "type": "integer" + }, + "redisDB": { + "description": "Redis port.", + "type": "integer" + }, + "statsdIngest": { + "description": "Statsd ingest port.", + "type": "integer" + }, + "statsdScrape": { + "description": "Statsd scrape port.", + "type": "integer" + }, + "pgbouncer": { + "description": "Pgbouncer port.", + "type": "integer" + }, + "pgbouncerScrape": { + "description": "Pgbouncer scrape port.", + "type": "integer" + } + } + }, + "quotas": { + "description": "Define any ResourceQuotas for namespace.", + "type": "object" + }, + "limits": { + "description": "Define default/max/min values for pods and containers in namespace.", + "type": "array" + }, + "podMutation": { + "description": "Settings for pod_mutation_hook", + "type": "object", + "properties": { + "tolerations": { + "description": "Tolerations provided here would be applied using pod_mutation_hook. So any pods spun up using KubernetesExecutor or KubernetesPodOperator will contain these tolerations.", + "type": "array" + }, + "affinity": { + "description": "Pods spun up would land in the node that matches the affinity.", + "type": "object" + } + } + }, + "cleanup": { + "description": "This runs as a CronJob to cleanup old pods.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable cleanup.", + "type": "boolean" + }, + "schedule": { + "description": "Cleanup schedule.", + "type": "string" + } + } + }, + "postgresql": { + "description": "Configuration for postgresql subchart.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable postgresql subchart.", + "type": "boolean" + }, + "postgresqlPassword": { + "description": "Postgresql password.", + "type": "string" + }, + "postgresqlUsername": { + "description": "Postgresql username.", + "type": "string" + } + } + }, + "config": { + "description": "Settings to go into the mounted airflow.cfg", + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": [ + "boolean", + "integer", + "number", + "string" + ] + } + } + }, + "dags": { + "description": "DAGs settings.", + "type": "object", + "properties": { + "persistence": { + "description": "Persistence configuration.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable persistent volume for storing dags.", + "type": "boolean" + }, + "size": { + "description": "Volume size for dags.", + "type": "string" + }, + "storageClassName": { + "description": "If using a custom storageClass, pass name here.", + "type": [ + "string", + "null" + ] + }, + "accessMode": { + "description": "Access mode of the persistent volume.", + "type": "string" + }, + "existingClaim": { + "description": "The name of an existing PVC to use.", + "type": [ + "string", + "null" + ] + } + } + }, + "gitSync": { + "description": "Git sync settings.", + "type": "object", + "properties": { + "enabled": { + "description": "Enable Git sync.", + "type": "boolean" + }, + "repo": { + "description": "Git repository.", + "type": "string" + }, + "branch": { + "description": "Git branch", + "type": "string" + }, + "rev": { + "description": "Git revision.", + "type": "string" + }, + "root": { + "description": "Root directory.", + "type": "string" + }, + "dest": { + "description": "Destination folder.", + "type": "string" + }, + "depth": { + "description": "Repository depth.", + "type": "integer" + }, + "maxFailures": { + "description": "The number of consecutive failures allowed before aborting.", + "type": "integer" + }, + "subPath": { + "description": "Subpath within the repo where dags are located.", + "type": "string" + }, + "wait": { + "description": "Interval between git sync attempts in seconds.", + "type": "integer" + }, + "containerRepository": { + "description": "Git sync image repository.", + "type": "string" + }, + "containerTag": { + "description": "Git sync image tag.", + "type": "string" + }, + "containerName": { + "description": "Git sync container name.", + "type": "string" + } + } + } + } + } + } +} diff --git a/chart/values.yaml b/chart/values.yaml index 513dc47..f7c5e67 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -31,7 +31,7 @@ airflowHome: "/opt/airflow" defaultAirflowRepository: apache/airflow # Default airflow tag to deploy -defaultAirflowTag: 1.10.10.1-alpha2-python3.6 +defaultAirflowTag: 1.10.12 # Select certain nodes for airflow pods. @@ -42,6 +42,59 @@ tolerations: [] # Add common labels to all objects and pods defined in this chart. labels: {} +# Ingress configuration +ingress: + # Enable ingress resource + enabled: false + + # Configs for the Ingress of the web Service + web: + # Annotations for the web Ingress + annotations: {} + + # The path for the web Ingress + path: "" + + # The hostname for the web Ingress + host: "" + + # configs for web Ingress TLS + tls: + # Enable TLS termination for the web Ingress + enabled: false + # the name of a pre-created Secret containing a TLS private key and certificate + secretName: "" + + # HTTP paths to add to the web Ingress before the default path + precedingPaths: [] + + # Http paths to add to the web Ingress after the default path + succeedingPaths: [] + + # Configs for the Ingress of the flower Service + flower: + # Annotations for the flower Ingress + annotations: {} + + # The path for the flower Ingress + path: "" + + # The hostname for the flower Ingress + host: "" + + # configs for web Ingress TLS + tls: + # Enable TLS termination for the flower Ingress + enabled: false + # the name of a pre-created Secret containing a TLS private key and certificate + secretName: "" + + # HTTP paths to add to the flower Ingress before the default path + precedingPaths: [] + + # Http paths to add to the flower Ingress after the default path + succeedingPaths: [] + # Network policy configuration networkPolicies: # Enabled network policies @@ -196,7 +249,7 @@ workers: pollingInterval: 5 # How many seconds KEDA will wait before scaling to zero. - # Note that HPA has a seperate cooldwon period for scale-downs + # Note that HPA has a separate cooldown period for scale-downs cooldownPeriod: 30 # Maximum number of workers created by keda @@ -280,12 +333,12 @@ webserver: extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi # Create initial user. defaultUser: @@ -316,18 +369,20 @@ webserver: service: type: ClusterIP + ## service annotations + annotations: {} # Flower settings flower: # Additional network policies as needed extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: type: ClusterIP @@ -338,12 +393,12 @@ statsd: # Additional network policies as needed extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: extraAnnotations: {} @@ -370,21 +425,13 @@ pgbouncer: config: maxUnavailable: 1 - # Limit the resources to pgbouncerExported. - # When you specify the resource request the scheduler uses this information to decide which node to place - # the Pod on. When you specify a resource limit for a Container, the kubelet enforces those limits so - # that the running container is not allowed to use more of that resource than the limit you set. - # See: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - # Example: - # - # resource: - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: extraAnnotations: {} @@ -430,13 +477,11 @@ redis: registry: secretName: ~ - # Example: - # connection: - # user: ~ - # pass: ~ - # host: ~ - # email: ~ connection: {} + # user: ~ + # pass: ~ + # host: ~ + # email: ~ # Elasticsearch logging configuration elasticsearch: @@ -445,14 +490,11 @@ elasticsearch: # A secret containing the connection secretName: ~ # Or an object representing the connection - # Example: - # connection: - # user: ~ - # pass: ~ - # host: ~ - # port: ~ connection: {} - + # user: ~ + # pass: ~ + # host: ~ + # port: ~ # All ports used by chart ports: @@ -562,6 +604,7 @@ config: namespace: '{{ .Release.Namespace }}' airflow_configmap: '{{ include "airflow_config" . }}' airflow_local_settings_configmap: '{{ include "airflow_config" . }}' + pod_template_file: '{{ include "airflow_pod_template_file" . }}/pod_template_file.yaml' worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' worker_container_image_pull_policy: '{{ .Values.images.airflow.pullPolicy }}' @@ -569,6 +612,7 @@ config: image_pull_secrets: '{{ template "registry_secret" . }}' dags_in_image: '{{ ternary "False" "True" (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}' delete_worker_pods: 'True' + multi_namespace_mode: '{{ .Values.multiNamespaceMode }}' delete_worker_pods_on_failure: 'False' run_as_user: '{{ .Values.uid }}' fs_group: '{{ .Values.gid }}' @@ -594,6 +638,10 @@ config: AIRFLOW__CORE__FERNET_KEY: '{{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}' # yamllint enable rule:line-length +multiNamespaceMode: 'False' + +podTemplate: ~ + # Git sync dags: persistence: @@ -635,7 +683,6 @@ dags: # GIT_SYNC_USERNAME: <base64_encoded_git_username> # GIT_SYNC_PASSWORD: <base64_encoded_git_password> # and specify the name of the secret below - # # credentialsSecret: git-credentials # # @@ -651,10 +698,8 @@ dags: # gitSshKey: <base64_encoded_data> # and specify the name of the secret below # sshKeySecret: airflow-ssh-secret - # # If you are using an ssh private key, you can additionally # specify the content of your known_hosts file, example: - # # knownHosts: | # <host1>,<ip1> <key1> # <host2>,<ip2> <key2>
