This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch chart/v1-2x-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/chart/v1-2x-test by this push:
new 3329cb0cef0 [chart/v1-2x-test] Add workers.celery.logGroomerSidecar
section (#65033) (#65083)
3329cb0cef0 is described below
commit 3329cb0cef043b9438fb7cdea7515c383a63c4de
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Apr 12 14:40:20 2026 +0200
[chart/v1-2x-test] Add workers.celery.logGroomerSidecar section (#65033)
(#65083)
* Add workers.celery.logGroomerSidecar section
* Add newsfragment
* Fix spellcheck & tests
(cherry picked from commit 556380f86e463b1a118f0db85a37e578343ea996)
Co-authored-by: Przemysław Mirowski
<[email protected]>
---
chart/docs/setting-resources-for-containers.rst | 2 +-
chart/newsfragments/65033.significant.rst | 1 +
chart/templates/NOTES.txt | 96 +++++++
chart/values.schema.json | 313 ++++++++++++++++++++-
chart/values.yaml | 74 ++++-
helm-tests/tests/chart_utils/log_groomer.py | 145 ++++++----
.../airflow_aux/test_container_lifecycle.py | 71 ++++-
.../tests/helm_tests/airflow_core/test_worker.py | 7 +
.../helm_tests/security/test_security_context.py | 25 +-
9 files changed, 661 insertions(+), 73 deletions(-)
diff --git a/chart/docs/setting-resources-for-containers.rst
b/chart/docs/setting-resources-for-containers.rst
index 600b690778d..1402d465d79 100644
--- a/chart/docs/setting-resources-for-containers.rst
+++ b/chart/docs/setting-resources-for-containers.rst
@@ -31,7 +31,7 @@ Possible containers where resources can be configured include:
* Main Airflow containers and their sidecars. You can add the resources for
these containers through the following parameters:
* ``workers.resources``
- * ``workers.logGroomerSidecar.resources``
+ * ``workers.celery.logGroomerSidecar.resources``
* ``workers.kerberosSidecar.resources``
* ``workers.kerberosInitContainer.resources``
* ``scheduler.resources``
diff --git a/chart/newsfragments/65033.significant.rst
b/chart/newsfragments/65033.significant.rst
new file mode 100644
index 00000000000..ee4ec2bfd20
--- /dev/null
+++ b/chart/newsfragments/65033.significant.rst
@@ -0,0 +1 @@
+``workers.logGroomerSidecar`` section is now deprecated in favor of
``workers.celery.logGroomerSidecar``. Please update your configuration
accordingly.
diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt
index baee62cb881..205fd39ec19 100644
--- a/chart/templates/NOTES.txt
+++ b/chart/templates/NOTES.txt
@@ -829,6 +829,102 @@ DEPRECATION WARNING:
{{- end }}
+{{- if not .Values.workers.logGroomerSidecar.enabled }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.enabled` has been renamed to
`workers.celery.logGroomerSidecar.enabled`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.logGroomerSidecar.command) }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.command` has been renamed to
`workers.celery.logGroomerSidecar.command`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if ne (.Values.workers.logGroomerSidecar.args | toJson) (list "bash"
"/clean-logs" | toJson) }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.args` has been renamed to
`workers.celery.logGroomerSidecar.args`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.logGroomerSidecar.retentionDays) 15 }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.retentionDays` has been renamed to
`workers.celery.logGroomerSidecar.retentionDays`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.logGroomerSidecar.retentionMinutes) 0 }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.retentionMinutes` has been renamed to
`workers.celery.logGroomerSidecar.retentionMinutes`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.logGroomerSidecar.frequencyMinutes) 15 }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.frequencyMinutes` has been renamed to
`workers.celery.logGroomerSidecar.frequencyMinutes`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.logGroomerSidecar.maxSizeBytes) 0 }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.maxSizeBytes` has been renamed to
`workers.celery.logGroomerSidecar.maxSizeBytes`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if ne (int .Values.workers.logGroomerSidecar.maxSizePercent) 0 }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.maxSizePercent` has been renamed to
`workers.celery.logGroomerSidecar.maxSizePercent`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.logGroomerSidecar.resources) }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.resources` has been renamed to
`workers.celery.logGroomerSidecar.resources`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if not (empty
.Values.workers.logGroomerSidecar.securityContexts.container) }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.securityContexts.container` has been renamed to
`workers.celery.logGroomerSidecar.securityContexts.container`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.logGroomerSidecar.env) }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.env` has been renamed to
`workers.celery.logGroomerSidecar.env`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
+{{- if not (empty .Values.workers.logGroomerSidecar.containerLifecycleHooks) }}
+
+ DEPRECATION WARNING:
+ `workers.logGroomerSidecar.containerLifecycleHooks` has been renamed to
`workers.celery.logGroomerSidecar.containerLifecycleHooks`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
{{- if not .Values.workers.waitForMigrations.enabled }}
DEPRECATION WARNING:
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 8d152146ca1..984149f26b2 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2436,8 +2436,150 @@
}
},
"logGroomerSidecar": {
- "$ref": "#/definitions/logGroomerConfigType",
- "description": "Configuration for Airflow Celery worker
log groomer sidecar"
+ "description": "Configuration for Airflow Celery worker
log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar``
instead).",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "enabled": {
+ "description": "Whether to deploy the Airflow log
groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.enabled``
instead).",
+ "type": "boolean",
+ "default": true
+ },
+ "command": {
+ "description": "Command to use when running the
Airflow log groomer sidecar (templated) (deprecated, use
``workers.celery.logGroomerSidecar.command`` instead).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": null
+ },
+ "args": {
+ "description": "Args to use when running the
Airflow log groomer sidecar (templated) (deprecated, use
``workers.celery.logGroomerSidecar.args`` instead).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": [
+ "bash",
+ "/clean-logs"
+ ]
+ },
+ "retentionDays": {
+ "description": "Number of days to retain the logs
when running the Airflow log groomer sidecar (deprecated, use
``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention
time is ``retentionDays`` + ``retentionMinutes``.",
+ "type": "integer",
+ "default": 15
+ },
+ "retentionMinutes": {
+ "description": "Number of minutes to retain the
logs when running the Airflow log groomer sidecar (deprecated, use
``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention
time is ``retentionDays`` + ``retentionMinutes``.",
+ "type": "integer",
+ "default": 0
+ },
+ "frequencyMinutes": {
+ "description": "Number of minutes between attempts
to groom the Airflow logs in log groomer sidecar (deprecated, use
``workers.celery.logGroomerSidecar.frequencyMinutes`` instead).",
+ "type": "integer",
+ "default": 15
+ },
+ "maxSizeBytes": {
+ "description": "Max size of logs directory in
bytes (deprecated, use ``workers.celery.logGroomerSidecar.maxSizeBytes``
instead). When exceeded, the log groomer reduces retention until size is under
limit. 0 = disabled.",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0
+ },
+ "maxSizePercent": {
+ "description": "Max size of logs as a percentage
of total disk space (deprecated, use
``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded,
the log groomer reduces retention until size is under limit. 0 = disabled.
Ignored if ``maxSizeBytes`` is set.",
+ "type": "integer",
+ "default": 0,
+ "minimum": 0,
+ "maximum": 100
+ },
+ "env": {
+ "description": "Add additional env vars to log
groomer sidecar container (templated) (deprecated, use
``workers.celery.logGroomerSidecar.env`` instead).",
+ "items": {
+ "$ref":
"#/definitions/io.k8s.api.core.v1.EnvVar"
+ },
+ "type": "array",
+ "default": [],
+ "x-kubernetes-patch-merge-key": "name",
+ "x-kubernetes-patch-strategy": "merge"
+ },
+ "resources": {
+ "description": "Resources for Airflow log groomer
sidecar (deprecated, use ``workers.celery.logGroomerSidecar.resources``
instead).",
+ "type": "object",
+ "default": {},
+ "examples": [
+ {
+ "limits": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ },
+ "requests": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ }
+ }
+ ],
+ "$ref":
"#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+ },
+ "containerLifecycleHooks": {
+ "description": "Container Lifecycle Hooks
definition for the log groomer sidecar (deprecated, use
``workers.celery.logGroomerSidecar.containerLifecycleHooks`` instead). If not
set, the values from global `containerLifecycleHooks` will be used.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.Lifecycle",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+ "postStart": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo postStart handler >
/usr/share/message"
+ ]
+ }
+ },
+ "preStop": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo preStop handler >
/usr/share/message"
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "securityContexts": {
+ "description": "Security context definition for
the log groomer sidecar (deprecated, use
``workers.celery.logGroomerSidecar.securityContexts`` instead). If not set, the
values from global `securityContexts` will be used.",
+ "type": "object",
+ "x-docsSection": "Kubernetes",
+ "properties": {
+ "container": {
+ "description": "Container security context
definition for the log groomer sidecar (deprecated, use
``workers.celery.logGroomerSidecar.securityContexts.container`` instead).",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.SecurityContext",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+ "allowPrivilegeEscalation": false,
+ "capabilities": {
+ "drop": [
+ "ALL"
+ ]
+ }
+ }
+ ]
+ }
+ }
+ }
+ }
},
"securityContext": {
"description": "Security context for the Airflow Celery
worker pods and pods created with pod-template-file (deprecated, use
``workers.celery.securityContexts`` and/or
``workers.kubernetes.securityContexts`` instead). If not set, the values from
`securityContext` will be used.",
@@ -3506,6 +3648,167 @@
"type": "string"
}
},
+ "logGroomerSidecar": {
+ "description": "Configuration for Airflow Celery
worker log groomer sidecar.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "enabled": {
+ "description": "Whether to deploy the
Airflow log groomer sidecar.",
+ "type": [
+ "boolean",
+ "null"
+ ],
+ "default": null
+ },
+ "command": {
+ "description": "Command to use when
running the Airflow log groomer sidecar (templated).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": null
+ },
+ "args": {
+ "description": "Args to use when running
the Airflow log groomer sidecar (templated).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": []
+ },
+ "retentionDays": {
+ "description": "Number of days to retain
the logs when running the Airflow log groomer sidecar. Total retention time is
``retentionDays`` + ``retentionMinutes``.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null
+ },
+ "retentionMinutes": {
+ "description": "Number of minutes to
retain the logs when running the Airflow log groomer sidecar. Total retention
time is ``retentionDays`` + ``retentionMinutes``.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null
+ },
+ "frequencyMinutes": {
+ "description": "Number of minutes between
attempts to groom the Airflow logs in log groomer sidecar.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null
+ },
+ "maxSizeBytes": {
+ "description": "Max size of logs directory
in bytes. When exceeded, the log groomer reduces retention until size is under
limit. 0 = disabled.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null,
+ "minimum": 0
+ },
+ "maxSizePercent": {
+ "description": "Max size of logs as a
percentage of total disk space. When exceeded, the log groomer reduces
retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes``
is set.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null,
+ "minimum": 0,
+ "maximum": 100
+ },
+ "env": {
+ "description": "Add additional env vars to
log groomer sidecar container (templated).",
+ "items": {
+ "$ref":
"#/definitions/io.k8s.api.core.v1.EnvVar"
+ },
+ "type": "array",
+ "default": [],
+ "x-kubernetes-patch-merge-key": "name",
+ "x-kubernetes-patch-strategy": "merge"
+ },
+ "resources": {
+ "description": "Resources for Airflow log
groomer sidecar.",
+ "type": "object",
+ "default": {},
+ "examples": [
+ {
+ "limits": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ },
+ "requests": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ }
+ }
+ ],
+ "$ref":
"#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+ },
+ "containerLifecycleHooks": {
+ "description": "Container Lifecycle Hooks
definition for the log groomer sidecar. If not set, the values from
``workers.containerLifecycleHooks`` will be used.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.Lifecycle",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+ "postStart": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo postStart
handler > /usr/share/message"
+ ]
+ }
+ },
+ "preStop": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo preStop handler
> /usr/share/message"
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "securityContexts": {
+ "description": "Security context
definition for the log groomer sidecar. If not set, the values from
``workers.securityContexts`` will be used.",
+ "type": "object",
+ "x-docsSection": "Kubernetes",
+ "properties": {
+ "container": {
+ "description": "Container security
context definition for the log groomer sidecar.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.SecurityContext",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+
"allowPrivilegeEscalation": false,
+ "capabilities": {
+ "drop": [
+ "ALL"
+ ]
+ }
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
"waitForMigrations": {
"description": "Configuration of
wait-for-airflow-migration init container for Airflow Celery workers.",
"type": "object",
@@ -14698,12 +15001,12 @@
]
},
"retentionDays": {
- "description": "Number of days to retain the logs when
running the Airflow log groomer sidecar. Total retention time is retentionDays
+ retentionMinutes.",
+ "description": "Number of days to retain the logs when
running the Airflow log groomer sidecar. Total retention time is
``retentionDays`` + ``retentionMinutes``.",
"type": "integer",
"default": 15
},
"retentionMinutes": {
- "description": "Number of minutes to retain the logs when
running the Airflow log groomer sidecar. Total retention time is retentionDays
+ retentionMinutes.",
+ "description": "Number of minutes to retain the logs when
running the Airflow log groomer sidecar. Total retention time is
``retentionDays`` + ``retentionMinutes``.",
"type": "integer",
"default": 0
},
@@ -14719,7 +15022,7 @@
"minimum": 0
},
"maxSizePercent": {
- "description": "Max size of logs as a percentage of total
disk space. When exceeded, the log groomer reduces retention until size is
under limit. 0 = disabled. Ignored if maxSizeBytes is set.",
+ "description": "Max size of logs as a percentage of total
disk space. When exceeded, the log groomer reduces retention until size is
under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.",
"type": "integer",
"default": 0,
"minimum": 0,
diff --git a/chart/values.yaml b/chart/values.yaml
index 8a3290df5aa..fa56e57bebc 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -1142,33 +1142,43 @@ workers:
labels: {}
# Log groomer configuration for Airflow Celery workers
+ # (deprecated, use `workers.celery.logGroomerSidecar` instead)
logGroomerSidecar:
# Whether to deploy the Airflow Celery worker log groomer sidecar
+ # (deprecated, use `workers.celery.logGroomerSidecar.enabled` instead)
enabled: true
# Command to use when running the Airflow Celery worker log groomer
sidecar (templated)
+ # (deprecated, use `workers.celery.logGroomerSidecar.command` instead)
command: ~
# Args to use when running the Airflow Celery worker log groomer sidecar
(templated)
+ # (deprecated, use `workers.celery.logGroomerSidecar.args` instead)
args: ["bash", "/clean-logs"]
# Number of days to retain logs
+ # (deprecated, use `workers.celery.logGroomerSidecar.retentionDays`
instead)
retentionDays: 15
# Number of minutes to retain logs.
# This can be used for finer granularity than days.
- # Total retention is retentionDays + retentionMinutes.
+ # Total retention is `retentionDays` + `retentionMinutes`.
+ # (deprecated, use `workers.celery.logGroomerSidecar.retentionMinutes`
instead)
retentionMinutes: 0
# Frequency to attempt to groom logs (in minutes)
+ # (deprecated, use `workers.celery.logGroomerSidecar.frequencyMinutes`
instead)
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
+ # (deprecated, use `workers.celery.logGroomerSidecar.maxSizeBytes` instead)
maxSizeBytes: 0
- # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if
maxSizeBytes is set.
+ # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if
`maxSizeBytes` is set.
+ # (deprecated, use `workers.celery.logGroomerSidecar.maxSizePercent`
instead)
maxSizePercent: 0
+ # (deprecated, use `workers.celery.logGroomerSidecar.resources` instead)
resources: {}
# limits:
# cpu: 100m
@@ -1177,12 +1187,19 @@ workers:
# cpu: 100m
# memory: 128Mi
- # Detailed default security context for logGroomerSidecar for container
level
+ # Detailed default security context for `logGroomerSidecar` for container
level
+ # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts`
instead)
securityContexts:
+ # (deprecated, use
`workers.celery.logGroomerSidecar.securityContexts.container` instead)
container: {}
+ # (deprecated, use `workers.celery.logGroomerSidecar.env` instead)
env: []
+ # Container level lifecycle hooks
+ # (deprecated, use
`workers.celery.logGroomerSidecar.containerLifecycleHooks` instead)
+ containerLifecycleHooks: {}
+
# Configuration of wait-for-airflow-migration init container for Airflow
Celery workers
# (deprecated, use `workers.celery.waitForMigrations` instead)
waitForMigrations:
@@ -1533,6 +1550,51 @@ workers:
# Pod annotations for the Airflow Celery workers (templated)
podAnnotations: {}
+ # Log groomer configuration for Airflow Celery workers
+ logGroomerSidecar:
+ # Whether to deploy the Airflow Celery worker log groomer sidecar
+ enabled: ~
+
+ # Command to use when running the Airflow Celery worker log groomer
sidecar (templated)
+ command: ~
+
+ # Args to use when running the Airflow Celery worker log groomer sidecar
(templated)
+ args: []
+
+ # Number of days to retain logs
+ retentionDays: ~
+
+ # Number of minutes to retain logs.
+ # This can be used for finer granularity than days.
+ # Total retention is `retentionDays` + `retentionMinutes`.
+ retentionMinutes: ~
+
+ # Frequency to attempt to groom logs (in minutes)
+ frequencyMinutes: ~
+
+ # Max size of logs in bytes. 0 = disabled
+ maxSizeBytes: ~
+
+ # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if
`maxSizeBytes` is set.
+ maxSizePercent: ~
+
+ resources: {}
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
+
+ # Detailed default security context for `logGroomerSidecar` for
container level
+ securityContexts:
+ container: {}
+
+ env: []
+
+ # Container level lifecycle hooks
+ containerLifecycleHooks: {}
+
# Configuration of wait-for-airflow-migration init container for Airflow
Celery workers
waitForMigrations:
# Whether to create init container to wait for db migrations
@@ -1911,7 +1973,7 @@ scheduler:
# Max size of logs in bytes. 0 = disabled
maxSizeBytes: 0
- # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if
maxSizeBytes is set.
+ # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if
`maxSizeBytes` is set.
maxSizePercent: 0
resources: {}
@@ -1922,7 +1984,7 @@ scheduler:
# cpu: 100m
# memory: 128Mi
- # Detailed default security context for logGroomerSidecar for container
level
+ # Detailed default security context for `logGroomerSidecar` for container
level
securityContexts:
container: {}
@@ -2845,7 +2907,7 @@ triggerer:
# cpu: 100m
# memory: 128Mi
- # Detailed default security context for logGroomerSidecar for container
level
+ # Detailed default security context for `logGroomerSidecar` for container
level
securityContexts:
container: {}
diff --git a/helm-tests/tests/chart_utils/log_groomer.py
b/helm-tests/tests/chart_utils/log_groomer.py
index 4d88c7da947..36a2c1eb7ae 100644
--- a/helm-tests/tests/chart_utils/log_groomer.py
+++ b/helm-tests/tests/chart_utils/log_groomer.py
@@ -26,18 +26,27 @@ class LogGroomerTestBase:
obj_name: str = ""
folder: str = ""
+ def get_show_only(self):
+ if self.obj_name == "workers-celery":
+ return [f"templates/{self.folder}/worker-deployment.yaml"]
+
+ return [f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
+
def test_log_groomer_collector_default_enabled(self):
if self.obj_name == "dag-processor":
values = {"dagProcessor": {"enabled": True}}
else:
values = None
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
+ if self.obj_name == "workers-celery":
+ container_name = "worker-log-groomer"
+ else:
+ container_name = f"{self.obj_name}-log-groomer"
+
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert len(jmespath.search("spec.template.spec.containers", docs[0]))
== 2
- assert f"{self.obj_name}-log-groomer" in [
+ assert container_name in [
c["name"] for c in
jmespath.search("spec.template.spec.containers", docs[0])
]
@@ -49,14 +58,18 @@ class LogGroomerTestBase:
"logGroomerSidecar": {"enabled": False},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {
+ "workers": {
+ "celery": {
+ "logGroomerSidecar": {"enabled": False},
+ }
+ }
+ }
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"enabled":
False}}}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
-
+ docs = render_chart(values=values, show_only=self.get_show_only())
actual = jmespath.search("spec.template.spec.containers", docs[0])
assert len(actual) == 1
@@ -67,9 +80,7 @@ class LogGroomerTestBase:
else:
values = None
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert jmespath.search("spec.template.spec.containers[1].command",
docs[0]) is None
assert jmespath.search("spec.template.spec.containers[1].args",
docs[0]) == ["bash", "/clean-logs"]
@@ -80,9 +91,7 @@ class LogGroomerTestBase:
else:
values = None
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert {"name": "AIRFLOW__LOG_RETENTION_DAYS", "value": "15"} in
jmespath.search(
"spec.template.spec.containers[1].env", docs[0]
@@ -96,6 +105,8 @@ class LogGroomerTestBase:
if self.obj_name == "dag-processor":
values = {"dagProcessor": {"enabled": True, "logGroomerSidecar":
{"env": env}}}
+ elif self.obj_name == "workers-celery":
+ values = {"workers": {"celery": {"logGroomerSidecar": {"env":
env}}}}
else:
values = {
"workers": {"logGroomerSidecar": {"env": env}},
@@ -103,9 +114,7 @@ class LogGroomerTestBase:
"triggerer": {"logGroomerSidecar": {"env": env}},
}
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert {"name": "APP_RELEASE_NAME", "value": "release-name-airflow"}
in jmespath.search(
"spec.template.spec.containers[1].env", docs[0]
@@ -124,16 +133,22 @@ class LogGroomerTestBase:
"logGroomerSidecar": {"command": command, "args": args},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {"workers": {"celery": {"logGroomerSidecar": {"command":
command, "args": args}}}}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"command":
command, "args": args}}}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert command ==
jmespath.search("spec.template.spec.containers[1].command", docs[0])
- assert args ==
jmespath.search("spec.template.spec.containers[1].args", docs[0])
+
+ if self.obj_name == "workers-celery" and args is None:
+ assert jmespath.search("spec.template.spec.containers[1].args",
docs[0]) == [
+ "bash",
+ "/clean-logs",
+ ]
+ else:
+ assert args ==
jmespath.search("spec.template.spec.containers[1].args", docs[0])
def test_log_groomer_command_and_args_overrides_are_templated(self):
if self.obj_name == "dag-processor":
@@ -146,6 +161,17 @@ class LogGroomerTestBase:
},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {
+ "workers": {
+ "celery": {
+ "logGroomerSidecar": {
+ "command": ["{{ .Release.Name }}"],
+ "args": ["{{ .Release.Service }}"],
+ }
+ }
+ }
+ }
else:
values = {
f"{self.folder}": {
@@ -156,10 +182,7 @@ class LogGroomerTestBase:
}
}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert jmespath.search("spec.template.spec.containers[1].command",
docs[0]) == ["release-name"]
assert jmespath.search("spec.template.spec.containers[1].args",
docs[0]) == ["Helm"]
@@ -170,13 +193,12 @@ class LogGroomerTestBase:
values = {
"dagProcessor": {"enabled": True, "logGroomerSidecar":
{"retentionDays": retention_days}}
}
+ elif self.obj_name == "workers-celery":
+ values = {"workers": {"celery": {"logGroomerSidecar":
{"retentionDays": retention_days}}}}
else:
values = {f"{self.folder}": {"logGroomerSidecar":
{"retentionDays": retention_days}}}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
if retention_result:
assert (
@@ -186,6 +208,15 @@ class LogGroomerTestBase:
)
== retention_result
)
+ elif self.obj_name == "workers-celery" and retention_result is None:
+ # Testing backward compatibility of move from workers to
workers.celery
+ assert (
+ jmespath.search(
+
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value
| [0]",
+ docs[0],
+ )
+ == "15"
+ )
else:
assert len(jmespath.search("spec.template.spec.containers[1].env",
docs[0])) == 2
@@ -198,13 +229,12 @@ class LogGroomerTestBase:
"logGroomerSidecar": {"frequencyMinutes":
frequency_minutes},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {"workers": {"celery": {"logGroomerSidecar":
{"frequencyMinutes": frequency_minutes}}}}
else:
values = {f"{self.folder}": {"logGroomerSidecar":
{"frequencyMinutes": frequency_minutes}}}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
if frequency_result:
assert (
@@ -214,6 +244,15 @@ class LogGroomerTestBase:
)
== frequency_result
)
+ elif self.obj_name == "workers-celery" and frequency_result is None:
+ # Testing backward compatibility of move from workers to
workers.celery
+ assert (
+ jmespath.search(
+
"spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value
| [0]",
+ docs[0],
+ )
+ == "15"
+ )
else:
assert len(jmespath.search("spec.template.spec.containers[1].env",
docs[0])) == 2
@@ -228,13 +267,12 @@ class LogGroomerTestBase:
"logGroomerSidecar": {"maxSizeBytes": max_size_bytes},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {"workers": {"celery": {"logGroomerSidecar":
{"maxSizeBytes": max_size_bytes}}}}
else:
values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes":
max_size_bytes}}}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
if max_size_result:
assert (
@@ -262,13 +300,12 @@ class LogGroomerTestBase:
"logGroomerSidecar": {"maxSizePercent": max_size_percent},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {"workers": {"celery": {"logGroomerSidecar":
{"maxSizePercent": max_size_percent}}}}
else:
values = {f"{self.folder}": {"logGroomerSidecar":
{"maxSizePercent": max_size_percent}}}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
if max_size_result:
assert (
@@ -300,6 +337,19 @@ class LogGroomerTestBase:
},
}
}
+ elif self.obj_name == "workers-celery":
+ values = {
+ "workers": {
+ "celery": {
+ "logGroomerSidecar": {
+ "resources": {
+ "requests": {"memory": "2Gi", "cpu": "1"},
+ "limits": {"memory": "3Gi", "cpu": "2"},
+ }
+ }
+ }
+ }
+ }
else:
values = {
f"{self.folder}": {
@@ -312,10 +362,7 @@ class LogGroomerTestBase:
}
}
- docs = render_chart(
- values=values,
-
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"],
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert jmespath.search("spec.template.spec.containers[1].resources",
docs[0]) == {
"limits": {
@@ -334,9 +381,7 @@ class LogGroomerTestBase:
else:
values = None
- docs = render_chart(
- values=values,
show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"]
- )
+ docs = render_chart(values=values, show_only=self.get_show_only())
assert (
jmespath.search("spec.template.spec.containers[1].env[?name=='AIRFLOW_HOME'].name
| [0]", docs[0])
diff --git
a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
index 04bef2de7df..f701e3f9ba8 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
@@ -265,25 +265,78 @@ class TestContainerLifecycleHooks:
# Test container lifecycle hooks for log-groomer-sidecar main container
@pytest.mark.parametrize("hook_type", ["preStop", "postStart"])
- def test_log_groomer_sidecar_container_setting(self, hook_type):
+ def test_log_groomer_sidecar_container_setting_scheduler(self, hook_type):
docs = render_chart(
name=RELEASE_NAME,
values={
"scheduler": {
"logGroomerSidecar": {"containerLifecycleHooks":
{hook_type: LIFECYCLE_TEMPLATE}}
},
- "workers": {
- "logGroomerSidecar": {"containerLifecycleHooks":
{hook_type: LIFECYCLE_TEMPLATE}}
- },
},
show_only=[
"templates/scheduler/scheduler-deployment.yaml",
+ ],
+ )
+
+ assert (
+
jmespath.search(f"spec.template.spec.containers[1].lifecycle.{hook_type}",
docs[0])
+ == LIFECYCLE_PARSED
+ )
+
+ @pytest.mark.parametrize(
+ ("workers_values", "expected"),
+ [
+ (
+ {"logGroomerSidecar": {"containerLifecycleHooks": {"preStop":
LIFECYCLE_TEMPLATE}}},
+ {"preStop": LIFECYCLE_PARSED},
+ ),
+ (
+ {"logGroomerSidecar": {"containerLifecycleHooks":
{"postStart": LIFECYCLE_TEMPLATE}}},
+ {"postStart": LIFECYCLE_PARSED},
+ ),
+ (
+ {
+ "celery": {
+ "logGroomerSidecar": {"containerLifecycleHooks":
{"preStop": LIFECYCLE_TEMPLATE}}
+ }
+ },
+ {"preStop": LIFECYCLE_PARSED},
+ ),
+ (
+ {
+ "celery": {
+ "logGroomerSidecar": {"containerLifecycleHooks":
{"postStart": LIFECYCLE_TEMPLATE}}
+ }
+ },
+ {"postStart": LIFECYCLE_PARSED},
+ ),
+ (
+ {
+ "logGroomerSidecar": {"containerLifecycleHooks":
{"postStart": LIFECYCLE_TEMPLATE}},
+ "celery": {
+ "logGroomerSidecar": {"containerLifecycleHooks":
{"preStop": LIFECYCLE_TEMPLATE}}
+ },
+ },
+ {"preStop": LIFECYCLE_PARSED},
+ ),
+ (
+ {
+ "logGroomerSidecar": {"containerLifecycleHooks":
{"preStop": LIFECYCLE_TEMPLATE}},
+ "celery": {
+ "logGroomerSidecar": {"containerLifecycleHooks":
{"postStart": LIFECYCLE_TEMPLATE}}
+ },
+ },
+ {"postStart": LIFECYCLE_PARSED},
+ ),
+ ],
+ )
+ def test_log_groomer_sidecar_container_setting(self, workers_values,
expected):
+ docs = render_chart(
+ name=RELEASE_NAME,
+ values={"workers": workers_values},
+ show_only=[
"templates/workers/worker-deployment.yaml",
],
)
- for doc in docs:
- assert (
-
jmespath.search(f"spec.template.spec.containers[1].lifecycle.{hook_type}", doc)
- == LIFECYCLE_PARSED
- )
+ assert jmespath.search("spec.template.spec.containers[1].lifecycle",
docs[0]) == expected
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
index 38bba315458..376a2d30c08 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
@@ -2290,6 +2290,13 @@ class TestWorkerLogGroomer(LogGroomerTestBase):
folder = "workers"
+class TestWorkerCeleryLogGroomer(LogGroomerTestBase):
+ """Worker Celery groomer."""
+
+ obj_name = "workers-celery"
+ folder = "workers"
+
+
class TestWorkerKedaAutoScaler:
"""Tests worker keda auto scaler."""
diff --git a/helm-tests/tests/helm_tests/security/test_security_context.py
b/helm-tests/tests/helm_tests/security/test_security_context.py
index a5e9d70816b..39e9281d04b 100644
--- a/helm-tests/tests/helm_tests/security/test_security_context.py
+++ b/helm-tests/tests/helm_tests/security/test_security_context.py
@@ -554,13 +554,34 @@ class TestSecurityContext:
assert ctx_value ==
jmespath.search("spec.template.spec.containers[0].securityContext", doc)
# Test securityContexts for log-groomer-sidecar main container
- def test_log_groomer_sidecar_container_setting(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {"logGroomerSidecar": {"securityContexts": {"container":
{"allowPrivilegeEscalation": False}}}},
+ {
+ "celery": {
+ "logGroomerSidecar": {
+ "securityContexts": {"container":
{"allowPrivilegeEscalation": False}}
+ }
+ }
+ },
+ {
+ "logGroomerSidecar": {"securityContexts": {"container":
{"runAsUser": 20}}},
+ "celery": {
+ "logGroomerSidecar": {
+ "securityContexts": {"container":
{"allowPrivilegeEscalation": False}}
+ }
+ },
+ },
+ ],
+ )
+ def test_log_groomer_sidecar_container_setting(self, workers_values):
ctx_value = {"allowPrivilegeEscalation": False}
spec = {"logGroomerSidecar": {"securityContexts": {"container":
ctx_value}}}
docs = render_chart(
values={
"scheduler": spec,
- "workers": spec,
+ "workers": workers_values,
"dagProcessor": spec,
"triggerer": spec,
},
