This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8d7c822ce7f Use standalone DAG processor in standalone command (#45698)
8d7c822ce7f is described below
commit 8d7c822ce7f1d5ce8b154b6370bbfd8cd2ca3827
Author: Jed Cunningham <[email protected]>
AuthorDate: Thu Jan 16 09:06:04 2025 -0700
Use standalone DAG processor in standalone command (#45698)
We are moving to only support a standalone DAG processor in Airflow 3,
so let's switch the standalone command to use it.
---
airflow/cli/commands/local_commands/standalone_command.py | 12 +++++++++++-
docs/apache-airflow/start.rst | 4 ++++
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/airflow/cli/commands/local_commands/standalone_command.py
b/airflow/cli/commands/local_commands/standalone_command.py
index f248990a3dd..6a3e658c149 100644
--- a/airflow/cli/commands/local_commands/standalone_command.py
+++ b/airflow/cli/commands/local_commands/standalone_command.py
@@ -30,6 +30,7 @@ from termcolor import colored
from airflow.configuration import conf
from airflow.executors import executor_constants
from airflow.executors.executor_loader import ExecutorLoader
+from airflow.jobs.dag_processor_job_runner import DagProcessorJobRunner
from airflow.jobs.job import most_recent_job
from airflow.jobs.scheduler_job_runner import SchedulerJobRunner
from airflow.jobs.triggerer_job_runner import TriggererJobRunner
@@ -76,6 +77,12 @@ class StandaloneCommand:
command=["scheduler"],
env=env,
)
+ self.subcommands["dag-processor"] = SubCommand(
+ self,
+ name="dag-processor",
+ command=["dag-processor"],
+ env=env,
+ )
self.subcommands["webserver"] = SubCommand(
self,
name="webserver",
@@ -147,6 +154,7 @@ class StandaloneCommand:
"fastapi-api": "magenta",
"webserver": "green",
"scheduler": "blue",
+ "dag-processor": "yellow",
"triggerer": "cyan",
"standalone": "white",
}
@@ -169,6 +177,7 @@ class StandaloneCommand:
We override some settings as part of being standalone.
"""
env = dict(os.environ)
+ env["AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR"] = "True"
# Make sure we're using a local executor flavour
executor_class, _ = ExecutorLoader.import_default_executor_cls()
@@ -205,6 +214,7 @@ class StandaloneCommand:
return (
self.port_open(self.web_server_port)
and self.job_running(SchedulerJobRunner)
+ and self.job_running(DagProcessorJobRunner)
and self.job_running(TriggererJobRunner)
)
@@ -228,7 +238,7 @@ class StandaloneCommand:
"""
Check if the given job name is running and heartbeating correctly.
- Used to tell if scheduler is alive.
+ Used to tell if a component is alive.
"""
recent = most_recent_job(job_runner_class.job_type)
if not recent:
diff --git a/docs/apache-airflow/start.rst b/docs/apache-airflow/start.rst
index df263263f1c..ea8d624d74d 100644
--- a/docs/apache-airflow/start.rst
+++ b/docs/apache-airflow/start.rst
@@ -132,6 +132,10 @@ the all-in-one ``standalone`` command, you can instead run:
airflow scheduler
+ airflow dag-processor
+
+ airflow triggerer
+
What's Next?
''''''''''''
From this point, you can head to the :doc:`/tutorial/index` section for
further examples or the :doc:`/howto/index` section if you're ready to get your
hands dirty.
