This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0c43e68 Automatically create section when migrating config (#16814)
0c43e68 is described below
commit 0c43e68b923d1f3712d32338f7235abcd84593bb
Author: Tzu-ping Chung <[email protected]>
AuthorDate: Mon Sep 13 02:01:00 2021 +0800
Automatically create section when migrating config (#16814)
Previously, if a config is migrated to a new section, the migration code
would crash with NoSectionError if the user does not add that section to
airflow.cfg after upgrading Airflow. This patch automatically creates an
empty section when that happens to avoid Airflow from crashing.
---
airflow/configuration.py | 6 ++++--
tests/core/test_configuration.py | 42 +++++++++++++++++++++++-----------------
2 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index c75dcb4..fb2078a 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -215,9 +215,9 @@ class AirflowConfigParser(ConfigParser):
for section, replacement in self.deprecated_values.items():
for name, info in replacement.items():
old, new, version = info
- current_value = self.get(section, name, fallback=None)
+ current_value = self.get(section, name, fallback="")
if self._using_old_value(old, current_value):
- new_value = re.sub(old, new, current_value)
+ new_value = old.sub(new, current_value)
self._update_env_var(section=section, name=name,
new_value=new_value)
self._create_future_warning(
name=name,
@@ -285,6 +285,8 @@ class AirflowConfigParser(ConfigParser):
# would be read and used instead of the value we set
env_var = self._env_var_name(section, name)
os.environ.pop(env_var, None)
+ if not self.has_section(section):
+ self.add_section(section)
self.set(section, name, new_value)
@staticmethod
diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py
index 470e63f..d651de1 100644
--- a/tests/core/test_configuration.py
+++ b/tests/core/test_configuration.py
@@ -51,7 +51,7 @@ from tests.test_utils.reset_warning_registry import
reset_warning_registry
'AIRFLOW__TESTCMDENV__NOTACOMMAND_CMD': 'echo -n "NOT OK"',
},
)
-class TestConf(unittest.TestCase):
+class TestConf:
def test_airflow_home_default(self):
with unittest.mock.patch.dict('os.environ'):
if 'AIRFLOW_HOME' in os.environ:
@@ -530,33 +530,39 @@ AIRFLOW_HOME = /root/airflow
if tmp:
os.environ['AIRFLOW__CELERY__RESULT_BACKEND'] = tmp
- def test_deprecated_values(self):
+ def test_deprecated_values_from_conf(self):
+ test_conf = AirflowConfigParser(default_config='')
+ # Guarantee we have deprecated settings, so we test the deprecation
+ # lookup even if we remove this explicit fallback
+ test_conf.deprecated_values = {
+ 'core': {'hostname_callable': (re.compile(r':'), r'.', '2.1')},
+ }
+ test_conf.read_dict({'core': {'hostname_callable': 'socket:getfqdn'}})
+
+ with pytest.warns(FutureWarning):
+ test_conf.validate()
+ assert test_conf.get('core', 'hostname_callable') ==
'socket.getfqdn'
+
+ @pytest.mark.parametrize(
+ "conf_dict",
+ [
+ {}, # Even if the section is absent from config file, environ
still needs replacing.
+ {'core': {'hostname_callable': 'socket:getfqdn'}},
+ ],
+ )
+ def test_deprecated_values_from_environ(self, conf_dict):
def make_config():
test_conf = AirflowConfigParser(default_config='')
# Guarantee we have a deprecated setting, so we test the
deprecation
# lookup even if we remove this explicit fallback
test_conf.deprecated_values = {
- 'core': {
- 'hostname_callable': (re.compile(r':'), r'.', '2.1'),
- },
+ 'core': {'hostname_callable': (re.compile(r':'), r'.', '2.1')},
}
- test_conf.read_dict(
- {
- 'core': {
- 'executor': 'SequentialExecutor',
- 'sql_alchemy_conn': 'sqlite://',
- 'hostname_callable': 'socket:getfqdn',
- },
- }
- )
+ test_conf.read_dict(conf_dict)
test_conf.validate()
return test_conf
with pytest.warns(FutureWarning):
- test_conf = make_config()
- assert test_conf.get('core', 'hostname_callable') ==
'socket.getfqdn'
-
- with pytest.warns(FutureWarning):
with unittest.mock.patch.dict('os.environ',
AIRFLOW__CORE__HOSTNAME_CALLABLE='socket:getfqdn'):
test_conf = make_config()
assert test_conf.get('core', 'hostname_callable') ==
'socket.getfqdn'
