From 5aa42e5e66a5a65528e3dd6218e17b59970dfc82 Mon Sep 17 00:00:00 2001 From: jimtng <2554958+jimtng@users.noreply.github.com> Date: Tue, 31 May 2022 14:45:18 +1000 Subject: [PATCH] Add variable substitutions for !include (#3510) --- esphome/components/substitutions/__init__.py | 33 +++++++------- esphome/yaml_util.py | 44 ++++++++++++++++++- .../fixtures/yaml_util/includes/included.yaml | 2 + .../fixtures/yaml_util/includes/list.yaml | 2 + .../fixtures/yaml_util/includes/scalar.yaml | 1 + .../fixtures/yaml_util/includetest.yaml | 17 +++++++ tests/unit_tests/test_yaml_util.py | 13 ++++++ 7 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 tests/unit_tests/fixtures/yaml_util/includes/included.yaml create mode 100644 tests/unit_tests/fixtures/yaml_util/includes/list.yaml create mode 100644 tests/unit_tests/fixtures/yaml_util/includes/scalar.yaml create mode 100644 tests/unit_tests/fixtures/yaml_util/includetest.yaml create mode 100644 tests/unit_tests/test_yaml_util.py diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 6188b14b35..5a3da1abbe 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -48,7 +48,7 @@ VARIABLE_PROG = re.compile( ) -def _expand_substitutions(substitutions, value, path): +def _expand_substitutions(substitutions, value, path, ignore_missing): if "$" not in value: return value @@ -66,13 +66,14 @@ def _expand_substitutions(substitutions, value, path): if name.startswith("{") and name.endswith("}"): name = name[1:-1] if name not in substitutions: - _LOGGER.warning( - "Found '%s' (see %s) which looks like a substitution, but '%s' was " - "not declared", - orig_value, - "->".join(str(x) for x in path), - name, - ) + if not ignore_missing: + _LOGGER.warning( + "Found '%s' (see %s) which looks like a substitution, but '%s' was " + "not declared", + orig_value, + "->".join(str(x) for x in path), + name, + ) i = j continue @@ -92,37 +93,37 @@ def _expand_substitutions(substitutions, value, path): return value -def _substitute_item(substitutions, item, path): +def _substitute_item(substitutions, item, path, ignore_missing): if isinstance(item, list): for i, it in enumerate(item): - sub = _substitute_item(substitutions, it, path + [i]) + sub = _substitute_item(substitutions, it, path + [i], ignore_missing) if sub is not None: item[i] = sub elif isinstance(item, dict): replace_keys = [] for k, v in item.items(): if path or k != CONF_SUBSTITUTIONS: - sub = _substitute_item(substitutions, k, path + [k]) + sub = _substitute_item(substitutions, k, path + [k], ignore_missing) if sub is not None: replace_keys.append((k, sub)) - sub = _substitute_item(substitutions, v, path + [k]) + sub = _substitute_item(substitutions, v, path + [k], ignore_missing) if sub is not None: item[k] = sub for old, new in replace_keys: item[new] = merge_config(item.get(old), item.get(new)) del item[old] elif isinstance(item, str): - sub = _expand_substitutions(substitutions, item, path) + sub = _expand_substitutions(substitutions, item, path, ignore_missing) if sub != item: return sub elif isinstance(item, core.Lambda): - sub = _expand_substitutions(substitutions, item.value, path) + sub = _expand_substitutions(substitutions, item.value, path, ignore_missing) if sub != item: item.value = sub return None -def do_substitution_pass(config, command_line_substitutions): +def do_substitution_pass(config, command_line_substitutions, ignore_missing=False): if CONF_SUBSTITUTIONS not in config and not command_line_substitutions: return @@ -151,4 +152,4 @@ def do_substitution_pass(config, command_line_substitutions): config[CONF_SUBSTITUTIONS] = substitutions # Move substitutions to the first place to replace substitutions in them correctly config.move_to_end(CONF_SUBSTITUTIONS, False) - _substitute_item(substitutions, config, []) + _substitute_item(substitutions, config, [], ignore_missing) diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 57009be57e..75aec0edc8 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -251,7 +251,49 @@ class ESPHomeLoader(yaml.SafeLoader): # pylint: disable=too-many-ancestors @_add_data_ref def construct_include(self, node): - return _load_yaml_internal(self._rel_path(node.value)) + def extract_file_vars(node): + fields = self.construct_yaml_map(node) + file = fields.get("file") + if file is None: + raise yaml.MarkedYAMLError("Must include 'file'", node.start_mark) + vars = fields.get("vars") + if vars: + vars = {k: str(v) for k, v in vars.items()} + return file, vars + + def substitute_vars(config, vars): + from esphome.const import CONF_SUBSTITUTIONS + from esphome.components import substitutions + + org_subs = None + result = config + if not isinstance(config, dict): + # when the included yaml contains a list or a scalar + # wrap it into an OrderedDict because do_substitution_pass expects it + result = OrderedDict([("yaml", config)]) + elif CONF_SUBSTITUTIONS in result: + org_subs = result.pop(CONF_SUBSTITUTIONS) + + result[CONF_SUBSTITUTIONS] = vars + # Ignore missing vars that refer to the top level substitutions + substitutions.do_substitution_pass(result, None, ignore_missing=True) + result.pop(CONF_SUBSTITUTIONS) + + if not isinstance(config, dict): + result = result["yaml"] # unwrap the result + elif org_subs: + result[CONF_SUBSTITUTIONS] = org_subs + return result + + if isinstance(node, yaml.nodes.MappingNode): + file, vars = extract_file_vars(node) + else: + file, vars = node.value, None + + result = _load_yaml_internal(self._rel_path(file)) + if vars: + result = substitute_vars(result, vars) + return result @_add_data_ref def construct_include_dir_list(self, node): diff --git a/tests/unit_tests/fixtures/yaml_util/includes/included.yaml b/tests/unit_tests/fixtures/yaml_util/includes/included.yaml new file mode 100644 index 0000000000..e9fca324a3 --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includes/included.yaml @@ -0,0 +1,2 @@ +--- +ssid: ${name} diff --git a/tests/unit_tests/fixtures/yaml_util/includes/list.yaml b/tests/unit_tests/fixtures/yaml_util/includes/list.yaml new file mode 100644 index 0000000000..2fb3838631 --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includes/list.yaml @@ -0,0 +1,2 @@ +--- +- ${var1} diff --git a/tests/unit_tests/fixtures/yaml_util/includes/scalar.yaml b/tests/unit_tests/fixtures/yaml_util/includes/scalar.yaml new file mode 100644 index 0000000000..ddd2156b5e --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includes/scalar.yaml @@ -0,0 +1 @@ +${var1} diff --git a/tests/unit_tests/fixtures/yaml_util/includetest.yaml b/tests/unit_tests/fixtures/yaml_util/includetest.yaml new file mode 100644 index 0000000000..959283df60 --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includetest.yaml @@ -0,0 +1,17 @@ +--- +substitutions: + name: original + +wifi: !include + file: includes/included.yaml + vars: + name: my_custom_ssid + +esphome: + # should be substituted as 'original', not overwritten by vars in the !include above + name: ${name} + name_add_mac_suffix: true + platform: esp8266 + board: !include { file: includes/scalar.yaml, vars: { var1: nodemcu } } + + libraries: !include { file: includes/list.yaml, vars: { var1: Wire } } diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py new file mode 100644 index 0000000000..8ee991f5b3 --- /dev/null +++ b/tests/unit_tests/test_yaml_util.py @@ -0,0 +1,13 @@ +from esphome import yaml_util +from esphome.components import substitutions + + +def test_include_with_vars(fixture_path): + yaml_file = fixture_path / "yaml_util" / "includetest.yaml" + + actual = yaml_util.load_yaml(yaml_file) + substitutions.do_substitution_pass(actual, None) + assert actual["esphome"]["name"] == "original" + assert actual["esphome"]["libraries"][0] == "Wire" + assert actual["esphome"]["board"] == "nodemcu" + assert actual["wifi"]["ssid"] == "my_custom_ssid"