Merge components in packages (#3555)

Co-authored-by: Paul Monigatti <pm@paul.pm>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

esphome/components/packages/__init__.py

@@ -165,7 +165,7 @@ def do_packages_pass(config: dict):
                 f"Packages must be a key to value mapping, got {type(packages)} instead"
             )
 
-        for package_name, package_config in packages.items():
+        for package_name, package_config in reversed(packages.items()):
             with cv.prepend_path(package_name):
                 recursive_package = package_config
                 if CONF_URL in package_config:

esphome/config.py

@@ -14,6 +14,7 @@ from esphome import core, yaml_util, loader
 import esphome.core.config as core_config
 from esphome.const import (
     CONF_ESPHOME,
+    CONF_ID,
     CONF_PLATFORM,
     CONF_PACKAGES,
     CONF_SUBSTITUTIONS,
@@ -24,6 +25,7 @@ from esphome.core import CORE, EsphomeError
 from esphome.helpers import indent
 from esphome.util import safe_print, OrderedDict
 
+from esphome.config_helpers import Extend
 from esphome.loader import get_component, get_platform, ComponentManifest
 from esphome.yaml_util import is_secret, ESPHomeDataBase, ESPForceValue
 from esphome.voluptuous_schema import ExtraKeysInvalid
@@ -334,6 +336,13 @@ class LoadValidationStep(ConfigValidationStep):
                 continue
             p_name = p_config.get("platform")
             if p_name is None:
+                p_id = p_config.get(CONF_ID)
+                if isinstance(p_id, Extend):
+                    result.add_str_error(
+                        f"Source for extension of ID '{p_id.value}' was not found.",
+                        path + [CONF_ID],
+                    )
+                    continue
                 result.add_str_error("No platform specified! See 'platform' key.", path)
                 continue
             # Remove temp output path and construct new one

esphome/config_helpers.py

@@ -1,10 +1,27 @@
 import json
 import os
 
+from esphome.const import CONF_ID
 from esphome.core import CORE
 from esphome.helpers import read_file
 
 
+class Extend:
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return f"!extend {self.value}"
+
+    def __eq__(self, b):
+        """
+        Check if two Extend objects contain the same ID.
+
+        Only used in unit tests.
+        """
+        return isinstance(b, Extend) and self.value == b.value
+
+
 def read_config_file(path: str) -> str:
     if CORE.vscode and (
         not CORE.ace or os.path.abspath(path) == os.path.abspath(CORE.config_path)
@@ -36,7 +53,25 @@ def merge_config(full_old, full_new):
         if isinstance(new, list):
             if not isinstance(old, list):
                 return new
-            return old + new
+            res = old.copy()
+            ids = {
+                v[CONF_ID]: i
+                for i, v in enumerate(res)
+                if CONF_ID in v and isinstance(v[CONF_ID], str)
+            }
+            for v in new:
+                if CONF_ID in v:
+                    new_id = v[CONF_ID]
+                    if isinstance(new_id, Extend):
+                        new_id = new_id.value
+                        if new_id in ids:
+                            v[CONF_ID] = new_id
+                            res[ids[new_id]] = merge(res[ids[new_id]], v)
+                            continue
+                    else:
+                        ids[new_id] = len(res)
+                res.append(v)
+            return res
         if new is None:
             return old
 

esphome/config_validation.py

@@ -13,6 +13,7 @@ import voluptuous as vol
 
 from esphome import core
 import esphome.codegen as cg
+from esphome.config_helpers import Extend
 from esphome.const import (
     ALLOWED_NAME_CHARS,
     CONF_AVAILABILITY,
@@ -490,6 +491,8 @@ def declare_id(type):
         if value is None:
             return core.ID(None, is_declaration=True, type=type)
 
+        if isinstance(value, Extend):
+            raise Invalid(f"Source for extension of ID '{value.value}' was not found.")
         return core.ID(validate_id_name(value), is_declaration=True, type=type)
 
     return validator

esphome/yaml_util.py

@@ -10,7 +10,7 @@ import yaml
 import yaml.constructor
 
 from esphome import core
-from esphome.config_helpers import read_config_file
+from esphome.config_helpers import read_config_file, Extend
 from esphome.core import (
     EsphomeError,
     IPAddress,
@@ -338,6 +338,10 @@ class ESPHomeLoader(yaml.SafeLoader):
         obj = self.construct_scalar(node)
         return add_class_to_obj(obj, ESPForceValue)
 
+    @_add_data_ref
+    def construct_extend(self, node):
+        return Extend(str(node.value))
+
 
 ESPHomeLoader.add_constructor("tag:yaml.org,2002:int", ESPHomeLoader.construct_yaml_int)
 ESPHomeLoader.add_constructor(
@@ -369,6 +373,7 @@ ESPHomeLoader.add_constructor(
 )
 ESPHomeLoader.add_constructor("!lambda", ESPHomeLoader.construct_lambda)
 ESPHomeLoader.add_constructor("!force", ESPHomeLoader.construct_force)
+ESPHomeLoader.add_constructor("!extend", ESPHomeLoader.construct_extend)
 
 
 def load_yaml(fname, clear_secrets=True):

tests/component_tests/packages/test_packages.py

@@ -0,0 +1,351 @@
+"""Tests for the packages component."""
+
+import pytest
+
+
+from esphome.const import (
+    CONF_DOMAIN,
+    CONF_ESPHOME,
+    CONF_FILTERS,
+    CONF_ID,
+    CONF_MULTIPLY,
+    CONF_NAME,
+    CONF_OFFSET,
+    CONF_PACKAGES,
+    CONF_PASSWORD,
+    CONF_PLATFORM,
+    CONF_SENSOR,
+    CONF_SSID,
+    CONF_UPDATE_INTERVAL,
+    CONF_WIFI,
+)
+from esphome.components.packages import do_packages_pass
+from esphome.config_helpers import Extend
+import esphome.config_validation as cv
+
+# Test strings
+TEST_DEVICE_NAME = "test_device_name"
+TEST_PLATFORM = "test_platform"
+TEST_WIFI_SSID = "test_wifi_ssid"
+TEST_PACKAGE_WIFI_SSID = "test_package_wifi_ssid"
+TEST_PACKAGE_WIFI_PASSWORD = "test_package_wifi_password"
+TEST_DOMAIN = "test_domain_name"
+TEST_SENSOR_PLATFORM_1 = "test_sensor_platform_1"
+TEST_SENSOR_PLATFORM_2 = "test_sensor_platform_2"
+TEST_SENSOR_NAME_1 = "test_sensor_name_1"
+TEST_SENSOR_NAME_2 = "test_sensor_name_2"
+TEST_SENSOR_ID_1 = "test_sensor_id_1"
+TEST_SENSOR_ID_2 = "test_sensor_id_2"
+TEST_SENSOR_UPDATE_INTERVAL = "test_sensor_update_interval"
+
+
+@pytest.fixture(name="basic_wifi")
+def fixture_basic_wifi():
+    return {
+        CONF_SSID: TEST_PACKAGE_WIFI_SSID,
+        CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD,
+    }
+
+
+@pytest.fixture(name="basic_esphome")
+def fixture_basic_esphome():
+    return {CONF_NAME: TEST_DEVICE_NAME, CONF_PLATFORM: TEST_PLATFORM}
+
+
+def test_package_unused(basic_esphome, basic_wifi):
+    """
+    Ensures do_package_pass does not change a config if packages aren't used.
+    """
+    config = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi}
+
+    actual = do_packages_pass(config)
+    assert actual == config
+
+
+def test_package_invalid_dict(basic_esphome, basic_wifi):
+    """
+    Ensures an error is raised if packages is not valid.
+
+    """
+    config = {CONF_ESPHOME: basic_esphome, CONF_PACKAGES: basic_wifi}
+
+    with pytest.raises(cv.Invalid):
+        do_packages_pass(config)
+
+
+def test_package_include(basic_wifi, basic_esphome):
+    """
+    Tests the simple case where an independent config present in a package is added to the top-level config as is.
+
+    In this test, the CONF_WIFI config is expected to be simply added to the top-level config.
+    """
+    config = {
+        CONF_ESPHOME: basic_esphome,
+        CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}},
+    }
+
+    expected = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi}
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_package_append(basic_wifi, basic_esphome):
+    """
+    Tests the case where a key is present in both a package and top-level config.
+
+    In this test, CONF_WIFI is defined in a package, and CONF_DOMAIN is added to it at the top level.
+    """
+    config = {
+        CONF_ESPHOME: basic_esphome,
+        CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}},
+        CONF_WIFI: {CONF_DOMAIN: TEST_DOMAIN},
+    }
+
+    expected = {
+        CONF_ESPHOME: basic_esphome,
+        CONF_WIFI: {
+            CONF_SSID: TEST_PACKAGE_WIFI_SSID,
+            CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD,
+            CONF_DOMAIN: TEST_DOMAIN,
+        },
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_package_override(basic_wifi, basic_esphome):
+    """
+    Ensures that the top-level configuration takes precedence over duplicate keys defined in a package.
+
+    In this test, CONF_SSID should be overwritten by that defined in the top-level config.
+    """
+    config = {
+        CONF_ESPHOME: basic_esphome,
+        CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}},
+        CONF_WIFI: {CONF_SSID: TEST_WIFI_SSID},
+    }
+
+    expected = {
+        CONF_ESPHOME: basic_esphome,
+        CONF_WIFI: {
+            CONF_SSID: TEST_WIFI_SSID,
+            CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD,
+        },
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_multiple_package_order():
+    """
+    Ensures that mutiple packages are merged in order.
+    """
+    config = {
+        CONF_PACKAGES: {
+            "package1": {
+                "logger": {
+                    "level": "DEBUG",
+                },
+            },
+            "package2": {
+                "logger": {
+                    "level": "VERBOSE",
+                },
+            },
+        },
+    }
+
+    expected = {
+        "logger": {
+            "level": "VERBOSE",
+        },
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_package_list_merge():
+    """
+    Ensures lists defined in both a package and the top-level config are merged correctly
+    """
+    config = {
+        CONF_PACKAGES: {
+            "package_sensors": {
+                CONF_SENSOR: [
+                    {
+                        CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
+                        CONF_NAME: TEST_SENSOR_NAME_1,
+                    },
+                    {
+                        CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
+                        CONF_NAME: TEST_SENSOR_NAME_2,
+                    },
+                ]
+            }
+        },
+        CONF_SENSOR: [
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_1},
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
+        ],
+    }
+
+    expected = {
+        CONF_SENSOR: [
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, CONF_NAME: TEST_SENSOR_NAME_1},
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, CONF_NAME: TEST_SENSOR_NAME_2},
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_1},
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
+        ]
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_package_list_merge_by_id():
+    """
+    Ensures that components with matching IDs are merged correctly.
+
+    In this test, a sensor is defined in a package, and a CONF_UPDATE_INTERVAL is added at the top level,
+    and a sensor name is overridden in another sensor.
+    """
+    config = {
+        CONF_PACKAGES: {
+            "package_sensors": {
+                CONF_SENSOR: [
+                    {
+                        CONF_ID: TEST_SENSOR_ID_1,
+                        CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
+                        CONF_NAME: TEST_SENSOR_NAME_1,
+                    },
+                    {
+                        CONF_ID: TEST_SENSOR_ID_2,
+                        CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
+                        CONF_NAME: TEST_SENSOR_NAME_2,
+                    },
+                ]
+            },
+            "package2": {
+                CONF_SENSOR: [
+                    {
+                        CONF_ID: Extend(TEST_SENSOR_ID_1),
+                        CONF_DOMAIN: "2",
+                    }
+                ],
+            },
+            "package3": {
+                CONF_SENSOR: [
+                    {
+                        CONF_ID: Extend(TEST_SENSOR_ID_1),
+                        CONF_DOMAIN: "3",
+                    }
+                ],
+            },
+        },
+        CONF_SENSOR: [
+            {
+                CONF_ID: Extend(TEST_SENSOR_ID_1),
+                CONF_UPDATE_INTERVAL: TEST_SENSOR_UPDATE_INTERVAL,
+            },
+            {CONF_ID: Extend(TEST_SENSOR_ID_2), CONF_NAME: TEST_SENSOR_NAME_1},
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
+        ],
+    }
+
+    expected = {
+        CONF_SENSOR: [
+            {
+                CONF_ID: TEST_SENSOR_ID_1,
+                CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
+                CONF_NAME: TEST_SENSOR_NAME_1,
+                CONF_UPDATE_INTERVAL: TEST_SENSOR_UPDATE_INTERVAL,
+                CONF_DOMAIN: "3",
+            },
+            {
+                CONF_ID: TEST_SENSOR_ID_2,
+                CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
+                CONF_NAME: TEST_SENSOR_NAME_1,
+            },
+            {CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
+        ]
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_package_merge_by_id_with_list():
+    """
+    Ensures that components with matching IDs are merged correctly when their configuration contains lists.
+
+    For example, a sensor with filters defined in both a package and the top level config should be merged.
+    """
+
+    config = {
+        CONF_PACKAGES: {
+            "sensors": {
+                CONF_SENSOR: [
+                    {CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 42.0}]}
+                ]
+            }
+        },
+        CONF_SENSOR: [
+            {CONF_ID: Extend(TEST_SENSOR_ID_1), CONF_FILTERS: [{CONF_OFFSET: 146.0}]}
+        ],
+    }
+
+    expected = {
+        CONF_SENSOR: [
+            {
+                CONF_ID: TEST_SENSOR_ID_1,
+                CONF_FILTERS: [{CONF_MULTIPLY: 42.0}, {CONF_OFFSET: 146.0}],
+            }
+        ]
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected
+
+
+def test_package_merge_by_missing_id():
+    """
+    Ensures that components with missing IDs are not merged.
+    """
+
+    config = {
+        CONF_PACKAGES: {
+            "sensors": {
+                CONF_SENSOR: [
+                    {CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 42.0}]},
+                ]
+            }
+        },
+        CONF_SENSOR: [
+            {CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 10.0}]},
+            {CONF_ID: Extend(TEST_SENSOR_ID_2), CONF_FILTERS: [{CONF_OFFSET: 146.0}]},
+        ],
+    }
+
+    expected = {
+        CONF_SENSOR: [
+            {
+                CONF_ID: TEST_SENSOR_ID_1,
+                CONF_FILTERS: [{CONF_MULTIPLY: 42.0}],
+            },
+            {
+                CONF_ID: TEST_SENSOR_ID_1,
+                CONF_FILTERS: [{CONF_MULTIPLY: 10.0}],
+            },
+            {
+                CONF_ID: Extend(TEST_SENSOR_ID_2),
+                CONF_FILTERS: [{CONF_OFFSET: 146.0}],
+            },
+        ]
+    }
+
+    actual = do_packages_pass(config)
+    assert actual == expected

tests/test1.yaml

@@ -1076,6 +1076,8 @@ sensor:
     id: ultrasonic_sensor1
   - platform: uptime
     name: Uptime Sensor
+  - id: !extend ${devicename}_uptime_pcg
+    unit_of_measurement: s
   - platform: wifi_signal
     name: WiFi Signal Sensor
     update_interval: 15s