Merge components in packages (#3555)

Co-authored-by: Paul Monigatti <pm@paul.pm>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
This commit is contained in:
Quentin Smith 2023-02-06 20:08:40 -05:00 committed by GitHub
parent 393ca64d70
commit 40df3aa55e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 408 additions and 3 deletions

View File

@ -165,7 +165,7 @@ def do_packages_pass(config: dict):
f"Packages must be a key to value mapping, got {type(packages)} instead"
)
for package_name, package_config in packages.items():
for package_name, package_config in reversed(packages.items()):
with cv.prepend_path(package_name):
recursive_package = package_config
if CONF_URL in package_config:

View File

@ -14,6 +14,7 @@ from esphome import core, yaml_util, loader
import esphome.core.config as core_config
from esphome.const import (
CONF_ESPHOME,
CONF_ID,
CONF_PLATFORM,
CONF_PACKAGES,
CONF_SUBSTITUTIONS,
@ -24,6 +25,7 @@ from esphome.core import CORE, EsphomeError
from esphome.helpers import indent
from esphome.util import safe_print, OrderedDict
from esphome.config_helpers import Extend
from esphome.loader import get_component, get_platform, ComponentManifest
from esphome.yaml_util import is_secret, ESPHomeDataBase, ESPForceValue
from esphome.voluptuous_schema import ExtraKeysInvalid
@ -334,6 +336,13 @@ class LoadValidationStep(ConfigValidationStep):
continue
p_name = p_config.get("platform")
if p_name is None:
p_id = p_config.get(CONF_ID)
if isinstance(p_id, Extend):
result.add_str_error(
f"Source for extension of ID '{p_id.value}' was not found.",
path + [CONF_ID],
)
continue
result.add_str_error("No platform specified! See 'platform' key.", path)
continue
# Remove temp output path and construct new one

View File

@ -1,10 +1,27 @@
import json
import os
from esphome.const import CONF_ID
from esphome.core import CORE
from esphome.helpers import read_file
class Extend:
def __init__(self, value):
self.value = value
def __str__(self):
return f"!extend {self.value}"
def __eq__(self, b):
"""
Check if two Extend objects contain the same ID.
Only used in unit tests.
"""
return isinstance(b, Extend) and self.value == b.value
def read_config_file(path: str) -> str:
if CORE.vscode and (
not CORE.ace or os.path.abspath(path) == os.path.abspath(CORE.config_path)
@ -36,7 +53,25 @@ def merge_config(full_old, full_new):
if isinstance(new, list):
if not isinstance(old, list):
return new
return old + new
res = old.copy()
ids = {
v[CONF_ID]: i
for i, v in enumerate(res)
if CONF_ID in v and isinstance(v[CONF_ID], str)
}
for v in new:
if CONF_ID in v:
new_id = v[CONF_ID]
if isinstance(new_id, Extend):
new_id = new_id.value
if new_id in ids:
v[CONF_ID] = new_id
res[ids[new_id]] = merge(res[ids[new_id]], v)
continue
else:
ids[new_id] = len(res)
res.append(v)
return res
if new is None:
return old

View File

@ -13,6 +13,7 @@ import voluptuous as vol
from esphome import core
import esphome.codegen as cg
from esphome.config_helpers import Extend
from esphome.const import (
ALLOWED_NAME_CHARS,
CONF_AVAILABILITY,
@ -490,6 +491,8 @@ def declare_id(type):
if value is None:
return core.ID(None, is_declaration=True, type=type)
if isinstance(value, Extend):
raise Invalid(f"Source for extension of ID '{value.value}' was not found.")
return core.ID(validate_id_name(value), is_declaration=True, type=type)
return validator

View File

@ -10,7 +10,7 @@ import yaml
import yaml.constructor
from esphome import core
from esphome.config_helpers import read_config_file
from esphome.config_helpers import read_config_file, Extend
from esphome.core import (
EsphomeError,
IPAddress,
@ -338,6 +338,10 @@ class ESPHomeLoader(yaml.SafeLoader):
obj = self.construct_scalar(node)
return add_class_to_obj(obj, ESPForceValue)
@_add_data_ref
def construct_extend(self, node):
return Extend(str(node.value))
ESPHomeLoader.add_constructor("tag:yaml.org,2002:int", ESPHomeLoader.construct_yaml_int)
ESPHomeLoader.add_constructor(
@ -369,6 +373,7 @@ ESPHomeLoader.add_constructor(
)
ESPHomeLoader.add_constructor("!lambda", ESPHomeLoader.construct_lambda)
ESPHomeLoader.add_constructor("!force", ESPHomeLoader.construct_force)
ESPHomeLoader.add_constructor("!extend", ESPHomeLoader.construct_extend)
def load_yaml(fname, clear_secrets=True):

View File

@ -0,0 +1,351 @@
"""Tests for the packages component."""
import pytest
from esphome.const import (
CONF_DOMAIN,
CONF_ESPHOME,
CONF_FILTERS,
CONF_ID,
CONF_MULTIPLY,
CONF_NAME,
CONF_OFFSET,
CONF_PACKAGES,
CONF_PASSWORD,
CONF_PLATFORM,
CONF_SENSOR,
CONF_SSID,
CONF_UPDATE_INTERVAL,
CONF_WIFI,
)
from esphome.components.packages import do_packages_pass
from esphome.config_helpers import Extend
import esphome.config_validation as cv
# Test strings
TEST_DEVICE_NAME = "test_device_name"
TEST_PLATFORM = "test_platform"
TEST_WIFI_SSID = "test_wifi_ssid"
TEST_PACKAGE_WIFI_SSID = "test_package_wifi_ssid"
TEST_PACKAGE_WIFI_PASSWORD = "test_package_wifi_password"
TEST_DOMAIN = "test_domain_name"
TEST_SENSOR_PLATFORM_1 = "test_sensor_platform_1"
TEST_SENSOR_PLATFORM_2 = "test_sensor_platform_2"
TEST_SENSOR_NAME_1 = "test_sensor_name_1"
TEST_SENSOR_NAME_2 = "test_sensor_name_2"
TEST_SENSOR_ID_1 = "test_sensor_id_1"
TEST_SENSOR_ID_2 = "test_sensor_id_2"
TEST_SENSOR_UPDATE_INTERVAL = "test_sensor_update_interval"
@pytest.fixture(name="basic_wifi")
def fixture_basic_wifi():
return {
CONF_SSID: TEST_PACKAGE_WIFI_SSID,
CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD,
}
@pytest.fixture(name="basic_esphome")
def fixture_basic_esphome():
return {CONF_NAME: TEST_DEVICE_NAME, CONF_PLATFORM: TEST_PLATFORM}
def test_package_unused(basic_esphome, basic_wifi):
"""
Ensures do_package_pass does not change a config if packages aren't used.
"""
config = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi}
actual = do_packages_pass(config)
assert actual == config
def test_package_invalid_dict(basic_esphome, basic_wifi):
"""
Ensures an error is raised if packages is not valid.
"""
config = {CONF_ESPHOME: basic_esphome, CONF_PACKAGES: basic_wifi}
with pytest.raises(cv.Invalid):
do_packages_pass(config)
def test_package_include(basic_wifi, basic_esphome):
"""
Tests the simple case where an independent config present in a package is added to the top-level config as is.
In this test, the CONF_WIFI config is expected to be simply added to the top-level config.
"""
config = {
CONF_ESPHOME: basic_esphome,
CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}},
}
expected = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi}
actual = do_packages_pass(config)
assert actual == expected
def test_package_append(basic_wifi, basic_esphome):
"""
Tests the case where a key is present in both a package and top-level config.
In this test, CONF_WIFI is defined in a package, and CONF_DOMAIN is added to it at the top level.
"""
config = {
CONF_ESPHOME: basic_esphome,
CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}},
CONF_WIFI: {CONF_DOMAIN: TEST_DOMAIN},
}
expected = {
CONF_ESPHOME: basic_esphome,
CONF_WIFI: {
CONF_SSID: TEST_PACKAGE_WIFI_SSID,
CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD,
CONF_DOMAIN: TEST_DOMAIN,
},
}
actual = do_packages_pass(config)
assert actual == expected
def test_package_override(basic_wifi, basic_esphome):
"""
Ensures that the top-level configuration takes precedence over duplicate keys defined in a package.
In this test, CONF_SSID should be overwritten by that defined in the top-level config.
"""
config = {
CONF_ESPHOME: basic_esphome,
CONF_PACKAGES: {"network": {CONF_WIFI: basic_wifi}},
CONF_WIFI: {CONF_SSID: TEST_WIFI_SSID},
}
expected = {
CONF_ESPHOME: basic_esphome,
CONF_WIFI: {
CONF_SSID: TEST_WIFI_SSID,
CONF_PASSWORD: TEST_PACKAGE_WIFI_PASSWORD,
},
}
actual = do_packages_pass(config)
assert actual == expected
def test_multiple_package_order():
"""
Ensures that mutiple packages are merged in order.
"""
config = {
CONF_PACKAGES: {
"package1": {
"logger": {
"level": "DEBUG",
},
},
"package2": {
"logger": {
"level": "VERBOSE",
},
},
},
}
expected = {
"logger": {
"level": "VERBOSE",
},
}
actual = do_packages_pass(config)
assert actual == expected
def test_package_list_merge():
"""
Ensures lists defined in both a package and the top-level config are merged correctly
"""
config = {
CONF_PACKAGES: {
"package_sensors": {
CONF_SENSOR: [
{
CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
CONF_NAME: TEST_SENSOR_NAME_1,
},
{
CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
CONF_NAME: TEST_SENSOR_NAME_2,
},
]
}
},
CONF_SENSOR: [
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_1},
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
],
}
expected = {
CONF_SENSOR: [
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, CONF_NAME: TEST_SENSOR_NAME_1},
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_1, CONF_NAME: TEST_SENSOR_NAME_2},
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_1},
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
]
}
actual = do_packages_pass(config)
assert actual == expected
def test_package_list_merge_by_id():
"""
Ensures that components with matching IDs are merged correctly.
In this test, a sensor is defined in a package, and a CONF_UPDATE_INTERVAL is added at the top level,
and a sensor name is overridden in another sensor.
"""
config = {
CONF_PACKAGES: {
"package_sensors": {
CONF_SENSOR: [
{
CONF_ID: TEST_SENSOR_ID_1,
CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
CONF_NAME: TEST_SENSOR_NAME_1,
},
{
CONF_ID: TEST_SENSOR_ID_2,
CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
CONF_NAME: TEST_SENSOR_NAME_2,
},
]
},
"package2": {
CONF_SENSOR: [
{
CONF_ID: Extend(TEST_SENSOR_ID_1),
CONF_DOMAIN: "2",
}
],
},
"package3": {
CONF_SENSOR: [
{
CONF_ID: Extend(TEST_SENSOR_ID_1),
CONF_DOMAIN: "3",
}
],
},
},
CONF_SENSOR: [
{
CONF_ID: Extend(TEST_SENSOR_ID_1),
CONF_UPDATE_INTERVAL: TEST_SENSOR_UPDATE_INTERVAL,
},
{CONF_ID: Extend(TEST_SENSOR_ID_2), CONF_NAME: TEST_SENSOR_NAME_1},
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
],
}
expected = {
CONF_SENSOR: [
{
CONF_ID: TEST_SENSOR_ID_1,
CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
CONF_NAME: TEST_SENSOR_NAME_1,
CONF_UPDATE_INTERVAL: TEST_SENSOR_UPDATE_INTERVAL,
CONF_DOMAIN: "3",
},
{
CONF_ID: TEST_SENSOR_ID_2,
CONF_PLATFORM: TEST_SENSOR_PLATFORM_1,
CONF_NAME: TEST_SENSOR_NAME_1,
},
{CONF_PLATFORM: TEST_SENSOR_PLATFORM_2, CONF_NAME: TEST_SENSOR_NAME_2},
]
}
actual = do_packages_pass(config)
assert actual == expected
def test_package_merge_by_id_with_list():
"""
Ensures that components with matching IDs are merged correctly when their configuration contains lists.
For example, a sensor with filters defined in both a package and the top level config should be merged.
"""
config = {
CONF_PACKAGES: {
"sensors": {
CONF_SENSOR: [
{CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 42.0}]}
]
}
},
CONF_SENSOR: [
{CONF_ID: Extend(TEST_SENSOR_ID_1), CONF_FILTERS: [{CONF_OFFSET: 146.0}]}
],
}
expected = {
CONF_SENSOR: [
{
CONF_ID: TEST_SENSOR_ID_1,
CONF_FILTERS: [{CONF_MULTIPLY: 42.0}, {CONF_OFFSET: 146.0}],
}
]
}
actual = do_packages_pass(config)
assert actual == expected
def test_package_merge_by_missing_id():
"""
Ensures that components with missing IDs are not merged.
"""
config = {
CONF_PACKAGES: {
"sensors": {
CONF_SENSOR: [
{CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 42.0}]},
]
}
},
CONF_SENSOR: [
{CONF_ID: TEST_SENSOR_ID_1, CONF_FILTERS: [{CONF_MULTIPLY: 10.0}]},
{CONF_ID: Extend(TEST_SENSOR_ID_2), CONF_FILTERS: [{CONF_OFFSET: 146.0}]},
],
}
expected = {
CONF_SENSOR: [
{
CONF_ID: TEST_SENSOR_ID_1,
CONF_FILTERS: [{CONF_MULTIPLY: 42.0}],
},
{
CONF_ID: TEST_SENSOR_ID_1,
CONF_FILTERS: [{CONF_MULTIPLY: 10.0}],
},
{
CONF_ID: Extend(TEST_SENSOR_ID_2),
CONF_FILTERS: [{CONF_OFFSET: 146.0}],
},
]
}
actual = do_packages_pass(config)
assert actual == expected

View File

@ -1076,6 +1076,8 @@ sensor:
id: ultrasonic_sensor1
- platform: uptime
name: Uptime Sensor
- id: !extend ${devicename}_uptime_pcg
unit_of_measurement: s
- platform: wifi_signal
name: WiFi Signal Sensor
update_interval: 15s