From 229bf719a2bc40f1ef297efb88684c97f47a458f Mon Sep 17 00:00:00 2001 From: Otto Winter Date: Fri, 7 May 2021 20:02:17 +0200 Subject: [PATCH] Implement external custom components installing from YAML (#1630) * Move components import loading to importlib MetaPathFinder and importlib.resources * Add external_components component * Fix * Fix * fix cv.url return * fix validate shorthand git * implement git refresh * Use finders from sys.path_hooks instead of looking for __init__.py * use github:// schema * error handling * add test * fix handling git output * revert file check handling * fix test * allow full component path be specified for local * fix test * fix path handling * lint Co-authored-by: Guillermo Ruffino --- .github/workflows/ci.yml | 2 +- .github/workflows/release-dev.yml | 2 +- .github/workflows/release.yml | 2 +- .../external_components/__init__.py | 197 ++++++++++++++++++ esphome/components/http_request/__init__.py | 2 +- esphome/config.py | 190 ++--------------- esphome/config_validation.py | 14 ++ esphome/const.py | 3 + esphome/{core.py => core/__init__.py} | 2 +- esphome/{core_config.py => core/config.py} | 0 esphome/helpers.py | 4 +- esphome/loader.py | 179 ++++++++++++++++ esphome/writer.py | 38 ++-- script/build_jsonschema.py | 2 +- tests/test4.yaml | 6 + 15 files changed, 451 insertions(+), 192 deletions(-) create mode 100644 esphome/components/external_components/__init__.py rename esphome/{core.py => core/__init__.py} (99%) rename esphome/{core_config.py => core/config.py} (100%) create mode 100644 esphome/loader.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed4343202c..d2230b3da7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: uses: actions/cache@v1 with: path: ~/.platformio - key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }} + key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }} restore-keys: | test-home-platformio-${{ matrix.test }}- - name: Set up environment diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index b4c4a8f17d..f0dc4bd0c0 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -112,7 +112,7 @@ jobs: uses: actions/cache@v1 with: path: ~/.platformio - key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }} + key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }} restore-keys: | test-home-platformio-${{ matrix.test }}- - name: Set up environment diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a1fd2dba24..1eca3be269 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,7 @@ jobs: uses: actions/cache@v1 with: path: ~/.platformio - key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }} + key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }} restore-keys: | test-home-platformio-${{ matrix.test }}- - name: Set up environment diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py new file mode 100644 index 0000000000..272812adcf --- /dev/null +++ b/esphome/components/external_components/__init__.py @@ -0,0 +1,197 @@ +import re +import logging +from pathlib import Path +import subprocess +import hashlib +import datetime + +import esphome.config_validation as cv +from esphome.const import ( + CONF_COMPONENTS, + CONF_SOURCE, + CONF_URL, + CONF_TYPE, + CONF_EXTERNAL_COMPONENTS, + CONF_PATH, +) +from esphome.core import CORE +from esphome import loader + +_LOGGER = logging.getLogger(__name__) + +DOMAIN = CONF_EXTERNAL_COMPONENTS + +TYPE_GIT = "git" +TYPE_LOCAL = "local" +CONF_REFRESH = "refresh" +CONF_REF = "ref" + + +def validate_git_ref(value): + if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None: + raise cv.Invalid("Not a valid git ref") + return value + + +GIT_SCHEMA = { + cv.Required(CONF_URL): cv.url, + cv.Optional(CONF_REF): validate_git_ref, +} +LOCAL_SCHEMA = { + cv.Required(CONF_PATH): cv.directory, +} + + +def validate_source_shorthand(value): + if not isinstance(value, str): + raise cv.Invalid("Shorthand only for strings") + try: + return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value}) + except cv.Invalid: + pass + # Regex for GitHub repo name with optional branch/tag + # Note: git allows other branch/tag names as well, but never seen them used before + m = re.match( + r"github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?", + value, + ) + if m is None: + raise cv.Invalid( + "Source is not a file system path or in expected github://username/name[@branch-or-tag] format!" + ) + conf = { + CONF_TYPE: TYPE_GIT, + CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git", + } + if m.group(3): + conf[CONF_REF] = m.group(3) + return SOURCE_SCHEMA(conf) + + +def validate_refresh(value: str): + if value.lower() == "always": + return validate_refresh("0s") + if value.lower() == "never": + return validate_refresh("1000y") + return cv.positive_time_period_seconds(value) + + +SOURCE_SCHEMA = cv.Any( + validate_source_shorthand, + cv.typed_schema( + { + TYPE_GIT: cv.Schema(GIT_SCHEMA), + TYPE_LOCAL: cv.Schema(LOCAL_SCHEMA), + } + ), +) + + +CONFIG_SCHEMA = cv.ensure_list( + { + cv.Required(CONF_SOURCE): SOURCE_SCHEMA, + cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, validate_refresh), + cv.Optional(CONF_COMPONENTS, default="all"): cv.Any( + "all", cv.ensure_list(cv.string) + ), + } +) + + +def to_code(config): + pass + + +def _compute_destination_path(key: str) -> Path: + base_dir = Path(CORE.config_dir) / ".esphome" / DOMAIN + h = hashlib.new("sha256") + h.update(key.encode()) + return base_dir / h.hexdigest()[:8] + + +def _handle_git_response(ret): + if ret.returncode != 0 and ret.stderr: + err_str = ret.stderr.decode("utf-8") + lines = [x.strip() for x in err_str.splitlines()] + if lines[-1].startswith("fatal:"): + raise cv.Invalid(lines[-1][len("fatal: ") :]) + raise cv.Invalid(err_str) + + +def _process_single_config(config: dict): + conf = config[CONF_SOURCE] + if conf[CONF_TYPE] == TYPE_GIT: + key = f"{conf[CONF_URL]}@{conf.get(CONF_REF)}" + repo_dir = _compute_destination_path(key) + if not repo_dir.is_dir(): + cmd = ["git", "clone", "--depth=1"] + if CONF_REF in conf: + cmd += ["--branch", conf[CONF_REF]] + cmd += [conf[CONF_URL], str(repo_dir)] + ret = subprocess.run(cmd, capture_output=True, check=False) + _handle_git_response(ret) + + else: + # Check refresh needed + file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD") + # On first clone, FETCH_HEAD does not exists + if not file_timestamp.exists(): + file_timestamp = Path(repo_dir / ".git" / "HEAD") + age = datetime.datetime.now() - datetime.datetime.fromtimestamp( + file_timestamp.stat().st_mtime + ) + if age.seconds > config[CONF_REFRESH].total_seconds: + _LOGGER.info("Executing git pull %s", key) + cmd = ["git", "pull"] + ret = subprocess.run( + cmd, cwd=repo_dir, capture_output=True, check=False + ) + _handle_git_response(ret) + + if (repo_dir / "esphome" / "components").is_dir(): + components_dir = repo_dir / "esphome" / "components" + elif (repo_dir / "components").is_dir(): + components_dir = repo_dir / "components" + else: + raise cv.Invalid( + "Could not find components folder for source. Please check the source contains a 'components' or 'esphome/components' folder", + [CONF_SOURCE], + ) + + elif conf[CONF_TYPE] == TYPE_LOCAL: + components_dir = Path(CORE.relative_config_path(conf[CONF_PATH])) + else: + raise NotImplementedError() + + if config[CONF_COMPONENTS] == "all": + num_components = len(list(components_dir.glob("*/__init__.py"))) + if num_components > 100: + # Prevent accidentally including all components from an esphome fork/branch + # In this case force the user to manually specify which components they want to include + raise cv.Invalid( + "This source is an ESPHome fork or branch. Please manually specify the components you want to import using the 'components' key", + [CONF_COMPONENTS], + ) + allowed_components = None + else: + for i, name in enumerate(config[CONF_COMPONENTS]): + expected = components_dir / name / "__init__.py" + if not expected.is_file(): + raise cv.Invalid( + f"Could not find __init__.py file for component {name}. Please check the component is defined by this source (search path: {expected})", + [CONF_COMPONENTS, i], + ) + allowed_components = config[CONF_COMPONENTS] + + loader.install_meta_finder(components_dir, allowed_components=allowed_components) + + +def do_external_components_pass(config: dict) -> None: + conf = config.get(DOMAIN) + if conf is None: + return + with cv.prepend_path(DOMAIN): + conf = CONFIG_SCHEMA(conf) + for i, c in enumerate(conf): + with cv.prepend_path(i): + _process_single_config(c) diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index dee3fe8f77..ef664a9d35 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -14,7 +14,7 @@ from esphome.const import ( CONF_URL, ) from esphome.core import CORE, Lambda -from esphome.core_config import PLATFORMIO_ESP8266_LUT +from esphome.core.config import PLATFORMIO_ESP8266_LUT DEPENDENCIES = ["network"] AUTO_LOAD = ["json"] diff --git a/esphome/config.py b/esphome/config.py index 5a135b3e37..a1fc07a21f 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -1,195 +1,34 @@ import collections -import importlib import logging import re -import os.path # pylint: disable=unused-import, wrong-import-order -import sys from contextlib import contextmanager import voluptuous as vol -from esphome import core, core_config, yaml_util +from esphome import core, yaml_util, loader +import esphome.core.config as core_config from esphome.const import ( CONF_ESPHOME, CONF_PLATFORM, - ESP_PLATFORMS, CONF_PACKAGES, CONF_SUBSTITUTIONS, + CONF_EXTERNAL_COMPONENTS, ) -from esphome.core import CORE, EsphomeError # noqa +from esphome.core import CORE, EsphomeError from esphome.helpers import indent from esphome.util import safe_print, OrderedDict -from typing import List, Optional, Tuple, Union # noqa -from esphome.core import ConfigType # noqa +from typing import List, Optional, Tuple, Union +from esphome.core import ConfigType +from esphome.loader import get_component, get_platform, ComponentManifest from esphome.yaml_util import is_secret, ESPHomeDataBase, ESPForceValue from esphome.voluptuous_schema import ExtraKeysInvalid from esphome.log import color, Fore _LOGGER = logging.getLogger(__name__) -_COMPONENT_CACHE = {} - - -class ComponentManifest: - def __init__(self, module, base_components_path, is_core=False, is_platform=False): - self.module = module - self._is_core = is_core - self.is_platform = is_platform - self.base_components_path = base_components_path - - @property - def is_platform_component(self): - return getattr(self.module, "IS_PLATFORM_COMPONENT", False) - - @property - def config_schema(self): - return getattr(self.module, "CONFIG_SCHEMA", None) - - @property - def multi_conf(self): - return getattr(self.module, "MULTI_CONF", False) - - @property - def to_code(self): - return getattr(self.module, "to_code", None) - - @property - def esp_platforms(self): - return getattr(self.module, "ESP_PLATFORMS", ESP_PLATFORMS) - - @property - def dependencies(self): - return getattr(self.module, "DEPENDENCIES", []) - - @property - def conflicts_with(self): - return getattr(self.module, "CONFLICTS_WITH", []) - - @property - def auto_load(self): - return getattr(self.module, "AUTO_LOAD", []) - - @property - def codeowners(self) -> List[str]: - return getattr(self.module, "CODEOWNERS", []) - - def _get_flags_set(self, name, config): - if not hasattr(self.module, name): - return set() - obj = getattr(self.module, name) - if callable(obj): - obj = obj(config) - if obj is None: - return set() - if not isinstance(obj, (list, tuple, set)): - obj = [obj] - return set(obj) - - @property - def source_files(self): - if self._is_core: - core_p = os.path.abspath(os.path.join(os.path.dirname(__file__), "core")) - source_files = core.find_source_files(os.path.join(core_p, "dummy")) - ret = {} - for f in source_files: - ret[f"esphome/core/{f}"] = os.path.join(core_p, f) - return ret - - source_files = core.find_source_files(self.module.__file__) - ret = {} - # Make paths absolute - directory = os.path.abspath(os.path.dirname(self.module.__file__)) - for x in source_files: - full_file = os.path.join(directory, x) - rel = os.path.relpath(full_file, self.base_components_path) - # Always use / for C++ include names - rel = rel.replace(os.sep, "/") - target_file = f"esphome/components/{rel}" - ret[target_file] = full_file - return ret - - -CORE_COMPONENTS_PATH = os.path.abspath( - os.path.join(os.path.dirname(__file__), "components") -) -_UNDEF = object() -CUSTOM_COMPONENTS_PATH = _UNDEF - - -def _mount_config_dir(): - global CUSTOM_COMPONENTS_PATH - if CUSTOM_COMPONENTS_PATH is not _UNDEF: - return - custom_path = os.path.abspath(os.path.join(CORE.config_dir, "custom_components")) - if not os.path.isdir(custom_path): - CUSTOM_COMPONENTS_PATH = None - return - if CORE.config_dir not in sys.path: - sys.path.insert(0, CORE.config_dir) - CUSTOM_COMPONENTS_PATH = custom_path - - -def _lookup_module(domain, is_platform): - if domain in _COMPONENT_CACHE: - return _COMPONENT_CACHE[domain] - - _mount_config_dir() - # First look for custom_components - try: - module = importlib.import_module(f"custom_components.{domain}") - except ImportError as e: - # ImportError when no such module - if "No module named" not in str(e): - _LOGGER.warning( - "Unable to import custom component %s:", domain, exc_info=True - ) - except Exception: # pylint: disable=broad-except - # Other error means component has an issue - _LOGGER.error("Unable to load custom component %s:", domain, exc_info=True) - return None - else: - # Found in custom components - manif = ComponentManifest( - module, CUSTOM_COMPONENTS_PATH, is_platform=is_platform - ) - _COMPONENT_CACHE[domain] = manif - return manif - - try: - module = importlib.import_module(f"esphome.components.{domain}") - except ImportError as e: - if "No module named" not in str(e): - _LOGGER.error("Unable to import component %s:", domain, exc_info=True) - return None - except Exception: # pylint: disable=broad-except - _LOGGER.error("Unable to load component %s:", domain, exc_info=True) - return None - else: - manif = ComponentManifest(module, CORE_COMPONENTS_PATH, is_platform=is_platform) - _COMPONENT_CACHE[domain] = manif - return manif - - -def get_component(domain): - assert "." not in domain - return _lookup_module(domain, False) - - -def get_platform(domain, platform): - full = f"{platform}.{domain}" - return _lookup_module(full, True) - - -_COMPONENT_CACHE["esphome"] = ComponentManifest( - core_config, - CORE_COMPONENTS_PATH, - is_core=True, - is_platform=False, -) - def iter_components(config): for domain, conf in config.items(): @@ -453,6 +292,9 @@ def recursive_check_replaceme(value): def validate_config(config, command_line_substitutions): result = Config() + loader.clear_component_meta_finders() + loader.install_custom_components_meta_finder() + # 0. Load packages if CONF_PACKAGES in config: from esphome.components.packages import do_packages_pass @@ -486,6 +328,18 @@ def validate_config(config, command_line_substitutions): except vol.Invalid as err: result.add_error(err) + # 1.2. Load external_components + if CONF_EXTERNAL_COMPONENTS in config: + from esphome.components.external_components import do_external_components_pass + + result.add_output_path([CONF_EXTERNAL_COMPONENTS], CONF_EXTERNAL_COMPONENTS) + try: + do_external_components_pass(config) + except vol.Invalid as err: + result.update(config) + result.add_error(err) + return result + if "esphomeyaml" in config: _LOGGER.warning( "The esphomeyaml section has been renamed to esphome in 1.11.0. " diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 4a65c59379..24c86e6713 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1556,3 +1556,17 @@ def polling_component_schema(default_update_interval): ): update_interval, } ) + + +def url(value): + import urllib.parse + + value = string_strict(value) + try: + parsed = urllib.parse.urlparse(value) + except ValueError as e: + raise Invalid("Not a valid URL") from e + + if not parsed.scheme or not parsed.netloc: + raise Invalid("Expected a URL scheme and host") + return parsed.geturl() diff --git a/esphome/const.py b/esphome/const.py index cece8df4e3..a8f8aa81f5 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -193,6 +193,7 @@ CONF_ESPHOME = "esphome" CONF_ETHERNET = "ethernet" CONF_EVENT = "event" CONF_EXPIRE_AFTER = "expire_after" +CONF_EXTERNAL_COMPONENTS = "external_components" CONF_EXTERNAL_VCC = "external_vcc" CONF_FALLING_EDGE = "falling_edge" CONF_FAMILY = "family" @@ -405,6 +406,7 @@ CONF_PAGE_ID = "page_id" CONF_PAGES = "pages" CONF_PANASONIC = "panasonic" CONF_PASSWORD = "password" +CONF_PATH = "path" CONF_PAYLOAD = "payload" CONF_PAYLOAD_AVAILABLE = "payload_available" CONF_PAYLOAD_NOT_AVAILABLE = "payload_not_available" @@ -514,6 +516,7 @@ CONF_SLEEP_DURATION = "sleep_duration" CONF_SLEEP_PIN = "sleep_pin" CONF_SLEEP_WHEN_DONE = "sleep_when_done" CONF_SONY = "sony" +CONF_SOURCE = "source" CONF_SPEED = "speed" CONF_SPEED_COMMAND_TOPIC = "speed_command_topic" CONF_SPEED_COUNT = "speed_count" diff --git a/esphome/core.py b/esphome/core/__init__.py similarity index 99% rename from esphome/core.py rename to esphome/core/__init__.py index ce7eaad6fb..47048478ef 100644 --- a/esphome/core.py +++ b/esphome/core/__init__.py @@ -23,7 +23,7 @@ from esphome.helpers import ensure_unique_string, is_hassio from esphome.util import OrderedDict if TYPE_CHECKING: - from .cpp_generator import MockObj, MockObjClass, Statement + from ..cpp_generator import MockObj, MockObjClass, Statement _LOGGER = logging.getLogger(__name__) diff --git a/esphome/core_config.py b/esphome/core/config.py similarity index 100% rename from esphome/core_config.py rename to esphome/core/config.py diff --git a/esphome/helpers.py b/esphome/helpers.py index b80d338eef..d9730f96a7 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -222,7 +222,7 @@ def write_file_if_changed(path: Union[Path, str], text: str): write_file(path, text) -def copy_file_if_changed(src, dst): +def copy_file_if_changed(src: os.PathLike, dst: os.PathLike) -> None: import shutil if file_compare(src, dst): @@ -240,7 +240,7 @@ def list_starts_with(list_, sub): return len(sub) <= len(list_) and all(list_[i] == x for i, x in enumerate(sub)) -def file_compare(path1, path2): +def file_compare(path1: os.PathLike, path2: os.PathLike) -> bool: """Return True if the files path1 and path2 have the same contents.""" import stat diff --git a/esphome/loader.py b/esphome/loader.py new file mode 100644 index 0000000000..c418008453 --- /dev/null +++ b/esphome/loader.py @@ -0,0 +1,179 @@ +import logging +import typing +from typing import Callable, List, Optional, Dict, Any, ContextManager +from types import ModuleType +import importlib +import importlib.util +import importlib.resources +import importlib.abc +import sys +from pathlib import Path + +from esphome.const import ESP_PLATFORMS, SOURCE_FILE_EXTENSIONS +import esphome.core.config +from esphome.core import CORE + +_LOGGER = logging.getLogger(__name__) + + +class SourceFile: + def __init__( + self, + package: importlib.resources.Package, + resource: importlib.resources.Resource, + ) -> None: + self._package = package + self._resource = resource + + def open_binary(self) -> typing.BinaryIO: + return importlib.resources.open_binary(self._package, self._resource) + + def path(self) -> ContextManager[Path]: + return importlib.resources.path(self._package, self._resource) + + +class ComponentManifest: + def __init__(self, module: ModuleType): + self.module = module + + @property + def package(self) -> str: + return self.module.__package__ + + @property + def is_platform(self) -> bool: + return len(self.module.__name__.split(".")) == 4 + + @property + def is_platform_component(self) -> bool: + return getattr(self.module, "IS_PLATFORM_COMPONENT", False) + + @property + def config_schema(self) -> Optional[Any]: + return getattr(self.module, "CONFIG_SCHEMA", None) + + @property + def multi_conf(self) -> bool: + return getattr(self.module, "MULTI_CONF", False) + + @property + def to_code(self) -> Optional[Callable[[Any], None]]: + return getattr(self.module, "to_code", None) + + @property + def esp_platforms(self) -> List[str]: + return getattr(self.module, "ESP_PLATFORMS", ESP_PLATFORMS) + + @property + def dependencies(self) -> List[str]: + return getattr(self.module, "DEPENDENCIES", []) + + @property + def conflicts_with(self) -> List[str]: + return getattr(self.module, "CONFLICTS_WITH", []) + + @property + def auto_load(self) -> List[str]: + return getattr(self.module, "AUTO_LOAD", []) + + @property + def codeowners(self) -> List[str]: + return getattr(self.module, "CODEOWNERS", []) + + @property + def source_files(self) -> Dict[Path, SourceFile]: + ret = {} + for resource in importlib.resources.contents(self.package): + if Path(resource).suffix not in SOURCE_FILE_EXTENSIONS: + continue + if not importlib.resources.is_resource(self.package, resource): + # Not a resource = this is a directory (yeah this is confusing) + continue + # Always use / for C++ include names + target_path = Path(*self.package.split(".")) / resource + ret[target_path] = SourceFile(self.package, resource) + return ret + + +class ComponentMetaFinder(importlib.abc.MetaPathFinder): + def __init__( + self, components_path: Path, allowed_components: Optional[List[str]] = None + ) -> None: + self._allowed_components = allowed_components + self._finders = [] + for hook in sys.path_hooks: + try: + finder = hook(str(components_path)) + except ImportError: + continue + self._finders.append(finder) + + def find_spec(self, fullname: str, path: Optional[List[str]], target=None): + if not fullname.startswith("esphome.components."): + return None + parts = fullname.split(".") + if len(parts) != 3: + # only handle direct components, not platforms + # platforms are handled automatically when parent is imported + return None + component = parts[2] + if ( + self._allowed_components is not None + and component not in self._allowed_components + ): + return None + + for finder in self._finders: + spec = finder.find_spec(fullname, target=target) + if spec is not None: + return spec + return None + + +def clear_component_meta_finders(): + sys.meta_path = [x for x in sys.meta_path if not isinstance(x, ComponentMetaFinder)] + + +def install_meta_finder( + components_path: Path, allowed_components: Optional[List[str]] = None +): + sys.meta_path.insert(0, ComponentMetaFinder(components_path, allowed_components)) + + +def install_custom_components_meta_finder(): + custom_components_dir = (Path(CORE.config_dir) / "custom_components").resolve() + install_meta_finder(custom_components_dir) + + +def _lookup_module(domain): + if domain in _COMPONENT_CACHE: + return _COMPONENT_CACHE[domain] + + try: + module = importlib.import_module(f"esphome.components.{domain}") + except ImportError as e: + if "No module named" not in str(e): + _LOGGER.error("Unable to import component %s:", domain, exc_info=True) + return None + except Exception: # pylint: disable=broad-except + _LOGGER.error("Unable to load component %s:", domain, exc_info=True) + return None + else: + manif = ComponentManifest(module) + _COMPONENT_CACHE[domain] = manif + return manif + + +def get_component(domain): + assert "." not in domain + return _lookup_module(domain) + + +def get_platform(domain, platform): + full = f"{platform}.{domain}" + return _lookup_module(full) + + +_COMPONENT_CACHE = {} +CORE_COMPONENTS_PATH = (Path(__file__).parent / "components").resolve() +_COMPONENT_CACHE["esphome"] = ComponentManifest(esphome.core.config) diff --git a/esphome/writer.py b/esphome/writer.py index ec772b5127..57698f8c25 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -1,6 +1,8 @@ import logging import os import re +from pathlib import Path +from typing import Dict from esphome.config import iter_components from esphome.const import ( @@ -24,6 +26,7 @@ from esphome.helpers import ( ) from esphome.storage_json import StorageJSON, storage_path from esphome.pins import ESP8266_FLASH_SIZES, ESP8266_LD_SCRIPTS +from esphome import loader _LOGGER = logging.getLogger(__name__) @@ -355,7 +358,7 @@ or use the custom_components folder. def copy_src_tree(): - source_files = {} + source_files: Dict[Path, loader.SourceFile] = {} for _, component, _ in iter_components(CORE.config): source_files.update(component.source_files) @@ -365,37 +368,40 @@ def copy_src_tree(): # Build #include list for esphome.h include_l = [] - for target, path in source_files_l: - if os.path.splitext(path)[1] in HEADER_FILE_EXTENSIONS: + for target, _ in source_files_l: + if target.suffix in HEADER_FILE_EXTENSIONS: include_l.append(f'#include "{target}"') include_l.append("") include_s = "\n".join(include_l) source_files_copy = source_files.copy() - source_files_copy.pop(DEFINES_H_TARGET) + ignore_targets = [Path(x) for x in (DEFINES_H_TARGET, VERSION_H_TARGET)] + for t in ignore_targets: + source_files_copy.pop(t) - for path in walk_files(CORE.relative_src_path("esphome")): - if os.path.splitext(path)[1] not in SOURCE_FILE_EXTENSIONS: + for fname in walk_files(CORE.relative_src_path("esphome")): + p = Path(fname) + if p.suffix not in SOURCE_FILE_EXTENSIONS: # Not a source file, ignore continue # Transform path to target path name - target = os.path.relpath(path, CORE.relative_src_path()).replace( - os.path.sep, "/" - ) - if target in (DEFINES_H_TARGET, VERSION_H_TARGET): + target = p.relative_to(CORE.relative_src_path()) + if target in ignore_targets: # Ignore defines.h, will be dealt with later continue if target not in source_files_copy: # Source file removed, delete target - os.remove(path) + p.unlink() else: - src_path = source_files_copy.pop(target) - copy_file_if_changed(src_path, path) + src_file = source_files_copy.pop(target) + with src_file.path() as src_path: + copy_file_if_changed(src_path, p) # Now copy new files - for target, src_path in source_files_copy.items(): - dst_path = CORE.relative_src_path(*target.split("/")) - copy_file_if_changed(src_path, dst_path) + for target, src_file in source_files_copy.items(): + dst_path = CORE.relative_src_path(*target.parts) + with src_file.path() as src_path: + copy_file_if_changed(src_path, dst_path) # Finally copy defines write_file_if_changed( diff --git a/script/build_jsonschema.py b/script/build_jsonschema.py index 2c9534b861..6d19e25e29 100644 --- a/script/build_jsonschema.py +++ b/script/build_jsonschema.py @@ -62,7 +62,7 @@ def add_definition_array_or_single_object(ref): def add_core(): - from esphome.core_config import CONFIG_SCHEMA + from esphome.core.config import CONFIG_SCHEMA base_props["esphome"] = get_jschema("esphome", CONFIG_SCHEMA.schema) diff --git a/tests/test4.yaml b/tests/test4.yaml index ed63a1ac14..e85cdfbc19 100644 --- a/tests/test4.yaml +++ b/tests/test4.yaml @@ -160,3 +160,9 @@ display: lambda: |- it.rectangle(0, 0, it.get_width(), it.get_height()); +external_components: + - source: github://esphome/esphome@dev + refresh: 1d + components: ["bh1750"] + - source: ../esphome/components + components: ["sntp"]