From 23f47d0ad29a3641217020a64b0ccef0aed6a605 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 23 Feb 2023 14:22:39 +1300 Subject: [PATCH] Initial stab at importing idf components (#4000) * Initial stab at importing idf components * Handle repo with multiple components Allow components directly from yaml * Actually use the refresh config var * Update esphome/components/esp32/__init__.py --- esphome/components/esp32/__init__.py | 86 ++++++++++++++++++- esphome/components/esp32/const.py | 5 ++ .../external_components/__init__.py | 74 ++-------------- esphome/config_validation.py | 63 ++++++++++++++ esphome/const.py | 3 + 5 files changed, 163 insertions(+), 68 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index f30fa9a7b2..0f16fc9293 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -4,29 +4,43 @@ from pathlib import Path import logging import os -from esphome.helpers import copy_file_if_changed, write_file_if_changed +from esphome.helpers import copy_file_if_changed, write_file_if_changed, mkdir_p from esphome.const import ( CONF_BOARD, + CONF_COMPONENTS, CONF_FRAMEWORK, + CONF_NAME, CONF_SOURCE, CONF_TYPE, CONF_VARIANT, CONF_VERSION, CONF_ADVANCED, + CONF_REFRESH, + CONF_PATH, + CONF_URL, + CONF_REF, CONF_IGNORE_EFUSE_MAC_CRC, KEY_CORE, KEY_FRAMEWORK_VERSION, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + TYPE_GIT, + TYPE_LOCAL, __version__, ) -from esphome.core import CORE, HexInt +from esphome.core import CORE, HexInt, TimePeriod import esphome.config_validation as cv import esphome.codegen as cg +from esphome import git from .const import ( # noqa KEY_BOARD, + KEY_COMPONENTS, KEY_ESP32, + KEY_PATH, + KEY_REF, + KEY_REFRESH, + KEY_REPO, KEY_SDKCONFIG_OPTIONS, KEY_VARIANT, VARIANT_ESP32C3, @@ -51,6 +65,7 @@ def set_core_data(config): if conf[CONF_TYPE] == FRAMEWORK_ESP_IDF: CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "esp-idf" CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] = {} + CORE.data[KEY_ESP32][KEY_COMPONENTS] = {} elif conf[CONF_TYPE] == FRAMEWORK_ARDUINO: CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino" CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse( @@ -104,6 +119,21 @@ def add_idf_sdkconfig_option(name: str, value: SdkconfigValueType): CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS][name] = value +def add_idf_component( + name: str, repo: str, ref: str = None, path: str = None, refresh: TimePeriod = None +): + """Add an esp-idf component to the project.""" + if not CORE.using_esp_idf: + raise ValueError("Not an esp-idf project") + if name not in CORE.data[KEY_ESP32][KEY_COMPONENTS]: + CORE.data[KEY_ESP32][KEY_COMPONENTS][name] = { + KEY_REPO: repo, + KEY_REF: ref, + KEY_PATH: path, + KEY_REFRESH: refresh, + } + + def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/espressif/arduino-esp32/releases) version to # a PIO platformio/framework-arduinoespressif32 value @@ -270,6 +300,18 @@ ESP_IDF_FRAMEWORK_SCHEMA = cv.All( cv.Optional(CONF_IGNORE_EFUSE_MAC_CRC, default=False): cv.boolean, } ), + cv.Optional(CONF_COMPONENTS, default=[]): cv.ensure_list( + cv.Schema( + { + cv.Required(CONF_NAME): cv.string_strict, + cv.Required(CONF_SOURCE): cv.SOURCE_SCHEMA, + cv.Optional(CONF_PATH): cv.string, + cv.Optional(CONF_REFRESH, default="1d"): cv.All( + cv.string, cv.source_refresh + ), + } + ) + ), } ), _esp_idf_check_versions, @@ -372,6 +414,19 @@ async def to_code(config): ), ) + for component in conf[CONF_COMPONENTS]: + source = component[CONF_SOURCE] + if source[CONF_TYPE] == TYPE_GIT: + add_idf_component( + name=component[CONF_NAME], + repo=source[CONF_URL], + ref=source.get(CONF_REF), + path=component.get(CONF_PATH), + refresh=component[CONF_REFRESH], + ) + elif source[CONF_TYPE] == TYPE_LOCAL: + _LOGGER.warning("Local components are not implemented yet.") + elif conf[CONF_TYPE] == FRAMEWORK_ARDUINO: cg.add_platformio_option("framework", "arduino") cg.add_build_flag("-DUSE_ARDUINO") @@ -468,6 +523,33 @@ def copy_files(): __version__, ) + if CORE.data[KEY_ESP32][KEY_COMPONENTS]: + import shutil + + shutil.rmtree(CORE.relative_build_path("components"), ignore_errors=True) + + components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS] + + for name, component in components.items(): + + repo_dir, _ = git.clone_or_update( + url=component[KEY_REPO], + ref=component[KEY_REF], + refresh=component[KEY_REFRESH], + domain="idf_components", + ) + mkdir_p(CORE.relative_build_path("components")) + component_dir = repo_dir + if component[KEY_PATH] is not None: + component_dir = component_dir / component[KEY_PATH] + + shutil.copytree( + component_dir, + CORE.relative_build_path(f"components/{name}"), + dirs_exist_ok=True, + ignore=shutil.ignore_patterns(".git", ".github"), + ) + dir = os.path.dirname(__file__) post_build_file = os.path.join(dir, "post_build.py.script") copy_file_if_changed( diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index d92b449ee9..d13df01d3a 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -4,6 +4,11 @@ KEY_ESP32 = "esp32" KEY_BOARD = "board" KEY_VARIANT = "variant" KEY_SDKCONFIG_OPTIONS = "sdkconfig_options" +KEY_COMPONENTS = "components" +KEY_REPO = "repo" +KEY_REF = "ref" +KEY_REFRESH = "refresh" +KEY_PATH = "path" VARIANT_ESP32 = "ESP32" VARIANT_ESP32S2 = "ESP32S2" diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py index 53fd337ed8..bbb703dc5c 100644 --- a/esphome/components/external_components/__init__.py +++ b/esphome/components/external_components/__init__.py @@ -1,90 +1,32 @@ -import re import logging from pathlib import Path import esphome.config_validation as cv +from esphome import git, loader from esphome.const import ( CONF_COMPONENTS, + CONF_EXTERNAL_COMPONENTS, + CONF_PASSWORD, + CONF_PATH, CONF_REF, CONF_REFRESH, CONF_SOURCE, - CONF_URL, CONF_TYPE, - CONF_EXTERNAL_COMPONENTS, - CONF_PATH, + CONF_URL, CONF_USERNAME, - CONF_PASSWORD, + TYPE_GIT, + TYPE_LOCAL, ) from esphome.core import CORE -from esphome import git, loader _LOGGER = logging.getLogger(__name__) DOMAIN = CONF_EXTERNAL_COMPONENTS -TYPE_GIT = "git" -TYPE_LOCAL = "local" - - -GIT_SCHEMA = { - cv.Required(CONF_URL): cv.url, - cv.Optional(CONF_REF): cv.git_ref, - cv.Optional(CONF_USERNAME): cv.string, - cv.Optional(CONF_PASSWORD): cv.string, -} -LOCAL_SCHEMA = { - cv.Required(CONF_PATH): cv.directory, -} - - -def validate_source_shorthand(value): - if not isinstance(value, str): - raise cv.Invalid("Shorthand only for strings") - try: - return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value}) - except cv.Invalid: - pass - # Regex for GitHub repo name with optional branch/tag - # Note: git allows other branch/tag names as well, but never seen them used before - m = re.match( - r"github://(?:([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?|pr#([0-9]+))", - value, - ) - if m is None: - raise cv.Invalid( - "Source is not a file system path, in expected github://username/name[@branch-or-tag] or github://pr#1234 format!" - ) - if m.group(4): - conf = { - CONF_TYPE: TYPE_GIT, - CONF_URL: "https://github.com/esphome/esphome.git", - CONF_REF: f"pull/{m.group(4)}/head", - } - else: - conf = { - CONF_TYPE: TYPE_GIT, - CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git", - } - if m.group(3): - conf[CONF_REF] = m.group(3) - - return SOURCE_SCHEMA(conf) - - -SOURCE_SCHEMA = cv.Any( - validate_source_shorthand, - cv.typed_schema( - { - TYPE_GIT: cv.Schema(GIT_SCHEMA), - TYPE_LOCAL: cv.Schema(LOCAL_SCHEMA), - } - ), -) - CONFIG_SCHEMA = cv.ensure_list( { - cv.Required(CONF_SOURCE): SOURCE_SCHEMA, + cv.Required(CONF_SOURCE): cv.SOURCE_SCHEMA, cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, cv.source_refresh), cv.Optional(CONF_COMPONENTS, default="all"): cv.Any( "all", cv.ensure_list(cv.string) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 7440f71790..a46d14053b 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -39,6 +39,11 @@ from esphome.const import ( CONF_UPDATE_INTERVAL, CONF_TYPE_ID, CONF_TYPE, + CONF_REF, + CONF_URL, + CONF_PATH, + CONF_USERNAME, + CONF_PASSWORD, ENTITY_CATEGORY_CONFIG, ENTITY_CATEGORY_DIAGNOSTIC, ENTITY_CATEGORY_NONE, @@ -46,6 +51,8 @@ from esphome.const import ( KEY_FRAMEWORK_VERSION, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + TYPE_GIT, + TYPE_LOCAL, ) from esphome.core import ( CORE, @@ -1820,3 +1827,59 @@ def suppress_invalid(): yield except vol.Invalid: pass + + +GIT_SCHEMA = { + Required(CONF_URL): url, + Optional(CONF_REF): git_ref, + Optional(CONF_USERNAME): string, + Optional(CONF_PASSWORD): string, +} +LOCAL_SCHEMA = { + Required(CONF_PATH): directory, +} + + +def validate_source_shorthand(value): + if not isinstance(value, str): + raise Invalid("Shorthand only for strings") + try: + return SOURCE_SCHEMA({CONF_TYPE: TYPE_LOCAL, CONF_PATH: value}) + except Invalid: + pass + # Regex for GitHub repo name with optional branch/tag + # Note: git allows other branch/tag names as well, but never seen them used before + m = re.match( + r"github://(?:([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)(?:@([a-zA-Z0-9\-_.\./]+))?|pr#([0-9]+))", + value, + ) + if m is None: + raise Invalid( + "Source is not a file system path, in expected github://username/name[@branch-or-tag] or github://pr#1234 format!" + ) + if m.group(4): + conf = { + CONF_TYPE: TYPE_GIT, + CONF_URL: "https://github.com/esphome/esphome.git", + CONF_REF: f"pull/{m.group(4)}/head", + } + else: + conf = { + CONF_TYPE: TYPE_GIT, + CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git", + } + if m.group(3): + conf[CONF_REF] = m.group(3) + + return SOURCE_SCHEMA(conf) + + +SOURCE_SCHEMA = Any( + validate_source_shorthand, + typed_schema( + { + TYPE_GIT: Schema(GIT_SCHEMA), + TYPE_LOCAL: Schema(LOCAL_SCHEMA), + } + ), +) diff --git a/esphome/const.py b/esphome/const.py index bd593bbc80..d821a76b75 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -797,6 +797,9 @@ CONF_X_GRID = "x_grid" CONF_Y_GRID = "y_grid" CONF_ZERO = "zero" +TYPE_GIT = "git" +TYPE_LOCAL = "local" + ENV_NOGITIGNORE = "ESPHOME_NOGITIGNORE" ENV_QUICKWIZARD = "ESPHOME_QUICKWIZARD"