Upstream/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2024-11-16 10:45:48 +01:00
Code Issues Projects Releases Wiki Activity

Allow using a git source for a package (#2193)

Browse Source
This commit is contained in:
Jesse Hills 2021-09-06 08:23:06 +12:00 committed by GitHub
parent ca12b8aa56
commit f9b0666adf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 231 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
esphome/components/external_components/__init__.py
View File

@ -1,13 +1,12 @@
import re
import logging
from pathlib import Path
import subprocess
import hashlib
import datetime
import esphome.config_validation as cv
from esphome.const import (
CONF_COMPONENTS,
CONF_REF,
CONF_REFRESH,
CONF_SOURCE,
CONF_URL,
CONF_TYPE,
@ -15,7 +14,7 @@ from esphome.const import (
CONF_PATH,
)
from esphome.core import CORE
from esphome import loader
from esphome import git, loader
_LOGGER = logging.getLogger(__name__)
@ -23,19 +22,11 @@ DOMAIN = CONF_EXTERNAL_COMPONENTS
TYPE_GIT = "git"
TYPE_LOCAL = "local"
CONF_REFRESH = "refresh"
CONF_REF = "ref"
def validate_git_ref(value):
if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
raise cv.Invalid("Not a valid git ref")
return value
GIT_SCHEMA = {
cv.Required(CONF_URL): cv.url,
cv.Optional(CONF_REF): validate_git_ref,
cv.Optional(CONF_REF): cv.git_ref,
}
LOCAL_SCHEMA = {
cv.Required(CONF_PATH): cv.directory,
@ -68,14 +59,6 @@ def validate_source_shorthand(value):
return SOURCE_SCHEMA(conf)
def validate_refresh(value: str):
if value.lower() == "always":
return validate_refresh("0s")
if value.lower() == "never":
return validate_refresh("1000y")
return cv.positive_time_period_seconds(value)
SOURCE_SCHEMA = cv.Any(
validate_source_shorthand,
cv.typed_schema(
@ -90,7 +73,7 @@ SOURCE_SCHEMA = cv.Any(
CONFIG_SCHEMA = cv.ensure_list(
{
cv.Required(CONF_SOURCE): SOURCE_SCHEMA,
cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, validate_refresh),
cv.Optional(CONF_REFRESH, default="1d"): cv.All(cv.string, cv.source_refresh),
cv.Optional(CONF_COMPONENTS, default="all"): cv.Any(
"all", cv.ensure_list(cv.string)
),
@ -102,65 +85,13 @@ async def to_code(config):
pass
def _compute_destination_path(key: str) -> Path:
base_dir = Path(CORE.config_dir) / ".esphome" / DOMAIN
h = hashlib.new("sha256")
h.update(key.encode())
return base_dir / h.hexdigest()[:8]
def _run_git_command(cmd, cwd=None):
try:
ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False)
except FileNotFoundError as err:
raise cv.Invalid(
"git is not installed but required for external_components.\n"
"Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git"
) from err
if ret.returncode != 0 and ret.stderr:
err_str = ret.stderr.decode("utf-8")
lines = [x.strip() for x in err_str.splitlines()]
if lines[-1].startswith("fatal:"):
raise cv.Invalid(lines[-1][len("fatal: ") :])
raise cv.Invalid(err_str)
def _process_git_config(config: dict, refresh) -> str:
key = f"{config[CONF_URL]}@{config.get(CONF_REF)}"
repo_dir = _compute_destination_path(key)
if not repo_dir.is_dir():
_LOGGER.info("Cloning %s", key)
_LOGGER.debug("Location: %s", repo_dir)
cmd = ["git", "clone", "--depth=1"]
if CONF_REF in config:
cmd += ["--branch", config[CONF_REF]]
cmd += ["--", config[CONF_URL], str(repo_dir)]
_run_git_command(cmd)
else:
# Check refresh needed
file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
# On first clone, FETCH_HEAD does not exists
if not file_timestamp.exists():
file_timestamp = Path(repo_dir / ".git" / "HEAD")
age = datetime.datetime.now() - datetime.datetime.fromtimestamp(
file_timestamp.stat().st_mtime
)
if age.total_seconds() > refresh.total_seconds:
_LOGGER.info("Updating %s", key)
_LOGGER.debug("Location: %s", repo_dir)
# Stash local changes (if any)
_run_git_command(
["git", "stash", "push", "--include-untracked"], str(repo_dir)
)
# Fetch remote ref
cmd = ["git", "fetch", "--", "origin"]
if CONF_REF in config:
cmd.append(config[CONF_REF])
_run_git_command(cmd, str(repo_dir))
# Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch)
_run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
repo_dir = git.clone_or_update(
url=config[CONF_URL],
ref=config.get(CONF_REF),
refresh=refresh,
domain=DOMAIN,
)
if (repo_dir / "esphome" / "components").is_dir():
components_dir = repo_dir / "esphome" / "components"

125
esphome/components/packages/__init__.py
View File

@ -1,6 +1,19 @@
import re
from pathlib import Path
from esphome.core import EsphomeError
from esphome import git, yaml_util
from esphome.const import (
CONF_FILE,
CONF_FILES,
CONF_PACKAGES,
CONF_REF,
CONF_REFRESH,
CONF_URL,
)
import esphome.config_validation as cv
from esphome.const import CONF_PACKAGES
DOMAIN = CONF_PACKAGES
def _merge_package(full_old, full_new):
@ -23,11 +36,119 @@ def _merge_package(full_old, full_new):
return merge(full_old, full_new)
def validate_git_package(config: dict):
new_config = config
for key, conf in config.items():
if CONF_URL in conf:
try:
conf = BASE_SCHEMA(conf)
if CONF_FILE in conf:
new_config[key][CONF_FILES] = [conf[CONF_FILE]]
del new_config[key][CONF_FILE]
except cv.MultipleInvalid as e:
with cv.prepend_path([key]):
raise e
except cv.Invalid as e:
raise cv.Invalid(
"Extra keys not allowed in git based package",
path=[key] + e.path,
) from e
return new_config
def validate_yaml_filename(value):
value = cv.string(value)
if not (value.endswith(".yaml") or value.endswith(".yml")):
raise cv.Invalid("Only YAML (.yaml / .yml) files are supported.")
return value
def validate_source_shorthand(value):
if not isinstance(value, str):
raise cv.Invalid("Shorthand only for strings")
m = re.match(
r"github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+)/([a-zA-Z0-9\-_.\./]+)(?:@([a-zA-Z0-9\-_.\./]+))?",
value,
)
if m is None:
raise cv.Invalid(
"Source is not a file system path or in expected github://username/name/[sub-folder/]file-path.yml[@branch-or-tag] format!"
)
conf = {
CONF_URL: f"https://github.com/{m.group(1)}/{m.group(2)}.git",
CONF_FILE: m.group(3),
}
if m.group(4):
conf[CONF_REF] = m.group(4)
# print(conf)
return BASE_SCHEMA(conf)
BASE_SCHEMA = cv.All(
cv.Schema(
{
cv.Required(CONF_URL): cv.url,
cv.Exclusive(CONF_FILE, "files"): validate_yaml_filename,
cv.Exclusive(CONF_FILES, "files"): cv.All(
cv.ensure_list(validate_yaml_filename),
cv.Length(min=1),
),
cv.Optional(CONF_REF): cv.git_ref,
cv.Optional(CONF_REFRESH, default="1d"): cv.All(
cv.string, cv.source_refresh
),
}
),
cv.has_at_least_one_key(CONF_FILE, CONF_FILES),
)
CONFIG_SCHEMA = cv.All(
cv.Schema(
{
str: cv.Any(validate_source_shorthand, BASE_SCHEMA, dict),
}
),
validate_git_package,
)
def _process_base_package(config: dict) -> dict:
repo_dir = git.clone_or_update(
url=config[CONF_URL],
ref=config.get(CONF_REF),
refresh=config[CONF_REFRESH],
domain=DOMAIN,
)
files: str = config[CONF_FILES]
packages = {}
for file in files:
yaml_file: Path = repo_dir / file
if not yaml_file.is_file():
raise cv.Invalid(f"{file} does not exist in repository", path=[CONF_FILES])
try:
packages[file] = yaml_util.load_yaml(yaml_file)
except EsphomeError as e:
raise cv.Invalid(
f"{file} is not a valid YAML file. Please check the file contents."
) from e
return {"packages": packages}
def do_packages_pass(config: dict):
if CONF_PACKAGES not in config:
return config
packages = config[CONF_PACKAGES]
with cv.prepend_path(CONF_PACKAGES):
packages = CONFIG_SCHEMA(packages)
if not isinstance(packages, dict):
raise cv.Invalid(
"Packages must be a key to value mapping, got {} instead"
@ -37,6 +158,8 @@ def do_packages_pass(config: dict):
for package_name, package_config in packages.items():
with cv.prepend_path(package_name):
recursive_package = package_config
if CONF_URL in package_config:
package_config = _process_base_package(package_config)
if isinstance(package_config, dict):
recursive_package = do_packages_pass(package_config)
config = _merge_package(recursive_package, config)

4
esphome/config.py
View File

@ -333,6 +333,8 @@ def validate_config(config, command_line_substitutions):
result.add_error(err)
return result
CORE.raw_config = config
# 1. Load substitutions
if CONF_SUBSTITUTIONS in config:
from esphome.components import substitutions
@ -348,6 +350,8 @@ def validate_config(config, command_line_substitutions):
result.add_error(err)
return result
CORE.raw_config = config
# 1.1. Check for REPLACEME special value
try:
recursive_check_replaceme(config)

25
esphome/config_validation.py
View File

@ -33,7 +33,6 @@ from esphome.const import (
CONF_UPDATE_INTERVAL,
CONF_TYPE_ID,
CONF_TYPE,
CONF_PACKAGES,
)
from esphome.core import (
CORE,
@ -1455,15 +1454,7 @@ class OnlyWith(Optional):
@property
def default(self):
# pylint: disable=unsupported-membership-test
if self._component in CORE.raw_config or (
CONF_PACKAGES in CORE.raw_config
and self._component
in [
k
for package in CORE.raw_config[CONF_PACKAGES].values()
for k in package.keys()
]
):
if self._component in CORE.raw_config:
return self._default
return vol.UNDEFINED
@ -1633,3 +1624,17 @@ def url(value):
if not parsed.scheme or not parsed.netloc:
raise Invalid("Expected a URL scheme and host")
return parsed.geturl()
def git_ref(value):
if re.match(r"[a-zA-Z0-9\-_.\./]+", value) is None:
raise Invalid("Not a valid git ref")
return value
def source_refresh(value: str):
if value.lower() == "always":
return source_refresh("0s")
if value.lower() == "never":
return source_refresh("1000y")
return positive_time_period_seconds(value)

3
esphome/const.py
View File

@ -235,6 +235,7 @@ CONF_FAN_WITH_COOLING = "fan_with_cooling"
CONF_FAN_WITH_HEATING = "fan_with_heating"
CONF_FAST_CONNECT = "fast_connect"
CONF_FILE = "file"
CONF_FILES = "files"
CONF_FILTER = "filter"
CONF_FILTER_OUT = "filter_out"
CONF_FILTERS = "filters"
@ -525,8 +526,10 @@ CONF_REACTIVE_POWER = "reactive_power"
CONF_REBOOT_TIMEOUT = "reboot_timeout"
CONF_RECEIVE_TIMEOUT = "receive_timeout"
CONF_RED = "red"
CONF_REF = "ref"
CONF_REFERENCE_RESISTANCE = "reference_resistance"
CONF_REFERENCE_TEMPERATURE = "reference_temperature"
CONF_REFRESH = "refresh"
CONF_REPEAT = "repeat"
CONF_REPOSITORY = "repository"
CONF_RESET_PIN = "reset_pin"

74
esphome/git.py Normal file
View File

@ -0,0 +1,74 @@
from pathlib import Path
import subprocess
import hashlib
import logging
from datetime import datetime
from esphome.core import CORE, TimePeriodSeconds
import esphome.config_validation as cv
_LOGGER = logging.getLogger(__name__)
def run_git_command(cmd, cwd=None):
try:
ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False)
except FileNotFoundError as err:
raise cv.Invalid(
"git is not installed but required for external_components.\n"
"Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git"
) from err
if ret.returncode != 0 and ret.stderr:
err_str = ret.stderr.decode("utf-8")
lines = [x.strip() for x in err_str.splitlines()]
if lines[-1].startswith("fatal:"):
raise cv.Invalid(lines[-1][len("fatal: ") :])
raise cv.Invalid(err_str)
def _compute_destination_path(key: str, domain: str) -> Path:
base_dir = Path(CORE.config_dir) / ".esphome" / domain
h = hashlib.new("sha256")
h.update(key.encode())
return base_dir / h.hexdigest()[:8]
def clone_or_update(
*, url: str, ref: str = None, refresh: TimePeriodSeconds, domain: str
) -> Path:
key = f"{url}@{ref}"
repo_dir = _compute_destination_path(key, domain)
if not repo_dir.is_dir():
_LOGGER.info("Cloning %s", key)
_LOGGER.debug("Location: %s", repo_dir)
cmd = ["git", "clone", "--depth=1"]
if ref is not None:
cmd += ["--branch", ref]
cmd += ["--", url, str(repo_dir)]
run_git_command(cmd)
else:
# Check refresh needed
file_timestamp = Path(repo_dir / ".git" / "FETCH_HEAD")
# On first clone, FETCH_HEAD does not exists
if not file_timestamp.exists():
file_timestamp = Path(repo_dir / ".git" / "HEAD")
age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime)
if age.total_seconds() > refresh.total_seconds:
_LOGGER.info("Updating %s", key)
_LOGGER.debug("Location: %s", repo_dir)
# Stash local changes (if any)
run_git_command(
["git", "stash", "push", "--include-untracked"], str(repo_dir)
)
# Fetch remote ref
cmd = ["git", "fetch", "--", "origin"]
if ref is not None:
cmd.append(ref)
run_git_command(cmd, str(repo_dir))
# Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch)
run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir))
return repo_dir