from genericpath import exists
import re
import json
import urllib
from typing import MutableMapping
from sphinx.util import logging
from docutils import nodes
# Instructions for building
# you must have checked out this project in the same folder of
# esphome and esphome-vscode so the SCHEMA_PATH below can find the source schemas
# This file is not processed by default as extension unless added.
# To add this extension from command line use:
# -Dextensions=github,seo,sitemap,components,schema_doc"
# also for improve performance running old version
# -d_build/.doctrees-schema
# will put caches in another dir and not overwrite the ones without schema
SCHEMA_PATH = "../esphome-vscode/server/src/schema/"
CONFIGURATION_VARIABLES = "Configuration variables:"
CONFIGURATION_OPTIONS = "Configuration options:"
PIN_CONFIGURATION_VARIABLES = "Pin configuration variables:"
COMPONENT_HUB = "Component/Hub"
JSON_DUMP_PRETTY = True
class Statistics:
props_documented = 0
enums_good = 0
enums_bad = 0
statistics = Statistics()
logger = logging.getLogger(__name__)
def setup(app):
import os
if not os.path.isfile(SCHEMA_PATH + "esphome.json"):
logger.info(f"{SCHEMA_PATH} not found. Not documenting schema.")
return
app.connect("doctree-resolved", doctree_resolved)
app.connect("build-finished", build_finished)
app.files = {}
return {"version": "1.0.0", "parallel_read_safe": True, "parallel_write_safe": True}
def find_platform_component(app, platform, component):
file_data = get_component_file(app, component)
return file_data[f"{component}.{platform}"]["schemas"]["CONFIG_SCHEMA"]
def doctree_resolved(app, doctree, docname):
if docname == "components/index":
# nothing useful here
return
handle_component(app, doctree, docname)
PLATFORMS_TITLES = {
"Sensor": "sensor",
"Binary Sensor": "binary_sensor",
"Text Sensor": "text_sensor",
"Output": "output",
"Cover": "cover",
"Button": "button",
"Select": "select",
"Fan": "fan",
"Lock": "lock",
"Number": "number",
"Climate": "climate",
"CAN Bus": "canbus",
"Stepper": "stepper",
"Switch": "switch",
"I²C": "i2c",
"Media Player": "media_player",
"Microphone": "microphone",
"Speaker": "speaker",
"Alarm Control Panel": "alarm_control_panel",
"Event": "event",
}
CUSTOM_DOCS = {
"components/globals": {
"Global Variables": "globals.schemas.CONFIG_SCHEMA",
},
"guides/configuration-types": {
"Pin Schema": [
"esp32.pin.schema",
"esp8266.pin.schema",
],
},
"components/binary_sensor/index": {
"Binary Sensor Filters": "binary_sensor.registry.filter",
},
"components/canbus": {
"_LoadSchema": False,
"Base CAN Bus Configuration": "canbus.schemas.CANBUS_SCHEMA",
},
"components/climate/climate_ir": {"_LoadSchema": False, "IR Remote Climate": []},
"components/display/index": {
"Images": "image.schemas.CONFIG_SCHEMA",
"Fonts": "font.schemas.CONFIG_SCHEMA",
"Color": "color.schemas.CONFIG_SCHEMA",
"Animation": "animation.schemas.CONFIG_SCHEMA",
},
"components/light/index": {
"Base Light Configuration": [
"light.schemas.ADDRESSABLE_LIGHT_SCHEMA",
"light.schemas.BINARY_LIGHT_SCHEMA",
"light.schemas.BRIGHTNESS_ONLY_LIGHT_SCHEMA",
"light.schemas.LIGHT_SCHEMA",
],
"Light Effects": "light.registry.effects",
},
"components/light/fastled": {
"_LoadSchema": False,
"Clockless": "fastled_clockless.platform.light.schemas.CONFIG_SCHEMA",
"SPI": "fastled_spi.platform.light.schemas.CONFIG_SCHEMA",
},
"components/binary_sensor/ttp229": {
"_LoadSchema": False,
},
"components/mcp230xx": {
"_LoadSchema": False,
PIN_CONFIGURATION_VARIABLES: "mcp23xxx.pin",
},
"components/mqtt": {
"MQTT Component Base Configuration": "core.schemas.MQTT_COMMAND_COMPONENT_SCHEMA",
"MQTTMessage": "mqtt.schemas.MQTT_MESSAGE_BASE",
},
"components/output/index": {
"Base Output Configuration": "output.schemas.FLOAT_OUTPUT_SCHEMA",
},
"components/remote_transmitter": {
"Remote Transmitter Actions": "remote_base.schemas.BASE_REMOTE_TRANSMITTER_SCHEMA",
},
"components/sensor/index": {
"Sensor Filters": "sensor.registry.filter",
},
"components/time": {
"_LoadSchema": False,
"Base Time Configuration": "time.schemas.TIME_SCHEMA",
"on_time Trigger": "time.schemas.TIME_SCHEMA.schema.config_vars.on_time.schema",
"Home Assistant Time Source": "homeassistant.platform.time.schemas.CONFIG_SCHEMA",
"SNTP Time Source": "sntp.platform.time.schemas.CONFIG_SCHEMA",
"GPS Time Source": "gps.platform.time.schemas.CONFIG_SCHEMA",
"DS1307 Time Source": "ds1307.platform.time.schemas.CONFIG_SCHEMA",
},
"components/wifi": {
"Connecting to Multiple Networks": "wifi.schemas.CONFIG_SCHEMA.schema.config_vars.networks.schema",
"Enterprise Authentication": "wifi.schemas.EAP_AUTH_SCHEMA",
},
"custom/custom_component": {
"Generic Custom Component": "custom_component.schemas.CONFIG_SCHEMA"
},
"components/esp32": {
"Arduino framework": "esp32.schemas.CONFIG_SCHEMA.schema.config_vars.framework.types.arduino",
"ESP-IDF framework": "esp32.schemas.CONFIG_SCHEMA.schema.config_vars.framework.types.esp-idf",
},
"components/sensor/airthings_ble": {
"_LoadSchema": False,
},
"components/sensor/radon_eye_ble": {
"_LoadSchema": False,
},
"components/sensor/xiaomi_ble": {
"_LoadSchema": False,
},
"components/sensor/xiaomi_miscale2": {
"_LoadSchema": False,
},
"components/mcp23Sxx": {
"_LoadSchema": False,
},
"components/display/lcd_display": {"_LoadSchema": False},
"components/display/ssd1306": {"_LoadSchema": False},
"components/display/ssd1322": {"_LoadSchema": False},
"components/display/ssd1325": {"_LoadSchema": False},
"components/display/ssd1327": {"_LoadSchema": False},
"components/display/ssd1351": {"_LoadSchema": False},
"components/copy": {"_LoadSchema": False},
"components/display_menu/index": {
"Display Menu": "display_menu_base.schemas.DISPLAY_MENU_BASE_SCHEMA",
"Select": "display_menu_base.schemas.MENU_TYPES.schema.config_vars.items.types.select",
"Menu": "display_menu_base.schemas.MENU_TYPES.schema.config_vars.items.types.menu",
"Number": "display_menu_base.schemas.MENU_TYPES.schema.config_vars.items.types.number",
"Switch": "display_menu_base.schemas.MENU_TYPES.schema.config_vars.items.types.switch",
"Custom": "display_menu_base.schemas.MENU_TYPES.schema.config_vars.items.types.custom",
},
"components/display_menu/lcd_menu": {
"LCD Menu": "lcd_menu.schemas.CONFIG_SCHEMA",
},
"components/alarm_control_panel/index": {
"Base Alarm Control Panel Configuration": "template.alarm_control_panel.schemas.CONFIG_SCHEMA",
},
"components/vbus": {
"custom VBus sensors": "vbus.platform.sensor.schemas.CONFIG_SCHEMA.types.custom",
"custom VBus binary sensors": "vbus.platform.binary_sensor.schemas.CONFIG_SCHEMA.types.custom",
},
"components/spi": {
"Generic SPI device component:": "spi_device.schemas.CONFIG_SCHEMA"
},
"components/libretiny": {"LibreTiny Platform": "bk72xx.schemas.CONFIG_SCHEMA"},
}
REQUIRED_OPTIONAL_TYPE_REGEX = r"(\(((\*\*Required\*\*)|(\*Optional\*))(,\s(.*))*)\):\s"
def get_node_title(node):
return list(node.traverse(nodes.title))[0].astext()
def read_file(fileName):
f = open(SCHEMA_PATH + fileName + ".json", "r", encoding="utf-8-sig")
str = f.read()
return json.loads(str)
def is_config_vars_title(title_text):
return title_text == CONFIGURATION_VARIABLES or title_text == CONFIGURATION_OPTIONS
class SchemaGeneratorVisitor(nodes.NodeVisitor):
def __init__(self, app, doctree, docname):
nodes.NodeVisitor.__init__(self, doctree)
self.app = app
self.doctree = doctree
self.docname = docname
self.path = docname.split("/")
self.json_component = None
self.props = None
self.platform = None
self.json_platform_component = None
self.title_id = None
self.props_section_title = None
self.find_registry = None
self.component = None
self.section_level = 0
self.file_schema = None
self.custom_doc = CUSTOM_DOCS.get(docname)
if self.path[0] == "components":
if len(self.path) == 2: # root component, e.g. dfplayer, logger
self.component = docname[11:]
if not self.custom_doc or self.custom_doc.get("_LoadSchema", True):
self.file_schema = get_component_file(app, self.component)
self.json_component = self.file_schema[self.component]["schemas"][
"CONFIG_SCHEMA"
]
elif self.path[1] == "display_menu": # weird folder naming
if self.path[2] == "index":
# weird component name mismatch
self.component = "display_menu_base"
else:
self.component = self.path[2]
self.file_schema = get_component_file(app, self.component)
self.json_component = self.file_schema[self.component]["schemas"][
"CONFIG_SCHEMA"
]
else: # sub component, e.g. output/esp8266_pwm
# components here might have a core / hub, eg. dallas, ads1115
# and then they can be a binary_sensor, sensor, etc.
self.platform = self.path[1]
self.component = self.path[2]
if self.component == "ssd1331":
self.component = "ssd1331_spi"
if not self.custom_doc or self.custom_doc.get("_LoadSchema", True):
if self.component == "index":
# these are e.g. sensor, binary sensor etc.
self.component = self.platform.replace(" ", "_").lower()
self.file_schema = get_component_file(app, self.component)
self.json_component = self.file_schema[self.component][
"schemas"
].get(self.component.upper() + "_SCHEMA")
pass
else:
self.json_component = get_component_file(app, self.component)
self.json_platform_component = find_platform_component(
app, self.platform, self.component
)
self.previous_title_text = "No title"
self.is_component_hub = False
# used in custom_docs when titles are mapped to array of components, this
# allows for same configuration text be applied to different json schemas
self.multi_component = None
# a stack for props, used when there are nested props to save high level props.
self.prop_stack = []
# The prop just filled in, used when there are nested props and need to know which
# want to dig
self.current_prop = None
# self.filled_props used to know when any prop is added to props,
# we dont invalidate props on exiting bullet lists but just when entering a new title
self.filled_props = False
# Found a Configuration variables: heading, this is to increase docs consistency
self.accept_props = False
self.bullet_list_level = 0
def set_component_description(self, description, componentName, platformName=None):
if platformName is not None:
platform = get_component_file(self.app, platformName)
platform[platformName]["components"][componentName.lower()][
"docs"
] = description
else:
core = get_component_file(self.app, "esphome")["core"]
if componentName in core["components"]:
core["components"][componentName]["docs"] = description
elif componentName in core["platforms"]:
core["platforms"][componentName]["docs"] = description
else:
if componentName != "display_menu_base":
raise ValueError(
"Cannot set description for component " + componentName
)
def visit_document(self, node):
# ESPHome page docs follows strict formatting guidelines which allows
# for docs to be parsed directly into yaml schema
if self.docname in ["components/sensor/binary_sensor_map"]:
# temporarily not supported
raise nodes.SkipChildren
if self.docname in ["components/climate/climate_ir"]:
# not much to do on the visit to the document, component will be found by title
return
if len(list(node.traverse(nodes.paragraph))) == 0:
# this is empty, not much to do
raise nodes.SkipChildren
self.props_section_title = get_node_title(node)
# Document first paragraph is description of this thing
description = self.getMarkdownParagraph(node)
if self.json_platform_component:
self.set_component_description(description, self.component, self.platform)
elif self.json_component:
self.set_component_description(description, self.component)
# for most components / platforms get the props, this allows for a less restrictive
# first title on the page
if self.json_component or self.json_platform_component:
if is_component_file(
self.app,
self.component,
):
self.props = self.find_props(
self.json_platform_component
if self.json_platform_component
else self.json_component,
True,
)
def visit_table(self, node):
if (
self.docname == "components/climate/climate_ir"
and len(CUSTOM_DOCS["components/climate/climate_ir"]["IR Remote Climate"])
== 0
):
# figure out multi components from table
table_rows = node[0][4]
for row in table_rows:
components_paths = [
components + ".platform.climate.schemas.CONFIG_SCHEMA"
for components in row[1].astext().split("\n")
]
CUSTOM_DOCS["components/climate/climate_ir"][
"IR Remote Climate"
] += components_paths
def depart_document(self, node):
pass
def visit_section(self, node):
self.section_level += 1
section_title = get_node_title(node)
if self.custom_doc and section_title in self.custom_doc:
r = self.custom_doc[section_title]
if ".registry." in r:
self.find_registry = r
def depart_section(self, node):
self.section_level -= 1
if self.section_level == 1:
self.find_registry = None
def unknown_visit(self, node):
pass
def unknown_departure(self, node):
pass
def visit_title(self, node):
title_text = node.astext()
if self.custom_doc is not None and title_text in self.custom_doc:
if isinstance(self.custom_doc[title_text], list):
self.multi_component = self.custom_doc[title_text]
self.filled_props = False
self.props = None
desc = self.getMarkdownParagraph(node.parent)
for c in self.multi_component:
if len(c.split(".")) == 2:
self.set_component_description(desc, c.split(".")[0])
return
json_component = self.find_component(self.custom_doc[title_text])
if not json_component:
return
if self.json_component is None:
self.json_component = json_component
parts = self.custom_doc[title_text].split(".")
if parts[0] not in ["core", "remote_base"] and parts[-1] != "pin":
if parts[1] == "platform":
self.set_component_description(
self.getMarkdownParagraph(node.parent), parts[0], parts[2]
)
else:
self.set_component_description(
self.getMarkdownParagraph(node.parent),
parts[0],
)
self.props_section_title = title_text
self.props = self.find_props(json_component)
return
elif title_text == COMPONENT_HUB:
# here comes docs for the component, make sure we have props of the component
# Needed for e.g. ads1115
self.props_section_title = f"{self.path[-1]} {title_text}"
json_component = self.get_component_schema(
self.path[-1] + ".CONFIG_SCHEMA"
).get("schema", {})
if json_component:
self.props = self.find_props(json_component)
self.set_component_description(
self.getMarkdownParagraph(node.parent), self.path[-1]
)
# mark this to retrieve components instead of platforms
self.is_component_hub = True
elif is_config_vars_title(title_text):
if not self.props and self.multi_component is None:
raise ValueError(
f'Found a "{title_text}": title after {self.previous_title_text}. Unknown object.'
)
elif title_text == "Over SPI" or title_text == "Over I²C":
suffix = "_spi" if "SPI" in title_text else "_i2c"
# these could be platform components, like the display's ssd1306
# but also there are components which are component/hub
# and there are non platform components with the SPI/I2C versions,
# like pn532, those need to be marked with the 'Component/Hub' title
component = self.path[-1] + suffix
self.props_section_title = self.path[-1] + " " + title_text
if self.platform is not None and not self.is_component_hub:
json_platform_component = find_platform_component(
self.app, self.platform, component
)
if not json_platform_component:
raise ValueError(
f"Cannot find platform {self.platform} component '{component}' after found title: '{title_text}'."
)
self.props = self.find_props(json_platform_component)
# Document first paragraph is description of this thing
json_platform_component["docs"] = self.getMarkdownParagraph(node.parent)
else:
json_component = self.get_component_schema(
component + ".CONFIG_SCHEMA"
).get("schema", {})
if not json_component:
raise ValueError(
f"Cannot find component '{component}' after found title: '{title_text}'."
)
self.props = self.find_props(json_component)
# Document first paragraph is description of this thing
self.set_component_description(
self.getMarkdownParagraph(node.parent), component
)
# Title is description of platform component, those ends with Sensor, Binary Sensor, Cover, etc.
elif (
len(
list(
filter(
lambda x: title_text.endswith(x), list(PLATFORMS_TITLES.keys())
)
)
)
> 0
):
if title_text in PLATFORMS_TITLES:
# this omits the name of the component, but we know the platform
platform_name = PLATFORMS_TITLES[title_text]
if self.path[-1] == "index":
component_name = self.path[-2]
else:
component_name = self.path[-1]
self.props_section_title = component_name + " " + title_text
else:
# # title first word is the component name
# component_name = title_text.split(" ")[0]
# # and the rest is the platform
# platform_name = PLATFORMS_TITLES.get(
# title_text[len(component_name) + 1 :]
# )
# if not platform_name:
# # Some general title which does not locate a component directly
# return
# self.props_section_title = title_text
for t in PLATFORMS_TITLES:
if title_text.endswith(t):
component_name = title_text[
0 : len(title_text) - len(t) - 1
].replace(" ", "_")
platform_name = PLATFORMS_TITLES[t]
if not platform_name:
# Some general title which does not locate a component directly
return
self.props_section_title = title_text
if not is_component_file(self.app, component_name):
return
c = find_platform_component(self.app, platform_name, component_name.lower())
if c:
self.json_platform_component = c
self.set_component_description(
self.getMarkdownParagraph(node.parent),
component_name,
platform_name,
)
# Now fill props for the platform element
try:
self.props = self.find_props(self.json_platform_component)
except KeyError:
raise ValueError("Cannot find platform props")
elif title_text.endswith("Component") or title_text.endswith("Bus"):
# if len(path) == 3 and path[2] == 'index':
# # skip platforms index, e.g. sensors/index
# continue
split_text = title_text.split(" ")
self.props_section_title = title_text
# some components are several components in a single platform doc
# e.g. ttp229 binary_sensor has two different named components.
component_name = (
"_".join(split_text[:-1]).lower().replace(".", "").replace("i²c", "i2c")
)
if component_name != self.platform and is_component_file(
self.app, component_name
):
f = get_component_file(self.app, component_name)
# Document first paragraph is description of this thing
description = self.getMarkdownParagraph(node.parent)
if component_name in f:
self.set_component_description(description, component_name)
c = f[component_name]
if c:
self.json_component = c["schemas"]["CONFIG_SCHEMA"]
try:
self.props = self.find_props(self.json_component)
self.multi_component = None
except KeyError:
raise ValueError(
"Cannot find props for component " + component_name
)
return
# component which are platforms in doc, used by: stepper and canbus, lcd_pcf8574
elif f"{component_name}.{self.path[1]}" in f:
self.set_component_description(
description, component_name, self.path[1]
)
self.json_platform_component = f[
f"{component_name}.{self.path[1]}"
]["schemas"]["CONFIG_SCHEMA"]
try:
self.props = self.find_props(self.json_platform_component)
except KeyError:
raise ValueError(
f"Cannot find props for platform {self.path[1]} component {self.component_name}"
)
return
elif title_text.endswith("Trigger"):
# Document first paragraph is description of this thing
description = self.getMarkdownParagraph(node.parent)
split_text = title_text.split(" ")
if len(split_text) != 2:
return
key = split_text[0]
if (
not self.props or not self.props.typed
): # props are right for typed components so far
c = self.json_component
if c:
if self.component in c:
c = c[self.component]["schemas"][
self.component.upper() + "_SCHEMA"
]
trigger_schema = self.find_props(c).get(key)
if trigger_schema is not None:
self.props = self.find_props(trigger_schema)
self.props_section_title = title_text
elif title_text == PIN_CONFIGURATION_VARIABLES:
self.component = self.find_component(self.path[-1] + ".pin")
self.props = self.find_props(self.component)
self.accept_props = True
if not self.component:
raise ValueError(
f'Found a "{PIN_CONFIGURATION_VARIABLES}" entry but could not find pin schema'
)
elif title_text.endswith("Action") or title_text.endswith("Condition"):
# Document first paragraph is description of this thing
description = self.getMarkdownParagraph(node.parent)
split_text = title_text.split(" ")
if len(split_text) != 2:
return
key = split_text[0]
component_parts = split_text[0].split(".")
if len(component_parts) == 3:
try:
cv = get_component_file(self.app, component_parts[1])[
component_parts[1] + "." + component_parts[0]
][split_text[1].lower()][component_parts[2]]
except KeyError:
logger.warn(
f"In {self.docname} cannot found schema of {title_text}"
)
cv = None
if cv is not None:
cv["docs"] = description
self.props = self.find_props(cv.get("schema", {}))
elif len(component_parts) == 2:
registry_name = ".".join(
[component_parts[0], "registry", split_text[1].lower()]
)
key = component_parts[1]
self.find_registry_prop(registry_name, key, description)
else:
registry_name = f"core.registry.{split_text[1].lower()}"
# f"automation.{split_text[1].upper()}_REGISTRY"
self.find_registry_prop(registry_name, key, description)
if self.section_level == 3 and self.find_registry:
name = title_text
if name.endswith(" Effect"):
name = title_text[: -len(" Effect")]
if name.endswith(" Light"):
name = name[: -len(" Light")]
key = name.replace(" ", "_").replace(".", "").lower()
description = self.getMarkdownParagraph(node.parent)
self.find_registry_prop(self.find_registry, key, description)
self.props_section_title = title_text
def get_component_schema(self, name):
parts = name.split(".")
schema_file = get_component_file(self.app, parts[0])
if parts[1] == "registry":
schema = schema_file.get(parts[0], {}).get(parts[2], {})
elif len(parts) == 3:
schema = (
schema_file.get(f"{parts[0]}.{parts[1]}")
.get("schemas", {})
.get(parts[2], {})
)
else:
schema = schema_file.get(parts[0], {}).get("schemas", {}).get(parts[1], {})
return schema
def get_component_config_var(self, name, key):
c = self.get_component_schema(name)
if key in c:
return c[key]
if "config_vars" not in c:
return c
if key in c["config_vars"]:
return c["config_vars"][c]
def find_registry_prop(self, registry_name, key, description):
c = self.get_component_schema(registry_name)
if key in c:
cv = c[key]
if cv is not None:
cv["docs"] = description
self.props = self.find_props(cv.get("schema", {}))
def depart_title(self, node):
if self.filled_props:
self.filled_props = False
self.props = None
self.current_prop = None
self.accept_props = False
self.multi_component = None
self.previous_title_text = node.astext()
self.title_id = node.parent["ids"][0]
def find_props_previous_title(self):
comp = self.json_component or self.json_platform_component
if comp:
props = self.find_props(comp)
if self.previous_title_text in props:
prop = props[self.previous_title_text]
if prop:
self.props = self.find_props(prop)
else:
# return fake dict so better errors are printed
self.props = {"__": "none"}
def visit_Text(self, node):
if self.multi_component:
return
if is_config_vars_title(node.astext()):
if not self.props:
self.find_props_previous_title()
if not self.props:
raise ValueError(
f'Found a "{node.astext()}" entry for unknown object after {self.previous_title_text}'
)
self.accept_props = True
raise nodes.SkipChildren
def depart_Text(self, node):
pass
def visit_paragraph(self, node):
if is_config_vars_title(node.astext()):
if not self.props and not self.multi_component:
self.find_props_previous_title()
if not self.props and not self.multi_component:
logger.info(
f"In {self.docname} / {self.previous_title_text} found a {node.astext()} title and there are no props."
)
# raise ValueError(
# f'Found a "{node.astext()}" entry for unknown object after {self.previous_title_text}'
# )
self.accept_props = True
raise nodes.SkipChildren
def depart_paragraph(self, node):
pass
def visit_bullet_list(self, node):
self.bullet_list_level = self.bullet_list_level + 1
if (
self.current_prop
and (self.props or self.multi_component)
and self.bullet_list_level > 1
):
self.prop_stack.append((self.current_prop, node))
self.accept_props = True
return
if not self.props and self.multi_component is None:
raise nodes.SkipChildren
def depart_bullet_list(self, node):
self.bullet_list_level = self.bullet_list_level - 1
if len(self.prop_stack) > 0:
stack_prop, stack_node = self.prop_stack[-1]
if stack_node == node:
self.prop_stack.pop()
self.filled_props = True
self.current_prop = stack_prop
def visit_list_item(self, node):
if self.accept_props and self.props:
self.filled_props = True
self.current_prop, found = self.update_prop(node, self.props)
if self.current_prop and not found:
logger.info(
f"In '{self.docname} {self.previous_title_text} Cannot find property {self.current_prop}"
)
elif self.multi_component:
# update prop for each component
found_any = False
self.current_prop = None
for c in self.multi_component:
props = self.find_props(self.find_component(c))
self.current_prop, found = self.update_prop(node, props)
if self.current_prop and found:
found_any = True
if self.current_prop and not found_any:
logger.info(
f"In '{self.docname} {self.previous_title_text} Cannot find property {self.current_prop}"
)
self.filled_props = True
def depart_list_item(self, node):
pass
def visit_literal(self, node):
raise nodes.SkipChildren
def depart_literal(self, node):
pass
def getMarkdown(self, node):
from markdown import Translator
t = Translator(
urllib.parse.urljoin(self.app.config.html_baseurl, self.docname + ".html"),
self.doctree,
)
node.walkabout(t)
return t.output.strip("\n")
def getMarkdownParagraph(self, node):
paragraph = list(node.traverse(nodes.paragraph))[0]
markdown = self.getMarkdown(paragraph)
param_type = None
# Check if there is type information for this item
try:
name_type = markdown[: markdown.index(": ") + 2]
ntr = re.search(
REQUIRED_OPTIONAL_TYPE_REGEX,
name_type,
re.IGNORECASE,
)
if ntr:
param_type = ntr.group(6)
if param_type:
markdown = (
f"**{param_type}**: {markdown[markdown.index(': ') + 2 :]}"
)
except ValueError:
# ': ' not found
pass
title = list(node.traverse(nodes.title))[0]
if len(title) > 0:
url = urllib.parse.urljoin(
self.app.config.html_baseurl,
self.docname + ".html#" + title.parent["ids"][0],
)
if (
self.props_section_title is not None
and self.props_section_title.endswith(title.astext())
):
markdown += f"\n\n*See also: [{self.props_section_title}]({url})*"
else:
markdown += f"\n\n*See also: [{self.getMarkdown(title)}]({url})*"
return markdown
def update_prop(self, node, props):
prop_name = None
for s_prop, n in self.prop_stack:
inner = props.get(s_prop)
if inner is not None and "schema" in inner:
props = self.Props(self, inner["schema"])
elif inner is not None and inner.get("type") == "typed":
# this is used in external_components
props = self.Props(self, inner)
elif inner is not None and inner.get("type") == "enum":
enum_raw = self.getMarkdown(node)
# the regex allow the format to have either a ":" or a " -" as the value / docs separator, value must be in `back ticks`
# also description is optional
enum_match = re.search(
r"\* `([^`]*)`((:| -) (.*))*", enum_raw, re.IGNORECASE
)
if enum_match:
enum_value = enum_match.group(1)
enum_docs = enum_match.group(4)
found = False
for name in inner["values"]:
if enum_value.upper().replace(" ", "_") == str(name).upper():
found = True
if enum_docs:
enum_docs = enum_docs.strip()
if inner["values"][name] is None:
inner["values"][name] = {"docs": enum_docs}
else:
inner["values"][name]["docs"] = enum_docs
statistics.props_documented += 1
statistics.enums_good += 1
if not found:
logger.info(
f"In '{self.docname} {self.previous_title_text} Property {s_prop} cannot find enum value {enum_value}"
)
else:
statistics.enums_bad += 1
logger.info(
f"In '{self.docname} {self.previous_title_text} Property {s_prop} unexpected enum member description format"
)
else:
# nothing to do?
return prop_name, False
raw = node.rawsource # this has the full raw rst code for this property
if not raw.startswith("**"):
# not bolded, most likely not a property definition,
# usually texts like 'All properties from...' etc
return prop_name, False
markdown = self.getMarkdown(node)
markdown += f"\n\n*See also: [{self.props_section_title}]({urllib.parse.urljoin(self.app.config.html_baseurl, self.docname +'.html#'+self.title_id)})*"
try:
name_type = markdown[: markdown.index(": ") + 2]
except ValueError:
logger.info(
f"In '{self.docname} {self.previous_title_text} Property format error. Missing ': ' in {raw}'"
)
return prop_name, False
# Example properties formats are:
# **prop_name** (**Required**, string): Long Description...
# **prop_name** (*Optional*, string): Long Description... Defaults to ``value``.
# **prop_name** (*Optional*): Long Description... Defaults to ``value``.
# **prop_name** can be a list of names separated by / e.g. **name1/name2** (*Optional*) see climate/pid/ threshold_low/threshold_high
PROP_NAME_REGEX = r"\*\*(\w*(?:/\w*)*)\*\*"
FULL_ITEM_PROP_NAME_TYPE_REGEX = (
r"\* " + PROP_NAME_REGEX + r"\s" + REQUIRED_OPTIONAL_TYPE_REGEX
)
ntr = re.search(
FULL_ITEM_PROP_NAME_TYPE_REGEX,
name_type,
re.IGNORECASE,
)
if ntr:
prop_name = ntr.group(1)
param_type = ntr.group(7)
else:
s2 = re.search(
FULL_ITEM_PROP_NAME_TYPE_REGEX,
markdown,
re.IGNORECASE,
)
if s2:
# this is e.g. when a property has a list inside, and the list inside are the options.
# just validate **prop_name**
s3 = re.search(r"\* " + PROP_NAME_REGEX + r"*:\s", name_type)
if s3 is not None:
prop_name = s3.group(1)
else:
logger.info(
f"In '{self.docname} {self.previous_title_text} Invalid list format: {node.rawsource}"
)
param_type = None
else:
logger.info(
f"In '{self.docname} {self.previous_title_text} Invalid property format: {node.rawsource}"
)
return prop_name, False
prop_names = str(prop_name)
for k in prop_names.split("/"):
config_var = props.get(k)
if not config_var:
# Create docs for common properties when descriptions are overridden
# in the most specific component.
if k in [
"id",
"name",
"internal",
# i2c
"address",
"i2c_id",
# polling component
"update_interval",
# uart
"uart_id",
# light
"effects",
"gamma_correct",
"default_transition_length",
"flash_transition_length",
"color_correct",
# display
"lambda",
"pages",
"rotation",
# spi
"spi_id",
"cs_pin",
# output (binary/float output)
"inverted",
"power_supply",
# climate
"receiver_id",
]:
config_var = props[k] = {}
else:
if self.path[1] == "esphome" and k in [
# deprecated esphome
"platform",
"board",
"arduino_version",
"esp8266_restore_from_flash",
]:
return prop_name, True
return prop_name, False
desc = markdown[markdown.index(": ") + 2 :].strip()
if param_type:
desc = "**" + param_type + "**: " + desc
config_var["docs"] = desc
statistics.props_documented += 1
return prop_name, True
def find_component(self, component_path):
path = component_path.split(".")
file_content = get_component_file(self.app, path[0])
if path[1] == "platform":
path[2] = f"{path[0]}.{path[2]}"
path = path[2:]
component = file_content
for p in path:
component = component.get(p, {})
return component
class Props(MutableMapping):
"""Smarter props dict.
Props are mostly a dict, however some constructs have two issues:
- An update is intended on an element which does not own a property, but it is based
on an schema that does have the property, those cases are handled by examining the extended
"""
def __init__(self, visitor, component, fail_silently=False):
self.visitor = visitor
self.component = component
self.store = self._get_props(component, fail_silently)
self.parent = None
self.typed = self.component.get("type") == "typed"
def _get_props(self, component, fail_silently):
# component is a 'schema' dict which has 'config_vars' and 'extends'
if not (
"config_vars" in component
or "extends" in component
or len(component) == 0
or component.get("type") == "typed"
):
if fail_silently:
return None
raise ValueError("Unexpected component data to get props")
props = component.get("config_vars")
return props
def _find_extended(self, component, key):
for extended in component.get("extends", []):
c = self.visitor.get_component_schema(extended)
if c.get("type") == "typed":
p = self.visitor.Props(self.visitor, c)
return p[key]
schema = c.get("schema", {})
for k, cv in schema.get("config_vars", {}).items():
if k == key:
return SetObservable(
cv,
setitem_callback=self._set_extended,
inner_key=key,
original_dict=schema.get("config_vars"),
)
ex1 = self._find_extended(schema, key)
if ex1:
return ex1
def _set_extended(self, inner_key, original_dict, key, value):
original_dict[inner_key][key] = value
def _iter_extended(self, component):
for extended in component.get("extends", []):
schema = self.visitor.get_component_schema(extended).get("schema", {})
for s in self._iter_extended(schema):
yield s
yield schema
def __getitem__(self, key):
if self.store and key in self.store:
return self.store[key]
extended = self._find_extended(self.component, key)
if extended is not None:
return extended
if self.component.get("type") == "typed":
return SetObservable(
{key: {"type": "string"}},
setitem_callback=self._set_typed,
inner_key=key,
original_dict={},
)
def _set_typed(self, inner_key, original_dict, key, value):
if inner_key == self.component.get("typed_key", "type"):
self.component[key] = value
else:
for tk, tv in self.component["types"].items():
for cv_k, cv_v in tv["config_vars"].items():
if cv_k == inner_key:
cv_v[key] = value
def __setitem__(self, key, value):
self.store[key] = value
def __delitem__(self, key):
self.store.pop(key)
def __iter__(self):
return iter(self.store)
def __len__(self):
len_extended = 0
if self.component.get("type"):
types = self.component.get("types")
for t, tv in types.items():
for s in self._iter_extended(types.get(t, {})):
len_extended += len(s.get("config_vars", {}))
len_extended += len(tv.get("config_vars", {}))
return len_extended
for s in self._iter_extended(self.component):
len_extended += len(s.get("config_vars", {}))
return len_extended + (len(self.store) if self.store else 0)
def find_props(self, component, fail_silently=False):
if component.get("type") in ["trigger", "schema"]:
# can have schema
if "schema" not in component:
return None
component = component.get("schema")
props = self.Props(self, component, fail_silently)
if props:
self.filled_props = False
self.accept_props = False
self.current_prop = None
return props
def handle_component(app, doctree, docname):
path = docname.split("/")
if path[0] == "components":
pass
elif docname not in CUSTOM_DOCS:
return
try:
v = SchemaGeneratorVisitor(app, doctree, docname)
doctree.walkabout(v)
except Exception as e:
err_str = f"In {docname}.rst: {str(e)}"
# if you put a breakpoint here get call-stack in the console by entering
# import traceback
# traceback.print_exc()
logger.warning(err_str)
def build_finished(app, exception):
# TODO: create report of missing descriptions
for fname, contents in app.files.items():
f = open(SCHEMA_PATH + fname + ".json", "w", newline="\n")
if JSON_DUMP_PRETTY:
f.write(json.dumps(contents, indent=2))
else:
f.write(json.dumps(contents, separators=(",", ":")))
str = f"Documented: {statistics.props_documented} Enums: {statistics.enums_good}/{statistics.enums_bad}"
logger.info(str)
class SetObservable(dict):
"""
a MyDict is like a dict except that when you set an item, before
doing so it will call a callback function that was passed in when the
MyDict instance was created
"""
def __init__(
self,
value,
setitem_callback=None,
inner_key=None,
original_dict=None,
*args,
**kwargs,
):
super(SetObservable, self).__init__(value, *args, **kwargs)
self._setitem_callback = setitem_callback
self.inner_key = inner_key
self.original_dict = original_dict
def __setitem__(self, key, value):
if self._setitem_callback:
self._setitem_callback(self.inner_key, self.original_dict, key, value)
super(SetObservable, self).__setitem__(key, value)
def is_component_file(app: SchemaGeneratorVisitor, component):
if component == "core" or component == "automation":
component = "esphome"
return exists(SCHEMA_PATH + component + ".json")
def get_component_file(app: SchemaGeneratorVisitor, component):
if component == "core" or component == "automation":
component = "esphome"
if component not in app.files:
app.files[component] = read_file(component)
return app.files[component]