From 234810e6f5e13cb525c9b549b0cd8a12d64b4996 Mon Sep 17 00:00:00 2001 From: Guillermo Ruffino Date: Sun, 7 Mar 2021 21:25:11 -0300 Subject: [PATCH] Dump schema (#1030) * updates * updates * add verify missing descriptions * updates * schema handling updates * add action and condition * add more checks for missing props * updates * support typed schema * better inner props handling and mqtt * updates * clean up * add links to titles on markdown * remove spelling not used * format black * format black * Better titles --- conf.py | 1 + markdown.py | 194 +++++++++++ schema_doc.py | 943 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1138 insertions(+) create mode 100644 markdown.py create mode 100644 schema_doc.py diff --git a/conf.py b/conf.py index 4b02a1e8e..3f2f786b7 100644 --- a/conf.py +++ b/conf.py @@ -40,6 +40,7 @@ extensions = [ "github", "seo", "sitemap", + "schema_doc", ] # Add any paths that contain templates here, relative to this directory. diff --git a/markdown.py b/markdown.py new file mode 100644 index 000000000..1043117e8 --- /dev/null +++ b/markdown.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python +# reStructuredText (RST) to GitHub-flavored Markdown converter + +import re +import sys +from docutils import core, nodes, writers +from urllib import parse + + +class Translator(nodes.NodeVisitor): + def __init__(self, base_url, document): + nodes.NodeVisitor.__init__(self, document) + self.output = "" + self.indent = 0 + self.preserve_newlines = False + self.base_url = base_url + + def write(self, text): + self.output += text.replace("\n", "\n" + " " * self.indent) + + def visit_document(self, node): + pass + + def depart_document(self, node): + pass + + def visit_section(self, node): + pass + + def depart_section(self, node): + # Skip all sections except the first one. + raise nodes.StopTraversal + + def visit_title(self, node): + self.version = re.match(r"(\d+\.\d+\.\d+).*", node.children[0]).group(1) + raise nodes.SkipChildren + + def visit_title_reference(self, node): + raise Exception( + f'Markdown conversion: Improper format or not supported feature in "{node.parent.rawsource}". Check "`"' + ) + + def depart_title(self, node): + pass + + def visit_Text(self, node): + if not self.preserve_newlines: + node = node.replace("\n", " ") + self.write(node) + + def depart_Text(self, node): + pass + + def visit_bullet_list(self, node): + self.write("\n") + pass + + def depart_bullet_list(self, node): + pass + + def visit_list_item(self, node): + self.write("* ") + self.indent += 2 + + def depart_list_item(self, node): + self.indent -= 2 + self.write("\n\n") + + def visit_paragraph(self, node): + pass + + def depart_paragraph(self, node): + pass + + def visit_reference(self, node): + # if not is_github_ref(node): + self.write("[") + + def depart_reference(self, node): + # if not is_github_ref(node): + # self.write('](' + node['refuri'] + ')') + refid = node.get("refid") + if refid: + self.write("](" + parse.urljoin(self.base_url, "#" + refid) + ")") + else: + refuri = node.get("refuri") + if refuri: + # if refuri.startswith('../'): + # refuri = parse.urljoin(self.base_url, refuri) + # self.write('](' + refuri + ')') + self.write("](" + parse.urljoin(self.base_url, refuri) + ")") + + def visit_emphasis(self, node): + self.write("*") + pass + + def depart_emphasis(self, node): + self.write("*") + pass + + def visit_target(self, node): + pass + + def depart_target(self, node): + pass + + def visit_literal(self, node): + self.write("`") + + def depart_literal(self, node): + self.write("`") + + def visit_literal_block(self, node): + self.write("\n\n```") + if "c++" in node["classes"]: + self.write("c++") + self.write("\n") + self.preserve_newlines = True + + def depart_literal_block(self, node): + self.write("\n```\n") + self.preserve_newlines = False + + def visit_inline(self, node): + pass + + def depart_inline(self, node): + pass + + def visit_image(self, node): + self.write("![](" + node["uri"] + ")") + + def depart_image(self, node): + pass + + def write_row(self, row, widths): + for i, entry in enumerate(row): + text = entry[0][0] if len(entry) > 0 else "" + if i != 0: + self.write("|") + self.write("{:{}}".format(text, widths[i])) + self.write("\n") + + def visit_table(self, node): + table = node.children[0] + colspecs = table[:-2] + thead = table[-2] + tbody = table[-1] + widths = [int(cs["colwidth"]) for cs in colspecs] + sep = "|".join(["-" * w for w in widths]) + "\n" + self.write("\n\n") + self.write_row(thead[0], widths) + self.write(sep) + for row in tbody: + self.write_row(row, widths) + raise nodes.SkipChildren + + def depart_table(self, node): + pass + + def visit_strong(self, node): + self.write("**") + pass + + def depart_strong(self, node): + self.write("**") + pass + + def visit_block_quote(self, node): + pass + + def depart_block_quote(self, node): + pass + + +class MDWriter(writers.Writer): + """GitHub-flavored markdown writer""" + + supported = ("md",) + """Formats this writer supports.""" + + def translate(self): + translator = Translator(self.document) + self.document.walkabout(translator) + self.output = (translator.output, translator.version) + + +def convert(rst_path): + """Converts RST file to Markdown.""" + return core.publish_file(source_path=rst_path, writer=MDWriter()) + + +if __name__ == "__main__": + convert(sys.argv[1]) diff --git a/schema_doc.py b/schema_doc.py new file mode 100644 index 000000000..fc3f9c54f --- /dev/null +++ b/schema_doc.py @@ -0,0 +1,943 @@ +import re +import json +import urllib + +from typing import MutableMapping +from sphinx.util import logging +from docutils import nodes + +SCHEMA_PATH = "../esphome_devices/schema.json" +CONFIGURATION_VARIABLES = "Configuration variables:" +COMPONENT_HUB = "Component/Hub" + +props_missing = 0 +props_verified = 0 +props_documented = 0 + + +def setup(app): + """Setup connects events to the sitemap builder""" + + import os + + if not os.path.isfile(SCHEMA_PATH): + logger = logging.getLogger(__name__) + logger.info(f"{SCHEMA_PATH} not found. Not documenting schema.") + else: + app.connect("doctree-resolved", doctree_resolved) + app.connect("build-finished", build_finished) + + f = open(SCHEMA_PATH, "r", encoding="utf-8-sig") + str = f.read() + app.jschema = json.loads(str) + + return {"version": "1.0.0", "parallel_read_safe": True, "parallel_write_safe": True} + + +def find_component(jschema, component): + return jschema["properties"].get(component) + + +def find_platform_component(jschema, platform, component): + platform_items = jschema["properties"][platform].get("items") + if not platform_items: + return None + ar = platform_items["allOf"] + for p in ar: + if "if" in p: + if p["if"]["properties"]["platform"]["const"] == component: + return p + + +def doctree_resolved(app, doctree, docname): + if docname == "components/index": + # nothing useful here + return + try: + handle_component(app, doctree, docname) + + except Exception as e: + err_str = f"In {docname}: {str(e)}" + logger = logging.getLogger(__name__) + logger.warning(err_str) + + +PLATFORMS_TITLES = { + "Sensor": "sensor", + "Binary Sensor": "binary_sensor", + "Text Sensor": "text_sensor", + "Output": "output", + "Cover": "cover", + "Climate": "climate", + "CAN Bus": "canbus", + "Stepper": "stepper", +} + +CUSTOM_DOCS = { + "guides/automations": {"Global Variables": "properties/globals"}, + "guides/configuration-types": { + "Color": "properties/color", + "Pin Schema": "definitions/PIN.GPIO_FULL_INPUT_PIN_SCHEMA", + }, + "components/climate/ir_climate": { + "IR Remote Climate": [ + "properties/climate/coolix", + "properties/climate/daikin", + "properties/climate/fujitsu_general", + "properties/climate/mitsubishi", + "properties/climate/tcl112", + "properties/climate/toshiba", + "properties/climate/yashima", + "properties/climate/whirlpool", + "properties/climate/climate_ir_lg", + "properties/climate/hitachi_ac344", + ] + }, + "components/display/index": { + "Images": "properties/image", + "Drawing Static Text": "properties/font", + "Color": "properties/color", + "Animation": "properties/animation", + }, + "components/light/index": { + "Base Light Configuration": [ + "definitions/light.ADDRESSABLE_LIGHT_SCHEMA", + "definitions/light.BINARY_LIGHT_SCHEMA", + "definitions/light.BRIGHTNESS_ONLY_LIGHT_SCHEMA", + "definitions/light.LIGHT_SCHEMA", + ] + }, + "components/light/fastled": { + "Clockless": "properties/light/fastled_clockless", + "SPI": "properties/light/fastled_spi", + }, + "components/mqtt": { + "MQTT Component Base Configuration": "definitions/CONFIG.MQTT_COMMAND_COMPONENT_SCHEMA", + }, + "components/output/index": { + "Base Output Configuration": "definitions/output.FLOAT_OUTPUT_SCHEMA" + }, + "components/remote_transmitter": { + "Remote Transmitter Actions": "definitions/REMOTE_BASE.BASE_REMOTE_TRANSMITTER_SCHEMA" + }, + "components/time": { + "Home Assistant Time Source": "properties/time/homeassistant", + "SNTP Time Source": "properties/time/sntp", + "GPS Time Source": "properties/time/gps", + "DS1307 Time Source": "properties/time/ds1307", + }, + "components/wifi": { + "Connecting to Multiple Networks": "definitions/wifi-networks", + "Enterprise Authentication": "definitions/wifi-networks/eap", + }, + "custom/custom_component": { + "Generic Custom Component": "properties/custom_component" + }, +} + + +class SchemaGeneratorVisitor(nodes.NodeVisitor): + def __init__(self, app, doctree, docname): + nodes.NodeVisitor.__init__(self, doctree) + self.app = app + self.doctree = doctree + self.docname = docname + self.path = docname.split("/") + self.json_component = None + self.props = None + self.platform = None + self.json_platform_component = None + self.json_base_config = None + self.title_id = None + self.props_section_title = None + if self.path[0] == "components": + if len(self.path) == 2: # root component, e.g. dfplayer, logger + component = docname[11:] + self.json_component = app.jschema["properties"].get(component) + else: # sub component, e.g. output/esp8266_pwm + + # components here might have a core / hub, eg. dallas, ads1115 + # and then they can be a binary_sensor, sensor, etc. + + component = self.path[2] + + if component == "ssd1331": + component = "ssd1331_spi" + + self.platform = self.path[1] + if component == "index": + # these are e.g. sensor, binary sensor etc. + p = self.platform.replace(" ", "_").lower() + self.json_component = find_component( + self.app.jschema, self.platform + ) + self.json_base_config = self.app.jschema["definitions"].get( + p + "." + p.upper() + "_SCHEMA" + ) + + else: + self.json_component = find_component(self.app.jschema, component) + + self.json_platform_component = find_platform_component( + self.app.jschema, self.platform, component + ) + + self.custom_doc = CUSTOM_DOCS.get(docname) + + self.previous_title_text = "No title" + + self.is_component_hub = False + + # used in custom_docs when titles are mapped to array of components, this + # allows for same configuration text be applied to different json schemas + self.multi_component = None + + # a stack for props, used when there are nested props to save high level props. + self.prop_stack = [] + + # The prop just filled in, used when there are nested props and need to know which + # want to dig + self.current_prop = None + + # self.filled_props used to know when any prop is added to props, + # we dont invalidate props on exiting bullet lists but just when entering a new title + self.filled_props = False + + # Found a Configuration variables: heading, this is to increase docs consistency + self.accept_props = False + + self.props_level = 0 + + def visit_document(self, node): + # ESPHome page docs follows strict formating guidelines which allows + # for docs to be parsed directly into yaml schema + + if self.docname in ["components/sensor/binary_sensor_map"]: + # temporarly not supported + raise nodes.SkipChildren + + if len(list(node.traverse(nodes.paragraph))) == 0: + # this is empty, not much to do + raise nodes.SkipChildren + + # Document first paragraph is description of this thing + description = self.getMarkdownParagraph(node) + + if self.json_platform_component: + self.json_platform_component["markdownDescription"] = description + elif self.json_component: + self.json_component["markdownDescription"] = description + + if self.json_base_config: + self.json_component = self.json_base_config + + # for most components / platforms get the props, this allows for a less restrictive + # first title on the page + if self.json_component or self.json_platform_component: + self.props = self.find_props( + self.json_platform_component + if self.json_platform_component + else self.json_component + ) + + if not self.props: + # get props for base components, Sensor, Binary Sensor, Light, etc. + + if len(self.path) == 2: + # "#/definitions/schema_canbus.CONFIG_SCHEMA" + self.json_base_config = self.app.jschema["definitions"].get( + f"{self.path[1]}.{self.path[1].upper()}_SCHEMA" + ) + if self.json_base_config: + self.props = self.find_props(self.json_base_config) + + def depart_document(self, node): + pass + + def visit_SEONode(self, node): + pass + + def depart_SEONode(self, node): + pass + + def visit_literal_block(self, node): + pass + + def depart_literal_block(self, node): + pass + + def visit_section(self, node): + pass + + def depart_section(self, node): + pass + + def unknown_visit(self, node): + pass + + def unknown_departure(self, node): + pass + + def visit_title(self, node): + title_text = node.astext() + if self.props_section_title is None: + self.props_section_title = title_text + + if "interval" in title_text: + title_text = title_text + if self.custom_doc is not None and title_text in self.custom_doc: + if isinstance(self.custom_doc[title_text], list): + self.multi_component = self.custom_doc[title_text] + + # TODO: add same markdown description to each? + + return + json_component = self.find_component(self.custom_doc[title_text]) + json_component["markdownDescription"] = self.getMarkdownParagraph( + node.parent + ) + self.props_section_title = title_text + self.props = self.find_props(json_component) + + return + + if title_text == COMPONENT_HUB: + # here comes docs for the component, make sure we have props of the component + # Needed for e.g. ads1115 + self.props_section_title = f"{self.path[-1]} {title_text}" + json_component = find_component(self.app.jschema, self.path[-1]) + if json_component: + self.props = self.find_props(json_component) + + json_component["markdownDescription"] = self.getMarkdownParagraph( + node.parent + ) + + # mark this to retrieve components instead of platforms + self.is_component_hub = True + + if title_text == CONFIGURATION_VARIABLES: + if not self.props and self.multi_component is None: + raise ValueError( + f"Found a Configuration variables: title after {self.previous_title_text}. Unkown object." + ) + + if title_text == "Over SPI" or title_text == "Over I²C": + suffix = "_spi" if "SPI" in title_text else "_i2c" + + # these could be platform components, like the display's ssd1306 + # but also there are components which are component/hub + # and there are non platform components with the SPI/I2C versions, + # like pn532, those need to be marked with the 'Component/Hub' title + component = self.path[-1] + suffix + + self.props_section_title = self.path[-1] + " " + title_text + + if self.platform is not None and not self.is_component_hub: + json_platform_component = find_platform_component( + self.app.jschema, self.platform, component + ) + if not json_platform_component: + raise ValueError( + f"Cannot find platform {self.platform} component '{component}' after found title: '{title_text}'." + ) + self.props = self.find_props(json_platform_component) + + # Document first paragraph is description of this thing + json_platform_component[ + "markdownDescription" + ] = self.getMarkdownParagraph(node.parent) + + else: + json_component = find_component(self.app.jschema, component) + if not json_component: + raise ValueError( + f"Cannot find component '{component}' after found title: '{title_text}'." + ) + self.props = self.find_props(json_component) + + # Document first paragraph is description of this thing + json_component["markdownDescription"] = self.getMarkdownParagraph( + node.parent + ) + + # Title is description of platform component, those ends with Sensor, Binary Sensor, Cover, etc. + if ( + len( + list( + filter( + lambda x: title_text.endswith(x), list(PLATFORMS_TITLES.keys()) + ) + ) + ) + > 0 + ): + if title_text in PLATFORMS_TITLES: + # this omits the name of the component, but we know the platform + platform_name = PLATFORMS_TITLES[title_text] + component_name = self.path[-1] + self.props_section_title = self.path[-1] + " " + title_text + else: + # title first word is the component name + component_name = title_text.split(" ")[0] + # and the rest is the platform + platform_name = PLATFORMS_TITLES.get( + title_text[len(component_name) + 1 :] + ) + if not platform_name: + # Some general title which does not locate a component directly + return + self.props_section_title = title_text + c = find_platform_component( + self.app.jschema, platform_name, component_name.lower() + ) + if c: + self.json_platform_component = c + + c["markdownDescription"] = self.getMarkdownParagraph(node.parent) + + # Now fill props for the platform element + try: + self.props = self.find_props(self.json_platform_component) + except KeyError: + raise ValueError("Cannot find platform props") + + if title_text.endswith("Component") or title_text.endswith("Bus"): + # if len(path) == 3 and path[2] == 'index': + # # skip platforms index, e.g. sensors/index + # continue + split_text = title_text.split(" ") + self.props_section_title = title_text + if len(split_text) == 2: + # some components are several components in a single platform doc + # e.g. ttp229 binary_sensor has two different named components. + component_name = split_text[0].lower().replace(".", "") + if component_name.lower() == self.platform: + return + c = find_component(self.app.jschema, component_name) + if c: + self.json_component = c + try: + self.props = self.find_props(self.json_component) + except KeyError: + raise ValueError( + "Cannot find props for component " + component_name + ) + return + c = find_platform_component( + self.app.jschema, self.path[1], component_name + ) + if c: + self.json_platform_component = c + try: + self.props = self.find_props(self.json_platform_component) + + except KeyError: + raise ValueError( + f"Cannot find props for platform {self.path[1]} component {self.component_name}" + ) + return + + if title_text.endswith("Action") or title_text.endswith("Condition"): + # Document first paragraph is description of this thing + description = self.getMarkdownParagraph(node.parent) + split_text = title_text.split(" ") + if len(split_text) != 2: + return + key = split_text[0] + registry_name = f"automation.{split_text[1].upper()}_REGISTRY" + registry = self.app.jschema["definitions"][registry_name]["anyOf"] + for action in registry: + if key in action["properties"]: + action["properties"][key]["markdownDescription"] = description + self.props = self.find_props(action["properties"][key]) + break + self.props_section_title = title_text + + if title_text.endswith("Trigger"): + # Document first paragraph is description of this thing + description = self.getMarkdownParagraph(node.parent) + split_text = title_text.split(" ") + if len(split_text) != 2: + return + key = split_text[0] + + # handles Time / on_time + c = self.json_base_config or self.json_component + if c: + trigger_schema = self.find_props(c).get(key) + self.props = self.find_props(trigger_schema) + self.props_section_title = title_text + + if self.docname == "components/light/index" and title_text.endswith("Effect"): + # Document first paragraph is description of this thing + + description = self.getMarkdownParagraph(node.parent) + name = title_text[: -len(" Effect")] + # accept Light Effect as ending (Automation Light Effect) + if name.endswith(" Light"): + name = name[: -len(" Light")] + + key = name.replace(" ", "_").replace(".", "").lower() + registry = self.app.jschema["definitions"]["light.EFFECTS_REGISTRY"][ + "anyOf" + ] + self.props_section_title = title_text + + for effect in registry: + if key in effect["properties"]: + effect["properties"][key]["markdownDescription"] = description + self.props = self.find_props(effect["properties"][key]) + return + raise ValueError("Cannot find Effect " + title_text) + + def depart_title(self, node): + if self.filled_props: + self.filled_props = False + self.props = None + self.current_prop = None + self.accept_props = False + self.multi_component = None + self.previous_title_text = node.astext() + self.title_id = node.parent["ids"][0] + + def find_props_previous_title(self): + comp = self.json_component or self.json_platform_component + if comp: + props = self.find_props(comp) + + if self.previous_title_text in props: + prop = props[self.previous_title_text] + if prop: + self.props = self.find_props(prop) + else: + # return fake dict so better errors are printed + self.props = {"__": "none"} + + def visit_Text(self, node): + if self.multi_component: + return + if node.astext() == "Configuration variables:": + if not self.props: + self.find_props_previous_title() + if not self.props: + raise ValueError( + f'Found a "Configuration variables:" entry for unknown object after {self.previous_title_text}' + ) + self.accept_props = True + raise nodes.SkipChildren + + def depart_Text(self, node): + pass + + def visit_paragraph(self, node): + if node.astext() == "Configuration variables:": + if not self.props: + self.find_props_previous_title() + if not self.props: + raise ValueError( + f'Found a "Configuration variables:" entry for unknown object after {self.previous_title_text}' + ) + self.accept_props = True + raise nodes.SkipChildren + + def depart_paragraph(self, node): + pass + + def visit_bullet_list(self, node): + self.props_level = self.props_level + 1 + if self.current_prop and self.props and self.props_level > 1: + + # if '$ref' in self.props[self.current_prop]: + # if self.props[self.current_prop]['$ref'].endswith('_SCHEMA'): + # nowhere put this props info... + # raise nodes.SkipChildren + + # this can be list of values, list of subproperties + + # deep configs + # e.g. wifi manual_ip, could also be a enum list + + # if this prop has a reference + prop = self.props[self.current_prop] + if isinstance(prop, dict): + if "$ref" in prop: + ref = self.get_ref(prop) + self.prop_stack.append(self.props) + self.props = self.find_props(ref) + self.accept_props = True + elif "properties" in prop: + self.prop_stack.append(self.props) + self.props = prop["properties"] + elif ( + "anyOf" in prop + and len(prop["anyOf"]) > 0 + and isinstance(self.get_ref(prop["anyOf"][0]), dict) + and "$ref" in self.get_ref(prop["anyOf"][0]) + ): + ref = self.get_ref(prop["anyOf"][0]) + self.prop_stack.append(self.props) + self.props = self.find_props(ref) + # nowhere put this props info... + # otherwise inner bullet list will be interpreted as property list + if self.props_level > 1: + raise nodes.SkipChildren + else: + # nowhere put this props info... + # otherwise inner bullet list will be interpreted as property list + if self.props_level > 1: + raise nodes.SkipChildren + + if not self.props and self.multi_component is None: + raise nodes.SkipChildren + + def depart_bullet_list(self, node): + self.props_level = self.props_level - 1 + if len(self.prop_stack) > 0: + self.props = self.prop_stack.pop() + self.filled_props = True + + def visit_list_item(self, node): + if self.accept_props and self.props: + self.filled_props = True + try: + self.current_prop = self.update_prop(node, self.props) + # print( + # f'{self.current_prop} updated from {node.rawsource[:40]}') + except Exception as e: + raise ValueError(f"In '{self.previous_title_text}' {str(e)}") + + elif self.multi_component: + # update prop for each component + for c in self.multi_component: + props = self.find_props(self.find_component(c)) + self.current_prop = self.update_prop(node, props) + self.filled_props = True + + def depart_list_item(self, node): + pass + + def visit_literal(self, node): + raise nodes.SkipChildren + + def depart_literal(self, node): + pass + + def getMarkdown(self, node): + from markdown import Translator + + t = Translator( + urllib.parse.urljoin(self.app.config.html_baseurl, self.docname + ".html"), + self.doctree, + ) + node.walkabout(t) + return t.output + + def getMarkdownParagraph(self, node): + paragraph = list(node.traverse(nodes.paragraph))[0] + markdown = self.getMarkdown(paragraph) + + title = list(node.traverse(nodes.title))[0] + if len(title) > 0: + url = urllib.parse.urljoin( + self.app.config.html_baseurl, + self.docname + ".html#" + title.parent["ids"][0], + ) + markdown += f"\n\n*See also: [{self.props_section_title}]({url})*" + return markdown + + def update_prop(self, node, props): + prop_name = None + + raw = node.rawsource # this has the full raw rst code for this property + + if not raw.startswith("**"): + # not bolded, most likely not a property definition, + # usually texts like 'All properties from...' etc + return None + + markdown = self.getMarkdown(node) + + markdown += f"\n\n*See also: [{self.props_section_title}]({urllib.parse.urljoin(self.app.config.html_baseurl, self.docname +'.html#'+self.title_id)})*" + + try: + name_type = markdown[: markdown.index(": ") + 2] + except ValueError: + raise ValueError(f'Property format error. Missing ": " in {raw}') + + # Example properties formats are: + # **name** (**Required**, string): Long Description... + # **name** (*Optional*, string): Long Description... Defaults to ``value``. + # **name** (*Optional*): Long Description... Defaults to ``value``. + + if "ads111" in self.docname: + self.docname = self.docname + + ntr = re.search( + r"\* \*\*(\w*)\*\*\s(\(((\*\*Required\*\*)|(\*Optional\*))(,\s(.*))*)\):\s", + name_type, + re.IGNORECASE, + ) + + if ntr: + prop_name = ntr.group(1) + param_type = ntr.group(7) + else: + s2 = re.search( + r"\* \*\*(\w*)\*\*\s(\(((\*\*Required\*\*)|(\*Optional\*))(,\s(.*))*)\):\s", + markdown, + re.IGNORECASE, + ) + if s2: + # this is e.g. when a property has a list inside, and the list inside are the options. + # just validate **prop_name** + s3 = re.search(r"\* \*\*(\w*)\*\*:\s", name_type) + prop_name = s3.group(1) + param_type = None + else: + raise ValueError(f"Invalid property format: {node.rawsource}") + + k = str(prop_name) + jprop = props.get(k) + if not jprop: + + # do not fail for common properties, + # however this should check prop is valid upstream + + if k in [ + "id", + "name", + "internal", + # i2c + "address", + "i2c_id", + # polling component + "update_interval", + # uart + "uart_id", + # ligth + "effects", + "gamma_correct", + "default_transition_length", + "color_correct", + # display + "lambda", + "dither", + "pages", + "rotation", + # spi + "spi_id", + "cs_pin", + # output (binary/float output) + "inverted", + "power_supply", + # climate + "receiver_id", + ]: + return + else: + raise ValueError(f"Cannot find property {k}") + + desc = markdown[markdown.index(": ") + 2 :].strip() + if param_type: + desc = param_type + ": " + desc + + # if '$ref' in jprop: + # self.get_ref(jprop)["markdownDescription"] = desc + # else: + jprop["markdownDescription"] = desc + global props_documented + props_documented = props_documented + 1 + + return prop_name + + def get_ref(self, node): + ref = node.get("$ref") + if ref and ref.startswith("#/definitions/"): + return self.app.jschema["definitions"][ref[14:]] + + def find_component(self, component_path): + path = component_path.split("/") + json_component = self.app.jschema[path[0]][path[1]] + + if len(path) > 2: + # a property path + props = self.find_props(json_component) + if props: + json_component = props.get(path[2]) + else: + # a platform sub element + json_component = find_platform_component( + self.app.jschema, path[1], path[2] + ) + return json_component + + class Props(MutableMapping): + """Smarter props dict. + + Props are mostly a dict, however some constructs have two issues: + - An update is intended on an element which does not own a property, but it is based + on an schema that does have the property, those cases can be handled + - An update is done in a typed schema + """ + + def __init__(self, visitor, component): + self.visitor = visitor + self.component = component + self.store = self._get_props(component) + self.parent = None + + def _get_props(self, component): + # find properties + if "then" in component: + component = component["then"] + props = component.get("properties") + ref = None + if not props: + arr = component.get("anyOf", component.get("allOf")) + if not arr: + if "$ref" in component: + return self._get_props(self.visitor.get_ref(component)) + return None + for x in arr: + props = x.get("properties") + if not ref: + ref = self.visitor.get_ref(x) + if props: + break + if not props and ref: + props = self._get_props(ref) + return props + + def __getitem__(self, key): + if self.store and key in self.store: + return self.store[key] + + if "then" in self.component: + # check if it's typed + schemas = self.component["then"].get("allOf") + if ( + isinstance(schemas, list) + and "properties" in schemas[0] + and "type" in schemas[0]["properties"] + ): + for s in schemas: + if "then" in s: + props = self._get_props(s.get("then")) + if key in props: + return SetObservable( + props[key], + setitem_callback=self._update_typed, + inner_key=key, + ) + return # key not found + + # check if it's a registry and need to reset store + # e.g. remote_receiver binary sensor + if "$ref" in self.component["then"]: + ref = self.visitor.get_ref(self.component["then"]) + prop_set = ref.get("anyOf") + if isinstance(prop_set, list): + for k in prop_set: + if key in k["properties"]: + self.store = k["properties"] + return self.store[key] + + def _update_typed(self, inner_key, key, value): + # Make sure we update all types + if "then" in self.component: + schemas = self.component["then"].get("allOf") + assert "type" in schemas[0].get("properties") + for s in schemas: + if "then" in s: + props = self._get_props(s.get("then")) + if inner_key in props: + props[inner_key][key] = value + + def __setitem__(self, key, value): + self.store[key] = value + + def __delitem__(self, key): + self.store.pop(key) + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) if self.store else 0 + + def find_props(self, component): + props = self.Props(self, component) + + if props: + self.filled_props = False + self.accept_props = False + self.current_prop = None + + return props + return + # find properties + if "then" in component: + component = component["then"] + props = component.get("properties") + ref = None + if not props: + arr = component.get("anyOf", component.get("allOf")) + if not arr: + if "$ref" in component: + return self.find_props(self.get_ref(component)) + return None + for x in arr: + props = x.get("properties") + if not ref: + ref = self.get_ref(x) + if props: + break + if not props and ref: + props = self.find_props(ref) + + if props: + self.filled_props = False + self.accept_props = False + self.current_prop = None + return props + + +def handle_component(app, doctree, docname): + path = docname.split("/") + if path[0] == "components": + pass + elif docname not in CUSTOM_DOCS: + return + + v = SchemaGeneratorVisitor(app, doctree, docname) + doctree.walkabout(v) + + +def build_finished(app, exception): + # TODO: create report of missing descriptions + + f = open(SCHEMA_PATH, "w") + f.write(json.dumps(app.jschema)) + + str = f"Documented: {props_documented}" + logger = logging.getLogger(__name__) + logger.info(str) + + +class SetObservable(dict): + """ + a MyDict is like a dict except that when you set an item, before + doing so it will call a callback function that was passed in when the + MyDict instance was created + """ + + def __init__(self, value, setitem_callback=None, inner_key=None, *args, **kwargs): + super(SetObservable, self).__init__(value, *args, **kwargs) + self._setitem_callback = setitem_callback + self.inner_key = inner_key + + def __setitem__(self, key, value): + if self._setitem_callback: + self._setitem_callback(self.inner_key, key, value) + super(SetObservable, self).__setitem__(key, value)