From f5ce2e594bc879f1bc8124f3849c9ade30145692 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 28 Dec 2021 09:18:23 +0100 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Docutils=20parser?= =?UTF-8?q?=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move Myst config above RST. Add logic for handling `line_length_limit` and `raw_enabled` settings, in-line with `docutils/parsers/recommonmark_wrapper.py` --- myst_parser/docutils_.py | 121 +++++++++++++++++++++++++++------------ setup.cfg | 1 + 2 files changed, 86 insertions(+), 36 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index b937ec97..3aeec3f5 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -3,13 +3,14 @@ .. include:: path/to/file.md :parser: myst_parser.docutils_ """ -from typing import Any, Callable, Iterable, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union from attr import Attribute from docutils import frontend, nodes from docutils.core import default_description, publish_cmdline from docutils.parsers.rst import Parser as RstParser from markdown_it.token import Token +from typing_extensions import Literal, get_args, get_origin from myst_parser.docutils_renderer import DocutilsRenderer from myst_parser.main import MdParserConfig, create_md_parser @@ -22,6 +23,19 @@ def _validate_int( return int(value) +def _create_validate_choice(choices: Sequence[str]) -> Callable[..., str]: + """Create a validator for a choice from a sequence of strings.""" + + def _validate( + setting, value, option_parser, config_parser=None, config_section=None + ): + if value not in choices: + raise ValueError(f"Expecting one of {choices!r}, got {value}.") + return value + + return _validate + + def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]: """Create a validator for a tuple of length `length`.""" @@ -68,10 +82,8 @@ def __repr__(self): """Names of settings that cannot be set in docutils.conf.""" -def _docutils_optparse_options_of_attribute( - at: Attribute, default: Any -) -> Tuple[dict, str]: - """Convert an ``MdParserConfig`` attribute into a Docutils optparse options dict.""" +def _attr_to_optparse_option(at: Attribute, default: Any) -> Tuple[dict, str]: + """Convert an ``attrs.Attribute`` into a Docutils optparse options dict.""" if at.type is int: return {"validator": _validate_int}, f"(type: int, default: {default})" if at.type is bool: @@ -80,7 +92,14 @@ def _docutils_optparse_options_of_attribute( }, f"(type: bool, default: {default})" if at.type is str: return {}, f"(type: str, default: '{default}')" - if at.type == Iterable[str] or at.name == "url_schemes": + if get_origin(at.type) is Literal and all( + isinstance(a, str) for a in get_args(at.type) + ): + args = get_args(at.type) + return { + "validator": _create_validate_choice(args), + }, f"(type: {'|'.join(args)}, default: {default!r})" + if at.type in (Iterable[str], Sequence[str]): return { "validator": frontend.validate_comma_separated_list }, f"(type: comma-delimited, default: '{','.join(default)}')" @@ -88,57 +107,64 @@ def _docutils_optparse_options_of_attribute( return { "validator": _create_validate_tuple(2) }, f"(type: str,str, default: '{','.join(default)}')" - if at.type == Union[int, type(None)] and at.default is None: + if at.type == Union[int, type(None)]: return { "validator": _validate_int, - "default": None, }, f"(type: null|int, default: {default})" - if at.type == Union[Iterable[str], type(None)] and at.default is None: + if at.type == Union[Iterable[str], type(None)]: + default_str = ",".join(default) if default else "" return { "validator": frontend.validate_comma_separated_list, - "default": None, - }, f"(type: comma-delimited, default: '{default or ','.join(default)}')" + }, f"(type: null|comma-delimited, default: {default_str!r})" raise AssertionError( f"Configuration option {at.name} not set up for use in docutils.conf." - f"Either add {at.name} to docutils_.DOCUTILS_EXCLUDED_ARGS," - "or add a new entry in _docutils_optparse_of_attribute." ) -def _docutils_setting_tuple_of_attribute( - attribute: Attribute, default: Any -) -> Tuple[str, Any, Any]: - """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple.""" - name = f"myst_{attribute.name}" +def attr_to_optparse_option( + attribute: Attribute, default: Any, prefix: str = "myst_" +) -> Tuple[str, List[str], Dict[str, Any]]: + """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple. + + :returns: A tuple of ``(help string, option flags, optparse kwargs)``. + """ + name = f"{prefix}{attribute.name}" flag = "--" + name.replace("_", "-") options = {"dest": name, "default": DOCUTILS_UNSET} - at_options, type_str = _docutils_optparse_options_of_attribute(attribute, default) + at_options, type_str = _attr_to_optparse_option(attribute, default) options.update(at_options) help_str = attribute.metadata.get("help", "") if attribute.metadata else "" return (f"{help_str} {type_str}", [flag], options) -def _myst_docutils_setting_tuples(): - """Return a list of Docutils setting for the MyST section.""" - defaults = MdParserConfig() +def create_myst_settings_spec( + excluded: Sequence[str], config_cls=MdParserConfig, prefix: str = "myst_" +): + """Return a list of Docutils setting for the docutils MyST section.""" + defaults = config_cls() return tuple( - _docutils_setting_tuple_of_attribute(at, getattr(defaults, at.name)) - for at in MdParserConfig.get_fields() - if at.name not in DOCUTILS_EXCLUDED_ARGS + attr_to_optparse_option(at, getattr(defaults, at.name), prefix) + for at in config_cls.get_fields() + if at.name not in excluded ) -def create_myst_config(settings: frontend.Values): - """Create a ``MdParserConfig`` from the given settings.""" +def create_myst_config( + settings: frontend.Values, + excluded: Sequence[str], + config_cls=MdParserConfig, + prefix: str = "myst_", +): + """Create a configuration instance from the given settings.""" values = {} - for attribute in MdParserConfig.get_fields(): - if attribute.name in DOCUTILS_EXCLUDED_ARGS: + for attribute in config_cls.get_fields(): + if attribute.name in excluded: continue - setting = f"myst_{attribute.name}" + setting = f"{prefix}{attribute.name}" val = getattr(settings, setting, DOCUTILS_UNSET) if val is not DOCUTILS_UNSET: values[attribute.name] = val - return MdParserConfig(**values) + return config_cls(**values) class Parser(RstParser): @@ -148,10 +174,10 @@ class Parser(RstParser): """Aliases this parser supports.""" settings_spec = ( - *RstParser.settings_spec, "MyST options", None, - _myst_docutils_setting_tuples(), + create_myst_settings_spec(DOCUTILS_EXCLUDED_ARGS), + *RstParser.settings_spec, ) """Runtime settings specification.""" @@ -165,11 +191,26 @@ def parse(self, inputstring: str, document: nodes.document) -> None: :param inputstring: The source string to parse :param document: The root docutils node to add AST elements to """ + + # check for exorbitantly long lines + for i, line in enumerate(inputstring.split("\n")): + if len(line) > document.settings.line_length_limit: + error = document.reporter.error( + f"Line {i+1} exceeds the line-length-limit:" + f" {document.settings.line_length_limit}." + ) + document.append(error) + return + + # create parsing configuration try: - config = create_myst_config(document.settings) - except (TypeError, ValueError) as error: - document.reporter.error(f"myst configuration invalid: {error.args[0]}") + config = create_myst_config(document.settings, DOCUTILS_EXCLUDED_ARGS) + except Exception as exc: + error = document.reporter.error(f"myst configuration invalid: {exc}") + document.append(error) config = MdParserConfig() + + # parse content parser = create_md_parser(config, DocutilsRenderer) parser.options["document"] = document env: dict = {} @@ -180,6 +221,14 @@ def parse(self, inputstring: str, document: nodes.document) -> None: tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens parser.renderer.render(tokens, parser.options, env) + # post-processing + + # replace raw nodes if raw is not allowed + if not document.settings.raw_enabled: + for node in document.traverse(nodes.raw): + warning = document.reporter.warning("Raw content disabled.") + node.parent.replace(node, warning) + def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): """Run the command line interface for a particular writer.""" diff --git a/setup.cfg b/setup.cfg index 4f7ddcbe..7b554f79 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ install_requires = mdit-py-plugins~=0.3.0 pyyaml sphinx>=3.1,<5 + typing-extensions python_requires = >=3.6 include_package_data = True zip_safe = True From 5bba10ebe9f30f0d65d446512830f6e3772474d8 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 28 Dec 2021 09:28:10 +0100 Subject: [PATCH 2/6] Update docutils_.py --- myst_parser/docutils_.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 3aeec3f5..f13f9d7d 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -192,6 +192,8 @@ def parse(self, inputstring: str, document: nodes.document) -> None: :param document: The root docutils node to add AST elements to """ + self.setup_parse(inputstring, document) + # check for exorbitantly long lines for i, line in enumerate(inputstring.split("\n")): if len(line) > document.settings.line_length_limit: @@ -229,6 +231,8 @@ def parse(self, inputstring: str, document: nodes.document) -> None: warning = document.reporter.warning("Raw content disabled.") node.parent.replace(node, warning) + self.finish_parse() + def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): """Run the command line interface for a particular writer.""" From 54dabd89b374bc3d6b8e2de40bfbb7adb9e7f2a7 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 28 Dec 2021 09:34:09 +0100 Subject: [PATCH 3/6] Update docutils_.py --- myst_parser/docutils_.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index f13f9d7d..f1e631f3 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -195,14 +195,15 @@ def parse(self, inputstring: str, document: nodes.document) -> None: self.setup_parse(inputstring, document) # check for exorbitantly long lines - for i, line in enumerate(inputstring.split("\n")): - if len(line) > document.settings.line_length_limit: - error = document.reporter.error( - f"Line {i+1} exceeds the line-length-limit:" - f" {document.settings.line_length_limit}." - ) - document.append(error) - return + if hasattr(document.settings, "line_length_limit"): + for i, line in enumerate(inputstring.split("\n")): + if len(line) > document.settings.line_length_limit: + error = document.reporter.error( + f"Line {i+1} exceeds the line-length-limit:" + f" {document.settings.line_length_limit}." + ) + document.append(error) + return # create parsing configuration try: @@ -226,7 +227,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None: # post-processing # replace raw nodes if raw is not allowed - if not document.settings.raw_enabled: + if not getattr(document.settings, "raw_enabled", True): for node in document.traverse(nodes.raw): warning = document.reporter.warning("Raw content disabled.") node.parent.replace(node, warning) From 6faee5de2b005410de20c2b9a4d9df8350cd779c Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 28 Dec 2021 09:44:26 +0100 Subject: [PATCH 4/6] Update test_docutils.py --- tests/test_docutils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 52d32810..46d10462 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -1,11 +1,14 @@ import io from textwrap import dedent +import attr import pytest from docutils import VersionInfo, __version_info__ +from typing_extensions import Literal from myst_parser.docutils_ import ( Parser, + attr_to_optparse_option, cli_html, cli_html5, cli_latex, @@ -15,6 +18,15 @@ from myst_parser.docutils_renderer import make_document +def test_attr_to_optparse_option(): + @attr.s + class Config: + name: Literal["a"] = attr.ib(default="default") + + output = attr_to_optparse_option(attr.fields(Config).name, "default") + assert len(output) == 3 + + def test_parser(): """Test calling `Parser.parse` directly.""" parser = Parser() From 8b6d2d8beb1dca588b281d26b860c0458457841b Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 28 Dec 2021 09:58:37 +0100 Subject: [PATCH 5/6] Update docutils_.py --- myst_parser/docutils_.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index f1e631f3..418ab6dd 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -85,37 +85,46 @@ def __repr__(self): def _attr_to_optparse_option(at: Attribute, default: Any) -> Tuple[dict, str]: """Convert an ``attrs.Attribute`` into a Docutils optparse options dict.""" if at.type is int: - return {"validator": _validate_int}, f"(type: int, default: {default})" + return {"metavar": "", "validator": _validate_int}, f"(default: {default})" if at.type is bool: return { - "validator": frontend.validate_boolean - }, f"(type: bool, default: {default})" + "metavar": "", + "validator": frontend.validate_boolean, + }, f"(default: {default})" if at.type is str: - return {}, f"(type: str, default: '{default}')" + return { + "metavar": "", + }, f"(default: '{default}')" if get_origin(at.type) is Literal and all( isinstance(a, str) for a in get_args(at.type) ): args = get_args(at.type) return { - "validator": _create_validate_choice(args), - }, f"(type: {'|'.join(args)}, default: {default!r})" + "metavar": f"<{'|'.join(repr(a) for a in args)}>", + "type": "choice", + "choices": args, + }, f"(default: {default!r})" if at.type in (Iterable[str], Sequence[str]): return { - "validator": frontend.validate_comma_separated_list - }, f"(type: comma-delimited, default: '{','.join(default)}')" + "metavar": "", + "validator": frontend.validate_comma_separated_list, + }, f"(default: '{','.join(default)}')" if at.type == Tuple[str, str]: return { - "validator": _create_validate_tuple(2) - }, f"(type: str,str, default: '{','.join(default)}')" + "metavar": "", + "validator": _create_validate_tuple(2), + }, f"(default: '{','.join(default)}')" if at.type == Union[int, type(None)]: return { + "metavar": "", "validator": _validate_int, - }, f"(type: null|int, default: {default})" + }, f"(default: {default})" if at.type == Union[Iterable[str], type(None)]: default_str = ",".join(default) if default else "" return { + "metavar": "", "validator": frontend.validate_comma_separated_list, - }, f"(type: null|comma-delimited, default: {default_str!r})" + }, f"(default: {default_str!r})" raise AssertionError( f"Configuration option {at.name} not set up for use in docutils.conf." ) From fb7e41a015b19703ed6bed68e7e0608d5b020393 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Tue, 28 Dec 2021 10:02:37 +0100 Subject: [PATCH 6/6] Update docutils_.py --- myst_parser/docutils_.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 418ab6dd..49502cd7 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -23,19 +23,6 @@ def _validate_int( return int(value) -def _create_validate_choice(choices: Sequence[str]) -> Callable[..., str]: - """Create a validator for a choice from a sequence of strings.""" - - def _validate( - setting, value, option_parser, config_parser=None, config_section=None - ): - if value not in choices: - raise ValueError(f"Expecting one of {choices!r}, got {value}.") - return value - - return _validate - - def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]: """Create a validator for a tuple of length `length`."""