diff --git a/.github/workflows/docutils_setup.py b/.github/workflows/docutils_setup.py index a9f8f2fa..9d745222 100755 --- a/.github/workflows/docutils_setup.py +++ b/.github/workflows/docutils_setup.py @@ -44,12 +44,12 @@ def modify_readme(content: str) -> str: if __name__ == "__main__": project_path = sys.argv[1] readme_path = sys.argv[2] - with open(project_path, "r") as f: + with open(project_path) as f: content = f.read() content = modify_toml(content) with open(project_path, "w") as f: f.write(content) - with open(readme_path, "r") as f: + with open(readme_path) as f: content = f.read() content = modify_readme(content) with open(readme_path, "w") as f: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 20d067a9..4bbe10c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -89,7 +89,7 @@ jobs: - name: Install dependencies run: | pip install . - pip install pytest~=6.2 pytest-param-files~=0.3.3 docutils==${{ matrix.docutils-version }} + pip install pytest~=6.2 pytest-param-files~=0.3.3 pygments docutils==${{ matrix.docutils-version }} - name: ensure sphinx is not installed run: | python -c "\ @@ -100,7 +100,7 @@ jobs: else: raise AssertionError()" - name: Run pytest for docutils-only tests - run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py + run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py - name: Run docutils CLI run: echo "test" | myst-docutils-html diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 589cd3e0..a0ca0fc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,12 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/asottile/pyupgrade + rev: v2.32.0 + hooks: + - id: pyupgrade + args: [--py37-plus] + - repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: @@ -44,7 +50,7 @@ repos: - id: mypy args: [--config-file=pyproject.toml] additional_dependencies: - - sphinx~=3.3 + - sphinx~=4.1 - markdown-it-py>=1.0.0,<3.0.0 - mdit-py-plugins~=0.3.0 files: > diff --git a/.readthedocs.yml b/.readthedocs.yml index cb2ff6ae..43b85263 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,7 +1,7 @@ version: 2 python: - version: 3 + version: "3" install: - method: pip path: . diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 118c5a83..f126fba7 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,11 +1,24 @@ -.bg-myst-one { - background-color: #52d16f3b; +/** Add a counter before subsections **/ +h1 { + counter-reset: subsection; + text-decoration: underline; } - -.bg-myst-two { - background-color: #e7dd7b73; +h2 { + counter-reset: subsubsection; +} +h2::before { + counter-increment: subsection; + content: counter(subsection) ". "; +} +h3::before { + counter-increment: subsubsection; + content: counter(subsection) "." counter(subsubsection) ". "; } -.bg-myst-three { - background-color: #e7b07b96; +/** No icon for admonitions with no-icon class */ +.admonition > .admonition-title, div.admonition.no-icon > .admonition-title::before { + content: ""; +} +.admonition > .admonition-title, div.admonition.no-icon > .admonition-title { + padding-left: .6rem; } diff --git a/docs/api/index.md b/docs/api/index.md deleted file mode 100644 index c646a562..00000000 --- a/docs/api/index.md +++ /dev/null @@ -1,14 +0,0 @@ -(api/main)= - -# MyST-Parser Python API - -MyST-Parser also has a Python API *via* the `myst_parser` package. -This allows you to parse MyST Markdown and render various outputs with Python. - -```{toctree} -:maxdepth: 2 - -parsers.md -renderers.md -reference.rst -``` diff --git a/docs/api/parsers.md b/docs/api/parsers.md deleted file mode 100644 index b675db2c..00000000 --- a/docs/api/parsers.md +++ /dev/null @@ -1,301 +0,0 @@ -# Parse MyST Markdown - -```{seealso} -- The MyST Parser package heavily uses the the [markdown-it-py](https://github.com/executablebooks/markdown-it-py) package. -- {ref}`The MyST-Parser API reference ` contains a more complete reference of this API. -``` - -## Parsing and rendering helper functions - -The MyST Parser comes bundled with some helper functions to quickly parse MyST Markdown and render its output. - -:::{important} -These APIs are primarily intended for testing and development purposes. -For proper parsing see {ref}`myst-sphinx` and {ref}`myst-docutils`. -::: - -### Parse MyST Markdown to HTML - -The following code parses markdown and renders as HTML using only the markdown-it parser -(i.e. no sphinx or docutils specific processing is done): - -```python -from myst_parser.main import to_html -to_html("some *text* {literal}`a`") -``` - - -```html -'

some text {literal}[a]

\n' -``` - - -### Parse MyST Markdown to docutils - -The following function renders your text as **docutils AST objects** (for example, for use with the Sphinx ecosystem): - -```python -from myst_parser.main import to_docutils -print(to_docutils("some *text* {literal}`a`").pformat()) -``` - -```xml - - - some - - text - - - a -``` - -:::{note} -This function only performs the initial parse of the AST, -without applying any transforms or post-processing. -See for example the [Sphinx core events](https://www.sphinx-doc.org/en/master/extdev/appapi.html?highlight=config-inited#sphinx-core-events). -::: - -### Parse MyST Markdown as `markdown-it` tokens - -The MyST Parser uses `markdown-it-py` tokens as an intermediate representation of your text. -Normally these tokens are then *rendered* into various outputs. -If you'd like direct access to the tokens, use the `to_tokens` function. -Here's an example of its use: - -```python -from pprint import pprint -from myst_parser.main import to_tokens - -for token in to_tokens("some *text*"): - print(token, "\n") -``` - - -```python -Token(type='paragraph_open', tag='p', nesting=1, attrs=None, map=[0, 1], level=0, children=None, content='', markup='', info='', meta={}, block=True, hidden=False) - -Token(type='inline', tag='', nesting=0, attrs=None, map=[0, 1], level=1, children=[Token(type='text', tag='', nesting=0, attrs=None, map=None, level=0, children=None, content='some ', markup='', info='', meta={}, block=False, hidden=False), Token(type='em_open', tag='em', nesting=1, attrs=None, map=None, level=0, children=None, content='', markup='*', info='', meta={}, block=False, hidden=False), Token(type='text', tag='', nesting=0, attrs=None, map=None, level=1, children=None, content='text', markup='', info='', meta={}, block=False, hidden=False), Token(type='em_close', tag='em', nesting=-1, attrs=None, map=None, level=0, children=None, content='', markup='*', info='', meta={}, block=False, hidden=False)], content='some *text*', markup='', info='', meta={}, block=True, hidden=False) - -Token(type='paragraph_close', tag='p', nesting=-1, attrs=None, map=None, level=0, children=None, content='', markup='', info='', meta={}, block=True, hidden=False) -``` - - -Each token is an abstract representation of a piece of MyST Markdown syntax. - -## Use the parser object for more control - -The MyST Parser is actually a `markdown-it-py` parser with several extensions pre-enabled that support the MyST syntax. -If you'd like more control over the parsing process, then you can directly use a `markdown-it-py` parser with MyST syntax extensions loaded. - -:::{seealso} -[`markdown-it-py`](https://markdown-it-py.readthedocs.io/) is an extensible Python parser and renderer for flavors of markdown. -It is inspired heavily by the [`markdown-it`](https://github.com/markdown-it/markdown-it) Javascript package. -See the documentation of these tools for more information. -::: - -### Load a parser - -To load one of these parsers for your own use, use the `create_md_parser` function. -Below we'll create such a parser and show that it is an instance of a `markdown-it-py` parser: - -```python -from markdown_it.renderer import RendererHTML -from myst_parser.main import create_md_parser, MdParserConfig -config = MdParserConfig() -parser = create_md_parser(config, RendererHTML) -parser -``` - - -```python -markdown_it.main.MarkdownIt() -``` - - -### List the active rules - -We can list the **currently active rules** for this parser. -Each rules maps onto a particular markdown syntax, and a Token. -To list the active rules, use the `get_active_rules` method: - -```python -pprint(parser.get_active_rules()) -``` - - -```python -{'block': ['front_matter', - 'table', - 'code', - 'math_block_label', - 'math_block', - 'fence', - 'myst_line_comment', - 'blockquote', - 'myst_block_break', - 'myst_target', - 'hr', - 'list', - 'footnote_def', - 'reference', - 'heading', - 'lheading', - 'html_block', - 'paragraph'], - 'core': ['normalize', 'block', 'inline'], - 'inline': ['text', - 'newline', - 'math_inline', - 'math_single', - 'escape', - 'myst_role', - 'backticks', - 'emphasis', - 'link', - 'image', - 'footnote_ref', - 'autolink', - 'html_inline', - 'entity'], - 'inline2': ['balance_pairs', 'emphasis', 'text_collapse']} -``` - - -### Parse and render markdown - -Once we have a Parser instance, we can use it to parse some markdown. -Use the `render` function to do so: - -```python -parser.render("*abc*") -``` - - -```html -'

abc

\n' -``` - - -### Disable and enable rules - -You can disable and enable rules for a parser using the `disable` and `enable` methods. -For example, below we'll disable the `emphasis` rule (which is what detected the `*abc*` syntax above) and re-render the text: - -```python -parser.disable("emphasis").render("*abc*") -``` - - -```html -'

*abc*

\n' -``` - - -As you can see, the parser no longer detected the `**` syntax as requiring an _emphasis_. - -### Turn off all block-level syntax - -If you'd like to use your parser *only* for in-line content, you may turn off all block-level syntax with the `renderInline` method: - -```python -parser.enable("emphasis").renderInline("- *abc*") -``` - - -```html -'- abc' -``` - - - -## The Token Stream - -When you parse markdown with the MyST Parser, the result is a flat stream of **Tokens**. -These are abstract representations of each type of syntax that the parser has detected. - -For example, below we'll show the token stream for some simple markdown: - -```python -from myst_parser.main import to_tokens -tokens = to_tokens(""" -Here's some *text* - -1. a list - -> a *quote*""") -[t.type for t in tokens] -``` - - -```python -['paragraph_open', - 'inline', - 'paragraph_close', - 'ordered_list_open', - 'list_item_open', - 'paragraph_open', - 'inline', - 'paragraph_close', - 'list_item_close', - 'ordered_list_close', - 'blockquote_open', - 'paragraph_open', - 'inline', - 'paragraph_close', - 'blockquote_close'] -``` - - -Note that these tokens are **flat**, although some of the tokens refer to one another (for example, Tokens with `_open` and `_close` represent the start/end of blocks). - -Tokens of type `inline` will have a `children` attribute that contains a list of the Tokens that they contain. -For example: - -```python -tokens[6] -``` - - -```python -Token(type='inline', tag='', nesting=0, attrs=None, map=[3, 4], level=3, children=[Token(type='text', tag='', nesting=0, attrs=None, map=None, level=0, children=None, content='a list', markup='', info='', meta={}, block=False, hidden=False)], content='a list', markup='', info='', meta={}, block=True, hidden=False) -``` - - -### Rendering tokens - -The list of Token objects can be rendered to a number of different outputs. -This involves first processing the Tokens, and then defining how each should be rendered in an output format (e.g., HTML or Docutils). - -For example, the sphinx renderer first converts the token to a nested structure, collapsing the opening/closing tokens into single tokens: - -```python -from markdown_it.token import nest_tokens -nested = nest_tokens(tokens) -[t.type for t in nested] -``` - - -```python -['paragraph_open', 'ordered_list_open', 'blockquote_open'] -``` - - -```python -print(nested[0].opening, end="\n\n") -print(nested[0].closing, end="\n\n") -print(nested[0].children, end="\n\n") -``` - - -```python -Token(type='paragraph_open', tag='p', nesting=1, attrs=None, map=[1, 2], level=0, children=None, content='', markup='', info='', meta={}, block=True, hidden=False) - -Token(type='paragraph_close', tag='p', nesting=-1, attrs=None, map=None, level=0, children=None, content='', markup='', info='', meta={}, block=True, hidden=False) - -[Token(type='inline', tag='', nesting=0, attrs=None, map=[1, 2], level=1, children=[Token(type='text', tag='', nesting=0, attrs=None, map=None, level=0, children=None, content="Here's some ", markup='', info='', meta={}, block=False, hidden=False), NestedTokens(opening=Token(type='em_open', tag='em', nesting=1, attrs=None, map=None, level=0, children=None, content='', markup='*', info='', meta={}, block=False, hidden=False), closing=Token(type='em_close', tag='em', nesting=-1, attrs=None, map=None, level=0, children=None, content='', markup='*', info='', meta={}, block=False, hidden=False), children=[Token(type='text', tag='', nesting=0, attrs=None, map=None, level=1, children=None, content='text', markup='', info='', meta={}, block=False, hidden=False)])], content="Here's some *text*", markup='', info='', meta={}, block=True, hidden=False)] -``` - - -It then renders each token to a Sphinx-based docutils object. -See [the renderers section](renderers.md) for more information about rendering tokens. diff --git a/docs/api/reference.rst b/docs/api/reference.rst index ffd17dd6..9ea58983 100644 --- a/docs/api/reference.rst +++ b/docs/api/reference.rst @@ -1,29 +1,49 @@ -============= -API Reference -============= +.. _api/main: -.. _api/directive: +========== +Python API +========== -Directive Parsing Reference ---------------------------- +Source text parsers +------------------- -.. automodule:: myst_parser.parse_directives - :members: +.. _api/docutils_parser: + +Docutils +........ + +.. autoclass:: myst_parser.docutils_.Parser + :members: parse + :undoc-members: + :member-order: bysource + :show-inheritance: + +.. _api/sphinx_parser: + +Sphinx +...... -MyST Renderers --------------- +.. autoclass:: myst_parser.parsers.sphinx_.MystParser + :members: supported, parse + :undoc-members: + :member-order: bysource + :show-inheritance: + :exclude-members: __init__ +.. _api/renderers: + +Markdown-it to docutils +----------------------- These renderers take the markdown-it parsed token stream and convert it to the docutils AST. The sphinx renderer is a subclass of the docutils one, -with some additional methods only available *via* sphinx -.e.g. multi-document cross-referencing. +with some additional methods only available *via* sphinx e.g. multi-document cross-referencing. Docutils ........ -.. autoclass:: myst_parser.docutils_renderer.DocutilsRenderer +.. autoclass:: myst_parser.mdit_to_docutils.base.DocutilsRenderer :special-members: __output__, __init__ :members: render, nested_render_text, add_line_and_source_path, current_node_context :undoc-members: @@ -34,15 +54,22 @@ Docutils Sphinx ...... -.. autoclass:: myst_parser.sphinx_renderer.SphinxRenderer +.. autoclass:: myst_parser.mdit_to_docutils.sphinx_.SphinxRenderer :special-members: __output__ :members: render_internal_link, render_math_block_label :undoc-members: :member-order: alphabetical :show-inheritance: -Mocking -....... +.. _api/directive: + +Directive and role processing +----------------------------- + +This module processes the content of a directive: + +.. automodule:: myst_parser.parsers.directives + :members: These classes are parsed to sphinx roles and directives, to mimic the original docutls rST specific parser elements, @@ -67,44 +94,3 @@ but instead run nested parsing with the markdown parser. :members: :undoc-members: :show-inheritance: - - -Additional Methods -.................. - -.. autofunction:: myst_parser.docutils_renderer.make_document - -.. autofunction:: myst_parser.docutils_renderer.html_meta_to_nodes - -.. autofunction:: myst_parser.sphinx_renderer.minimal_sphinx_app - -.. autofunction:: myst_parser.sphinx_renderer.mock_sphinx_env - - -.. _api/docutils_parser: - -Docutils Parser Reference -------------------------- - -.. autoclass:: myst_parser.docutils_.Parser - :members: parse - :undoc-members: - :member-order: bysource - :show-inheritance: - -.. _api/sphinx_parser: - -Sphinx Parser Reference ------------------------ - -This class builds on the :py:class:`~myst_parser.sphinx_renderer.SphinxRenderer` -to generate a parser for Sphinx, using the :ref:`Sphinx parser API `: - -.. autoclass:: myst_parser.sphinx_parser.MystParser - :members: supported, parse - :undoc-members: - :member-order: bysource - :show-inheritance: - :exclude-members: __init__ - -.. _api/renderers: diff --git a/docs/api/renderers.md b/docs/api/renderers.md deleted file mode 100644 index bdc9d60f..00000000 --- a/docs/api/renderers.md +++ /dev/null @@ -1,146 +0,0 @@ -# Render outputs - -There are a few different ways to render MyST Parser tokens into different outputs. -This section covers a few common ones. - -## The `docutils` renderer - -The `myst_parser.docutils_renderer.DocutilsRenderer` converts a token directly to the `docutils.document` representation of the document, converting roles and directives to a `docutils.nodes` if a converter can be found for the given name. - -````python -from myst_parser.main import to_docutils - -document = to_docutils(""" -Here's some *text* - -1. a list - -> a quote - -{emphasis}`content` - -```{sidebar} my sidebar -content -``` -""") - -print(document.pformat()) -```` - -```xml - - - Here's some - - text - - - - a list - - - a quote - - - content - - - my sidebar - <paragraph> - content -``` - -## The Sphinx renderer - -The `myst_parser.sphinx_renderer.SphinxRenderer` builds on the `DocutilsRenderer` to add sphinx specific nodes, e.g. for cross-referencing between documents. - -To use the sphinx specific roles and directives outside of a `sphinx-build`, they must first be loaded with the `in_sphinx_env` option. - -````python -document = to_docutils(""" -Here's some *text* - -1. a list - -> a quote - -{ref}`target` - -```{glossary} my gloassary -name - definition -``` -""", - in_sphinx_env=True) -print(document.pformat()) -```` - -```xml -<document source="notset"> - <paragraph> - Here's some - <emphasis> - text - <enumerated_list> - <list_item> - <paragraph> - a list - <block_quote> - <paragraph> - a quote - <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="target" reftype="ref" refwarn="True"> - <inline classes="xref std std-ref"> - target - <glossary> - <definition_list classes="glossary"> - <definition_list_item> - <term ids="term-my-gloassary"> - my gloassary - <index entries="('single',\ 'my\ gloassary',\ 'term-my-gloassary',\ 'main',\ None)"> - <term ids="term-name"> - name - <index entries="('single',\ 'name',\ 'term-name',\ 'main',\ None)"> - <definition> - <paragraph> - definition -``` - -### Set Sphinx configuration for testing - -You can also set Sphinx configuration *via* `sphinx_conf`. This is a dictionary representation of the contents of the Sphinx `conf.py`. - -```{warning} -This feature is only meant for simple testing. -It will fail for extensions that require the full -Sphinx build process and/or access to external files. -``` - -`````python -document = to_docutils(""" -````{tabs} - -```{tab} Apples - -Apples are green, or sometimes red. -``` -```` -""", - in_sphinx_env=True, - conf={"extensions": ["sphinx_tabs.tabs"]} -) -print(document.pformat()) -````` - -```xml -<document source="notset"> - <container classes="sphinx-tabs"> - <container> - <a classes="item"> - <container> - <paragraph> - Apples - <container classes="ui bottom attached sphinx-tab tab segment sphinx-data-tab-0-0 active"> - <paragraph> - Apples are green, or sometimes red. -``` diff --git a/docs/conf.py b/docs/conf.py index 34f5034c..e2d0c5dc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,15 +4,16 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from datetime import date + from sphinx.application import Sphinx -from sphinx.util.docutils import SphinxDirective from myst_parser import __version__ # -- Project information ----------------------------------------------------- project = "MyST Parser" -copyright = "2020, Executable Book Project" +copyright = f"{date.today().year}, Executable Book Project" author = "Executable Book Project" version = __version__ @@ -29,8 +30,7 @@ "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", - "sphinxcontrib.bibtex", - "sphinx_panels", + "sphinx_design", "sphinxext.rediraffe", "sphinxcontrib.mermaid", "sphinxext.opengraph", @@ -55,11 +55,13 @@ html_favicon = "_static/logo-square.svg" html_title = "" html_theme_options = { + "home_page_in_toc": True, "github_url": "https://github.com/executablebooks/MyST-Parser", "repository_url": "https://github.com/executablebooks/MyST-Parser", - "use_edit_page_button": True, "repository_branch": "master", "path_to_docs": "docs", + "use_repository_button": True, + "use_edit_page_button": True, } # OpenGraph metadata ogp_site_url = "https://myst-parser.readthedocs.io/en/latest" @@ -93,54 +95,25 @@ myst_heading_anchors = 2 myst_footnote_transition = True myst_dmath_double_inline = True -panels_add_bootstrap_css = False -bibtex_bibfiles = ["examples/references.bib"] + rediraffe_redirects = { "using/intro.md": "sphinx/intro.md", + "sphinx/intro.md": "intro.md", "using/use_api.md": "api/index.md", + "api/index.md": "api/reference.rst", "using/syntax.md": "syntax/syntax.md", "using/syntax-optional.md": "syntax/optional.md", "using/reference.md": "syntax/reference.md", + "sphinx/reference.md": "configuration.md", + "sphinx/index.md": "faq/index.md", + "sphinx/use.md": "faq/index.md", + "sphinx/faq.md": "faq/index.md", + "explain/index.md": "develop/background.md", } suppress_warnings = ["myst.strikethrough"] -def run_apidoc(app): - """generate apidoc - - See: https://github.com/rtfd/readthedocs.org/issues/1139 - """ - import os - import shutil - - import sphinx - from sphinx.ext import apidoc - - logger = sphinx.util.logging.getLogger(__name__) - logger.info("running apidoc") - # get correct paths - this_folder = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) - api_folder = os.path.join(this_folder, "_api") - module_path = os.path.normpath(os.path.join(this_folder, "../")) - ignore_paths = ["../setup.py", "../conftest.py", "../tests"] - ignore_paths = [ - os.path.normpath(os.path.join(this_folder, p)) for p in ignore_paths - ] - - if os.path.exists(api_folder): - shutil.rmtree(api_folder) - os.mkdir(api_folder) - - argv = ["-M", "--separate", "-o", api_folder, module_path] + ignore_paths - - apidoc.main(argv) - - # we don't use this - if os.path.exists(os.path.join(api_folder, "modules.rst")): - os.remove(os.path.join(api_folder, "modules.rst")) - - intersphinx_mapping = { "python": ("https://docs.python.org/3.7", None), "sphinx": ("https://www.sphinx-doc.org/en/master", None), @@ -170,6 +143,12 @@ def run_apidoc(app): ("py:class", "docutils.parsers.rst.directives.misc.Include"), ("py:class", "docutils.parsers.rst.Parser"), ("py:class", "docutils.utils.Reporter"), + ("py:class", "nodes.Element"), + ("py:class", "nodes.Node"), + ("py:class", "nodes.system_message"), + ("py:class", "Directive"), + ("py:class", "Include"), + ("py:class", "StringList"), ("py:class", "DocutilsRenderer"), ("py:class", "MockStateMachine"), ] @@ -177,32 +156,13 @@ def run_apidoc(app): def setup(app: Sphinx): """Add functions to the Sphinx setup.""" + from myst_parser._docs import ( + DirectiveDoc, + DocutilsCliHelpDirective, + MystConfigDirective, + ) - class DocutilsCliHelpDirective(SphinxDirective): - """Directive to print the docutils CLI help.""" - - has_content = False - required_arguments = 0 - optional_arguments = 0 - final_argument_whitespace = False - option_spec = {} - - def run(self): - """Run the directive.""" - import io - - from docutils import nodes - from docutils.frontend import OptionParser - - from myst_parser.docutils_ import Parser as DocutilsParser - - stream = io.StringIO() - OptionParser( - components=(DocutilsParser,), - usage="myst-docutils-<writer> [options] [<source> [<destination>]]", - ).print_help(stream) - return [nodes.literal_block("", stream.getvalue())] - - # app.connect("builder-inited", run_apidoc) app.add_css_file("custom.css") + app.add_directive("myst-config", MystConfigDirective) app.add_directive("docutils-cli-help", DocutilsCliHelpDirective) + app.add_directive("doc-directive", DirectiveDoc) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..a87ce0bd --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,106 @@ +(sphinx/config-options)= +# Configuration + +MyST parsing can be configured at both the global and individual document level, +with the most specific configuration taking precedence. + +## Global configuration + +Overriding the default configuration at the global level is achieved by specifying variables in the Sphinx `conf.py` file. +All `myst_parser` global configuration variables are prefixed with `myst_`, e.g. + +```python +myst_enable_extensions = ["deflist"] +``` + +:::{seealso} +Configuration in Docutils, in the [](docutils.md) section. +::: + +```{myst-config} +:sphinx: +:scope: global +``` + +### Extensions + +Configuration specific to syntax extensions: + +```{myst-config} +:sphinx: +:extensions: +:scope: global +``` + +## Local configuration + +```{versionadded} 0.18 +``` + +The following configuration variables are available at the document level. +These can be set in the document [front matter](syntax/frontmatter), under the `myst` key, e.g. + +```yaml +--- +myst: + enable_extensions: ["deflist"] +--- +``` + +```{myst-config} +:sphinx: +:scope: local +``` + +### Extensions + +Configuration specific to syntax extensions: + +```{myst-config} +:sphinx: +:extensions: +:scope: local +``` + +## List of syntax extensions + +Full details in the [](syntax/extensions) section. + +amsmath +: enable direct parsing of [amsmath](https://ctan.org/pkg/amsmath) LaTeX equations + +colon_fence +: Enable code fences using `:::` delimiters, [see here](syntax/colon_fence) for details + +deflist +: Enable definition lists, [see here](syntax/definition-lists) for details + +dollarmath +: Enable parsing of dollar `$` and `$$` encapsulated math + +fieldlist +: Enable field lists, [see here](syntax/fieldlists) for details + +html_admonition +: Convert `<div class="admonition">` elements to sphinx admonition nodes, see the [HTML admonition syntax](syntax/html-admonition) for details + +html_image +: Convert HTML `<img>` elements to sphinx image nodes, [see here](syntax/images) for details + +linkify +: Automatically identify "bare" web URLs and add hyperlinks + +replacements +: Automatically convert some common typographic texts + +smartquotes +: Automatically convert standard quotations to their opening/closing variants + +strikethrough +: Enable strikethrough syntax, [see here](syntax/strikethrough) for details + +substitution +: Substitute keys, [see here](syntax/substitutions) for details + +tasklist +: Add check-boxes to the start of list items, [see here](syntax/tasklists) for details diff --git a/docs/explain/index.md b/docs/develop/background.md similarity index 99% rename from docs/explain/index.md rename to docs/develop/background.md index 1ac48703..4be19e03 100644 --- a/docs/explain/index.md +++ b/docs/develop/background.md @@ -1,4 +1,4 @@ -# Background and explanation +# Background These sections discuss high-level questions about the MyST ecosystem, and explain a few decisions made in the project. diff --git a/docs/develop/index.md b/docs/develop/index.md index 9c7ba25f..f3a3f977 100644 --- a/docs/develop/index.md +++ b/docs/develop/index.md @@ -1,4 +1,4 @@ -# Contribute to MyST +# Contribute This section covers documentation relevant to developing and maintaining the MyST codebase, and some guidelines for how you can contribute. diff --git a/docs/docutils.md b/docs/docutils.md index 0e5c9411..b4d04800 100644 --- a/docs/docutils.md +++ b/docs/docutils.md @@ -1,6 +1,6 @@ (myst-docutils)= -# MyST with Docutils +# Single Page Builds ```{versionadded} 0.16.0 ``` diff --git a/docs/examples/heavy_tails.md b/docs/examples/heavy_tails.md deleted file mode 100644 index 50e2f811..00000000 --- a/docs/examples/heavy_tails.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -orphan: true ---- - -# DUMMY DOCUMENT FOR TESTING :doc: ROLE diff --git a/docs/examples/htop_again.png b/docs/examples/htop_again.png deleted file mode 100644 index 59168b56..00000000 Binary files a/docs/examples/htop_again.png and /dev/null differ diff --git a/docs/examples/index.md b/docs/examples/index.md deleted file mode 100644 index f80832da..00000000 --- a/docs/examples/index.md +++ /dev/null @@ -1,32 +0,0 @@ -# Example pages - -MyST brings all of the features of reStructuredText into markdown. As an example, -The pages below have the same final product, but are written in either reStructuredText -or MyST markdown. You can browse the raw content of each page by clicking the -"download" button at the top of each page, or see the raw content below. - -```{toctree} ---- -caption: Compare rST and MyST versions -maxdepth: 1 ---- -wealth_dynamics_rst.rst -wealth_dynamics_md.md -``` - -## Raw content of each document above - -The following tabs show the raw content of each of the above documents, for quick -comparison. - -````{tabbed} MyST Markdown -Raw source for {doc}`wealth_dynamics_md` -```{literalinclude} wealth_dynamics_md.md -``` -```` - -````{tabbed} reStructuredText -Raw source for {doc}`wealth_dynamics_rst` -```{literalinclude} wealth_dynamics_rst.rst -``` -```` diff --git a/docs/examples/kesten_processes.md b/docs/examples/kesten_processes.md deleted file mode 100644 index 3589f61f..00000000 --- a/docs/examples/kesten_processes.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -orphan: true ---- -# DUMMY DOCUMENT FOR TESTING :doc: ROLE diff --git a/docs/examples/references.bib b/docs/examples/references.bib deleted file mode 100644 index 9cf5b215..00000000 --- a/docs/examples/references.bib +++ /dev/null @@ -1,12 +0,0 @@ -### -Single example from the QuantEcon Bib -### -@article{benhabib2018skewed, - title={Skewed wealth distributions: Theory and empirics}, - author={Benhabib, Jess and Bisin, Alberto}, - journal={Journal of Economic Literature}, - volume={56}, - number={4}, - pages={1261--91}, - year={2018} -} diff --git a/docs/examples/wealth_dynamics_md.md b/docs/examples/wealth_dynamics_md.md deleted file mode 100644 index 3af6672b..00000000 --- a/docs/examples/wealth_dynamics_md.md +++ /dev/null @@ -1,482 +0,0 @@ -# Wealth Distribution Dynamics in MyST - -> {sub-ref}`today` | {sub-ref}`wordcount-minutes` min read - -```{note} -You can {download}`Download the source file for this page <./wealth_dynamics_md.md>` -``` - -```{contents} -:depth: 2 -``` - -In addition to what's in Anaconda, this lecture will need the following -libraries: - -```{code-block} ipython ---- -class: hide-output ---- -!pip install --upgrade quantecon -``` - -## Overview - -This notebook gives an introduction to wealth distribution dynamics, -with a focus on - -- modeling and computing the wealth distribution via simulation, -- measures of inequality such as the Lorenz curve and Gini - coefficient, and -- how inequality is affected by the properties of wage income and - returns on assets. - -The wealth distribution in many countries exhibits a Pareto tail - -- See {doc}`this lecture <heavy_tails>` for a - definition. -- For a review of the empirical evidence, see, for example, - {cite}`md-benhabib2018skewed`. - -### A Note on Assumptions - -The evolution of wealth for any given household depends on their savings -behavior. - -We will use the following imports. - -```{code-block} python -import numpy as np -import matplotlib.pyplot as plt -%matplotlib inline - -import quantecon as qe -from numba import njit, jitclass, float64, prange -``` - -## Lorenz Curves and the Gini Coefficient - -Before we investigate wealth dynamics, we briefly review some measures -of inequality. - -### Lorenz Curves - -One popular graphical measure of inequality is the [Lorenz curve](https://en.wikipedia.org/wiki/Lorenz_curve). - -The package [QuantEcon.py](https://github.com/QuantEcon/QuantEcon.py), -already imported above, contains a function to compute Lorenz curves. - -To illustrate, suppose that - -```{code-block} python -n = 10_000 # size of sample -w = np.exp(np.random.randn(n)) # lognormal draws -``` - -is data representing the wealth of 10,000 households. - -We can compute and plot the Lorenz curve as follows: - -```{code-block} python -f_vals, l_vals = qe.lorenz_curve(w) - -fig, ax = plt.subplots() -ax.plot(f_vals, l_vals, label='Lorenz curve, lognormal sample') -ax.plot(f_vals, f_vals, label='Lorenz curve, equality') -ax.legend() -plt.show() -``` - -This curve can be understood as follows: if point $(x,y)$ lies on the -curve, it means that, collectively, the bottom $(100x)\%$ of the -population holds $(100y)\%$ of the wealth. - -```{code-block} python -a_vals = (1, 2, 5) # Pareto tail index -n = 10_000 # size of each sample -fig, ax = plt.subplots() -for a in a_vals: - u = np.random.uniform(size=n) - y = u**(-1/a) # distributed as Pareto with tail index a - f_vals, l_vals = qe.lorenz_curve(y) - ax.plot(f_vals, l_vals, label=f'$a = {a}$') -ax.plot(f_vals, f_vals, label='equality') -ax.legend() -plt.show() -``` - -You can see that, as the tail parameter of the Pareto distribution -increases, inequality decreases. - -This is to be expected, because a higher tail index implies less weight -in the tail of the Pareto distribution. - -### The Gini Coefficient - -The definition and interpretation of the Gini coefficient can be found -on the corresponding [Wikipedia page](https://en.wikipedia.org/wiki/Gini\_coefficient). - -A value of 0 indicates perfect equality (corresponding the case where -the Lorenz curve matches the 45 degree line) and a value of 1 indicates -complete inequality (all wealth held by the richest household). - -The [QuantEcon.py](https://github.com/QuantEcon/QuantEcon.py) library -contains a function to calculate the Gini coefficient. - -We can test it on the Weibull distribution with parameter $a$, where the -Gini coefficient is known to be - -$$G = 1 - 2^{-1/a}$$ - -Let's see if the Gini coefficient computed from a simulated sample -matches this at each fixed value of $a$. - -```{code-block} python -a_vals = range(1, 20) -ginis = [] -ginis_theoretical = [] -n = 100 - -fig, ax = plt.subplots() -for a in a_vals: - y = np.random.weibull(a, size=n) - ginis.append(qe.gini_coefficient(y)) - ginis_theoretical.append(1 - 2**(-1/a)) -ax.plot(a_vals, ginis, label='estimated gini coefficient') -ax.plot(a_vals, ginis_theoretical, label='theoretical gini coefficient') -ax.legend() -ax.set_xlabel("Weibull parameter $a$") -ax.set_ylabel("Gini coefficient") -plt.show() -``` - -The simulation shows that the fit is good. - -## A Model of Wealth Dynamics - -Having discussed inequality measures, let us now turn to wealth -dynamics. - -The model we will study is - -```{math} ---- -label: md:wealth_dynam_ah ---- -w_{t+1} = (1 + r_{t+1}) s(w_t) + y_{t+1} -``` - -where - -- $w_t$ is wealth at time $t$ for a given household, -- $r_t$ is the rate of return of financial assets, -- $y_t$ is current non-financial (e.g., labor) income and -- $s(w_t)$ is current wealth net of consumption - -Letting $\{z_t\}$ be a correlated state process of the form - -$$z_{t+1} = a z_t + b + \sigma_z \epsilon_{t+1}$$ - -we'll assume that - -$$R_t := 1 + r_t = c_r \exp(z_t) + \exp(\mu_r + \sigma_r \xi_t)$$ - -and - -$$y_t = c_y \exp(z_t) + \exp(\mu_y + \sigma_y \zeta_t)$$ - -Here $\{ (\epsilon_t, \xi_t, \zeta_t) \}$ is IID and standard normal in -$\mathbb R^3$. - -(md:sav_ah)= - -```{math} ---- -label: md:sav_ah ---- -s(w) = s_0 w \cdot \mathbb 1\{w \geq \hat w\} -``` - - -where $s_0$ is a positive constant. - -## Implementation - -Here's some type information to help Numba. - -```{code-block} python -wealth_dynamics_data = [ - ('w_hat', float64), # savings parameter - ('s_0', float64), # savings parameter - ('c_y', float64), # labor income parameter - ('μ_y', float64), # labor income paraemter - ('σ_y', float64), # labor income parameter - ('c_r', float64), # rate of return parameter - ('μ_r', float64), # rate of return parameter - ('σ_r', float64), # rate of return parameter - ('a', float64), # aggregate shock parameter - ('b', float64), # aggregate shock parameter - ('σ_z', float64), # aggregate shock parameter - ('z_mean', float64), # mean of z process - ('z_var', float64), # variance of z process - ('y_mean', float64), # mean of y process - ('R_mean', float64) # mean of R process -] -``` - -Here's a class that stores instance data and implements methods that -update the aggregate state and household wealth. - -```{code-block} python -@jitclass(wealth_dynamics_data) -class WealthDynamics: - - def __init__(self, - w_hat=1.0, - s_0=0.75, - c_y=1.0, - μ_y=1.0, - σ_y=0.2, - c_r=0.05, - μ_r=0.1, - σ_r=0.5, - a=0.5, - b=0.0, - σ_z=0.1): - - self.w_hat, self.s_0 = w_hat, s_0 - self.c_y, self.μ_y, self.σ_y = c_y, μ_y, σ_y - self.c_r, self.μ_r, self.σ_r = c_r, μ_r, σ_r - self.a, self.b, self.σ_z = a, b, σ_z - - # Record stationary moments - self.z_mean = b / (1 - a) - self.z_var = σ_z**2 / (1 - a**2) - exp_z_mean = np.exp(self.z_mean + self.z_var / 2) - self.R_mean = c_r * exp_z_mean + np.exp(μ_r + σ_r**2 / 2) - self.y_mean = c_y * exp_z_mean + np.exp(μ_y + σ_y**2 / 2) - - # Test a stability condition that ensures wealth does not diverge - # to infinity. - α = self.R_mean * self.s_0 - if α >= 1: - raise ValueError("Stability condition failed.") - - def parameters(self): - """ - Collect and return parameters. - """ - parameters = (self.w_hat, self.s_0, - self.c_y, self.μ_y, self.σ_y, - self.c_r, self.μ_r, self.σ_r, - self.a, self.b, self.σ_z) - return parameters - - def update_states(self, w, z): - """ - Update one period, given current wealth w and persistent - state z. - """ - - # Simplify names - params = self.parameters() - w_hat, s_0, c_y, μ_y, σ_y, c_r, μ_r, σ_r, a, b, σ_z = params - zp = a * z + b + σ_z * np.random.randn() - - # Update wealth - y = c_y * np.exp(zp) + np.exp(μ_y + σ_y * np.random.randn()) - wp = y - if w >= w_hat: - R = c_r * np.exp(zp) + np.exp(μ_r + σ_r * np.random.randn()) - wp += R * s_0 * w - return wp, zp -``` - -Here's function to simulate the time series of wealth for in individual -households. - -```{code-block} python -@njit -def wealth_time_series(wdy, w_0, n): - """ - Generate a single time series of length n for wealth given - initial value w_0. - - The initial persistent state z_0 for each household is drawn from - the stationary distribution of the AR(1) process. - - * wdy: an instance of WealthDynamics - * w_0: scalar - * n: int - - - """ - z = wdy.z_mean + np.sqrt(wdy.z_var) * np.random.randn() - w = np.empty(n) - w[0] = w_0 - for t in range(n-1): - w[t+1], z = wdy.update_states(w[t], z) - return w -``` - -Now here's function to simulate a cross section of households forward -in time. - -Note the use of parallelization to speed up computation. - -```{code-block} python -@njit(parallel=True) -def update_cross_section(wdy, w_distribution, shift_length=500): - """ - Shifts a cross-section of household forward in time - - * wdy: an instance of WealthDynamics - * w_distribution: array_like, represents current cross-section - - Takes a current distribution of wealth values as w_distribution - and updates each w_t in w_distribution to w_{t+j}, where - j = shift_length. - - Returns the new distribution. - - """ - new_distribution = np.empty_like(w_distribution) - - # Update each household - for i in prange(len(new_distribution)): - z = wdy.z_mean + np.sqrt(wdy.z_var) * np.random.randn() - w = w_distribution[i] - for t in range(shift_length-1): - w, z = wdy.update_states(w, z) - new_distribution[i] = w - return new_distribution -``` - -Parallelization is very effective in the function above because the time -path of each household can be calculated independently once the path for -the aggregate state is known. - -## Applications - -Let's try simulating the model at different parameter values and -investigate the implications for the wealth distribution. - -### Time Series - -Let's look at the wealth dynamics of an individual household. - -```{code-block} python -wdy = WealthDynamics() - -ts_length = 200 -w = wealth_time_series(wdy, wdy.y_mean, ts_length) - -fig, ax = plt.subplots() -ax.plot(w) -plt.show() -``` - -Notice the large spikes in wealth over time. - -Such spikes are similar to what we observed in time series when -{doc}`we studied Kesten processes<kesten_processes>`. - -### Inequality Measures - -Let's look at how inequality varies with returns on financial assets. - -The next function generates a cross section and then computes the Lorenz -curve and Gini coefficient. - -```{code-block} python -def generate_lorenz_and_gini(wdy, num_households=100_000, T=500): - """ - Generate the Lorenz curve data and gini coefficient corresponding to a - WealthDynamics mode by simulating num_households forward to time T. - """ - ψ_0 = np.ones(num_households) * wdy.y_mean - z_0 = wdy.z_mean - - ψ_star = update_cross_section(wdy, ψ_0, shift_length=T) - return qe.gini_coefficient(ψ_star), qe.lorenz_curve(ψ_star) -``` - -Now we investigate how the Lorenz curves associated with the wealth -distribution change as return to savings varies. - -The code below plots Lorenz curves for three different values of $\mu_r$. - -If you are running this yourself, note that it will take one or two -minutes to execute. - -This is unavoidable because we are executing a CPU intensive task. - -In fact the code, which is JIT compiled and parallelized, runs extremely -fast relative to the number of computations. - -```{code-block} python -fig, ax = plt.subplots() -μ_r_vals = (0.0, 0.025, 0.05) -gini_vals = [] - -for μ_r in μ_r_vals: - wdy = WealthDynamics(μ_r=μ_r) - gv, (f_vals, l_vals) = generate_lorenz_and_gini(wdy) - ax.plot(f_vals, l_vals, label=f'$\psi^*$ at $\mu_r = {μ_r:0.2}$') - gini_vals.append(gv) - -ax.plot(f_vals, f_vals, label='equality') -ax.legend(loc="upper left") -plt.show() -``` - -The Lorenz curve shifts downwards as returns on financial income rise, -indicating a rise in inequality. - -(htop_again)= - -```{image} htop_again.png ---- -scale: 80 ---- -``` - -Now let's check the Gini coefficient. - -```{code-block} python -fig, ax = plt.subplots() -ax.plot(μ_r_vals, gini_vals, label='gini coefficient') -ax.set_xlabel("$\mu_r$") -ax.legend() -plt.show() -``` - -Once again, we see that inequality increases as returns on financial -income rise. - -Let's finish this section by investigating what happens when we change -the volatility term $\sigma_r$ in financial returns. - -```{code-block} python -fig, ax = plt.subplots() -σ_r_vals = (0.35, 0.45, 0.52) -gini_vals = [] - -for σ_r in σ_r_vals: - wdy = WealthDynamics(σ_r=σ_r) - gv, (f_vals, l_vals) = generate_lorenz_and_gini(wdy) - ax.plot(f_vals, l_vals, label=f'$\psi^*$ at $\sigma_r = {σ_r:0.2}$') - gini_vals.append(gv) - -ax.plot(f_vals, f_vals, label='equality') -ax.legend(loc="upper left") -plt.show() -``` - -We see that greater volatility has the effect of increasing inequality -in this model. - -```{bibliography} references.bib -:labelprefix: md -:keyprefix: md- -``` diff --git a/docs/examples/wealth_dynamics_rst.rst b/docs/examples/wealth_dynamics_rst.rst deleted file mode 100644 index f9b64024..00000000 --- a/docs/examples/wealth_dynamics_rst.rst +++ /dev/null @@ -1,493 +0,0 @@ -.. highlight:: python3 - - -*********************************** -Wealth Distribution Dynamics in rST -*********************************** - - -.. note:: - - You can - :download:`Download the source file for this page <./wealth_dynamics_rst.rst>` - -.. contents:: :depth: 2 - -In addition to what's in Anaconda, this lecture will need the following libraries: - -.. code-block:: ipython - :class: hide-output - - !pip install --upgrade quantecon - - -Overview -======== - -This notebook gives an introduction to wealth distribution dynamics, with a -focus on - -* modeling and computing the wealth distribution via simulation, - -* measures of inequality such as the Lorenz curve and Gini coefficient, and - -* how inequality is affected by the properties of wage income and returns on assets. - -The wealth distribution in many countries exhibits a Pareto tail - -* See :doc:`this lecture <heavy_tails>` for a definition. - -* For a review of the empirical evidence, see, for example, :cite:`benhabib2018skewed`. - - - -A Note on Assumptions ---------------------- - -The evolution of wealth for any given household depends on their -savings behavior. - -We will use the following imports. - -.. code:: ipython3 - - import numpy as np - import matplotlib.pyplot as plt - %matplotlib inline - - import quantecon as qe - from numba import njit, jitclass, float64, prange - - -Lorenz Curves and the Gini Coefficient -====================================== - -Before we investigate wealth dynamics, we briefly review some measures of -inequality. - -Lorenz Curves -------------- - -One popular graphical measure of inequality is the `Lorenz curve -<https://en.wikipedia.org/wiki/Lorenz_curve>`__. - -The package `QuantEcon.py <https://github.com/QuantEcon/QuantEcon.py>`__, -already imported above, contains a function to compute Lorenz curves. - -To illustrate, suppose that - -.. code:: ipython3 - - n = 10_000 # size of sample - w = np.exp(np.random.randn(n)) # lognormal draws - -is data representing the wealth of 10,000 households. - -We can compute and plot the Lorenz curve as follows: - -.. code:: ipython3 - - f_vals, l_vals = qe.lorenz_curve(w) - - fig, ax = plt.subplots() - ax.plot(f_vals, l_vals, label='Lorenz curve, lognormal sample') - ax.plot(f_vals, f_vals, label='Lorenz curve, equality') - ax.legend() - plt.show() - -This curve can be understood as follows: if point :math:`(x,y)` lies on the curve, it means that, collectively, the bottom :math:`(100x)\%` of the population holds :math:`(100y)\%` of the wealth. - - -.. code:: ipython3 - - a_vals = (1, 2, 5) # Pareto tail index - n = 10_000 # size of each sample - fig, ax = plt.subplots() - for a in a_vals: - u = np.random.uniform(size=n) - y = u**(-1/a) # distributed as Pareto with tail index a - f_vals, l_vals = qe.lorenz_curve(y) - ax.plot(f_vals, l_vals, label=f'$a = {a}$') - ax.plot(f_vals, f_vals, label='equality') - ax.legend() - plt.show() - -You can see that, as the tail parameter of the Pareto distribution increases, inequality decreases. - -This is to be expected, because a higher tail index implies less weight in the tail of the Pareto distribution. - - - -The Gini Coefficient --------------------- - -The definition and interpretation of the Gini coefficient can be found on the -corresponding `Wikipedia page -<https://en.wikipedia.org/wiki/Gini_coefficient>`__. - -A value of 0 indicates perfect equality (corresponding the case where the -Lorenz curve matches the 45 degree line) and a value of 1 indicates complete -inequality (all wealth held by the richest household). - -The `QuantEcon.py <https://github.com/QuantEcon/QuantEcon.py>`__ library -contains a function to calculate the Gini coefficient. - -We can test it on the Weibull distribution with parameter :math:`a`, where the -Gini coefficient is known to be - -.. math:: G = 1 - 2^{-1/a} - -Let's see if the Gini coefficient computed from a simulated sample matches -this at each fixed value of :math:`a`. - - - -.. code:: ipython3 - - a_vals = range(1, 20) - ginis = [] - ginis_theoretical = [] - n = 100 - - fig, ax = plt.subplots() - for a in a_vals: - y = np.random.weibull(a, size=n) - ginis.append(qe.gini_coefficient(y)) - ginis_theoretical.append(1 - 2**(-1/a)) - ax.plot(a_vals, ginis, label='estimated gini coefficient') - ax.plot(a_vals, ginis_theoretical, label='theoretical gini coefficient') - ax.legend() - ax.set_xlabel("Weibull parameter $a$") - ax.set_ylabel("Gini coefficient") - plt.show() - -The simulation shows that the fit is good. - - - -A Model of Wealth Dynamics -========================== - -Having discussed inequality measures, let us now turn to wealth dynamics. - -The model we will study is - -.. math:: - :label: wealth_dynam_ah - - w_{t+1} = (1 + r_{t+1}) s(w_t) + y_{t+1} - -where - -- :math:`w_t` is wealth at time :math:`t` for a given household, -- :math:`r_t` is the rate of return of financial assets, -- :math:`y_t` is current non-financial (e.g., labor) income and -- :math:`s(w_t)` is current wealth net of consumption - -Letting :math:`\{z_t\}` be a correlated state process of the form - -.. math:: z_{t+1} = a z_t + b + \sigma_z \epsilon_{t+1} - -we’ll assume that - -.. math:: R_t := 1 + r_t = c_r \exp(z_t) + \exp(\mu_r + \sigma_r \xi_t) - -and - -.. math:: y_t = c_y \exp(z_t) + \exp(\mu_y + \sigma_y \zeta_t) - -Here :math:`\{ (\epsilon_t, \xi_t, \zeta_t) \}` is IID and standard -normal in :math:`\mathbb R^3`. - - -.. math:: - :label: sav_ah - - s(w) = s_0 w \cdot \mathbb 1\{w \geq \hat w\} - -where :math:`s_0` is a positive constant. - - -Implementation -============== - -Here's some type information to help Numba. - -.. code:: ipython3 - - wealth_dynamics_data = [ - ('w_hat', float64), # savings parameter - ('s_0', float64), # savings parameter - ('c_y', float64), # labor income parameter - ('μ_y', float64), # labor income paraemter - ('σ_y', float64), # labor income parameter - ('c_r', float64), # rate of return parameter - ('μ_r', float64), # rate of return parameter - ('σ_r', float64), # rate of return parameter - ('a', float64), # aggregate shock parameter - ('b', float64), # aggregate shock parameter - ('σ_z', float64), # aggregate shock parameter - ('z_mean', float64), # mean of z process - ('z_var', float64), # variance of z process - ('y_mean', float64), # mean of y process - ('R_mean', float64) # mean of R process - ] - -Here's a class that stores instance data and implements methods that update -the aggregate state and household wealth. - -.. code:: ipython3 - - @jitclass(wealth_dynamics_data) - class WealthDynamics: - - def __init__(self, - w_hat=1.0, - s_0=0.75, - c_y=1.0, - μ_y=1.0, - σ_y=0.2, - c_r=0.05, - μ_r=0.1, - σ_r=0.5, - a=0.5, - b=0.0, - σ_z=0.1): - - self.w_hat, self.s_0 = w_hat, s_0 - self.c_y, self.μ_y, self.σ_y = c_y, μ_y, σ_y - self.c_r, self.μ_r, self.σ_r = c_r, μ_r, σ_r - self.a, self.b, self.σ_z = a, b, σ_z - - # Record stationary moments - self.z_mean = b / (1 - a) - self.z_var = σ_z**2 / (1 - a**2) - exp_z_mean = np.exp(self.z_mean + self.z_var / 2) - self.R_mean = c_r * exp_z_mean + np.exp(μ_r + σ_r**2 / 2) - self.y_mean = c_y * exp_z_mean + np.exp(μ_y + σ_y**2 / 2) - - # Test a stability condition that ensures wealth does not diverge - # to infinity. - α = self.R_mean * self.s_0 - if α >= 1: - raise ValueError("Stability condition failed.") - - def parameters(self): - """ - Collect and return parameters. - """ - parameters = (self.w_hat, self.s_0, - self.c_y, self.μ_y, self.σ_y, - self.c_r, self.μ_r, self.σ_r, - self.a, self.b, self.σ_z) - return parameters - - def update_states(self, w, z): - """ - Update one period, given current wealth w and persistent - state z. - """ - - # Simplify names - params = self.parameters() - w_hat, s_0, c_y, μ_y, σ_y, c_r, μ_r, σ_r, a, b, σ_z = params - zp = a * z + b + σ_z * np.random.randn() - - # Update wealth - y = c_y * np.exp(zp) + np.exp(μ_y + σ_y * np.random.randn()) - wp = y - if w >= w_hat: - R = c_r * np.exp(zp) + np.exp(μ_r + σ_r * np.random.randn()) - wp += R * s_0 * w - return wp, zp - - -Here's function to simulate the time series of wealth for in individual households. - -.. code:: ipython3 - - @njit - def wealth_time_series(wdy, w_0, n): - """ - Generate a single time series of length n for wealth given - initial value w_0. - - The initial persistent state z_0 for each household is drawn from - the stationary distribution of the AR(1) process. - - * wdy: an instance of WealthDynamics - * w_0: scalar - * n: int - - - """ - z = wdy.z_mean + np.sqrt(wdy.z_var) * np.random.randn() - w = np.empty(n) - w[0] = w_0 - for t in range(n-1): - w[t+1], z = wdy.update_states(w[t], z) - return w - - -Now here's function to simulate a cross section of households forward in time. - -Note the use of parallelization to speed up computation. - -.. code:: ipython3 - - @njit(parallel=True) - def update_cross_section(wdy, w_distribution, shift_length=500): - """ - Shifts a cross-section of household forward in time - - * wdy: an instance of WealthDynamics - * w_distribution: array_like, represents current cross-section - - Takes a current distribution of wealth values as w_distribution - and updates each w_t in w_distribution to w_{t+j}, where - j = shift_length. - - Returns the new distribution. - - """ - new_distribution = np.empty_like(w_distribution) - - # Update each household - for i in prange(len(new_distribution)): - z = wdy.z_mean + np.sqrt(wdy.z_var) * np.random.randn() - w = w_distribution[i] - for t in range(shift_length-1): - w, z = wdy.update_states(w, z) - new_distribution[i] = w - return new_distribution - -Parallelization is very effective in the function above because the time path -of each household can be calculated independently once the path for the -aggregate state is known. - - - - -Applications -============ - -Let's try simulating the model at different parameter values and investigate -the implications for the wealth distribution. - - -Time Series ------------ - -Let's look at the wealth dynamics of an individual household. - -.. code:: ipython3 - - wdy = WealthDynamics() - - ts_length = 200 - w = wealth_time_series(wdy, wdy.y_mean, ts_length) - - fig, ax = plt.subplots() - ax.plot(w) - plt.show() - -Notice the large spikes in wealth over time. - -Such spikes are similar to what we observed in time series when :doc:`we studied Kesten processes <kesten_processes>`. - - - -Inequality Measures -------------------- - - -Let's look at how inequality varies with returns on financial assets. - -The next function generates a cross section and then computes the Lorenz -curve and Gini coefficient. - -.. code:: ipython3 - - def generate_lorenz_and_gini(wdy, num_households=100_000, T=500): - """ - Generate the Lorenz curve data and gini coefficient corresponding to a - WealthDynamics mode by simulating num_households forward to time T. - """ - ψ_0 = np.ones(num_households) * wdy.y_mean - z_0 = wdy.z_mean - - ψ_star = update_cross_section(wdy, ψ_0, shift_length=T) - return qe.gini_coefficient(ψ_star), qe.lorenz_curve(ψ_star) - -Now we investigate how the Lorenz curves associated with the wealth distribution change as return to savings varies. - -The code below plots Lorenz curves for three different values of :math:`\mu_r`. - -If you are running this yourself, note that it will take one or two minutes to execute. - -This is unavoidable because we are executing a CPU intensive task. - -In fact the code, which is JIT compiled and parallelized, runs extremely fast relative to the number of computations. - -.. code:: ipython3 - - fig, ax = plt.subplots() - μ_r_vals = (0.0, 0.025, 0.05) - gini_vals = [] - - for μ_r in μ_r_vals: - wdy = WealthDynamics(μ_r=μ_r) - gv, (f_vals, l_vals) = generate_lorenz_and_gini(wdy) - ax.plot(f_vals, l_vals, label=f'$\psi^*$ at $\mu_r = {μ_r:0.2}$') - gini_vals.append(gv) - - ax.plot(f_vals, f_vals, label='equality') - ax.legend(loc="upper left") - plt.show() - -The Lorenz curve shifts downwards as returns on financial income rise, indicating a rise in inequality. - - -.. _htop_again: - -.. figure:: htop_again.png - :scale: 80 - - -Now let's check the Gini coefficient. - -.. code:: ipython3 - - fig, ax = plt.subplots() - ax.plot(μ_r_vals, gini_vals, label='gini coefficient') - ax.set_xlabel("$\mu_r$") - ax.legend() - plt.show() - -Once again, we see that inequality increases as returns on financial income -rise. - -Let's finish this section by investigating what happens when we change the -volatility term :math:`\sigma_r` in financial returns. - - -.. code:: ipython3 - - fig, ax = plt.subplots() - σ_r_vals = (0.35, 0.45, 0.52) - gini_vals = [] - - for σ_r in σ_r_vals: - wdy = WealthDynamics(σ_r=σ_r) - gv, (f_vals, l_vals) = generate_lorenz_and_gini(wdy) - ax.plot(f_vals, l_vals, label=f'$\psi^*$ at $\sigma_r = {σ_r:0.2}$') - gini_vals.append(gv) - - ax.plot(f_vals, f_vals, label='equality') - ax.legend(loc="upper left") - plt.show() - - -We see that greater volatility has the effect of increasing inequality in this model. - -.. bibliography:: references.bib diff --git a/docs/sphinx/use.md b/docs/faq/index.md similarity index 78% rename from docs/sphinx/use.md rename to docs/faq/index.md index 1678d522..e4d45815 100644 --- a/docs/sphinx/use.md +++ b/docs/faq/index.md @@ -1,13 +1,13 @@ -# Sphinx extension usage guide +(myst-sphinx)= -These sections describe some common scenarios and use-cases for writing MyST with Sphinx. +# FAQ -:::{seealso} -For an introduction to using MyST with Sphinx, see [](intro.md). -::: +## How-tos + +These sections describe some common scenarios and use-cases for writing MyST with Sphinx. (howto/include-rst)= -## Include rST files into a Markdown file +### Include rST files into a Markdown file As explained in [this section](syntax/directives/parsing), all MyST directives will parse their content as Markdown. Therefore, using the conventional `include` directive, will parse the file contents as Markdown: @@ -33,7 +33,7 @@ To include rST, we must first "wrap" the directive in the [eval-rst directive](s ``` (howto/include-md)= -## Include Markdown files into an rST file +### Include Markdown files into an rST file To include a MyST file within a ReStructuredText file, we can use the `parser` option of the `include` directive: @@ -46,12 +46,12 @@ To include a MyST file within a ReStructuredText file, we can use the `parser` o The `parser` option requires `docutils>=0.17` ``` -## Use MyST in Jupyter Notebooks +### Use MyST in Jupyter Notebooks The [MyST-NB](https://myst-nb.readthedocs.io) tool provides a Sphinx extension for parsing **Jupyter Notebooks written with MyST Markdown**. It includes features like automatically executing notebooks during documentation builds, storing notebook cell outputs in order to insert them elsewhere in your documentation, and more. See the [MyST-NB documentation](https://myst-nb.readthedocs.io) for more information. (howto/include-readme)= -## Include a file from outside the docs folder (like README.md) +### Include a file from outside the docs folder (like README.md) You can include a file, including one from outside the project using e.g.: @@ -100,7 +100,7 @@ If you encounter any issues with this feature, please don't hesitate to report i ::: (howto/autodoc)= -## Use `sphinx.ext.autodoc` in Markdown files +### Use `sphinx.ext.autodoc` in Markdown files The [Sphinx extension `autodoc`](sphinx:sphinx.ext.autodoc), which pulls in code documentation from docstrings, is currently hard-coded to parse reStructuredText. It is therefore incompatible with MyST's Markdown parser. @@ -131,7 +131,7 @@ We hope to support Markdown in the future, see [GitHub issue #228](https://githu ``` (howto/autosectionlabel)= -## Automatically create targets for section headers +### Automatically create targets for section headers :::{important} @@ -173,7 +173,7 @@ like so: ``` (howto/warnings)= -## Suppress warnings +### Suppress warnings In general, if your build logs any warnings, you should either fix them or [raise an Issue](https://github.com/executablebooks/MyST-Parser/issues/new/choose) if you think the warning is erroneous. However, in some circumstances if you wish to suppress the warning you can use the [`suppress_warnings`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-suppress_warnings) configuration option. @@ -195,7 +195,7 @@ suppress_warnings = ["myst.header"] ``` -## Sphinx-specific page front matter +### Sphinx-specific page front matter Sphinx intercepts front matter and stores them within the global environment (as discussed [in the deflists documentation](https://www.sphinx-doc.org/en/master/usage/restructuredtext/field-lists.html)). @@ -225,6 +225,43 @@ orphan: true This is an orphan document, not specified in any toctrees. ``` -## Migrate pre-existing rST into MyST +### Migrate pre-existing rST into MyST If you've already got some reStructuredText files that you'd like to convert into MyST Markdown, try the [`rst-to-myst`](https://github.com/executablebooks/rst-to-myst) tool, which allows you to convert single rST files to MyST markdown documents. + +## Disable Markdown syntax for the parser + +If you'd like to either enable or disable custom markdown syntax, use `myst_disable_syntax`. +Anything in this list will no longer be parsed by the MyST parser. + +For example, to disable the `emphasis` in-line syntax, use this configuration: + +```python +myst_disable_syntax = ["emphasis"] +``` + +emphasis syntax will now be disabled. For example, the following will be rendered +*without* any italics: + +```md +*emphasis is now disabled* +``` + +For a list of all the syntax elements you can disable, see the [markdown-it parser guide](markdown_it:using). + +## Common errors and questions + +These are common issues and gotchas that people may experience when using the MyST Sphinx extension. + +### What markup language should I use inside directives? + +If you need to parse content *inside* of another block of content (for example, the +content inside a **note directive**), note that the MyST parser will be used for this +nested parsing as well. + +### Why doesn't my role/directive recognize markdown link syntax? + +There are some roles/directives that _hard-code_ syntax into +their behavior. For example, many roles allow you to supply titles for links like so: +`` {role}`My title <myref>` ``. While this looks like reStructuredText, the role may +be explicitly expecting the `My title <myref>` structure, and so MyST will behave the same way. diff --git a/docs/sphinx/snippets/include-md.md b/docs/faq/snippets/include-md.md similarity index 100% rename from docs/sphinx/snippets/include-md.md rename to docs/faq/snippets/include-md.md diff --git a/docs/sphinx/snippets/include-rst.rst b/docs/faq/snippets/include-rst.rst similarity index 100% rename from docs/sphinx/snippets/include-rst.rst rename to docs/faq/snippets/include-rst.rst diff --git a/docs/index.md b/docs/index.md index 93d0fa67..a87e43f7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,123 +1,145 @@ -# MyST - Markedly Structured Text <img src="_static/logo-square.svg" width=40 /> +--- +sd_hide_title: true +--- -[![PyPI][pypi-badge]][pypi-link] -[![Conda][conda-badge]][conda-link] +# Overview -**MyST is a rich and extensible flavor of Markdown meant for technical documentation and publishing**. +::::{grid} +:reverse: +:gutter: 3 4 4 4 +:margin: 1 2 1 2 -MyST is a flavor of markdown that is designed for simplicity, flexibility, and extensibility. Here are a few major features +:::{grid-item} +:columns: 12 4 4 4 -:::{panels} -:container: +full-width text-center -:column: col-lg-4 px-2 py-2 -:card: +```{image} _static/logo-square.svg +:width: 200px +:class: sd-m-auto +``` -**[CommonMark compliant](commonmark-block-tokens)** ✔ -^^^ -MyST is a superset of [CommonMark Markdown][commonmark]. Any CommonMark document is also MyST-compliant. ---- +::: -**[Extra syntax for authoring](extended-block-tokens)** ✍ -^^^ -MyST extends CommonMark with [syntax meant for scholarly writing and technical documentation](extended-block-tokens). +:::{grid-item} +:columns: 12 8 8 8 +:child-align: justify +:class: sd-fs-5 ---- -**[Extendable syntax](syntax/directives)** 🚀 -^^^ -MyST provides [roles](syntax/roles) and [directives](syntax/directives), allowing you to extend MyST's functionality. +```{rubric} MyST - Markedly Structured Text - Parser +``` ---- -**[Compatible with Sphinx](sphinx/index.md)** 📄 -^^^ -MyST is inspired by Sphinx, and comes with [its own Sphinx parser](sphinx/index.md). -[Write your Sphinx docs in Markdown](sphinx:usage/quickstart), or convert existing [RST to Markdown][rst-to-myst] -from the CLI or [using an interactive web interface][mystyc]! +A Sphinx and Docutils extension to parse MyST, +a rich and extensible flavour of Markdown for authoring technical and scientific documentation. ---- -**[Hackable with Python](api/index.md)** 🐍 -^^^ -This MyST parser is built on top of the [`markdown-it-py` package][markdown-it-py], an pluggable Python parser for Markdown. +```{button-ref} intro +:ref-type: doc +:color: primary +:class: sd-rounded-pill + +Get Started +``` + +::: + +:::: --- -**[Hackable with Javascript][markdown-it-myst]** 🌍 -^^^ -The [Javascript parser][markdown-it-myst] builds on [markdown-it][markdown-it], and allows you to parse MyST in websites. + +::::{grid} 1 2 2 3 +:gutter: 1 1 1 2 + +:::{grid-item-card} {octicon}`markdown;1.5em;sd-mr-1` CommonMark-plus +:link: syntax/core +:link-type: ref + +MyST extends the CommonMark syntax specification, to support technical authoring features such as tables and footnotes. + ++++ +[Learn more »](syntax/core) ::: -## Find the right documentation resources +:::{grid-item-card} {octicon}`plug;1.5em;sd-mr-1` Sphinx compatible +:link: roles-directives +:link-type: ref -This documentation is organized into a few major sections. **Tutorials** are step-by-step introductory guides to MyST Markdown. **Topic Guides** cover specific areas in more depth, and are organized as discrete "how-to" sections. **Reference** sections describe the API/syntax/etc of the MyST Parser in detail. +Use the MyST role and directive syntax to harness the full capability of Sphinx, such as admonitions and figures, and all existing Sphinx extensions. -In addition, here are a few pointers to help you get started. ++++ +[Learn more »](roles-directives) +::: -:::{panels} -:container: full-width -:column: col-lg-4 p-2 ---- -:header: bg-myst-one -**Get started with MyST** -^^^ -**[](sphinx/intro.md)**: a step-by-step tutorial. +:::{grid-item-card} {octicon}`tools;1.5em;sd-mr-1` Highly configurable +:link: configuration +:link-type: doc + +MyST-parser can be configured at both the global and individual document level, +to modify parsing behaviour and access extended syntax features. + ++++ +[Learn more »](configuration) +::: -**[](syntax/syntax.md)**: discusses major MyST syntax components. +:::: -**[The Sphinx guide](sphinx/index.md)**: how to use MyST with your Sphinx documentation. --- -:header: bg-myst-two -**Learn more about MyST** -^^^ -**[](syntax/optional.md)**: additional syntax you can enable for extra features. +```{rubric} Additional resources +``` -**[The Python API guide](api/index.md)**: parsing and rendering MyST with Python. +[MyST-Markdown VS Code extension](https://marketplace.visualstudio.com/items?itemName=ExecutableBookProject.myst-highlight) +: For MyST extended syntax highlighting and authoring tools. -**[](explain/index.md)**: background understanding and discussions of MyST markdown. ---- -:header: bg-myst-three +[Convert existing ReStructuredText files to Markdown][rst-to-myst] +: Use the [rst-to-myst] CLI or [the MySTyc interactive web interface](https://mystyc.herokuapp.com) -**Get inspired** -^^^ -**[Jupyter Book](https://jupyterbook.org)**: An open source project for building beautiful, publication-quality books and documents from computational material, built on top of the MyST Parser. +[MyST-NB](https://myst-nb.readthedocs.io) +: A Sphinx and Docutils extension for compiling Jupyter Notebooks into high quality documentation formats, built on top of the MyST-Parser. -**[The Jupyter Book gallery](https://gallery.jupyterbook.org)**: examples of documents built with MyST. -::: +[Jupyter Book](https://jupyterbook.org) +: An open source project for building beautiful, publication-quality books and documents from computational material, built on top of the MyST-Parser and MyST-NB. -```{toctree} -:hidden: -sphinx/intro.md +[The Jupyter Book gallery](https://gallery.jupyterbook.org) +: Examples of documents built with MyST. + +[Javascript MyST parser][mystjs] +: The [mystjs] Javascript parser, allows you to parse MyST in websites. + +[markdown-it-py] +: A CommonMark-compliant and extensible Markdown parser, used by MyST-Parser to parse source text to tokens. + +```{rubric} Acknowledgements ``` +The MyST markdown language and MyST parser are both supported by the open community, +[The Executable Book Project](https://executablebooks.org). + ```{toctree} -:caption: MyST Syntax :hidden: -syntax/syntax -syntax/optional -syntax/reference +intro.md ``` ```{toctree} :hidden: -:caption: Topic Guides -explain/index.md -sphinx/index.md +:caption: Guides + +syntax/syntax +syntax/optional +syntax/roles-and-directives.md +configuration.md docutils.md -api/index.md +faq/index.md develop/index.md ``` ```{toctree} :hidden: -:caption: About the project -examples/index.md +:caption: Reference + develop/_changelog.md -GitHub repo <https://github.com/executablebooks/myst-parser> +syntax/reference +develop/background.md +api/reference.rst ``` -## Acknowledgements - -The MyST markdown language and MyST parser are both supported by the open community, -[The Executable Book Project](https://executablebooks.org). - [commonmark]: https://commonmark.org/ [github-ci]: https://github.com/executablebooks/MyST-Parser/workflows/continuous-integration/badge.svg?branch=master [github-link]: https://github.com/executablebooks/MyST-Parser @@ -133,7 +155,6 @@ The MyST markdown language and MyST parser are both supported by the open commun [black-link]: https://github.com/ambv/black [github-badge]: https://img.shields.io/github/stars/executablebooks/myst-parser?label=github [markdown-it-py]: https://markdown-it-py.readthedocs.io/ -[markdown-it-myst]: https://github.com/executablebooks/markdown-it-myst [markdown-it]: https://markdown-it.github.io/ [rst-to-myst]: https://rst-to-myst.readthedocs.io -[mystyc]: https://mystyc.herokuapp.com +[mystjs]: https://github.com/executablebooks/mystjs diff --git a/docs/intro.md b/docs/intro.md new file mode 100644 index 00000000..b0b8d429 --- /dev/null +++ b/docs/intro.md @@ -0,0 +1,250 @@ +(intro/get-started)= +# Get Started + +This page describes how to get started with the MyST parser, with a focus on enabling it in the Sphinx documentation engine. + +## Installation + +[![PyPI][pypi-badge]][pypi-link] +[![Conda][conda-badge]][conda-link] + +To install use [pip](https://pip.pypa.io): + +```bash +pip install myst-parser +``` + +or [Conda](https://docs.conda.io): + +```bash +conda install -c conda-forge myst-parser +``` + +[pypi-badge]: https://img.shields.io/pypi/v/myst-parser.svg +[pypi-link]: https://pypi.org/project/myst-parser +[conda-badge]: https://anaconda.org/conda-forge/myst-parser/badges/version.svg +[conda-link]: https://anaconda.org/conda-forge/myst-parser + +(intro/sphinx)= +## Enable MyST in Sphinx + +To get started with Sphinx, see their [Quickstart Guide](https://www.sphinx-doc.org/en/master/usage/quickstart.html). + +To use the MyST parser in Sphinx, simply add the following to your `conf.py` file: + +```python +extensions = ["myst_parser"] +``` + +This will activate the MyST Parser extension, causing all documents with the `.md` extension to be parsed as MyST. + +:::{tip} +To parse single documents, see the [](docutils.md) section +::: + +(intro/writing)= +## Write a CommonMark document + +MyST is an extension of [CommonMark Markdown](https://commonmark.org/), +that includes [additional syntax](../syntax/syntax.md) for technical authoring, +which integrates with Docutils and Sphinx. + +To start off, create an empty file called `myfile.md` and give it a markdown title and text. + +```md +# My nifty title + +Some **text**! +``` + +To parse to HTML, try the CLI: + +```html +$ myst-docutils-html5 --stylesheet= myfile.md +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +<meta charset="utf-8"/> +<meta name="viewport" content="width=device-width, initial-scale=1" /> +<meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" /> +<title>My nifty title + + + +
+

My nifty title

+ +

Some text!

+
+ + +``` + +To include this document within a Sphinx project, +include `myfile.md` in a [`toctree` directive](sphinx:toctree-directive) on an index page. + +## Extend CommonMark with roles and directives + +MyST allows any Sphinx role or directive to be used in a document. +These are extensions points allowing for richer features, such as admonitions and figures. + +For example, add an `admonition` directive and `sup` role to your Markdown page, like so: + +````md +# My nifty title + +Some **text**! + +```{admonition} Here's my title +:class: tip + +Here's my admonition content.{sup}`1` +``` +```` + +Then convert to HTML: + +```html +$ myst-docutils-html5 --stylesheet= myfile.md +... +
+

Here's my title

+

Here's my admonition content.1

+
+... +``` + +:::{seealso} +The full [](syntax/roles-and-directives.md) section +::: + +(intro/reference)= +## Cross-referencing + +MyST-Parser offers powerful cross-referencing features, to link to documents, headers, figures and more. + +For example, to add a section *reference target*, and reference it: + +```md +(header-label)= +# A header + +[My reference](header-label) +``` + +```html +$ myst-docutils-html5 --stylesheet= myfile.md +... + +

A header

+ +

My reference

+... +``` + +:::{seealso} +The [](syntax/referencing) section,\ +and the [ReadTheDocs cross-referencing](https://docs.readthedocs.io/en/stable/guides/cross-referencing-with-sphinx.html) documentation +::: + +## Configuring MyST-Parser + +The [](configuration.md) section contains a complete list of configuration options for the MyST-Parser. + +These can be applied globally, e.g. in the sphinx `conf.py`: + +```python +myst_enable_extensions = [ + "colon_fence", +] +``` + +Or they can be applied to specific documents, at the top of the document: + +```yaml +--- +myst: + enable_extensions: ["colon_fence"] +--- +``` + +## Extending Sphinx + +The other way to extend MyST in Sphinx is to install Sphinx extensions that define new roles, directives, etc. + +For example, let's install the `sphinxcontib.mermaid` extension, +which will allow us to generate [Mermaid diagrams](https://mermaid-js.github.io/mermaid/#/) with MyST. + +First, install `sphinxcontrib.mermaid`: + +```shell +pip install sphinxcontrib-mermaid +``` + +Next, add it to your list of extensions in `conf.py`: + +```python +extensions = [ + "myst_parser", + "sphinxcontrib.mermaid", +] +``` + +Now, add a **mermaid directive** to your markdown file. +For example: + +````md +# My nifty title + +Some **text**! + +```{admonition} Here's my title +:class: warning + +Here's my admonition content +``` + +(section-two)= +## Here's another section + +And some more content. + +% This comment won't make it into the outputs! +And here's {ref}`a reference to this section `. +I can also reference the section {ref}`section-two` without specifying my title. + +:::{note} +And here's a note with a colon fence! +::: + +And finally, here's a cool mermaid diagram! + +```{mermaid} +sequenceDiagram + participant Alice + participant Bob + Alice->John: Hello John, how are you? + loop Healthcheck + John->John: Fight against hypochondria + end + Note right of John: Rational thoughts
prevail... + John-->Alice: Great! + John->Bob: How about you? + Bob-->John: Jolly good! +``` +```` + +When you build your documentation, you should see something like this: + +```{mermaid} +sequenceDiagram + participant Alice + participant Bob + Alice->John: Hello John, how are you? + loop Healthcheck + John->John: Fight against hypochondria + end + Note right of John: Rational thoughts
prevail... + John-->Alice: Great! + John->Bob: How about you? + Bob-->John: Jolly good! +``` diff --git a/docs/sphinx/faq.md b/docs/sphinx/faq.md deleted file mode 100644 index a799b168..00000000 --- a/docs/sphinx/faq.md +++ /dev/null @@ -1,16 +0,0 @@ -# Common errors and questions - -These are common issues and gotchas that people may experience when using the MyST Sphinx extension. - -## What markup language should I use inside directives? - -If you need to parse content *inside* of another block of content (for example, the -content inside a **note directive**), note that the MyST parser will be used for this -nested parsing as well. - -## Why doesn't my role/directive recognize markdown link syntax? - -There are some roles/directives that _hard-code_ syntax into -their behavior. For example, many roles allow you to supply titles for links like so: -`` {role}`My title ` ``. While this looks like reStructuredText, the role may -be explicitly expecting the `My title ` structure, and so MyST will behave the same way. diff --git a/docs/sphinx/index.md b/docs/sphinx/index.md deleted file mode 100644 index afe7694c..00000000 --- a/docs/sphinx/index.md +++ /dev/null @@ -1,17 +0,0 @@ -(myst-sphinx)= - -# MyST with Sphinx - -The MyST Parser comes bundled with a Sphinx extension that allows you to write Sphinx documentation entirely in MyST (or, in a combination of rST and MyST). -The following sections cover some major functionality of the Sphinx extension. - -:::{seealso} -For an introduction to MyST with Sphinx, see [](intro.md). -::: - -```{toctree} -use.md -roles-and-directives.md -reference.md -faq.md -``` diff --git a/docs/sphinx/intro.md b/docs/sphinx/intro.md deleted file mode 100644 index 7c7f46b3..00000000 --- a/docs/sphinx/intro.md +++ /dev/null @@ -1,423 +0,0 @@ -(intro/get-started)= -# Get started with MyST in Sphinx - -This page describes how to get started with the MyST parser, with a focus on enabling it in the Sphinx documentation engine. - -## Install the MyST Parser - -[![PyPI][pypi-badge]][pypi-link] -[![Conda][conda-badge]][conda-link] - -Installing the MyST parser provides access to two tools: - -* A Python library that can parse MyST markdown, and render it to a number of output formats (in particular, `docutils` format for use with Sphinx). -* A Sphinx extension that that utilizes the above tool in order to parse MyST Markdown in your documentation. - -To install the MyST parser, run the following in a -[Conda environment](https://docs.conda.io) (recommended): - -```bash -conda install -c conda-forge myst-parser -``` - -or - -```bash -pip install myst-parser -``` - -[pypi-badge]: https://img.shields.io/pypi/v/myst-parser.svg -[pypi-link]: https://pypi.org/project/myst-parser -[conda-badge]: https://anaconda.org/conda-forge/myst-parser/badges/version.svg -[conda-link]: https://anaconda.org/conda-forge/myst-parser - -(parse-with-sphinx)= -## Enable MyST in Sphinx - -Sphinx is a documentation generator for building a website or book from multiple source documents and assets. To get started with Sphinx, see their [Quickstart Guide](https://www.sphinx-doc.org/en/master/usage/quickstart.html). This guide assumes that you've already got a pre-existing Sphinx site that builds properly. - -To use the MyST parser in Sphinx, simply add the following to your `conf.py` file: - -```python -extensions = ["myst_parser"] -``` - -This will activate the MyST Parser extension, causing all documents with the `.md` extension to be parsed as MyST. - -:::{admonition} You can use both MyST and reStructuredText -:class: tip - -Activating the MyST parser will simply *enable* parsing markdown files with MyST, and the rST parser that ships with Sphinx will still work the same way. -Files ending with `.md` will be parsed as MyST, and files ending in `.rst` will be parsed as reStructuredText. -::: - -(intro/writing)= -## Write your first markdown document - -Now that you've enabled the `myst-parser` in Sphinx, you can write MyST markdown in a file that ends with `.md` extension for your pages. - -:::{note} -MyST markdown is a mixture of two flavors of markdown: - -It supports all the syntax of **[CommonMark Markdown](https://commonmark.org/)** at its -base. This is a community standard flavor of markdown used across many projects. - -In addition, it includes **[several extensions](../syntax/syntax.md) to CommonMark**. -These add extra syntax features for technical writing, such as the roles and directives used by Sphinx. -::: - -To start off, create an empty file called `myfile.md` and give it a markdown title and text. - -```md -# My nifty title - -Some **text**! -``` - -In the "main document" of your Sphinx project (the landing page of your Sphinx documentation), include `myfile.md` in a `toctree` directive so that it is included in your documentation: - -```rst -.. toctree:: - - myfile.md -``` - -Now build your site: - -```bash -make html -``` - -and navigate to your landing page. -You should see a link to the page generated from `myfile.md`. -Clicking that link should take you to your rendered Markdown! - -## Extend markdown with a directive - -The most important functionality available with MyST markdown is writing **directives**. -Directives are kind-of like functions that are designed for writing content. -Sphinx and reStructuredText use directives extensively. -Here's how a directive looks in MyST markdown: - -````{margin} Alternative options syntax -If you've got a lot of options for your directive, or have a value that is really -long (e.g., that spans multiple lines), then you can also wrap your options in -`---` lines and write them as YAML. For example: - -```yaml ---- -key1: val1 -key2: | - val line 1 - val line 2 ---- -``` -```` - -```` -```{directivename} -:optionname: - - -``` -```` - -For those who are familiar with reStructuredText, you can find [a mapping from MyST directive syntax to rST syntax here](syntax/directives). - - -As seen above, there are four main parts to consider when writing directives. - -* **the directive name** is kind of like the function name. Different names trigger - different functionality. They are wrapped in `{}` brackets. -* **directive arguments** come just after the directive name. They can be used - to trigger behavior in the directive. -* **directive options** come just after the first line of the directive. They also - control behavior of the directive. -* **directive content** is markdown that you put inside the directive. The directive - often displays the content in a special way. - -For example, add an **`admonition`** directive to your markdown page, like so: - - -````md -# My nifty title - -Some **text**! - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` -```` - -Re-build your Sphinx site and you should see the new admonition box show up. - -As you can see, we've used each of the four pieces described above to configure this -directive. Here's how the directive looks when rendered: - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` - -:::{seealso} -For more information about using directives with MyST, see {ref}`syntax/directives`. -::: - -(sphinx/intro:reference)= -## Reference a section label with a role - -Roles are another core Sphinx tool. They behave similarly to directives, but are given -in-line with text instead of in a separate block. They have the following form: - -```md -{rolename}`role content` -``` - -Roles are a bit more simple than directives, though some roles allow for more complex syntax inside their content area. -For example, the `ref` role is used to make references to other sections of your documentation, and allows you to specify the displayed text as well as the reference itself within the role: - -```md -{ref}`My displayed text ` -``` - -For example, let's add a **section reference** to your markdown file. -To do this, we'll first need to add a **label** to a section of your page. -To do so, use the following structure: - -```md -(label-name)= -## Some header -``` - -Add this to your markdown file from above, like so: - -````md -# My nifty title - -Some **text**! - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` - -(section-two)= -## Here's another section - -And some more content. -```` - -Because your new section has a label (`section-two`), you can reference it with the `ref` role. -Add it to your markdown file like so: - - -```md -(label-name)= -## Some header -``` - -Add this to your markdown file from above, like so: - -````md -# My nifty title - -Some **text**! - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` - -(section-two)= -## Here's another section - -And some more content. - -And here's {ref}`a reference to this section `. -I can also reference the section {ref}`section-two` without specifying my title. -```` - -Re-build your documentation and you should see the references automatically inserted. -Here's an example of how the `ref` roles look in the final output: - -Here's a reference to {ref}`sphinx/intro:reference`. - -:::{seealso} -For more information about roles, see {ref}`syntax/roles`. -::: - -## Add a comment using extra MyST syntax - -There are many other kinds of syntax in MyST to make writing more productive and enjoyable. -Let's play around with a couple of options. - -First, try writing a **comment**. -This can be done by adding a line starting with `%` to your markdown file. -For example, try adding a comment to your markdown file, like so: - -````md -# My nifty title - -Some **text**! - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` - -(section-two)= -## Here's another section - -And some more content. - -% This comment won't make it into the outputs! -And here's {ref}`a reference to this section `. -I can also reference the section {ref}`section-two` without specifying my title. -```` - -Re-build your documentation - the comment should _not_ be present in the output. - -## Extending MyST via configuration - -Thus far we have covered the basic MyST syntax with Sphinx. -However, there are a few ways that you can _extend_ this base syntax and get new functionality. -The first is to enable some "out of the box" extensions with the MyST parser. -These add new syntax that aren't part of "core MyST" but that are useful nonetheless (and may become part of core MyST one day). - -Let's extend the base MyST syntax to enable **fences for directives**. -This allows you to define a directive with `:::` in addition to ` ``` `. -This is useful for directives that have markdown in their content. -By using `:::`, a non-MyST markdown renderer will still be able to render what is inside (instead of displaying it as a code block). - -To activate extensions, add a list to your `conf.py` file that contains the extensions you'd like to activate. -For example, to activate the "colon code fences" extension, add the following to your `conf.py` file: - -```python -myst_enable_extensions = [ - "colon_fence", -] -``` - -You may now use `:::` to define directives. -For example, modify your markdown file like so: - -````md -# My nifty title - -Some **text**! - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` - -(section-two)= -## Here's another section - -And some more content. - -% This comment won't make it into the outputs! -And here's {ref}`a reference to this section `. -I can also reference the section {ref}`section-two` without specifying my title. - -:::{note} -And here's a note with a colon fence! -::: -```` - -It should render as a "note block" in your output when you build your site. - -## Install a new Sphinx extension and use its functionality - -The other way to extend MyST in Sphinx is to install Sphinx extensions that define new directives. -Directives are kind of like "functions" in Sphinx, and installing a new package can add new directives to use in your content. - -For example, let's install the `sphinxcontib.mermaid` extension, which will allow us to generate [Mermaid diagrams](https://mermaid-js.github.io/mermaid/#/) with MyST. - -First, install `sphinxcontrib.mermaid`: - -```shell -pip install sphinxcontrib-mermaid -``` - -Next, add it to your list of extensions in `conf.py`: - -```python -extensions = [ - "myst_parser", - "sphinxcontrib.mermaid", -] -``` - -Now, add a **mermaid directive** to your markdown file. -For example: - -````md -# My nifty title - -Some **text**! - -```{admonition} Here's my title -:class: warning - -Here's my admonition content -``` - -(section-two)= -## Here's another section - -And some more content. - -% This comment won't make it into the outputs! -And here's {ref}`a reference to this section `. -I can also reference the section {ref}`section-two` without specifying my title. - -:::{note} -And here's a note with a colon fence! -::: - -And finally, here's a cool mermaid diagram! - -```{mermaid} -sequenceDiagram - participant Alice - participant Bob - Alice->John: Hello John, how are you? - loop Healthcheck - John->John: Fight against hypochondria - end - Note right of John: Rational thoughts
prevail... - John-->Alice: Great! - John->Bob: How about you? - Bob-->John: Jolly good! -``` -```` - -When you build your documentation, you should see something like this: - -```{mermaid} -sequenceDiagram - participant Alice - participant Bob - Alice->John: Hello John, how are you? - loop Healthcheck - John->John: Fight against hypochondria - end - Note right of John: Rational thoughts
prevail... - John-->Alice: Great! - John->Bob: How about you? - Bob-->John: Jolly good! -``` - -## Next steps - Learn more about MyST Syntax - -In this tutorial we've covered some of the basics of MyST Markdown, how to enable and use it with Sphinx, and how to extend it for new use-cases. -There is much more functionality in MyST (and in the Sphinx ecosystem) that we haven't covered here. -For more information, see the [documentation on MyST Syntax](../syntax/syntax.md) and the [documentation about using MyST with Sphinx](../sphinx/index.md). diff --git a/docs/sphinx/reference.md b/docs/sphinx/reference.md deleted file mode 100644 index c77f559d..00000000 --- a/docs/sphinx/reference.md +++ /dev/null @@ -1,129 +0,0 @@ -(sphinx/config-options)= -# Sphinx configuration options - -You can control the behaviour of the MyST parser in Sphinx by modifying your `conf.py` file. -To do so, use the keywords beginning `myst_`. - -`````{list-table} -:header-rows: 1 - -* - Option - - Default - - Description -* - `myst_commonmark_only` - - `False` - - If `True` convert text as strict [CommonMark](https://spec.commonmark.org/) (all options below are then ignored). Note that strict CommonMark is unable to parse any directives, including the `toctree` directive, thus limiting MyST parser to single-page documentations. Use in conjunction with [sphinx-external-toc](https://github.com/executablebooks/sphinx-external-toc) Sphinx extension to counter this limitation. -* - `myst_gfm_only` - - `False` - - If `True` convert text as strict [GitHub-flavored Markdown](https://github.github.com/gfm/) (all options below are then ignored). -* - `myst_disable_syntax` - - () - - List of markdown syntax elements to disable, see the [markdown-it parser guide](markdown_it:using). -* - `myst_enable_extensions` - - `["dollarmath"]` - - Enable Markdown extensions, [see here](../syntax/optional.md) for details. -* - `myst_all_links_external` - - `False` - - If `True`, all Markdown links `[text](link)` are treated as external. -* - `myst_url_schemes` - - `None` - - [URI schemes](https://en.wikipedia.org/wiki/List_of_URI_schemes) that will be recognised as external URLs in `[](scheme:loc)` syntax, or set `None` to recognise all. - Other links will be resolved as internal cross-references. -* - `myst_ref_domains` - - `None` - - If a list, then only these [sphinx domains](sphinx:domain) will be searched for when resolving Markdown links like `[text](reference)`. -* - `myst_linkify_fuzzy_links` - - `True` - - If `False`, only links that contain a scheme (such as `http`) will be recognised as external links. -* - `myst_title_to_header` - - `False` - - If `True`, the `title` key of a document front-matter is converted to a header at the top of the document. -* - `myst_heading_anchors` - - `None` - - Enable auto-generated heading anchors, up to a maximum level, [see here](syntax/header-anchors) for details. -* - `myst_heading_slug_func` - - `None` - - Use the specified function to auto-generate heading anchors, [see here](syntax/header-anchors) for details. -* - `myst_number_code_blocks` - - `()` - - Add line numbers to code blocks with these languages, [see here](syntax/code-blocks) for details. -* - `myst_substitutions` - - `{}` - - A mapping of keys to substitutions, used globally for all MyST documents when the "substitution" extension is enabled. -* - `myst_html_meta` - - `{}` - - A mapping of keys to HTML metadata, used globally for all MyST documents. See [](syntax/html_meta). -* - `myst_footnote_transition` - - `True` - - Place a transition before any footnotes. -* - `myst_words_per_minute` - - `200` - - Reading speed used to calculate `` {sub-ref}`wordcount-minutes` `` -````` - -List of extensions: - -- "amsmath": enable direct parsing of [amsmath](https://ctan.org/pkg/amsmath) LaTeX equations -- "colon_fence": Enable code fences using `:::` delimiters, [see here](syntax/colon_fence) for details -- "deflist": Enable definition lists, [see here](syntax/definition-lists) for details -- "dollarmath": Enable parsing of dollar `$` and `$$` encapsulated math -- "html_admonition": Convert `
` elements to sphinx admonition nodes, see the [HTML admonition syntax](syntax/html-admonition) for details -- "fieldlist": Enable field lists, [see here](syntax/fieldlists) for details -- "html_image": Convert HTML `` elements to sphinx image nodes, see the [image syntax](syntax/images) for details -- "linkify": automatically identify "bare" web URLs and add hyperlinks -- "replacements": automatically convert some common typographic texts -- "smartquotes": automatically convert standard quotations to their opening/closing variants -- "substitution": substitute keys, see the [substitutions syntax](syntax/substitutions) for details -- "tasklist": add check-boxes to the start of list items, see the [tasklist syntax](syntax/tasklists) for details - -Math specific, when `"dollarmath"` activated, see the [Math syntax](syntax/math) for more details: - -`````{list-table} -:header-rows: 1 - -* - Option - - Default - - Description -* - `myst_dmath_double_inline` - - `False` - - Allow display math (i.e. `$$`) within an inline context -* - `myst_dmath_allow_labels` - - `True` - - Parse `$$...$$ (label)` syntax -* - `myst_dmath_allow_space` - - `True` - - If False then inline math will only be parsed if there are no initial/final spaces, - e.g. `$a$` but not `$ a$` or `$a $` -* - `myst_dmath_allow_digits` - - `True` - - If False then inline math will only be parsed if there are no initial/final digits, - e.g. `$a$` but not `1$a$` or `$a$2` (this is useful for using `$` as currency) -* - `myst_amsmath_enable` - - `False` - - Enable direct parsing of [amsmath LaTeX environments](https://ctan.org/pkg/amsmath) -* - `myst_update_mathjax` - - `True` - - If using [sphinx.ext.mathjax](https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.mathjax) (the default) then `mathjax_config` will be updated, - to ignore `$` delimiters and LaTeX environments, which should instead be handled by - `myst_dmath_enable` and `myst_amsmath_enable` respectively. -````` - -## Disable markdown syntax for the parser - -If you'd like to either enable or disable custom markdown syntax, use `myst_disable_syntax`. -Anything in this list will no longer be parsed by the MyST parser. - -For example, to disable the `emphasis` in-line syntax, use this configuration: - -```python -myst_disable_syntax = ["emphasis"] -``` - -emphasis syntax will now be disabled. For example, the following will be rendered -*without* any italics: - -```md -*emphasis is now disabled* -``` - -For a list of all the syntax elements you can disable, see the [markdown-it parser guide](markdown_it:using). diff --git a/docs/sphinx/roles-and-directives.md b/docs/sphinx/roles-and-directives.md deleted file mode 100644 index 63c86a31..00000000 --- a/docs/sphinx/roles-and-directives.md +++ /dev/null @@ -1,23 +0,0 @@ -# Special roles and directives - -This section contains information about special roles and directives that come bundled with the MyST Parser Sphinx extension. - -## Insert the date and reading time - -```{versionadded} 0.14.0 -The `sub-ref` role and word counting. -``` - -You may insert the "last updated" date and estimated reading time into your document via substitution definitions, which can be accessed *via* the `sub-ref` role. - -For example: - -```markdown -> {sub-ref}`today` | {sub-ref}`wordcount-words` words | {sub-ref}`wordcount-minutes` min read -``` - -> {sub-ref}`today` | {sub-ref}`wordcount-words` words | {sub-ref}`wordcount-minutes` min read - -`today` is replaced by either the date on which the document is parsed, with the format set by [`today_fmt`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-today_fmt), or the `today` variable if set in the configuration file. - -The reading speed is computed using the `myst_words_per_minute` configuration (see the [Sphinx configuration options](sphinx/config-options)). diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index 9854d09a..4045eb2e 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -1,22 +1,23 @@ --- -substitutions: - key1: I'm a **substitution** - key2: | - ```{note} - {{ key1 }} - ``` - key3a: fishy - key3: | - ```{image} img/fun-fish.png - :alt: fishy - :width: 200px - ``` - key4: example +myst: + substitutions: + key1: I'm a **substitution** + key2: | + ```{note} + {{ key1 }} + ``` + key3a: fishy + key3: | + ```{image} img/fun-fish.png + :alt: fishy + :width: 200px + ``` + key4: example --- -(syntax/optional)= +(syntax/extensions)= -# Optional MyST Syntaxes +# Syntax Extensions MyST-Parser is highly configurable, utilising the inherent "plugability" of the [markdown-it-py](markdown_it:index) parser. The following syntaxes are optional (disabled by default) and can be enabled *via* the sphinx `conf.py` (see also [](sphinx/config-options)). @@ -74,7 +75,7 @@ text | converted ``--`` | -- ``---`` | --- -(syntax/strikethough)= +(syntax/strikethrough)= ## Strikethrough @@ -283,18 +284,19 @@ or at the top of the file, in the front-matter section (see [this section](synta ````yaml --- -substitutions: - key1: "I'm a **substitution**" - key2: | - ```{note} - {{ key1 }} - ``` - key3: | - ```{image} img/fun-fish.png - :alt: fishy - :width: 200px - ``` - key4: example +myst: + substitutions: + key1: "I'm a **substitution**" + key2: | + ```{note} + {{ key1 }} + ``` + key3: | + ```{image} img/fun-fish.png + :alt: fishy + :width: 200px + ``` + key4: example --- ```` @@ -304,7 +306,8 @@ Keys in the front-matter will override ones in the `conf.py`. You can use these substitutions inline or as blocks, and you can even nest substitutions in other substitutions (but circular references are prohibited): -:::{tabbed} Markdown Input +::::{tab-set} +:::{tab-item} Markdown Input ```md Inline: {{ key1 }} @@ -321,7 +324,7 @@ Block level: ::: -:::{tabbed} Rendered Output +:::{tab-item} Rendered Output Inline: {{ key1 }} Block level: @@ -333,6 +336,7 @@ Block level: | {{key2}} | {{key3}} | ::: +:::: :::{important} @@ -407,7 +411,8 @@ you can also use `:::` delimiters to denote code fences, instead of ```` ``` ``` Using colons instead of back-ticks has the benefit of allowing the content to be rendered correctly, when you are working in any standard Markdown editor. It is ideal for admonition type directives (as documented in [Directives](syntax/directives)) or tables with titles, for example: -````{tabbed} Markdown Input +::::::{tab-set} +:::::{tab-item} Markdown Input ```md :::{note} This text is **standard** _Markdown_ @@ -423,9 +428,9 @@ abc | mnp | xyz ::: ``` -```` +::::: -````{tabbed} Rendered Output +:::::{tab-item} Rendered Output :::{note} This text is **standard** _Markdown_ @@ -440,7 +445,8 @@ abc | mnp | xyz 123 | 456 | 789 ::: -```` +::::: +:::::: Similar to normal directives, these directives can also be nested: @@ -503,7 +509,7 @@ You can then insert markdown links directly to anchors that are generated from y For example `[](#auto-generated-header-anchors)`: [](#auto-generated-header-anchors). The paths to other files should be relative to the current file, for example -`[**link text**](./syntax.md#the-myst-syntax-guide)`: [**link text**](./syntax.md#the-myst-syntax-guide). +`[**link text**](./syntax.md#core-syntax)`: [**link text**](./syntax.md#core-syntax). ### Anchor slug structure diff --git a/docs/syntax/reference.md b/docs/syntax/reference.md index 91ff7b8d..1e53b43e 100644 --- a/docs/syntax/reference.md +++ b/docs/syntax/reference.md @@ -1,4 +1,5 @@ -# MyST Syntax Reference +(syntax-tokens)= +# Syntax tokens This page serves as a reference for the syntax that makes of MyST Markdown. @@ -13,8 +14,6 @@ Block tokens span multiple lines of content. They are broken down into two secti - {ref}`extended-block-tokens` contains *extra* tokens that are not in CommonMark. - {ref}`commonmark-block-tokens` contains CommonMark tokens that also work, for reference. -In addition to these summaries of block-level syntax, see {ref}`extra-markdown-syntax`. - :::{note} Because MyST markdown was inspired by functionality that exists in reStructuredText, we have shown equivalent rST syntax for many MyST markdown features below. @@ -164,8 +163,6 @@ sections below: - {ref}`extended-span-tokens` contains *extra* tokens that are not in CommonMark. - {ref}`commonmark-span-tokens` contains CommonMark tokens that also work, for reference. -In addition to these summaries of inline syntax, see {ref}`extra-markdown-syntax`. - (extended-span-tokens)= ### Extended inline tokens diff --git a/docs/syntax/roles-and-directives.md b/docs/syntax/roles-and-directives.md new file mode 100644 index 00000000..4b3d80a4 --- /dev/null +++ b/docs/syntax/roles-and-directives.md @@ -0,0 +1,421 @@ +(roles-directives)= + +# Roles and Directives + +Roles and directives provide a way to extend the syntax of MyST in an unbound manner, +by interpreting a chuck of text as a specific type of markup, according to its name. + +Mostly all [docutils roles](https://docutils.sourceforge.io/docs/ref/rst/roles.html), [docutils directives](https://docutils.sourceforge.io/docs/ref/rst/directives.html), [sphinx roles](https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html), or [sphinx directives](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html) can be used in MyST. + +## Syntax + +(syntax/directives)= + +### Directives - a block-level extension point + +Directives syntax is defined with triple-backticks and curly-brackets. +It is effectively a Markdown code fence with curly brackets around the language, and a directive name in place of a language name. +Here is the basic structure: + +`````{list-table} +--- +header-rows: 1 +--- +* - MyST + - reStructuredText +* - ````md + ```{directivename} arguments + --- + key1: val1 + key2: val2 + --- + This is + directive content + ``` + ```` + - ```rst + .. directivename:: arguments + :key1: val1 + :key2: val2 + + This is + directive content + ``` +````` + +For example, the following code: + +````md +```{admonition} This is my admonition +This is my note +``` +```` + +Will generate this admonition: + +```{admonition} This is my admonition +This is my note +``` + +#### Parameterizing directives + +For directives that take parameters as input, there are two ways to parameterize them. +In each case, the options themselves are given as `key: value` pairs. An example of +each is shown below: + +**Using YAML frontmatter**. A block of YAML front-matter just after the +first line of the directive will be parsed as options for the directive. This needs to be +surrounded by `---` lines. Everything in between will be parsed by YAML and +passed as keyword arguments to your directive. For example: + +````md +```{code-block} python +--- +lineno-start: 10 +emphasize-lines: 1, 3 +caption: | + This is my + multi-line caption. It is *pretty nifty* ;-) +--- +a = 2 +print('my 1st line') +print(f'my {a}nd line') +``` +```` + +```{code-block} python +--- +lineno-start: 10 +emphasize-lines: 1, 3 +caption: | + This is my + multi-line caption. It is *pretty nifty* ;-) +--- +a = 2 +print('my 1st line') +print(f'my {a}nd line') +``` + +**Short-hand options with `:` characters**. If you only need one or two options for your +directive and wish to save lines, you may also specify directive options as a collection +of lines just after the first line of the directive, each preceding with `:`. Then the +leading `:` is removed from each line, and the rest is parsed as YAML. + +For example: + +````md +```{code-block} python +:lineno-start: 10 +:emphasize-lines: 1, 3 + +a = 2 +print('my 1st line') +print(f'my {a}nd line') +``` +```` + +(syntax/directives/parsing)= + +#### How directives parse content + +Some directives parse the content that is in their content block. +MyST parses this content **as Markdown**. + +This means that MyST markdown can be written in the content areas of any directives written in MyST markdown. For example: + +````md +```{admonition} My markdown link +Here is [markdown link syntax](https://jupyter.org) +``` +```` + +```{admonition} My markdown link +Here is [markdown link syntax](https://jupyter.org) +``` + +As a short-hand for directives that require no arguments, and when no parameter options are used (see below), +you may start the content directly after the directive name. + +````md +```{note} Notes require **no** arguments, so content can start here. +``` +```` + +```{note} Notes require **no** arguments, so content can start here. +``` + +For special cases, MySt also offers the `eval-rst` directive. +This will parse the content **as ReStructuredText**: + +````md +```{eval-rst} +.. figure:: img/fun-fish.png + :width: 100px + :name: rst-fun-fish + + Party time! + +A reference from inside: :ref:`rst-fun-fish` + +A reference from outside: :ref:`syntax/directives/parsing` +``` +```` + +```{eval-rst} +.. figure:: img/fun-fish.png + :width: 100px + :name: rst-fun-fish + + Party time! + +A reference from inside: :ref:`rst-fun-fish` + +A reference from outside: :ref:`syntax/directives/parsing` +``` + +Note how the text is integrated into the rest of the document, so we can also reference [party fish](rst-fun-fish) anywhere else in the documentation. + +#### Nesting directives + +You can nest directives by ensuring that the tick-lines corresponding to the +outermost directive are longer than the tick-lines for the inner directives. +For example, nest a warning inside a note block like so: + +`````md +````{note} +The next info should be nested +```{warning} +Here's my warning +``` +```` +````` + +Here's how it looks rendered: + +````{note} +The next info should be nested +```{warning} +Here's my warning +``` +```` + +You can indent inner-code fences, so long as they aren't indented by more than 3 spaces. +Otherwise, they will be rendered as "raw code" blocks: + +`````md +````{note} +The warning block will be properly-parsed + + ```{warning} + Here's my warning + ``` + +But the next block will be parsed as raw text + + ```{warning} + Here's my raw text warning that isn't parsed... + ``` +```` +````` + +````{note} +The warning block will be properly-parsed + + ```{warning} + Here's my warning + ``` + +But the next block will be parsed as raw text + + ```{warning} + Here's my raw text warning that isn't parsed... + ``` +```` + +This can really be abused if you'd like ;-) + +``````{note} +The next info should be nested +`````{warning} +Here's my warning +````{admonition} Yep another admonition +```python +# All this fuss was about this boring python?! +print('yep!') +``` +```` +````` +`````` + +#### Markdown-friendly directives + +Want to use syntax that renders correctly in standard Markdown editors? +See [the extended syntax option](syntax/colon_fence). + +```md +:::{note} +This text is **standard** *Markdown* +::: +``` + +:::{note} +This text is **standard** *Markdown* +::: + +(syntax/roles)= + +### Roles - an in-line extension point + +Roles are similar to directives - they allow you to define arbitrary new functionality, but they are used *in-line*. +To define an in-line role, use the following form: + +````{list-table} +--- +header-rows: 1 +--- +* - MyST + - reStructuredText +* - ````md + {role-name}`role content` + ```` + - ```rst + :role-name:`role content` + ``` +```` + +For example, the following code: + +```md +Since Pythagoras, we know that {math}`a^2 + b^2 = c^2` +``` + +Becomes: + +Since Pythagoras, we know that {math}`a^2 + b^2 = c^2` + +You can use roles to do things like reference equations and other items in +your book. For example: + +````md +```{math} e^{i\pi} + 1 = 0 +--- +label: euler +--- +``` + +Euler's identity, equation {math:numref}`euler`, was elected one of the +most beautiful mathematical formulas. +```` + +Becomes: + +```{math} e^{i\pi} + 1 = 0 +--- +label: euler +--- +``` + +Euler's identity, equation {math:numref}`euler`, was elected one of the +most beautiful mathematical formulas. + +#### How roles parse content + +The content of roles is parsed differently depending on the role that you've used. +Some roles expect inputs that will be used to change functionality. For example, +the `ref` role will assume that input content is a reference to some other part of the +site. However, other roles may use the MyST parser to parse the input as content. + +Some roles also **extend their functionality** depending on the content that you pass. +For example, following the `ref` example above, if you pass a string like this: +`Content to display `, then the `ref` will display `Content to display` and use +`myref` as the reference to look up. + +How roles parse this content depends on the author that created the role. + +## Common roles and directives + +:::{admonition} {material-regular}`engineering;1.5rem;sd-mr-1` Currently Under Construction +:class: no-icon +Check back for more... +::: + +### ToC Trees + +```{doc-directive} contents +Insert a table of contents tree of the documents headings. +``` + +```{doc-directive} toctree +Inserts a Sphinx "Table of Contents" tree, containing a list of (relative) child document paths. +``` + +### Admonitions + +```{doc-directive} admonition +Create a generic "callout" box, containing the content. +``` + +```{doc-directive} note +Create a "callout" box, specific to notes, containing the content. +``` + +Other admonitions (same structure as `note`): `attention`, `caution`, `danger`, `error`, `hint`, `important`, `tip`, `warning`. + +Sphinx only: `deprecated`, `versionadded`, `versionchanged`. + +### Images and Figures + +```{doc-directive} image +Insert an image, from a (relative) path or URL. +``` + +```{doc-directive} figure +Insert an image, from a (relative) path or URL, +with a caption (first paragraph), and optional legend (subsequent content). +``` + +```{doc-directive} table +Insert a (MyST) table with a caption. +``` + +### Tables + +```{doc-directive} list-table +Create a table from data in a uniform two-level bullet list. +``` + +```{doc-directive} csv-table +Create a table from CSV (comma-separated values) data. +``` + +### Code + +```{doc-directive} code-block +Syntax highlight a block of code, according to the language. +``` + +(syntax/roles/special)= + +### MyST only + +This section contains information about special roles and directives that come bundled with the MyST Parser Sphinx extension. + +#### Insert the date and reading time + +```{versionadded} 0.14.0 +The `sub-ref` role and word counting. +``` + +You may insert the "last updated" date and estimated reading time into your document via substitution definitions, which can be accessed *via* the `sub-ref` role. + +For example: + +```markdown +> {sub-ref}`today` | {sub-ref}`wordcount-words` words | {sub-ref}`wordcount-minutes` min read +``` + +> {sub-ref}`today` | {sub-ref}`wordcount-words` words | {sub-ref}`wordcount-minutes` min read + +`today` is replaced by either the date on which the document is parsed, with the format set by [`today_fmt`](https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-today_fmt), or the `today` variable if set in the configuration file. + +The reading speed is computed using the `myst_words_per_minute` configuration (see the [Sphinx configuration options](sphinx/config-options)). diff --git a/docs/syntax/syntax.md b/docs/syntax/syntax.md index 38436191..877afadc 100644 --- a/docs/syntax/syntax.md +++ b/docs/syntax/syntax.md @@ -1,398 +1,59 @@ -(example_syntax)= +(syntax/core)= -# The MyST Syntax Guide +# Core Syntax -> {sub-ref}`today` | {sub-ref}`wordcount-minutes` min read +## Introduction -As a base, MyST adheres to the [CommonMark specification](https://spec.commonmark.org/). -For this, it uses the [markdown-it-py](https://github.com/executablebooks/markdown-it-py) parser, -which is a well-structured markdown parser for Python that is CommonMark-compliant -and also extensible. +MyST is a strict superset of the [CommonMark syntax specification](https://spec.commonmark.org/). +It adds features focussed on scientific and technical documentation authoring, as detailed below. -MyST adds several new syntax options to CommonMark in order to be used -with Sphinx, the documentation generation engine used extensively in the Python -ecosystem. - -Below is a summary of the syntax 'tokens' parsed, -and further details of a few major extensions from the CommonMark flavor of markdown. +In addition, the roles and directives syntax provide inline/block-level extension points for plugins. +This is detailed further in the [Roles and Directives](roles-directives) section. :::{seealso} -- For an introduction to writing Directives and Roles with MyST markdown, see {ref}`intro/writing`. -- Check out the [MyST-Markdown VS Code extension](https://marketplace.visualstudio.com/items?itemName=ExecutableBookProject.myst-highlight), for MyST extended syntax highlighting. -::: - -MyST builds on the tokens defined by markdown-it, to extend the syntax -described in the [CommonMark Spec](https://spec.commonmark.org/0.29/), which the parser is tested against. - -% TODO link to markdown-it documentation - -(syntax/directives)= - -## Directives - a block-level extension point - -Directives syntax is defined with triple-backticks and curly-brackets. It -is effectively a code block with curly brackets around the language, and -a directive name in place of a language name. It is similar to how RMarkdown -defines "runnable cells". Here is the basic structure: - -`````{list-table} ---- -header-rows: 1 ---- -* - MyST - - reStructuredText -* - ````md - ```{directivename} arguments - --- - key1: val1 - key2: val2 - --- - This is - directive content - ``` - ```` - - ```rst - .. directivename:: arguments - :key1: val1 - :key2: val2 - - This is - directive content - ``` -````` - -For example, the following code: - -````md -```{admonition} This is my admonition -This is my note -``` -```` - -Will generate this admonition: - -```{admonition} This is my admonition -This is my note -``` - -### Parameterizing directives - -For directives that take parameters as input, there are two ways to parameterize them. -In each case, the options themselves are given as `key: value` pairs. An example of -each is shown below: - -**Using YAML frontmatter**. A block of YAML front-matter just after the -first line of the directive will be parsed as options for the directive. This needs to be -surrounded by `---` lines. Everything in between will be parsed by YAML and -passed as keyword arguments to your directive. For example: - -````md -```{code-block} python ---- -lineno-start: 10 -emphasize-lines: 1, 3 -caption: | - This is my - multi-line caption. It is *pretty nifty* ;-) ---- -a = 2 -print('my 1st line') -print(f'my {a}nd line') -``` -```` - -```{code-block} python ---- -lineno-start: 10 -emphasize-lines: 1, 3 -caption: | - This is my - multi-line caption. It is *pretty nifty* ;-) ---- -a = 2 -print('my 1st line') -print(f'my {a}nd line') -``` - -**Short-hand options with `:` characters**. If you only need one or two options for your -directive and wish to save lines, you may also specify directive options as a collection -of lines just after the first line of the directive, each preceding with `:`. Then the -leading `:` is removed from each line, and the rest is parsed as YAML. - -For example: - -````md -```{code-block} python -:lineno-start: 10 -:emphasize-lines: 1, 3 - -a = 2 -print('my 1st line') -print(f'my {a}nd line') -``` -```` - -(syntax/directives/parsing)= -### How directives parse content - -Some directives parse the content that is in their content block. -MyST parses this content **as Markdown**. - -This means that MyST markdown can be written in the content areas of any directives written in MyST markdown. For example: - -````md -```{admonition} My markdown link -Here is [markdown link syntax](https://jupyter.org) -``` -```` - -```{admonition} My markdown link -Here is [markdown link syntax](https://jupyter.org) -``` - -As a short-hand for directives that require no arguments, and when no parameter options are used (see below), -you may start the content directly after the directive name. - -````md -```{note} Notes require **no** arguments, so content can start here. -``` -```` - -```{note} Notes require **no** arguments, so content can start here. -``` - -For special cases, MySt also offers the `eval-rst` directive. -This will parse the content **as ReStructuredText**: - -````md -```{eval-rst} -.. figure:: img/fun-fish.png - :width: 100px - :name: rst-fun-fish - - Party time! - -A reference from inside: :ref:`rst-fun-fish` - -A reference from outside: :ref:`syntax/directives/parsing` -``` -```` - -```{eval-rst} -.. figure:: img/fun-fish.png - :width: 100px - :name: rst-fun-fish - - Party time! - -A reference from inside: :ref:`rst-fun-fish` - -A reference from outside: :ref:`syntax/directives/parsing` -``` - -Note how the text is integrated into the rest of the document, so we can also reference [party fish](rst-fun-fish) anywhere else in the documentation. - -### Nesting directives - -You can nest directives by ensuring that the tick-lines corresponding to the -outermost directive are longer than the tick-lines for the inner directives. -For example, nest a warning inside a note block like so: - -`````md -````{note} -The next info should be nested -```{warning} -Here's my warning -``` -```` -````` - -Here's how it looks rendered: - -````{note} -The next info should be nested -```{warning} -Here's my warning -``` -```` - -You can indent inner-code fences, so long as they aren't indented by more than 3 spaces. -Otherwise, they will be rendered as "raw code" blocks: - -`````md -````{note} -The warning block will be properly-parsed - - ```{warning} - Here's my warning - ``` - -But the next block will be parsed as raw text - - ```{warning} - Here's my raw text warning that isn't parsed... - ``` -```` -````` - -````{note} -The warning block will be properly-parsed - - ```{warning} - Here's my warning - ``` - -But the next block will be parsed as raw text - - ```{warning} - Here's my raw text warning that isn't parsed... - ``` -```` - -This can really be abused if you'd like ;-) - -``````{note} -The next info should be nested -`````{warning} -Here's my warning -````{admonition} Yep another admonition -```python -# All this fuss was about this boring python?! -print('yep!') -``` -```` -````` -`````` - -### Markdown-friendly directives - -Want to use syntax that renders correctly in standard Markdown editors? -See [the extended syntax option](syntax/colon_fence). - -```md -:::{note} -This text is **standard** _Markdown_ -::: -``` - -:::{note} -This text is **standard** _Markdown_ +The [syntax token reference tables](syntax-tokens) ::: -(syntax/roles)= - -## Roles - an in-line extension point - -Roles are similar to directives - they allow you to define arbitrary new -functionality, but they are used *in-line*. To define an in-line -role, use the following form: - -````{list-table} ---- -header-rows: 1 ---- -* - MyST - - reStructuredText -* - ````md - {role-name}`role content` - ```` - - ```rst - :role-name:`role content` - ``` -```` - -For example, the following code: +(syntax/commonmark)= + +## CommonMark + +The [CommonMark syntax specification](https://spec.commonmark.org/) details the full set of syntax rules. +Here we provide a summary of most features: + +Element | Syntax +--------------- | ------------------------------------------- +Heading | `# H1` to `###### H6` +Bold | `**bold**` +Italic | `*italic*` +Inline Code | `` `code` `` +Autolink | `` +URL Link | `[title](https://www.example.com)` +Image | `![alt](https://www.example.com/image.png)` +Reference Link | `[title][link]` +Link Definition | `[link]: https://www.example.com` +Thematic break | `---` +Blockquote | `> quote` +Ordered List | `1. item` +Unordered List | `- item` +Code Fence | opening ```` ```lang ```` to closing ```` ``` ```` -```md -Since Pythagoras, we know that {math}`a^2 + b^2 = c^2` -``` - -Becomes: - -Since Pythagoras, we know that {math}`a^2 + b^2 = c^2` - -You can use roles to do things like reference equations and other items in -your book. For example: - -````md -```{math} e^{i\pi} + 1 = 0 ---- -label: euler ---- -``` +(syntax/frontmatter)= -Euler's identity, equation {math:numref}`euler`, was elected one of the -most beautiful mathematical formulas. -```` +## Front Matter -Becomes: +This is a [YAML](https://en.wikipedia.org/wiki/YAML) block at the start of the document, as used for example in [jekyll](https://jekyllrb.com/docs/front-matter/). +The document should start with three or more `---` markers, and YAML is parsed until a closing `---` marker is found: -```{math} e^{i\pi} + 1 = 0 +```yaml --- -label: euler +key1: value +key2: [value1, value2] +key3: + subkey1: value --- ``` -Euler's identity, equation {math:numref}`euler`, was elected one of the -most beautiful mathematical formulas. - -### How roles parse content - -The content of roles is parsed differently depending on the role that you've used. -Some roles expect inputs that will be used to change functionality. For example, -the `ref` role will assume that input content is a reference to some other part of the -site. However, other roles may use the MyST parser to parse the input as content. - -Some roles also **extend their functionality** depending on the content that you pass. -For example, following the `ref` example above, if you pass a string like this: -`Content to display `, then the `ref` will display `Content to display` and use -`myref` as the reference to look up. - -How roles parse this content depends on the author that created the role. - -(syntax/roles/special)= - -(extra-markdown-syntax)= -## Extra markdown syntax - -In addition to roles and directives, MyST supports extra markdown syntax that doesn't -exist in CommonMark. In most cases, these are syntactic short-cuts to calling -roles and directives. We'll cover some common ones below. - -This table describes the rST and MyST equivalents: - -````{list-table} ---- -header-rows: 1 ---- -* - Type - - MyST - - reStructuredText -* - Front matter - - ```md - --- - key: val - --- - ``` - - ```md - :key: val - ``` -* - Comments - - `% comment` - - `.. comment` -* - Targets - - `(mytarget)=` - - `.. _mytarget:` -```` - -(syntax/frontmatter)= - -## Front Matter - -This is a YAML block at the start of the document, as used for example in -[jekyll](https://jekyllrb.com/docs/front-matter/). - - :::{seealso} Top-matter is also used for the [substitution syntax extension](syntax/substitutions), and can be used to store information for blog posting (see [ablog's myst-parser support](https://ablog.readthedocs.io/en/latest/manual/markdown/)). @@ -429,7 +90,8 @@ This is equivalent to using the [RST `meta` directive](https://www.sphinx-doc.or HTML metadata can also be added globally in the `conf.py` *via* the `myst_html_meta` variable, in which case it will be added to all MyST documents. For each document, the `myst_html_meta` dict will be updated by the document level front-matter `html_meta`, with the front-matter taking precedence. -:::{tabbed} Sphinx Configuration +::::{tab-set} +:::{tab-item} Sphinx Configuration ```python language = "en" @@ -443,21 +105,22 @@ myst_html_meta = { ::: -:::{tabbed} MyST Front-Matter +:::{tab-item} MyST Front-Matter ```yaml --- -html_meta: - "description lang=en": "metadata description" - "description lang=fr": "description des métadonnées" - "keywords": "Sphinx, MyST" - "property=og:locale": "en_US" +myst: + html_meta: + "description lang=en": "metadata description" + "description lang=fr": "description des métadonnées" + "keywords": "Sphinx, MyST" + "property=og:locale": "en_US" --- ``` ::: -:::{tabbed} RestructuredText +:::{tab-item} RestructuredText ```restructuredtext .. meta:: @@ -469,7 +132,7 @@ html_meta: ::: -:::{tabbed} HTML Output +:::{tab-item} HTML Output ```html @@ -481,7 +144,7 @@ html_meta: ``` ::: - +:::: (syntax/comments)= @@ -516,6 +179,10 @@ a line another line ```` +:::{tip} +Comments are equivalent to the RST syntax: `.. my comment`. +::: + (syntax/blockbreaks)= ## Block Breaks @@ -602,8 +269,7 @@ By default, the reference will use the text of the target (such as the section t {ref}`my text ` ``` -For example, see this ref: {ref}`syntax/targets`, and here's a ref back to the top of -this page: {ref}`my text `. +For example, see this ref: {ref}`syntax/targets`, and here's a ref back to the top of this page: {ref}`my text `. Alternatively using the markdown syntax: @@ -620,7 +286,7 @@ is equivalent to using the [any inline role](https://www.sphinx-doc.org/en/maste but can also accept "nested" syntax (like bold text) and will recognise document paths that include extensions (e.g. `syntax/syntax` or `syntax/syntax.md`) Using the same example, see this ref: [](syntax/targets), here is a reference back to the top of -this page: [my text with **nested** $\alpha$ syntax](example_syntax), and here is a reference to another page (`[](../sphinx/intro.md)`): [](../sphinx/intro.md). +this page: [my text with **nested** $\alpha$ syntax](syntax/core), and here is a reference to another page (`[](../intro.md)`): [](../intro.md). ```{note} If you wish to have the target's title inserted into your text, you can @@ -629,7 +295,7 @@ markdown: `[](syntax.md)` will result in: [](syntax.md). ``` (syntax/code-blocks)= -## Code blocks +## Code syntax highlighting Code blocks contain a language identifier, which is used to determine the language of the code. This language is used to determine the syntax highlighting, using an available [pygments lexer](https://pygments.org/docs/lexers/). diff --git a/example-include.md b/example-include.md index 4d23ee51..3f85dc54 100644 --- a/example-include.md +++ b/example-include.md @@ -1,2 +1,2 @@ -[Used in how-to](docs/sphinx/use.md) +[Used in how-to](docs/faq/index.md) ![alt](docs/_static/logo-wide.svg) diff --git a/myst_parser/__init__.py b/myst_parser/__init__.py index 2be73019..507213b7 100644 --- a/myst_parser/__init__.py +++ b/myst_parser/__init__.py @@ -1,67 +1,10 @@ """An extended commonmark compliant parser, with bridges to docutils & sphinx.""" -from typing import TYPE_CHECKING, Any - __version__ = "0.17.2" -if TYPE_CHECKING: - from sphinx.application import Sphinx - - def setup(app): - """Initialize Sphinx extension.""" - from myst_parser.sphinx_parser import MystParser - - app.add_source_suffix(".md", "markdown") - app.add_source_parser(MystParser) - - setup_sphinx(app) + """Initialize the Sphinx extension.""" + from myst_parser.sphinx_ext.main import setup_sphinx + setup_sphinx(app, load_parser=True) return {"version": __version__, "parallel_read_safe": True} - - -def setup_sphinx(app: "Sphinx"): - """Initialize all settings and transforms in Sphinx.""" - # we do this separately to setup, - # so that it can be called by external packages like myst_nb - from myst_parser.directives import FigureMarkdown, SubstitutionReferenceRole - from myst_parser.main import MdParserConfig - from myst_parser.mathjax import override_mathjax - from myst_parser.myst_refs import MystReferenceResolver - - app.add_role("sub-ref", SubstitutionReferenceRole()) - app.add_directive("figure-md", FigureMarkdown) - - app.add_post_transform(MystReferenceResolver) - - for name, default, field in MdParserConfig().as_triple(): - if not field.metadata.get("docutils_only", False): - # TODO add types? - app.add_config_value(f"myst_{name}", default, "env", types=Any) - - app.connect("builder-inited", create_myst_config) - app.connect("builder-inited", override_mathjax) - - -def create_myst_config(app): - from sphinx.util import logging - - # Ignore type checkers because the attribute is dynamically assigned - from sphinx.util.console import bold # type: ignore[attr-defined] - - from myst_parser.main import MdParserConfig - - logger = logging.getLogger(__name__) - - values = { - name: app.config[f"myst_{name}"] - for name, _, field in MdParserConfig().as_triple() - if not field.metadata.get("docutils_only", False) - } - - try: - app.env.myst_config = MdParserConfig(**values) - logger.info(bold("myst v%s:") + " %s", __version__, app.env.myst_config) - except (TypeError, ValueError) as error: - logger.error("myst configuration invalid: %s", error.args[0]) - app.env.myst_config = MdParserConfig() diff --git a/myst_parser/_docs.py b/myst_parser/_docs.py new file mode 100644 index 00000000..a7c46a34 --- /dev/null +++ b/myst_parser/_docs.py @@ -0,0 +1,198 @@ +"""Code to use internally, for documentation.""" +from __future__ import annotations + +import io +from typing import Sequence, Union + +from docutils import nodes +from docutils.frontend import OptionParser +from docutils.parsers.rst import directives +from sphinx.directives import other +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective +from typing_extensions import get_args, get_origin + +from .config.main import MdParserConfig +from .parsers.docutils_ import Parser as DocutilsParser + +logger = logging.getLogger(__name__) + + +class _ConfigBase(SphinxDirective): + """Directive to automate rendering of the configuration.""" + + @staticmethod + def table_header(): + return [ + "```````{list-table}", + ":header-rows: 1", + ":widths: 15 10 20", + "", + "* - Name", + " - Type", + " - Description", + ] + + @staticmethod + def field_default(value): + default = " ".join(f"{value!r}".splitlines()) + return default + + @staticmethod + def field_type(field): + ftypes: Sequence[str] + if get_origin(field.type) is Union: + ftypes = get_args(field.type) + else: + ftypes = [field.type] + ctype = " | ".join( + str("None" if ftype == type(None) else ftype) # type: ignore # noqa: E721 + for ftype in ftypes + ) + ctype = " ".join(ctype.splitlines()) + ctype = ctype.replace("typing.", "") + ctype = ctype.replace("typing_extensions.", "") + for tname in ("str", "int", "float", "bool"): + ctype = ctype.replace(f"", tname) + return ctype + + +class MystConfigDirective(_ConfigBase): + + option_spec = { + "sphinx": directives.flag, + "extensions": directives.flag, + "scope": lambda x: directives.choice(x, ["global", "local"]), + } + + def run(self): + """Run the directive.""" + config = MdParserConfig() + text = self.table_header() + count = 0 + for name, value, field in config.as_triple(): + + # filter by sphinx options + if "sphinx" in self.options and field.metadata.get("sphinx_exclude"): + continue + + if "extensions" in self.options: + if not field.metadata.get("extension"): + continue + else: + if field.metadata.get("extension"): + continue + + if self.options.get("scope") == "local": + if field.metadata.get("global_only"): + continue + + if self.options.get("scope") == "global": + name = f"myst_{name}" + + description = " ".join(field.metadata.get("help", "").splitlines()) + if field.metadata.get("extension"): + description = f"{field.metadata.get('extension')}: {description}" + default = self.field_default(value) + ctype = self.field_type(field) + text.extend( + [ + f"* - `{name}`", + f" - `{ctype}`", + f" - {description} (default: `{default}`)", + ] + ) + + count += 1 + + if not count: + return [] + + text.append("```````") + node = nodes.Element() + self.state.nested_parse(text, 0, node) + return node.children + + +class DocutilsCliHelpDirective(SphinxDirective): + """Directive to print the docutils CLI help.""" + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + + def run(self): + """Run the directive.""" + stream = io.StringIO() + OptionParser( + components=(DocutilsParser,), + usage="myst-docutils- [options] [ []]", + ).print_help(stream) + return [nodes.literal_block("", stream.getvalue())] + + +class DirectiveDoc(SphinxDirective): + """Load and document a directive.""" + + required_arguments = 1 # name of the directive + has_content = True + + def run(self): + """Run the directive.""" + name = self.arguments[0] + # load the directive class + klass, _ = directives.directive( + name, self.state.memo.language, self.state.document + ) + if klass is None: + logger.warning(f"Directive {name} not found.", line=self.lineno) + return [] + content = " ".join(self.content) + text = f"""\ +:Name: `{name}` +:Description: {content} +:Arguments: {klass.required_arguments} required, {klass.optional_arguments} optional +:Content: {'yes' if klass.has_content else 'no'} +:Options: +""" + if klass.option_spec: + text += " name | type\n -----|------\n" + for key, func in klass.option_spec.items(): + text += f" {key} | {convert_opt(name, func)}\n" + node = nodes.Element() + self.state.nested_parse(text.splitlines(), 0, node) + return node.children + + +def convert_opt(name, func): + """Convert an option function to a string.""" + if func is directives.flag: + return "flag" + if func is directives.unchanged: + return "text" + if func is directives.unchanged_required: + return "text" + if func is directives.class_option: + return "space-delimited list" + if func is directives.uri: + return "URI" + if func is directives.path: + return "path" + if func is int: + return "integer" + if func is directives.positive_int: + return "integer (positive)" + if func is directives.nonnegative_int: + return "integer (non-negative)" + if func is directives.positive_int_list: + return "space/comma-delimited list of integers (positive)" + if func is directives.percentage: + return "percentage" + if func is directives.length_or_unitless: + return "length or unitless" + if func is directives.length_or_percentage_or_unitless: + return "length, percentage or unitless" + if func is other.int_or_nothing: + return "integer" + return "" diff --git a/myst_parser/cli.py b/myst_parser/cli.py index 267b373c..a9c6ab09 100644 --- a/myst_parser/cli.py +++ b/myst_parser/cli.py @@ -3,7 +3,8 @@ from markdown_it.renderer import RendererHTML -from myst_parser.main import MdParserConfig, create_md_parser +from myst_parser.config.main import MdParserConfig +from myst_parser.parsers.mdit import create_md_parser def print_anchors(args=None): diff --git a/myst_parser/config/__init__.py b/myst_parser/config/__init__.py new file mode 100644 index 00000000..898f9cef --- /dev/null +++ b/myst_parser/config/__init__.py @@ -0,0 +1 @@ +"""This module holds the global configuration for the parser ``MdParserConfig``.""" diff --git a/myst_parser/dc_validators.py b/myst_parser/config/dc_validators.py similarity index 59% rename from myst_parser/dc_validators.py rename to myst_parser/config/dc_validators.py index c0c69ef2..765cfb93 100644 --- a/myst_parser/dc_validators.py +++ b/myst_parser/config/dc_validators.py @@ -2,10 +2,28 @@ from __future__ import annotations import dataclasses as dc -from typing import Any, Callable, Sequence, Type +from typing import Any, Sequence +from typing_extensions import Protocol -def validate_fields(inst): + +def validate_field(inst: Any, field: dc.Field, value: Any) -> None: + """Validate the field of a dataclass, + according to a `validator` function set in the field.metadata. + + The validator function should take as input (inst, field, value) and + raise an exception if the value is invalid. + """ + if "validator" not in field.metadata: + return + if isinstance(field.metadata["validator"], list): + for validator in field.metadata["validator"]: + validator(inst, field, value) + else: + field.metadata["validator"](inst, field, value) + + +def validate_fields(inst: Any) -> None: """Validate the fields of a dataclass, according to `validator` functions set in the field metadata. @@ -15,19 +33,17 @@ def validate_fields(inst): raise an exception if the value is invalid. """ for field in dc.fields(inst): - if "validator" not in field.metadata: - continue - if isinstance(field.metadata["validator"], list): - for validator in field.metadata["validator"]: - validator(inst, field, getattr(inst, field.name)) - else: - field.metadata["validator"](inst, field, getattr(inst, field.name)) + validate_field(inst, field, getattr(inst, field.name)) -ValidatorType = Callable[[Any, dc.Field, Any], None] +class ValidatorType(Protocol): + def __call__( + self, inst: bytes, field: dc.Field, value: Any, suffix: str = "" + ) -> None: + ... -def instance_of(type: Type[Any] | tuple[Type[Any], ...]) -> ValidatorType: +def instance_of(type: type[Any] | tuple[type[Any], ...]) -> ValidatorType: """ A validator that raises a `TypeError` if the initializer is called with a wrong type for this particular attribute (checks are performed using @@ -36,13 +52,14 @@ def instance_of(type: Type[Any] | tuple[Type[Any], ...]) -> ValidatorType: :param type: The type to check for. """ - def _validator(inst, attr, value): + def _validator(inst, field, value, suffix=""): """ We use a callable class to be able to change the ``__repr__``. """ if not isinstance(value, type): raise TypeError( - f"'{attr.name}' must be {type!r} (got {value!r} that is a {value.__class__!r})." + f"'{field.name}{suffix}' must be of type {type!r} " + f"(got {value!r} that is a {value.__class__!r})." ) return _validator @@ -55,16 +72,16 @@ def optional(validator: ValidatorType) -> ValidatorType: the sub-validator. """ - def _validator(inst, attr, value): + def _validator(inst, field, value, suffix=""): if value is None: return - validator(inst, attr, value) + validator(inst, field, value, suffix=suffix) return _validator -def is_callable(inst, attr, value): +def is_callable(inst, field, value, suffix=""): """ A validator that raises a `TypeError` if the initializer is called with a value for this particular attribute @@ -72,7 +89,7 @@ def is_callable(inst, attr, value): """ if not callable(value): raise TypeError( - f"'{attr.name}' must be callable " + f"'{field.name}{suffix}' must be callable " f"(got {value!r} that is a {value.__class__!r})." ) @@ -86,14 +103,16 @@ def in_(options: Sequence) -> ValidatorType: :param options: Allowed options. """ - def _validator(inst, attr, value): + def _validator(inst, field, value, suffix=""): try: in_options = value in options except TypeError: # e.g. `1 in "abc"` in_options = False if not in_options: - raise ValueError(f"'{attr.name}' must be in {options!r} (got {value!r})") + raise ValueError( + f"'{field.name}{suffix}' must be in {options!r} (got {value!r})" + ) return _validator @@ -108,12 +127,12 @@ def deep_iterable( :param iterable_validator: Validator to apply to iterable itself """ - def _validator(inst, attr, value): + def _validator(inst, field, value, suffix=""): if iterable_validator is not None: - iterable_validator(inst, attr, value) + iterable_validator(inst, field, value, suffix=suffix) - for member in value: - member_validator(inst, attr, member) + for idx, member in enumerate(value): + member_validator(inst, field, member, suffix=f"{suffix}[{idx}]") return _validator @@ -131,12 +150,12 @@ def deep_mapping( :param mapping_validator: Validator to apply to top-level mapping attribute (optional) """ - def _validator(inst, attr, value): + def _validator(inst, field: dc.Field, value, suffix=""): if mapping_validator is not None: - mapping_validator(inst, attr, value) + mapping_validator(inst, field, value) for key in value: - key_validator(inst, attr, key) - value_validator(inst, attr, value[key]) + key_validator(inst, field, key, suffix=f"{suffix}[{key!r}]") + value_validator(inst, field, value[key], suffix=f"{suffix}[{key!r}]") return _validator diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py new file mode 100644 index 00000000..c1f1c7f1 --- /dev/null +++ b/myst_parser/config/main.py @@ -0,0 +1,408 @@ +"""The configuration for the myst parser.""" +import dataclasses as dc +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + Optional, + Sequence, + Tuple, + Union, + cast, +) + +from .dc_validators import ( + deep_iterable, + deep_mapping, + in_, + instance_of, + is_callable, + optional, + validate_field, + validate_fields, +) + + +def check_extensions(_, __, value): + if not isinstance(value, Iterable): + raise TypeError(f"'enable_extensions' not iterable: {value}") + diff = set(value).difference( + [ + "amsmath", + "colon_fence", + "deflist", + "dollarmath", + "fieldlist", + "html_admonition", + "html_image", + "linkify", + "replacements", + "smartquotes", + "strikethrough", + "substitution", + "tasklist", + ] + ) + if diff: + raise ValueError(f"'enable_extensions' items not recognised: {diff}") + + +def check_sub_delimiters(_, __, value): + if (not isinstance(value, (tuple, list))) or len(value) != 2: + raise TypeError(f"myst_sub_delimiters is not a tuple of length 2: {value}") + for delim in value: + if (not isinstance(delim, str)) or len(delim) != 1: + raise TypeError( + f"myst_sub_delimiters does not contain strings of length 1: {value}" + ) + + +@dc.dataclass() +class MdParserConfig: + """Configuration options for the Markdown Parser. + + Note in the sphinx configuration these option names are prepended with ``myst_`` + """ + + # TODO replace commonmark_only, gfm_only with a single option + + commonmark_only: bool = dc.field( + default=False, + metadata={ + "validator": instance_of(bool), + "help": "Use strict CommonMark parser", + }, + ) + gfm_only: bool = dc.field( + default=False, + metadata={ + "validator": instance_of(bool), + "help": "Use strict Github Flavoured Markdown parser", + }, + ) + + enable_extensions: Sequence[str] = dc.field( + default_factory=list, + metadata={"validator": check_extensions, "help": "Enable syntax extensions"}, + ) + + disable_syntax: Iterable[str] = dc.field( + default_factory=list, + metadata={ + "validator": deep_iterable(instance_of(str), instance_of((list, tuple))), + "help": "Disable Commonmark syntax elements", + }, + ) + + all_links_external: bool = dc.field( + default=False, + metadata={ + "validator": instance_of(bool), + "help": "Parse all links as simple hyperlinks", + }, + ) + + # see https://en.wikipedia.org/wiki/List_of_URI_schemes + url_schemes: Optional[Iterable[str]] = dc.field( + default=cast(Optional[Iterable[str]], ("http", "https", "mailto", "ftp")), + metadata={ + "validator": optional( + deep_iterable(instance_of(str), instance_of((list, tuple))) + ), + "help": "URL scheme prefixes identified as external links", + }, + ) + + ref_domains: Optional[Iterable[str]] = dc.field( + default=None, + metadata={ + "validator": optional( + deep_iterable(instance_of(str), instance_of((list, tuple))) + ), + "help": "Sphinx domain names to search in for link references", + }, + ) + + highlight_code_blocks: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Syntax highlight code blocks with pygments", + "docutils_only": True, + }, + ) + + number_code_blocks: Sequence[str] = dc.field( + default_factory=list, + metadata={ + "validator": deep_iterable(instance_of(str), instance_of((list, tuple))), + "help": "Add line numbers to code blocks with these languages", + }, + ) + + title_to_header: bool = dc.field( + default=False, + metadata={ + "validator": instance_of(bool), + "help": "Convert a `title` field in the top-matter to a H1 header", + }, + ) + + heading_anchors: Optional[int] = dc.field( + default=None, + metadata={ + "validator": optional(in_([1, 2, 3, 4, 5, 6, 7])), + "help": "Heading level depth to assign HTML anchors", + }, + ) + + heading_slug_func: Optional[Callable[[str], str]] = dc.field( + default=None, + metadata={ + "validator": optional(is_callable), + "help": "Function for creating heading anchors", + "global_only": True, + }, + ) + + html_meta: Dict[str, str] = dc.field( + default_factory=dict, + repr=False, + metadata={ + "validator": deep_mapping( + instance_of(str), instance_of(str), instance_of(dict) + ), + "merge_topmatter": True, + "help": "HTML meta tags", + }, + ) + + footnote_transition: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Place a transition before any footnotes", + }, + ) + + words_per_minute: int = dc.field( + default=200, + metadata={ + "validator": instance_of(int), + "help": "For reading speed calculations", + }, + ) + + # Extension specific + + substitutions: Dict[str, Union[str, int, float]] = dc.field( + default_factory=dict, + repr=False, + metadata={ + "validator": deep_mapping( + instance_of(str), instance_of((str, int, float)), instance_of(dict) + ), + "merge_topmatter": True, + "help": "Substitutions mapping", + "extension": "substitutions", + }, + ) + + sub_delimiters: Tuple[str, str] = dc.field( + default=("{", "}"), + metadata={ + "validator": check_sub_delimiters, + "help": "Substitution delimiters", + "extension": "substitutions", + }, + ) + + linkify_fuzzy_links: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Recognise URLs without schema prefixes", + "extension": "linkify", + }, + ) + + dmath_allow_labels: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Parse `$$...$$ (label)`", + "extension": "dollarmath", + }, + ) + dmath_allow_space: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Allow initial/final spaces in `$ ... $`", + "extension": "dollarmath", + }, + ) + dmath_allow_digits: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Allow initial/final digits `1$ ...$2`", + "extension": "dollarmath", + }, + ) + dmath_double_inline: bool = dc.field( + default=False, + metadata={ + "validator": instance_of(bool), + "help": "Parse inline `$$ ... $$`", + "extension": "dollarmath", + }, + ) + + update_mathjax: bool = dc.field( + default=True, + metadata={ + "validator": instance_of(bool), + "help": "Update sphinx.ext.mathjax configuration to ignore `$` delimiters", + "extension": "dollarmath", + "global_only": True, + }, + ) + + mathjax_classes: str = dc.field( + default="tex2jax_process|mathjax_process|math|output_area", + metadata={ + "validator": instance_of(str), + "help": "MathJax classes to add to math HTML", + "extension": "dollarmath", + "global_only": True, + }, + ) + + def __post_init__(self): + validate_fields(self) + + def copy(self, **kwargs: Any) -> "MdParserConfig": + """Return a new object replacing specified fields with new values. + + Note: initiating the copy will also validate the new fields. + """ + return dc.replace(self, **kwargs) + + @classmethod + def get_fields(cls) -> Tuple[dc.Field, ...]: + """Return all attribute fields in this class.""" + return dc.fields(cls) + + def as_dict(self, dict_factory=dict) -> dict: + """Return a dictionary of field name -> value.""" + return dc.asdict(self, dict_factory=dict_factory) + + def as_triple(self) -> Iterable[Tuple[str, Any, dc.Field]]: + """Yield triples of (name, value, field).""" + fields = {f.name: f for f in dc.fields(self.__class__)} + for name, value in dc.asdict(self).items(): + yield name, value, fields[name] + + +def merge_file_level( + config: MdParserConfig, + topmatter: Dict[str, Any], + warning: Callable[[str, str], None], +) -> MdParserConfig: + """Merge the file-level topmatter with the global config. + + :param config: Global config. + :param topmatter: Topmatter from the file. + :param warning: Function to call with a warning (type, message). + :returns: A new config object + """ + # get updates + updates: Dict[str, Any] = {} + myst = topmatter.get("myst", {}) + if not isinstance(myst, dict): + warning("topmatter", f"'myst' key not a dict: {type(myst)}") + else: + updates = myst + + # allow html_meta and substitutions at top-level for back-compatibility + if "html_meta" in topmatter: + warning( + "topmatter", + "top-level 'html_meta' key is deprecated, " + "place under 'myst' key instead", + ) + updates["html_meta"] = topmatter["html_meta"] + if "substitutions" in topmatter: + warning( + "topmatter", + "top-level 'substitutions' key is deprecated, " + "place under 'myst' key instead", + ) + updates["substitutions"] = topmatter["substitutions"] + + new = config.copy() + + # validate each update + fields = {name: (value, field) for name, value, field in config.as_triple()} + for name, value in updates.items(): + + if name not in fields: + warning("topmatter", f"Unknown field: {name}") + continue + + old_value, field = fields[name] + + try: + validate_field(new, field, value) + except Exception as exc: + warning("topmatter", str(exc)) + continue + + if field.metadata.get("merge_topmatter"): + value = {**old_value, **value} + + setattr(new, name, value) + + return new + + +class TopmatterReadError(Exception): + """Topmatter parsing error.""" + + +def read_topmatter(text: Union[str, Iterator[str]]) -> Optional[Dict[str, Any]]: + """Read the (optional) YAML topmatter from a source string. + + This is identified by the first line starting with `---`, + then read up to a terminating line of `---`, or `...`. + + :param source: The source string to read from + :return: The topmatter + """ + import yaml + + if isinstance(text, str): + if not text.startswith("---"): # skip creating the line list in memory + return None + text = (line for line in text.splitlines()) + try: + if not next(text).startswith("---"): + return None + except StopIteration: + return None + top_matter = [] + for line in text: + if line.startswith("---") or line.startswith("..."): + break + top_matter.append(line.rstrip() + "\n") + try: + metadata = yaml.safe_load("".join(top_matter)) + assert isinstance(metadata, dict) + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as err: + raise TopmatterReadError("Malformed YAML") from err + if not isinstance(metadata, dict): + raise TopmatterReadError(f"YAML is not a dict: {type(metadata)}") + return metadata diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index f0a7001f..6f2cc848 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -3,266 +3,4 @@ .. include:: path/to/file.md :parser: myst_parser.docutils_ """ -from dataclasses import Field -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union - -from docutils import frontend, nodes -from docutils.core import default_description, publish_cmdline -from docutils.parsers.rst import Parser as RstParser -from markdown_it.token import Token -from typing_extensions import Literal, get_args, get_origin - -from myst_parser.docutils_renderer import DocutilsRenderer -from myst_parser.main import MdParserConfig, create_md_parser - - -def _validate_int( - setting, value, option_parser, config_parser=None, config_section=None -) -> int: - """Validate an integer setting.""" - return int(value) - - -def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]: - """Create a validator for a tuple of length `length`.""" - - def _validate( - setting, value, option_parser, config_parser=None, config_section=None - ): - string_list = frontend.validate_comma_separated_list( - setting, value, option_parser, config_parser, config_section - ) - if len(string_list) != length: - raise ValueError( - f"Expecting {length} items in {setting}, got {len(string_list)}." - ) - return tuple(string_list) - - return _validate - - -class Unset: - """A sentinel class for unset settings.""" - - def __repr__(self): - return "UNSET" - - -DOCUTILS_UNSET = Unset() -"""Sentinel for arguments not set through docutils.conf.""" - - -DOCUTILS_EXCLUDED_ARGS = ( - # docutils.conf can't represent callables - "heading_slug_func", - # docutils.conf can't represent dicts - "html_meta", - "substitutions", - # we can't add substitutions so not needed - "sub_delimiters", - # sphinx only options - "heading_anchors", - "ref_domains", - "update_mathjax", - "mathjax_classes", -) -"""Names of settings that cannot be set in docutils.conf.""" - - -def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]: - """Convert a field into a Docutils optparse options dict.""" - if at.type is int: - return {"metavar": "", "validator": _validate_int}, f"(default: {default})" - if at.type is bool: - return { - "metavar": "", - "validator": frontend.validate_boolean, - }, f"(default: {default})" - if at.type is str: - return { - "metavar": "", - }, f"(default: '{default}')" - if get_origin(at.type) is Literal and all( - isinstance(a, str) for a in get_args(at.type) - ): - args = get_args(at.type) - return { - "metavar": f"<{'|'.join(repr(a) for a in args)}>", - "type": "choice", - "choices": args, - }, f"(default: {default!r})" - if at.type in (Iterable[str], Sequence[str]): - return { - "metavar": "", - "validator": frontend.validate_comma_separated_list, - }, f"(default: '{','.join(default)}')" - if at.type == Tuple[str, str]: - return { - "metavar": "", - "validator": _create_validate_tuple(2), - }, f"(default: '{','.join(default)}')" - if at.type == Union[int, type(None)]: - return { - "metavar": "", - "validator": _validate_int, - }, f"(default: {default})" - if at.type == Union[Iterable[str], type(None)]: - default_str = ",".join(default) if default else "" - return { - "metavar": "", - "validator": frontend.validate_comma_separated_list, - }, f"(default: {default_str!r})" - raise AssertionError( - f"Configuration option {at.name} not set up for use in docutils.conf." - ) - - -def attr_to_optparse_option( - attribute: Field, default: Any, prefix: str = "myst_" -) -> Tuple[str, List[str], Dict[str, Any]]: - """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple. - - :returns: A tuple of ``(help string, option flags, optparse kwargs)``. - """ - name = f"{prefix}{attribute.name}" - flag = "--" + name.replace("_", "-") - options = {"dest": name, "default": DOCUTILS_UNSET} - at_options, type_str = _attr_to_optparse_option(attribute, default) - options.update(at_options) - help_str = attribute.metadata.get("help", "") if attribute.metadata else "" - return (f"{help_str} {type_str}", [flag], options) - - -def create_myst_settings_spec( - excluded: Sequence[str], config_cls=MdParserConfig, prefix: str = "myst_" -): - """Return a list of Docutils setting for the docutils MyST section.""" - defaults = config_cls() - return tuple( - attr_to_optparse_option(at, getattr(defaults, at.name), prefix) - for at in config_cls.get_fields() - if at.name not in excluded - ) - - -def create_myst_config( - settings: frontend.Values, - excluded: Sequence[str], - config_cls=MdParserConfig, - prefix: str = "myst_", -): - """Create a configuration instance from the given settings.""" - values = {} - for attribute in config_cls.get_fields(): - if attribute.name in excluded: - continue - setting = f"{prefix}{attribute.name}" - val = getattr(settings, setting, DOCUTILS_UNSET) - if val is not DOCUTILS_UNSET: - values[attribute.name] = val - return config_cls(**values) - - -class Parser(RstParser): - """Docutils parser for Markedly Structured Text (MyST).""" - - supported: Tuple[str, ...] = ("md", "markdown", "myst") - """Aliases this parser supports.""" - - settings_spec = ( - "MyST options", - None, - create_myst_settings_spec(DOCUTILS_EXCLUDED_ARGS), - *RstParser.settings_spec, - ) - """Runtime settings specification.""" - - config_section = "myst parser" - config_section_dependencies = ("parsers",) - translate_section_name = None - - def parse(self, inputstring: str, document: nodes.document) -> None: - """Parse source text. - - :param inputstring: The source string to parse - :param document: The root docutils node to add AST elements to - """ - - self.setup_parse(inputstring, document) - - # check for exorbitantly long lines - if hasattr(document.settings, "line_length_limit"): - for i, line in enumerate(inputstring.split("\n")): - if len(line) > document.settings.line_length_limit: - error = document.reporter.error( - f"Line {i+1} exceeds the line-length-limit:" - f" {document.settings.line_length_limit}." - ) - document.append(error) - return - - # create parsing configuration - try: - config = create_myst_config(document.settings, DOCUTILS_EXCLUDED_ARGS) - except Exception as exc: - error = document.reporter.error(f"myst configuration invalid: {exc}") - document.append(error) - config = MdParserConfig() - - # parse content - parser = create_md_parser(config, DocutilsRenderer) - parser.options["document"] = document - env: dict = {} - tokens = parser.parse(inputstring, env) - if not tokens or tokens[0].type != "front_matter": - # we always add front matter, so that we can merge it with global keys, - # specified in the sphinx configuration - tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens - parser.renderer.render(tokens, parser.options, env) - - # post-processing - - # replace raw nodes if raw is not allowed - if not getattr(document.settings, "raw_enabled", True): - for node in document.traverse(nodes.raw): - warning = document.reporter.warning("Raw content disabled.") - node.parent.replace(node, warning) - - self.finish_parse() - - -def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): - """Run the command line interface for a particular writer.""" - publish_cmdline( - parser=Parser(), - writer_name=writer_name, - description=( - f"Generates {writer_description} from standalone MyST sources.\n{default_description}" - ), - argv=argv, - ) - - -def cli_html(argv: Optional[List[str]] = None) -> None: - """Cmdline entrypoint for converting MyST to HTML.""" - _run_cli("html", "(X)HTML documents", argv) - - -def cli_html5(argv: Optional[List[str]] = None): - """Cmdline entrypoint for converting MyST to HTML5.""" - _run_cli("html5", "HTML5 documents", argv) - - -def cli_latex(argv: Optional[List[str]] = None): - """Cmdline entrypoint for converting MyST to LaTeX.""" - _run_cli("latex", "LaTeX documents", argv) - - -def cli_xml(argv: Optional[List[str]] = None): - """Cmdline entrypoint for converting MyST to XML.""" - _run_cli("xml", "Docutils-native XML", argv) - - -def cli_pseudoxml(argv: Optional[List[str]] = None): - """Cmdline entrypoint for converting MyST to pseudo-XML.""" - _run_cli("pseudoxml", "pseudo-XML", argv) +from myst_parser.parsers.docutils_ import Parser # noqa: F401 diff --git a/myst_parser/main.py b/myst_parser/main.py deleted file mode 100644 index 982fe851..00000000 --- a/myst_parser/main.py +++ /dev/null @@ -1,442 +0,0 @@ -"""This module holds the global configuration for the parser ``MdParserConfig``, -and the ``create_md_parser`` function, which creates a parser from the config. -""" -import dataclasses as dc -from typing import Any, Callable, Dict, Iterable, Optional, Sequence, Tuple, Union, cast - -from markdown_it import MarkdownIt -from markdown_it.renderer import RendererHTML, RendererProtocol -from mdit_py_plugins.amsmath import amsmath_plugin -from mdit_py_plugins.anchors import anchors_plugin -from mdit_py_plugins.colon_fence import colon_fence_plugin -from mdit_py_plugins.deflist import deflist_plugin -from mdit_py_plugins.dollarmath import dollarmath_plugin -from mdit_py_plugins.field_list import fieldlist_plugin -from mdit_py_plugins.footnote import footnote_plugin -from mdit_py_plugins.front_matter import front_matter_plugin -from mdit_py_plugins.myst_blocks import myst_block_plugin -from mdit_py_plugins.myst_role import myst_role_plugin -from mdit_py_plugins.substitution import substitution_plugin -from mdit_py_plugins.tasklists import tasklists_plugin -from mdit_py_plugins.wordcount import wordcount_plugin - -from . import __version__ # noqa: F401 -from .dc_validators import ( - deep_iterable, - deep_mapping, - in_, - instance_of, - is_callable, - optional, - validate_fields, -) - - -def check_extensions(_, __, value): - if not isinstance(value, Iterable): - raise TypeError(f"myst_enable_extensions not iterable: {value}") - diff = set(value).difference( - [ - "dollarmath", - "amsmath", - "deflist", - "fieldlist", - "html_admonition", - "html_image", - "colon_fence", - "smartquotes", - "replacements", - "linkify", - "strikethrough", - "substitution", - "tasklist", - ] - ) - if diff: - raise ValueError(f"myst_enable_extensions not recognised: {diff}") - - -def check_sub_delimiters(_, __, value): - if (not isinstance(value, (tuple, list))) or len(value) != 2: - raise TypeError(f"myst_sub_delimiters is not a tuple of length 2: {value}") - for delim in value: - if (not isinstance(delim, str)) or len(delim) != 1: - raise TypeError( - f"myst_sub_delimiters does not contain strings of length 1: {value}" - ) - - -@dc.dataclass() -class MdParserConfig: - """Configuration options for the Markdown Parser. - - Note in the sphinx configuration these option names are prepended with ``myst_`` - """ - - commonmark_only: bool = dc.field( - default=False, - metadata={ - "validator": instance_of(bool), - "help": "Use strict CommonMark parser", - }, - ) - gfm_only: bool = dc.field( - default=False, - metadata={ - "validator": instance_of(bool), - "help": "Use strict Github Flavoured Markdown parser", - }, - ) - - enable_extensions: Sequence[str] = dc.field( - default_factory=list, - metadata={"validator": check_extensions, "help": "Enable extensions"}, - ) - - linkify_fuzzy_links: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "linkify: recognise URLs without schema prefixes", - }, - ) - - dmath_allow_labels: bool = dc.field( - default=True, - metadata={"validator": instance_of(bool), "help": "Parse `$$...$$ (label)`"}, - ) - dmath_allow_space: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "dollarmath: allow initial/final spaces in `$ ... $`", - }, - ) - dmath_allow_digits: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "dollarmath: allow initial/final digits `1$ ...$2`", - }, - ) - dmath_double_inline: bool = dc.field( - default=False, - metadata={ - "validator": instance_of(bool), - "help": "dollarmath: parse inline `$$ ... $$`", - }, - ) - - update_mathjax: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "Update sphinx.ext.mathjax configuration", - }, - ) - - mathjax_classes: str = dc.field( - default="tex2jax_process|mathjax_process|math|output_area", - metadata={ - "validator": instance_of(str), - "help": "MathJax classes to add to math HTML", - }, - ) - - disable_syntax: Iterable[str] = dc.field( - default_factory=list, - metadata={ - "validator": deep_iterable(instance_of(str), instance_of((list, tuple))), - "help": "Disable syntax elements", - }, - ) - - all_links_external: bool = dc.field( - default=False, - metadata={ - "validator": instance_of(bool), - "help": "Parse all links as simple hyperlinks", - }, - ) - - # see https://en.wikipedia.org/wiki/List_of_URI_schemes - url_schemes: Optional[Iterable[str]] = dc.field( - default=cast(Optional[Iterable[str]], ("http", "https", "mailto", "ftp")), - metadata={ - "validator": optional( - deep_iterable(instance_of(str), instance_of((list, tuple))) - ), - "help": "URL scheme prefixes identified as external links", - }, - ) - - ref_domains: Optional[Iterable[str]] = dc.field( - default=None, - metadata={ - "validator": optional( - deep_iterable(instance_of(str), instance_of((list, tuple))) - ), - "help": "Sphinx domain names to search in for references", - }, - ) - - highlight_code_blocks: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "Syntax highlight code blocks with pygments", - "docutils_only": True, - }, - ) - - number_code_blocks: Sequence[str] = dc.field( - default_factory=list, - metadata={ - "validator": deep_iterable(instance_of(str), instance_of((list, tuple))), - "help": "Add line numbers to code blocks with these languages", - }, - ) - - title_to_header: bool = dc.field( - default=False, - metadata={ - "validator": instance_of(bool), - "help": "Convert a `title` field in the top-matter to a H1 header", - }, - ) - - heading_anchors: Optional[int] = dc.field( - default=None, - metadata={ - "validator": optional(in_([1, 2, 3, 4, 5, 6, 7])), - "help": "Heading level depth to assign HTML anchors", - }, - ) - - heading_slug_func: Optional[Callable[[str], str]] = dc.field( - default=None, - metadata={ - "validator": optional(is_callable), - "help": "Function for creating heading anchors", - }, - ) - - html_meta: Dict[str, str] = dc.field( - default_factory=dict, - repr=False, - metadata={ - "validator": deep_mapping( - instance_of(str), instance_of(str), instance_of(dict) - ), - "help": "HTML meta tags", - }, - ) - - footnote_transition: bool = dc.field( - default=True, - metadata={ - "validator": instance_of(bool), - "help": "Place a transition before any footnotes", - }, - ) - - substitutions: Dict[str, Union[str, int, float]] = dc.field( - default_factory=dict, - repr=False, - metadata={ - "validator": deep_mapping( - instance_of(str), instance_of((str, int, float)), instance_of(dict) - ), - "help": "Substitutions", - }, - ) - - sub_delimiters: Tuple[str, str] = dc.field( - default=("{", "}"), - metadata={"validator": check_sub_delimiters, "help": "Substitution delimiters"}, - ) - - words_per_minute: int = dc.field( - default=200, - metadata={ - "validator": instance_of(int), - "help": "For reading speed calculations", - }, - ) - - def __post_init__(self): - validate_fields(self) - - @classmethod - def get_fields(cls) -> Tuple[dc.Field, ...]: - """Return all attribute fields in this class.""" - return dc.fields(cls) - - def as_dict(self, dict_factory=dict) -> dict: - """Return a dictionary of field name -> value.""" - return dc.asdict(self, dict_factory=dict_factory) - - def as_triple(self) -> Iterable[Tuple[str, Any, dc.Field]]: - """Yield triples of (name, value, field).""" - fields = {f.name: f for f in dc.fields(self.__class__)} - for name, value in dc.asdict(self).items(): - yield name, value, fields[name] - - -def default_parser(config: MdParserConfig): - raise NotImplementedError( - "default_parser has been deprecated and replaced by create_md_parser." - "You must also supply the renderer class directly to create_md_parser." - ) - - -def create_md_parser( - config: MdParserConfig, renderer: Callable[[MarkdownIt], RendererProtocol] -) -> MarkdownIt: - """Return a Markdown parser with the required MyST configuration.""" - - # TODO warn if linkify required and linkify-it-py not installed - # (currently the parse will unceremoniously except) - - if config.commonmark_only: - # see https://spec.commonmark.org/ - md = MarkdownIt("commonmark", renderer_cls=renderer).use( - wordcount_plugin, per_minute=config.words_per_minute - ) - md.options.update({"myst_config": config}) - return md - - if config.gfm_only: - # see https://github.github.com/gfm/ - md = ( - MarkdownIt("commonmark", renderer_cls=renderer) - # note, strikethrough currently only supported tentatively for HTML - .enable("strikethrough") - .enable("table") - .use(tasklists_plugin) - .enable("linkify") - .use(wordcount_plugin, per_minute=config.words_per_minute) - ) - md.options.update({"linkify": True, "myst_config": config}) - return md - - md = ( - MarkdownIt("commonmark", renderer_cls=renderer) - .enable("table") - .use(front_matter_plugin) - .use(myst_block_plugin) - .use(myst_role_plugin) - .use(footnote_plugin) - .use(wordcount_plugin, per_minute=config.words_per_minute) - .disable("footnote_inline") - # disable this for now, because it need a new implementation in the renderer - .disable("footnote_tail") - ) - - typographer = False - if "smartquotes" in config.enable_extensions: - md.enable("smartquotes") - typographer = True - if "replacements" in config.enable_extensions: - md.enable("replacements") - typographer = True - if "linkify" in config.enable_extensions: - md.enable("linkify") - if md.linkify is not None: - md.linkify.set({"fuzzy_link": config.linkify_fuzzy_links}) - if "strikethrough" in config.enable_extensions: - md.enable("strikethrough") - if "dollarmath" in config.enable_extensions: - md.use( - dollarmath_plugin, - allow_labels=config.dmath_allow_labels, - allow_space=config.dmath_allow_space, - allow_digits=config.dmath_allow_digits, - double_inline=config.dmath_double_inline, - ) - if "colon_fence" in config.enable_extensions: - md.use(colon_fence_plugin) - if "amsmath" in config.enable_extensions: - md.use(amsmath_plugin) - if "deflist" in config.enable_extensions: - md.use(deflist_plugin) - if "fieldlist" in config.enable_extensions: - md.use(fieldlist_plugin) - if "tasklist" in config.enable_extensions: - md.use(tasklists_plugin) - if "substitution" in config.enable_extensions: - md.use(substitution_plugin, *config.sub_delimiters) - if config.heading_anchors is not None: - md.use( - anchors_plugin, - max_level=config.heading_anchors, - slug_func=config.heading_slug_func, - ) - for name in config.disable_syntax: - md.disable(name, True) - - md.options.update( - { - "typographer": typographer, - "linkify": "linkify" in config.enable_extensions, - "myst_config": config, - } - ) - - return md - - -def to_docutils( - text: str, - parser_config: Optional[MdParserConfig] = None, - *, - options=None, - env=None, - document=None, - in_sphinx_env: bool = False, - conf=None, - srcdir=None, -): - """Render text to the docutils AST (before transforms) - - :param text: the text to render - :param options: options to update the parser with - :param env: The sandbox environment for the parse - (will contain e.g. reference definitions) - :param document: the docutils root node to use (otherwise a new one will be created) - :param in_sphinx_env: initialise a minimal sphinx environment (useful for testing) - :param conf: the sphinx conf.py as a dictionary - :param srcdir: to parse to the mock sphinx env - - :returns: docutils document - """ - from myst_parser.docutils_renderer import make_document - from myst_parser.sphinx_renderer import SphinxRenderer - - md = create_md_parser(parser_config or MdParserConfig(), SphinxRenderer) - if options: - md.options.update(options) - md.options["document"] = document or make_document() - if in_sphinx_env: - from myst_parser.sphinx_renderer import mock_sphinx_env - - with mock_sphinx_env(conf=conf, srcdir=srcdir, document=md.options["document"]): - return md.render(text, env) - else: - return md.render(text, env) - - -def to_html(text: str, env=None, config: Optional[MdParserConfig] = None): - """Render text to HTML directly using markdown-it-py. - - This is mainly for test purposes only. - """ - config = config or MdParserConfig() - md = create_md_parser(config, RendererHTML) - return md.render(text, env) - - -def to_tokens(text: str, env=None, config: Optional[MdParserConfig] = None): - config = config or MdParserConfig() - md = create_md_parser(config, RendererHTML) - return md.parse(text, env) diff --git a/myst_parser/mdit_to_docutils/__init__.py b/myst_parser/mdit_to_docutils/__init__.py new file mode 100644 index 00000000..0b9307f0 --- /dev/null +++ b/myst_parser/mdit_to_docutils/__init__.py @@ -0,0 +1 @@ +"""Conversion of Markdown-it tokens to docutils AST.""" diff --git a/myst_parser/docutils_renderer.py b/myst_parser/mdit_to_docutils/base.py similarity index 91% rename from myst_parser/docutils_renderer.py rename to myst_parser/mdit_to_docutils/base.py index 4494b549..099b5ad1 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1,26 +1,15 @@ +"""Convert Markdown-it tokens to docutils nodes.""" +from __future__ import annotations + import inspect import json import os import re from collections import OrderedDict from contextlib import contextmanager -from copy import deepcopy from datetime import date, datetime from types import ModuleType -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterator, - List, - MutableMapping, - Optional, - Sequence, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Iterator, MutableMapping, Sequence, cast from urllib.parse import urlparse import jinja2 @@ -43,7 +32,7 @@ from markdown_it.token import Token from markdown_it.tree import SyntaxTreeNode -from myst_parser.main import MdParserConfig +from myst_parser.config.main import MdParserConfig from myst_parser.mocking import ( MockIncludeDirective, MockingError, @@ -52,8 +41,8 @@ MockState, MockStateMachine, ) +from ..parsers.directives import DirectiveParsingError, parse_directive_text from .html_to_nodes import html_to_nodes -from .parse_directives import DirectiveParsingError, parse_directive_text from .utils import is_external_url if TYPE_CHECKING: @@ -69,7 +58,7 @@ def make_document(source_path="notset", parser_cls=RSTParser) -> nodes.document: REGEX_DIRECTIVE_START = re.compile(r"^[\s]{0,3}([`]{3,10}|[~]{3,10}|[:]{3,10})\{") -def token_line(token: SyntaxTreeNode, default: Optional[int] = None) -> int: +def token_line(token: SyntaxTreeNode, default: int | None = None) -> int: """Retrieve the initial line of a token.""" if not getattr(token, "map", None): if default is not None: @@ -78,6 +67,27 @@ def token_line(token: SyntaxTreeNode, default: Optional[int] = None) -> int: return token.map[0] # type: ignore[index] +def create_warning( + document: nodes.document, + message: str, + *, + line: int | None = None, + append_to: nodes.Element | None = None, + wtype: str = "myst", + subtype: str = "other", +) -> nodes.system_message | None: + """Generate a warning, logging if it is necessary. + + Note this is overridden in the ``SphinxRenderer``, + to handle suppressed warning types. + """ + kwargs = {"line": line} if line is not None else {} + msg_node = document.reporter.warning(f"{message} [{wtype}.{subtype}]", **kwargs) + if append_to is not None: + append_to.append(msg_node) + return msg_node + + class DocutilsRenderer(RendererProtocol): """A markdown-it-py renderer to populate (in-place) a `docutils.document` AST. @@ -115,7 +125,7 @@ def __getattr__(self, name: str): ) def setup_render( - self, options: Dict[str, Any], env: MutableMapping[str, Any] + self, options: dict[str, Any], env: MutableMapping[str, Any] ) -> None: """Setup the renderer with per render variables.""" self.md_env = env @@ -130,12 +140,12 @@ def setup_render( self.document.settings.language_code ) # a mapping of heading levels to its currently associated node - self._level_to_elem: Dict[int, Union[nodes.document, nodes.section]] = { + self._level_to_elem: dict[int, nodes.document | nodes.section] = { 0: self.document } @property - def sphinx_env(self) -> Optional["BuildEnvironment"]: + def sphinx_env(self) -> BuildEnvironment | None: """Return the sphinx env, if using Sphinx.""" try: return self.document.settings.env @@ -146,23 +156,26 @@ def create_warning( self, message: str, *, - line: Optional[int] = None, - append_to: Optional[nodes.Element] = None, + line: int | None = None, + append_to: nodes.Element | None = None, wtype: str = "myst", subtype: str = "other", - ) -> Optional[nodes.system_message]: + ) -> nodes.system_message | None: """Generate a warning, logging if it is necessary. Note this is overridden in the ``SphinxRenderer``, to handle suppressed warning types. """ - kwargs = {"line": line} if line is not None else {} - msg_node = self.reporter.warning(f"{message} [{wtype}.{subtype}]", **kwargs) - if append_to is not None: - append_to.append(msg_node) - return msg_node + return create_warning( + self.document, + message, + line=line, + append_to=append_to, + wtype=wtype, + subtype=subtype, + ) - def _render_tokens(self, tokens: List[Token]) -> None: + def _render_tokens(self, tokens: list[Token]) -> None: """Render the tokens.""" # propagate line number down to inline elements for token in tokens: @@ -213,8 +226,24 @@ def render( containing additional metadata like reference info """ self.setup_render(options, md_env) - + self._render_initialise() self._render_tokens(list(tokens)) + self._render_finalise() + return self.document + + def _render_initialise(self) -> None: + """Initialise the render of the document.""" + self.current_node.extend( + html_meta_to_nodes( + self.md_config.html_meta, + document=self.document, + line=0, + reporter=self.reporter, + ) + ) + + def _render_finalise(self) -> None: + """Finalise the render of the document.""" # log warnings for duplicate reference definitions # "duplicate_refs": [{"href": "ijk", "label": "B", "map": [4, 5], "title": ""}], @@ -255,34 +284,29 @@ def render( else: self.render_footnote_reference(foot_ref_tokens[0]) - self.add_document_wordcount() - - return self.document - - def add_document_wordcount(self) -> None: - """Add the wordcount, generated by the ``mdit_py_plugins.wordcount_plugin``.""" - + # Add the wordcount, generated by the ``mdit_py_plugins.wordcount_plugin``. wordcount_metadata = self.md_env.get("wordcount", {}) - if not wordcount_metadata: - return - - # save the wordcount to the sphinx BuildEnvironment metadata - if self.sphinx_env is not None: - meta = self.sphinx_env.metadata.setdefault(self.sphinx_env.docname, {}) - meta["wordcount"] = wordcount_metadata - - # now add the wordcount as substitution definitions, - # so we can reference them in the document - for key in ("words", "minutes"): - value = wordcount_metadata.get(key, None) - if value is None: - continue - substitution_node = nodes.substitution_definition( - str(value), nodes.Text(str(value)) - ) - substitution_node.source = self.document["source"] - substitution_node["names"].append(f"wordcount-{key}") - self.document.note_substitution_def(substitution_node, f"wordcount-{key}") + if wordcount_metadata: + + # save the wordcount to the sphinx BuildEnvironment metadata + if self.sphinx_env is not None: + meta = self.sphinx_env.metadata.setdefault(self.sphinx_env.docname, {}) + meta["wordcount"] = wordcount_metadata + + # now add the wordcount as substitution definitions, + # so we can reference them in the document + for key in ("words", "minutes"): + value = wordcount_metadata.get(key, None) + if value is None: + continue + substitution_node = nodes.substitution_definition( + str(value), nodes.Text(str(value)) + ) + substitution_node.source = self.document["source"] + substitution_node["names"].append(f"wordcount-{key}") + self.document.note_substitution_def( + substitution_node, f"wordcount-{key}" + ) def nested_render_text( self, text: str, lineno: int, inline: bool = False, allow_headings: bool = True @@ -349,7 +373,7 @@ def add_line_and_source_path(self, node, token: SyntaxTreeNode) -> None: node.source = self.document["source"] def add_line_and_source_path_r( - self, nodes: List[nodes.Element], token: SyntaxTreeNode + self, nodes: list[nodes.Element], token: SyntaxTreeNode ) -> None: """Copy the line number and document source path to the docutils nodes, and recursively to all descendants. @@ -394,7 +418,7 @@ def update_section_level_state(self, section: nodes.section, level: int) -> None if section_level <= level } - def renderInlineAsText(self, tokens: List[SyntaxTreeNode]) -> str: + def renderInlineAsText(self, tokens: list[SyntaxTreeNode]) -> str: """Special kludge for image `alt` attributes to conform CommonMark spec. Don't try to use it! Spec requires to show `alt` content with stripped markup, @@ -492,12 +516,12 @@ def render_code_inline(self, token: SyntaxTreeNode) -> None: def create_highlighted_code_block( self, text: str, - lexer_name: Optional[str], + lexer_name: str | None, number_lines: bool = False, lineno_start: int = 1, - source: Optional[str] = None, - line: Optional[int] = None, - node_cls: Type[nodes.Element] = nodes.literal_block, + source: str | None = None, + line: int | None = None, + node_cls: type[nodes.Element] = nodes.literal_block, ) -> nodes.Element: """Create a literal block with syntax highlighting. @@ -768,66 +792,45 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None: """Pass document front matter data.""" position = token_line(token, default=0) - if not isinstance(token.content, dict): + if isinstance(token.content, str): try: data = yaml.safe_load(token.content) - assert isinstance(data, dict), "not dict" - except ( - AssertionError, - yaml.parser.ParserError, - yaml.scanner.ScannerError, - ) as error: - msg_node = self.reporter.error( - "Front matter block:\n" + str(error), line=position + except (yaml.parser.ParserError, yaml.scanner.ScannerError): + self.create_warning( + "Malformed YAML", + line=position, + append_to=self.current_node, + subtype="topmatter", ) - msg_node += nodes.literal_block(token.content, token.content) - self.current_node.append(msg_node) return else: - data = deepcopy(token.content) + data = token.content - substitutions = data.pop("substitutions", {}) - html_meta = data.pop("html_meta", {}) + if not isinstance(data, dict): + self.create_warning( + f"YAML is not a dict: {type(data)}", + line=position, + append_to=self.current_node, + subtype="topmatter", + ) + return - if data: + fields = { + k: v + for k, v in data.items() + if k not in ("myst", "mystnb", "substitutions", "html_meta") + } + if fields: field_list = self.dict_to_fm_field_list( - data, language_code=self.document.settings.language_code + fields, language_code=self.document.settings.language_code ) self.current_node.append(field_list) - if isinstance(substitutions, dict): - self.document.fm_substitutions = substitutions - else: - msg_node = self.reporter.error( - "Front-matter 'substitutions' is not a dict", line=position - ) - msg_node += nodes.literal_block(token.content, token.content) - self.current_node.append(msg_node) - - if not isinstance(html_meta, dict): - msg_node = self.reporter.error( - "Front-matter 'html_meta' is not a dict", line=position - ) - msg_node += nodes.literal_block(token.content, token.content) - self.current_node.append(msg_node) - - self.current_node.extend( - html_meta_to_nodes( - { - **self.md_config.html_meta, - **html_meta, - }, - document=self.document, - line=position, - reporter=self.reporter, - ) - ) - if data.get("title") and self.md_config.title_to_header: self.nested_render_text(f"# {data['title']}", 0) def dict_to_fm_field_list( - self, data: Dict[str, Any], language_code: str, line: int = 0 + self, data: dict[str, Any], language_code: str, line: int = 0 ) -> nodes.field_list: """Render each key/val pair as a docutils ``field_node``. @@ -1004,7 +1007,7 @@ def render_footnote_ref(self, token: SyntaxTreeNode) -> None: """ target = token.meta["label"] - refnode = nodes.footnote_reference("[^{}]".format(target)) + refnode = nodes.footnote_reference(f"[^{target}]") self.add_line_and_source_path(refnode, token) if not target.isdigit(): refnode["auto"] = 1 @@ -1066,7 +1069,7 @@ def render_myst_role(self, token: SyntaxTreeNode) -> None: self.current_node += nodes else: message = self.reporter.error( - 'Unknown interpreted text role "{}".'.format(name), line=lineno + f'Unknown interpreted text role "{name}".', line=lineno ) problematic = inliner.problematic(text, rawsource, message) self.current_node += problematic @@ -1192,7 +1195,7 @@ def render_directive(self, token: SyntaxTreeNode) -> None: def run_directive( self, name: str, first_line: str, content: str, position: int - ) -> List[nodes.Element]: + ) -> list[nodes.Element]: """Run a directive and return the generated nodes. :param name: the name of the directive @@ -1207,13 +1210,13 @@ def run_directive( self.document.current_line = position # get directive class - output: Tuple[Directive, list] = directives.directive( + output: tuple[Directive, list] = directives.directive( name, self.language_module_rst, self.document ) directive_class, messages = output if not directive_class: error = self.reporter.error( - 'Unknown directive type "{}".\n'.format(name), + f'Unknown directive type "{name}".\n', # nodes.literal_block(content, content), line=position, ) @@ -1231,7 +1234,7 @@ def run_directive( ) except DirectiveParsingError as error: error = self.reporter.error( - "Directive '{}': {}".format(name, error), + f"Directive '{name}': {error}", nodes.literal_block(content, content), line=position, ) @@ -1290,7 +1293,7 @@ def run_directive( assert isinstance( result, list - ), 'Directive "{}" must return a list of nodes.'.format(name) + ), f'Directive "{name}" must return a list of nodes.' for i in range(len(result)): assert isinstance( result[i], nodes.Node @@ -1322,10 +1325,7 @@ def render_substitution(self, token: SyntaxTreeNode, inline: bool) -> None: position = token_line(token) # front-matter substitutions take priority over config ones - variable_context: Dict[str, Any] = { - **self.md_config.substitutions, - **getattr(self.document, "fm_substitutions", {}), - } + variable_context: dict[str, Any] = {**self.md_config.substitutions} if self.sphinx_env is not None: variable_context["env"] = self.sphinx_env @@ -1378,8 +1378,8 @@ def render_substitution(self, token: SyntaxTreeNode, inline: bool) -> None: def html_meta_to_nodes( - data: Dict[str, Any], document: nodes.document, line: int, reporter: Reporter -) -> List[Union[nodes.pending, nodes.system_message]]: + data: dict[str, Any], document: nodes.document, line: int, reporter: Reporter +) -> list[nodes.pending | nodes.system_message]: """Replicate the `meta` directive, by converting a dictionary to a list of pending meta nodes diff --git a/myst_parser/html_to_nodes.py b/myst_parser/mdit_to_docutils/html_to_nodes.py similarity index 92% rename from myst_parser/html_to_nodes.py rename to myst_parser/mdit_to_docutils/html_to_nodes.py index 5d05ea0d..2cc30667 100644 --- a/myst_parser/html_to_nodes.py +++ b/myst_parser/mdit_to_docutils/html_to_nodes.py @@ -1,13 +1,15 @@ """Convert HTML to docutils nodes.""" +from __future__ import annotations + import re -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from docutils import nodes -from .parse_html import Data, tokenize_html +from myst_parser.parsers.parse_html import Data, tokenize_html if TYPE_CHECKING: - from .docutils_renderer import DocutilsRenderer + from .base import DocutilsRenderer def make_error( @@ -32,7 +34,7 @@ def make_error( ) -def default_html(text: str, source: str, line_number: int) -> List[nodes.Element]: +def default_html(text: str, source: str, line_number: int) -> list[nodes.Element]: raw_html = nodes.raw("", text, format="html") raw_html.source = source raw_html.line = line_number @@ -40,8 +42,8 @@ def default_html(text: str, source: str, line_number: int) -> List[nodes.Element def html_to_nodes( - text: str, line_number: int, renderer: "DocutilsRenderer" -) -> List[nodes.Element]: + text: str, line_number: int, renderer: DocutilsRenderer +) -> list[nodes.Element]: """Convert HTML to docutils nodes.""" if renderer.md_config.gfm_only: text, _ = RE_FLOW.subn(lambda s: s.group(0).replace("<", "<"), text) diff --git a/myst_parser/sphinx_renderer.py b/myst_parser/mdit_to_docutils/sphinx_.py similarity index 58% rename from myst_parser/sphinx_renderer.py rename to myst_parser/mdit_to_docutils/sphinx_.py index 379fbb94..3c1bc237 100644 --- a/myst_parser/sphinx_renderer.py +++ b/myst_parser/mdit_to_docutils/sphinx_.py @@ -1,35 +1,55 @@ -import copy +"""Convert Markdown-it tokens to docutils nodes, including sphinx specific elements.""" +from __future__ import annotations + import os -import tempfile -from contextlib import contextmanager -from io import StringIO from pathlib import Path -from typing import Optional, cast +from typing import cast from urllib.parse import unquote from uuid import uuid4 from docutils import nodes -from docutils.parsers.rst import directives, roles from markdown_it.tree import SyntaxTreeNode from sphinx import addnodes -from sphinx.application import Sphinx, builtin_extensions -from sphinx.config import Config from sphinx.domains.math import MathDomain from sphinx.domains.std import StandardDomain from sphinx.environment import BuildEnvironment -from sphinx.events import EventManager -from sphinx.project import Project -from sphinx.registry import SphinxComponentRegistry from sphinx.util import logging -from sphinx.util.docutils import additional_nodes, sphinx_domains, unregister_node from sphinx.util.nodes import clean_astext -from sphinx.util.tags import Tags -from myst_parser.docutils_renderer import DocutilsRenderer +from myst_parser.mdit_to_docutils.base import DocutilsRenderer LOGGER = logging.getLogger(__name__) +def create_warning( + document: nodes.document, + message: str, + *, + line: int | None = None, + append_to: nodes.Element | None = None, + wtype: str = "myst", + subtype: str = "other", +) -> nodes.system_message | None: + """Generate a warning, logging it if necessary. + + If the warning type is listed in the ``suppress_warnings`` configuration, + then ``None`` will be returned and no warning logged. + """ + message = f"{message} [{wtype}.{subtype}]" + kwargs = {"line": line} if line is not None else {} + + if logging.is_suppressed_warning( + wtype, subtype, document.settings.env.app.config.suppress_warnings + ): + return None + + msg_node = document.reporter.warning(message, **kwargs) + if append_to is not None: + append_to.append(msg_node) + + return None + + class SphinxRenderer(DocutilsRenderer): """A markdown-it-py renderer to populate (in-place) a `docutils.document` AST. @@ -45,26 +65,24 @@ def create_warning( self, message: str, *, - line: Optional[int] = None, - append_to: Optional[nodes.Element] = None, + line: int | None = None, + append_to: nodes.Element | None = None, wtype: str = "myst", subtype: str = "other", - ) -> Optional[nodes.system_message]: + ) -> nodes.system_message | None: """Generate a warning, logging it if necessary. If the warning type is listed in the ``suppress_warnings`` configuration, then ``None`` will be returned and no warning logged. """ - message = f"{message} [{wtype}.{subtype}]" - kwargs = {"line": line} if line is not None else {} - - if not logging.is_suppressed_warning( - wtype, subtype, self.doc_env.app.config.suppress_warnings - ): - msg_node = self.reporter.warning(message, **kwargs) - if append_to is not None: - append_to.append(msg_node) - return None + return create_warning( + self.document, + message, + line=line, + append_to=append_to, + wtype=wtype, + subtype=subtype, + ) def render_internal_link(self, token: SyntaxTreeNode) -> None: """Render link token `[text](link "title")`, @@ -225,114 +243,3 @@ def add_math_target(self, node: nodes.math_block) -> nodes.target: target = nodes.target("", "", ids=[node_id]) self.document.note_explicit_target(target) return target - - -def minimal_sphinx_app( - configuration=None, sourcedir=None, with_builder=False, raise_on_warning=False -): - """Create a minimal Sphinx environment; loading sphinx roles, directives, etc.""" - - class MockSphinx(Sphinx): - """Minimal sphinx init to load roles and directives.""" - - def __init__(self, confoverrides=None, srcdir=None, raise_on_warning=False): - self.extensions = {} - self.registry = SphinxComponentRegistry() - try: - self.html_themes = {} - except AttributeError: - # changed to property in sphinx 4.1 - pass - self.events = EventManager(self) - - # logging - self.verbosity = 0 - self._warncount = 0 - self.warningiserror = raise_on_warning - self._status = StringIO() - self._warning = StringIO() - logging.setup(self, self._status, self._warning) - - self.tags = Tags([]) - self.config = Config({}, confoverrides or {}) - self.config.pre_init_values() - self._init_i18n() - for extension in builtin_extensions: - self.registry.load_extension(self, extension) - # fresh env - self.doctreedir = "" - self.srcdir = srcdir - self.confdir = None - self.outdir = "" - self.project = Project(srcdir=srcdir, source_suffix={".md": "markdown"}) - self.project.docnames = {"mock_docname"} - self.env = BuildEnvironment(self) - self.env.setup(self) - self.env.temp_data["docname"] = "mock_docname" - # Ignore type checkers because we disrespect superclass typing here - self.builder = None # type: ignore[assignment] - - if not with_builder: - return - - # this code is only required for more complex parsing with extensions - for extension in self.config.extensions: - self.setup_extension(extension) - buildername = "dummy" - self.preload_builder(buildername) - self.config.init_values() - self.events.emit("config-inited", self.config) - - with tempfile.TemporaryDirectory() as tempdir: - # creating a builder attempts to make the doctreedir - self.doctreedir = tempdir - self.builder = self.create_builder(buildername) - self.doctreedir = "" - - app = MockSphinx( - confoverrides=configuration, srcdir=sourcedir, raise_on_warning=raise_on_warning - ) - return app - - -@contextmanager -def mock_sphinx_env( - conf=None, srcdir=None, document=None, with_builder=False, raise_on_warning=False -): - """Set up an environment, to parse sphinx roles/directives, - outside of a `sphinx-build`. - - :param conf: a dictionary representation of the sphinx `conf.py` - :param srcdir: a path to a source directory - (for example, can be used for `include` statements) - - This primarily copies the code in `sphinx.util.docutils.docutils_namespace` - and `sphinx.util.docutils.sphinx_domains`. - """ - # store currently loaded roles/directives, so we can revert on exit - _directives = copy.copy(directives._directives) - _roles = copy.copy(roles._roles) - # Monkey-patch directive and role dispatch, - # so that sphinx domain-specific markup takes precedence. - app = minimal_sphinx_app( - configuration=conf, - sourcedir=srcdir, - with_builder=with_builder, - raise_on_warning=raise_on_warning, - ) - _sphinx_domains = sphinx_domains(app.env) - _sphinx_domains.enable() - if document is not None: - document.settings.env = app.env - try: - yield app - finally: - # revert loaded roles/directives - directives._directives = _directives - roles._roles = _roles - # TODO unregister nodes (see `sphinx.util.docutils.docutils_namespace`) - for node in list(additional_nodes): - unregister_node(node) - additional_nodes.discard(node) - # revert directive/role function (see `sphinx.util.docutils.sphinx_domains`) - _sphinx_domains.disable() diff --git a/myst_parser/utils.py b/myst_parser/mdit_to_docutils/utils.py similarity index 100% rename from myst_parser/utils.py rename to myst_parser/mdit_to_docutils/utils.py diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index 15786a50..b22475d8 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -1,11 +1,13 @@ """This module provides classes to Mock the core components of the docutils.RSTParser, the key difference being that nested parsing treats the text as Markdown not rST. """ +from __future__ import annotations + import os import re import sys from pathlib import Path -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type +from typing import TYPE_CHECKING, Any from docutils import nodes from docutils.parsers.rst import Directive, DirectiveError @@ -15,10 +17,10 @@ from docutils.statemachine import StringList from docutils.utils import unescape -from .parse_directives import parse_directive_text +from .parsers.directives import parse_directive_text if TYPE_CHECKING: - from .docutils_renderer import DocutilsRenderer + from .mdit_to_docutils.base import DocutilsRenderer class MockingError(Exception): @@ -31,7 +33,7 @@ class MockInliner: This is parsed to role functions. """ - def __init__(self, renderer: "DocutilsRenderer"): + def __init__(self, renderer: DocutilsRenderer): """Initialize the mock inliner.""" self._renderer = renderer # here we mock that the `parse` method has already been called @@ -59,7 +61,7 @@ def problematic( def parse( self, text: str, lineno: int, memo: Any, parent: nodes.Node - ) -> Tuple[List[nodes.Node], List[nodes.system_message]]: + ) -> tuple[list[nodes.Node], list[nodes.system_message]]: """Parse the text and return a list of nodes.""" # note the only place this is normally called, # is by `RSTState.inline_text`, or in directives: `self.state.inline_text`, @@ -86,7 +88,7 @@ def __getattr__(self, name: str): cls=type(self).__name__, name=name ) raise MockingError(msg).with_traceback(sys.exc_info()[2]) - msg = "{cls} has no attribute {name}".format(cls=type(self).__name__, name=name) + msg = f"{type(self).__name__} has no attribute {name}" raise MockingError(msg).with_traceback(sys.exc_info()[2]) @@ -100,8 +102,8 @@ class MockState: def __init__( self, - renderer: "DocutilsRenderer", - state_machine: "MockStateMachine", + renderer: DocutilsRenderer, + state_machine: MockStateMachine, lineno: int, ): self._renderer = renderer @@ -115,7 +117,7 @@ class Struct: document = self.document reporter = self.document.reporter language = renderer.language_module_rst - title_styles: List[str] = [] + title_styles: list[str] = [] section_level = max(renderer._level_to_elem) section_bubble_up_kludge = False inliner = self.inliner @@ -126,9 +128,9 @@ def parse_directive_block( self, content: StringList, line_offset: int, - directive: Type[Directive], + directive: type[Directive], option_presets: dict, - ) -> Tuple[list, dict, StringList, int]: + ) -> tuple[list, dict, StringList, int]: """Parse the full directive text :returns: (arguments, options, content, content_offset) @@ -189,7 +191,7 @@ def parse_target(self, block, block_text, lineno: int): def inline_text( self, text: str, lineno: int - ) -> Tuple[List[nodes.Element], List[nodes.Element]]: + ) -> tuple[list[nodes.Element], list[nodes.Element]]: """Parse text with only inline rules. :returns: (list of nodes, list of messages) @@ -199,7 +201,7 @@ def inline_text( # U+2014 is an em-dash: attribution_pattern = re.compile("^((?:---?(?!-)|\u2014) *)(.+)") - def block_quote(self, lines: List[str], line_offset: int) -> List[nodes.Element]: + def block_quote(self, lines: list[str], line_offset: int) -> list[nodes.Element]: """Parse a block quote, which is a block of text, followed by an (optional) attribution. @@ -284,7 +286,7 @@ class MockStateMachine: This is parsed to the `Directives.run()` method. """ - def __init__(self, renderer: "DocutilsRenderer", lineno: int): + def __init__(self, renderer: DocutilsRenderer, lineno: int): self._renderer = renderer self._lineno = lineno self.document = renderer.document @@ -293,11 +295,11 @@ def __init__(self, renderer: "DocutilsRenderer", lineno: int): self.node: nodes.Element = renderer.current_node self.match_titles: bool = True - def get_source(self, lineno: Optional[int] = None): + def get_source(self, lineno: int | None = None): """Return document source path.""" return self.document["source"] - def get_source_and_line(self, lineno: Optional[int] = None): + def get_source_and_line(self, lineno: int | None = None): """Return (source path, line) tuple for current or given line number.""" return self.document["source"], lineno or self._lineno @@ -310,7 +312,7 @@ def __getattr__(self, name: str): cls=type(self).__name__, name=name ) raise MockingError(msg).with_traceback(sys.exc_info()[2]) - msg = "{cls} has no attribute {name}".format(cls=type(self).__name__, name=name) + msg = f"{type(self).__name__} has no attribute {name}" raise MockingError(msg).with_traceback(sys.exc_info()[2]) @@ -324,12 +326,12 @@ class MockIncludeDirective: def __init__( self, - renderer: "DocutilsRenderer", + renderer: DocutilsRenderer, name: str, klass: Include, arguments: list, options: dict, - body: List[str], + body: list[str], lineno: int, ): self.renderer = renderer @@ -341,12 +343,12 @@ def __init__( self.body = body self.lineno = lineno - def run(self) -> List[nodes.Element]: + def run(self) -> list[nodes.Element]: from docutils.parsers.rst.directives.body import CodeBlock, NumberLines if not self.document.settings.file_insertion_enabled: - raise DirectiveError(2, 'Directive "{}" disabled.'.format(self.name)) + raise DirectiveError(2, f'Directive "{self.name}" disabled.') source_dir = Path(self.document["source"]).absolute().parent include_arg = "".join([s.strip() for s in self.arguments[0].splitlines()]) diff --git a/myst_parser/parsers/__init__.py b/myst_parser/parsers/__init__.py new file mode 100644 index 00000000..26fbfcab --- /dev/null +++ b/myst_parser/parsers/__init__.py @@ -0,0 +1 @@ +"""Parsers of MyST Markdown source text to docutils AST.""" diff --git a/myst_parser/parse_directives.py b/myst_parser/parsers/directives.py similarity index 94% rename from myst_parser/parse_directives.py rename to myst_parser/parsers/directives.py index b053deae..e275c843 100644 --- a/myst_parser/parse_directives.py +++ b/myst_parser/parsers/directives.py @@ -33,10 +33,12 @@ This is to allow for separation between the option block and content. """ +from __future__ import annotations + import datetime import re from textwrap import dedent -from typing import Any, Callable, Dict, List, Tuple, Type +from typing import Any, Callable import yaml from docutils.parsers.rst import Directive @@ -50,11 +52,11 @@ class DirectiveParsingError(Exception): def parse_directive_text( - directive_class: Type[Directive], + directive_class: type[Directive], first_line: str, content: str, validate_options: bool = True, -) -> Tuple[List[str], dict, List[str], int]: +) -> tuple[list[str], dict, list[str], int]: """Parse (and validate) the full directive text. :param first_line: The text on the same line as the directive name. @@ -103,10 +105,10 @@ def parse_directive_text( def parse_directive_options( - content: str, directive_class: Type[Directive], validate: bool = True + content: str, directive_class: type[Directive], validate: bool = True ): """Parse (and validate) the directive option section.""" - options: Dict[str, Any] = {} + options: dict[str, Any] = {} if content.startswith("---"): content = "\n".join(content.splitlines()[1:]) match = re.search(r"^-{3,}", content, re.MULTILINE) @@ -143,7 +145,7 @@ def parse_directive_options( return content, options # check options against spec - options_spec = directive_class.option_spec # type: Dict[str, Callable] + options_spec: dict[str, Callable] = directive_class.option_spec for name, value in list(options.items()): try: convertor = options_spec[name] @@ -179,7 +181,7 @@ def parse_directive_arguments(directive, arg_text): arguments = arg_text.split() if len(arguments) < required: raise DirectiveParsingError( - "{} argument(s) required, {} supplied".format(required, len(arguments)) + f"{required} argument(s) required, {len(arguments)} supplied" ) elif len(arguments) > required + optional: if directive.final_argument_whitespace: diff --git a/myst_parser/parsers/docutils_.py b/myst_parser/parsers/docutils_.py new file mode 100644 index 00000000..aaef5e2f --- /dev/null +++ b/myst_parser/parsers/docutils_.py @@ -0,0 +1,275 @@ +"""MyST Markdown parser for docutils.""" +from dataclasses import Field +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union + +from docutils import frontend, nodes +from docutils.core import default_description, publish_cmdline +from docutils.parsers.rst import Parser as RstParser +from typing_extensions import Literal, get_args, get_origin + +from myst_parser.config.main import ( + MdParserConfig, + TopmatterReadError, + merge_file_level, + read_topmatter, +) +from myst_parser.mdit_to_docutils.base import DocutilsRenderer, create_warning +from myst_parser.parsers.mdit import create_md_parser + + +def _validate_int( + setting, value, option_parser, config_parser=None, config_section=None +) -> int: + """Validate an integer setting.""" + return int(value) + + +def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]: + """Create a validator for a tuple of length `length`.""" + + def _validate( + setting, value, option_parser, config_parser=None, config_section=None + ): + string_list = frontend.validate_comma_separated_list( + setting, value, option_parser, config_parser, config_section + ) + if len(string_list) != length: + raise ValueError( + f"Expecting {length} items in {setting}, got {len(string_list)}." + ) + return tuple(string_list) + + return _validate + + +class Unset: + """A sentinel class for unset settings.""" + + def __repr__(self): + return "UNSET" + + +DOCUTILS_UNSET = Unset() +"""Sentinel for arguments not set through docutils.conf.""" + + +DOCUTILS_EXCLUDED_ARGS = ( + # docutils.conf can't represent callables + "heading_slug_func", + # docutils.conf can't represent dicts + "html_meta", + "substitutions", + # we can't add substitutions so not needed + "sub_delimiters", + # sphinx only options + "heading_anchors", + "ref_domains", + "update_mathjax", + "mathjax_classes", +) +"""Names of settings that cannot be set in docutils.conf.""" + + +def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]: + """Convert a field into a Docutils optparse options dict.""" + if at.type is int: + return {"metavar": "", "validator": _validate_int}, f"(default: {default})" + if at.type is bool: + return { + "metavar": "", + "validator": frontend.validate_boolean, + }, f"(default: {default})" + if at.type is str: + return { + "metavar": "", + }, f"(default: '{default}')" + if get_origin(at.type) is Literal and all( + isinstance(a, str) for a in get_args(at.type) + ): + args = get_args(at.type) + return { + "metavar": f"<{'|'.join(repr(a) for a in args)}>", + "type": "choice", + "choices": args, + }, f"(default: {default!r})" + if at.type in (Iterable[str], Sequence[str]): + return { + "metavar": "", + "validator": frontend.validate_comma_separated_list, + }, f"(default: '{','.join(default)}')" + if at.type == Tuple[str, str]: + return { + "metavar": "", + "validator": _create_validate_tuple(2), + }, f"(default: '{','.join(default)}')" + if at.type == Union[int, type(None)]: + return { + "metavar": "", + "validator": _validate_int, + }, f"(default: {default})" + if at.type == Union[Iterable[str], type(None)]: + default_str = ",".join(default) if default else "" + return { + "metavar": "", + "validator": frontend.validate_comma_separated_list, + }, f"(default: {default_str!r})" + raise AssertionError( + f"Configuration option {at.name} not set up for use in docutils.conf." + ) + + +def attr_to_optparse_option( + attribute: Field, default: Any, prefix: str = "myst_" +) -> Tuple[str, List[str], Dict[str, Any]]: + """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple. + + :returns: A tuple of ``(help string, option flags, optparse kwargs)``. + """ + name = f"{prefix}{attribute.name}" + flag = "--" + name.replace("_", "-") + options = {"dest": name, "default": DOCUTILS_UNSET} + at_options, type_str = _attr_to_optparse_option(attribute, default) + options.update(at_options) + help_str = attribute.metadata.get("help", "") if attribute.metadata else "" + return (f"{help_str} {type_str}", [flag], options) + + +def create_myst_settings_spec( + excluded: Sequence[str], config_cls=MdParserConfig, prefix: str = "myst_" +): + """Return a list of Docutils setting for the docutils MyST section.""" + defaults = config_cls() + return tuple( + attr_to_optparse_option(at, getattr(defaults, at.name), prefix) + for at in config_cls.get_fields() + if at.name not in excluded + ) + + +def create_myst_config( + settings: frontend.Values, + excluded: Sequence[str], + config_cls=MdParserConfig, + prefix: str = "myst_", +): + """Create a configuration instance from the given settings.""" + values = {} + for attribute in config_cls.get_fields(): + if attribute.name in excluded: + continue + setting = f"{prefix}{attribute.name}" + val = getattr(settings, setting, DOCUTILS_UNSET) + if val is not DOCUTILS_UNSET: + values[attribute.name] = val + return config_cls(**values) + + +class Parser(RstParser): + """Docutils parser for Markedly Structured Text (MyST).""" + + supported: Tuple[str, ...] = ("md", "markdown", "myst") + """Aliases this parser supports.""" + + settings_spec = ( + "MyST options", + None, + create_myst_settings_spec(DOCUTILS_EXCLUDED_ARGS), + *RstParser.settings_spec, + ) + """Runtime settings specification.""" + + config_section = "myst parser" + config_section_dependencies = ("parsers",) + translate_section_name = None + + def parse(self, inputstring: str, document: nodes.document) -> None: + """Parse source text. + + :param inputstring: The source string to parse + :param document: The root docutils node to add AST elements to + """ + + self.setup_parse(inputstring, document) + + # check for exorbitantly long lines + if hasattr(document.settings, "line_length_limit"): + for i, line in enumerate(inputstring.split("\n")): + if len(line) > document.settings.line_length_limit: + error = document.reporter.error( + f"Line {i+1} exceeds the line-length-limit:" + f" {document.settings.line_length_limit}." + ) + document.append(error) + return + + # create parsing configuration from the global config + try: + config = create_myst_config(document.settings, DOCUTILS_EXCLUDED_ARGS) + except Exception as exc: + error = document.reporter.error(f"Global myst configuration invalid: {exc}") + document.append(error) + config = MdParserConfig() + + # update the global config with the file-level config + try: + topmatter = read_topmatter(inputstring) + except TopmatterReadError: + pass # this will be reported during the render + else: + if topmatter: + warning = lambda wtype, msg: create_warning( # noqa: E731 + document, msg, line=1, append_to=document, subtype=wtype + ) + config = merge_file_level(config, topmatter, warning) + + # parse content + parser = create_md_parser(config, DocutilsRenderer) + parser.options["document"] = document + parser.render(inputstring) + + # post-processing + + # replace raw nodes if raw is not allowed + if not getattr(document.settings, "raw_enabled", True): + for node in document.traverse(nodes.raw): + warning = document.reporter.warning("Raw content disabled.") + node.parent.replace(node, warning) + + self.finish_parse() + + +def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): + """Run the command line interface for a particular writer.""" + publish_cmdline( + parser=Parser(), + writer_name=writer_name, + description=( + f"Generates {writer_description} from standalone MyST sources.\n{default_description}" + ), + argv=argv, + ) + + +def cli_html(argv: Optional[List[str]] = None) -> None: + """Cmdline entrypoint for converting MyST to HTML.""" + _run_cli("html", "(X)HTML documents", argv) + + +def cli_html5(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to HTML5.""" + _run_cli("html5", "HTML5 documents", argv) + + +def cli_latex(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to LaTeX.""" + _run_cli("latex", "LaTeX documents", argv) + + +def cli_xml(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to XML.""" + _run_cli("xml", "Docutils-native XML", argv) + + +def cli_pseudoxml(argv: Optional[List[str]] = None): + """Cmdline entrypoint for converting MyST to pseudo-XML.""" + _run_cli("pseudoxml", "pseudo-XML", argv) diff --git a/myst_parser/parsers/mdit.py b/myst_parser/parsers/mdit.py new file mode 100644 index 00000000..249acd68 --- /dev/null +++ b/myst_parser/parsers/mdit.py @@ -0,0 +1,120 @@ +"""This module holds the ``create_md_parser`` function, +which creates a parser from the config. +""" +from __future__ import annotations + +from typing import Callable + +from markdown_it import MarkdownIt +from markdown_it.renderer import RendererProtocol +from mdit_py_plugins.amsmath import amsmath_plugin +from mdit_py_plugins.anchors import anchors_plugin +from mdit_py_plugins.colon_fence import colon_fence_plugin +from mdit_py_plugins.deflist import deflist_plugin +from mdit_py_plugins.dollarmath import dollarmath_plugin +from mdit_py_plugins.field_list import fieldlist_plugin +from mdit_py_plugins.footnote import footnote_plugin +from mdit_py_plugins.front_matter import front_matter_plugin +from mdit_py_plugins.myst_blocks import myst_block_plugin +from mdit_py_plugins.myst_role import myst_role_plugin +from mdit_py_plugins.substitution import substitution_plugin +from mdit_py_plugins.tasklists import tasklists_plugin +from mdit_py_plugins.wordcount import wordcount_plugin + +from myst_parser.config.main import MdParserConfig + + +def create_md_parser( + config: MdParserConfig, renderer: Callable[[MarkdownIt], RendererProtocol] +) -> MarkdownIt: + """Return a Markdown parser with the required MyST configuration.""" + + # TODO warn if linkify required and linkify-it-py not installed + # (currently the parse will unceremoniously except) + + if config.commonmark_only: + # see https://spec.commonmark.org/ + md = MarkdownIt("commonmark", renderer_cls=renderer).use( + wordcount_plugin, per_minute=config.words_per_minute + ) + md.options.update({"myst_config": config}) + return md + + if config.gfm_only: + # see https://github.github.com/gfm/ + md = ( + MarkdownIt("commonmark", renderer_cls=renderer) + # note, strikethrough currently only supported tentatively for HTML + .enable("strikethrough") + .enable("table") + .use(tasklists_plugin) + .enable("linkify") + .use(wordcount_plugin, per_minute=config.words_per_minute) + ) + md.options.update({"linkify": True, "myst_config": config}) + return md + + md = ( + MarkdownIt("commonmark", renderer_cls=renderer) + .enable("table") + .use(front_matter_plugin) + .use(myst_block_plugin) + .use(myst_role_plugin) + .use(footnote_plugin) + .use(wordcount_plugin, per_minute=config.words_per_minute) + .disable("footnote_inline") + # disable this for now, because it need a new implementation in the renderer + .disable("footnote_tail") + ) + + typographer = False + if "smartquotes" in config.enable_extensions: + md.enable("smartquotes") + typographer = True + if "replacements" in config.enable_extensions: + md.enable("replacements") + typographer = True + if "linkify" in config.enable_extensions: + md.enable("linkify") + if md.linkify is not None: + md.linkify.set({"fuzzy_link": config.linkify_fuzzy_links}) + if "strikethrough" in config.enable_extensions: + md.enable("strikethrough") + if "dollarmath" in config.enable_extensions: + md.use( + dollarmath_plugin, + allow_labels=config.dmath_allow_labels, + allow_space=config.dmath_allow_space, + allow_digits=config.dmath_allow_digits, + double_inline=config.dmath_double_inline, + ) + if "colon_fence" in config.enable_extensions: + md.use(colon_fence_plugin) + if "amsmath" in config.enable_extensions: + md.use(amsmath_plugin) + if "deflist" in config.enable_extensions: + md.use(deflist_plugin) + if "fieldlist" in config.enable_extensions: + md.use(fieldlist_plugin) + if "tasklist" in config.enable_extensions: + md.use(tasklists_plugin) + if "substitution" in config.enable_extensions: + md.use(substitution_plugin, *config.sub_delimiters) + if config.heading_anchors is not None: + md.use( + anchors_plugin, + max_level=config.heading_anchors, + slug_func=config.heading_slug_func, + ) + for name in config.disable_syntax: + md.disable(name, True) + + md.options.update( + { + "typographer": typographer, + "linkify": "linkify" in config.enable_extensions, + "myst_config": config, + } + ) + + return md diff --git a/myst_parser/parse_html.py b/myst_parser/parsers/parse_html.py similarity index 89% rename from myst_parser/parse_html.py rename to myst_parser/parsers/parse_html.py index d881f7fd..7539e42f 100644 --- a/myst_parser/parse_html.py +++ b/myst_parser/parsers/parse_html.py @@ -17,11 +17,13 @@ (see https://html.spec.whatwg.org/multipage/syntax.html#optional-tags) """ +from __future__ import annotations + import inspect import itertools from collections import abc, deque from html.parser import HTMLParser -from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Type, Union +from typing import Any, Callable, Iterable, Iterator class Attribute(dict): @@ -32,7 +34,7 @@ def __getitem__(self, key: str) -> str: return self.get(key, "") @property - def classes(self) -> List[str]: + def classes(self) -> list[str]: """Return 'class' attribute as list.""" return self["class"].split() @@ -47,24 +49,24 @@ class Element(abc.MutableSequence): All xml/html entities inherit from this class. """ - def __init__(self, name: str = "", attr: Optional[dict] = None) -> None: + def __init__(self, name: str = "", attr: dict | None = None) -> None: """Initialise the element.""" self.name = name self.attrs: Attribute = Attribute(attr or {}) - self._parent: Optional[Element] = None - self._children: List[Element] = [] + self._parent: Element | None = None + self._children: list[Element] = [] @property - def parent(self) -> Optional["Element"]: + def parent(self) -> Element | None: """Return parent.""" return self._parent @property - def children(self) -> List["Element"]: + def children(self) -> list[Element]: """Return copy of children.""" return self._children[:] - def reset_children(self, children: List["Element"], deepcopy: bool = False): + def reset_children(self, children: list[Element], deepcopy: bool = False): new_children = [] for i, item in enumerate(children): assert isinstance(item, Element) @@ -77,10 +79,10 @@ def reset_children(self, children: List["Element"], deepcopy: bool = False): new_children.append(item) self._children = new_children - def __getitem__(self, index: int) -> "Element": # type: ignore[override] + def __getitem__(self, index: int) -> Element: # type: ignore[override] return self._children[index] - def __setitem__(self, index: int, item: "Element"): # type: ignore[override] + def __setitem__(self, index: int, item: Element): # type: ignore[override] assert isinstance(item, Element) if item._parent is not None and item._parent != self: raise AssertionError(f"different parent already set for: {item!r}") @@ -93,18 +95,17 @@ def __delitem__(self, index: int): # type: ignore[override] def __len__(self) -> int: return self._children.__len__() - def __iter__(self) -> Iterator["Element"]: - for child in self._children: - yield child + def __iter__(self) -> Iterator[Element]: + yield from self._children - def insert(self, index: int, item: "Element"): + def insert(self, index: int, item: Element): assert isinstance(item, Element) if item._parent is not None and item._parent != self: raise AssertionError(f"different parent already set for: {item!r}") item._parent = self return self._children.insert(index, item) - def deepcopy(self) -> "Element": + def deepcopy(self) -> Element: """Recursively copy and remove parent.""" _copy = self.__class__(self.name, self.attrs) for child in self: @@ -121,7 +122,7 @@ def __repr__(self) -> str: def render( self, - tag_overrides: Optional[Dict[str, Callable[["Element", dict], str]]] = None, + tag_overrides: dict[str, Callable[[Element, dict], str]] | None = None, **kwargs, ) -> str: """Returns a HTML string representation of the element. @@ -138,16 +139,15 @@ def __str__(self) -> str: def __eq__(self, item: Any) -> bool: return item is self - def walk(self, include_self: bool = False) -> Iterator["Element"]: + def walk(self, include_self: bool = False) -> Iterator[Element]: """Walk through the xml/html AST.""" if include_self: yield self for child in self: yield child - for ancestor in child.walk(): - yield ancestor + yield from child.walk() - def strip(self, inplace: bool = False, recurse: bool = False) -> "Element": + def strip(self, inplace: bool = False, recurse: bool = False) -> Element: """Return copy with all `Data` tokens that only contain whitespace / newlines removed. """ @@ -168,12 +168,12 @@ def strip(self, inplace: bool = False, recurse: bool = False) -> "Element": def find( self, - identifier: Union[str, Type["Element"]], - attrs: Optional[dict] = None, - classes: Optional[Iterable[str]] = None, + identifier: str | type[Element], + attrs: dict | None = None, + classes: Iterable[str] | None = None, include_self: bool = False, recurse: bool = True, - ) -> Iterator["Element"]: + ) -> Iterator[Element]: """Find all elements that match name and specific attributes.""" iterator = self.walk() if recurse else self if include_self: @@ -207,7 +207,7 @@ class Tag(Element): def render( self, - tag_overrides: Optional[Dict[str, Callable[[Element, dict], str]]] = None, + tag_overrides: dict[str, Callable[[Element, dict], str]] | None = None, **kwargs, ) -> str: if tag_overrides and self.name in tag_overrides: @@ -226,7 +226,7 @@ class XTag(Element): def render( self, - tag_overrides: Optional[Dict[str, Callable[[Element, dict], str]]] = None, + tag_overrides: dict[str, Callable[[Element, dict], str]] | None = None, **kwargs, ) -> str: if tag_overrides is not None and self.name in tag_overrides: @@ -252,7 +252,7 @@ def __repr__(self) -> str: text = text[:17] + "..." return f"{self.__class__.__name__}({text!r})" - def deepcopy(self) -> "TerminalElement": + def deepcopy(self) -> TerminalElement: """Copy and remove parent.""" _copy = self.__class__(self.data) return _copy @@ -300,7 +300,7 @@ def render(self, **kwargs) -> str: # type: ignore[override] return f"&{self.data};" -class Tree(object): +class Tree: """The engine class to generate the AST tree.""" def __init__(self, name: str = ""): @@ -342,7 +342,7 @@ def nest_vtag(self, name: str, attrs: dict): item = VoidTag(name, attrs) top.append(item) - def nest_terminal(self, klass: Type[TerminalElement], data: str): + def nest_terminal(self, klass: type[TerminalElement], data: str): """Nest the data onto the tree.""" top = self.last() item = klass(data) diff --git a/myst_parser/parsers/sphinx_.py b/myst_parser/parsers/sphinx_.py new file mode 100644 index 00000000..fff098f3 --- /dev/null +++ b/myst_parser/parsers/sphinx_.py @@ -0,0 +1,69 @@ +"""MyST Markdown parser for sphinx.""" +from __future__ import annotations + +from docutils import nodes +from docutils.parsers.rst import Parser as RstParser +from sphinx.parsers import Parser as SphinxParser +from sphinx.util import logging + +from myst_parser.config.main import ( + MdParserConfig, + TopmatterReadError, + merge_file_level, + read_topmatter, +) +from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer, create_warning +from myst_parser.parsers.mdit import create_md_parser + +SPHINX_LOGGER = logging.getLogger(__name__) + + +class MystParser(SphinxParser): + """Sphinx parser for Markedly Structured Text (MyST).""" + + supported: tuple[str, ...] = ("md", "markdown", "myst") + """Aliases this parser supports.""" + + settings_spec = RstParser.settings_spec + """Runtime settings specification. + + Defines runtime settings and associated command-line options, as used by + `docutils.frontend.OptionParser`. This is a concatenation of tuples of: + + - Option group title (string or `None` which implies no group, just a list + of single options). + + - Description (string or `None`). + + - A sequence of option tuples + """ + + config_section = "myst parser" + config_section_dependencies = ("parsers",) + translate_section_name = None + + def parse(self, inputstring: str, document: nodes.document) -> None: + """Parse source text. + + :param inputstring: The source string to parse + :param document: The root docutils node to add AST elements to + + """ + # get the global config + config: MdParserConfig = document.settings.env.myst_config + + # update the global config with the file-level config + try: + topmatter = read_topmatter(inputstring) + except TopmatterReadError: + pass # this will be reported during the render + else: + if topmatter: + warning = lambda wtype, msg: create_warning( # noqa: E731 + document, msg, line=1, append_to=document, subtype=wtype + ) + config = merge_file_level(config, topmatter, warning) + + parser = create_md_parser(config, SphinxRenderer) + parser.options["document"] = document + parser.render(inputstring) diff --git a/myst_parser/sphinx_.py b/myst_parser/sphinx_.py index e7979c80..b0850865 100644 --- a/myst_parser/sphinx_.py +++ b/myst_parser/sphinx_.py @@ -3,4 +3,4 @@ .. include:: path/to/file.md :parser: myst_parser.sphinx_ """ -from myst_parser.sphinx_parser import MystParser as Parser # noqa: F401 +from myst_parser.parsers.sphinx_ import MystParser as Parser # noqa: F401 diff --git a/myst_parser/sphinx_ext/__init__.py b/myst_parser/sphinx_ext/__init__.py new file mode 100644 index 00000000..1bfeb71a --- /dev/null +++ b/myst_parser/sphinx_ext/__init__.py @@ -0,0 +1 @@ +"""Sphinx extension for myst_parser.""" diff --git a/myst_parser/directives.py b/myst_parser/sphinx_ext/directives.py similarity index 98% rename from myst_parser/directives.py rename to myst_parser/sphinx_ext/directives.py index 4cb90306..39ca2c65 100644 --- a/myst_parser/directives.py +++ b/myst_parser/sphinx_ext/directives.py @@ -29,7 +29,7 @@ class SubstitutionReferenceRole(SphinxRole): def run(self) -> Tuple[List[nodes.Node], List[nodes.system_message]]: subref_node = nodes.substitution_reference(self.rawtext, self.text) - self.set_source_info(subref_node, self.lineno) # type: ignore[arg-type] + self.set_source_info(subref_node, self.lineno) subref_node["refname"] = nodes.fully_normalize_name(self.text) return [subref_node], [] diff --git a/myst_parser/sphinx_ext/main.py b/myst_parser/sphinx_ext/main.py new file mode 100644 index 00000000..f5aeffc1 --- /dev/null +++ b/myst_parser/sphinx_ext/main.py @@ -0,0 +1,60 @@ +"""The setup for the sphinx extension.""" +from typing import Any + +from sphinx.application import Sphinx + + +def setup_sphinx(app: Sphinx, load_parser=False): + """Initialize all settings and transforms in Sphinx.""" + # we do this separately to setup, + # so that it can be called by external packages like myst_nb + from myst_parser.config.main import MdParserConfig + from myst_parser.parsers.sphinx_ import MystParser + from myst_parser.sphinx_ext.directives import ( + FigureMarkdown, + SubstitutionReferenceRole, + ) + from myst_parser.sphinx_ext.mathjax import override_mathjax + from myst_parser.sphinx_ext.myst_refs import MystReferenceResolver + + if load_parser: + app.add_source_suffix(".md", "markdown") + app.add_source_parser(MystParser) + + app.add_role("sub-ref", SubstitutionReferenceRole()) + app.add_directive("figure-md", FigureMarkdown) + + app.add_post_transform(MystReferenceResolver) + + for name, default, field in MdParserConfig().as_triple(): + if not field.metadata.get("docutils_only", False): + # TODO add types? + app.add_config_value(f"myst_{name}", default, "env", types=Any) + + app.connect("builder-inited", create_myst_config) + app.connect("builder-inited", override_mathjax) + + +def create_myst_config(app): + from sphinx.util import logging + + # Ignore type checkers because the attribute is dynamically assigned + from sphinx.util.console import bold # type: ignore[attr-defined] + + from myst_parser import __version__ + from myst_parser.config.main import MdParserConfig + + logger = logging.getLogger(__name__) + + values = { + name: app.config[f"myst_{name}"] + for name, _, field in MdParserConfig().as_triple() + if not field.metadata.get("docutils_only", False) + } + + try: + app.env.myst_config = MdParserConfig(**values) + logger.info(bold("myst v%s:") + " %s", __version__, app.env.myst_config) + except (TypeError, ValueError) as error: + logger.error("myst configuration invalid: %s", error.args[0]) + app.env.myst_config = MdParserConfig() diff --git a/myst_parser/mathjax.py b/myst_parser/sphinx_ext/mathjax.py similarity index 97% rename from myst_parser/mathjax.py rename to myst_parser/sphinx_ext/mathjax.py index e430c3ba..260f0080 100644 --- a/myst_parser/mathjax.py +++ b/myst_parser/sphinx_ext/mathjax.py @@ -53,14 +53,14 @@ def override_mathjax(app: Sphinx): if "dollarmath" not in app.config["myst_enable_extensions"]: return - if not app.env.myst_config.update_mathjax: # type: ignore[attr-defined] + if not app.env.myst_config.update_mathjax: # type: ignore return - mjax_classes = app.env.myst_config.mathjax_classes # type: ignore[attr-defined] + mjax_classes = app.env.myst_config.mathjax_classes # type: ignore if "mathjax3_config" in app.config: # sphinx 4 + mathjax 3 - app.config.mathjax3_config = app.config.mathjax3_config or {} # type: ignore[attr-defined] + app.config.mathjax3_config = app.config.mathjax3_config or {} # type: ignore app.config.mathjax3_config.setdefault("options", {}) if ( "processHtmlClass" in app.config.mathjax3_config["options"] diff --git a/myst_parser/myst_refs.py b/myst_parser/sphinx_ext/myst_refs.py similarity index 99% rename from myst_parser/myst_refs.py rename to myst_parser/sphinx_ext/myst_refs.py index e7847957..81675a1b 100644 --- a/myst_parser/myst_refs.py +++ b/myst_parser/sphinx_ext/myst_refs.py @@ -117,6 +117,8 @@ def resolve_myst_ref( # get allowed domains for referencing ref_domains = self.env.config.myst_ref_domains + assert self.app.builder + # next resolve for any other standard reference objects if ref_domains is None or "std" in ref_domains: stddomain = cast(StandardDomain, self.env.get_domain("std")) @@ -246,6 +248,7 @@ def _resolve_ref_nested( if not docname: return None + assert self.app.builder return make_refnode(self.app.builder, fromdocname, docname, labelid, innernode) def _resolve_doc_nested( @@ -277,4 +280,5 @@ def _resolve_doc_nested( caption = clean_astext(self.env.titles[docname]) innernode = nodes.inline(caption, caption, classes=["doc"]) + assert self.app.builder return make_refnode(self.app.builder, fromdocname, docname, "", innernode) diff --git a/myst_parser/sphinx_parser.py b/myst_parser/sphinx_parser.py deleted file mode 100644 index 7799d498..00000000 --- a/myst_parser/sphinx_parser.py +++ /dev/null @@ -1,80 +0,0 @@ -import time -from os import path -from typing import Tuple - -from docutils import nodes -from docutils.core import publish_doctree -from docutils.parsers.rst import Parser as RstParser -from markdown_it.token import Token -from sphinx.application import Sphinx -from sphinx.io import SphinxStandaloneReader -from sphinx.parsers import Parser as SphinxParser -from sphinx.util import logging -from sphinx.util.docutils import sphinx_domains - -from myst_parser.main import create_md_parser -from myst_parser.sphinx_renderer import SphinxRenderer - -SPHINX_LOGGER = logging.getLogger(__name__) - - -class MystParser(SphinxParser): - """Sphinx parser for Markedly Structured Text (MyST).""" - - supported: Tuple[str, ...] = ("md", "markdown", "myst") - """Aliases this parser supports.""" - - settings_spec = RstParser.settings_spec - """Runtime settings specification. - - Defines runtime settings and associated command-line options, as used by - `docutils.frontend.OptionParser`. This is a concatenation of tuples of: - - - Option group title (string or `None` which implies no group, just a list - of single options). - - - Description (string or `None`). - - - A sequence of option tuples - """ - - config_section = "myst parser" - config_section_dependencies = ("parsers",) - translate_section_name = None - - def parse(self, inputstring: str, document: nodes.document) -> None: - """Parse source text. - - :param inputstring: The source string to parse - :param document: The root docutils node to add AST elements to - - """ - config = document.settings.env.myst_config - parser = create_md_parser(config, SphinxRenderer) - parser.options["document"] = document - env: dict = {} - tokens = parser.parse(inputstring, env) - if not tokens or tokens[0].type != "front_matter": - # we always add front matter, so that we can merge it with global keys, - # specified in the sphinx configuration - tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens - parser.renderer.render(tokens, parser.options, env) - - -def parse(app: Sphinx, text: str, docname: str = "index") -> nodes.document: - """Parse a string as MystMarkdown with Sphinx application.""" - app.env.temp_data["docname"] = docname - app.env.all_docs[docname] = time.time() - reader = SphinxStandaloneReader() - reader.setup(app) - parser = MystParser() - parser.set_application(app) - with sphinx_domains(app.env): - return publish_doctree( - text, - path.join(app.srcdir, docname + ".md"), - reader=reader, - parser=parser, - parser_name="markdown", - settings_overrides={"env": app.env, "gettext_compact": True}, - ) diff --git a/pyproject.toml b/pyproject.toml index 5a36d88b..487d334e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,29 +55,29 @@ linkify = ["linkify-it-py~=1.0"] rtd = [ "ipython", "sphinx-book-theme", - "sphinx-panels", - "sphinxcontrib-bibtex~=2.4", + "sphinx-design", "sphinxext-rediraffe~=0.2.7", "sphinxcontrib.mermaid~=0.7.1", "sphinxext-opengraph~=0.6.3", ] testing = [ "beautifulsoup4", - "coverage", + "coverage[toml]", "docutils~=0.17.0", # this version changes some HTML tags "pytest>=6,<7", "pytest-cov", "pytest-regressions", "pytest-param-files~=0.3.4", + "sphinx-pytest", ] [project.scripts] myst-anchors = "myst_parser.cli:print_anchors" -myst-docutils-html = "myst_parser.docutils_:cli_html" -myst-docutils-html5 = "myst_parser.docutils_:cli_html5" -myst-docutils-latex = "myst_parser.docutils_:cli_latex" -myst-docutils-xml = "myst_parser.docutils_:cli_xml" -myst-docutils-pseudoxml = "myst_parser.docutils_:cli_pseudoxml" +myst-docutils-html = "myst_parser.parsers.docutils_:cli_html" +myst-docutils-html5 = "myst_parser.parsers.docutils_:cli_html5" +myst-docutils-latex = "myst_parser.parsers.docutils_:cli_latex" +myst-docutils-xml = "myst_parser.parsers.docutils_:cli_xml" +myst-docutils-pseudoxml = "myst_parser.parsers.docutils_:cli_pseudoxml" [tool.flit.module] name = "myst_parser" @@ -105,3 +105,6 @@ warn_unused_ignores = true [[tool.mypy.overrides]] module = ["docutils.*", "yaml.*"] ignore_missing_imports = true + +[tool.coverage.run] +omit = ["*/_docs.py"] diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 11b20c47..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Top-level configuration for pytest.""" -try: - import sphinx # noqa: F401 -except ImportError: - pass -else: - # only use when Sphinx is installed, to allow testing myst-docutils - pytest_plugins = "sphinx.testing.fixtures" diff --git a/tests/test_commonmark/test_commonmark.py b/tests/test_commonmark/test_commonmark.py index 63e0c89a..3cee1eb8 100644 --- a/tests/test_commonmark/test_commonmark.py +++ b/tests/test_commonmark/test_commonmark.py @@ -5,8 +5,10 @@ import os import pytest +from markdown_it.renderer import RendererHTML -from myst_parser.main import to_html +from myst_parser.config.main import MdParserConfig +from myst_parser.parsers.mdit import create_md_parser with open( os.path.join(os.path.dirname(__file__), "commonmark.json"), encoding="utf8" @@ -27,7 +29,8 @@ def test_commonmark(entry): "Thematic breaks on the first line conflict with front matter syntax" ) test_case = entry["markdown"] - output = to_html(test_case) + md = create_md_parser(MdParserConfig(), RendererHTML) + output = md.render(test_case) if entry["example"] == 593: # this doesn't have any bearing on the output diff --git a/tests/test_docutils.py b/tests/test_docutils.py index b206958c..f6f5d36f 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -6,7 +6,8 @@ from docutils import VersionInfo, __version_info__ from typing_extensions import Literal -from myst_parser.docutils_ import ( +from myst_parser.mdit_to_docutils.base import make_document +from myst_parser.parsers.docutils_ import ( Parser, attr_to_optparse_option, cli_html, @@ -15,7 +16,6 @@ cli_pseudoxml, cli_xml, ) -from myst_parser.docutils_renderer import make_document def test_attr_to_optparse_option(): diff --git a/tests/test_html/test_html_to_nodes.py b/tests/test_html/test_html_to_nodes.py index 9774a32f..207a6274 100644 --- a/tests/test_html/test_html_to_nodes.py +++ b/tests/test_html/test_html_to_nodes.py @@ -4,8 +4,8 @@ import pytest from docutils import nodes -from myst_parser.html_to_nodes import html_to_nodes -from myst_parser.main import MdParserConfig +from myst_parser.config.main import MdParserConfig +from myst_parser.mdit_to_docutils.html_to_nodes import html_to_nodes FIXTURE_PATH = Path(__file__).parent diff --git a/tests/test_html/test_parse_html.py b/tests/test_html/test_parse_html.py index 5b765887..3b4cdc1a 100644 --- a/tests/test_html/test_parse_html.py +++ b/tests/test_html/test_parse_html.py @@ -2,7 +2,7 @@ import pytest -from myst_parser.parse_html import tokenize_html +from myst_parser.parsers.parse_html import tokenize_html FIXTURE_PATH = Path(__file__).parent diff --git a/tests/test_renderers/fixtures/amsmath.md b/tests/test_renderers/fixtures/amsmath.md index 861e6256..fe3713ea 100644 --- a/tests/test_renderers/fixtures/amsmath.md +++ b/tests/test_renderers/fixtures/amsmath.md @@ -1,55 +1,51 @@ --------------------------------- Single Line: . \begin{equation} a \end{equation} . - + - + \begin{equation} a \end{equation} . --------------------------------- Multi Line: . \begin{equation} a \end{equation} . - + - + \begin{equation} a \end{equation} . --------------------------------- Multi Line no number: . \begin{equation*} a \end{equation*} . - + \begin{equation*} a \end{equation*} . --------------------------------- In list: . - \begin{equation} a = 1 \end{equation} . - + - + \begin{equation} a = 1 \end{equation} diff --git a/tests/test_renderers/fixtures/containers.md b/tests/test_renderers/fixtures/containers.md index 89bb699b..f67bea84 100644 --- a/tests/test_renderers/fixtures/containers.md +++ b/tests/test_renderers/fixtures/containers.md @@ -1,18 +1,16 @@ --------------------------------- Basic note: . ::: {note} *hallo* ::: . - + hallo . --------------------------------- Admonition with options: . ::: {admonition} A **title** @@ -21,10 +19,10 @@ Admonition with options: *hallo* ::: . - + - A + A <strong> title <paragraph> diff --git a/tests/test_renderers/fixtures/definition_lists.md b/tests/test_renderers/fixtures/definition_lists.md index 33529228..c064fa2c 100644 --- a/tests/test_renderers/fixtures/definition_lists.md +++ b/tests/test_renderers/fixtures/definition_lists.md @@ -1,4 +1,3 @@ --------------------------- Simple: . Term **1** @@ -18,7 +17,7 @@ Term 3 : other . -<document source="notset"> +<document source="<src>/index.md"> <definition_list classes="simple myst"> <definition_list_item> <term> diff --git a/tests/test_renderers/fixtures/directive_options.md b/tests/test_renderers/fixtures/directive_options.md index 838a1091..b9ae64a8 100644 --- a/tests/test_renderers/fixtures/directive_options.md +++ b/tests/test_renderers/fixtures/directive_options.md @@ -3,82 +3,76 @@ Test Directive 1: ```{restructuredtext-test-directive} ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=[], options={}, content: None . ------------------------------ Test Directive 2: . ```{restructuredtext-test-directive} foo ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=[], options={}, content: <literal_block xml:space="preserve"> foo . ------------------------------ Test Directive 3: . ```{restructuredtext-test-directive} foo ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=['foo'], options={}, content: None . ------------------------------ Test Directive 4: . ```{restructuredtext-test-directive} foo bar ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=['foo'], options={}, content: <literal_block xml:space="preserve"> bar . ------------------------------ Test Directive 5: . ```{restructuredtext-test-directive} foo bar ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=['foo bar'], options={}, content: None . ------------------------------ Test Directive 6: . ```{restructuredtext-test-directive} foo bar baz ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=['foo bar'], options={}, content: <literal_block xml:space="preserve"> baz . ------------------------------ Test Directive 7: . ```{restructuredtext-test-directive} @@ -86,15 +80,14 @@ Test Directive 7: foo ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=[], options={}, content: <literal_block xml:space="preserve"> foo . ------------------------------ Test Directive Options 1: . ```{restructuredtext-test-directive} @@ -105,15 +98,14 @@ option2: b foo ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=[], options={'option1': 'a', 'option2': 'b'}, content: <literal_block xml:space="preserve"> foo . ------------------------------ Test Directive Options 2: . ```{restructuredtext-test-directive} @@ -122,15 +114,14 @@ Test Directive Options 2: foo ``` . -<document source="notset"> - <system_message level="1" line="1" source="notset" type="INFO"> +<document source="<src>/index.md"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> Directive processed. Type="restructuredtext-test-directive", arguments=[], options={'option1': 'a', 'option2': 'b'}, content: <literal_block xml:space="preserve"> foo . ------------------------------ Test Directive Options Error: . ```{restructuredtext-test-directive} @@ -139,8 +130,8 @@ Test Directive Options Error: foo ``` . -<document source="notset"> - <system_message level="3" line="1" source="notset" type="ERROR"> +<document source="<src>/index.md"> + <system_message level="3" line="1" source="<src>/index.md" type="ERROR"> <paragraph> Directive 'restructuredtext-test-directive': Invalid options YAML: mapping values are not allowed here in "<unicode string>", line 2, column 8: @@ -152,18 +143,16 @@ foo foo . - ------------------------------ Unknown Directive: . ```{unknown} ``` . -<document source="notset"> - <system_message level="3" line="1" source="notset" type="ERROR"> +<document source="<src>/index.md"> + <system_message level="3" line="1" source="<src>/index.md" type="ERROR"> <paragraph> Unknown directive type "unknown". - <system_message level="1" line="1" source="notset" type="INFO"> + <system_message level="1" line="1" source="<src>/index.md" type="INFO"> <paragraph> No directive entry for "unknown" in module "docutils.parsers.rst.languages.en". Trying "unknown" as canonical directive name. diff --git a/tests/test_renderers/fixtures/docutil_syntax_elements.md b/tests/test_renderers/fixtures/docutil_syntax_elements.md index 93718030..9b59f3a1 100644 --- a/tests/test_renderers/fixtures/docutil_syntax_elements.md +++ b/tests/test_renderers/fixtures/docutil_syntax_elements.md @@ -1,4 +1,3 @@ ---------------------------- Raw . foo @@ -8,7 +7,6 @@ foo foo . ---------------------------- Hard-break . foo\ @@ -24,7 +22,6 @@ bar bar . ---------------------------- Strong: . **foo** @@ -35,7 +32,6 @@ Strong: foo . ---------------------------- Emphasis . *foo* @@ -46,7 +42,6 @@ Emphasis foo . ---------------------------- Escaped Emphasis: . \*foo* @@ -56,7 +51,6 @@ Escaped Emphasis: *foo* . --------------------------- Mixed Inline . a *b* **c** `abc` \\* @@ -75,7 +69,6 @@ a *b* **c** `abc` \\* \* . --------------------------- Inline Code: . `foo` @@ -86,7 +79,6 @@ Inline Code: foo . --------------------------- Heading: . # foo @@ -97,7 +89,6 @@ Heading: foo . --------------------------- Heading Levels: . # a @@ -120,8 +111,6 @@ Heading Levels: d . - --------------------------- Block Code: . foo @@ -131,7 +120,6 @@ Block Code: foo . --------------------------- Fenced Code: . ```sh @@ -143,7 +131,6 @@ foo foo . --------------------------- Fenced Code no language: . ``` @@ -155,7 +142,6 @@ foo foo . --------------------------- Fenced Code no language with trailing whitespace: . ``` @@ -167,7 +153,6 @@ foo foo . --------------------------- Image empty: . ![]() @@ -177,7 +162,6 @@ Image empty: <image alt="" uri=""> . --------------------------- Image with alt and title: . ![alt](src "title") @@ -187,7 +171,6 @@ Image with alt and title: <image alt="alt" title="title" uri="src"> . --------------------------- Image with escapable html: . ![alt](http://www.google<>.com) @@ -197,7 +180,6 @@ Image with escapable html: <image alt="alt" uri="http://www.google%3C%3E.com"> . --------------------------- Block Quote: . > *foo* @@ -209,7 +191,6 @@ Block Quote: foo . --------------------------- Bullet List: . - *foo* @@ -227,7 +208,6 @@ Bullet List: bar . --------------------------- Nested Bullets . - a @@ -253,7 +233,6 @@ Nested Bullets d . --------------------------- Enumerated List: . 1. *foo* @@ -286,7 +265,6 @@ para enumerator . --------------------------- Nested Enumrated List: . 1. a @@ -307,7 +285,6 @@ Nested Enumrated List: c . --------------------------- Sphinx Role containing backtick: . {code}``a=1{`}`` @@ -318,7 +295,6 @@ Sphinx Role containing backtick: a=1{`} . --------------------------- Target: . (target)= @@ -327,7 +303,6 @@ Target: <target ids="target" names="target"> . --------------------------- Target with whitespace: . (target with space)= @@ -336,7 +311,6 @@ Target with whitespace: <target ids="target-with-space" names="target\ with\ space"> . --------------------------- Referencing: . (target)= @@ -370,7 +344,6 @@ Title alt3 . --------------------------- Comments: . line 1 @@ -386,7 +359,6 @@ line 2 line 2 . --------------------------- Block Break: . +++ string @@ -396,7 +368,6 @@ Block Break: string . --------------------------- Link Reference: . [name][key] @@ -409,7 +380,6 @@ Link Reference: name . --------------------------- Link Reference short version: . [name] @@ -422,7 +392,6 @@ Link Reference short version: name . --------------------------- Block Quotes: . ```{epigraph} @@ -443,7 +412,6 @@ a b*c* b . --------------------------- Link Definition in directive: . ```{note} @@ -459,7 +427,6 @@ Link Definition in directive: a . --------------------------- Link Definition in nested directives: . ```{note} @@ -486,7 +453,6 @@ Link Definition in nested directives: <note> . --------------------------- Footnotes: . [^a] @@ -504,7 +470,6 @@ Footnotes: text . --------------------------- Footnotes nested blocks: . [^a] @@ -548,7 +513,6 @@ finish c . --------------------------- Front Matter: . --- @@ -583,7 +547,6 @@ c: {"d": 2} . --------------------------- Front Matter Biblio: . --- @@ -702,7 +665,6 @@ other: Something else Something else . --------------------------- Front Matter Bad Yaml: . --- @@ -710,26 +672,20 @@ a: { --- . <document source="notset"> - <system_message level="3" line="1" source="notset" type="ERROR"> + <system_message level="2" line="1" source="notset" type="WARNING"> <paragraph> - Front matter block: - while parsing a flow node - expected the node content, but found '<stream end>' - in "<unicode string>", line 1, column 5: - a: { - ^ - <literal_block xml:space="preserve"> - a: { + Malformed YAML [myst.topmatter] . Front Matter HTML Meta . --- -html_meta: - keywords: Sphinx, documentation, builder - description lang=en: An amusing story - description lang=fr: Un histoire amusant - http-equiv=Content-Type: text/html; charset=ISO-8859-1 +myst: + html_meta: + keywords: Sphinx, documentation, builder + description lang=en: An amusing story + description lang=fr: Un histoire amusant + http-equiv=Content-Type: text/html; charset=ISO-8859-1 --- . <document source="notset"> @@ -767,7 +723,6 @@ html_meta: <meta content="text/html; charset=ISO-8859-1" http-equiv="Content-Type"> . --------------------------- Full Test: . --- diff --git a/tests/test_renderers/fixtures/dollarmath.md b/tests/test_renderers/fixtures/dollarmath.md index d3acabcb..e3e5fd23 100644 --- a/tests/test_renderers/fixtures/dollarmath.md +++ b/tests/test_renderers/fixtures/dollarmath.md @@ -1,21 +1,19 @@ --------------------------- Inline Math: . $foo$ . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <math> foo . --------------------------- Inline Math, multi-line: . a $foo bar$ b . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> a <math> @@ -24,42 +22,38 @@ bar$ b b . --------------------------- Inline Math, multi-line with line break (invalid): . a $foo bar$ b . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> a $foo <paragraph> bar$ b . --------------------------- Math Block: . $$foo$$ . -<document source="notset"> +<document source="<src>/index.md"> <math_block nowrap="False" number="True" xml:space="preserve"> foo . --------------------------- Math Block With Equation Label: . $$foo$$ (abc) . -<document source="notset"> +<document source="<src>/index.md"> <target ids="equation-abc"> - <math_block docname="mock_docname" label="abc" nowrap="False" number="1" xml:space="preserve"> + <math_block docname="index" label="abc" nowrap="False" number="1" xml:space="preserve"> foo . --------------------------- Math Block multiple: . $$ @@ -70,12 +64,12 @@ $$ b = 2 $$ (a) . -<document source="notset"> +<document source="<src>/index.md"> <math_block nowrap="False" number="True" xml:space="preserve"> a = 1 <target ids="equation-a"> - <math_block docname="mock_docname" label="a" nowrap="False" number="1" xml:space="preserve"> + <math_block docname="index" label="a" nowrap="False" number="1" xml:space="preserve"> b = 2 . diff --git a/tests/test_renderers/fixtures/eval_rst.md b/tests/test_renderers/fixtures/eval_rst.md index 421aeb01..9f21bd19 100644 --- a/tests/test_renderers/fixtures/eval_rst.md +++ b/tests/test_renderers/fixtures/eval_rst.md @@ -4,7 +4,7 @@ eval-rst link `MyST Parser <https://myst-parser.readthedocs.io/>`_ ``` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <reference name="MyST Parser" refuri="https://myst-parser.readthedocs.io/"> MyST Parser @@ -16,7 +16,7 @@ eval-rst bold ```{eval-rst} **bold** . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <strong> bold diff --git a/tests/test_renderers/fixtures/reporter_warnings.md b/tests/test_renderers/fixtures/reporter_warnings.md index 22b53329..e9998b90 100644 --- a/tests/test_renderers/fixtures/reporter_warnings.md +++ b/tests/test_renderers/fixtures/reporter_warnings.md @@ -37,24 +37,43 @@ Bad Front Matter: a: { --- . -<string>:1: (ERROR/3) Front matter block: -while parsing a flow node -expected the node content, but found '<stream end>' - in "<unicode string>", line 1, column 5: - a: { - ^ +<string>:1: (WARNING/2) Malformed YAML [myst.topmatter] +. + +Unknown Front Matter myst key: +. +--- +myst: + unknown: true +--- +. +<string>:1: (WARNING/2) Unknown field: unknown [myst.topmatter] +. + +Invalid Front Matter myst key: +. +--- +myst: + title_to_header: 1 + url_schemes: [1] + substitutions: + key: [] +--- +. +<string>:1: (WARNING/2) 'title_to_header' must be of type <class 'bool'> (got 1 that is a <class 'int'>). [myst.topmatter] +<string>:1: (WARNING/2) 'url_schemes[0]' must be of type <class 'str'> (got 1 that is a <class 'int'>). [myst.topmatter] +<string>:1: (WARNING/2) 'substitutions['key']' must be of type (<class 'str'>, <class 'int'>, <class 'float'>) (got [] that is a <class 'list'>). [myst.topmatter] . Bad HTML Meta . --- -html_meta: - empty: - name noequals: value +myst: + html_meta: + name noequals: value --- . -<string>:: (ERROR/3) Error parsing meta tag attribute "empty": No content. <string>:: (ERROR/3) Error parsing meta tag attribute "name noequals": no '=' in noequals. . diff --git a/tests/test_renderers/fixtures/sphinx_directives.md b/tests/test_renderers/fixtures/sphinx_directives.md index 72107736..16a5fa74 100644 --- a/tests/test_renderers/fixtures/sphinx_directives.md +++ b/tests/test_renderers/fixtures/sphinx_directives.md @@ -1,28 +1,25 @@ --------------------------------- default-role (`sphinx.directives.DefaultRole`): . ```{default-role} ``` . -<document source="notset"> +<document source="<src>/index.md"> . --------------------------------- default-domain (`sphinx.directives.DefaultDomain`): . ```{default-domain} mydomain ``` . -<document source="notset"> +<document source="<src>/index.md"> . --------------------------------- SPHINX4 object (`sphinx.directives.ObjectDescription`): . ```{object} something ``` . -<document source="notset"> +<document source="<src>/index.md"> <index entries=""> <desc classes="object" desctype="object" domain="" noindex="False" objtype="object"> <desc_signature classes="sig sig-object"> @@ -31,17 +28,15 @@ SPHINX4 object (`sphinx.directives.ObjectDescription`): <desc_content> . --------------------------------- highlight (`sphinx.directives.code.Highlight`): . ```{highlight} something ``` . -<document source="notset"> +<document source="<src>/index.md"> <highlightlang force="False" lang="something" linenothreshold="9223372036854775807"> . --------------------------------- code-block (`sphinx.directives.code.CodeBlock`): . ```{code-block} @@ -50,23 +45,20 @@ code-block (`sphinx.directives.code.CodeBlock`): a=1 ``` . -<document source="notset"> +<document source="<src>/index.md"> <literal_block force="False" highlight_args="{}" language="default" xml:space="preserve"> a=1 . --------------------------------- sourcecode (`sphinx.directives.code.CodeBlock`): . ```{sourcecode} ``` . -<document source="notset"> +<document source="<src>/index.md"> <literal_block force="False" highlight_args="{}" language="default" xml:space="preserve"> . --------------------------------- -literalinclude (`sphinx.directives.code.LiteralInclude`): SKIP: Tested in sphinx builds . ```{literalinclude} /path/to/file @@ -78,56 +70,50 @@ SKIP: Tested in sphinx builds Include file '/srcdir/path/to/file' not found or reading it failed . --------------------------------- toctree (`sphinx.directives.other.TocTree`): . ```{toctree} ``` . -<document source="notset"> +<document source="<src>/index.md"> <compound classes="toctree-wrapper"> - <toctree caption="True" entries="" glob="False" hidden="False" includefiles="" includehidden="False" maxdepth="-1" numbered="0" parent="mock_docname" titlesonly="False"> + <toctree caption="True" entries="" glob="False" hidden="False" includefiles="" includehidden="False" maxdepth="-1" numbered="0" parent="index" titlesonly="False"> . --------------------------------- sectionauthor (`sphinx.directives.other.Author`): . ```{sectionauthor} bob geldof ``` . -<document source="notset"> +<document source="<src>/index.md"> . --------------------------------- moduleauthor (`sphinx.directives.other.Author`): . ```{moduleauthor} ringo starr ``` . -<document source="notset"> +<document source="<src>/index.md"> . --------------------------------- codeauthor (`sphinx.directives.other.Author`): . ```{codeauthor} paul mcartney ``` . -<document source="notset"> +<document source="<src>/index.md"> . --------------------------------- index (`sphinx.directives.other.Index`): . ```{index} something ``` . -<document source="notset"> +<document source="<src>/index.md"> <index entries="('single',\ 'something',\ 'index-0',\ '',\ None)" inline="False"> <target ids="index-0"> . --------------------------------- seealso (`sphinx.directives.other.SeeAlso`): . ```{seealso} @@ -135,34 +121,31 @@ seealso (`sphinx.directives.other.SeeAlso`): a ``` . -<document source="notset"> +<document source="<src>/index.md"> <seealso> <paragraph> a . --------------------------------- tabularcolumns (`sphinx.directives.other.TabularColumns`): . ```{tabularcolumns} spec ``` . -<document source="notset"> +<document source="<src>/index.md"> <tabular_col_spec spec="spec"> . --------------------------------- centered (`sphinx.directives.other.Centered`): . ```{centered} text ``` . -<document source="notset"> +<document source="<src>/index.md"> <centered> text . --------------------------------- acks (`sphinx.directives.other.Acks`): . ```{acks} @@ -170,7 +153,7 @@ acks (`sphinx.directives.other.Acks`): - name ``` . -<document source="notset"> +<document source="<src>/index.md"> <acks> <bullet_list bullet="-"> <list_item> @@ -178,7 +161,6 @@ acks (`sphinx.directives.other.Acks`): name . --------------------------------- SPHINX4 hlist (`sphinx.directives.other.HList`): . ```{hlist} @@ -186,7 +168,7 @@ SPHINX4 hlist (`sphinx.directives.other.HList`): - item ``` . -<document source="notset"> +<document source="<src>/index.md"> <hlist ncolumns="2"> <hlistcol> <bullet_list> @@ -197,18 +179,15 @@ SPHINX4 hlist (`sphinx.directives.other.HList`): <bullet_list> . --------------------------------- only (`sphinx.directives.other.Only`): . ```{only} expr ``` . -<document source="notset"> +<document source="<src>/index.md"> <only expr="expr"> . --------------------------------- -include (`sphinx.directives.other.Include`): SKIP: Tested in sphinx builds . ```{include} path/to/include @@ -217,7 +196,6 @@ SKIP: Tested in sphinx builds <document source="notset"> . --------------------------------- figure (`sphinx.directives.patches.Figure`): . ```{figure} path/to/figure @@ -227,7 +205,7 @@ figure (`sphinx.directives.patches.Figure`): legend ``` . -<document source="notset"> +<document source="<src>/index.md"> <figure> <image uri="path/to/figure"> <caption> @@ -238,8 +216,6 @@ legend legend . --------------------------------- -meta (`sphinx.directives.patches.Meta`): SKIP: MockingError: MockState has not yet implemented attribute 'nested_list_parse' . ```{meta} @@ -249,7 +225,6 @@ foo <document source="notset"> . --------------------------------- table (`sphinx.directives.patches.RSTTable`): . ```{table} *title* @@ -260,7 +235,7 @@ table (`sphinx.directives.patches.RSTTable`): | 1 | 2 | ``` . -<document source="notset"> +<document source="<src>/index.md"> <table classes="colwidths-auto" ids="name" names="name"> <title> <emphasis> @@ -286,7 +261,6 @@ table (`sphinx.directives.patches.RSTTable`): 2 . --------------------------------- csv-table (`sphinx.directives.patches.CSVTable`): . ```{csv-table} @@ -294,7 +268,7 @@ csv-table (`sphinx.directives.patches.CSVTable`): "Albatross", 2.99, "On a stick!" ``` . -<document source="notset"> +<document source="<src>/index.md"> <table> <tgroup cols="3"> <colspec colwidth="33"> @@ -313,7 +287,6 @@ csv-table (`sphinx.directives.patches.CSVTable`): On a stick! . --------------------------------- list-table (`sphinx.directives.patches.ListTable`): . ```{list-table} @@ -321,7 +294,7 @@ list-table (`sphinx.directives.patches.ListTable`): * - item ``` . -<document source="notset"> +<document source="<src>/index.md"> <table> <tgroup cols="1"> <colspec colwidth="100"> @@ -332,7 +305,6 @@ list-table (`sphinx.directives.patches.ListTable`): item . --------------------------------- code (`sphinx.directives.patches.Code`): . ```{code} python @@ -340,60 +312,56 @@ code (`sphinx.directives.patches.Code`): a ``` . -<document source="notset"> +<document source="<src>/index.md"> <literal_block force="False" highlight_args="{}" language="python" xml:space="preserve"> a . --------------------------------- math (`sphinx.directives.patches.MathDirective`): . ```{math} ``` . -<document source="notset"> - <math_block docname="mock_docname" label="True" nowrap="False" number="True" xml:space="preserve"> +<document source="<src>/index.md"> + <math_block docname="index" label="True" nowrap="False" number="True" xml:space="preserve"> . --------------------------------- deprecated (`sphinx.domains.changeset.VersionChange`): . ```{deprecated} 0.3 ``` . -<document source="notset"> +<document source="<src>/index.md"> <versionmodified type="deprecated" version="0.3"> <paragraph translatable="False"> <inline classes="versionmodified deprecated"> Deprecated since version 0.3. . --------------------------------- versionadded (`sphinx.domains.changeset.VersionChange`): . ```{versionadded} 0.2 ``` . -<document source="notset"> +<document source="<src>/index.md"> <versionmodified type="versionadded" version="0.2"> <paragraph translatable="False"> <inline classes="versionmodified added"> New in version 0.2. . --------------------------------- versionchanged (`sphinx.domains.changeset.VersionChange`): . ```{versionchanged} 0.1 ``` . -<document source="notset"> +<document source="<src>/index.md"> <versionmodified type="versionchanged" version="0.1"> <paragraph translatable="False"> <inline classes="versionmodified changed"> Changed in version 0.1. . --------------------------------- + glossary (`sphinx.domains.std.Glossary`): . ```{glossary} @@ -403,7 +371,7 @@ term 2 : B Definition of both terms. ``` . -<document source="notset"> +<document source="<src>/index.md"> <glossary> <definition_list classes="glossary"> <definition_list_item> @@ -418,25 +386,23 @@ term 2 : B Definition of both terms. . --------------------------------- SPHINX3 productionlist (`sphinx.domains.std.ProductionList`): . ```{productionlist} try_stmt: try1_stmt | try2_stmt ``` . -<document source="notset"> +<document source="<src>/index.md"> <productionlist> <production ids="grammar-token-try_stmt grammar-token-try-stmt" tokenname="try_stmt" xml:space="preserve"> try1_stmt | try2_stmt . --------------------------------- SPHINX4 cmdoption (`sphinx.domains.std.Cmdoption`): . ```{cmdoption} a ``` . -<document source="notset"> +<document source="<src>/index.md"> <index entries="('pair',\ 'command\ line\ option;\ a',\ 'cmdoption-arg-a',\ '',\ None)"> <desc classes="std cmdoption" desctype="cmdoption" domain="std" noindex="False" objtype="cmdoption"> <desc_signature allnames="a" classes="sig sig-object" ids="cmdoption-arg-a"> @@ -446,13 +412,12 @@ SPHINX4 cmdoption (`sphinx.domains.std.Cmdoption`): <desc_content> . --------------------------------- SPHINX4 rst:directive (`sphinx.domains.rst.ReSTDirective`): . ```{rst:directive} a ``` . -<document source="notset"> +<document source="<src>/index.md"> <index entries="('single',\ 'a\ (directive)',\ 'directive-a',\ '',\ None)"> <desc classes="rst directive" desctype="directive" domain="rst" noindex="False" objtype="directive"> <desc_signature classes="sig sig-object" ids="directive-a"> @@ -461,13 +426,12 @@ SPHINX4 rst:directive (`sphinx.domains.rst.ReSTDirective`): <desc_content> . --------------------------------- SPHINX4 rst:directive:option (`sphinx.domains.rst.ReSTDirectiveOption`): . ```{rst:directive:option} a ``` . -<document source="notset"> +<document source="<src>/index.md"> <index entries="('single',\ ':a:\ (directive\ option)',\ 'directive-option-a',\ '',\ 'A')"> <desc classes="rst directive:option" desctype="directive:option" domain="rst" noindex="False" objtype="directive:option"> <desc_signature classes="sig sig-object" ids="directive-option-a directive:option--a"> diff --git a/tests/test_renderers/fixtures/sphinx_roles.md b/tests/test_renderers/fixtures/sphinx_roles.md index ee9c74c9..37683dbe 100644 --- a/tests/test_renderers/fixtures/sphinx_roles.md +++ b/tests/test_renderers/fixtures/sphinx_roles.md @@ -1,196 +1,179 @@ --------------------------------- c:func (`sphinx.domains.c.CXRefRole`): . {c:func}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="c" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> + <pending_xref refdoc="index" refdomain="c" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> <literal classes="xref c c-func"> a() . --------------------------------- c:member (`sphinx.domains.c.CObject`): . {c:member}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="c" refexplicit="False" reftarget="a" reftype="member" refwarn="False"> + <pending_xref refdoc="index" refdomain="c" refexplicit="False" reftarget="a" reftype="member" refwarn="False"> <literal classes="xref c c-member"> a . --------------------------------- c:macro (`sphinx.domains.c.CObject`): . {c:macro}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="c" refexplicit="False" reftarget="a" reftype="macro" refwarn="False"> + <pending_xref refdoc="index" refdomain="c" refexplicit="False" reftarget="a" reftype="macro" refwarn="False"> <literal classes="xref c c-macro"> a . --------------------------------- c:data (`sphinx.domains.c.CXRefRole`): . {c:data}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="c" refexplicit="False" reftarget="a" reftype="data" refwarn="False"> + <pending_xref refdoc="index" refdomain="c" refexplicit="False" reftarget="a" reftype="data" refwarn="False"> <literal classes="xref c c-data"> a . --------------------------------- c:type (`sphinx.domains.c.CObject`): . {c:type}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="c" refexplicit="False" reftarget="a" reftype="type" refwarn="False"> + <pending_xref refdoc="index" refdomain="c" refexplicit="False" reftarget="a" reftype="type" refwarn="False"> <literal classes="xref c c-type"> a . --------------------------------- cpp:any (`sphinx.domains.cpp.CPPXRefRole`): . {cpp:any}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="any" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="any" refwarn="False"> <literal classes="xref cpp cpp-any"> a . --------------------------------- cpp:class (`sphinx.domains.cpp.CPPClassObject`): . {cpp:class}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="class" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="class" refwarn="False"> <literal classes="xref cpp cpp-class"> a . --------------------------------- cpp:struct (`sphinx.domains.cpp.CPPClassObject`): . {cpp:struct}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="struct" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="struct" refwarn="False"> <literal classes="xref cpp cpp-struct"> a . --------------------------------- cpp:union (`sphinx.domains.cpp.CPPUnionObject`): . {cpp:union}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="union" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="union" refwarn="False"> <literal classes="xref cpp cpp-union"> a . --------------------------------- cpp:func (`sphinx.domains.cpp.CPPXRefRole`): . {cpp:func}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> <literal classes="xref cpp cpp-func"> a() . --------------------------------- cpp:member (`sphinx.domains.cpp.CPPMemberObject`): . {cpp:member}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="member" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="member" refwarn="False"> <literal classes="xref cpp cpp-member"> a . --------------------------------- cpp:var (`sphinx.domains.cpp.CPPMemberObject`): . {cpp:var}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="var" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="var" refwarn="False"> <literal classes="xref cpp cpp-var"> a . --------------------------------- cpp:type (`sphinx.domains.cpp.CPPTypeObject`): . {cpp:type}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="type" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="type" refwarn="False"> <literal classes="xref cpp cpp-type"> a . --------------------------------- cpp:concept (`sphinx.domains.cpp.CPPConceptObject`): . {cpp:concept}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="concept" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="concept" refwarn="False"> <literal classes="xref cpp cpp-concept"> a . --------------------------------- cpp:enum (`sphinx.domains.cpp.CPPEnumObject`): . {cpp:enum}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="enum" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="enum" refwarn="False"> <literal classes="xref cpp cpp-enum"> a . --------------------------------- cpp:enumerator (`sphinx.domains.cpp.CPPEnumeratorObject`): . {cpp:enumerator}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="cpp" refexplicit="False" reftarget="a" reftype="enumerator" refwarn="False"> + <pending_xref refdoc="index" refdomain="cpp" refexplicit="False" reftarget="a" reftype="enumerator" refwarn="False"> <literal classes="xref cpp cpp-enumerator"> a . --------------------------------- SKIP cpp:expr (`sphinx.domains.cpp.CPPExprRole`): . {cpp:expr}`a` @@ -203,7 +186,6 @@ SKIP cpp:expr (`sphinx.domains.cpp.CPPExprRole`): a . --------------------------------- SKIP cpp:texpr (`sphinx.domains.cpp.CPPExprRole`): . {cpp:texpr}`a` @@ -216,369 +198,337 @@ SKIP cpp:texpr (`sphinx.domains.cpp.CPPExprRole`): a . --------------------------------- js:func (`sphinx.domains.javascript.JSXRefRole`): . {js:func}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref js:module="True" js:object="True" refdoc="mock_docname" refdomain="js" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> + <pending_xref js:module="True" js:object="True" refdoc="index" refdomain="js" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> <literal classes="xref js js-func"> a() . --------------------------------- js:meth (`sphinx.domains.javascript.JSXRefRole`): . {js:meth}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref js:module="True" js:object="True" refdoc="mock_docname" refdomain="js" refexplicit="False" reftarget="a" reftype="meth" refwarn="False"> + <pending_xref js:module="True" js:object="True" refdoc="index" refdomain="js" refexplicit="False" reftarget="a" reftype="meth" refwarn="False"> <literal classes="xref js js-meth"> a() . --------------------------------- js:class (`sphinx.domains.javascript.JSConstructor`): . {js:class}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref js:module="True" js:object="True" refdoc="mock_docname" refdomain="js" refexplicit="False" reftarget="a" reftype="class" refwarn="False"> + <pending_xref js:module="True" js:object="True" refdoc="index" refdomain="js" refexplicit="False" reftarget="a" reftype="class" refwarn="False"> <literal classes="xref js js-class"> a() . --------------------------------- js:data (`sphinx.domains.javascript.JSObject`): . {js:data}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref js:module="True" js:object="True" refdoc="mock_docname" refdomain="js" refexplicit="False" reftarget="a" reftype="data" refwarn="False"> + <pending_xref js:module="True" js:object="True" refdoc="index" refdomain="js" refexplicit="False" reftarget="a" reftype="data" refwarn="False"> <literal classes="xref js js-data"> a . --------------------------------- js:attr (`sphinx.domains.javascript.JSXRefRole`): . {js:attr}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref js:module="True" js:object="True" refdoc="mock_docname" refdomain="js" refexplicit="False" reftarget="a" reftype="attr" refwarn="False"> + <pending_xref js:module="True" js:object="True" refdoc="index" refdomain="js" refexplicit="False" reftarget="a" reftype="attr" refwarn="False"> <literal classes="xref js js-attr"> a . --------------------------------- js:mod (`sphinx.domains.javascript.JSXRefRole`): . {js:mod}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref js:module="True" js:object="True" refdoc="mock_docname" refdomain="js" refexplicit="False" reftarget="a" reftype="mod" refwarn="False"> + <pending_xref js:module="True" js:object="True" refdoc="index" refdomain="js" refexplicit="False" reftarget="a" reftype="mod" refwarn="False"> <literal classes="xref js js-mod"> a . --------------------------------- eq (`sphinx.domains.math.MathReferenceRole`): . {eq}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="math" refexplicit="False" reftarget="a" reftype="eq" refwarn="True"> + <pending_xref refdoc="index" refdomain="math" refexplicit="False" reftarget="a" reftype="eq" refwarn="True"> <literal classes="xref eq"> a . --------------------------------- math:numref (`sphinx.domains.math.MathReferenceRole`): . {math:numref}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="math" refexplicit="False" reftarget="a" reftype="numref" refwarn="False"> + <pending_xref refdoc="index" refdomain="math" refexplicit="False" reftarget="a" reftype="numref" refwarn="False"> <literal classes="xref math math-numref"> a . --------------------------------- py:data (`sphinx.domains.python.PyVariable`): . {py:data}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="data" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="data" refwarn="False"> <literal classes="xref py py-data"> a . --------------------------------- py:exc (`sphinx.domains.python.PyXRefRole`): . {py:exc}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="exc" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="exc" refwarn="False"> <literal classes="xref py py-exc"> a . --------------------------------- py:func (`sphinx.domains.python.PyXRefRole`): . {py:func}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="func" refwarn="False"> <literal classes="xref py py-func"> a() . --------------------------------- py:class (`sphinx.domains.python.PyClasslike`): . {py:class}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="class" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="class" refwarn="False"> <literal classes="xref py py-class"> a . --------------------------------- py:const (`sphinx.domains.python.PyXRefRole`): . {py:const}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="const" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="const" refwarn="False"> <literal classes="xref py py-const"> a . --------------------------------- py:attr (`sphinx.domains.python.PyXRefRole`): . {py:attr}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="attr" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="attr" refwarn="False"> <literal classes="xref py py-attr"> a . --------------------------------- py:meth (`sphinx.domains.python.PyXRefRole`): . {py:meth}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="meth" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="meth" refwarn="False"> <literal classes="xref py py-meth"> a() . --------------------------------- py:mod (`sphinx.domains.python.PyXRefRole`): . {py:mod}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="mod" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="mod" refwarn="False"> <literal classes="xref py py-mod"> a . --------------------------------- py:obj (`sphinx.domains.python.PyXRefRole`): . {py:obj}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref py:class="True" py:module="True" refdoc="mock_docname" refdomain="py" refexplicit="False" reftarget="a" reftype="obj" refwarn="False"> + <pending_xref py:class="True" py:module="True" refdoc="index" refdomain="py" refexplicit="False" reftarget="a" reftype="obj" refwarn="False"> <literal classes="xref py py-obj"> a . --------------------------------- rst:role (`sphinx.domains.rst.ReSTRole`): . {rst:role}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="rst" refexplicit="False" reftarget="a" reftype="role" refwarn="False"> + <pending_xref refdoc="index" refdomain="rst" refexplicit="False" reftarget="a" reftype="role" refwarn="False"> <literal classes="xref rst rst-role"> a . --------------------------------- program (`sphinx.domains.std.Program`): . {program}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <literal_strong classes="program"> a . --------------------------------- option (`sphinx.domains.std.Cmdoption`): . {option}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="option" refwarn="True" std:program="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="option" refwarn="True" std:program="True"> <literal classes="xref std std-option"> a . --------------------------------- envvar (`sphinx.domains.std.EnvVarXRefRole`): . {envvar}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <index entries="('single',\ 'a',\ 'index-0',\ '',\ None) ('single',\ 'environment\ variable;\ a',\ 'index-0',\ '',\ None)"> <target ids="index-0"> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="envvar" refwarn="False"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="envvar" refwarn="False"> <literal classes="xref std std-envvar"> a . --------------------------------- index (`sphinx.roles.Index`): . {index}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <index entries="('single',\ 'a',\ 'index-0',\ '',\ None)"> <target ids="index-0"> a . --------------------------------- download (`sphinx.roles.XRefRole`): . {download}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <download_reference refdoc="mock_docname" refdomain="" refexplicit="False" reftarget="a" reftype="download" refwarn="False"> + <download_reference refdoc="index" refdomain="" refexplicit="False" reftarget="a" reftype="download" refwarn="False"> <literal classes="xref download"> a . --------------------------------- any (`sphinx.roles.AnyXRefRole`): . {any}`a <alt text>` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="" refexplicit="True" reftarget="alt text" reftype="any" refwarn="True"> + <pending_xref refdoc="index" refdomain="" refexplicit="True" reftarget="alt text" reftype="any" refwarn="True"> <literal classes="xref any"> a . --------------------------------- pep (`sphinx.roles.PEP`): . {pep}`1` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <index entries="('single',\ 'Python\ Enhancement\ Proposals;\ PEP\ 1',\ 'index-0',\ '',\ None)"> <target ids="index-0"> - <reference classes="pep" internal="False" refuri="http://www.python.org/dev/peps/pep-0001/"> + <reference classes="pep" internal="False" refuri="https://peps.python.org/pep-0001/"> <strong> PEP 1 . --------------------------------- rfc (`sphinx.roles.RFC`): . {rfc}`1` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <index entries="('single',\ 'RFC;\ RFC\ 1',\ 'index-0',\ '',\ None)"> <target ids="index-0"> - <reference classes="rfc" internal="False" refuri="http://tools.ietf.org/html/rfc1.html"> + <reference classes="rfc" internal="False" refuri="https://datatracker.ietf.org/doc/html/rfc1.html"> <strong> RFC 1 . --------------------------------- guilabel (`sphinx.roles.GUILabel`): . {guilabel}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <inline classes="guilabel" rawtext=":guilabel:`a`"> a . --------------------------------- menuselection (`sphinx.roles.MenuSelection`): . {menuselection}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <inline classes="menuselection" rawtext=":menuselection:`a`"> a . --------------------------------- file (`sphinx.roles.EmphasizedLiteral`): . {file}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <literal classes="file" role="file"> a . --------------------------------- samp (`sphinx.roles.EmphasizedLiteral`): . {samp}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <literal classes="samp" role="samp"> a . --------------------------------- -abbr (`sphinx.roles.Abbreviation`): SKIP: Non-deterministic output . {abbr}`a` @@ -589,55 +539,50 @@ SKIP: Non-deterministic output a . --------------------------------- rst:dir (`sphinx.roles.XRefRole`): . {rst:dir}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="rst" refexplicit="False" reftarget="a" reftype="dir" refwarn="False"> + <pending_xref refdoc="index" refdomain="rst" refexplicit="False" reftarget="a" reftype="dir" refwarn="False"> <literal classes="xref rst rst-dir"> a . --------------------------------- token (`sphinx.roles.XRefRole`): . {token}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="token" refwarn="False"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="token" refwarn="False"> <literal classes="xref std std-token"> a . --------------------------------- term (`sphinx.roles.XRefRole`): . {term}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="term" refwarn="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="term" refwarn="True"> <inline classes="xref std std-term"> a . --------------------------------- ref (`sphinx.roles.XRefRole`): . {ref}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="ref" refwarn="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="ref" refwarn="True"> <inline classes="xref std std-ref"> a . --------------------------------- ref with line breaks (`sphinx.roles.XRefRole`): . {ref}`some @@ -646,45 +591,42 @@ text a custom reference>` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="True" reftarget="and a custom reference" reftype="ref" refwarn="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="True" reftarget="and a custom reference" reftype="ref" refwarn="True"> <inline classes="xref std std-ref"> some text . --------------------------------- numref (`sphinx.roles.XRefRole`): . {numref}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="numref" refwarn="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="numref" refwarn="True"> <literal classes="xref std std-numref"> a . --------------------------------- keyword (`sphinx.roles.XRefRole`): . {keyword}`a` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="False" reftarget="a" reftype="keyword" refwarn="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="False" reftarget="a" reftype="keyword" refwarn="True"> <literal classes="xref std std-keyword"> a . --------------------------------- doc (`sphinx.roles.XRefRole`): . {doc}`this lecture <heavy_tails>` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="std" refexplicit="True" reftarget="heavy_tails" reftype="doc" refwarn="True"> + <pending_xref refdoc="index" refdomain="std" refexplicit="True" reftarget="heavy_tails" reftype="doc" refwarn="True"> <inline classes="xref std std-doc"> this lecture . diff --git a/tests/test_renderers/fixtures/sphinx_syntax_elements.md b/tests/test_renderers/fixtures/sphinx_syntax_elements.md index d706780c..1ac085c2 100644 --- a/tests/test_renderers/fixtures/sphinx_syntax_elements.md +++ b/tests/test_renderers/fixtures/sphinx_syntax_elements.md @@ -1,20 +1,18 @@ ---------------------------- Raw . foo . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> foo . ---------------------------- Hard-break . foo\ bar . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> foo <raw format="html" xml:space="preserve"> @@ -24,44 +22,40 @@ bar bar . ---------------------------- Strong: . **foo** . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <strong> foo . ---------------------------- Emphasis . *foo* . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <emphasis> foo . ---------------------------- Escaped Emphasis: . \*foo* . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> *foo* . --------------------------- Mixed Inline . a *b* **c** `abc` \\* . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> a <emphasis> @@ -75,29 +69,26 @@ a *b* **c** `abc` \\* \* . --------------------------- Inline Code: . `foo` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <literal> foo . --------------------------- Heading: . # foo . -<document source="notset"> +<document source="<src>/index.md"> <section ids="foo" names="foo"> <title> foo . --------------------------- Heading Levels: . # a @@ -105,7 +96,7 @@ Heading Levels: ### c # d . -<document source="notset"> +<document source="<src>/index.md"> <section ids="a" names="a"> <title> a @@ -120,102 +111,92 @@ Heading Levels: d . - --------------------------- Block Code: . foo . -<document source="notset"> +<document source="<src>/index.md"> <literal_block language="none" xml:space="preserve"> foo . --------------------------- Fenced Code: . ```sh foo ``` . -<document source="notset"> +<document source="<src>/index.md"> <literal_block language="sh" xml:space="preserve"> foo . --------------------------- Fenced Code no language: . ``` foo ``` . -<document source="notset"> +<document source="<src>/index.md"> <literal_block language="default" xml:space="preserve"> foo . --------------------------- Fenced Code no language with trailing whitespace: . ``` foo ``` . -<document source="notset"> +<document source="<src>/index.md"> <literal_block language="default" xml:space="preserve"> foo . --------------------------- Image empty: . ![]() . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <image alt="" uri=""> . --------------------------- Image with alt and title: . ![alt](src "title") . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <image alt="alt" title="title" uri="src"> . --------------------------- Image with escapable html: . ![alt](http://www.google<>.com) . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <image alt="alt" uri="http://www.google%3C%3E.com"> . --------------------------- Block Quote: . > *foo* . -<document source="notset"> +<document source="<src>/index.md"> <block_quote> <paragraph> <emphasis> foo . --------------------------- Bullet List: . - *foo* * bar . -<document source="notset"> +<document source="<src>/index.md"> <bullet_list bullet="-"> <list_item> <paragraph> @@ -227,7 +208,6 @@ Bullet List: bar . --------------------------- Nested Bullets . - a @@ -235,7 +215,7 @@ Nested Bullets - c - d . -<document source="notset"> +<document source="<src>/index.md"> <bullet_list bullet="-"> <list_item> <paragraph> @@ -253,7 +233,6 @@ Nested Bullets d . --------------------------- Enumerated List: . 1. *foo* @@ -265,7 +244,7 @@ para 10. starting 11. enumerator . -<document source="notset"> +<document source="<src>/index.md"> <enumerated_list enumtype="arabic" prefix="" suffix="."> <list_item> <paragraph> @@ -286,14 +265,13 @@ para enumerator . --------------------------- Nested Enumrated List: . 1. a 2. b 1. c . -<document source="notset"> +<document source="<src>/index.md"> <enumerated_list enumtype="arabic" prefix="" suffix="."> <list_item> <paragraph> @@ -307,36 +285,32 @@ Nested Enumrated List: c . --------------------------- Sphinx Role containing backtick: . {code}``a=1{`}`` . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <literal classes="code"> a=1{`} . --------------------------- Target: . (target)= . -<document source="notset"> +<document source="<src>/index.md"> <target ids="target" names="target"> . --------------------------- Target with whitespace: . (target with space)= . -<document source="notset"> +<document source="<src>/index.md"> <target ids="target-with-space" names="target\ with\ space"> . --------------------------- Referencing: . (target)= @@ -352,35 +326,34 @@ Title [alt3](#target3) . -<document source="notset"> +<document source="<src>/index.md"> <target ids="target" names="target"> <section ids="title" names="title"> <title> Title <paragraph> - <pending_xref refdoc="mock_docname" refdomain="True" refexplicit="True" reftarget="target" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="target" reftype="myst" refwarn="True"> <inline classes="xref myst"> alt1 <paragraph> - <pending_xref refdoc="mock_docname" refdomain="True" refexplicit="False" reftarget="target2" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="target2" reftype="myst" refwarn="True"> <inline classes="xref myst"> <paragraph> <reference refuri="https://www.google.com"> alt2 <paragraph> - <pending_xref refdoc="mock_docname" refdomain="True" refexplicit="True" reftarget="#target3" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="#target3" reftype="myst" refwarn="True"> <inline classes="xref myst"> alt3 . --------------------------- Comments: . line 1 % a comment line 2 . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> line 1 <comment xml:space="preserve"> @@ -389,43 +362,39 @@ line 2 line 2 . --------------------------- Block Break: . +++ string . -<document source="notset"> +<document source="<src>/index.md"> <comment classes="block_break" xml:space="preserve"> string . --------------------------- Link Reference: . [name][key] [key]: https://www.google.com "a title" . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <reference refuri="https://www.google.com" title="a title"> name . --------------------------- Link Reference short version: . [name] [name]: https://www.google.com "a title" . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <reference refuri="https://www.google.com" title="a title"> name . --------------------------- Block Quotes: . ```{epigraph} @@ -434,7 +403,7 @@ a b*c* -- a**b** ``` . -<document source="notset"> +<document source="<src>/index.md"> <block_quote classes="epigraph"> <paragraph> a b @@ -446,7 +415,6 @@ a b*c* b . --------------------------- Link Definition in directive: . ```{note} @@ -455,15 +423,14 @@ Link Definition in directive: [a]: link . -<document source="notset"> +<document source="<src>/index.md"> <note> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="True" refexplicit="True" reftarget="link" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="link" reftype="myst" refwarn="True"> <inline classes="xref myst"> a . --------------------------- Link Definition in nested directives: . ```{note} @@ -479,11 +446,11 @@ Link Definition in nested directives: [ref2]: link ``` . -<document source="notset"> +<document source="<src>/index.md"> <note> <note> <paragraph> - <pending_xref refdoc="mock_docname" refdomain="True" refexplicit="True" reftarget="link" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="True" refexplicit="True" reftarget="link" reftype="myst" refwarn="True"> <inline classes="xref myst"> ref1 @@ -491,14 +458,13 @@ Link Definition in nested directives: <note> . --------------------------- Footnotes: . [^a] [^a]: footnote*text* . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <footnote_reference auto="1" ids="id1" refname="a"> <transition classes="footnotes"> @@ -509,7 +475,6 @@ Footnotes: text . --------------------------- Footnotes nested blocks: . [^a] @@ -527,7 +492,7 @@ xyz finish . -<document source="notset"> +<document source="<src>/index.md"> <paragraph> <footnote_reference auto="1" ids="id1" refname="a"> <paragraph> @@ -553,7 +518,6 @@ finish c . --------------------------- Front Matter: . --- @@ -563,7 +527,7 @@ c: d: 2 --- . -<document source="notset"> +<document source="<src>/index.md"> <field_list> <field> <field_name> @@ -588,7 +552,6 @@ c: {"d": 2} . --------------------------- Front Matter Biblio: . --- @@ -612,7 +575,7 @@ abstract: other: Something else --- . -<document source="notset"> +<document source="<src>/index.md"> <field_list> <field> <field_name> @@ -638,11 +601,11 @@ other: Something else <field_body> <paragraph> 1 Cedar Park Close - + Thundersley - + Essex - + <field> <field_name> contact @@ -685,16 +648,16 @@ other: Something else dedication <field_body> <paragraph> - To my + To my <emphasis> homies - + <field> <field_name> abstract <field_body> <paragraph> - Something something + Something something <strong> dark side @@ -707,37 +670,30 @@ other: Something else Something else . --------------------------- Front Matter Bad Yaml: . --- a: { --- . -<document source="notset"> - <system_message level="3" line="1" source="notset" type="ERROR"> +<document source="<src>/index.md"> + <system_message level="2" line="1" source="<src>/index.md" type="WARNING"> <paragraph> - Front matter block: - while parsing a flow node - expected the node content, but found '<stream end>' - in "<unicode string>", line 1, column 5: - a: { - ^ - <literal_block xml:space="preserve"> - a: { + Malformed YAML [myst.topmatter] . Front Matter HTML Meta . --- -html_meta: - keywords: Sphinx, documentation, builder - description lang=en: An amusing story - description lang=fr: Un histoire amusant - http-equiv=Content-Type: text/html; charset=ISO-8859-1 +myst: + html_meta: + keywords: Sphinx, documentation, builder + description lang=en: An amusing story + description lang=fr: Un histoire amusant + http-equiv=Content-Type: text/html; charset=ISO-8859-1 --- . -<document source="notset"> +<document source="<src>/index.md"> <pending> .. internal attributes: .transform: docutils.transforms.components.Filter @@ -772,7 +728,6 @@ html_meta: <meta content="text/html; charset=ISO-8859-1" http-equiv="Content-Type"> . --------------------------- Full Test: . --- @@ -799,7 +754,7 @@ a = 1 [](target) . -<document source="notset"> +<document source="<src>/index.md"> <field_list> <field> <field_name> @@ -840,6 +795,6 @@ a = 1 <literal_block language="::python" xml:space="preserve"> a = 1 <paragraph> - <pending_xref refdoc="mock_docname" refdomain="True" refexplicit="False" reftarget="target" reftype="myst" refwarn="True"> + <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="target" reftype="myst" refwarn="True"> <inline classes="xref myst"> . diff --git a/tests/test_renderers/fixtures/tables.md b/tests/test_renderers/fixtures/tables.md index 65238d3a..f24a86cc 100644 --- a/tests/test_renderers/fixtures/tables.md +++ b/tests/test_renderers/fixtures/tables.md @@ -1,11 +1,10 @@ --------------------------- Simple: . a|b -|- 1|2 . -<document source="notset"> +<document source="<src>/index.md"> <table classes="colwidths-auto"> <tgroup cols="2"> <colspec colwidth="50.0"> @@ -28,13 +27,12 @@ a|b 2 . --------------------------- Header only: . | abc | def | | --- | --- | . -<document source="notset"> +<document source="<src>/index.md"> <table classes="colwidths-auto"> <tgroup cols="2"> <colspec colwidth="50.0"> @@ -49,14 +47,13 @@ Header only: def . --------------------------- Aligned: . a | b | c :-|:-:| -: 1 | 2 | 3 . -<document source="notset"> +<document source="<src>/index.md"> <table classes="colwidths-auto"> <tgroup cols="3"> <colspec colwidth="33.33"> @@ -86,14 +83,13 @@ a | b | c 3 . --------------------------- Nested syntax: . | *a* | __*b*__ | | --- | -------- | |c | {sub}`x` | . -<document source="notset"> +<document source="<src>/index.md"> <table classes="colwidths-auto"> <tgroup cols="2"> <colspec colwidth="50.0"> @@ -120,14 +116,13 @@ Nested syntax: x . --------------------------- External links: . a|b |-|-| [link-a](https://www.google.com/)|[link-b](https://www.python.org/) . -<document source="notset"> +<document source="<src>/index.md"> <table classes="colwidths-auto"> <tgroup cols="2"> <colspec colwidth="50.0"> @@ -150,4 +145,4 @@ a|b <paragraph> <reference refuri="https://www.python.org/"> link-b -. \ No newline at end of file +. diff --git a/tests/test_renderers/test_error_reporting.py b/tests/test_renderers/test_error_reporting.py index 72327fdb..2547e85f 100644 --- a/tests/test_renderers/test_error_reporting.py +++ b/tests/test_renderers/test_error_reporting.py @@ -5,7 +5,7 @@ import pytest from docutils.core import publish_doctree -from myst_parser.docutils_ import Parser +from myst_parser.parsers.docutils_ import Parser FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") diff --git a/tests/test_renderers/test_fixtures_docutils.py b/tests/test_renderers/test_fixtures_docutils.py index b1897b1e..d80792f3 100644 --- a/tests/test_renderers/test_fixtures_docutils.py +++ b/tests/test_renderers/test_fixtures_docutils.py @@ -9,43 +9,68 @@ import pytest from docutils.core import Publisher, publish_doctree -from myst_parser.docutils_ import Parser -from myst_parser.docutils_renderer import DocutilsRenderer, make_document -from myst_parser.main import MdParserConfig, create_md_parser +from myst_parser.parsers.docutils_ import Parser FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") @pytest.mark.param_file(FIXTURE_PATH / "docutil_syntax_elements.md") -def test_syntax_elements(file_params): - parser = create_md_parser( - MdParserConfig(highlight_code_blocks=False), DocutilsRenderer +def test_syntax_elements(file_params, monkeypatch): + """Test conversion of Markdown to docutils AST (before transforms are applied).""" + + def _apply_transforms(self): + pass + + monkeypatch.setattr(Publisher, "apply_transforms", _apply_transforms) + + doctree = publish_doctree( + file_params.content, + source_path="notset", + parser=Parser(), + settings_overrides={"myst_highlight_code_blocks": False}, ) - parser.options["document"] = document = make_document() - parser.render(file_params.content) + # in docutils 0.18 footnote ids have changed - outcome = document.pformat().replace('"footnote-reference-1"', '"id1"') + outcome = doctree.pformat().replace('"footnote-reference-1"', '"id1"') file_params.assert_expected(outcome, rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "docutil_roles.md") -def test_docutils_roles(file_params): - """Test output of docutils roles.""" - parser = create_md_parser(MdParserConfig(), DocutilsRenderer) - parser.options["document"] = document = make_document() - parser.render(file_params.content) - file_params.assert_expected(document.pformat(), rstrip_lines=True) +def test_docutils_roles(file_params, monkeypatch): + """Test conversion of Markdown to docutils AST (before transforms are applied).""" + + def _apply_transforms(self): + pass + + monkeypatch.setattr(Publisher, "apply_transforms", _apply_transforms) + + doctree = publish_doctree( + file_params.content, + source_path="notset", + parser=Parser(), + ) + + file_params.assert_expected(doctree.pformat(), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "docutil_directives.md") -def test_docutils_directives(file_params): +def test_docutils_directives(file_params, monkeypatch): """Test output of docutils directives.""" if "SKIP" in file_params.description: # line-block directive not yet supported pytest.skip(file_params.description) - parser = create_md_parser(MdParserConfig(), DocutilsRenderer) - parser.options["document"] = document = make_document() - parser.render(file_params.content) - file_params.assert_expected(document.pformat(), rstrip_lines=True) + + def _apply_transforms(self): + pass + + monkeypatch.setattr(Publisher, "apply_transforms", _apply_transforms) + + doctree = publish_doctree( + file_params.content, + source_path="notset", + parser=Parser(), + ) + + file_params.assert_expected(doctree.pformat(), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "docutil_syntax_extensions.txt") diff --git a/tests/test_renderers/test_fixtures_sphinx.py b/tests/test_renderers/test_fixtures_sphinx.py index 11427806..abb57aa1 100644 --- a/tests/test_renderers/test_fixtures_sphinx.py +++ b/tests/test_renderers/test_fixtures_sphinx.py @@ -2,44 +2,44 @@ Note, the output AST is before any transforms are applied. """ +from __future__ import annotations + import re import sys from pathlib import Path import pytest import sphinx +from sphinx_pytest.plugin import CreateDoctree -from myst_parser.main import MdParserConfig, to_docutils -from myst_parser.sphinx_renderer import SphinxRenderer, mock_sphinx_env +from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") -def test_minimal_sphinx(): - with mock_sphinx_env(conf={"author": "bob geldof"}, with_builder=True) as app: - assert app.config["author"] == "bob geldof" - - @pytest.mark.param_file(FIXTURE_PATH / "sphinx_syntax_elements.md") -def test_syntax_elements(file_params): - document = to_docutils(file_params.content, in_sphinx_env=True) - file_params.assert_expected(document.pformat(), rstrip_lines=True) +def test_syntax_elements(file_params, sphinx_doctree_no_tr: CreateDoctree): + sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "tables.md") -def test_tables(file_params): - document = to_docutils(file_params.content, in_sphinx_env=True) - file_params.assert_expected(document.pformat(), rstrip_lines=True) +def test_tables(file_params, sphinx_doctree_no_tr: CreateDoctree): + sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "directive_options.md") -def test_directive_options(file_params): - document = to_docutils(file_params.content) - file_params.assert_expected(document.pformat(), rstrip_lines=True) +def test_directive_options(file_params, sphinx_doctree_no_tr: CreateDoctree): + sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "sphinx_directives.md") -def test_sphinx_directives(file_params): +def test_sphinx_directives(file_params, sphinx_doctree_no_tr: CreateDoctree): # TODO fix skipped directives # TODO test domain directives if file_params.title.startswith("SKIP"): @@ -48,77 +48,77 @@ def test_sphinx_directives(file_params): pytest.skip(file_params.title) elif file_params.title.startswith("SPHINX4") and sphinx.version_info[0] < 4: pytest.skip(file_params.title) - document = to_docutils(file_params.content, in_sphinx_env=True).pformat() + + sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) + pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index") # see https://github.com/sphinx-doc/sphinx/issues/9827 - document = document.replace('<glossary sorted="False">', "<glossary>") + pformat = pformat.replace('<glossary sorted="False">', "<glossary>") # see https://github.com/executablebooks/MyST-Parser/issues/522 if sys.maxsize == 2147483647: - document = document.replace('"2147483647"', '"9223372036854775807"') - file_params.assert_expected(document, rstrip_lines=True) + pformat = pformat.replace('"2147483647"', '"9223372036854775807"') + file_params.assert_expected(pformat, rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "sphinx_roles.md") -def test_sphinx_roles(file_params): +def test_sphinx_roles(file_params, sphinx_doctree_no_tr: CreateDoctree): if file_params.title.startswith("SKIP"): pytest.skip(file_params.title) elif file_params.title.startswith("SPHINX4") and sphinx.version_info[0] < 4: pytest.skip(file_params.title) - document = to_docutils(file_params.content, in_sphinx_env=True) - actual = document.pformat() + + sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) + pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index") # sphinx 3 adds a parent key - actual = re.sub('cpp:parent_key="[^"]*"', 'cpp:parent_key=""', actual) + pformat = re.sub('cpp:parent_key="[^"]*"', 'cpp:parent_key=""', pformat) # sphinx >= 4.5.0 adds a trailing slash to PEP URLs, # see https://github.com/sphinx-doc/sphinx/commit/658689433eacc9eb - actual = actual.replace( + pformat = pformat.replace( ' refuri="http://www.python.org/dev/peps/pep-0001">', ' refuri="http://www.python.org/dev/peps/pep-0001/">', ) - file_params.assert_expected(actual, rstrip_lines=True) + file_params.assert_expected(pformat, rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "dollarmath.md") -def test_dollarmath(file_params, monkeypatch): - document = to_docutils( - file_params.content, - MdParserConfig(enable_extensions=["dollarmath"]), - in_sphinx_env=True, +def test_dollarmath(file_params, sphinx_doctree_no_tr: CreateDoctree): + sphinx_doctree_no_tr.set_conf( + {"extensions": ["myst_parser"], "myst_enable_extensions": ["dollarmath"]} ) - file_params.assert_expected(document.pformat(), rstrip_lines=True) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "amsmath.md") -def test_amsmath(file_params, monkeypatch): +def test_amsmath(file_params, sphinx_doctree_no_tr: CreateDoctree, monkeypatch): monkeypatch.setattr(SphinxRenderer, "_random_label", lambda self: "mock-uuid") - document = to_docutils( - file_params.content, - MdParserConfig(enable_extensions=["amsmath"]), - in_sphinx_env=True, + sphinx_doctree_no_tr.set_conf( + {"extensions": ["myst_parser"], "myst_enable_extensions": ["amsmath"]} ) - file_params.assert_expected(document.pformat(), rstrip_lines=True) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "containers.md") -def test_containers(file_params, monkeypatch): +def test_containers(file_params, sphinx_doctree_no_tr: CreateDoctree, monkeypatch): monkeypatch.setattr(SphinxRenderer, "_random_label", lambda self: "mock-uuid") - document = to_docutils( - file_params.content, - MdParserConfig(enable_extensions=["colon_fence"]), - in_sphinx_env=True, + sphinx_doctree_no_tr.set_conf( + {"extensions": ["myst_parser"], "myst_enable_extensions": ["colon_fence"]} ) - file_params.assert_expected(document.pformat(), rstrip_lines=True) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "eval_rst.md") -def test_evalrst_elements(file_params): - document = to_docutils(file_params.content, in_sphinx_env=True) - file_params.assert_expected(document.pformat(), rstrip_lines=True) +def test_evalrst_elements(file_params, sphinx_doctree_no_tr: CreateDoctree): + sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) @pytest.mark.param_file(FIXTURE_PATH / "definition_lists.md") -def test_definition_lists(file_params): - document = to_docutils( - file_params.content, - MdParserConfig(enable_extensions=["deflist"]), - in_sphinx_env=True, +def test_definition_lists(file_params, sphinx_doctree_no_tr: CreateDoctree): + sphinx_doctree_no_tr.set_conf( + {"extensions": ["myst_parser"], "myst_enable_extensions": ["deflist"]} ) - file_params.assert_expected(document.pformat(), rstrip_lines=True) + result = sphinx_doctree_no_tr(file_params.content, "index.md") + file_params.assert_expected(result.pformat("index"), rstrip_lines=True) diff --git a/tests/test_renderers/test_include_directive.py b/tests/test_renderers/test_include_directive.py index 20e1a9d6..f02b246f 100644 --- a/tests/test_renderers/test_include_directive.py +++ b/tests/test_renderers/test_include_directive.py @@ -1,44 +1,52 @@ import os +from io import StringIO from pathlib import Path import pytest +from docutils.core import publish_doctree -from myst_parser.docutils_renderer import make_document -from myst_parser.main import to_docutils +from myst_parser.parsers.docutils_ import Parser FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") @pytest.mark.param_file(FIXTURE_PATH / "mock_include.md") -def test_render(file_params, tmp_path): +def test_render(file_params, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + tmp_path.joinpath("other.md").write_text("a\nb\nc") tmp_path.joinpath("fmatter.md").write_text("---\na: 1\n---\nb") - document = make_document(str(tmp_path / "test.md")) - to_docutils( - file_params.content, document=document, in_sphinx_env=True, srcdir=str(tmp_path) + + doctree = publish_doctree( + file_params.content, + parser=Parser(), + settings_overrides={"myst_highlight_code_blocks": False}, ) - output = document.pformat().replace(str(tmp_path) + os.sep, "tmpdir" + "/").rstrip() + + doctree["source"] = "tmpdir/test.md" + output = doctree.pformat().replace(str(tmp_path) + os.sep, "tmpdir" + "/").rstrip() + file_params.assert_expected(output, rstrip=True) @pytest.mark.param_file(FIXTURE_PATH / "mock_include_errors.md") -def test_errors(file_params, tmp_path): +def test_errors(file_params, tmp_path, monkeypatch): if file_params.title.startswith("Non-existent path") and os.name == "nt": pytest.skip("tmp_path not converted correctly on Windows") + monkeypatch.chdir(tmp_path) + tmp_path.joinpath("bad.md").write_text("{a}`b`") - document = make_document(str(tmp_path / "test.md")) - messages = [] - - def observer(msg_node): - if msg_node["level"] > 1: - messages.append( - msg_node.astext().replace(str(tmp_path) + os.sep, "tmpdir" + "/") - ) - - document.reporter.attach_observer(observer) - document.reporter.halt_level = 6 - to_docutils( - file_params.content, document=document, in_sphinx_env=True, srcdir=str(tmp_path) + + report_stream = StringIO() + publish_doctree( + file_params.content, + source_path=str(tmp_path / "test.md"), + parser=Parser(), + settings_overrides={"halt_level": 6, "warning_stream": report_stream}, + ) + + file_params.assert_expected( + report_stream.getvalue().replace(str(tmp_path) + os.sep, "tmpdir" + "/"), + rstrip=True, ) - file_params.assert_expected("\n".join(messages), rstrip=True) diff --git a/tests/test_renderers/test_myst_config.py b/tests/test_renderers/test_myst_config.py index fee43de3..0f58cd76 100644 --- a/tests/test_renderers/test_myst_config.py +++ b/tests/test_renderers/test_myst_config.py @@ -6,7 +6,7 @@ import pytest from docutils.core import Publisher, publish_doctree -from myst_parser.docutils_ import Parser +from myst_parser.parsers.docutils_ import Parser FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") diff --git a/tests/test_renderers/test_myst_refs.py b/tests/test_renderers/test_myst_refs.py index c56b256c..083b34a2 100644 --- a/tests/test_renderers/test_myst_refs.py +++ b/tests/test_renderers/test_myst_refs.py @@ -1,12 +1,5 @@ -import os - import pytest -from sphinx.application import Sphinx -from sphinx.errors import SphinxWarning - -from myst_parser.main import MdParserConfig -from myst_parser.sphinx_parser import parse -from myst_parser.sphinx_renderer import mock_sphinx_env +from sphinx_pytest.plugin import CreateDoctree @pytest.mark.parametrize( @@ -23,24 +16,23 @@ ("ref_colon", "(ref:colon)=\n# Title\n[](ref:colon)", False), ], ) -def test_parse(test_name, text, should_warn, file_regression): +def test_parse( + test_name: str, + text: str, + should_warn: bool, + sphinx_doctree: CreateDoctree, + file_regression, +): + sphinx_doctree.set_conf({"extensions": ["myst_parser"]}) + result = sphinx_doctree(text, "index.md") + assert not result.warnings - with mock_sphinx_env( - conf={"extensions": ["myst_parser"]}, - srcdir="root", - with_builder=True, - raise_on_warning=True, - ) as app: # type: Sphinx - app.env.myst_config = MdParserConfig() - document = parse(app, text, docname="index") - if should_warn: - with pytest.raises(SphinxWarning): - app.env.apply_post_transforms(document, "index") - else: - app.env.apply_post_transforms(document, "index") + doctree = result.get_resolved_doctree("index") - content = document.pformat() - # windows fix - content = content.replace("root" + os.sep + "index.md", "root/index.md") + if should_warn: + assert result.warnings + else: + assert not result.warnings - file_regression.check(content, basename=test_name, extension=".xml") + doctree["source"] = "root/index.md" + file_regression.check(doctree.pformat(), basename=test_name, extension=".xml") diff --git a/tests/test_renderers/test_myst_refs/duplicate.xml b/tests/test_renderers/test_myst_refs/duplicate.xml index 35d4e8df..755906c8 100644 --- a/tests/test_renderers/test_myst_refs/duplicate.xml +++ b/tests/test_renderers/test_myst_refs/duplicate.xml @@ -1,7 +1,9 @@ -<document ids="title index" names="title index" source="root/index.md" title="Title"> - <title> - Title +<document source="root/index.md"> <target refid="index"> - <paragraph> - <pending_xref refdoc="index" refdomain="True" refexplicit="False" reftarget="index" reftype="myst" refwarn="True"> - <inline classes="xref myst"> + <section ids="title index" names="title index"> + <title> + Title + <paragraph> + <reference internal="True" refid="index"> + <inline classes="std std-ref"> + Title diff --git a/tests/test_renderers/test_myst_refs/missing.xml b/tests/test_renderers/test_myst_refs/missing.xml index 6c4b5be3..6bc72ade 100644 --- a/tests/test_renderers/test_myst_refs/missing.xml +++ b/tests/test_renderers/test_myst_refs/missing.xml @@ -1,4 +1,3 @@ <document source="root/index.md"> <paragraph> - <pending_xref refdoc="index" refdomain="" refexplicit="False" reftarget="ref" reftype="myst" refwarn="True"> - <inline classes="xref myst"> + <inline classes="xref myst"> diff --git a/tests/test_renderers/test_myst_refs/ref.xml b/tests/test_renderers/test_myst_refs/ref.xml index 78e2a49e..e4ae200d 100644 --- a/tests/test_renderers/test_myst_refs/ref.xml +++ b/tests/test_renderers/test_myst_refs/ref.xml @@ -1,8 +1,9 @@ -<document ids="title ref" names="title ref" source="root/index.md" title="Title"> - <title> - Title +<document source="root/index.md"> <target refid="ref"> - <paragraph> - <reference internal="True" refid="ref"> - <inline classes="std std-ref"> - Title + <section ids="title ref" names="title ref"> + <title> + Title + <paragraph> + <reference internal="True" refid="ref"> + <inline classes="std std-ref"> + Title diff --git a/tests/test_renderers/test_myst_refs/ref_colon.xml b/tests/test_renderers/test_myst_refs/ref_colon.xml index affccd4a..f1e9923b 100644 --- a/tests/test_renderers/test_myst_refs/ref_colon.xml +++ b/tests/test_renderers/test_myst_refs/ref_colon.xml @@ -1,8 +1,9 @@ -<document ids="title ref-colon" names="title ref:colon" source="root/index.md" title="Title"> - <title> - Title +<document source="root/index.md"> <target refid="ref-colon"> - <paragraph> - <reference internal="True" refid="ref-colon"> - <inline classes="std std-ref"> - Title + <section ids="title ref-colon" names="title ref:colon"> + <title> + Title + <paragraph> + <reference internal="True" refid="ref-colon"> + <inline classes="std std-ref"> + Title diff --git a/tests/test_renderers/test_myst_refs/ref_nested.xml b/tests/test_renderers/test_myst_refs/ref_nested.xml index 34f99c83..be69ef22 100644 --- a/tests/test_renderers/test_myst_refs/ref_nested.xml +++ b/tests/test_renderers/test_myst_refs/ref_nested.xml @@ -1,9 +1,10 @@ -<document ids="title ref" names="title ref" source="root/index.md" title="Title"> - <title> - Title +<document source="root/index.md"> <target refid="ref"> - <paragraph> - <reference internal="True" refid="ref"> - <inline classes="std std-ref"> - <emphasis> - text + <section ids="title ref" names="title ref"> + <title> + Title + <paragraph> + <reference internal="True" refid="ref"> + <inline classes="std std-ref"> + <emphasis> + text diff --git a/tests/test_renderers/test_parse_directives.py b/tests/test_renderers/test_parse_directives.py index c85e6637..068a3d77 100644 --- a/tests/test_renderers/test_parse_directives.py +++ b/tests/test_renderers/test_parse_directives.py @@ -3,7 +3,7 @@ from docutils.parsers.rst.directives.admonitions import Note from docutils.parsers.rst.directives.body import Rubric -from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text +from myst_parser.parsers.directives import DirectiveParsingError, parse_directive_text @pytest.mark.parametrize( diff --git a/tests/test_sphinx/conftest.py b/tests/test_sphinx/conftest.py index 7930b651..e7fb75a8 100644 --- a/tests/test_sphinx/conftest.py +++ b/tests/test_sphinx/conftest.py @@ -66,7 +66,7 @@ def read( outpath = path(os.path.join(str(app.srcdir), "_build", buildername, filename)) if not outpath.exists(): - raise IOError("no output file exists: {}".format(outpath)) + raise OSError(f"no output file exists: {outpath}") try: # introduced in sphinx 3.0 diff --git a/tests/test_sphinx/sourcedirs/substitutions/index.md b/tests/test_sphinx/sourcedirs/substitutions/index.md index 5c8aa727..e1dd672a 100644 --- a/tests/test_sphinx/sourcedirs/substitutions/index.md +++ b/tests/test_sphinx/sourcedirs/substitutions/index.md @@ -1,22 +1,23 @@ --- -substitutions: - text: "- text" - text_with_nest: > - output - with *Markdown* - {{ nested }} - nested: nested substitution - admonition: | - prefix - - ```{note} - A note {{ nested }} - ``` - inline_admonition: | - ```{note} - Inline note - ``` - override: Overridden by front matter +myst: + substitutions: + text: "- text" + text_with_nest: > + output + with *Markdown* + {{ nested }} + nested: nested substitution + admonition: | + prefix + + ```{note} + A note {{ nested }} + ``` + inline_admonition: | + ```{note} + Inline note + ``` + override: Overridden by front matter --- diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py index e3025bcd..e8615a64 100644 --- a/tests/test_sphinx/test_sphinx_builds.py +++ b/tests/test_sphinx/test_sphinx_builds.py @@ -203,7 +203,7 @@ def test_extended_syntaxes( monkeypatch, ): """test setting addition configuration values.""" - from myst_parser.sphinx_renderer import SphinxRenderer + from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer monkeypatch.setattr(SphinxRenderer, "_random_label", lambda self: "mock-uuid") app.build()