diff --git a/docs/use/markdown.md b/docs/use/markdown.md index 98ce1f24..c4442abf 100644 --- a/docs/use/markdown.md +++ b/docs/use/markdown.md @@ -103,6 +103,22 @@ b = "Python code!" print(f"{a} {b}") ``` +**Code from Files:** + +`myst_nb` provides a convenience feature for importing executable code into a `{code-cell}` +from a file. This can be useful when you want to share code between documents. To do this +you specify a `load` metadata attribute such as: + +````md +```{code-cell} ipython3 +:load: +``` +```` + +```{warning} +This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive. +``` + ### Syntax for markdown Anything in-between code cells will be treated as markdown. You can use any markdown diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 52b6bc3c..0a78b41b 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -2,10 +2,11 @@ from typing import Callable, Iterable, Optional import attr +from pathlib import Path import nbformat as nbf from sphinx.environment import BuildEnvironment -from sphinx.util import import_object +from sphinx.util import import_object, logging import yaml from myst_parser.main import MdParserConfig @@ -14,6 +15,8 @@ CODE_DIRECTIVE = "{code-cell}" RAW_DIRECTIVE = "{raw-cell}" +LOGGER = logging.getLogger(__name__) + @attr.s class NbConverter: @@ -55,18 +58,26 @@ def get_nb_converter( # If there is no source text then we assume a MyST Notebook if source_iter is None: + # Check if docname exists return NbConverter( lambda text: myst_to_notebook( - text, config=env.myst_config, add_source_map=True + text, + config=env.myst_config, + add_source_map=True, + path=path, ), env.myst_config, ) # Given the source lines, we check it can be recognised as a MyST Notebook if is_myst_notebook(source_iter): + # Check if docname exists return NbConverter( lambda text: myst_to_notebook( - text, config=env.myst_config, add_source_map=True + text, + config=env.myst_config, + add_source_map=True, + path=path, ), env.myst_config, ) @@ -120,6 +131,10 @@ class MystMetadataParsingError(Exception): """Error when parsing metadata from myst formatted text""" +class LoadFileParsingError(Exception): + """Error when parsing files for code-blocks/code-cells""" + + def strip_blank_lines(text): text = text.rstrip() while text and text.startswith("\n"): @@ -174,12 +189,32 @@ def read_cell_metadata(token, cell_index): return metadata +def load_code_from_file(nb_path, file_name, token, body_lines): + """load source code from a file.""" + if nb_path is None: + raise LoadFileParsingError("path to notebook not supplied for :load:") + file_path = Path(nb_path).parent.joinpath(file_name).resolve() + if len(body_lines): + line = token.map[0] if token.map else 0 + msg = ( + f"{nb_path}:{line} content of code-cell is being overwritten by " + f":load: {file_name}" + ) + LOGGER.warning(msg) + try: + body_lines = file_path.read_text().split("\n") + except Exception: + raise LoadFileParsingError("Can't read file from :load: {}".format(file_path)) + return body_lines + + def myst_to_notebook( text, config: MdParserConfig, code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, + path: Optional[str] = None, ): """Convert text written in the myst format to a notebook. @@ -188,6 +223,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. + :param path: path to notebook (required for :load:) :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML @@ -248,6 +284,11 @@ def _flush_markdown(start_line, token, md_metadata): if token.type == "fence" and token.info.startswith(code_directive): _flush_markdown(md_start_line, token, md_metadata) options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code") + # Parse :load: or load: tags and populate body with contents of file + if "load" in options: + body_lines = load_code_from_file( + path, options["load"], token, body_lines + ) meta = nbf.from_dict(options) source_map.append(token.map[0] + 1) notebook.cells.append( diff --git a/myst_nb/parser.py b/myst_nb/parser.py index c7287a53..51d48136 100644 --- a/myst_nb/parser.py +++ b/myst_nb/parser.py @@ -49,7 +49,7 @@ def parse(self, inputstring: str, document: nodes.document): self.env = document.settings.env # type: BuildEnvironment converter = get_nb_converter( - self.env.doc2path(self.env.docname, False), + self.env.doc2path(self.env.docname, True), self.env, inputstring.splitlines(keepends=True), ) diff --git a/tests/nb_fixtures/reporter_warnings.txt b/tests/nb_fixtures/reporter_warnings.txt index b20287f9..3324011f 100644 --- a/tests/nb_fixtures/reporter_warnings.txt +++ b/tests/nb_fixtures/reporter_warnings.txt @@ -66,5 +66,5 @@ cells: [a]: c . -source/path:20004: (WARNING/2) Duplicate reference definition: A +source/path:20004: (WARNING/2) Duplicate reference definition: A [myst.ref] . \ No newline at end of file diff --git a/tests/notebooks/mystnb_codecell_file.md b/tests/notebooks/mystnb_codecell_file.md new file mode 100644 index 00000000..eae2841f --- /dev/null +++ b/tests/notebooks/mystnb_codecell_file.md @@ -0,0 +1,17 @@ +--- +jupytext: + text_representation: + extension: .md + format_name: myst +kernelspec: + display_name: Python 3 + language: python + name: python3 +author: Matt +--- + +# a title + +```{code-cell} ipython3 +:load: mystnb_codecell_file.py +``` diff --git a/tests/notebooks/mystnb_codecell_file.py b/tests/notebooks/mystnb_codecell_file.py new file mode 100644 index 00000000..efc9904d --- /dev/null +++ b/tests/notebooks/mystnb_codecell_file.py @@ -0,0 +1,3 @@ +# flake8: noqa + +import numpy as np diff --git a/tests/notebooks/mystnb_codecell_file_warnings.md b/tests/notebooks/mystnb_codecell_file_warnings.md new file mode 100644 index 00000000..4d243615 --- /dev/null +++ b/tests/notebooks/mystnb_codecell_file_warnings.md @@ -0,0 +1,19 @@ +--- +jupytext: + text_representation: + extension: .md + format_name: myst +kernelspec: + display_name: Python 3 + language: python + name: python3 +author: Aakash +--- + +# a title + +```{code-cell} ipython3 +:load: mystnb_codecell_file.py +i = 10 +print(i) +``` diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py new file mode 100644 index 00000000..7dc203b5 --- /dev/null +++ b/tests/test_mystnb_features.py @@ -0,0 +1,68 @@ +import pytest + +from sphinx.util.fileutil import copy_asset_file + + +@pytest.mark.sphinx_params( + "mystnb_codecell_file.md", + conf={"jupyter_execute_notebooks": "cache", "source_suffix": {".md": "myst-nb"}}, +) +def test_codecell_file(sphinx_run, file_regression, check_nbs, get_test_path): + asset_path = get_test_path("mystnb_codecell_file.py") + copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) + sphinx_run.build() + assert sphinx_run.warnings() == "" + assert set(sphinx_run.app.env.metadata["mystnb_codecell_file"].keys()) == { + "jupytext", + "kernelspec", + "author", + "source_map", + "language_info", + } + assert sphinx_run.app.env.metadata["mystnb_codecell_file"]["author"] == "Matt" + assert ( + sphinx_run.app.env.metadata["mystnb_codecell_file"]["kernelspec"] + == '{"display_name": "Python 3", "language": "python", "name": "python3"}' + ) + file_regression.check( + sphinx_run.get_nb(), check_fn=check_nbs, extension=".ipynb", encoding="utf8" + ) + file_regression.check( + sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8" + ) + + +@pytest.mark.sphinx_params( + "mystnb_codecell_file_warnings.md", + conf={"jupyter_execute_notebooks": "force", "source_suffix": {".md": "myst-nb"}}, +) +def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test_path): + asset_path = get_test_path("mystnb_codecell_file.py") + copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) + sphinx_run.build() + assert ( + "mystnb_codecell_file_warnings.md:14 content of code-cell " + "is being overwritten by :load: mystnb_codecell_file.py" + in sphinx_run.warnings() + ) + assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == { + "jupytext", + "kernelspec", + "author", + "source_map", + "language_info", + } + assert ( + sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["author"] + == "Aakash" + ) + assert ( + sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["kernelspec"] + == '{"display_name": "Python 3", "language": "python", "name": "python3"}' + ) + file_regression.check( + sphinx_run.get_nb(), check_fn=check_nbs, extension=".ipynb", encoding="utf8" + ) + file_regression.check( + sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8" + ) diff --git a/tests/test_mystnb_features/test_codecell_file.ipynb b/tests/test_mystnb_features/test_codecell_file.ipynb new file mode 100644 index 00000000..b89804b9 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file.ipynb @@ -0,0 +1,56 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# a title" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "file": "mystnb_codecell_file.py" + }, + "outputs": [], + "source": [ + "# flake8: noqa\n", + "\n", + "import numpy as np\n" + ] + } + ], + "metadata": { + "author": "Matt", + "jupytext": { + "text_representation": { + "extension": ".md", + "format_name": "myst" + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + }, + "source_map": [ + 11, + 15 + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_mystnb_features/test_codecell_file.xml b/tests/test_mystnb_features/test_codecell_file.xml new file mode 100644 index 00000000..545ad7b2 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file.xml @@ -0,0 +1,10 @@ + +
+ + a title + <CellNode cell_type="code" classes="cell"> + <CellInputNode classes="cell_input"> + <literal_block language="ipython3" xml:space="preserve"> + # flake8: noqa + + import numpy as np diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb new file mode 100644 index 00000000..37f6c148 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb @@ -0,0 +1,56 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# a title" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "file": "mystnb_codecell_file.py" + }, + "outputs": [], + "source": [ + "# flake8: noqa\n", + "\n", + "import numpy as np\n" + ] + } + ], + "metadata": { + "author": "Aakash", + "jupytext": { + "text_representation": { + "extension": ".md", + "format_name": "myst" + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "source_map": [ + 11, + 15 + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.xml b/tests/test_mystnb_features/test_codecell_file_warnings.xml new file mode 100644 index 00000000..5c43ddd7 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file_warnings.xml @@ -0,0 +1,10 @@ +<document source="mystnb_codecell_file_warnings"> + <section ids="a-title" names="a\ title"> + <title> + a title + <CellNode cell_type="code" classes="cell"> + <CellInputNode classes="cell_input"> + <literal_block language="ipython3" xml:space="preserve"> + # flake8: noqa + + import numpy as np