✨ NEW: Add file load option for code-cells

docs/use/markdown.md

@@ -103,6 +103,22 @@ b = "Python code!"
 print(f"{a} {b}")
 ```
 
+**Code from Files:**
+
+`myst_nb` provides a convenience feature for importing executable code into a `{code-cell}`
+from a file. This can be useful when you want to share code between documents. To do this
+you specify a `load` metadata attribute such as:
+
+````md
+```{code-cell} ipython3
+:load: <path>
+```
+````
+
+```{warning}
+This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive.
+```
+
 ### Syntax for markdown
 
 Anything in-between code cells will be treated as markdown. You can use any markdown

myst_nb/converter.py

@@ -2,10 +2,11 @@
 from typing import Callable, Iterable, Optional
 
 import attr
+from pathlib import Path
 
 import nbformat as nbf
 from sphinx.environment import BuildEnvironment
-from sphinx.util import import_object
+from sphinx.util import import_object, logging
 import yaml
 
 from myst_parser.main import MdParserConfig
@@ -14,6 +15,8 @@
 CODE_DIRECTIVE = "{code-cell}"
 RAW_DIRECTIVE = "{raw-cell}"
 
+LOGGER = logging.getLogger(__name__)
+
 
 @attr.s
 class NbConverter:
@@ -55,18 +58,26 @@ def get_nb_converter(
 
     # If there is no source text then we assume a MyST Notebook
     if source_iter is None:
+        # Check if docname exists
         return NbConverter(
             lambda text: myst_to_notebook(
-                text, config=env.myst_config, add_source_map=True
+                text,
+                config=env.myst_config,
+                add_source_map=True,
+                path=path,
             ),
             env.myst_config,
         )
 
     # Given the source lines, we check it can be recognised as a MyST Notebook
     if is_myst_notebook(source_iter):
+        # Check if docname exists
         return NbConverter(
             lambda text: myst_to_notebook(
-                text, config=env.myst_config, add_source_map=True
+                text,
+                config=env.myst_config,
+                add_source_map=True,
+                path=path,
             ),
             env.myst_config,
         )
@@ -120,6 +131,10 @@ class MystMetadataParsingError(Exception):
     """Error when parsing metadata from myst formatted text"""
 
 
+class LoadFileParsingError(Exception):
+    """Error when parsing files for code-blocks/code-cells"""
+
+
 def strip_blank_lines(text):
     text = text.rstrip()
     while text and text.startswith("\n"):
@@ -174,12 +189,32 @@ def read_cell_metadata(token, cell_index):
     return metadata
 
 
+def load_code_from_file(nb_path, file_name, token, body_lines):
+    """load source code from a file."""
+    if nb_path is None:
+        raise LoadFileParsingError("path to notebook not supplied for :load:")
+    file_path = Path(nb_path).parent.joinpath(file_name).resolve()
+    if len(body_lines):
+        line = token.map[0] if token.map else 0
+        msg = (
+            f"{nb_path}:{line} content of code-cell is being overwritten by "
+            f":load: {file_name}"
+        )
+        LOGGER.warning(msg)
+    try:
+        body_lines = file_path.read_text().split("\n")
+    except Exception:
+        raise LoadFileParsingError("Can't read file from :load: {}".format(file_path))
+    return body_lines
+
+
 def myst_to_notebook(
     text,
     config: MdParserConfig,
     code_directive=CODE_DIRECTIVE,
     raw_directive=RAW_DIRECTIVE,
     add_source_map=False,
+    path: Optional[str] = None,
 ):
     """Convert text written in the myst format to a notebook.
 
@@ -188,6 +223,7 @@ def myst_to_notebook(
     :param raw_directive: the name of the directive to search for containing raw cells
     :param add_source_map: add a `source_map` key to the notebook metadata,
         which is a list of the starting source line number for each cell.
+    :param path: path to notebook (required for :load:)
 
     :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML
 
@@ -248,6 +284,11 @@ def _flush_markdown(start_line, token, md_metadata):
         if token.type == "fence" and token.info.startswith(code_directive):
             _flush_markdown(md_start_line, token, md_metadata)
             options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code")
+            # Parse :load: or load: tags and populate body with contents of file
+            if "load" in options:
+                body_lines = load_code_from_file(
+                    path, options["load"], token, body_lines
+                )
             meta = nbf.from_dict(options)
             source_map.append(token.map[0] + 1)
             notebook.cells.append(

myst_nb/parser.py

@@ -49,7 +49,7 @@ def parse(self, inputstring: str, document: nodes.document):
         self.env = document.settings.env  # type: BuildEnvironment
 
         converter = get_nb_converter(
-            self.env.doc2path(self.env.docname, False),
+            self.env.doc2path(self.env.docname, True),
             self.env,
             inputstring.splitlines(keepends=True),
         )

tests/nb_fixtures/reporter_warnings.txt

@@ -66,5 +66,5 @@ cells:
 
         [a]: c
 .
-source/path:20004: (WARNING/2) Duplicate reference definition: A
+source/path:20004: (WARNING/2) Duplicate reference definition: A [myst.ref]
 .

tests/notebooks/mystnb_codecell_file.md

@@ -0,0 +1,17 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+author: Matt
+---
+
+# a title
+
+```{code-cell} ipython3
+:load: mystnb_codecell_file.py
+```

tests/notebooks/mystnb_codecell_file.py

@@ -0,0 +1,3 @@
+# flake8: noqa
+
+import numpy as np

tests/notebooks/mystnb_codecell_file_warnings.md

@@ -0,0 +1,19 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+kernelspec:
+  display_name: Python 3
+  language: python
+  name: python3
+author: Aakash
+---
+
+# a title
+
+```{code-cell} ipython3
+:load: mystnb_codecell_file.py
+i = 10
+print(i)
+```

tests/test_mystnb_features.py

@@ -0,0 +1,68 @@
+import pytest
+
+from sphinx.util.fileutil import copy_asset_file
+
+
+@pytest.mark.sphinx_params(
+    "mystnb_codecell_file.md",
+    conf={"jupyter_execute_notebooks": "cache", "source_suffix": {".md": "myst-nb"}},
+)
+def test_codecell_file(sphinx_run, file_regression, check_nbs, get_test_path):
+    asset_path = get_test_path("mystnb_codecell_file.py")
+    copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir))
+    sphinx_run.build()
+    assert sphinx_run.warnings() == ""
+    assert set(sphinx_run.app.env.metadata["mystnb_codecell_file"].keys()) == {
+        "jupytext",
+        "kernelspec",
+        "author",
+        "source_map",
+        "language_info",
+    }
+    assert sphinx_run.app.env.metadata["mystnb_codecell_file"]["author"] == "Matt"
+    assert (
+        sphinx_run.app.env.metadata["mystnb_codecell_file"]["kernelspec"]
+        == '{"display_name": "Python 3", "language": "python", "name": "python3"}'
+    )
+    file_regression.check(
+        sphinx_run.get_nb(), check_fn=check_nbs, extension=".ipynb", encoding="utf8"
+    )
+    file_regression.check(
+        sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8"
+    )
+
+
+@pytest.mark.sphinx_params(
+    "mystnb_codecell_file_warnings.md",
+    conf={"jupyter_execute_notebooks": "force", "source_suffix": {".md": "myst-nb"}},
+)
+def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test_path):
+    asset_path = get_test_path("mystnb_codecell_file.py")
+    copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir))
+    sphinx_run.build()
+    assert (
+        "mystnb_codecell_file_warnings.md:14 content of code-cell "
+        "is being overwritten by :load: mystnb_codecell_file.py"
+        in sphinx_run.warnings()
+    )
+    assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == {
+        "jupytext",
+        "kernelspec",
+        "author",
+        "source_map",
+        "language_info",
+    }
+    assert (
+        sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["author"]
+        == "Aakash"
+    )
+    assert (
+        sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["kernelspec"]
+        == '{"display_name": "Python 3", "language": "python", "name": "python3"}'
+    )
+    file_regression.check(
+        sphinx_run.get_nb(), check_fn=check_nbs, extension=".ipynb", encoding="utf8"
+    )
+    file_regression.check(
+        sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8"
+    )

tests/test_mystnb_features/test_codecell_file.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# a title"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "file": "mystnb_codecell_file.py"
+   },
+   "outputs": [],
+   "source": [
+    "# flake8: noqa\n",
+    "\n",
+    "import numpy as np\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "author": "Matt",
+  "jupytext": {
+   "text_representation": {
+    "extension": ".md",
+    "format_name": "myst"
+   }
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.9"
+  },
+  "source_map": [
+   11,
+   15
+  ]
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

tests/test_mystnb_features/test_codecell_file.xml

@@ -0,0 +1,10 @@
+<document source="mystnb_codecell_file">
+    <section ids="a-title" names="a\ title">
+        <title>
+            a title
+        <CellNode cell_type="code" classes="cell">
+            <CellInputNode classes="cell_input">
+                <literal_block language="ipython3" xml:space="preserve">
+                    # flake8: noqa
+
+                    import numpy as np

tests/test_mystnb_features/test_codecell_file_warnings.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# a title"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "file": "mystnb_codecell_file.py"
+   },
+   "outputs": [],
+   "source": [
+    "# flake8: noqa\n",
+    "\n",
+    "import numpy as np\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "author": "Aakash",
+  "jupytext": {
+   "text_representation": {
+    "extension": ".md",
+    "format_name": "myst"
+   }
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  },
+  "source_map": [
+   11,
+   15
+  ]
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}