From 541a1109caa4ccb302bb6b7ae9b978e028c1a177 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Thu, 31 Dec 2020 10:41:21 +0000 Subject: [PATCH 01/20] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20UPDATE:=20myst-parse?= =?UTF-8?q?r~=3D0.13.1=20(#284)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/tests.yml | 1 + docs/conf.py | 12 +- docs/examples/basic.md | 14 +- docs/examples/custom-formats.Rmd | 2 +- myst_nb/__init__.py | 2 +- setup.py | 6 +- .../complex_outputs_unrun_timeout.ipynb | 463 ------------------ tests/notebooks/sleep_10.ipynb | 41 ++ .../notebooks/sleep_10_metadata_timeout.ipynb | 43 ++ tests/test_execute.py | 4 +- tox.ini | 5 +- 11 files changed, 113 insertions(+), 480 deletions(-) delete mode 100644 tests/notebooks/complex_outputs_unrun_timeout.ipynb create mode 100644 tests/notebooks/sleep_10.ipynb create mode 100644 tests/notebooks/sleep_10_metadata_timeout.ipynb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b999e43e..8cf61b07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,6 +23,7 @@ jobs: tests: strategy: + fail-fast: false matrix: os: [ubuntu-latest] python-version: [3.6, 3.7, 3.8] diff --git a/docs/conf.py b/docs/conf.py index f88e8702..890f6652 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,6 +61,7 @@ "repository_branch": "master", "use_edit_page_button": True, "path_to_docs": "docs/", + "show_navbar_depth": 2, } intersphinx_mapping = { @@ -96,10 +97,13 @@ execution_show_tb = "READTHEDOCS" in os.environ execution_timeout = 60 # Note: 30 was timing out on RTD -myst_admonition_enable = True -myst_amsmath_enable = True -myst_html_img_enable = True -myst_deflist_enable = True +myst_enable_extensions = [ + "amsmath", + "colon_fence", + "deflist", + "dollarmath", + "html_image", +] myst_url_schemes = ("http", "https", "mailto") panels_add_bootstrap_css = False diff --git a/docs/examples/basic.md b/docs/examples/basic.md index 9e331bef..f7b9de56 100644 --- a/docs/examples/basic.md +++ b/docs/examples/basic.md @@ -30,9 +30,13 @@ See the [MyST configuration options](myst:intro/config-options) for the full set To build documentation from this notebook, the following options are set: ```python -myst_admonition_enable = True -myst_amsmath_enable = True -myst_html_img_enable = True +myst_enable_extensions = [ + "amsmath", + "colon_fence", + "deflist", + "dollarmath", + "html_image", +] myst_url_schemes = ("http", "https", "mailto") ``` @@ -47,7 +51,7 @@ For example, here's the MyST-NB logo: ![myst-nb logo](../_static/logo.png) -With the `myst_html_img_enable=True` ([see here](myst:syntax/images)), you can even add HTML img tags with attributes: +By adding `"html_image"` to the `myst_enable_extensions` list in the sphinx configuration ([see here](myst:syntax/images)), you can even add HTML `img` tags with attributes: ```html logo @@ -70,7 +74,7 @@ It was generated with this code ([as explained here](myst:syntax/admonitions)): ::::: -If you wish to use "bare" LaTeX equations, then you should set `myst_amsmath_enable = True` in the sphinx configuration. +If you wish to use "bare" LaTeX equations, then you should add `"amsmath"` to the `myst_enable_extensions` list in the sphinx configuration. This is [explained here](myst:syntax/amsmath), and works as such: ```latex diff --git a/docs/examples/custom-formats.Rmd b/docs/examples/custom-formats.Rmd index a43afe7f..3d37c215 100644 --- a/docs/examples/custom-formats.Rmd +++ b/docs/examples/custom-formats.Rmd @@ -71,7 +71,7 @@ nb_custom_formats = { ``` :::{important} -For full compatibility with `myst-nb`, `jupytext>=1.6.0` should be used. +For full compatibility with `myst-nb`, `jupytext>=1.8.0` should be used. ::: For example: diff --git a/myst_nb/__init__.py b/myst_nb/__init__.py index c3763926..c0e2178f 100644 --- a/myst_nb/__init__.py +++ b/myst_nb/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.10.1" +__version__ = "0.11.0" from collections.abc import Sequence import os diff --git a/setup.py b/setup.py index 44301b49..d717cebc 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ python_requires=">=3.6", package_data={"myst_nb": ["_static/*"]}, install_requires=[ - "myst-parser~=0.12.9", + "myst-parser~=0.13.1", "docutils>=0.15", "sphinx>=2,<4", # TODO 0.3.2 requires some changes to the pytests @@ -74,10 +74,10 @@ "numpy", "sympy", "pandas", - "jupytext>=1.6.0", + "jupytext~=1.8.0", ], "rtd": [ - "jupytext>=1.6.0", + "jupytext~=1.8.0", "coconut~=1.4.3", "sphinxcontrib-bibtex", "ipywidgets", diff --git a/tests/notebooks/complex_outputs_unrun_timeout.ipynb b/tests/notebooks/complex_outputs_unrun_timeout.ipynb deleted file mode 100644 index 572fc144..00000000 --- a/tests/notebooks/complex_outputs_unrun_timeout.ipynb +++ /dev/null @@ -1,463 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "init_cell": true, - "slideshow": { - "slide_type": "skip" - } - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "pd.set_option('display.latex.repr', True)\n", - "import sympy as sym\n", - "sym.init_printing(use_latex=True)\n", - "import numpy as np\n", - "from IPython.display import Image, Latex" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "# Markdown" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "## General" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "Some markdown text.\n", - "\n", - "A list:\n", - "\n", - "- something\n", - "- something else\n", - "\n", - "A numbered list\n", - "\n", - "1. something\n", - "2. something else\n", - "\n", - "non-ascii characters TODO" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": {} - }, - "source": [ - "This is a long section of text, which we only want in a document (not a presentation)\n", - "some text\n", - "some more text\n", - "some more text\n", - "some more text\n", - "some more text\n", - "some more text\n", - "some more text\n", - "some more text\n", - "some more text\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true, - "slideonly": true - } - }, - "source": [ - "This is an abbreviated section of the document text, which we only want in a presentation\n", - "\n", - "- summary of document text" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "## References and Citations" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "References to \\cref{fig:example}, \\cref{tbl:example}, =@eqn:example_sympy and \\cref{code:example_mpl}.\n", - "\n", - "A latex citation.\\cite{zelenyak_molecular_2016}\n", - "\n", - "A html citation.(Kirkeminde, 2012) " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "## Todo notes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "slide": true - } - }, - "source": [ - "\\todo[inline]{an inline todo}\n", - "\n", - "Some text.\\todo{a todo in the margins}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Text Output" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ipub": { - "text": { - "format": { - "backgroundcolor": "\\color{blue!10}" - } - } - } - }, - "outputs": [], - "source": [ - "print(\"\"\"\n", - "This is some printed text,\n", - "with a nicely formatted output.\n", - "\"\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Images and Figures" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Displaying a plot with its code" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "caption": "fig:example_mpl" - } - }, - "source": [ - "A matplotlib figure, with the caption set in the markdowncell above the figure." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "caption": "code:example_mpl" - } - }, - "source": [ - "The plotting code for a matplotlib figure (\\cref{fig:example_mpl})." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "# Tables (with pandas)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "caption": "code:example_pd" - } - }, - "source": [ - "The plotting code for a pandas Dataframe table (\\cref{tbl:example})." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ipub": { - "code": { - "asfloat": true, - "caption": "", - "label": "code:example_pd", - "placement": "H", - "widefigure": false - }, - "table": { - "alternate": "gray!20", - "caption": "An example of a table created with pandas dataframe.", - "label": "tbl:example", - "placement": "H" - } - } - }, - "outputs": [], - "source": [ - "np.random.seed(0) \n", - "df = pd.DataFrame(np.random.rand(3,4),columns=['a','b','c','d'])\n", - "df.a = ['$\\delta$','x','y']\n", - "df.b = ['l','m','n']\n", - "df.set_index(['a','b'])\n", - "df.round(3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Equations (with ipython or sympy)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ipub": { - "equation": { - "label": "eqn:example_ipy" - } - } - }, - "outputs": [], - "source": [ - "Latex('$$ a = b+c $$')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ipub": { - "caption": "code:example_sym" - } - }, - "source": [ - "The plotting code for a sympy equation (=@eqn:example_sympy)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ipub": { - "code": { - "asfloat": true, - "caption": "", - "label": "code:example_sym", - "placement": "H", - "widefigure": false - }, - "equation": { - "environment": "equation", - "label": "eqn:example_sympy" - } - } - }, - "outputs": [], - "source": [ - "y = sym.Function('y')\n", - "n = sym.symbols(r'\\alpha')\n", - "f = y(n)-2*y(n-1/sym.pi)-5*y(n-2)\n", - "sym.rsolve(f,y(n),[1,4])" - ] - } - ], - "metadata": { - "celltoolbar": "Edit Metadata", - "hide_input": false, - "execution": { - "timeout": 1 - }, - "ipub": { - "bibliography": "example.bib", - "biboptions": [ - "super", - "sort" - ], - "bibstyle": "unsrtnat", - "language": "portuges", - "listcode": true, - "listfigures": true, - "listtables": true, - "pandoc": { - "at_notation": true, - "use_numref": true - }, - "sphinx": { - "bib_title": "My Bibliography" - }, - "titlepage": { - "author": "Authors Name", - "email": "authors@email.com", - "institution": [ - "Institution1", - "Institution2" - ], - "logo": "logo_example.png", - "subtitle": "Sub-Title", - "supervisors": [ - "First Supervisor", - "Second Supervisor" - ], - "tagline": "A tagline for the report.", - "title": "Main-Title" - }, - "toc": { - "depth": 2 - } - }, - "jupytext": { - "notebook_metadata_filter": "ipub" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - }, - "latex_envs": { - "LaTeX_envs_menu_present": true, - "autocomplete": true, - "bibliofile": "example.bib", - "cite_by": "apalike", - "current_citInitial": 1, - "eqLabelWithNumbers": true, - "eqNumInitial": 1, - "hotkeys": { - "equation": "Ctrl-E", - "itemize": "Ctrl-I" - }, - "labels_anchors": false, - "latex_user_defs": false, - "report_style_numbering": false, - "user_envs_cfg": true - }, - "nav_menu": {}, - "toc": { - "colors": { - "hover_highlight": "#DAA520", - "navigate_num": "#000000", - "navigate_text": "#333333", - "running_highlight": "#FF0000", - "selected_highlight": "#FFD700", - "sidebar_border": "#EEEEEE", - "wrapper_background": "#FFFFFF" - }, - "moveMenuLeft": true, - "nav_menu": { - "height": "161px", - "width": "252px" - }, - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 4, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": true, - "widenNotebook": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/tests/notebooks/sleep_10.ipynb b/tests/notebooks/sleep_10.ipynb new file mode 100644 index 00000000..7473264a --- /dev/null +++ b/tests/notebooks/sleep_10.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "init_cell": true + }, + "outputs": [], + "source": [ + "import time\n", + "time.sleep(10)" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "hide_input": false, + "execution": {}, + "jupytext": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/notebooks/sleep_10_metadata_timeout.ipynb b/tests/notebooks/sleep_10_metadata_timeout.ipynb new file mode 100644 index 00000000..946cb38a --- /dev/null +++ b/tests/notebooks/sleep_10_metadata_timeout.ipynb @@ -0,0 +1,43 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "init_cell": true + }, + "outputs": [], + "source": [ + "import time\n", + "time.sleep(10)" + ] + } + ], + "metadata": { + "celltoolbar": "Edit Metadata", + "hide_input": false, + "execution": { + "timeout": 1 + }, + "jupytext": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_execute.py b/tests/test_execute.py index b6ddb2ba..42216438 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -228,7 +228,7 @@ def test_relative_path_force(sphinx_run, file_regression, check_nbs): # Execution timeout configuration @pytest.mark.sphinx_params( - "complex_outputs_unrun.ipynb", + "sleep_10.ipynb", conf={"jupyter_execute_notebooks": "cache", "execution_timeout": 1}, ) def test_execution_timeout(sphinx_run, file_regression, check_nbs): @@ -239,7 +239,7 @@ def test_execution_timeout(sphinx_run, file_regression, check_nbs): @pytest.mark.sphinx_params( - "complex_outputs_unrun_timeout.ipynb", + "sleep_10_metadata_timeout.ipynb", conf={"jupyter_execute_notebooks": "cache", "execution_timeout": 60}, ) def test_execution_metadata_timeout(sphinx_run, file_regression, check_nbs): diff --git a/tox.ini b/tox.ini index 63a4450d..29f61ffc 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,10 @@ commands = pytest {posargs} extras = rtd deps = ipython<=7.11.0 # required by coconut -whitelist_externals = rm +whitelist_externals = + echo + rm commands = clean: rm -rf docs/_build sphinx-build -nW --keep-going -b {posargs:html} docs/ docs/_build/{posargs:html} +commands_post = echo "open file://{toxinidir}/docs/_build/{posargs:html}/index.html" From 6ada3b5e26efa99ed3fda09ad8f86a076af3b2b9 Mon Sep 17 00:00:00 2001 From: mmcky Date: Tue, 5 Jan 2021 13:53:59 +1100 Subject: [PATCH 02/20] enable parsing of a file option for code-cells --- myst_nb/converter.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 2781c1e6..98d1218e 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -2,6 +2,8 @@ from typing import Callable, Iterable, Optional import attr +import re +from pathlib import Path import nbformat as nbf from sphinx.environment import BuildEnvironment @@ -57,7 +59,7 @@ def get_nb_converter( if source_iter is None: return NbConverter( lambda text: myst_to_notebook( - text, config=env.myst_config, add_source_map=True + text, config=env.myst_config, add_source_map=True, context=env ), env.myst_config, ) @@ -66,7 +68,7 @@ def get_nb_converter( if is_myst_notebook(source_iter): return NbConverter( lambda text: myst_to_notebook( - text, config=env.myst_config, add_source_map=True + text, config=env.myst_config, add_source_map=True, context=env ), env.myst_config, ) @@ -120,6 +122,10 @@ class MystMetadataParsingError(Exception): """Error when parsing metadata from myst formatted text""" +class MystFileParsingError(Exception): + """Error when parsing files for code-blocks/code-cells""" + + def strip_blank_lines(text): text = text.rstrip() while text and text.startswith("\n"): @@ -180,6 +186,7 @@ def myst_to_notebook( code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, + context=None, ): """Convert text written in the myst format to a notebook. @@ -188,6 +195,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. + :param: context: passthrough Sphinx Builder Environment :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML @@ -248,6 +256,14 @@ def _flush_markdown(start_line, token, md_metadata): if token.type == "fence" and token.info.startswith(code_directive): _flush_markdown(md_start_line, token, md_metadata) options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code") + # Parse :file or file: tags and populate body with contents of file + if "file" in options: + fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() + flpath = Path(context.srcdir + "/" + fl).resolve() + try: + body_lines = flpath.read_text().split("\n") + except Exception: + raise MystFileParsingError("Can't read file: {}".format(flpath)) meta = nbf.from_dict(options) source_map.append(token.map[0] + 1) notebook.cells.append( From 8c0acbe0989f7bb446d8a35d65037dbc12ebe487 Mon Sep 17 00:00:00 2001 From: mmcky Date: Wed, 6 Jan 2021 13:11:53 +1100 Subject: [PATCH 03/20] add test for codecell_file in a mystnb test set --- tests/notebooks/mystnb_codecell_file.md | 17 +++++++++++++++ tests/notebooks/mystnb_codecell_file.py | 3 +++ tests/test_mystnb_features.py | 28 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 tests/notebooks/mystnb_codecell_file.md create mode 100644 tests/notebooks/mystnb_codecell_file.py create mode 100644 tests/test_mystnb_features.py diff --git a/tests/notebooks/mystnb_codecell_file.md b/tests/notebooks/mystnb_codecell_file.md new file mode 100644 index 00000000..a077982b --- /dev/null +++ b/tests/notebooks/mystnb_codecell_file.md @@ -0,0 +1,17 @@ +--- +jupytext: + text_representation: + extension: .md + format_name: myst +kernelspec: + display_name: Python 3 + language: python + name: python3 +author: Matt +--- + +# a title + +```{code-cell} ipython3 +:file: mystnb_codecell_file.py +``` diff --git a/tests/notebooks/mystnb_codecell_file.py b/tests/notebooks/mystnb_codecell_file.py new file mode 100644 index 00000000..efc9904d --- /dev/null +++ b/tests/notebooks/mystnb_codecell_file.py @@ -0,0 +1,3 @@ +# flake8: noqa + +import numpy as np diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py new file mode 100644 index 00000000..32d60a14 --- /dev/null +++ b/tests/test_mystnb_features.py @@ -0,0 +1,28 @@ +import pytest + + +@pytest.mark.sphinx_params( + "mystnb_codecell_file.md", + conf={"jupyter_execute_notebooks": "cache", "source_suffix": {".md": "myst-nb"}}, +) +def test_codecell_file(sphinx_run, file_regression, check_nbs): + sphinx_run.build() + assert sphinx_run.warnings() == "" + assert set(sphinx_run.app.env.metadata["mystnb_codecell_file"].keys()) == { + "jupytext", + "kernelspec", + "author", + "source_map", + "language_info", + } + assert sphinx_run.app.env.metadata["mystnb_codecell_file"]["author"] == "Matt" + assert ( + sphinx_run.app.env.metadata["mystnb_codecell_file"]["kernelspec"] + == '{"display_name": "Python 3", "language": "python", "name": "python3"}' + ) + file_regression.check( + sphinx_run.get_nb(), check_fn=check_nbs, extension=".ipynb", encoding="utf8" + ) + file_regression.check( + sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8" + ) From c253c55ae0e12c6fcb8100da1cf3a216042458d8 Mon Sep 17 00:00:00 2001 From: mmcky Date: Wed, 6 Jan 2021 13:17:23 +1100 Subject: [PATCH 04/20] add docs for myst_nb feature --- docs/use/markdown.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/use/markdown.md b/docs/use/markdown.md index 98ce1f24..776d0826 100644 --- a/docs/use/markdown.md +++ b/docs/use/markdown.md @@ -103,6 +103,22 @@ b = "Python code!" print(f"{a} {b}") ``` +**Code from Files:** + +`myst_nb` provides a convenience feature for importing executable code into a `{code-cell}` +from a file. This can be useful when you want to share code between documents. To do this +you specify a `file` metadata attribute such as: + +````md +```{code-cell} ipython3 +:file: +``` +```` + +```{warning} +This is a feature provided by `myst_nb` and is **not** part of the core `MyST` markup specification. +``` + ### Syntax for markdown Anything in-between code cells will be treated as markdown. You can use any markdown From 48129adc49b98a0085ca45f8373e4b9d347b19d5 Mon Sep 17 00:00:00 2001 From: mmcky Date: Thu, 4 Feb 2021 15:48:52 +1100 Subject: [PATCH 05/20] add test py file for test --- tests/test_mystnb_features.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index 32d60a14..b2d647b1 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -1,11 +1,15 @@ import pytest +from sphinx.util.fileutil import copy_asset_file + @pytest.mark.sphinx_params( "mystnb_codecell_file.md", conf={"jupyter_execute_notebooks": "cache", "source_suffix": {".md": "myst-nb"}}, ) -def test_codecell_file(sphinx_run, file_regression, check_nbs): +def test_codecell_file(sphinx_run, file_regression, check_nbs, get_test_path): + asset_path = get_test_path("mystnb_codecell_file.py") + copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) sphinx_run.build() assert sphinx_run.warnings() == "" assert set(sphinx_run.app.env.metadata["mystnb_codecell_file"].keys()) == { From 347653f0ecbb5ed76adeaa79c57ad4519f77ca1b Mon Sep 17 00:00:00 2001 From: mmcky Date: Tue, 9 Feb 2021 14:00:20 +1100 Subject: [PATCH 06/20] add test solutions for regression checks --- .../test_codecell_file.ipynb | 56 +++++++++++++++++++ .../test_codecell_file.xml | 10 ++++ 2 files changed, 66 insertions(+) create mode 100644 tests/test_mystnb_features/test_codecell_file.ipynb create mode 100644 tests/test_mystnb_features/test_codecell_file.xml diff --git a/tests/test_mystnb_features/test_codecell_file.ipynb b/tests/test_mystnb_features/test_codecell_file.ipynb new file mode 100644 index 00000000..b89804b9 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file.ipynb @@ -0,0 +1,56 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# a title" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "file": "mystnb_codecell_file.py" + }, + "outputs": [], + "source": [ + "# flake8: noqa\n", + "\n", + "import numpy as np\n" + ] + } + ], + "metadata": { + "author": "Matt", + "jupytext": { + "text_representation": { + "extension": ".md", + "format_name": "myst" + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + }, + "source_map": [ + 11, + 15 + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_mystnb_features/test_codecell_file.xml b/tests/test_mystnb_features/test_codecell_file.xml new file mode 100644 index 00000000..545ad7b2 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file.xml @@ -0,0 +1,10 @@ + +
+ + a title + <CellNode cell_type="code" classes="cell"> + <CellInputNode classes="cell_input"> + <literal_block language="ipython3" xml:space="preserve"> + # flake8: noqa + + import numpy as np From 32c67fe8f8f5700132be7f1ff96e5d1af6887709 Mon Sep 17 00:00:00 2001 From: AakashGC <aakashgfude@gmail.com> Date: Wed, 10 Feb 2021 11:48:08 +1100 Subject: [PATCH 07/20] added warning for code with --- myst_nb/converter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index c41f3ad0..051e8ef1 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -7,7 +7,7 @@ import nbformat as nbf from sphinx.environment import BuildEnvironment -from sphinx.util import import_object +from sphinx.util import import_object, logging import yaml from myst_parser.main import MdParserConfig @@ -16,6 +16,7 @@ CODE_DIRECTIVE = "{code-cell}" RAW_DIRECTIVE = "{raw-cell}" +LOGGER = logging.getLogger(__name__) @attr.s class NbConverter: @@ -260,6 +261,8 @@ def _flush_markdown(start_line, token, md_metadata): if "file" in options: fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() flpath = Path(context.srcdir + "/" + fl).resolve() + if len(body_lines): + LOGGER.warning(("content of code-cell is being overwritten by `file`"),location=(context.srcdir + "/" + context.docname, token.map)) try: body_lines = flpath.read_text().split("\n") except Exception: From 39bca9413b3d36e2f351dfa93e3448d7b195f31c Mon Sep 17 00:00:00 2001 From: AakashGC <aakashgfude@gmail.com> Date: Wed, 10 Feb 2021 12:18:19 +1100 Subject: [PATCH 08/20] added a test for codecell file --- .../mystnb_codecell_file_warnings.md | 19 +++++++ tests/test_mystnb_features.py | 28 ++++++++++ .../test_codecell_file_warnings.ipynb | 56 +++++++++++++++++++ .../test_codecell_file_warnings.xml | 10 ++++ 4 files changed, 113 insertions(+) create mode 100644 tests/notebooks/mystnb_codecell_file_warnings.md create mode 100644 tests/test_mystnb_features/test_codecell_file_warnings.ipynb create mode 100644 tests/test_mystnb_features/test_codecell_file_warnings.xml diff --git a/tests/notebooks/mystnb_codecell_file_warnings.md b/tests/notebooks/mystnb_codecell_file_warnings.md new file mode 100644 index 00000000..fe3d471a --- /dev/null +++ b/tests/notebooks/mystnb_codecell_file_warnings.md @@ -0,0 +1,19 @@ +--- +jupytext: + text_representation: + extension: .md + format_name: myst +kernelspec: + display_name: Python 3 + language: python + name: python3 +author: Aakash +--- + +# a title + +```{code-cell} ipython3 +:file: mystnb_codecell_file.py +i = 10 +print(i) +``` diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index b2d647b1..37a383e4 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -30,3 +30,31 @@ def test_codecell_file(sphinx_run, file_regression, check_nbs, get_test_path): file_regression.check( sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8" ) + +@pytest.mark.sphinx_params( + "mystnb_codecell_file_warnings.md", + conf={"jupyter_execute_notebooks": "force", "source_suffix": {".md": "myst-nb"}}, +) +def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test_path): + asset_path = get_test_path("mystnb_codecell_file.py") + copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) + sphinx_run.build() + assert "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by `file`" in sphinx_run.warnings() + assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == { + "jupytext", + "kernelspec", + "author", + "source_map", + "language_info", + } + assert sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["author"] == "Aakash" + assert ( + sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["kernelspec"] + == '{"display_name": "Python 3", "language": "python", "name": "python3"}' + ) + file_regression.check( + sphinx_run.get_nb(), check_fn=check_nbs, extension=".ipynb", encoding="utf8" + ) + file_regression.check( + sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8" + ) diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb new file mode 100644 index 00000000..9a008386 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb @@ -0,0 +1,56 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# a title" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "file": "mystnb_codecell_file.py" + }, + "outputs": [], + "source": [ + "# flake8: noqa\n", + "\n", + "import numpy as np\n" + ] + } + ], + "metadata": { + "author": "Aakash", + "jupytext": { + "text_representation": { + "extension": ".md", + "format_name": "myst" + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "source_map": [ + 11, + 15 + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.xml b/tests/test_mystnb_features/test_codecell_file_warnings.xml new file mode 100644 index 00000000..5c43ddd7 --- /dev/null +++ b/tests/test_mystnb_features/test_codecell_file_warnings.xml @@ -0,0 +1,10 @@ +<document source="mystnb_codecell_file_warnings"> + <section ids="a-title" names="a\ title"> + <title> + a title + <CellNode cell_type="code" classes="cell"> + <CellInputNode classes="cell_input"> + <literal_block language="ipython3" xml:space="preserve"> + # flake8: noqa + + import numpy as np From 3c19a28a2ee08df568fa43d31f3a682a2314bbe0 Mon Sep 17 00:00:00 2001 From: AakashGC <aakashgfude@gmail.com> Date: Wed, 10 Feb 2021 12:21:37 +1100 Subject: [PATCH 09/20] pre-commit check --- myst_nb/converter.py | 6 +++++- tests/test_mystnb_features.py | 11 +++++++++-- .../test_codecell_file_warnings.ipynb | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 051e8ef1..26dd5767 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -18,6 +18,7 @@ LOGGER = logging.getLogger(__name__) + @attr.s class NbConverter: func: Callable[[str], nbf.NotebookNode] = attr.ib() @@ -262,7 +263,10 @@ def _flush_markdown(start_line, token, md_metadata): fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() flpath = Path(context.srcdir + "/" + fl).resolve() if len(body_lines): - LOGGER.warning(("content of code-cell is being overwritten by `file`"),location=(context.srcdir + "/" + context.docname, token.map)) + LOGGER.warning( + ("content of code-cell is being overwritten by `file`"), + location=(context.srcdir + "/" + context.docname, token.map), + ) try: body_lines = flpath.read_text().split("\n") except Exception: diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index 37a383e4..9196b02c 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -31,6 +31,7 @@ def test_codecell_file(sphinx_run, file_regression, check_nbs, get_test_path): sphinx_run.get_doctree().pformat(), extension=".xml", encoding="utf8" ) + @pytest.mark.sphinx_params( "mystnb_codecell_file_warnings.md", conf={"jupyter_execute_notebooks": "force", "source_suffix": {".md": "myst-nb"}}, @@ -39,7 +40,10 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test asset_path = get_test_path("mystnb_codecell_file.py") copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) sphinx_run.build() - assert "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by `file`" in sphinx_run.warnings() + assert ( + "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by `file`" # noqaE501 + in sphinx_run.warnings() + ) assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == { "jupytext", "kernelspec", @@ -47,7 +51,10 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test "source_map", "language_info", } - assert sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["author"] == "Aakash" + assert ( + sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["author"] + == "Aakash" + ) assert ( sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["kernelspec"] == '{"display_name": "Python 3", "language": "python", "name": "python3"}' diff --git a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb index 9a008386..37f6c148 100644 --- a/tests/test_mystnb_features/test_codecell_file_warnings.ipynb +++ b/tests/test_mystnb_features/test_codecell_file_warnings.ipynb @@ -53,4 +53,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} From 24e0475759ee06bbb0cb4003a51d2f4cb3b1f76b Mon Sep 17 00:00:00 2001 From: mmcky <mmcky@users.noreply.github.com> Date: Tue, 16 Feb 2021 15:29:20 +1100 Subject: [PATCH 10/20] Update docs/use/markdown.md Co-authored-by: Chris Sewell <chrisj_sewell@hotmail.com> --- docs/use/markdown.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/use/markdown.md b/docs/use/markdown.md index 776d0826..eb21dd69 100644 --- a/docs/use/markdown.md +++ b/docs/use/markdown.md @@ -116,7 +116,7 @@ you specify a `file` metadata attribute such as: ```` ```{warning} -This is a feature provided by `myst_nb` and is **not** part of the core `MyST` markup specification. +This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. ``` ### Syntax for markdown From 12a11a361856e25f677ee09c25733314aae0e1d4 Mon Sep 17 00:00:00 2001 From: mmcky <mamckay@gmail.com> Date: Wed, 17 Feb 2021 11:18:50 +1100 Subject: [PATCH 11/20] remove context from myst_to_notebook and pass file_srcdir --- myst_nb/converter.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 26dd5767..d4d1246c 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -61,7 +61,10 @@ def get_nb_converter( if source_iter is None: return NbConverter( lambda text: myst_to_notebook( - text, config=env.myst_config, add_source_map=True, context=env + text, + config=env.myst_config, + add_source_map=True, + file_srcdir=env.srcdir, ), env.myst_config, ) @@ -70,7 +73,10 @@ def get_nb_converter( if is_myst_notebook(source_iter): return NbConverter( lambda text: myst_to_notebook( - text, config=env.myst_config, add_source_map=True, context=env + text, + config=env.myst_config, + add_source_map=True, + file_srcdir=env.srcdir, ), env.myst_config, ) @@ -124,7 +130,7 @@ class MystMetadataParsingError(Exception): """Error when parsing metadata from myst formatted text""" -class MystFileParsingError(Exception): +class CodeFileParsingError(Exception): """Error when parsing files for code-blocks/code-cells""" @@ -188,7 +194,7 @@ def myst_to_notebook( code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, - context=None, + file_srcdir=".", ): """Convert text written in the myst format to a notebook. @@ -197,7 +203,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. - :param: context: passthrough Sphinx Builder Environment + :param: file_srcdir: Support for :file: import option for code-cell (myst_nb) :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML @@ -258,19 +264,21 @@ def _flush_markdown(start_line, token, md_metadata): if token.type == "fence" and token.info.startswith(code_directive): _flush_markdown(md_start_line, token, md_metadata) options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code") - # Parse :file or file: tags and populate body with contents of file + # Parse :file: or file: tags and populate body with contents of file if "file" in options: fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() - flpath = Path(context.srcdir + "/" + fl).resolve() + flpath = Path(file_srcdir + "/" + fl).resolve() if len(body_lines): LOGGER.warning( ("content of code-cell is being overwritten by `file`"), - location=(context.srcdir + "/" + context.docname, token.map), + location=(flpath, token.map), ) try: body_lines = flpath.read_text().split("\n") except Exception: - raise MystFileParsingError("Can't read file: {}".format(flpath)) + raise CodeFileParsingError( + "Can't read code file: {}".format(flpath) + ) meta = nbf.from_dict(options) source_map.append(token.map[0] + 1) notebook.cells.append( From 32523ca835cf60fa0d469d3e2c983e1288c5a5d6 Mon Sep 17 00:00:00 2001 From: AakashGC <aakashgfude@gmail.com> Date: Wed, 17 Feb 2021 12:29:05 +1100 Subject: [PATCH 12/20] corrected a fixture --- tests/nb_fixtures/reporter_warnings.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nb_fixtures/reporter_warnings.txt b/tests/nb_fixtures/reporter_warnings.txt index b20287f9..3324011f 100644 --- a/tests/nb_fixtures/reporter_warnings.txt +++ b/tests/nb_fixtures/reporter_warnings.txt @@ -66,5 +66,5 @@ cells: [a]: c . -source/path:20004: (WARNING/2) Duplicate reference definition: A +source/path:20004: (WARNING/2) Duplicate reference definition: A [myst.ref] . \ No newline at end of file From 2c5a6186689cbedf9d81b9a5c8864eb7d4d25062 Mon Sep 17 00:00:00 2001 From: mmcky <mamckay@gmail.com> Date: Wed, 17 Feb 2021 12:57:29 +1100 Subject: [PATCH 13/20] pass docname for warning context --- myst_nb/converter.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index d4d1246c..aba6c732 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -64,6 +64,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, + docname=env.docname, file_srcdir=env.srcdir, ), env.myst_config, @@ -76,6 +77,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, + docname=env.docname, file_srcdir=env.srcdir, ), env.myst_config, @@ -194,6 +196,7 @@ def myst_to_notebook( code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, + docname=None, file_srcdir=".", ): """Convert text written in the myst format to a notebook. @@ -203,6 +206,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. + :param: docname: current name of document being converted :param: file_srcdir: Support for :file: import option for code-cell (myst_nb) :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML @@ -269,10 +273,8 @@ def _flush_markdown(start_line, token, md_metadata): fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() flpath = Path(file_srcdir + "/" + fl).resolve() if len(body_lines): - LOGGER.warning( - ("content of code-cell is being overwritten by `file`"), - location=(flpath, token.map), - ) + msg = f"content of code-cell is being overwritten by :file: {fl}" + LOGGER.warning(msg, location=(docname, token.map)) try: body_lines = flpath.read_text().split("\n") except Exception: From 3207b34525b49cb6a4709ffd16366611d1e447af Mon Sep 17 00:00:00 2001 From: mmcky <mamckay@gmail.com> Date: Wed, 17 Feb 2021 13:31:20 +1100 Subject: [PATCH 14/20] fix context issue for :file: option --- myst_nb/converter.py | 16 ++++++---------- tests/test_mystnb_features.py | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index aba6c732..7d7b5999 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -64,8 +64,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - docname=env.docname, - file_srcdir=env.srcdir, + file_context=env, ), env.myst_config, ) @@ -77,8 +76,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - docname=env.docname, - file_srcdir=env.srcdir, + file_context=env, ), env.myst_config, ) @@ -196,8 +194,7 @@ def myst_to_notebook( code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, - docname=None, - file_srcdir=".", + file_context=None, ): """Convert text written in the myst format to a notebook. @@ -206,8 +203,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. - :param: docname: current name of document being converted - :param: file_srcdir: Support for :file: import option for code-cell (myst_nb) + :param: file_context: sphinx context for :file: code-cell option (myst_nb) :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML @@ -271,10 +267,10 @@ def _flush_markdown(start_line, token, md_metadata): # Parse :file: or file: tags and populate body with contents of file if "file" in options: fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() - flpath = Path(file_srcdir + "/" + fl).resolve() + flpath = Path(file_context.srcdir + "/" + fl).resolve() if len(body_lines): msg = f"content of code-cell is being overwritten by :file: {fl}" - LOGGER.warning(msg, location=(docname, token.map)) + LOGGER.warning(msg, location=(file_context.docname, token.map)) try: body_lines = flpath.read_text().split("\n") except Exception: diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index 9196b02c..84b0e3ed 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -41,7 +41,7 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) sphinx_run.build() assert ( - "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by `file`" # noqaE501 + "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by :file: mystnb_codecell_file.py" # noqaE501 in sphinx_run.warnings() ) assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == { From ffc4f92ca7d6e68f448ad2492fbd0968e44ec7a1 Mon Sep 17 00:00:00 2001 From: mmcky <mamckay@gmail.com> Date: Thu, 18 Feb 2021 12:48:52 +1100 Subject: [PATCH 15/20] update :file: to :load: and associted language --- myst_nb/converter.py | 22 +++++++++---------- tests/notebooks/mystnb_codecell_file.md | 2 +- .../mystnb_codecell_file_warnings.md | 2 +- tests/test_mystnb_features.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 7d7b5999..c747a913 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -64,7 +64,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - file_context=env, + load_context=env, ), env.myst_config, ) @@ -76,7 +76,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - file_context=env, + load_context=env, ), env.myst_config, ) @@ -130,7 +130,7 @@ class MystMetadataParsingError(Exception): """Error when parsing metadata from myst formatted text""" -class CodeFileParsingError(Exception): +class LoadFileParsingError(Exception): """Error when parsing files for code-blocks/code-cells""" @@ -194,7 +194,7 @@ def myst_to_notebook( code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, - file_context=None, + load_context=None, ): """Convert text written in the myst format to a notebook. @@ -264,18 +264,18 @@ def _flush_markdown(start_line, token, md_metadata): if token.type == "fence" and token.info.startswith(code_directive): _flush_markdown(md_start_line, token, md_metadata) options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code") - # Parse :file: or file: tags and populate body with contents of file - if "file" in options: + # Parse :load: or load: tags and populate body with contents of file + if "load" in options: fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() - flpath = Path(file_context.srcdir + "/" + fl).resolve() + flpath = Path(load_context.srcdir + "/" + fl).resolve() if len(body_lines): - msg = f"content of code-cell is being overwritten by :file: {fl}" - LOGGER.warning(msg, location=(file_context.docname, token.map)) + msg = f"content of code-cell is being overwritten by :load: {fl}" + LOGGER.warning(msg, location=(load_context.docname, token.map)) try: body_lines = flpath.read_text().split("\n") except Exception: - raise CodeFileParsingError( - "Can't read code file: {}".format(flpath) + raise LoadFileParsingError( + "Can't read file from :load: {}".format(flpath) ) meta = nbf.from_dict(options) source_map.append(token.map[0] + 1) diff --git a/tests/notebooks/mystnb_codecell_file.md b/tests/notebooks/mystnb_codecell_file.md index a077982b..eae2841f 100644 --- a/tests/notebooks/mystnb_codecell_file.md +++ b/tests/notebooks/mystnb_codecell_file.md @@ -13,5 +13,5 @@ author: Matt # a title ```{code-cell} ipython3 -:file: mystnb_codecell_file.py +:load: mystnb_codecell_file.py ``` diff --git a/tests/notebooks/mystnb_codecell_file_warnings.md b/tests/notebooks/mystnb_codecell_file_warnings.md index fe3d471a..4d243615 100644 --- a/tests/notebooks/mystnb_codecell_file_warnings.md +++ b/tests/notebooks/mystnb_codecell_file_warnings.md @@ -13,7 +13,7 @@ author: Aakash # a title ```{code-cell} ipython3 -:file: mystnb_codecell_file.py +:load: mystnb_codecell_file.py i = 10 print(i) ``` diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index 84b0e3ed..d58a290c 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -41,7 +41,7 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) sphinx_run.build() assert ( - "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by :file: mystnb_codecell_file.py" # noqaE501 + "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by :load: mystnb_codecell_file.py" # noqaE501 in sphinx_run.warnings() ) assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == { From 2a57b65aac72a004eedc3a597d3b1fe54f96837d Mon Sep 17 00:00:00 2001 From: mmcky <mamckay@gmail.com> Date: Thu, 18 Feb 2021 12:51:08 +1100 Subject: [PATCH 16/20] update docs for :load: --- docs/use/markdown.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/use/markdown.md b/docs/use/markdown.md index eb21dd69..c4442abf 100644 --- a/docs/use/markdown.md +++ b/docs/use/markdown.md @@ -107,16 +107,16 @@ print(f"{a} {b}") `myst_nb` provides a convenience feature for importing executable code into a `{code-cell}` from a file. This can be useful when you want to share code between documents. To do this -you specify a `file` metadata attribute such as: +you specify a `load` metadata attribute such as: ````md ```{code-cell} ipython3 -:file: <path> +:load: <path> ``` ```` ```{warning} -This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. +This is an experimental feature that is **not** part of the core `MyST` markup specification, and may be removed in the future. Using `:load:` will also overwrite any code written into the directive. ``` ### Syntax for markdown From 8621c99f62698060e347cb93dcff747b5c51f282 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 22 Feb 2021 09:09:21 +0100 Subject: [PATCH 17/20] fix the code --- myst_nb/converter.py | 37 +++++++++++++++++++++-------------- tests/test_mystnb_features.py | 2 +- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index c747a913..0856d71d 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -1,5 +1,5 @@ import json -from typing import Callable, Iterable, Optional +from typing import Callable, Iterable, Optional, Tuple import attr import re @@ -64,7 +64,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - load_context=env, + load_context=(env.srcdir, env.docname), ), env.myst_config, ) @@ -76,7 +76,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - load_context=env, + load_context=(env.srcdir, env.docname), ), env.myst_config, ) @@ -188,13 +188,30 @@ def read_cell_metadata(token, cell_index): return metadata +def load_code_from_file(token, body_lines, load_context): + """load source code from a file.""" + if not load_context: + raise LoadFileParsingError("load context not supplied for :load:") + srcdir, docname = load_context + file_name = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() + file_path = Path(srcdir).joinpath(file_name).resolve() + if len(body_lines): + msg = f"content of code-cell is being overwritten by :load: {file_name}" + LOGGER.warning(msg, location=(docname, token.map[0] if token.map else 0)) + try: + body_lines = file_path.read_text().split("\n") + except Exception: + raise LoadFileParsingError("Can't read file from :load: {}".format(file_path)) + return body_lines + + def myst_to_notebook( text, config: MdParserConfig, code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, - load_context=None, + load_context: Optional[Tuple[str, str]] = None, ): """Convert text written in the myst format to a notebook. @@ -266,17 +283,7 @@ def _flush_markdown(start_line, token, md_metadata): options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code") # Parse :load: or load: tags and populate body with contents of file if "load" in options: - fl = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() - flpath = Path(load_context.srcdir + "/" + fl).resolve() - if len(body_lines): - msg = f"content of code-cell is being overwritten by :load: {fl}" - LOGGER.warning(msg, location=(load_context.docname, token.map)) - try: - body_lines = flpath.read_text().split("\n") - except Exception: - raise LoadFileParsingError( - "Can't read file from :load: {}".format(flpath) - ) + body_lines = load_code_from_file(token, body_lines, load_context) meta = nbf.from_dict(options) source_map.append(token.map[0] + 1) notebook.cells.append( diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index d58a290c..b25cf0c7 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -41,7 +41,7 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) sphinx_run.build() assert ( - "mystnb_codecell_file_warnings.md:[14, 19]: WARNING: content of code-cell is being overwritten by :load: mystnb_codecell_file.py" # noqaE501 + "mystnb_codecell_file_warnings.md:14: WARNING: content of code-cell is being overwritten by :load: mystnb_codecell_file.py" # noqaE501 in sphinx_run.warnings() ) assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == { From 43cc5ccbcd4f9fc6b0736e1899fb2150ec0e316a Mon Sep 17 00:00:00 2001 From: mmcky <mamckay@gmail.com> Date: Mon, 22 Feb 2021 21:35:30 +1100 Subject: [PATCH 18/20] add check for docname when master_doc is passed --- myst_nb/converter.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 0856d71d..a43bb749 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -59,24 +59,34 @@ def get_nb_converter( # If there is no source text then we assume a MyST Notebook if source_iter is None: + # Check if docname exists + try: + docname = env.docname + except KeyError: + docname = env.config.master_doc return NbConverter( lambda text: myst_to_notebook( text, config=env.myst_config, add_source_map=True, - load_context=(env.srcdir, env.docname), + load_context=(env.srcdir, docname), ), env.myst_config, ) # Given the source lines, we check it can be recognised as a MyST Notebook if is_myst_notebook(source_iter): + # Check if docname exists + try: + docname = env.docname + except KeyError: + docname = env.config.master_doc return NbConverter( lambda text: myst_to_notebook( text, config=env.myst_config, add_source_map=True, - load_context=(env.srcdir, env.docname), + load_context=(env.srcdir, docname), ), env.myst_config, ) @@ -220,7 +230,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. - :param: file_context: sphinx context for :file: code-cell option (myst_nb) + :param: load_context: sphinx context for :load: code-cell option (myst_nb) :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML From 7bcc728ba57e237d4a0f107baa105e2f25e1542e Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 22 Feb 2021 11:56:08 +0100 Subject: [PATCH 19/20] remove duplication --- myst_nb/converter.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index a43bb749..8a035b38 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -57,13 +57,14 @@ def get_nb_converter( ) return a + try: + docname = env.docname + except KeyError: + docname = env.config.master_doc + # If there is no source text then we assume a MyST Notebook if source_iter is None: # Check if docname exists - try: - docname = env.docname - except KeyError: - docname = env.config.master_doc return NbConverter( lambda text: myst_to_notebook( text, @@ -77,10 +78,6 @@ def get_nb_converter( # Given the source lines, we check it can be recognised as a MyST Notebook if is_myst_notebook(source_iter): # Check if docname exists - try: - docname = env.docname - except KeyError: - docname = env.config.master_doc return NbConverter( lambda text: myst_to_notebook( text, From f64a21020eac71a1451d8aef9d351cbcfa912da7 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Mon, 22 Feb 2021 13:09:40 +0100 Subject: [PATCH 20/20] rewrite --- myst_nb/converter.py | 38 +++++++++++++++++------------------ myst_nb/parser.py | 2 +- tests/test_mystnb_features.py | 3 ++- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/myst_nb/converter.py b/myst_nb/converter.py index 8a035b38..0a78b41b 100644 --- a/myst_nb/converter.py +++ b/myst_nb/converter.py @@ -1,8 +1,7 @@ import json -from typing import Callable, Iterable, Optional, Tuple +from typing import Callable, Iterable, Optional import attr -import re from pathlib import Path import nbformat as nbf @@ -57,11 +56,6 @@ def get_nb_converter( ) return a - try: - docname = env.docname - except KeyError: - docname = env.config.master_doc - # If there is no source text then we assume a MyST Notebook if source_iter is None: # Check if docname exists @@ -70,7 +64,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - load_context=(env.srcdir, docname), + path=path, ), env.myst_config, ) @@ -83,7 +77,7 @@ def get_nb_converter( text, config=env.myst_config, add_source_map=True, - load_context=(env.srcdir, docname), + path=path, ), env.myst_config, ) @@ -195,16 +189,18 @@ def read_cell_metadata(token, cell_index): return metadata -def load_code_from_file(token, body_lines, load_context): +def load_code_from_file(nb_path, file_name, token, body_lines): """load source code from a file.""" - if not load_context: - raise LoadFileParsingError("load context not supplied for :load:") - srcdir, docname = load_context - file_name = re.search(r"( .*?\.[\w:]+)", token.content).group(0).lstrip() - file_path = Path(srcdir).joinpath(file_name).resolve() + if nb_path is None: + raise LoadFileParsingError("path to notebook not supplied for :load:") + file_path = Path(nb_path).parent.joinpath(file_name).resolve() if len(body_lines): - msg = f"content of code-cell is being overwritten by :load: {file_name}" - LOGGER.warning(msg, location=(docname, token.map[0] if token.map else 0)) + line = token.map[0] if token.map else 0 + msg = ( + f"{nb_path}:{line} content of code-cell is being overwritten by " + f":load: {file_name}" + ) + LOGGER.warning(msg) try: body_lines = file_path.read_text().split("\n") except Exception: @@ -218,7 +214,7 @@ def myst_to_notebook( code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, add_source_map=False, - load_context: Optional[Tuple[str, str]] = None, + path: Optional[str] = None, ): """Convert text written in the myst format to a notebook. @@ -227,7 +223,7 @@ def myst_to_notebook( :param raw_directive: the name of the directive to search for containing raw cells :param add_source_map: add a `source_map` key to the notebook metadata, which is a list of the starting source line number for each cell. - :param: load_context: sphinx context for :load: code-cell option (myst_nb) + :param path: path to notebook (required for :load:) :raises MystMetadataParsingError if the metadata block is not valid JSON/YAML @@ -290,7 +286,9 @@ def _flush_markdown(start_line, token, md_metadata): options, body_lines = read_fenced_cell(token, len(notebook.cells), "Code") # Parse :load: or load: tags and populate body with contents of file if "load" in options: - body_lines = load_code_from_file(token, body_lines, load_context) + body_lines = load_code_from_file( + path, options["load"], token, body_lines + ) meta = nbf.from_dict(options) source_map.append(token.map[0] + 1) notebook.cells.append( diff --git a/myst_nb/parser.py b/myst_nb/parser.py index c7287a53..51d48136 100644 --- a/myst_nb/parser.py +++ b/myst_nb/parser.py @@ -49,7 +49,7 @@ def parse(self, inputstring: str, document: nodes.document): self.env = document.settings.env # type: BuildEnvironment converter = get_nb_converter( - self.env.doc2path(self.env.docname, False), + self.env.doc2path(self.env.docname, True), self.env, inputstring.splitlines(keepends=True), ) diff --git a/tests/test_mystnb_features.py b/tests/test_mystnb_features.py index b25cf0c7..7dc203b5 100644 --- a/tests/test_mystnb_features.py +++ b/tests/test_mystnb_features.py @@ -41,7 +41,8 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test copy_asset_file(str(asset_path), str(sphinx_run.app.srcdir)) sphinx_run.build() assert ( - "mystnb_codecell_file_warnings.md:14: WARNING: content of code-cell is being overwritten by :load: mystnb_codecell_file.py" # noqaE501 + "mystnb_codecell_file_warnings.md:14 content of code-cell " + "is being overwritten by :load: mystnb_codecell_file.py" in sphinx_run.warnings() ) assert set(sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"].keys()) == {