From 2c4df67d25e4507b0f61db3024b8b6216b5459f0 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Wed, 9 Dec 2020 12:14:04 +0100
Subject: [PATCH 01/18] Implement classes to represent recipe metadata

---
 esmvalcore/experimental/recipe_info.py | 159 +++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 esmvalcore/experimental/recipe_info.py

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
new file mode 100644
index 0000000000..0ddb0e3aa8
--- /dev/null
+++ b/esmvalcore/experimental/recipe_info.py
@@ -0,0 +1,159 @@
+"""Recipe metadata."""
+
+import textwrap
+from pathlib import Path
+
+import yaml
+
+from esmvalcore._citation import REFERENCES_PATH
+from esmvalcore._config import TAGS
+
+# TODO: Look into `functools.cached_property` for lazy evaluation (python 3.8+)
+
+
+class Author:
+    """Contains author information."""
+    def __init__(self, name: str, institute: str, orcid: str = None):
+        self.name = name
+        self.institute = institute
+        self.orcid = orcid
+
+    def __repr__(self):
+        """Return canonical string representation."""
+        return (f'{self.__class__.__name__}({self.name!r},'
+                ' institute={self.institute!r}, orcid={self.orcid!r})')
+
+    def __str__(self):
+        """Return string representation."""
+        s = f'{self.name} ({self.institute})'
+        if self.orcid:
+            s += f'\n{self.orcid}'
+        return s
+
+    @classmethod
+    def from_tag(cls, tag: str):
+        """Return an instance of Author from a tag (TAGS).
+
+        The author tags are defined in `config-references.yml`.
+        """
+        mapping = TAGS['authors'][tag]
+
+        name = ' '.join(reversed(mapping['name'].split(', ')))
+        institute = mapping.get('institute', 'No affiliation')
+        orcid = mapping['orcid']
+
+        return cls(name=name, institute=institute, orcid=orcid)
+
+
+class Project:
+    """Contains author information."""
+    def __init__(self, project):
+        self.project = project
+
+    def __repr__(self):
+        """Return canonical string representation."""
+        return f'{self.__class__.__name__}({self.project!r})'
+
+    def __str__(self):
+        """Return string representation."""
+        s = f'{self.project}'
+        return s
+
+    @classmethod
+    def from_tag(cls, tag):
+        """Return an instance of Project from a tag (TAGS).
+
+        The project tags are defined in `config-references.yml`.
+        """
+        project = TAGS['projects'][tag]
+        return cls(project=project)
+
+
+class RecipeInfo():
+    """Contains Recipe metadata."""
+    def __init__(self, path):
+        self.path = path
+
+        self._mapping = None
+        self._authors = None
+        self._projects = None
+        self._references = None
+        self._description = None
+
+    def __repr__(self):
+        """Return canonical string representation."""
+        return f'{self.__class__.__name__}({str(self.path)!r})'
+
+    def __str__(self):
+        """Return string representation."""
+        return self.to_markdown()
+
+    def to_markdown(self) -> str:
+        """Return markdown formatted string."""
+        s = f'## {self.name}'
+
+        s += '\n\n'
+        s += f'{self.description}'
+
+        s += '\n\n### Authors'
+        for author in self.authors:
+            s += f'\n- {author}'
+
+        if self.projects:
+            s += '\n\n### Projects'
+            for project in self.projects:
+                s += f'\n- {project}'
+
+        if self.references:
+            s += '\n\n### References'
+            for reference in self.references:
+                s += f'\n- {reference}'
+
+        s += '\n'
+
+        return s
+
+    @property
+    def mapping(self):
+        if self._mapping is None:
+            self._mapping = yaml.safe_load(open(self.path, 'r'))
+        return self._mapping
+
+    @property
+    def name(self):
+        """Name of the recipe."""
+        return self.path.stem.replace('_', ' ').capitalize()
+
+    @property
+    def description(self) -> str:
+        """Recipe description."""
+        if self._description is None:
+            description = self.mapping['documentation']['description']
+            self._description = '\n'.join(textwrap.wrap(description))
+        return self._description
+
+    @property
+    def authors(self) -> tuple:
+        """List of recipe authors."""
+        if self._authors is None:
+            tags = self.mapping['documentation']['authors']
+            self._authors = tuple(Author.from_tag(tag) for tag in tags)
+        return self._authors
+
+    @property
+    def projects(self) -> tuple:
+        """List of recipe projects."""
+        if self._projects is None:
+            tags = self.mapping['documentation'].get('projects', [])
+            self._projects = tuple(Project.from_tag(tag) for tag in tags)
+        return self._projects
+
+    @property
+    def references(self) -> tuple:
+        """List of project references."""
+        if self._references is None:
+            references = self.mapping['documentation'].get('references', [])
+            self._references = tuple(
+                Path(REFERENCES_PATH, f'{reference}.bibtex')
+                for reference in references)
+        return self._references

From 98d03f8152caf791d3a709b46234b636e9360ad4 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Wed, 9 Dec 2020 12:14:28 +0100
Subject: [PATCH 02/18] Add utilities to list / find recipes

---
 esmvalcore/experimental/utils.py | 46 ++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 esmvalcore/experimental/utils.py

diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
new file mode 100644
index 0000000000..ab7dec234a
--- /dev/null
+++ b/esmvalcore/experimental/utils.py
@@ -0,0 +1,46 @@
+"""ESMValCore utilities."""
+
+import re
+from pathlib import Path
+
+from esmvalcore._config import DIAGNOSTICS_PATH
+
+from .recipe_info import RecipeInfo
+
+
+def get_recipes_list() -> list:
+    """Return a list of all available recipes."""
+    rootdir = Path(DIAGNOSTICS_PATH, 'recipes')
+    return list(rootdir.glob('**/*.yml'))
+
+
+def find_recipes(query: str) -> list:
+    """Search for recipes matching the search query or pattern.
+
+    This function will search the recipe description, author list, and
+    project information.
+
+    Parameters
+    ----------
+    query : str
+        String to search for, e.g. `find_recipes('righi')` will return all
+        matching that author. Can be a `regex` pattern.
+
+    Returns
+    -------
+    recipes : list
+        Returns a list of recipes matching the search query.
+    """
+    recipes = get_recipes_list()
+
+    query = re.compile(query, flags=re.IGNORECASE)
+
+    result = []
+
+    for recipe in recipes:
+        recipe_info = RecipeInfo(recipe)
+        match = re.search(query, recipe_info.info())
+        if match:
+            result.append(recipe_info)
+
+    return result

From f0d91fe7a837fae0992c16c6ab00257df73ffc03 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Wed, 9 Dec 2020 12:15:09 +0100
Subject: [PATCH 03/18] Expose functions to experimental API

---
 esmvalcore/experimental/__init__.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py
index 5614ce7770..9760e5f88f 100644
--- a/esmvalcore/experimental/__init__.py
+++ b/esmvalcore/experimental/__init__.py
@@ -8,7 +8,13 @@
     '\n  More info: https://github.com/ESMValGroup/ESMValCore/issues/498', )
 
 from .config import CFG  # noqa: E402
+from .recipe_info import RecipeInfo  # noqa: E402
+from .utils import find_recipes, get_recipes_list  # noqa: E402
 
 __all__ = [
     'CFG',
+    'find_recipes',
+    'get_recipes_list',
+    'find_recipes',
+    'RecipeInfo',
 ]

From ecf831e75c2122f417231e0d8592c273a966b6db Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Thu, 10 Dec 2020 15:34:29 +0100
Subject: [PATCH 04/18] Render bibtex references using pybtex / IPython magic

pybtex is a bibtex parser / renderer for python (https://pybtex.org/)
---
 esmvalcore/experimental/recipe_info.py | 77 +++++++++++++++++++++++---
 esmvalcore/experimental/utils.py       |  2 +-
 setup.py                               |  6 ++
 3 files changed, 77 insertions(+), 8 deletions(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 0ddb0e3aa8..82232e15ee 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -3,7 +3,9 @@
 import textwrap
 from pathlib import Path
 
+import pybtex
 import yaml
+from pybtex.database.input import bibtex
 
 from esmvalcore._citation import REFERENCES_PATH
 from esmvalcore._config import TAGS
@@ -11,6 +13,10 @@
 # TODO: Look into `functools.cached_property` for lazy evaluation (python 3.8+)
 
 
+class RenderError(BaseException):
+    """Error during rendering of object."""
+
+
 class Author:
     """Contains author information."""
     def __init__(self, name: str, institute: str, orcid: str = None):
@@ -21,13 +27,14 @@ def __init__(self, name: str, institute: str, orcid: str = None):
     def __repr__(self):
         """Return canonical string representation."""
         return (f'{self.__class__.__name__}({self.name!r},'
-                ' institute={self.institute!r}, orcid={self.orcid!r})')
+                f' institute={self.institute!r}, orcid={self.orcid!r})')
 
     def __str__(self):
         """Return string representation."""
-        s = f'{self.name} ({self.institute})'
+        s = f'{self.name} ({self.institute}'
         if self.orcid:
-            s += f'\n{self.orcid}'
+            s += f'; {self.orcid}'
+        s += ')'
         return s
 
     @classmethod
@@ -69,6 +76,60 @@ def from_tag(cls, tag):
         return cls(project=project)
 
 
+class Reference:
+    def __init__(self, filename):
+        parser = bibtex.Parser(strict=False)
+        try:
+            bib_data = parser.parse_file(filename)
+        except Exception as err:
+            raise IOError(f'Error parsing {filename}: {err}') from None
+
+        if len(bib_data.entries) > 1:
+            raise NotImplementedError(
+                f'{self.__class__.__name__} cannot handle bibtex files '
+                'with more than 1 entry.')
+
+        self._bib_data = bib_data
+        self._key, self._entry = list(bib_data.entries.items())[0]
+        self._filename = filename
+
+    @classmethod
+    def from_tag(cls, tag):
+        filename = Path(REFERENCES_PATH, f'{tag}.bibtex')
+        return cls(filename)
+
+    def __repr__(self):
+        """Return canonical string representation."""
+        return f'{self.__class__.__name__}({self._key!r})'
+
+    def __str__(self):
+        """Return string representation."""
+        return self.render(backend='plaintext')
+
+    def _repr_markdown_(self):
+        """Represent using markdown renderer in a notebook environment."""
+        return self.render(backend='markdown')
+
+    def render(self, backend: str = 'plaintext') -> str:
+        """
+        Parameters
+        ----------
+        backend : str
+            Must be one of: 'plaintext', 'markdown', 'html', 'latex'
+        """
+        style = 'plain'  # alpha, plain, unsrt, unsrtalpha
+        backend = pybtex.plugin.find_plugin('pybtex.backends', backend)()
+        style = pybtex.plugin.find_plugin('pybtex.style.formatting', style)()
+
+        try:
+            formatter = style.format_entry(self._key, self._entry)
+            rendered = formatter.text.render(backend)
+        except Exception as e:
+            raise RenderError(f'Could not render {self._key!r}: {e}') from None
+
+        return rendered
+
+
 class RecipeInfo():
     """Contains Recipe metadata."""
     def __init__(self, path):
@@ -84,6 +145,10 @@ def __repr__(self):
         """Return canonical string representation."""
         return f'{self.__class__.__name__}({str(self.path)!r})'
 
+    def _repr_markdown_(self):
+        """Represent using markdown renderer in a notebook environment."""
+        return self.to_markdown()
+
     def __str__(self):
         """Return string representation."""
         return self.to_markdown()
@@ -152,8 +217,6 @@ def projects(self) -> tuple:
     def references(self) -> tuple:
         """List of project references."""
         if self._references is None:
-            references = self.mapping['documentation'].get('references', [])
-            self._references = tuple(
-                Path(REFERENCES_PATH, f'{reference}.bibtex')
-                for reference in references)
+            tags = self.mapping['documentation'].get('references', [])
+            self._references = tuple(Reference.from_tag(tag) for tag in tags)
         return self._references
diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index ab7dec234a..3256ceff5b 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -39,7 +39,7 @@ def find_recipes(query: str) -> list:
 
     for recipe in recipes:
         recipe_info = RecipeInfo(recipe)
-        match = re.search(query, recipe_info.info())
+        match = re.search(query, recipe_info.to_markdown())
         if match:
             result.append(recipe_info)
 
diff --git a/setup.py b/setup.py
index 3554240a63..7c677195a2 100755
--- a/setup.py
+++ b/setup.py
@@ -76,6 +76,11 @@
         'yamllint',
         'yapf',
     ],
+    # Dependencies for API
+    # Use pip install .[api] to install
+    'api': [
+        'pybtex',
+    ]
 }
 
 
@@ -210,6 +215,7 @@ def read_description(filename):
     extras_require={
         'develop': REQUIREMENTS['develop'] + REQUIREMENTS['test'],
         'test': REQUIREMENTS['test'],
+        'api': REQUIREMENTS['api'],
     },
     entry_points={
         'console_scripts': [

From e5ac16acf2b10aa7c9fb7231d94b4ef86a0b54f3 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Thu, 10 Dec 2020 17:09:48 +0100
Subject: [PATCH 05/18] Tweak code and add documentation

---
 esmvalcore/experimental/__init__.py    |  5 +-
 esmvalcore/experimental/recipe_info.py | 80 ++++++++++++++++++--------
 esmvalcore/experimental/utils.py       | 12 ++--
 3 files changed, 63 insertions(+), 34 deletions(-)

diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py
index 9760e5f88f..1ab8f3162e 100644
--- a/esmvalcore/experimental/__init__.py
+++ b/esmvalcore/experimental/__init__.py
@@ -9,12 +9,11 @@
 
 from .config import CFG  # noqa: E402
 from .recipe_info import RecipeInfo  # noqa: E402
-from .utils import find_recipes, get_recipes_list  # noqa: E402
+from .utils import find_recipes, get_recipes  # noqa: E402
 
 __all__ = [
     'CFG',
-    'find_recipes',
-    'get_recipes_list',
+    'get_recipes',
     'find_recipes',
     'RecipeInfo',
 ]
diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 82232e15ee..0c92ac66d0 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -24,12 +24,12 @@ def __init__(self, name: str, institute: str, orcid: str = None):
         self.institute = institute
         self.orcid = orcid
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         """Return canonical string representation."""
         return (f'{self.__class__.__name__}({self.name!r},'
                 f' institute={self.institute!r}, orcid={self.orcid!r})')
 
-    def __str__(self):
+    def __str__(self) -> str:
         """Return string representation."""
         s = f'{self.name} ({self.institute}'
         if self.orcid:
@@ -37,6 +37,10 @@ def __str__(self):
         s += ')'
         return s
 
+    def _repr_markdown_(self):
+        """Represent using markdown renderer in a notebook environment."""
+        return str(self)
+
     @classmethod
     def from_tag(cls, tag: str):
         """Return an instance of Author from a tag (TAGS).
@@ -54,20 +58,20 @@ def from_tag(cls, tag: str):
 
 class Project:
     """Contains author information."""
-    def __init__(self, project):
+    def __init__(self, project: str):
         self.project = project
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         """Return canonical string representation."""
         return f'{self.__class__.__name__}({self.project!r})'
 
-    def __str__(self):
+    def __str__(self) -> str:
         """Return string representation."""
         s = f'{self.project}'
         return s
 
     @classmethod
-    def from_tag(cls, tag):
+    def from_tag(cls, tag: str):
         """Return an instance of Project from a tag (TAGS).
 
         The project tags are defined in `config-references.yml`.
@@ -77,6 +81,7 @@ def from_tag(cls, tag):
 
 
 class Reference:
+    """Contains reference information."""
     def __init__(self, filename):
         parser = bibtex.Parser(strict=False)
         try:
@@ -94,7 +99,12 @@ def __init__(self, filename):
         self._filename = filename
 
     @classmethod
-    def from_tag(cls, tag):
+    def from_tag(cls, tag: str):
+        """Return an instance of Reference from a bibtex tag.
+
+        The bibtex tags resolved as
+        `esmvaltool/references/{tag}.bibtex`.
+        """
         filename = Path(REFERENCES_PATH, f'{tag}.bibtex')
         return cls(filename)
 
@@ -104,21 +114,22 @@ def __repr__(self):
 
     def __str__(self):
         """Return string representation."""
-        return self.render(backend='plaintext')
+        return self.render(renderer='plaintext')
 
     def _repr_markdown_(self):
         """Represent using markdown renderer in a notebook environment."""
-        return self.render(backend='markdown')
+        return self.render(renderer='markdown')
 
-    def render(self, backend: str = 'plaintext') -> str:
+    def render(self, renderer: str = 'plaintext') -> str:
         """
         Parameters
         ----------
-        backend : str
+        renderer : str
+            Choose the Renderer for the string representation.
             Must be one of: 'plaintext', 'markdown', 'html', 'latex'
         """
         style = 'plain'  # alpha, plain, unsrt, unsrtalpha
-        backend = pybtex.plugin.find_plugin('pybtex.backends', backend)()
+        backend = pybtex.plugin.find_plugin('pybtex.backends', renderer)()
         style = pybtex.plugin.find_plugin('pybtex.style.formatting', style)()
 
         try:
@@ -131,8 +142,14 @@ def render(self, backend: str = 'plaintext') -> str:
 
 
 class RecipeInfo():
-    """Contains Recipe metadata."""
-    def __init__(self, path):
+    """Contains Recipe metadata.
+
+    Parameters
+    ----------
+    path : pathlike
+        Path to the recipe.
+    """
+    def __init__(self, path: str):
         self.path = path
 
         self._mapping = None
@@ -141,20 +158,32 @@ def __init__(self, path):
         self._references = None
         self._description = None
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         """Return canonical string representation."""
-        return f'{self.__class__.__name__}({str(self.path)!r})'
+        return f'{self.__class__.__name__}({self.name!r})'
 
-    def _repr_markdown_(self):
+    def _repr_markdown_(self) -> str:
         """Represent using markdown renderer in a notebook environment."""
-        return self.to_markdown()
+        return self.render('markdown')
 
-    def __str__(self):
+    def __str__(self) -> str:
         """Return string representation."""
-        return self.to_markdown()
+        return self.render('plaintext')
 
     def to_markdown(self) -> str:
         """Return markdown formatted string."""
+        return self.render('markdown')
+
+    def render(self, renderer: str = 'plaintext') -> str:
+        """Return formatted string.
+
+        Parameters
+        ----------
+        renderer : str
+            Choose the renderer for the string representation.
+            Must be one of 'plaintext', 'markdown'
+        """
+        bullet = '\n - '
         s = f'## {self.name}'
 
         s += '\n\n'
@@ -162,30 +191,31 @@ def to_markdown(self) -> str:
 
         s += '\n\n### Authors'
         for author in self.authors:
-            s += f'\n- {author}'
+            s += f'{bullet}{author}'
 
         if self.projects:
             s += '\n\n### Projects'
             for project in self.projects:
-                s += f'\n- {project}'
+                s += f'{bullet}{project}'
 
         if self.references:
             s += '\n\n### References'
             for reference in self.references:
-                s += f'\n- {reference}'
+                s += bullet + reference.render(renderer)
 
         s += '\n'
 
         return s
 
     @property
-    def mapping(self):
+    def mapping(self) -> dict:
+        """Dictionary representation of the recipe."""
         if self._mapping is None:
             self._mapping = yaml.safe_load(open(self.path, 'r'))
         return self._mapping
 
     @property
-    def name(self):
+    def name(self) -> str:
         """Name of the recipe."""
         return self.path.stem.replace('_', ' ').capitalize()
 
diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index 3256ceff5b..e423bb9608 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -8,10 +8,11 @@
 from .recipe_info import RecipeInfo
 
 
-def get_recipes_list() -> list:
+def get_recipes() -> list:
     """Return a list of all available recipes."""
     rootdir = Path(DIAGNOSTICS_PATH, 'recipes')
-    return list(rootdir.glob('**/*.yml'))
+    files = rootdir.glob('**/*.yml')
+    return [RecipeInfo(file) for file in files]
 
 
 def find_recipes(query: str) -> list:
@@ -31,16 +32,15 @@ def find_recipes(query: str) -> list:
     recipes : list
         Returns a list of recipes matching the search query.
     """
-    recipes = get_recipes_list()
+    recipes = get_recipes()
 
     query = re.compile(query, flags=re.IGNORECASE)
 
     result = []
 
     for recipe in recipes:
-        recipe_info = RecipeInfo(recipe)
-        match = re.search(query, recipe_info.to_markdown())
+        match = re.search(query, str(recipe))
         if match:
-            result.append(recipe_info)
+            result.append(recipe)
 
     return result

From 80c08a57f5f7eb5c4f63355de68b6bdbd8fd05cc Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 11:50:22 +0100
Subject: [PATCH 06/18] Make `find_recipes` a method on RecipeList

RecipeList is a container for recipes.
---
 esmvalcore/experimental/__init__.py |  4 +--
 esmvalcore/experimental/utils.py    | 56 ++++++++++++++---------------
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py
index 1ab8f3162e..cf0151cce1 100644
--- a/esmvalcore/experimental/__init__.py
+++ b/esmvalcore/experimental/__init__.py
@@ -9,11 +9,11 @@
 
 from .config import CFG  # noqa: E402
 from .recipe_info import RecipeInfo  # noqa: E402
-from .utils import find_recipes, get_recipes  # noqa: E402
+from .utils import RecipeList, get_recipes  # noqa: E402
 
 __all__ = [
     'CFG',
     'get_recipes',
-    'find_recipes',
     'RecipeInfo',
+    'RecipeList',
 ]
diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index e423bb9608..8879ddd6f6 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -8,39 +8,39 @@
 from .recipe_info import RecipeInfo
 
 
-def get_recipes() -> list:
-    """Return a list of all available recipes."""
-    rootdir = Path(DIAGNOSTICS_PATH, 'recipes')
-    files = rootdir.glob('**/*.yml')
-    return [RecipeInfo(file) for file in files]
-
+class RecipeList(list):
+    """Container for recipes."""
+    def find(self, query: str):
+        """Search for recipes matching the search query or pattern.
 
-def find_recipes(query: str) -> list:
-    """Search for recipes matching the search query or pattern.
+        This function will search the recipe description, author list, and
+        project information.
 
-    This function will search the recipe description, author list, and
-    project information.
+        Parameters
+        ----------
+        query : str
+            String to search for, e.g. `find_recipes('righi')` will return all
+            matching that author. Can be a `regex` pattern.
 
-    Parameters
-    ----------
-    query : str
-        String to search for, e.g. `find_recipes('righi')` will return all
-        matching that author. Can be a `regex` pattern.
+        Returns
+        -------
+        recipes : RecipeList
+            Returns a list of recipes matching the search query.
+        """
+        query = re.compile(query, flags=re.IGNORECASE)
 
-    Returns
-    -------
-    recipes : list
-        Returns a list of recipes matching the search query.
-    """
-    recipes = get_recipes()
+        matches = RecipeList()
 
-    query = re.compile(query, flags=re.IGNORECASE)
+        for recipe in self:
+            match = re.search(query, str(recipe))
+            if match:
+                matches.append(recipe)
 
-    result = []
+        return matches
 
-    for recipe in recipes:
-        match = re.search(query, str(recipe))
-        if match:
-            result.append(recipe)
 
-    return result
+def get_recipes() -> list:
+    """Return a list of all available recipes."""
+    rootdir = Path(DIAGNOSTICS_PATH, 'recipes')
+    files = rootdir.glob('**/*.yml')
+    return RecipeList(RecipeInfo(file) for file in files)

From 32055c3bd3dfd33a0756fc0e4dff880189a99979 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 11:52:49 +0100
Subject: [PATCH 07/18] Add maintainer to RecipeInfo

Because maintainers are also defined as authors in
config-references.yml, rename class Author -> Contributor
---
 esmvalcore/experimental/recipe_info.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 0c92ac66d0..49c9a13bd9 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -17,8 +17,8 @@ class RenderError(BaseException):
     """Error during rendering of object."""
 
 
-class Author:
-    """Contains author information."""
+class Contributor:
+    """Contains contributor (author or maintainer) information."""
     def __init__(self, name: str, institute: str, orcid: str = None):
         self.name = name
         self.institute = institute
@@ -43,9 +43,10 @@ def _repr_markdown_(self):
 
     @classmethod
     def from_tag(cls, tag: str):
-        """Return an instance of Author from a tag (TAGS).
+        """Return an instance of Contributor from a tag (TAGS).
 
-        The author tags are defined in `config-references.yml`.
+        Contributors are defined by author tags in
+        `config-references.yml`.
         """
         mapping = TAGS['authors'][tag]
 
@@ -154,6 +155,7 @@ def __init__(self, path: str):
 
         self._mapping = None
         self._authors = None
+        self._maintainers = None
         self._projects = None
         self._references = None
         self._description = None
@@ -189,7 +191,7 @@ def render(self, renderer: str = 'plaintext') -> str:
         s += '\n\n'
         s += f'{self.description}'
 
-        s += '\n\n### Authors'
+        s += '\n\n### Contributors'
         for author in self.authors:
             s += f'{bullet}{author}'
 
@@ -232,9 +234,18 @@ def authors(self) -> tuple:
         """List of recipe authors."""
         if self._authors is None:
             tags = self.mapping['documentation']['authors']
-            self._authors = tuple(Author.from_tag(tag) for tag in tags)
+            self._authors = tuple(Contributor.from_tag(tag) for tag in tags)
         return self._authors
 
+    @property
+    def maintainers(self) -> tuple:
+        """List of recipe maintainers."""
+        if self._maintainers is None:
+            tags = self.mapping['documentation']['maintainer']
+            self._maintainers = tuple(
+                Contributor.from_tag(tag) for tag in tags)
+        return self._maintainers
+
     @property
     def projects(self) -> tuple:
         """List of recipe projects."""

From d442f37ba77534c51307b4faedb74731f3b858a1 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 16:16:10 +0100
Subject: [PATCH 08/18] Add function to get single recipe by its handle

---
 esmvalcore/experimental/__init__.py |  5 ++--
 esmvalcore/experimental/utils.py    | 42 ++++++++++++++++++++++++-----
 2 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py
index cf0151cce1..e7078ae06b 100644
--- a/esmvalcore/experimental/__init__.py
+++ b/esmvalcore/experimental/__init__.py
@@ -9,11 +9,12 @@
 
 from .config import CFG  # noqa: E402
 from .recipe_info import RecipeInfo  # noqa: E402
-from .utils import RecipeList, get_recipes  # noqa: E402
+from .utils import RecipeList, get_all_recipes, get_recipe  # noqa: E402
 
 __all__ = [
     'CFG',
-    'get_recipes',
+    'get_all_recipes',
+    'get_recipe',
     'RecipeInfo',
     'RecipeList',
 ]
diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index 8879ddd6f6..3f16ecf50f 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -19,13 +19,13 @@ def find(self, query: str):
         Parameters
         ----------
         query : str
-            String to search for, e.g. `find_recipes('righi')` will return all
-            matching that author. Can be a `regex` pattern.
+            String to search for, e.g. ``find_recipes('righi')`` will return
+            all matching that author. Can be a `regex` pattern.
 
         Returns
         -------
-        recipes : RecipeList
-            Returns a list of recipes matching the search query.
+        RecipeList
+            List of recipes matching the search query.
         """
         query = re.compile(query, flags=re.IGNORECASE)
 
@@ -39,8 +39,38 @@ def find(self, query: str):
         return matches
 
 
-def get_recipes() -> list:
-    """Return a list of all available recipes."""
+def get_all_recipes() -> list:
+    """Return a list of all available recipes.
+
+    Returns
+    -------
+    RecipeList
+        List of available recipes
+    """
     rootdir = Path(DIAGNOSTICS_PATH, 'recipes')
     files = rootdir.glob('**/*.yml')
     return RecipeList(RecipeInfo(file) for file in files)
+
+
+def get_recipe(name: str) -> 'RecipeInfo':
+    """Get a recipe by its name.
+
+    Parameters
+    ----------
+    name : str
+        Name of the recipe, i.e. ``examples/recipe_python.yml``
+
+    Returns
+    -------
+    RecipeInfo
+    """
+    locations = Path(), Path(DIAGNOSTICS_PATH, 'recipes')
+    filenames = name, name + '.yml'
+
+    for location in locations:
+        for filename in filenames:
+            try_path = Path(location, filename)
+            if try_path.exists():
+                return RecipeInfo(try_path)
+
+    raise FileNotFoundError(f'Could not find `{name}')

From 60a86bd93acc061c2aef3067f854f8e321ae3ce8 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 16:17:35 +0100
Subject: [PATCH 09/18] Tweak variable names

---
 esmvalcore/experimental/recipe_info.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 49c9a13bd9..5ae65880e7 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -151,9 +151,11 @@ class RecipeInfo():
         Path to the recipe.
     """
     def __init__(self, path: str):
-        self.path = path
+        self.path = Path(path)
+        if not self.path.exists():
+            raise FileNotFoundError(f'Cannot find recipe: `{path}`.')
 
-        self._mapping = None
+        self._data = None
         self._authors = None
         self._maintainers = None
         self._projects = None
@@ -210,11 +212,11 @@ def render(self, renderer: str = 'plaintext') -> str:
         return s
 
     @property
-    def mapping(self) -> dict:
+    def data(self) -> dict:
         """Dictionary representation of the recipe."""
-        if self._mapping is None:
-            self._mapping = yaml.safe_load(open(self.path, 'r'))
-        return self._mapping
+        if self._data is None:
+            self._data = yaml.safe_load(open(self.path, 'r'))
+        return self._data
 
     @property
     def name(self) -> str:
@@ -225,7 +227,7 @@ def name(self) -> str:
     def description(self) -> str:
         """Recipe description."""
         if self._description is None:
-            description = self.mapping['documentation']['description']
+            description = self.data['documentation']['description']
             self._description = '\n'.join(textwrap.wrap(description))
         return self._description
 
@@ -233,7 +235,7 @@ def description(self) -> str:
     def authors(self) -> tuple:
         """List of recipe authors."""
         if self._authors is None:
-            tags = self.mapping['documentation']['authors']
+            tags = self.data['documentation']['authors']
             self._authors = tuple(Contributor.from_tag(tag) for tag in tags)
         return self._authors
 
@@ -241,7 +243,7 @@ def authors(self) -> tuple:
     def maintainers(self) -> tuple:
         """List of recipe maintainers."""
         if self._maintainers is None:
-            tags = self.mapping['documentation']['maintainer']
+            tags = self.data['documentation']['maintainer']
             self._maintainers = tuple(
                 Contributor.from_tag(tag) for tag in tags)
         return self._maintainers
@@ -250,7 +252,7 @@ def maintainers(self) -> tuple:
     def projects(self) -> tuple:
         """List of recipe projects."""
         if self._projects is None:
-            tags = self.mapping['documentation'].get('projects', [])
+            tags = self.data['documentation'].get('projects', [])
             self._projects = tuple(Project.from_tag(tag) for tag in tags)
         return self._projects
 
@@ -258,6 +260,6 @@ def projects(self) -> tuple:
     def references(self) -> tuple:
         """List of project references."""
         if self._references is None:
-            tags = self.mapping['documentation'].get('references', [])
+            tags = self.data['documentation'].get('references', [])
             self._references = tuple(Reference.from_tag(tag) for tag in tags)
         return self._references

From e7a6da3265d3fbe1deb65623dcfe3e95ce1cbdb7 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 16:18:36 +0100
Subject: [PATCH 10/18] Add and improve documentation

---
 doc/api/esmvalcore.api.config.rst      | 141 +++++++++++++++++++++++++
 doc/api/esmvalcore.api.recipe_info.rst |  22 ++++
 doc/api/esmvalcore.api.rst             | 137 +-----------------------
 doc/api/esmvalcore.api.utils.rst       |  63 +++++++++++
 esmvalcore/experimental/recipe_info.py |  20 ++--
 5 files changed, 243 insertions(+), 140 deletions(-)
 create mode 100644 doc/api/esmvalcore.api.config.rst
 create mode 100644 doc/api/esmvalcore.api.recipe_info.rst
 create mode 100644 doc/api/esmvalcore.api.utils.rst

diff --git a/doc/api/esmvalcore.api.config.rst b/doc/api/esmvalcore.api.config.rst
new file mode 100644
index 0000000000..5c694c8d7f
--- /dev/null
+++ b/doc/api/esmvalcore.api.config.rst
@@ -0,0 +1,141 @@
+.. _api_config:
+
+Configuration
+=============
+
+This section describes the config submodule of the API.
+
+Config
+******
+
+Configuration of ESMValCore/Tool is done via the ``Config`` object.
+The global configuration can be imported from the ``esmvalcore.experimental`` module as ``CFG``:
+
+.. code-block:: python
+
+    >>> from esmvalcore.experimental import CFG
+    >>> CFG
+    Config({'auxiliary_data_dir': PosixPath('/home/user/auxiliary_data'),
+            'compress_netcdf': False,
+            'config_developer_file': None,
+            'config_file': PosixPath('/home/user/.esmvaltool/config-user.yml'),
+            'drs': {'CMIP5': 'default', 'CMIP6': 'default'},
+            'exit_on_warning': False,
+            'log_level': 'info',
+            'max_parallel_tasks': None,
+            'output_dir': PosixPath('/home/user/esmvaltool_output'),
+            'output_file_type': 'png',
+            'profile_diagnostic': False,
+            'remove_preproc_dir': True,
+            'rootpath': {'CMIP5': '~/default_inputpath',
+                         'CMIP6': '~/default_inputpath',
+                         'default': '~/default_inputpath'},
+            'save_intermediary_cubes': False,
+            'write_netcdf': True,
+            'write_plots': True})
+
+The parameters for the user configuration file are listed `here <https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#user-configuration-file>`__.
+
+``CFG`` is essentially a python dictionary with a few extra functions, similar to ``matplotlib.rcParams``.
+This means that values can be updated like this:
+
+.. code-block:: python
+
+    >>> CFG['output_dir'] = '~/esmvaltool_output'
+    >>> CFG['output_dir']
+    PosixPath('/home/user/esmvaltool_output')
+
+Notice that ``CFG`` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory.
+All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key:
+
+.. code-block:: python
+
+    >>> CFG['otoptu_dri'] = '~/esmvaltool_output'
+    InvalidConfigParameter: `otoptu_dri` is not a valid config parameter.
+
+Or, if the value entered cannot be converted to the expected type:
+
+.. code-block:: python
+
+    >>> CFG['max_years'] = '🐜'
+    InvalidConfigParameter: Key `max_years`: Could not convert '🐜' to int
+
+``Config`` is also flexible, so it tries to correct the type of your input if possible:
+
+.. code-block:: python
+
+    >>> CFG['max_years'] = '123'  # str
+    >>> type(CFG['max_years'])
+    int
+
+By default, the config is loaded from the default location (``/home/user/.esmvaltool/config-user.yml``).
+If it does not exist, it falls back to the default values.
+to load a different file:
+
+.. code-block:: python
+
+    >>> CFG.load_from_file('~/my-config.yml')
+
+Or to reload the current config:
+
+.. code-block:: python
+
+    >>> CFG.reload()
+
+
+Session
+*******
+
+Recipes and diagnostics will be run in their own directories.
+This behaviour can be controlled via the ``Session`` object.
+A ``Session`` can be initiated from the global ``Config``.
+
+.. code-block:: python
+
+    >>> session = CFG.start_session(name='my_session')
+
+A ``Session`` is very similar to the config.
+It is also a dictionary, and copies all the keys from the ``Config``.
+At this moment, ``session`` is essentially a copy of ``CFG``:
+
+.. code-block:: python
+
+    >>> print(session == CFG)
+    True
+    >>> session['output_dir'] = '~/my_output_dir'
+    >>> print(session == CFG)  # False
+    False
+
+A ``Session`` also knows about the directories where the data will stored.
+The session name is used to prefix the directories.
+
+.. code-block:: python
+
+    >>> session.session_dir
+    /home/user/my_output_dir/my_session_20201203_155821
+    >>> session.run_dir
+    /home/user/my_output_dir/my_session_20201203_155821/run
+    >>> session.work_dir
+    /home/user/my_output_dir/my_session_20201203_155821/work
+    >>> session.preproc_dir
+    /home/user/my_output_dir/my_session_20201203_155821/preproc
+    >>> session.plot_dir
+    /home/user/my_output_dir/my_session_20201203_155821/plots
+
+Unlike the global configuration, of which only one can exist, multiple sessions can be initiated from the ``Config``.
+
+
+API reference
+*************
+
+.. autoclass:: esmvalcore.experimental.config.CFG
+    :no-inherited-members:
+    :no-show-inheritance:
+
+.. autoclass:: esmvalcore.experimental.config.Config
+    :no-inherited-members:
+    :no-show-inheritance:
+
+.. autoclass:: esmvalcore.experimental.config.Session
+    :no-inherited-members:
+    :no-show-inheritance:
diff --git a/doc/api/esmvalcore.api.recipe_info.rst b/doc/api/esmvalcore.api.recipe_info.rst
new file mode 100644
index 0000000000..e73ceb25af
--- /dev/null
+++ b/doc/api/esmvalcore.api.recipe_info.rst
@@ -0,0 +1,22 @@
+.. _api_recipe_info:
+
+Recipe Metadata
+===============
+
+This section describes the :py:mod:`esmvalcore.experimental.recipe_info` submodule of the API.
+
+Recipe metadata
+***************
+
+:py:class:`esmvalcore.experimental.recipe_info.RecipeInfo` info is a class that holds metadata from a recipe.
+
+.. code-block:: python
+
+    >>> RecipeInfo('path/to/my_recipe.yml')
+    recipe = RecipeInfo('My recipe')
+
+
+API reference
+*************
+
+.. automodule:: esmvalcore.experimental.recipe_info
diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst
index 8870ef67e0..0f23a0ebda 100644
--- a/doc/api/esmvalcore.api.rst
+++ b/doc/api/esmvalcore.api.rst
@@ -7,137 +7,8 @@ This page describes the new ESMValCore API.
 The API module is available in the submodule ``esmvalcore.experimental``.
 The API is under development, so use at your own risk!
 
-Config
-******
+.. toctree::
 
-Configuration of ESMValCore/Tool is done via the ``Config`` object.
-The global configuration can be imported from the ``esmvalcore.experimental`` module as ``CFG``:
-
-.. code-block:: python
-
-    >>> from esmvalcore.experimental import CFG
-    >>> CFG
-    Config({'auxiliary_data_dir': PosixPath('/home/user/auxiliary_data'),
-            'compress_netcdf': False,
-            'config_developer_file': None,
-            'config_file': PosixPath('/home/user/.esmvaltool/config-user.yml'),
-            'drs': {'CMIP5': 'default', 'CMIP6': 'default'},
-            'exit_on_warning': False,
-            'log_level': 'info',
-            'max_parallel_tasks': None,
-            'output_dir': PosixPath('/home/user/esmvaltool_output'),
-            'output_file_type': 'png',
-            'profile_diagnostic': False,
-            'remove_preproc_dir': True,
-            'rootpath': {'CMIP5': '~/default_inputpath',
-                         'CMIP6': '~/default_inputpath',
-                         'default': '~/default_inputpath'},
-            'save_intermediary_cubes': False,
-            'write_netcdf': True,
-            'write_plots': True})
-
-The parameters for the user configuration file are listed `here <https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#user-configuration-file>`__.
-
-``CFG`` is essentially a python dictionary with a few extra functions, similar to ``matplotlib.rcParams``.
-This means that values can be updated like this:
-
-.. code-block:: python
-
-    >>> CFG['output_dir'] = '~/esmvaltool_output'
-    >>> CFG['output_dir']
-    PosixPath('/home/user/esmvaltool_output')
-
-Notice that ``CFG`` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory.
-All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key:
-
-.. code-block:: python
-
-    >>> CFG['otoptu_dri'] = '~/esmvaltool_output'
-    InvalidConfigParameter: `otoptu_dri` is not a valid config parameter.
-
-Or, if the value entered cannot be converted to the expected type:
-
-.. code-block:: python
-
-    >>> CFG['max_years'] = '🐜'
-    InvalidConfigParameter: Key `max_years`: Could not convert '🐜' to int
-
-``Config`` is also flexible, so it tries to correct the type of your input if possible:
-
-.. code-block:: python
-
-    >>> CFG['max_years'] = '123'  # str
-    >>> type(CFG['max_years'])
-    int
-
-By default, the config is loaded from the default location (``/home/user/.esmvaltool/config-user.yml``).
-If it does not exist, it falls back to the default values.
-to load a different file:
-
-.. code-block:: python
-
-    >>> CFG.load_from_file('~/my-config.yml')
-
-Or to reload the current config:
-
-.. code-block:: python
-
-    >>> CFG.reload()
-
-
-Session
-*******
-
-Recipes and diagnostics will be run in their own directories.
-This behaviour can be controlled via the ``Session`` object.
-A ``Session`` can be initiated from the global ``Config``.
-
-.. code-block:: python
-
-    >>> session = CFG.start_session(name='my_session')
-
-A ``Session`` is very similar to the config.
-It is also a dictionary, and copies all the keys from the ``Config``.
-At this moment, ``session`` is essentially a copy of ``CFG``:
-
-.. code-block:: python
-
-    >>> print(session == CFG)
-    True
-    >>> session['output_dir'] = '~/my_output_dir'
-    >>> print(session == CFG)  # False
-    False
-
-A ``Session`` also knows about the directories where the data will stored.
-The session name is used to prefix the directories.
-
-.. code-block:: python
-
-    >>> session.session_dir
-    /home/user/my_output_dir/my_session_20201203_155821
-    >>> session.run_dir
-    /home/user/my_output_dir/my_session_20201203_155821/run
-    >>> session.work_dir
-    /home/user/my_output_dir/my_session_20201203_155821/work
-    >>> session.preproc_dir
-    /home/user/my_output_dir/my_session_20201203_155821/preproc
-    >>> session.plot_dir
-    /home/user/my_output_dir/my_session_20201203_155821/plots
-
-Unlike the global configuration, of which only one can exist, multiple sessions can be initiated from the ``Config``.
-
-
-API reference
-*************
-
-.. autoclass:: esmvalcore.experimental.config.CFG
-    :no-inherited-members:
-    :no-show-inheritance:
-
-.. autoclass:: esmvalcore.experimental.config.Config
-    :no-inherited-members:
-    :no-show-inheritance:
-
-.. autoclass:: esmvalcore.experimental.config.Session
-    :no-inherited-members:
-    :no-show-inheritance:
+   esmvalcore.api.config
+   esmvalcore.api.recipe_info
+   esmvalcore.api.utils
diff --git a/doc/api/esmvalcore.api.utils.rst b/doc/api/esmvalcore.api.utils.rst
new file mode 100644
index 0000000000..554f141484
--- /dev/null
+++ b/doc/api/esmvalcore.api.utils.rst
@@ -0,0 +1,63 @@
+.. _api_utils:
+
+
+
+Utils
+=====
+
+This section describes the utilities submodule of the API.
+
+Finding recipes
+***************
+
+One of the first thing we may want to do, is to simply get one of the recipes available in ``ESMValTool``
+
+If you already know which recipe you want to load, call :py:func:`esmvalcore.experimental.utils.get_recipe`.
+
+.. code-block:: python
+
+    from esmvalcore.experimental import get_recipe
+    >>> get_recipe('examples/recipe_python')
+    RecipeInfo('Recipe python')
+
+Call the :py:func:`esmvalcore.experimental.utils.get_all_recipes` function to get a list of all available recipes.
+
+.. code-block:: python
+
+    >>> from esmvalcore.experimental import get_all_recipes
+    >>> recipes = get_all_recipes()
+    >>> recipes
+    [RecipeInfo('Recipe perfmetrics cmip5 4cds'),
+     RecipeInfo('Recipe martin18grl'),
+     ...
+     RecipeInfo('Recipe wflow'),
+     RecipeInfo('Recipe pcrglobwb')]
+
+To search for a specific recipe, you can use the :py:meth:`esmvalcore.experimental.utils.RecipeList.find` method. This takes a search query that looks through the recipe metadata and returns any matches. The query can be a regex pattern, so you can make it as complex as you like.
+
+.. code-block:: python
+
+    >>> results = recipes.find('climwip')
+    [RecipeInfo('Recipe climwip')]
+
+The recipes are loaded in a :py:class:`esmvalcore.experimental.recipe_info.RecipeInfo` object, which knows about the documentation, authors, project, and related references of the recipe. It resolves all the tags, so that it knows which institute an author belongs to and which references are associated with the recipe.
+
+This means you can search for something like this:
+
+.. code-block:: python
+
+    >>> recipes.find('Geophysical Research Letters')
+    [RecipeInfo('Recipe martin18grl'),
+     RecipeInfo('Recipe climwip'),
+     RecipeInfo('Recipe ecs constraints'),
+     RecipeInfo('Recipe ecs scatter'),
+     RecipeInfo('Recipe ecs'),
+     RecipeInfo('Recipe seaice')]
+
+
+API reference
+*************
+
+.. automodule:: esmvalcore.experimental.utils
+    :no-inherited-members:
+    :no-show-inheritance:
diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 5ae65880e7..8c7501feaa 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -43,10 +43,10 @@ def _repr_markdown_(self):
 
     @classmethod
     def from_tag(cls, tag: str):
-        """Return an instance of Contributor from a tag (TAGS).
+        """Return an instance of Contributor from a tag (``TAGS``).
 
         Contributors are defined by author tags in
-        `config-references.yml`.
+        ``config-references.yml``.
         """
         mapping = TAGS['authors'][tag]
 
@@ -73,9 +73,9 @@ def __str__(self) -> str:
 
     @classmethod
     def from_tag(cls, tag: str):
-        """Return an instance of Project from a tag (TAGS).
+        """Return an instance of Project from a tag (``TAGS``).
 
-        The project tags are defined in `config-references.yml`.
+        The project tags are defined in ``config-references.yml``.
         """
         project = TAGS['projects'][tag]
         return cls(project=project)
@@ -104,7 +104,7 @@ def from_tag(cls, tag: str):
         """Return an instance of Reference from a bibtex tag.
 
         The bibtex tags resolved as
-        `esmvaltool/references/{tag}.bibtex`.
+        ``esmvaltool/references/{tag}.bibtex``.
         """
         filename = Path(REFERENCES_PATH, f'{tag}.bibtex')
         return cls(filename)
@@ -122,12 +122,18 @@ def _repr_markdown_(self):
         return self.render(renderer='markdown')
 
     def render(self, renderer: str = 'plaintext') -> str:
-        """
+        """Render the reference.
+
         Parameters
         ----------
         renderer : str
-            Choose the Renderer for the string representation.
+            Choose the renderer for the string representation.
             Must be one of: 'plaintext', 'markdown', 'html', 'latex'
+
+        Returns
+        -------
+        str
+            Rendered reference
         """
         style = 'plain'  # alpha, plain, unsrt, unsrtalpha
         backend = pybtex.plugin.find_plugin('pybtex.backends', renderer)()

From 3cb93401d235170573be631d39aed49b231041dd Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 16:48:45 +0100
Subject: [PATCH 11/18] Add tests

---
 package/meta.yaml                     |  3 ++-
 setup.py                              |  7 +------
 tests/unit/experimental/test_utils.py | 22 ++++++++++++++++++++++
 3 files changed, 25 insertions(+), 7 deletions(-)
 create mode 100644 tests/unit/experimental/test_utils.py

diff --git a/package/meta.yaml b/package/meta.yaml
index f5ee7471e7..d69e26856e 100644
--- a/package/meta.yaml
+++ b/package/meta.yaml
@@ -51,6 +51,7 @@ requirements:
     - numpy
     - prov
     - psutil
+    - pybtex
     - pydot
     - pyyaml
     - requests
@@ -82,7 +83,7 @@ test:
     - esmvalcore.cmor.check
     - esmvalcore.cmor.fix
     - esmvalcore.preprocessor
-
+    - esmvalcore.experimental
 
 about:
   home: https://www.esmvaltool.org
diff --git a/setup.py b/setup.py
index 7c677195a2..03e4482cc7 100755
--- a/setup.py
+++ b/setup.py
@@ -39,6 +39,7 @@
         'numpy',
         'prov[dot]',
         'psutil',
+        'pybtex',
         'pyyaml',
         'requests',
         'scitools-iris>=2.2',
@@ -76,11 +77,6 @@
         'yamllint',
         'yapf',
     ],
-    # Dependencies for API
-    # Use pip install .[api] to install
-    'api': [
-        'pybtex',
-    ]
 }
 
 
@@ -215,7 +211,6 @@ def read_description(filename):
     extras_require={
         'develop': REQUIREMENTS['develop'] + REQUIREMENTS['test'],
         'test': REQUIREMENTS['test'],
-        'api': REQUIREMENTS['api'],
     },
     entry_points={
         'console_scripts': [
diff --git a/tests/unit/experimental/test_utils.py b/tests/unit/experimental/test_utils.py
new file mode 100644
index 0000000000..311d9c1546
--- /dev/null
+++ b/tests/unit/experimental/test_utils.py
@@ -0,0 +1,22 @@
+import pytest
+
+from esmvalcore.experimental.recipe_info import RecipeInfo
+from esmvalcore.experimental.utils import get_all_recipes, get_recipe
+"""
+The behaviour of these tests are somewhat unpredictable, depending on
+whether ESMValTool is installed or not, which defines the location of
+``DIAGNOSTICS_PATH``.
+"""
+
+
+@pytest.mark.xfail('FileNotFoundError')
+def test_get_recipe():
+    """Get single recipe."""
+    recipe = get_recipe('examples/recipe_python.yml')
+    assert isinstance(recipe, RecipeInfo)
+
+
+def test_get_all_recipes():
+    """Get all recipes."""
+    recipes = get_all_recipes()
+    assert isinstance(recipes, list)

From 2084cd126aff6b294f20e3e6a1548d952d386467 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 16:58:52 +0100
Subject: [PATCH 12/18] Address linter issues

---
 esmvalcore/experimental/recipe_info.py | 45 ++++++++++++++------------
 esmvalcore/experimental/utils.py       |  1 +
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 8c7501feaa..cb70b752b2 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -19,6 +19,7 @@ class RenderError(BaseException):
 
 class Contributor:
     """Contains contributor (author or maintainer) information."""
+
     def __init__(self, name: str, institute: str, orcid: str = None):
         self.name = name
         self.institute = institute
@@ -31,11 +32,11 @@ def __repr__(self) -> str:
 
     def __str__(self) -> str:
         """Return string representation."""
-        s = f'{self.name} ({self.institute}'
+        string = f'{self.name} ({self.institute}'
         if self.orcid:
-            s += f'; {self.orcid}'
-        s += ')'
-        return s
+            string += f'; {self.orcid}'
+        string += ')'
+        return string
 
     def _repr_markdown_(self):
         """Represent using markdown renderer in a notebook environment."""
@@ -59,6 +60,7 @@ def from_tag(cls, tag: str):
 
 class Project:
     """Contains author information."""
+
     def __init__(self, project: str):
         self.project = project
 
@@ -68,8 +70,8 @@ def __repr__(self) -> str:
 
     def __str__(self) -> str:
         """Return string representation."""
-        s = f'{self.project}'
-        return s
+        string = f'{self.project}'
+        return string
 
     @classmethod
     def from_tag(cls, tag: str):
@@ -83,6 +85,7 @@ def from_tag(cls, tag: str):
 
 class Reference:
     """Contains reference information."""
+
     def __init__(self, filename):
         parser = bibtex.Parser(strict=False)
         try:
@@ -142,8 +145,9 @@ def render(self, renderer: str = 'plaintext') -> str:
         try:
             formatter = style.format_entry(self._key, self._entry)
             rendered = formatter.text.render(backend)
-        except Exception as e:
-            raise RenderError(f'Could not render {self._key!r}: {e}') from None
+        except Exception as err:
+            raise RenderError(
+                f'Could not render {self._key!r}: {err}') from None
 
         return rendered
 
@@ -156,6 +160,7 @@ class RecipeInfo():
     path : pathlike
         Path to the recipe.
     """
+
     def __init__(self, path: str):
         self.path = Path(path)
         if not self.path.exists():
@@ -194,32 +199,32 @@ def render(self, renderer: str = 'plaintext') -> str:
             Must be one of 'plaintext', 'markdown'
         """
         bullet = '\n - '
-        s = f'## {self.name}'
+        string = f'## {self.name}'
 
-        s += '\n\n'
-        s += f'{self.description}'
+        string += '\n\n'
+        string += f'{self.description}'
 
-        s += '\n\n### Contributors'
+        string += '\n\n### Contributors'
         for author in self.authors:
-            s += f'{bullet}{author}'
+            string += f'{bullet}{author}'
 
         if self.projects:
-            s += '\n\n### Projects'
+            string += '\n\n### Projects'
             for project in self.projects:
-                s += f'{bullet}{project}'
+                string += f'{bullet}{project}'
 
         if self.references:
-            s += '\n\n### References'
+            string += '\n\n### References'
             for reference in self.references:
-                s += bullet + reference.render(renderer)
+                string += bullet + reference.render(renderer)
 
-        s += '\n'
+        string += '\n'
 
-        return s
+        return string
 
     @property
     def data(self) -> dict:
-        """Dictionary representation of the recipe."""
+        """Return dictionary representation of the recipe."""
         if self._data is None:
             self._data = yaml.safe_load(open(self.path, 'r'))
         return self._data
diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index 3f16ecf50f..e864a1d40a 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -10,6 +10,7 @@
 
 class RecipeList(list):
     """Container for recipes."""
+
     def find(self, query: str):
         """Search for recipes matching the search query or pattern.
 

From c066735a20ac2dad665917401fa85b5e5a56a76e Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Fri, 11 Dec 2020 17:02:51 +0100
Subject: [PATCH 13/18] Add pybtex to readthedocs requirements

---
 doc/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/requirements.txt b/doc/requirements.txt
index 7d67ccad6e..0094f9fec0 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -6,6 +6,7 @@ numpy
 pandas
 pillow
 prov[dot]
+pybtex
 pyyaml
 scipy
 shapely[vectorized]

From c32fc2427a573952591a21e456d132b67e4bed48 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Mon, 14 Dec 2020 11:27:30 +0100
Subject: [PATCH 14/18] Add option to specify sub-directory when getting
 recipes

---
 esmvalcore/experimental/utils.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index e864a1d40a..99330c87f0 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -10,7 +10,6 @@
 
 class RecipeList(list):
     """Container for recipes."""
-
     def find(self, query: str):
         """Search for recipes matching the search query or pattern.
 
@@ -40,16 +39,24 @@ def find(self, query: str):
         return matches
 
 
-def get_all_recipes() -> list:
+def get_all_recipes(subdir: str = None) -> list:
     """Return a list of all available recipes.
 
+    Parameters
+    ----------
+    subdir : str
+        Sub-directory of the ``DIAGNOSTICS_PATH`` to look for
+        recipes, e.g. ``get_all_recipes(subdir='examples')``.
+
     Returns
     -------
     RecipeList
         List of available recipes
     """
+    if not subdir:
+        subdir = '**'
     rootdir = Path(DIAGNOSTICS_PATH, 'recipes')
-    files = rootdir.glob('**/*.yml')
+    files = rootdir.glob(f'{subdir}/*.yml')
     return RecipeList(RecipeInfo(file) for file in files)
 
 

From 8713e60d5ecf8f55ff2ab2901c472df01c991f2b Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Mon, 14 Dec 2020 11:28:04 +0100
Subject: [PATCH 15/18] Improve test coverage

---
 esmvalcore/experimental/recipe_info.py      | 13 ++----
 tests/unit/experimental/test_recipe_info.py | 48 +++++++++++++++++++++
 tests/unit/experimental/test_utils.py       | 32 +++++++++++---
 3 files changed, 76 insertions(+), 17 deletions(-)
 create mode 100644 tests/unit/experimental/test_recipe_info.py

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index cb70b752b2..5fdfd75cab 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -19,7 +19,6 @@ class RenderError(BaseException):
 
 class Contributor:
     """Contains contributor (author or maintainer) information."""
-
     def __init__(self, name: str, institute: str, orcid: str = None):
         self.name = name
         self.institute = institute
@@ -46,8 +45,8 @@ def _repr_markdown_(self):
     def from_tag(cls, tag: str):
         """Return an instance of Contributor from a tag (``TAGS``).
 
-        Contributors are defined by author tags in
-        ``config-references.yml``.
+        Contributors are defined by author tags in ``config-
+        references.yml``.
         """
         mapping = TAGS['authors'][tag]
 
@@ -60,7 +59,6 @@ def from_tag(cls, tag: str):
 
 class Project:
     """Contains author information."""
-
     def __init__(self, project: str):
         self.project = project
 
@@ -85,13 +83,9 @@ def from_tag(cls, tag: str):
 
 class Reference:
     """Contains reference information."""
-
     def __init__(self, filename):
         parser = bibtex.Parser(strict=False)
-        try:
-            bib_data = parser.parse_file(filename)
-        except Exception as err:
-            raise IOError(f'Error parsing {filename}: {err}') from None
+        bib_data = parser.parse_file(filename)
 
         if len(bib_data.entries) > 1:
             raise NotImplementedError(
@@ -160,7 +154,6 @@ class RecipeInfo():
     path : pathlike
         Path to the recipe.
     """
-
     def __init__(self, path: str):
         self.path = Path(path)
         if not self.path.exists():
diff --git a/tests/unit/experimental/test_recipe_info.py b/tests/unit/experimental/test_recipe_info.py
new file mode 100644
index 0000000000..887b70c5f8
--- /dev/null
+++ b/tests/unit/experimental/test_recipe_info.py
@@ -0,0 +1,48 @@
+import pytest
+
+from esmvalcore.experimental import get_recipe
+from esmvalcore.experimental.recipe_info import Contributor, Project, Reference
+
+pytest.importorskip(
+    'esmvaltool',
+    reason='The behaviour of these tests depends on what ``DIAGNOSTICS_PATH``'
+    'points to. This is defined by a forward-reference to ESMValTool, which'
+    'is not installed in the CI, but likely to be available in a developer'
+    'or user installation.')
+
+
+def test_contributor():
+    """Coverage test for Contributor."""
+    contributor = Contributor.from_tag('righi_mattia')
+
+    assert contributor.name == 'Mattia Righi'
+    assert contributor.institute == 'DLR, Germany'
+    assert contributor.orcid.startswith('https://orcid.org/')
+    assert isinstance(repr(contributor), str)
+    assert isinstance(str(contributor), str)
+
+
+def test_reference():
+    """Coverage test for Reference."""
+    reference = Reference.from_tag('acknow_project')
+
+    assert isinstance(repr(reference), str)
+    assert isinstance(str(reference), str)
+    assert isinstance(reference.render('markdown'), str)
+
+
+def test_project():
+    """Coverage test for Project."""
+    project = Project.from_tag('esmval')
+
+    assert isinstance(repr(project), str)
+    assert isinstance(str(project), str)
+
+
+def test_recipe_info():
+    """Coverage test for RecipeInfo."""
+    recipe = get_recipe('examples/recipe_python')
+
+    assert isinstance(repr(recipe), str)
+    assert isinstance(str(recipe), str)
+    assert isinstance(recipe.to_markdown(), str)
diff --git a/tests/unit/experimental/test_utils.py b/tests/unit/experimental/test_utils.py
index 311d9c1546..33dcc0d14f 100644
--- a/tests/unit/experimental/test_utils.py
+++ b/tests/unit/experimental/test_utils.py
@@ -1,15 +1,20 @@
 import pytest
 
 from esmvalcore.experimental.recipe_info import RecipeInfo
-from esmvalcore.experimental.utils import get_all_recipes, get_recipe
-"""
-The behaviour of these tests are somewhat unpredictable, depending on
-whether ESMValTool is installed or not, which defines the location of
-``DIAGNOSTICS_PATH``.
-"""
+from esmvalcore.experimental.utils import (
+    RecipeList,
+    get_all_recipes,
+    get_recipe,
+)
+
+pytest.importorskip(
+    'esmvaltool',
+    reason='The behaviour of these tests depends on what ``DIAGNOSTICS_PATH``'
+    'points to. This is defined by a forward-reference to ESMValTool, which'
+    'is not installed in the CI, but likely to be available in a developer'
+    'or user installation.')
 
 
-@pytest.mark.xfail('FileNotFoundError')
 def test_get_recipe():
     """Get single recipe."""
     recipe = get_recipe('examples/recipe_python.yml')
@@ -20,3 +25,16 @@ def test_get_all_recipes():
     """Get all recipes."""
     recipes = get_all_recipes()
     assert isinstance(recipes, list)
+
+    recipes.find('')
+
+
+def test_recipe_list_find():
+    """Get all recipes."""
+    recipes = get_all_recipes(subdir='examples')
+
+    assert isinstance(recipes, RecipeList)
+
+    result = recipes.find('python')
+
+    assert isinstance(result, RecipeList)

From 21c31d2b1aadef75c0b5da92515b494d5b8b0855 Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Mon, 14 Dec 2020 11:54:33 +0100
Subject: [PATCH 16/18] Fix codacy issues

---
 esmvalcore/experimental/recipe_info.py | 6 ++++--
 esmvalcore/experimental/utils.py       | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 5fdfd75cab..17cbfa4ceb 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -10,8 +10,6 @@
 from esmvalcore._citation import REFERENCES_PATH
 from esmvalcore._config import TAGS
 
-# TODO: Look into `functools.cached_property` for lazy evaluation (python 3.8+)
-
 
 class RenderError(BaseException):
     """Error during rendering of object."""
@@ -19,6 +17,7 @@ class RenderError(BaseException):
 
 class Contributor:
     """Contains contributor (author or maintainer) information."""
+
     def __init__(self, name: str, institute: str, orcid: str = None):
         self.name = name
         self.institute = institute
@@ -59,6 +58,7 @@ def from_tag(cls, tag: str):
 
 class Project:
     """Contains author information."""
+
     def __init__(self, project: str):
         self.project = project
 
@@ -83,6 +83,7 @@ def from_tag(cls, tag: str):
 
 class Reference:
     """Contains reference information."""
+
     def __init__(self, filename):
         parser = bibtex.Parser(strict=False)
         bib_data = parser.parse_file(filename)
@@ -154,6 +155,7 @@ class RecipeInfo():
     path : pathlike
         Path to the recipe.
     """
+
     def __init__(self, path: str):
         self.path = Path(path)
         if not self.path.exists():
diff --git a/esmvalcore/experimental/utils.py b/esmvalcore/experimental/utils.py
index 99330c87f0..3e2edb8649 100644
--- a/esmvalcore/experimental/utils.py
+++ b/esmvalcore/experimental/utils.py
@@ -10,6 +10,7 @@
 
 class RecipeList(list):
     """Container for recipes."""
+
     def find(self, query: str):
         """Search for recipes matching the search query or pattern.
 

From 10534fba8d964c641c9898c98803dfbcc2dc6bf5 Mon Sep 17 00:00:00 2001
From: Stef Smeets <stefsmeets@users.noreply.github.com>
Date: Wed, 16 Dec 2020 15:24:34 +0000
Subject: [PATCH 17/18] Update esmvalcore/experimental/recipe_info.py

Co-authored-by: Niels Drost <n.drost@esciencecenter.nl>
---
 esmvalcore/experimental/recipe_info.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 17cbfa4ceb..6c7059ee17 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -57,7 +57,7 @@ def from_tag(cls, tag: str):
 
 
 class Project:
-    """Contains author information."""
+    """Contains project information."""
 
     def __init__(self, project: str):
         self.project = project

From cbeb9df2ab0a6978da2efc0a05ecc8e88943b57f Mon Sep 17 00:00:00 2001
From: Stef Smeets <s.smeets@esciencecenter.nl>
Date: Wed, 16 Dec 2020 16:36:49 +0100
Subject: [PATCH 18/18] Fix author string and add maintainers

---
 esmvalcore/experimental/recipe_info.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/esmvalcore/experimental/recipe_info.py b/esmvalcore/experimental/recipe_info.py
index 6c7059ee17..5fe7ac830f 100644
--- a/esmvalcore/experimental/recipe_info.py
+++ b/esmvalcore/experimental/recipe_info.py
@@ -199,10 +199,14 @@ def render(self, renderer: str = 'plaintext') -> str:
         string += '\n\n'
         string += f'{self.description}'
 
-        string += '\n\n### Contributors'
+        string += '\n\n### Authors'
         for author in self.authors:
             string += f'{bullet}{author}'
 
+        string += '\n\n### Maintainers'
+        for maintainer in self.maintainers:
+            string += f'{bullet}{maintainer}'
+
         if self.projects:
             string += '\n\n### Projects'
             for project in self.projects: