Implement importable config object in experimental API submodule

doc/api/esmvalcore.api.rst

@@ -0,0 +1,143 @@
+.. _experimental_api:
+
+Experimental API
+================
+
+This page describes the new ESMValCore API.
+The API module is available in the submodule ``esmvalcore.experimental``.
+The API is under development, so use at your own risk!
+
+Config
+******
+
+Configuration of ESMValCore/Tool is done via the ``Config`` object.
+The global configuration can be imported from the ``esmvalcore.experimental`` module as ``CFG``:
+
+.. code-block:: python
+
+    >>> from esmvalcore.experimental import CFG
+    >>> CFG
+    Config({'auxiliary_data_dir': PosixPath('/home/user/auxiliary_data'),
+            'compress_netcdf': False,
+            'config_developer_file': None,
+            'config_file': PosixPath('/home/user/.esmvaltool/config-user.yml'),
+            'drs': {'CMIP5': 'default', 'CMIP6': 'default'},
+            'exit_on_warning': False,
+            'log_level': 'info',
+            'max_parallel_tasks': None,
+            'output_dir': PosixPath('/home/user/esmvaltool_output'),
+            'output_file_type': 'png',
+            'profile_diagnostic': False,
+            'remove_preproc_dir': True,
+            'rootpath': {'CMIP5': '~/default_inputpath',
+                         'CMIP6': '~/default_inputpath',
+                         'default': '~/default_inputpath'},
+            'save_intermediary_cubes': False,
+            'write_netcdf': True,
+            'write_plots': True})
+
+The parameters for the user configuration file are listed `here <https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#user-configuration-file>`__.
+
+``CFG`` is essentially a python dictionary with a few extra functions, similar to ``matplotlib.rcParams``.
+This means that values can be updated like this:
+
+.. code-block:: python
+
+    >>> CFG['output_dir'] = '~/esmvaltool_output'
+    >>> CFG['output_dir']
+    PosixPath('/home/user/esmvaltool_output')
+
+Notice that ``CFG`` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory.
+All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key:
+
+.. code-block:: python
+
+    >>> CFG['otoptu_dri'] = '~/esmvaltool_output'
+    InvalidConfigParameter: `otoptu_dri` is not a valid config parameter.
+
+Or, if the value entered cannot be converted to the expected type:
+
+.. code-block:: python
+
+    >>> CFG['max_years'] = '🐜'
+    InvalidConfigParameter: Key `max_years`: Could not convert '🐜' to int
+
+``Config`` is also flexible, so it tries to correct the type of your input if possible:
+
+.. code-block:: python
+
+    >>> CFG['max_years'] = '123'  # str
+    >>> type(CFG['max_years'])
+    int
+
+By default, the config is loaded from the default location (``/home/user/.esmvaltool/config-user.yml``).
+If it does not exist, it falls back to the default values.
+to load a different file:
+
+.. code-block:: python
+
+    >>> CFG.load_from_file('~/my-config.yml')
+
+Or to reload the current config:
+
+.. code-block:: python
+
+    >>> CFG.reload()
+
+
+Session
+*******
+
+Recipes and diagnostics will be run in their own directories.
+This behaviour can be controlled via the ``Session`` object.
+A ``Session`` can be initiated from the global ``Config``.
+
+.. code-block:: python
+
+    >>> session = CFG.start_session(name='my_session')
+
+A ``Session`` is very similar to the config.
+It is also a dictionary, and copies all the keys from the ``Config``.
+At this moment, ``session`` is essentially a copy of ``CFG``:
+
+.. code-block:: python
+
+    >>> print(session == CFG)
+    True
+    >>> session['output_dir'] = '~/my_output_dir'
+    >>> print(session == CFG)  # False
+    False
+
+A ``Session`` also knows about the directories where the data will stored.
+The session name is used to prefix the directories.
+
+.. code-block:: python
+
+    >>> session.session_dir
+    /home/user/my_output_dir/my_session_20201203_155821
+    >>> session.run_dir
+    /home/user/my_output_dir/my_session_20201203_155821/run
+    >>> session.work_dir
+    /home/user/my_output_dir/my_session_20201203_155821/work
+    >>> session.preproc_dir
+    /home/user/my_output_dir/my_session_20201203_155821/preproc
+    >>> session.plot_dir
+    /home/user/my_output_dir/my_session_20201203_155821/plots
+
+Unlike the global configuration, of which only one can exist, multiple sessions can be initiated from the ``Config``.
+
+
+API reference
+*************
+
+.. autoclass:: esmvalcore.experimental.config.CFG
+    :no-inherited-members:
+    :no-show-inheritance:
+
+.. autoclass:: esmvalcore.experimental.config.Config
+    :no-inherited-members:
+    :no-show-inheritance:
+
+.. autoclass:: esmvalcore.experimental.config.Session
+    :no-inherited-members:
+    :no-show-inheritance:

doc/api/esmvalcore.rst

@@ -8,3 +8,4 @@ library. This section documents the public API of ESMValCore.
 
    esmvalcore.cmor
    esmvalcore.preprocessor
+   esmvalcore.api

esmvalcore/_config.py

@@ -99,10 +99,7 @@ def read_config_user_file(config_file, folder_name, options=None):
     cfg['run_dir'] = os.path.join(cfg['output_dir'], 'run')
 
     # Read developer configuration file
-    cfg_developer = read_config_developer_file(cfg['config_developer_file'])
-    for key, value in cfg_developer.items():
-        CFG[key] = value
-    read_cmor_tables(CFG)
+    load_config_developer(cfg['config_developer_file'])
 
     return cfg
 
@@ -121,7 +118,6 @@ def _normalize_path(path):
     -------
     str:
         Normalized path
-
     """
     if path is None:
         return None
@@ -142,6 +138,14 @@ def read_config_developer_file(cfg_file=None):
     return cfg
 
 
+def load_config_developer(cfg_file=None):
+    """Load the config developer file and initialize CMOR tables."""
+    cfg_developer = read_config_developer_file(cfg_file)
+    for key, value in cfg_developer.items():
+        CFG[key] = value
+    read_cmor_tables(CFG)
+
+
 def configure_logging(cfg_file=None, output_dir=None, console_log_level=None):
     """Set up logging."""
     if cfg_file is None:

esmvalcore/_recipe.py

@@ -1033,7 +1033,7 @@ def _initialize_variables(self, raw_variable, raw_datasets):
             variable.update(dataset)
 
             variable['recipe_dataset_index'] = index
-            if 'end_year' in variable and 'max_years' in self._cfg:
+            if 'end_year' in variable and self._cfg.get('max_years'):
                 variable['end_year'] = min(
                     variable['end_year'],
                     variable['start_year'] + self._cfg['max_years'] - 1)

esmvalcore/config-user.yml

@@ -41,19 +41,19 @@ profile_diagnostic: false
 # Rootpaths to the data from different projects (lists are also possible)
 # these are generic entries to better allow you to enter your own
 # For site-specific entries, see below
-#rootpath:
-#  CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2]
-#  OBS: ~/obs_inputpath
-#  RAWOBS: ~/rawobs_inputpath
-#  default: ~/default_inputpath
-#  CORDEX: ~/default_inputpath
+# rootpath:
+#   CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2]
+#   OBS: ~/obs_inputpath
+#   RAWOBS: ~/rawobs_inputpath
+#   default: ~/default_inputpath
+#   CORDEX: ~/default_inputpath
 
 # Directory structure for input data: [default]/BADC/DKRZ/ETHZ/etc
 # See config-developer.yml for definitions.
-#drs:
-#  CMIP5: default
-#  CORDEX: default
-#  OBS: default
+# drs:
+#   CMIP5: default
+#   CORDEX: default
+#   OBS: default
 
 # Site-specific entries: Jasmin
 # Uncomment the lines below to locate data on JASMIN

esmvalcore/experimental/__init__.py

@@ -0,0 +1,14 @@
+"""ESMValCore experimental API module."""
+
+from ._warnings import warnings
+
+warnings.warn(
+    '\n  Thank you for trying out the new ESMValCore API.'
+    '\n  Note that this API is experimental and may be subject to change.'
+    '\n  More info: https://github.com/ESMValGroup/ESMValCore/issues/498', )
+
+from .config import CFG  # noqa: E402
+
+__all__ = [
+    'CFG',
+]

esmvalcore/experimental/_exceptions.py

@@ -0,0 +1,24 @@
+"""ESMValCore exceptions."""
+
+import sys
+
+
+class SuppressedError(Exception):
+    """Errors subclassed from SuppressedError hide the full traceback.
+
+    This can be used for simple user-facing errors that do not need the
+    full traceback.
+    """
+
+
+def _suppressed_hook(error, message, traceback):
+    """https://stackoverflow.com/a/27674608."""
+    if issubclass(error, SuppressedError):
+        # Print only the message and hide the traceback
+        print(f'{error.__name__}: {message}'.format(error.__name__, message))
+    else:
+        # Print full traceback
+        sys.__excepthook__(error, message, traceback)
+
+
+sys.excepthook = _suppressed_hook

esmvalcore/experimental/_warnings.py

@@ -0,0 +1,16 @@
+"""ESMValTool warnings."""
+
+import warnings
+
+
+def _warning_formatter(message,
+                       category,
+                       filename,
+                       lineno,
+                       file=None,
+                       line=None):
+    """Patch warning formatting to not mention itself."""
+    return f'{filename}:{lineno}: {category.__name__}: {message}\n'
+
+
+warnings.formatwarning = _warning_formatter

esmvalcore/experimental/config/__init__.py

@@ -0,0 +1,9 @@
+"""ESMValTool config module."""
+
+from ._config_object import CFG, Config, Session
+
+__all__ = [
+    'CFG',
+    'Config',
+    'Session',
+]