diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst new file mode 100644 index 0000000000..8870ef67e0 --- /dev/null +++ b/doc/api/esmvalcore.api.rst @@ -0,0 +1,143 @@ +.. _experimental_api: + +Experimental API +================ + +This page describes the new ESMValCore API. +The API module is available in the submodule ``esmvalcore.experimental``. +The API is under development, so use at your own risk! + +Config +****** + +Configuration of ESMValCore/Tool is done via the ``Config`` object. +The global configuration can be imported from the ``esmvalcore.experimental`` module as ``CFG``: + +.. code-block:: python + + >>> from esmvalcore.experimental import CFG + >>> CFG + Config({'auxiliary_data_dir': PosixPath('/home/user/auxiliary_data'), + 'compress_netcdf': False, + 'config_developer_file': None, + 'config_file': PosixPath('/home/user/.esmvaltool/config-user.yml'), + 'drs': {'CMIP5': 'default', 'CMIP6': 'default'}, + 'exit_on_warning': False, + 'log_level': 'info', + 'max_parallel_tasks': None, + 'output_dir': PosixPath('/home/user/esmvaltool_output'), + 'output_file_type': 'png', + 'profile_diagnostic': False, + 'remove_preproc_dir': True, + 'rootpath': {'CMIP5': '~/default_inputpath', + 'CMIP6': '~/default_inputpath', + 'default': '~/default_inputpath'}, + 'save_intermediary_cubes': False, + 'write_netcdf': True, + 'write_plots': True}) + +The parameters for the user configuration file are listed `here `__. + +``CFG`` is essentially a python dictionary with a few extra functions, similar to ``matplotlib.rcParams``. +This means that values can be updated like this: + +.. code-block:: python + + >>> CFG['output_dir'] = '~/esmvaltool_output' + >>> CFG['output_dir'] + PosixPath('/home/user/esmvaltool_output') + +Notice that ``CFG`` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory. +All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key: + +.. code-block:: python + + >>> CFG['otoptu_dri'] = '~/esmvaltool_output' + InvalidConfigParameter: `otoptu_dri` is not a valid config parameter. + +Or, if the value entered cannot be converted to the expected type: + +.. code-block:: python + + >>> CFG['max_years'] = '🐜' + InvalidConfigParameter: Key `max_years`: Could not convert '🐜' to int + +``Config`` is also flexible, so it tries to correct the type of your input if possible: + +.. code-block:: python + + >>> CFG['max_years'] = '123' # str + >>> type(CFG['max_years']) + int + +By default, the config is loaded from the default location (``/home/user/.esmvaltool/config-user.yml``). +If it does not exist, it falls back to the default values. +to load a different file: + +.. code-block:: python + + >>> CFG.load_from_file('~/my-config.yml') + +Or to reload the current config: + +.. code-block:: python + + >>> CFG.reload() + + +Session +******* + +Recipes and diagnostics will be run in their own directories. +This behaviour can be controlled via the ``Session`` object. +A ``Session`` can be initiated from the global ``Config``. + +.. code-block:: python + + >>> session = CFG.start_session(name='my_session') + +A ``Session`` is very similar to the config. +It is also a dictionary, and copies all the keys from the ``Config``. +At this moment, ``session`` is essentially a copy of ``CFG``: + +.. code-block:: python + + >>> print(session == CFG) + True + >>> session['output_dir'] = '~/my_output_dir' + >>> print(session == CFG) # False + False + +A ``Session`` also knows about the directories where the data will stored. +The session name is used to prefix the directories. + +.. code-block:: python + + >>> session.session_dir + /home/user/my_output_dir/my_session_20201203_155821 + >>> session.run_dir + /home/user/my_output_dir/my_session_20201203_155821/run + >>> session.work_dir + /home/user/my_output_dir/my_session_20201203_155821/work + >>> session.preproc_dir + /home/user/my_output_dir/my_session_20201203_155821/preproc + >>> session.plot_dir + /home/user/my_output_dir/my_session_20201203_155821/plots + +Unlike the global configuration, of which only one can exist, multiple sessions can be initiated from the ``Config``. + + +API reference +************* + +.. autoclass:: esmvalcore.experimental.config.CFG + :no-inherited-members: + :no-show-inheritance: + +.. autoclass:: esmvalcore.experimental.config.Config + :no-inherited-members: + :no-show-inheritance: + +.. autoclass:: esmvalcore.experimental.config.Session + :no-inherited-members: + :no-show-inheritance: diff --git a/doc/api/esmvalcore.rst b/doc/api/esmvalcore.rst index 7d1a3b3728..b9d2127688 100644 --- a/doc/api/esmvalcore.rst +++ b/doc/api/esmvalcore.rst @@ -8,3 +8,4 @@ library. This section documents the public API of ESMValCore. esmvalcore.cmor esmvalcore.preprocessor + esmvalcore.api diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 9f433e5d6c..1d9559dfdf 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -99,10 +99,7 @@ def read_config_user_file(config_file, folder_name, options=None): cfg['run_dir'] = os.path.join(cfg['output_dir'], 'run') # Read developer configuration file - cfg_developer = read_config_developer_file(cfg['config_developer_file']) - for key, value in cfg_developer.items(): - CFG[key] = value - read_cmor_tables(CFG) + load_config_developer(cfg['config_developer_file']) return cfg @@ -121,7 +118,6 @@ def _normalize_path(path): ------- str: Normalized path - """ if path is None: return None @@ -142,6 +138,14 @@ def read_config_developer_file(cfg_file=None): return cfg +def load_config_developer(cfg_file=None): + """Load the config developer file and initialize CMOR tables.""" + cfg_developer = read_config_developer_file(cfg_file) + for key, value in cfg_developer.items(): + CFG[key] = value + read_cmor_tables(CFG) + + def configure_logging(cfg_file=None, output_dir=None, console_log_level=None): """Set up logging.""" if cfg_file is None: diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index c89fa61aea..c1100139e6 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1033,7 +1033,7 @@ def _initialize_variables(self, raw_variable, raw_datasets): variable.update(dataset) variable['recipe_dataset_index'] = index - if 'end_year' in variable and 'max_years' in self._cfg: + if 'end_year' in variable and self._cfg.get('max_years'): variable['end_year'] = min( variable['end_year'], variable['start_year'] + self._cfg['max_years'] - 1) diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index 805a3783c9..9f6526d3f5 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -41,19 +41,19 @@ profile_diagnostic: false # Rootpaths to the data from different projects (lists are also possible) # these are generic entries to better allow you to enter your own # For site-specific entries, see below -#rootpath: -# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] -# OBS: ~/obs_inputpath -# RAWOBS: ~/rawobs_inputpath -# default: ~/default_inputpath -# CORDEX: ~/default_inputpath +# rootpath: +# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] +# OBS: ~/obs_inputpath +# RAWOBS: ~/rawobs_inputpath +# default: ~/default_inputpath +# CORDEX: ~/default_inputpath # Directory structure for input data: [default]/BADC/DKRZ/ETHZ/etc # See config-developer.yml for definitions. -#drs: -# CMIP5: default -# CORDEX: default -# OBS: default +# drs: +# CMIP5: default +# CORDEX: default +# OBS: default # Site-specific entries: Jasmin # Uncomment the lines below to locate data on JASMIN diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py new file mode 100644 index 0000000000..5614ce7770 --- /dev/null +++ b/esmvalcore/experimental/__init__.py @@ -0,0 +1,14 @@ +"""ESMValCore experimental API module.""" + +from ._warnings import warnings + +warnings.warn( + '\n Thank you for trying out the new ESMValCore API.' + '\n Note that this API is experimental and may be subject to change.' + '\n More info: https://github.com/ESMValGroup/ESMValCore/issues/498', ) + +from .config import CFG # noqa: E402 + +__all__ = [ + 'CFG', +] diff --git a/esmvalcore/experimental/_exceptions.py b/esmvalcore/experimental/_exceptions.py new file mode 100644 index 0000000000..437ed124cc --- /dev/null +++ b/esmvalcore/experimental/_exceptions.py @@ -0,0 +1,24 @@ +"""ESMValCore exceptions.""" + +import sys + + +class SuppressedError(Exception): + """Errors subclassed from SuppressedError hide the full traceback. + + This can be used for simple user-facing errors that do not need the + full traceback. + """ + + +def _suppressed_hook(error, message, traceback): + """https://stackoverflow.com/a/27674608.""" + if issubclass(error, SuppressedError): + # Print only the message and hide the traceback + print(f'{error.__name__}: {message}'.format(error.__name__, message)) + else: + # Print full traceback + sys.__excepthook__(error, message, traceback) + + +sys.excepthook = _suppressed_hook diff --git a/esmvalcore/experimental/_warnings.py b/esmvalcore/experimental/_warnings.py new file mode 100644 index 0000000000..0795667be1 --- /dev/null +++ b/esmvalcore/experimental/_warnings.py @@ -0,0 +1,16 @@ +"""ESMValTool warnings.""" + +import warnings + + +def _warning_formatter(message, + category, + filename, + lineno, + file=None, + line=None): + """Patch warning formatting to not mention itself.""" + return f'{filename}:{lineno}: {category.__name__}: {message}\n' + + +warnings.formatwarning = _warning_formatter diff --git a/esmvalcore/experimental/config/__init__.py b/esmvalcore/experimental/config/__init__.py new file mode 100644 index 0000000000..17998a0b60 --- /dev/null +++ b/esmvalcore/experimental/config/__init__.py @@ -0,0 +1,9 @@ +"""ESMValTool config module.""" + +from ._config_object import CFG, Config, Session + +__all__ = [ + 'CFG', + 'Config', + 'Session', +] diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py new file mode 100644 index 0000000000..0bd78259d7 --- /dev/null +++ b/esmvalcore/experimental/config/_config_object.py @@ -0,0 +1,187 @@ +"""Importable config object.""" + +from datetime import datetime +from pathlib import Path + +import yaml + +import esmvalcore + +from ._config_validators import _validators +from ._validated_config import ValidatedConfig + +URL = ('https://docs.esmvaltool.org/projects/' + 'ESMValCore/en/latest/quickstart/configure.html') + + +class Config(ValidatedConfig): + """ESMValTool configuration object. + + Do not instantiate this class directly, but use + :obj:`esmvalcore.experimental.CFG` instead. + """ + + _validate = _validators + _warn_if_missing = ( + ('drs', URL), + ('rootpath', URL), + ) + + @classmethod + def _load_user_config(cls, filename: str, raise_exception: bool = True): + """Load user configuration from the given file. + + The config is cleared and updated in-place. + + Parameters + ---------- + filename: pathlike + Name of the config file, must be yaml format + raise_exception : bool + Raise an exception if `filename` can not be found (default). + Otherwise, silently pass and use the default configuration. This + setting is necessary for the case where + `.esmvaltool/config-user.yml` has not been defined (i.e. first + start). + """ + new = cls() + + try: + mapping = _read_config_file(filename) + mapping['config_file'] = filename + except IOError: + if raise_exception: + raise + mapping = {} + + new.update(CFG_DEFAULT) + new.update(mapping) + + return new + + @classmethod + def _load_default_config(cls, filename: str): + """Load the default configuration.""" + new = cls() + + mapping = _read_config_file(filename) + new.update(mapping) + + return new + + def load_from_file(self, filename): + """Load user configuration from the given file.""" + path = Path(filename).expanduser() + if not path.exists(): + try_path = USER_CONFIG_DIR / filename + if try_path.exists(): + path = try_path + else: + raise FileNotFoundError(f'Cannot find: `{filename}`' + f'locally or in `{try_path}`') + + self.clear() + self.update(Config._load_user_config(path)) + + def reload(self): + """Reload the config file.""" + filename = self.get('config_file', DEFAULT_CONFIG) + self.load_from_file(filename) + + def start_session(self, name: str): + """Start a new session from this configuration object. + + Parameters + ---------- + name: str + Name of the session. + + Returns + ------- + Session + """ + return Session(config=self.copy(), name=name) + + +class Session(ValidatedConfig): + """Container class for session configuration and directory information. + + Do not instantiate this class directly, but use + :obj:`CFG.start_session` instead. + + Parameters + ---------- + config : dict + Dictionary with configuration settings. + name : str + Name of the session to initialize, for example, the name of the + recipe (default='session'). + """ + + _validate = _validators + + def __init__(self, config: dict, name: str = 'session'): + super().__init__(config) + self.set_session_name(name) + + def set_session_name(self, name: str = 'session'): + """Set the name for the session. + + The `name` is used to name the session directory, e.g. + `session_20201208_132800/`. The date is suffixed automatically. + """ + now = datetime.utcnow().strftime("%Y%m%d_%H%M%S") + self.session_name = f"{name}_{now}" + + @property + def session_dir(self): + """Return session directory.""" + return self['output_dir'] / self.session_name + + @property + def preproc_dir(self): + """Return preproc directory.""" + return self.session_dir / 'preproc' + + @property + def work_dir(self): + """Return work directory.""" + return self.session_dir / 'work' + + @property + def plot_dir(self): + """Return plot directory.""" + return self.session_dir / 'plots' + + @property + def run_dir(self): + """Return run directory.""" + return self.session_dir / 'run' + + @property + def config_dir(self): + """Return user config directory.""" + return USER_CONFIG_DIR + + +def _read_config_file(config_file): + """Read config user file and store settings in a dictionary.""" + config_file = Path(config_file) + if not config_file.exists(): + raise IOError(f'Config file `{config_file}` does not exist.') + + with open(config_file, 'r') as file: + cfg = yaml.safe_load(file) + + return cfg + + +DEFAULT_CONFIG_DIR = Path(esmvalcore.__file__).parent +DEFAULT_CONFIG = DEFAULT_CONFIG_DIR / 'config-user.yml' + +USER_CONFIG_DIR = Path.home() / '.esmvaltool' +USER_CONFIG = USER_CONFIG_DIR / 'config-user.yml' + +# initialize placeholders +CFG_DEFAULT = Config._load_default_config(DEFAULT_CONFIG) +CFG = Config._load_user_config(USER_CONFIG, raise_exception=False) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py new file mode 100644 index 0000000000..bd55a6ffff --- /dev/null +++ b/esmvalcore/experimental/config/_config_validators.py @@ -0,0 +1,286 @@ +"""List of config validators.""" + +import warnings +from collections.abc import Iterable +from functools import lru_cache +from pathlib import Path + +from esmvalcore import __version__ as current_version +from esmvalcore._config import load_config_developer +from esmvalcore._recipe import TASKSEP +from esmvalcore.cmor.check import CheckLevels + + +class ValidationError(ValueError): + """Custom validation error.""" + + +# Custom warning, because DeprecationWarning is hidden by default +class ESMValToolDeprecationWarning(UserWarning): + """Configuration key has been deprecated.""" + + +# The code for this function was taken from matplotlib (v3.3) and modified +# to fit the needs of ESMValCore. Matplotlib is licenced under the terms of +# the the 'Python Software Foundation License' +# (https://www.python.org/psf/license) +def _make_type_validator(cls, *, allow_none=False): + """Construct a type validator for `cls`. + + Return a validator that converts inputs to *cls* or raises (and + possibly allows ``None`` as well). + """ + def validator(inp): + looks_like_none = isinstance(inp, str) and (inp.lower() == "none") + if (allow_none and (inp is None or looks_like_none)): + return None + try: + return cls(inp) + except ValueError as err: + if isinstance(cls, type): + raise ValidationError( + f'Could not convert {repr(inp)} to {cls.__name__}' + ) from err + raise + + validator.__name__ = f"validate_{cls.__name__}" + if allow_none: + validator.__name__ += "_or_None" + validator.__qualname__ = (validator.__qualname__.rsplit(".", 1)[0] + "." + + validator.__name__) + return validator + + +# The code for this function was taken from matplotlib (v3.3) and modified +# to fit the needs of ESMValCore. Matplotlib is licenced under the terms of +# the the 'Python Software Foundation License' +# (https://www.python.org/psf/license) +@lru_cache() +def _listify_validator(scalar_validator, + allow_stringlist=False, + *, + n_items=None, + docstring=None): + """Apply the validator to a list.""" + def func(inp): + if isinstance(inp, str): + try: + inp = [ + scalar_validator(val.strip()) for val in inp.split(',') + if val.strip() + ] + except Exception: + if allow_stringlist: + # Sometimes, a list of colors might be a single string + # of single-letter colornames. So give that a shot. + inp = [ + scalar_validator(val.strip()) for val in inp + if val.strip() + ] + else: + raise + # Allow any ordered sequence type -- generators, np.ndarray, pd.Series + # -- but not sets, whose iteration order is non-deterministic. + elif isinstance(inp, + Iterable) and not isinstance(inp, (set, frozenset)): + # The condition on this list comprehension will preserve the + # behavior of filtering out any empty strings (behavior was + # from the original validate_stringlist()), while allowing + # any non-string/text scalar values such as numbers and arrays. + inp = [ + scalar_validator(val) for val in inp + if not isinstance(val, str) or val + ] + else: + raise ValidationError( + f"Expected str or other non-set iterable, but got {inp}") + if n_items is not None and len(inp) != n_items: + raise ValidationError(f"Expected {n_items} values, " + f"but there are {len(inp)} values in {inp}") + return inp + + try: + func.__name__ = "{}list".format(scalar_validator.__name__) + except AttributeError: # class instance. + func.__name__ = "{}List".format(type(scalar_validator).__name__) + func.__qualname__ = func.__qualname__.rsplit(".", + 1)[0] + "." + func.__name__ + if docstring is not None: + docstring = scalar_validator.__doc__ + func.__doc__ = docstring + return func + + +def validate_bool(value, allow_none=False): + """Check if the value can be evaluate as a boolean.""" + if (value is None) and allow_none: + return value + if not isinstance(value, bool): + raise ValidationError(f"Could not convert `{value}` to `bool`") + return value + + +def validate_path(value, allow_none=False): + """Return a `Path` object.""" + if (value is None) and allow_none: + return value + try: + path = Path(value).expanduser().absolute() + except TypeError as err: + raise ValidationError(f"Expected a path, but got {value}") from err + else: + return path + + +def validate_positive(value): + """Check if number is positive.""" + if value is not None and value <= 0: + raise ValidationError(f'Expected a positive number, but got {value}') + return value + + +def _chain_validator(*funcs): + """Chain a series of validators.""" + def chained(value): + for func in funcs: + value = func(value) + return value + + return chained + + +validate_string = _make_type_validator(str) +validate_string_or_none = _make_type_validator(str, allow_none=True) +validate_stringlist = _listify_validator(validate_string, + docstring='Return a list of strings.') +validate_int = _make_type_validator(int) +validate_int_or_none = _make_type_validator(int, allow_none=True) +validate_float = _make_type_validator(float) +validate_floatlist = _listify_validator(validate_float, + docstring='Return a list of floats.') + +validate_dict = _make_type_validator(dict) + +validate_path_or_none = _make_type_validator(validate_path, allow_none=True) + +validate_pathlist = _listify_validator(validate_path, + docstring='Return a list of paths.') + +validate_int_positive = _chain_validator(validate_int, validate_positive) +validate_int_positive_or_none = _make_type_validator(validate_int_positive, + allow_none=True) + + +def validate_oldstyle_rootpath(value): + """Validate `rootpath` mapping.""" + mapping = validate_dict(value) + return mapping + + +def validate_oldstyle_drs(value): + """Validate `drs` mapping.""" + mapping = validate_dict(value) + return mapping + + +def validate_config_developer(value): + """Validate and load config developer path.""" + path = validate_path_or_none(value) + + load_config_developer(path) + + return path + + +def validate_check_level(value): + """Validate CMOR level check.""" + if isinstance(value, str): + try: + value = CheckLevels[value.upper()] + except KeyError: + raise ValidationError( + f'`{value}` is not a valid strictness level') from None + + else: + value = CheckLevels(value) + + return value + + +def validate_diagnostics(diagnostics): + """Validate diagnostic location.""" + if isinstance(diagnostics, str): + diagnostics = diagnostics.strip().split(' ') + return { + pattern if TASKSEP in pattern else pattern + TASKSEP + '*' + for pattern in diagnostics or () + } + + +def deprecate(func, variable, version: str = None): + """Wrap function to mark variables to be deprecated. + + This will give a warning if the function will be/has been deprecated. + + Parameters + ---------- + func: + Validator function to wrap + variable: str + Name of the variable to deprecate + version: str + Version to deprecate the variable in, should be something + like '2.2.3' + """ + if not version: + version = 'a future version' + + if current_version >= version: + warnings.warn(f"`{variable}` has been removed in {version}", + ESMValToolDeprecationWarning) + else: + warnings.warn(f"`{variable}` will be removed in {version}.", + ESMValToolDeprecationWarning, + stacklevel=2) + + return func + + +_validators = { + # deprecate in 2.2.0 + 'write_plots': deprecate(validate_bool, 'write_plots', '2.2.0'), + 'write_netcdf': deprecate(validate_bool, 'write_netcdf', '2.2.0'), + 'output_file_type': deprecate(validate_string, 'output_file_type', + '2.2.0'), + + # From user config + 'log_level': validate_string, + 'exit_on_warning': validate_bool, + 'output_dir': validate_path, + 'auxiliary_data_dir': validate_path, + 'compress_netcdf': validate_bool, + 'save_intermediary_cubes': validate_bool, + 'remove_preproc_dir': validate_bool, + 'max_parallel_tasks': validate_int_or_none, + 'config_developer_file': validate_config_developer, + 'profile_diagnostic': validate_bool, + 'run_diagnostic': validate_bool, + + # From CLI + "skip-nonexistent": validate_bool, + "diagnostics": validate_diagnostics, + "check_level": validate_check_level, + "synda_download": validate_bool, + 'max_years': validate_int_positive_or_none, + 'max_datasets': validate_int_positive_or_none, + + # From recipe + 'write_ncl_interface': validate_bool, + + # oldstyle + 'rootpath': validate_oldstyle_rootpath, + 'drs': validate_oldstyle_drs, + + # config location + 'config_file': validate_path, +} diff --git a/esmvalcore/experimental/config/_validated_config.py b/esmvalcore/experimental/config/_validated_config.py new file mode 100644 index 0000000000..e3366d25a1 --- /dev/null +++ b/esmvalcore/experimental/config/_validated_config.py @@ -0,0 +1,90 @@ +"""Config validation objects.""" + +import pprint +import warnings +from collections.abc import MutableMapping + +from .._exceptions import SuppressedError +from ._config_validators import ValidationError + + +class InvalidConfigParameter(SuppressedError): + """Config parameter is invalid.""" + + +class MissingConfigParameter(UserWarning): + """Config parameter is missing.""" + + +# The code for this class was take from matplotlib (v3.3) and modified to +# fit the needs of ESMValCore. Matplotlib is licenced under the terms of +# the the 'Python Software Foundation License' +# (https://www.python.org/psf/license) +class ValidatedConfig(MutableMapping, dict): + """Based on `matplotlib.rcParams`.""" + + _validate = {} + _warn_if_missing = () + + # validate values on the way in + def __init__(self, *args, **kwargs): + super().__init__() + self.update(*args, **kwargs) + self.check_missing() + + def __setitem__(self, key, val): + """Map key to value.""" + try: + cval = self._validate[key](val) + except ValidationError as verr: + raise InvalidConfigParameter(f"Key `{key}`: {verr}") from None + except KeyError: + raise InvalidConfigParameter( + f"`{key}` is not a valid config parameter.") from None + + dict.__setitem__(self, key, cval) + + def __getitem__(self, key): + """Return value mapped by key.""" + return dict.__getitem__(self, key) + + def __repr__(self): + """Return canonical string representation.""" + class_name = self.__class__.__name__ + indent = len(class_name) + 1 + repr_split = pprint.pformat(dict(self), indent=1, + width=80 - indent).split('\n') + repr_indented = ('\n' + ' ' * indent).join(repr_split) + return '{}({})'.format(class_name, repr_indented) + + def __str__(self): + """Return string representation.""" + return '\n'.join(map('{0[0]}: {0[1]}'.format, sorted(self.items()))) + + def __iter__(self): + """Yield sorted list of keys.""" + yield from sorted(dict.__iter__(self)) + + def __len__(self): + """Return number of config keys.""" + return dict.__len__(self) + + def __delitem__(self, key): + """Delete key/value from config.""" + dict.__delitem__(self, key) + + def check_missing(self): + """Check and warn for missing variables.""" + for (key, more_info) in self._warn_if_missing: + if key not in self: + more_info = f' ({more_info})' if more_info else '' + warnings.warn(f'`{key}` is not defined{more_info}', + MissingConfigParameter) + + def copy(self): + """Copy the keys/values of this object to a dict.""" + return {k: dict.__getitem__(self, k) for k in self} + + def clear(self): + """Clear Config.""" + dict.clear(self) diff --git a/tests/unit/experimental/test_config.py b/tests/unit/experimental/test_config.py new file mode 100644 index 0000000000..ac74b3b856 --- /dev/null +++ b/tests/unit/experimental/test_config.py @@ -0,0 +1,251 @@ +from pathlib import Path + +import numpy as np +import pytest + +from esmvalcore import __version__ as current_version +from esmvalcore.experimental.config._config_object import Config +from esmvalcore.experimental.config._config_validators import ( + _listify_validator, + deprecate, + validate_bool, + validate_check_level, + validate_diagnostics, + validate_float, + validate_int, + validate_int_or_none, + validate_int_positive_or_none, + validate_path, + validate_path_or_none, + validate_positive, + validate_string, + validate_string_or_none, +) +from esmvalcore.experimental.config._validated_config import ( + InvalidConfigParameter, ) + + +def generate_validator_testcases(valid): + # The code for this function was taken from matplotlib (v3.3) and modified + # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of + # the the 'Python Software Foundation License' + # (https://www.python.org/psf/license) + + validation_tests = ( + { + 'validator': validate_bool, + 'success': ((True, True), (False, False)), + 'fail': ((_, ValueError) for _ in ('fail', 2, -1, [])) + }, + { + 'validator': validate_check_level, + 'success': ( + (1, 1), + (5, 5), + ('dEBUG', 1), + ('default', 3), + ), + 'fail': ( + (6, ValueError), + (0, ValueError), + ('fail', ValueError), + ), + }, + { + 'validator': + validate_diagnostics, + 'success': ( + ('/', {'/'}), + ('a ', {'a/*'}), + ('/ a ', {'/', 'a/*'}), + ('/ a a', {'/', 'a/*'}), + (('/', 'a'), {'/', 'a/*'}), + ([], set()), + ), + 'fail': ( + (1, TypeError), + ([1, 2], TypeError), + ), + }, + { + 'validator': + _listify_validator(validate_float, n_items=2), + 'success': + ((_, [1.5, 2.5]) + for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), + np.array((1.5, 2.5)))), + 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + }, + { + 'validator': + _listify_validator(validate_float, n_items=2), + 'success': + ((_, [1.5, 2.5]) + for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), + np.array((1.5, 2.5)))), + 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + }, + { + 'validator': + _listify_validator(validate_int, n_items=2), + 'success': + ((_, [1, 2]) + for _ in ('1, 2', [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2)))), + 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + }, + { + 'validator': validate_int_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + { + 'validator': validate_int_positive_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + { + 'validator': + validate_path, + 'success': ( + ('a/b/c', Path.cwd() / 'a' / 'b' / 'c'), + ('/a/b/c/', Path('/', 'a', 'b', 'c')), + ('~/', Path.home()), + ), + 'fail': ( + (None, ValueError), + (123, ValueError), + (False, ValueError), + ([], ValueError), + ), + }, + { + 'validator': validate_path_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + { + 'validator': validate_positive, + 'success': ( + (0.1, 0.1), + (1, 1), + (1.5, 1.5), + ), + 'fail': ( + (0, ValueError), + (-1, ValueError), + ('fail', TypeError), + ), + }, + { + 'validator': + _listify_validator(validate_string), + 'success': ( + ('', []), + ('a,b', ['a', 'b']), + ('abc', ['abc']), + ('abc, ', ['abc']), + ('abc, ,', ['abc']), + (['a', 'b'], ['a', 'b']), + (('a', 'b'), ['a', 'b']), + (iter(['a', 'b']), ['a', 'b']), + (np.array(['a', 'b']), ['a', 'b']), + ((1, 2), ['1', '2']), + (np.array([1, 2]), ['1', '2']), + ), + 'fail': ( + (set(), ValueError), + (1, ValueError), + ) + }, + { + 'validator': validate_string_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + ) + + for validator_dict in validation_tests: + validator = validator_dict['validator'] + if valid: + for arg, target in validator_dict['success']: + yield validator, arg, target + else: + for arg, error_type in validator_dict['fail']: + yield validator, arg, error_type + + +@pytest.mark.parametrize('validator, arg, target', + generate_validator_testcases(True)) +def test_validator_valid(validator, arg, target): + res = validator(arg) + assert res == target + + +@pytest.mark.parametrize('validator, arg, exception_type', + generate_validator_testcases(False)) +def test_validator_invalid(validator, arg, exception_type): + with pytest.raises(exception_type): + validator(arg) + + +@pytest.mark.parametrize('version', (current_version, '0.0.1', '9.9.9')) +def test_deprecate(version): + def test_func(): + pass + + # This always warns + with pytest.warns(UserWarning): + f = deprecate(test_func, 'test_var', version) + + assert callable(f) + + +def test_config_class(): + config = { + 'log_level': 'info', + 'exit_on_warning': False, + 'output_file_type': 'png', + 'output_dir': './esmvaltool_output', + 'auxiliary_data_dir': './auxiliary_data', + 'save_intermediary_cubes': False, + 'remove_preproc_dir': True, + 'max_parallel_tasks': None, + 'profile_diagnostic': False, + 'rootpath': { + 'CMIP6': '~/data/CMIP6' + }, + 'drs': { + 'CMIP6': 'default' + }, + } + + cfg = Config(config) + + assert isinstance(cfg['output_dir'], Path) + assert isinstance(cfg['auxiliary_data_dir'], Path) + + from esmvalcore._config import CFG as CFG_DEV + assert CFG_DEV + + +def test_config_update(): + config = Config({'output_dir': 'directory'}) + fail_dict = {'output_dir': 123} + + with pytest.raises(InvalidConfigParameter): + config.update(fail_dict) + + +def test_config_init(): + config = Config() + assert isinstance(config, dict) + + +def test_session(): + config = Config({'output_dir': 'config'}) + + session = config.start_session('recipe_name') + assert session == config + + session['output_dir'] = 'session' + session != config