From eeffeed6d57761313eadeb849c11ca0c471aa3bd Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 13:44:36 +0100 Subject: [PATCH 01/45] Add hidden defaults and make config-user.yml valid as default fallback --- esmvalcore/config-user.yml | 39 ++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index 805a3783c9..b51c8d0f4a 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -37,23 +37,42 @@ config_developer_file: null # Get profiling information for diagnostics # Only available for Python diagnostics profile_diagnostic: false +# Maximum number of years to use. +max_years: null +# Run the diagnostic +run_diagnostic: true +# If True, the run will not fail if some datasets are not available. +skip-nonexistent: False +# Only run the selected diagnostics from the recipe. +diagnostics: null +# Configure the sensitivity of the CMOR check. +# Possible values are: +# `ignore` (all errors will be reported as warnings), +# `relaxed` (only fail if there are critical errors), +# `default` (fail if there are any errors), +# `strict` (fail if there are any warnings). +check_level: default +# If True, the tool will try to download missing data using Synda. +synda_download: False +# Maximum number of datasets to use. +max_datasets: null # Rootpaths to the data from different projects (lists are also possible) # these are generic entries to better allow you to enter your own # For site-specific entries, see below -#rootpath: -# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] -# OBS: ~/obs_inputpath -# RAWOBS: ~/rawobs_inputpath -# default: ~/default_inputpath -# CORDEX: ~/default_inputpath +rootpath: + CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] + OBS: ~/obs_inputpath + RAWOBS: ~/rawobs_inputpath + default: ~/default_inputpath + CORDEX: ~/default_inputpath # Directory structure for input data: [default]/BADC/DKRZ/ETHZ/etc # See config-developer.yml for definitions. -#drs: -# CMIP5: default -# CORDEX: default -# OBS: default +drs: + CMIP5: default + CORDEX: default + OBS: default # Site-specific entries: Jasmin # Uncomment the lines below to locate data on JASMIN From 71dc0daee5a5a5c6318904d45eb9e38ccbfd4b78 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 13:45:56 +0100 Subject: [PATCH 02/45] Expose functionality to load config developer --- esmvalcore/_config.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 9f433e5d6c..fade94027c 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -99,10 +99,7 @@ def read_config_user_file(config_file, folder_name, options=None): cfg['run_dir'] = os.path.join(cfg['output_dir'], 'run') # Read developer configuration file - cfg_developer = read_config_developer_file(cfg['config_developer_file']) - for key, value in cfg_developer.items(): - CFG[key] = value - read_cmor_tables(CFG) + load_config_developer(cfg['config_developer_file']) return cfg @@ -142,6 +139,15 @@ def read_config_developer_file(cfg_file=None): return cfg +def load_config_developer(cfg_file=None): + """Load the config developer file into the CFG object and initialize cmor + tables.""" + cfg_developer = read_config_developer_file(cfg_file) + for key, value in cfg_developer.items(): + CFG[key] = value + read_cmor_tables(CFG) + + def configure_logging(cfg_file=None, output_dir=None, console_log_level=None): """Set up logging.""" if cfg_file is None: From 04d56cb34fca88cf593b275701f89b739e5fc2f3 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 13:49:39 +0100 Subject: [PATCH 03/45] Implement importable config object for future API --- esmvalcore/__init__.py | 1 + esmvalcore/future/__init__.py | 5 + esmvalcore/future/_exceptions.py | 23 ++ esmvalcore/future/config/__init__.py | 5 + esmvalcore/future/config/_config_object.py | 163 +++++++++++ .../future/config/_config_validators.py | 270 ++++++++++++++++++ esmvalcore/future/config/_validated_config.py | 73 +++++ 7 files changed, 540 insertions(+) create mode 100644 esmvalcore/future/__init__.py create mode 100644 esmvalcore/future/_exceptions.py create mode 100644 esmvalcore/future/config/__init__.py create mode 100644 esmvalcore/future/config/_config_object.py create mode 100644 esmvalcore/future/config/_config_validators.py create mode 100644 esmvalcore/future/config/_validated_config.py diff --git a/esmvalcore/__init__.py b/esmvalcore/__init__.py index 4d1bb19255..4941523f4b 100644 --- a/esmvalcore/__init__.py +++ b/esmvalcore/__init__.py @@ -9,6 +9,7 @@ __all__ = [ '__version__', + 'future' 'cmor', 'preprocessor', ] diff --git a/esmvalcore/future/__init__.py b/esmvalcore/future/__init__.py new file mode 100644 index 0000000000..5d272c9a69 --- /dev/null +++ b/esmvalcore/future/__init__.py @@ -0,0 +1,5 @@ +from .config import CFG + +__all__ = [ + 'CFG', +] diff --git a/esmvalcore/future/_exceptions.py b/esmvalcore/future/_exceptions.py new file mode 100644 index 0000000000..2edfbafeaf --- /dev/null +++ b/esmvalcore/future/_exceptions.py @@ -0,0 +1,23 @@ +import sys + + +class SuppressedError(Exception): + """Errors subclassed from SuppressedError hide the full traceback. + + This can be used for simple user-facing errors that do not need the + full traceback. + """ + pass + + +def _suppressed_hook(error, message, traceback): + """https://stackoverflow.com/a/27674608.""" + if issubclass(error, SuppressedError): + # Print only the message and hide the traceback + print(f'{error.__name__}: {message}'.format(error.__name__, message)) + else: + # Print full traceback + sys.__excepthook__(error, message, traceback) + + +sys.excepthook = _suppressed_hook diff --git a/esmvalcore/future/config/__init__.py b/esmvalcore/future/config/__init__.py new file mode 100644 index 0000000000..457c408639 --- /dev/null +++ b/esmvalcore/future/config/__init__.py @@ -0,0 +1,5 @@ +from ._config_object import CFG + +__all__ = [ + 'CFG', +] diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py new file mode 100644 index 0000000000..0a3973014b --- /dev/null +++ b/esmvalcore/future/config/_config_object.py @@ -0,0 +1,163 @@ +import os +from datetime import datetime +from pathlib import Path + +import yaml + +import esmvalcore + +from ._config_validators import _validators +from ._validated_config import ValidatedConfig + + +class ESMValCoreConfig(ValidatedConfig): + """The ESMValCore config object.""" + validate = _validators + + @staticmethod + def load_from_file(filename): + """Reload user configuration from the given file.""" + path = Path(filename).expanduser() + if not path.exists(): + try_path = USER_CONFIG_DIR / filename + if try_path.exists(): + path = try_path + else: + raise FileNotFoundError(f'No such file: `{filename}`') + + _load_user_config(path) + + def start_session(self, name): + return Session(name, self.copy()) + + +class Session(ValidatedConfig): + """This class holds information about the current session. Different + session directories can be accessed. + + Parameters + ---------- + name : str + Name of the session to initialize, for example, the name of the + recipe (default='session'). + """ + validate = _validators + + def __init__(self, name: str = 'session', *args, **kwargs): + super().__init__(*args, **kwargs) + self.init_session_dir(name) + + def init_session_dir(self, name: str = 'session'): + """Initialize session. + + The `name` is used to name the working directory, e.g. + `recipe_example_20200916/`. If no name is given, such as in an + interactive session, defaults to `session`. + """ + now = datetime.utcnow().strftime("%Y%m%d_%H%M%S") + session_name = f"{name}_{now}" + self._session_dir = self['output_dir'] / session_name + + @property + def session_dir(self): + return self._session_dir + + @property + def preproc_dir(self): + return self.session_dir / 'preproc' + + @property + def work_dir(self): + return self.session_dir / 'work' + + @property + def plot_dir(self): + return self.session_dir / 'plots' + + @property + def run_dir(self): + return self.session_dir / 'run' + + @property + def config_dir(self): + return USER_CONFIG_DIR + + +def read_config_file(config_file): + """Read config user file and store settings in a dictionary.""" + config_file = Path(config_file) + if not config_file.exists(): + raise IOError(f'Config file `{config_file}` does not exist.') + + with open(config_file, 'r') as file: + cfg = yaml.safe_load(file) + + return cfg + + +def _load_default_config(filename: str): + """Load the default configuration.""" + mapping = read_config_file(filename) + + global config_default + + CFG_default.update(mapping) + + +def _load_user_config(filename: str, raise_exception: bool = True): + """Load user configuration from the given file (`filename`). + + The config cleared and updated in-place. + + Parameters + ---------- + raise_exception : bool + Raise an exception if `filename` can not be found (default). + Otherwise, silently pass and use the default configuration. This + setting is necessary for the case where `.esmvalcore/config-user.yml` + has not been defined (i.e. first start). + """ + try: + mapping = read_config_file(filename) + except IOError: + if raise_exception: + raise + mapping = {} + + mapping['config_file'] = filename + + global CFG + global CFG_orig + + CFG.clear() + CFG.update(CFG_default) + CFG.update(mapping) + + CFG_orig = ESMValCoreConfig(CFG.copy()) + + +def get_user_config_location(): + """Check if environment variable `ESMVALTOOL_CONFIG` exists, otherwise use + the default config location.""" + try: + config_location = Path(os.environ['ESMVALTOOL_CONFIG']) + except KeyError: + config_location = USER_CONFIG_DIR / 'config-user.yml' + + return config_location + + +DEFAULT_CONFIG_DIR = Path(esmvalcore.__file__).parent +DEFAULT_CONFIG = DEFAULT_CONFIG_DIR / 'config-user.yml' + +USER_CONFIG_DIR = Path.home() / '.esmvaltool' +USER_CONFIG = get_user_config_location() + +# initialize placeholders +CFG_default = ESMValCoreConfig() +CFG = ESMValCoreConfig() +CFG_orig = ESMValCoreConfig() + +# update config objects +_load_default_config(DEFAULT_CONFIG) +_load_user_config(USER_CONFIG, raise_exception=False) diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/future/config/_config_validators.py new file mode 100644 index 0000000000..d6995d9a18 --- /dev/null +++ b/esmvalcore/future/config/_config_validators.py @@ -0,0 +1,270 @@ +from collections.abc import Iterable +from functools import lru_cache +from pathlib import Path + + +def _make_type_validator(cls, *, allow_none=False): + """Return a validator that converts inputs to *cls* or raises (and possibly + allows ``None`` as well).""" + + # The code for this function was taken from matplotlib (v3.3) and modified + # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of + # the the 'Python Software Foundation License' + # (https://www.python.org/psf/license) + + def validator(s): + if (allow_none + and (s is None or isinstance(s, str) and s.lower() == "none")): + return None + try: + return cls(s) + except ValueError as e: + if isinstance(cls, type): + raise ValueError( + f'Could not convert {repr(s)} to {cls.__name__}') from e + else: + raise + + validator.__name__ = f"validate_{cls.__name__}" + if allow_none: + validator.__name__ += "_or_None" + validator.__qualname__ = (validator.__qualname__.rsplit(".", 1)[0] + "." + + validator.__name__) + return validator + + +@lru_cache +def _listify_validator(scalar_validator, + allow_stringlist=False, + *, + n=None, + doc=None): + """Apply the validator to a list.""" + + # The code for this function was taken from matplotlib (v3.3) and modified + # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of + # the the 'Python Software Foundation License' + # (https://www.python.org/psf/license) + + def f(s): + if isinstance(s, str): + try: + val = [ + scalar_validator(v.strip()) for v in s.split(',') + if v.strip() + ] + except Exception: + if allow_stringlist: + # Sometimes, a list of colors might be a single string + # of single-letter colornames. So give that a shot. + val = [scalar_validator(v.strip()) for v in s if v.strip()] + else: + raise + # Allow any ordered sequence type -- generators, np.ndarray, pd.Series + # -- but not sets, whose iteration order is non-deterministic. + elif isinstance(s, Iterable) and not isinstance(s, (set, frozenset)): + # The condition on this list comprehension will preserve the + # behavior of filtering out any empty strings (behavior was + # from the original validate_stringlist()), while allowing + # any non-string/text scalar values such as numbers and arrays. + val = [ + scalar_validator(v) for v in s if not isinstance(v, str) or v + ] + else: + raise ValueError( + f"Expected str or other non-set iterable, but got {s}") + if n is not None and len(val) != n: + raise ValueError( + f"Expected {n} values, but there are {len(val)} values in {s}") + return val + + try: + f.__name__ = "{}list".format(scalar_validator.__name__) + except AttributeError: # class instance. + f.__name__ = "{}List".format(type(scalar_validator).__name__) + f.__qualname__ = f.__qualname__.rsplit(".", 1)[0] + "." + f.__name__ + f.__doc__ = doc if doc is not None else scalar_validator.__doc__ + return f + + +def validate_bool(value, allow_none=False): + """Check if the value can be evaluate as a boolean.""" + if (value is None) and allow_none: + return value + if not isinstance(value, bool): + raise ValueError(f"Could not convert `{value}` to `bool`") + return value + + +def validate_path(value, allow_none=False): + """Return a path object.""" + if (value is None) and allow_none: + return value + try: + path = Path(value).expanduser().absolute() + except TypeError as e: + raise ValueError(f"Expected a path, but got {value}") from e + else: + return path + + +def validate_positive(value): + """Check if number is positive.""" + if value is not None and value <= 0: + raise ValueError(f'Expected a positive number, but got {value}') + return value + + +def _chain_validator(*funcs): + """Chain a series of validators.""" + def chained(value): + for func in funcs: + value = func(value) + return value + + return chained + + +validate_string = _make_type_validator(str) +validate_string_or_none = _make_type_validator(str, allow_none=True) +validate_stringlist = _listify_validator(validate_string, + doc='return a list of strings') +validate_int = _make_type_validator(int) +validate_int_or_none = _make_type_validator(int, allow_none=True) +validate_float = _make_type_validator(float) +validate_floatlist = _listify_validator(validate_float, + doc='return a list of floats') + +validate_dict = _make_type_validator(dict) + +validate_path_or_none = _make_type_validator(validate_path, allow_none=True) + +validate_pathlist = _listify_validator(validate_path, + doc='return a list of paths') + +validate_int_positive = _chain_validator(validate_int, validate_positive) +validate_int_positive_or_none = _make_type_validator(validate_int_positive, + allow_none=True) + + +def validate_oldstyle_rootpath(value): + mapping = validate_dict(value) + return mapping + + +def validate_oldstyle_drs(value): + mapping = validate_dict(value) + return mapping + + +def validate_config_developer(value): + from esmvalcore._config import load_config_developer + path = validate_path_or_none(value) + + load_config_developer(path) + + return path + + +def validate_check_level(value): + from esmvalcore.cmor.check import CheckLevels + + if isinstance(value, str): + try: + value = CheckLevels[value.upper()] + except KeyError: + raise ValueError(f'`{value}` is not a valid strictness level') + + else: + value = CheckLevels(value) + + return value + + +def validate_diagnostics(diagnostics): + from esmvalcore._recipe import TASKSEP + + if isinstance(diagnostics, str): + diagnostics = diagnostics.strip().split(' ') + return { + pattern if TASKSEP in pattern else pattern + TASKSEP + '*' + for pattern in diagnostics or () + } + + +class ESMValToolDeprecationWarning(UserWarning): + # Custom warning, because DeprecationWarning is hidden by default + pass + + +def deprecate(func, variable, version: str = None): + """Wrapper function to mark variables to be deprecated. + + This will give a warning if the function will be/has been deprecated. + + Parameters + ---------- + func: + Validator function to wrap + variable: str + Name of the variable to deprecate + version: str + Version to deprecate the variable in, should be something + like '2.2.3' + """ + import warnings + + from esmvalcore import __version__ as current_version + + if not version: + version = 'a future version' + + if current_version >= version: + warnings.warn(f"`{variable}` has been deprecated in {version}", + ESMValToolDeprecationWarning) + else: + warnings.warn(f"`{variable}` will be deprecated in {version}.", + ESMValToolDeprecationWarning, + stacklevel=2) + + return func + + +_validators = { + # deprecate in 2.2.0 + 'write_plots': deprecate(validate_bool, 'write_plots', '2.2.0'), + 'write_netcdf': deprecate(validate_bool, 'write_netcdf', '2.2.0'), + 'output_file_type': deprecate(validate_string, 'output_file_type', + '2.2.0'), + + # From user config + 'log_level': validate_string, + 'exit_on_warning': validate_bool, + 'output_dir': validate_path, + 'auxiliary_data_dir': validate_path, + 'compress_netcdf': validate_bool, + 'save_intermediary_cubes': validate_bool, + 'remove_preproc_dir': validate_bool, + 'max_parallel_tasks': validate_int_or_none, + 'config_developer_file': validate_config_developer, + 'profile_diagnostic': validate_bool, + 'run_diagnostic': validate_bool, + + # From CLI + "skip-nonexistent": validate_bool, + "diagnostics": validate_diagnostics, + "check_level": validate_check_level, + "synda_download": validate_bool, + 'max_years': validate_int_positive_or_none, + 'max_datasets': validate_int_positive_or_none, + + # From recipe + 'write_ncl_interface': validate_bool, + + # oldstyle + 'rootpath': validate_oldstyle_rootpath, + 'drs': validate_oldstyle_drs, + + # config location + 'config_file': validate_path, +} diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py new file mode 100644 index 0000000000..542f977e7a --- /dev/null +++ b/esmvalcore/future/config/_validated_config.py @@ -0,0 +1,73 @@ +import pprint +import re +from collections.abc import MutableMapping + +from .._exceptions import SuppressedError + + +class InvalidConfigParameter(SuppressedError): + pass + + +class ValidatedConfig(MutableMapping, dict): + """Based on `matplotlib.rcParams`.""" + # The code for this class was take from matplotlib (v3.3) and modified to + # fit the needs of ESMValCore. Matplotlib is licenced under the terms of + # the the 'Python Software Foundation License' + # (https://www.python.org/psf/license) + + validate = {} + + # validate values on the way in + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + + def __setitem__(self, key, val): + try: + cval = self.validate[key](val) + except ValueError as ve: + raise InvalidConfigParameter(f"Key `{key}`: {ve}") from None + except KeyError: + raise InvalidConfigParameter( + f"`{key}` is not a valid config parameter.") from None + + dict.__setitem__(self, key, cval) + + def __getitem__(self, key): + return dict.__getitem__(self, key) + + def __repr__(self): + class_name = self.__class__.__name__ + indent = len(class_name) + 1 + repr_split = pprint.pformat(dict(self), indent=1, + width=80 - indent).split('\n') + repr_indented = ('\n' + ' ' * indent).join(repr_split) + return '{}({})'.format(class_name, repr_indented) + + def __str__(self): + return '\n'.join(map('{0[0]}: {0[1]}'.format, sorted(self.items()))) + + def __iter__(self): + """Yield sorted list of keys.""" + yield from sorted(dict.__iter__(self)) + + def __len__(self): + return dict.__len__(self) + + def find_all(self, pattern): + """Return the subset of this Config dictionary whose keys match, using + `re.search` with the given `pattern`. + + Changes to the returned dictionary are *not* propagated to the + parent Config dictionary. + """ + pattern_re = re.compile(pattern) + return self.__class__((key, value) for key, value in self.items() + if pattern_re.search(key)) + + def copy(self): + return {k: dict.__getitem__(self, k) for k in self} + + def clear(self): + """Clear Config dictionary.""" + dict.clear(self) From 076880698ce27d932e80b46f722096b6e25e8d57 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 14:31:17 +0100 Subject: [PATCH 04/45] Add tests for config + validation --- tests/unit/future/test_config.py | 250 +++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 tests/unit/future/test_config.py diff --git a/tests/unit/future/test_config.py b/tests/unit/future/test_config.py new file mode 100644 index 0000000000..499e192507 --- /dev/null +++ b/tests/unit/future/test_config.py @@ -0,0 +1,250 @@ +from pathlib import Path + +import numpy as np +import pytest + +from esmvalcore import __version__ as current_version +from esmvalcore.future.config._config_object import ESMValCoreConfig +from esmvalcore.future.config._config_validators import ( + _listify_validator, + deprecate, + validate_bool, + validate_check_level, + validate_diagnostics, + validate_float, + validate_int, + validate_int_or_none, + validate_int_positive_or_none, + validate_path, + validate_path_or_none, + validate_positive, + validate_string, + validate_string_or_none, +) +from esmvalcore.future.config._validated_config import InvalidConfigParameter + + +def generate_validator_testcases(valid): + # The code for this function was taken from matplotlib (v3.3) and modified + # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of + # the the 'Python Software Foundation License' + # (https://www.python.org/psf/license) + + validation_tests = ( + { + 'validator': validate_bool, + 'success': ((True, True), (False, False)), + 'fail': ((_, ValueError) for _ in ('fail', 2, -1, [])) + }, + { + 'validator': validate_check_level, + 'success': ( + (1, 1), + (5, 5), + ('dEBUG', 1), + ('default', 3), + ), + 'fail': ( + (6, ValueError), + (0, ValueError), + ('fail', ValueError), + ), + }, + { + 'validator': + validate_diagnostics, + 'success': ( + ('/', {'/'}), + ('a ', {'a/*'}), + ('/ a ', {'/', 'a/*'}), + ('/ a a', {'/', 'a/*'}), + (('/', 'a'), {'/', 'a/*'}), + ([], set()), + ), + 'fail': ( + (1, TypeError), + ([1, 2], TypeError), + ), + }, + { + 'validator': + _listify_validator(validate_float, n=2), + 'success': + ((_, [1.5, 2.5]) + for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), + np.array((1.5, 2.5)))), + 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + }, + { + 'validator': + _listify_validator(validate_float, n=2), + 'success': + ((_, [1.5, 2.5]) + for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), + np.array((1.5, 2.5)))), + 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + }, + { + 'validator': + _listify_validator(validate_int, n=2), + 'success': + ((_, [1, 2]) + for _ in ('1, 2', [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2)))), + 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) + }, + { + 'validator': validate_int_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + { + 'validator': validate_int_positive_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + { + 'validator': + validate_path, + 'success': ( + ('a/b/c', Path.cwd() / 'a' / 'b' / 'c'), + ('/a/b/c/', Path('/', 'a', 'b', 'c')), + ('~/', Path.home()), + ), + 'fail': ( + (None, ValueError), + (123, ValueError), + (False, ValueError), + ([], ValueError), + ), + }, + { + 'validator': validate_path_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + { + 'validator': validate_positive, + 'success': ( + (0.1, 0.1), + (1, 1), + (1.5, 1.5), + ), + 'fail': ( + (0, ValueError), + (-1, ValueError), + ('fail', TypeError), + ), + }, + { + 'validator': + _listify_validator(validate_string), + 'success': ( + ('', []), + ('a,b', ['a', 'b']), + ('abc', ['abc']), + ('abc, ', ['abc']), + ('abc, ,', ['abc']), + (['a', 'b'], ['a', 'b']), + (('a', 'b'), ['a', 'b']), + (iter(['a', 'b']), ['a', 'b']), + (np.array(['a', 'b']), ['a', 'b']), + ((1, 2), ['1', '2']), + (np.array([1, 2]), ['1', '2']), + ), + 'fail': ( + (set(), ValueError), + (1, ValueError), + ) + }, + { + 'validator': validate_string_or_none, + 'success': ((None, None), ), + 'fail': (), + }, + ) + + for validator_dict in validation_tests: + validator = validator_dict['validator'] + if valid: + for arg, target in validator_dict['success']: + yield validator, arg, target + else: + for arg, error_type in validator_dict['fail']: + yield validator, arg, error_type + + +@pytest.mark.parametrize('validator, arg, target', + generate_validator_testcases(True)) +def test_validator_valid(validator, arg, target): + res = validator(arg) + assert res == target + + +@pytest.mark.parametrize('validator, arg, exception_type', + generate_validator_testcases(False)) +def test_validator_invalid(validator, arg, exception_type): + with pytest.raises(exception_type): + validator(arg) + + +@pytest.mark.parametrize('version', (current_version, '0.0.1', '9.9.9')) +def test_deprecate(version): + def test_func(): + pass + + # This always warns + with pytest.warns(UserWarning): + f = deprecate(test_func, 'test_var', version) + + assert callable(f) + + +def test_config_class(): + config = { + 'log_level': 'info', + 'exit_on_warning': False, + 'output_file_type': 'png', + 'output_dir': './esmvaltool_output', + 'auxiliary_data_dir': './auxiliary_data', + 'save_intermediary_cubes': False, + 'remove_preproc_dir': True, + 'max_parallel_tasks': None, + 'profile_diagnostic': False, + 'rootpath': { + 'CMIP6': '~/data/CMIP6' + }, + 'drs': { + 'CMIP6': 'default' + }, + } + + cfg = ESMValCoreConfig(config) + + assert isinstance(cfg['output_dir'], Path) + assert isinstance(cfg['auxiliary_data_dir'], Path) + + from esmvalcore._config import CFG as CFG_DEV + assert CFG_DEV + + +def test_config_update(): + config = ESMValCoreConfig({'output_dir': 'directory'}) + fail_dict = {'output_dir': 123} + + with pytest.raises(InvalidConfigParameter): + config.update(fail_dict) + + +def test_config_init(): + config = ESMValCoreConfig() + assert isinstance(config, dict) + + +def test_session(): + config = ESMValCoreConfig({'output_dir': 'config'}) + + session = config.start_session('recipe_name') + assert session == config + + session['output_dir'] = 'session' + session != config From fc2e1f252e2238a74f1874d67351b7892ac35a5e Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 15:18:26 +0100 Subject: [PATCH 05/45] Fix for default config file --- esmvalcore/future/config/_config_object.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 0a3973014b..43887f901a 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -119,13 +119,12 @@ def _load_user_config(filename: str, raise_exception: bool = True): """ try: mapping = read_config_file(filename) + mapping['config_file'] = filename except IOError: if raise_exception: raise mapping = {} - mapping['config_file'] = filename - global CFG global CFG_orig From b5ab46b0a0beee7a0e51408b52f422779f6c25f6 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 15:26:54 +0100 Subject: [PATCH 06/45] Add parenthesis to lru_cache decorator for Py3.7 compatibility In Py3.8 it is OK to use `lru_cache` without parentheses, so Py3.7: `@lru_cache()`, Py3.8: `@lru_cache` --- esmvalcore/future/config/_config_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/future/config/_config_validators.py index d6995d9a18..23b920605e 100644 --- a/esmvalcore/future/config/_config_validators.py +++ b/esmvalcore/future/config/_config_validators.py @@ -33,7 +33,7 @@ def validator(s): return validator -@lru_cache +@lru_cache() def _listify_validator(scalar_validator, allow_stringlist=False, *, From efb4ac408640039811d7f476c836b294e58bd634 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 15:54:46 +0100 Subject: [PATCH 07/45] Tackle linter issues --- esmvalcore/__init__.py | 1 - esmvalcore/future/__init__.py | 2 ++ esmvalcore/future/_exceptions.py | 3 +- esmvalcore/future/config/__init__.py | 2 ++ esmvalcore/future/config/_config_object.py | 29 +++++++++++++++---- .../future/config/_config_validators.py | 17 ++++++----- esmvalcore/future/config/_validated_config.py | 7 +++-- 7 files changed, 45 insertions(+), 16 deletions(-) diff --git a/esmvalcore/__init__.py b/esmvalcore/__init__.py index 4941523f4b..4d1bb19255 100644 --- a/esmvalcore/__init__.py +++ b/esmvalcore/__init__.py @@ -9,7 +9,6 @@ __all__ = [ '__version__', - 'future' 'cmor', 'preprocessor', ] diff --git a/esmvalcore/future/__init__.py b/esmvalcore/future/__init__.py index 5d272c9a69..abe577bd6f 100644 --- a/esmvalcore/future/__init__.py +++ b/esmvalcore/future/__init__.py @@ -1,3 +1,5 @@ +"""ESMValCore future API module.""" + from .config import CFG __all__ = [ diff --git a/esmvalcore/future/_exceptions.py b/esmvalcore/future/_exceptions.py index 2edfbafeaf..437ed124cc 100644 --- a/esmvalcore/future/_exceptions.py +++ b/esmvalcore/future/_exceptions.py @@ -1,3 +1,5 @@ +"""ESMValCore exceptions.""" + import sys @@ -7,7 +9,6 @@ class SuppressedError(Exception): This can be used for simple user-facing errors that do not need the full traceback. """ - pass def _suppressed_hook(error, message, traceback): diff --git a/esmvalcore/future/config/__init__.py b/esmvalcore/future/config/__init__.py index 457c408639..7ac457d864 100644 --- a/esmvalcore/future/config/__init__.py +++ b/esmvalcore/future/config/__init__.py @@ -1,3 +1,5 @@ +"""ESMValTool config module.""" + from ._config_object import CFG __all__ = [ diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 43887f901a..4de464eef8 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -1,3 +1,5 @@ +"""Importable config object.""" + import os from datetime import datetime from pathlib import Path @@ -11,7 +13,8 @@ class ESMValCoreConfig(ValidatedConfig): - """The ESMValCore config object.""" + """ESMValTool configuration object.""" + validate = _validators @staticmethod @@ -27,13 +30,23 @@ def load_from_file(filename): _load_user_config(path) - def start_session(self, name): + def start_session(self, name: str): + """Start a new session from this configuration object. + + Parameters + ---------- + name: str + Name of the session. + + Returns + ------- + Session + """ return Session(name, self.copy()) class Session(ValidatedConfig): - """This class holds information about the current session. Different - session directories can be accessed. + """Container class for session configuration and directory information. Parameters ---------- @@ -60,26 +73,32 @@ def init_session_dir(self, name: str = 'session'): @property def session_dir(self): + """Return session directory.""" return self._session_dir @property def preproc_dir(self): + """Return preproc directory.""" return self.session_dir / 'preproc' @property def work_dir(self): + """Return work directory.""" return self.session_dir / 'work' @property def plot_dir(self): + """Return plot directory.""" return self.session_dir / 'plots' @property def run_dir(self): + """Return run directory.""" return self.session_dir / 'run' @property def config_dir(self): + """Return user config directory.""" return USER_CONFIG_DIR @@ -99,7 +118,7 @@ def _load_default_config(filename: str): """Load the default configuration.""" mapping = read_config_file(filename) - global config_default + global CFG_default CFG_default.update(mapping) diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/future/config/_config_validators.py index 23b920605e..baa926099c 100644 --- a/esmvalcore/future/config/_config_validators.py +++ b/esmvalcore/future/config/_config_validators.py @@ -1,3 +1,6 @@ +"""List of config validators.""" + +import warnings from collections.abc import Iterable from functools import lru_cache from pathlib import Path @@ -18,10 +21,10 @@ def validator(s): return None try: return cls(s) - except ValueError as e: + except ValueError as err: if isinstance(cls, type): raise ValueError( - f'Could not convert {repr(s)} to {cls.__name__}') from e + f'Could not convert {repr(s)} to {cls.__name__}') from err else: raise @@ -102,8 +105,8 @@ def validate_path(value, allow_none=False): return value try: path = Path(value).expanduser().absolute() - except TypeError as e: - raise ValueError(f"Expected a path, but got {value}") from e + except TypeError as err: + raise ValueError(f"Expected a path, but got {value}") from err else: return path @@ -173,7 +176,8 @@ def validate_check_level(value): try: value = CheckLevels[value.upper()] except KeyError: - raise ValueError(f'`{value}` is not a valid strictness level') + raise ValueError( + f'`{value}` is not a valid strictness level') from None else: value = CheckLevels(value) @@ -193,6 +197,7 @@ def validate_diagnostics(diagnostics): class ESMValToolDeprecationWarning(UserWarning): + """Configuration key has been deprecated.""" # Custom warning, because DeprecationWarning is hidden by default pass @@ -212,8 +217,6 @@ def deprecate(func, variable, version: str = None): Version to deprecate the variable in, should be something like '2.2.3' """ - import warnings - from esmvalcore import __version__ as current_version if not version: diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py index 542f977e7a..89fbd2c387 100644 --- a/esmvalcore/future/config/_validated_config.py +++ b/esmvalcore/future/config/_validated_config.py @@ -1,3 +1,5 @@ +"""Config validation objects.""" + import pprint import re from collections.abc import MutableMapping @@ -25,8 +27,8 @@ def __init__(self, *args, **kwargs): def __setitem__(self, key, val): try: cval = self.validate[key](val) - except ValueError as ve: - raise InvalidConfigParameter(f"Key `{key}`: {ve}") from None + except ValueError as verr: + raise InvalidConfigParameter(f"Key `{key}`: {verr}") from None except KeyError: raise InvalidConfigParameter( f"`{key}` is not a valid config parameter.") from None @@ -66,6 +68,7 @@ def find_all(self, pattern): if pattern_re.search(key)) def copy(self): + """Copy the keys this object to a dict.""" return {k: dict.__getitem__(self, k) for k in self} def clear(self): From 486247b7e2daed4fa91b7fc7f59b44a0fc62f17b Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 20 Nov 2020 16:13:58 +0100 Subject: [PATCH 08/45] Tackle linter issues --- esmvalcore/future/config/_config_object.py | 7 +++- .../future/config/_config_validators.py | 37 ++++++++----------- esmvalcore/future/config/_validated_config.py | 14 ++++--- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 4de464eef8..0eb0b107b1 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -54,6 +54,7 @@ class Session(ValidatedConfig): Name of the session to initialize, for example, the name of the recipe (default='session'). """ + validate = _validators def __init__(self, name: str = 'session', *args, **kwargs): @@ -124,12 +125,14 @@ def _load_default_config(filename: str): def _load_user_config(filename: str, raise_exception: bool = True): - """Load user configuration from the given file (`filename`). + """Load user configuration from the given file. - The config cleared and updated in-place. + The config is cleared and updated in-place. Parameters ---------- + filename: pathlike + Name of the config file, must be yaml format raise_exception : bool Raise an exception if `filename` can not be found (default). Otherwise, silently pass and use the default configuration. This diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/future/config/_config_validators.py index baa926099c..2404274c7f 100644 --- a/esmvalcore/future/config/_config_validators.py +++ b/esmvalcore/future/config/_config_validators.py @@ -6,15 +6,13 @@ from pathlib import Path +# The code for this function was taken from matplotlib (v3.3) and modified +# to fit the needs of ESMValCore. Matplotlib is licenced under the terms of +# the the 'Python Software Foundation License' +# (https://www.python.org/psf/license) def _make_type_validator(cls, *, allow_none=False): """Return a validator that converts inputs to *cls* or raises (and possibly allows ``None`` as well).""" - - # The code for this function was taken from matplotlib (v3.3) and modified - # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of - # the the 'Python Software Foundation License' - # (https://www.python.org/psf/license) - def validator(s): if (allow_none and (s is None or isinstance(s, str) and s.lower() == "none")): @@ -36,6 +34,10 @@ def validator(s): return validator +# The code for this function was taken from matplotlib (v3.3) and modified +# to fit the needs of ESMValCore. Matplotlib is licenced under the terms of +# the the 'Python Software Foundation License' +# (https://www.python.org/psf/license) @lru_cache() def _listify_validator(scalar_validator, allow_stringlist=False, @@ -43,13 +45,7 @@ def _listify_validator(scalar_validator, n=None, doc=None): """Apply the validator to a list.""" - - # The code for this function was taken from matplotlib (v3.3) and modified - # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of - # the the 'Python Software Foundation License' - # (https://www.python.org/psf/license) - - def f(s): + def fnc(s): if isinstance(s, str): try: val = [ @@ -82,12 +78,12 @@ def f(s): return val try: - f.__name__ = "{}list".format(scalar_validator.__name__) + fnc.__name__ = "{}list".format(scalar_validator.__name__) except AttributeError: # class instance. - f.__name__ = "{}List".format(type(scalar_validator).__name__) - f.__qualname__ = f.__qualname__.rsplit(".", 1)[0] + "." + f.__name__ - f.__doc__ = doc if doc is not None else scalar_validator.__doc__ - return f + fnc.__name__ = "{}List".format(type(scalar_validator).__name__) + fnc.__qualname__ = fnc.__qualname__.rsplit(".", 1)[0] + "." + fnc.__name__ + fnc.__doc__ = doc if doc is not None else scalar_validator.__doc__ + return fnc def validate_bool(value, allow_none=False): @@ -196,14 +192,13 @@ def validate_diagnostics(diagnostics): } +# Custom warning, because DeprecationWarning is hidden by default class ESMValToolDeprecationWarning(UserWarning): """Configuration key has been deprecated.""" - # Custom warning, because DeprecationWarning is hidden by default - pass def deprecate(func, variable, version: str = None): - """Wrapper function to mark variables to be deprecated. + """Wrap function to mark variables to be deprecated. This will give a warning if the function will be/has been deprecated. diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py index 89fbd2c387..c0fc8774bc 100644 --- a/esmvalcore/future/config/_validated_config.py +++ b/esmvalcore/future/config/_validated_config.py @@ -8,15 +8,16 @@ class InvalidConfigParameter(SuppressedError): + """Config parameter is invalid.""" pass +# The code for this class was take from matplotlib (v3.3) and modified to +# fit the needs of ESMValCore. Matplotlib is licenced under the terms of +# the the 'Python Software Foundation License' +# (https://www.python.org/psf/license) class ValidatedConfig(MutableMapping, dict): """Based on `matplotlib.rcParams`.""" - # The code for this class was take from matplotlib (v3.3) and modified to - # fit the needs of ESMValCore. Matplotlib is licenced under the terms of - # the the 'Python Software Foundation License' - # (https://www.python.org/psf/license) validate = {} @@ -57,8 +58,9 @@ def __len__(self): return dict.__len__(self) def find_all(self, pattern): - """Return the subset of this Config dictionary whose keys match, using - `re.search` with the given `pattern`. + """Return the subset of this Config dictionary whose keys match. + + Uses `re.search` with the given `pattern`. Changes to the returned dictionary are *not* propagated to the parent Config dictionary. From c6542c77342f8cf6da2122e987a6d10564c273c4 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 14:58:55 +0100 Subject: [PATCH 09/45] Address Codacy issues --- esmvalcore/future/config/_config_object.py | 21 +++++++++++-------- .../future/config/_config_validators.py | 14 ++++++++++--- esmvalcore/future/config/_validated_config.py | 4 +++- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 0eb0b107b1..d6c8a06fb7 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -119,9 +119,9 @@ def _load_default_config(filename: str): """Load the default configuration.""" mapping = read_config_file(filename) - global CFG_default + global CFG_DEFAULT - CFG_default.update(mapping) + CFG_DEFAULT.update(mapping) def _load_user_config(filename: str, raise_exception: bool = True): @@ -148,18 +148,21 @@ def _load_user_config(filename: str, raise_exception: bool = True): mapping = {} global CFG - global CFG_orig + global CFG_ORIG CFG.clear() - CFG.update(CFG_default) + CFG.update(CFG_DEFAULT) CFG.update(mapping) - CFG_orig = ESMValCoreConfig(CFG.copy()) + CFG_ORIG = ESMValCoreConfig(CFG.copy()) def get_user_config_location(): - """Check if environment variable `ESMVALTOOL_CONFIG` exists, otherwise use - the default config location.""" + """Get the user config location by looking in the expected places. + + Check if environment variable `ESMVALTOOL_CONFIG` exists, otherwise + use the default location in the user config dir (`~/.esmvaltool`). + """ try: config_location = Path(os.environ['ESMVALTOOL_CONFIG']) except KeyError: @@ -175,9 +178,9 @@ def get_user_config_location(): USER_CONFIG = get_user_config_location() # initialize placeholders -CFG_default = ESMValCoreConfig() +CFG_DEFAULT = ESMValCoreConfig() CFG = ESMValCoreConfig() -CFG_orig = ESMValCoreConfig() +CFG_ORIG = ESMValCoreConfig() # update config objects _load_default_config(DEFAULT_CONFIG) diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/future/config/_config_validators.py index 2404274c7f..228dc355f4 100644 --- a/esmvalcore/future/config/_config_validators.py +++ b/esmvalcore/future/config/_config_validators.py @@ -11,8 +11,11 @@ # the the 'Python Software Foundation License' # (https://www.python.org/psf/license) def _make_type_validator(cls, *, allow_none=False): - """Return a validator that converts inputs to *cls* or raises (and possibly - allows ``None`` as well).""" + """Construct a type validator for `cls`. + + Return a validator that converts inputs to *cls* or raises (and + possibly allows ``None`` as well). + """ def validator(s): if (allow_none and (s is None or isinstance(s, str) and s.lower() == "none")): @@ -96,7 +99,7 @@ def validate_bool(value, allow_none=False): def validate_path(value, allow_none=False): - """Return a path object.""" + """Return a `Path` object.""" if (value is None) and allow_none: return value try: @@ -147,16 +150,19 @@ def chained(value): def validate_oldstyle_rootpath(value): + """Validate `rootpath` mapping.""" mapping = validate_dict(value) return mapping def validate_oldstyle_drs(value): + """Validate `drs` mapping.""" mapping = validate_dict(value) return mapping def validate_config_developer(value): + """Validate and load config developer path.""" from esmvalcore._config import load_config_developer path = validate_path_or_none(value) @@ -166,6 +172,7 @@ def validate_config_developer(value): def validate_check_level(value): + """Validate CMOR level check.""" from esmvalcore.cmor.check import CheckLevels if isinstance(value, str): @@ -182,6 +189,7 @@ def validate_check_level(value): def validate_diagnostics(diagnostics): + """Validate diagnostic location.""" from esmvalcore._recipe import TASKSEP if isinstance(diagnostics, str): diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py index c0fc8774bc..c7502c3c9e 100644 --- a/esmvalcore/future/config/_validated_config.py +++ b/esmvalcore/future/config/_validated_config.py @@ -9,7 +9,6 @@ class InvalidConfigParameter(SuppressedError): """Config parameter is invalid.""" - pass # The code for this class was take from matplotlib (v3.3) and modified to @@ -57,6 +56,9 @@ def __iter__(self): def __len__(self): return dict.__len__(self) + def __del__(self, key): + dict.__delitem__(self, key) + def find_all(self, pattern): """Return the subset of this Config dictionary whose keys match. From 1ddb0c06f3855264f8ff03efc3c7f9f3c6decc93 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 16:00:27 +0100 Subject: [PATCH 10/45] Fix typo __del__ -> __delitem__ --- esmvalcore/future/config/_validated_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py index c7502c3c9e..7d6978ef7a 100644 --- a/esmvalcore/future/config/_validated_config.py +++ b/esmvalcore/future/config/_validated_config.py @@ -56,7 +56,7 @@ def __iter__(self): def __len__(self): return dict.__len__(self) - def __del__(self, key): + def __delitem__(self, key): dict.__delitem__(self, key) def find_all(self, pattern): From 73717e6a491eac4d4c88b596407bd2b42d3b2963 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 16:03:55 +0100 Subject: [PATCH 11/45] Change how session object gets initialized The last version was a bad idea mixing *args/**kwargs with a name arg, now the intention is more clear. --- esmvalcore/future/config/_config_object.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index d6c8a06fb7..7fbb01fdad 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -42,7 +42,7 @@ def start_session(self, name: str): ------- Session """ - return Session(name, self.copy()) + return Session(name, config=self.copy()) class Session(ValidatedConfig): @@ -53,12 +53,14 @@ class Session(ValidatedConfig): name : str Name of the session to initialize, for example, the name of the recipe (default='session'). + config : dict + Dictionary with configuration settings. """ validate = _validators - def __init__(self, name: str = 'session', *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, name: str = 'session', config: dict={}): + super().__init__(config) self.init_session_dir(name) def init_session_dir(self, name: str = 'session'): From 5a30a5f116b90cc311b5d740d00194f06578b351 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 16:25:36 +0100 Subject: [PATCH 12/45] Adjust variable names to address Codacy issues --- .../future/config/_config_validators.py | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/future/config/_config_validators.py index 228dc355f4..dc216ba434 100644 --- a/esmvalcore/future/config/_config_validators.py +++ b/esmvalcore/future/config/_config_validators.py @@ -16,18 +16,18 @@ def _make_type_validator(cls, *, allow_none=False): Return a validator that converts inputs to *cls* or raises (and possibly allows ``None`` as well). """ - def validator(s): - if (allow_none - and (s is None or isinstance(s, str) and s.lower() == "none")): + def validator(inp): + looks_like_none = isinstance(inp, str) and (inp.lower() == "none") + if (allow_none and (inp is None or looks_like_none)): return None try: - return cls(s) + return cls(inp) except ValueError as err: if isinstance(cls, type): raise ValueError( - f'Could not convert {repr(s)} to {cls.__name__}') from err - else: - raise + f'Could not convert {repr(inp)} to {cls.__name__}' + ) from err + raise validator.__name__ = f"validate_{cls.__name__}" if allow_none: @@ -45,48 +45,56 @@ def validator(s): def _listify_validator(scalar_validator, allow_stringlist=False, *, - n=None, - doc=None): + n_items=None, + docstring=None): """Apply the validator to a list.""" - def fnc(s): - if isinstance(s, str): + def func(inp): + if isinstance(inp, str): try: - val = [ - scalar_validator(v.strip()) for v in s.split(',') - if v.strip() + inp = [ + scalar_validator(val.strip()) for val in inp.split(',') + if val.strip() ] except Exception: if allow_stringlist: # Sometimes, a list of colors might be a single string # of single-letter colornames. So give that a shot. - val = [scalar_validator(v.strip()) for v in s if v.strip()] + inp = [ + scalar_validator(val.strip()) for val in inp + if val.strip() + ] else: raise # Allow any ordered sequence type -- generators, np.ndarray, pd.Series # -- but not sets, whose iteration order is non-deterministic. - elif isinstance(s, Iterable) and not isinstance(s, (set, frozenset)): + elif isinstance(inp, + Iterable) and not isinstance(inp, (set, frozenset)): # The condition on this list comprehension will preserve the # behavior of filtering out any empty strings (behavior was # from the original validate_stringlist()), while allowing # any non-string/text scalar values such as numbers and arrays. - val = [ - scalar_validator(v) for v in s if not isinstance(v, str) or v + inp = [ + scalar_validator(val) for val in inp + if not isinstance(val, str) or val ] else: raise ValueError( - f"Expected str or other non-set iterable, but got {s}") - if n is not None and len(val) != n: - raise ValueError( - f"Expected {n} values, but there are {len(val)} values in {s}") - return val + f"Expected str or other non-set iterable, but got {inp}") + if n_items is not None and len(inp) != n_items: + raise ValueError(f"Expected {n_items} values, " + f"but there are {len(inp)} values in {inp}") + return inp try: - fnc.__name__ = "{}list".format(scalar_validator.__name__) + func.__name__ = "{}list".format(scalar_validator.__name__) except AttributeError: # class instance. - fnc.__name__ = "{}List".format(type(scalar_validator).__name__) - fnc.__qualname__ = fnc.__qualname__.rsplit(".", 1)[0] + "." + fnc.__name__ - fnc.__doc__ = doc if doc is not None else scalar_validator.__doc__ - return fnc + func.__name__ = "{}List".format(type(scalar_validator).__name__) + func.__qualname__ = func.__qualname__.rsplit(".", + 1)[0] + "." + func.__name__ + if docstring is not None: + docstring = scalar_validator.__doc__ + func.__doc__ = docstring + return func def validate_bool(value, allow_none=False): @@ -130,19 +138,19 @@ def chained(value): validate_string = _make_type_validator(str) validate_string_or_none = _make_type_validator(str, allow_none=True) validate_stringlist = _listify_validator(validate_string, - doc='return a list of strings') + docstring='Return a list of strings.') validate_int = _make_type_validator(int) validate_int_or_none = _make_type_validator(int, allow_none=True) validate_float = _make_type_validator(float) validate_floatlist = _listify_validator(validate_float, - doc='return a list of floats') + docstring='Return a list of floats.') validate_dict = _make_type_validator(dict) validate_path_or_none = _make_type_validator(validate_path, allow_none=True) validate_pathlist = _listify_validator(validate_path, - doc='return a list of paths') + docstring='Return a list of paths.') validate_int_positive = _chain_validator(validate_int, validate_positive) validate_int_positive_or_none = _make_type_validator(validate_int_positive, From ff3cbeef8c7b36652130f973077aebbfa401110a Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 16:38:38 +0100 Subject: [PATCH 13/45] Address Codacy issues --- esmvalcore/future/config/_config_object.py | 8 ++++---- esmvalcore/future/config/_validated_config.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 7fbb01fdad..44f1ffc06e 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -42,7 +42,7 @@ def start_session(self, name: str): ------- Session """ - return Session(name, config=self.copy()) + return Session(config=self.copy(), name=name) class Session(ValidatedConfig): @@ -50,16 +50,16 @@ class Session(ValidatedConfig): Parameters ---------- + config : dict + Dictionary with configuration settings. name : str Name of the session to initialize, for example, the name of the recipe (default='session'). - config : dict - Dictionary with configuration settings. """ validate = _validators - def __init__(self, name: str = 'session', config: dict={}): + def __init__(self, config: dict, name: str = 'session'): super().__init__(config) self.init_session_dir(name) diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py index 7d6978ef7a..2e89955cde 100644 --- a/esmvalcore/future/config/_validated_config.py +++ b/esmvalcore/future/config/_validated_config.py @@ -22,6 +22,7 @@ class ValidatedConfig(MutableMapping, dict): # validate values on the way in def __init__(self, *args, **kwargs): + super().__init__() self.update(*args, **kwargs) def __setitem__(self, key, val): From 31579572bd355d695af1db02557f1811f486b6f1 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 16:45:32 +0100 Subject: [PATCH 14/45] Update tests to reflect variable name change --- tests/unit/future/test_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/future/test_config.py b/tests/unit/future/test_config.py index 499e192507..ee815341b7 100644 --- a/tests/unit/future/test_config.py +++ b/tests/unit/future/test_config.py @@ -68,7 +68,7 @@ def generate_validator_testcases(valid): }, { 'validator': - _listify_validator(validate_float, n=2), + _listify_validator(validate_float, n_items=2), 'success': ((_, [1.5, 2.5]) for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), @@ -77,7 +77,7 @@ def generate_validator_testcases(valid): }, { 'validator': - _listify_validator(validate_float, n=2), + _listify_validator(validate_float, n_items=2), 'success': ((_, [1.5, 2.5]) for _ in ('1.5, 2.5', [1.5, 2.5], [1.5, 2.5], (1.5, 2.5), @@ -86,7 +86,7 @@ def generate_validator_testcases(valid): }, { 'validator': - _listify_validator(validate_int, n=2), + _listify_validator(validate_int, n_items=2), 'success': ((_, [1, 2]) for _ in ('1, 2', [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2)))), From 75399756e74faa111b4e18a99c90b56368773b30 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 23 Nov 2020 16:46:44 +0100 Subject: [PATCH 15/45] Fix docstring --- esmvalcore/_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index fade94027c..1d9559dfdf 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -118,7 +118,6 @@ def _normalize_path(path): ------- str: Normalized path - """ if path is None: return None @@ -140,8 +139,7 @@ def read_config_developer_file(cfg_file=None): def load_config_developer(cfg_file=None): - """Load the config developer file into the CFG object and initialize cmor - tables.""" + """Load the config developer file and initialize CMOR tables.""" cfg_developer = read_config_developer_file(cfg_file) for key, value in cfg_developer.items(): CFG[key] = value From 1b9e0b9f923e53ce4d8d57cb1a2aa4177c87a8d1 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 24 Nov 2020 12:05:17 +0100 Subject: [PATCH 16/45] Default to None if 'max_years' not in config --- esmvalcore/_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 3e54de06a2..c18dc6157a 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1017,7 +1017,7 @@ def _initialize_variables(self, raw_variable, raw_datasets): variable.update(dataset) variable['recipe_dataset_index'] = index - if 'end_year' in variable and 'max_years' in self._cfg: + if 'end_year' in variable and self._cfg.get('max_years'): variable['end_year'] = min( variable['end_year'], variable['start_year'] + self._cfg['max_years'] - 1) From fcc9e39b7d5691a9818fe1c27fb47910c853ce7e Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 26 Nov 2020 16:33:00 +0100 Subject: [PATCH 17/45] Revert adding hidden config parameters to config-user.yml This will prevent undefined behaviour, because some values must be checked/validated which is now done in main, which may not always happen --- esmvalcore/config-user.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index b51c8d0f4a..8871dadee6 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -37,25 +37,6 @@ config_developer_file: null # Get profiling information for diagnostics # Only available for Python diagnostics profile_diagnostic: false -# Maximum number of years to use. -max_years: null -# Run the diagnostic -run_diagnostic: true -# If True, the run will not fail if some datasets are not available. -skip-nonexistent: False -# Only run the selected diagnostics from the recipe. -diagnostics: null -# Configure the sensitivity of the CMOR check. -# Possible values are: -# `ignore` (all errors will be reported as warnings), -# `relaxed` (only fail if there are critical errors), -# `default` (fail if there are any errors), -# `strict` (fail if there are any warnings). -check_level: default -# If True, the tool will try to download missing data using Synda. -synda_download: False -# Maximum number of datasets to use. -max_datasets: null # Rootpaths to the data from different projects (lists are also possible) # these are generic entries to better allow you to enter your own From 43f70826d2035587bcac4a7475934a431464da96 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 26 Nov 2020 16:59:15 +0100 Subject: [PATCH 18/45] Add warning for unstable API --- esmvalcore/future/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/esmvalcore/future/__init__.py b/esmvalcore/future/__init__.py index abe577bd6f..7ae0e69e05 100644 --- a/esmvalcore/future/__init__.py +++ b/esmvalcore/future/__init__.py @@ -1,5 +1,12 @@ """ESMValCore future API module.""" +import warnings + +warnings.warn( + '\n Thank you for trying out the new ESMValCore API.' + '\n Note that this API is experimental and may be subject to change.' + '\n More info: https://github.com/ESMValGroup/ESMValCore/issues/498', ) + from .config import CFG __all__ = [ From 9fd553f0e95add9f2d2b48ef0b6d435440a9cc30 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 26 Nov 2020 16:59:43 +0100 Subject: [PATCH 19/45] Remove unneeded code --- esmvalcore/future/config/_config_object.py | 4 ---- esmvalcore/future/config/_validated_config.py | 13 ------------- 2 files changed, 17 deletions(-) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 44f1ffc06e..63f5b5894f 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -150,14 +150,11 @@ def _load_user_config(filename: str, raise_exception: bool = True): mapping = {} global CFG - global CFG_ORIG CFG.clear() CFG.update(CFG_DEFAULT) CFG.update(mapping) - CFG_ORIG = ESMValCoreConfig(CFG.copy()) - def get_user_config_location(): """Get the user config location by looking in the expected places. @@ -182,7 +179,6 @@ def get_user_config_location(): # initialize placeholders CFG_DEFAULT = ESMValCoreConfig() CFG = ESMValCoreConfig() -CFG_ORIG = ESMValCoreConfig() # update config objects _load_default_config(DEFAULT_CONFIG) diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/future/config/_validated_config.py index 2e89955cde..2bb3411e04 100644 --- a/esmvalcore/future/config/_validated_config.py +++ b/esmvalcore/future/config/_validated_config.py @@ -1,7 +1,6 @@ """Config validation objects.""" import pprint -import re from collections.abc import MutableMapping from .._exceptions import SuppressedError @@ -60,18 +59,6 @@ def __len__(self): def __delitem__(self, key): dict.__delitem__(self, key) - def find_all(self, pattern): - """Return the subset of this Config dictionary whose keys match. - - Uses `re.search` with the given `pattern`. - - Changes to the returned dictionary are *not* propagated to the - parent Config dictionary. - """ - pattern_re = re.compile(pattern) - return self.__class__((key, value) for key, value in self.items() - if pattern_re.search(key)) - def copy(self): """Copy the keys this object to a dict.""" return {k: dict.__getitem__(self, k) for k in self} From f3346a261f2e1bf9378bbb12504366cb73800eae Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 26 Nov 2020 17:04:27 +0100 Subject: [PATCH 20/45] Add function to reload the config file --- esmvalcore/future/config/_config_object.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/future/config/_config_object.py index 63f5b5894f..2cdf239404 100644 --- a/esmvalcore/future/config/_config_object.py +++ b/esmvalcore/future/config/_config_object.py @@ -30,6 +30,11 @@ def load_from_file(filename): _load_user_config(path) + def reload(self): + """Reload the config file.""" + filename = self.get('config_file', DEFAULT_CONFIG) + self.load_from_file(filename) + def start_session(self, name: str): """Start a new session from this configuration object. From 41b158339ae093bba1480fb37f5f276286e578b3 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 11:16:16 +0100 Subject: [PATCH 21/45] Rename submodule future -> experimental --- esmvalcore/{future => experimental}/__init__.py | 0 esmvalcore/{future => experimental}/_exceptions.py | 0 esmvalcore/{future => experimental}/config/__init__.py | 0 esmvalcore/{future => experimental}/config/_config_object.py | 0 esmvalcore/{future => experimental}/config/_config_validators.py | 0 esmvalcore/{future => experimental}/config/_validated_config.py | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename esmvalcore/{future => experimental}/__init__.py (100%) rename esmvalcore/{future => experimental}/_exceptions.py (100%) rename esmvalcore/{future => experimental}/config/__init__.py (100%) rename esmvalcore/{future => experimental}/config/_config_object.py (100%) rename esmvalcore/{future => experimental}/config/_config_validators.py (100%) rename esmvalcore/{future => experimental}/config/_validated_config.py (100%) diff --git a/esmvalcore/future/__init__.py b/esmvalcore/experimental/__init__.py similarity index 100% rename from esmvalcore/future/__init__.py rename to esmvalcore/experimental/__init__.py diff --git a/esmvalcore/future/_exceptions.py b/esmvalcore/experimental/_exceptions.py similarity index 100% rename from esmvalcore/future/_exceptions.py rename to esmvalcore/experimental/_exceptions.py diff --git a/esmvalcore/future/config/__init__.py b/esmvalcore/experimental/config/__init__.py similarity index 100% rename from esmvalcore/future/config/__init__.py rename to esmvalcore/experimental/config/__init__.py diff --git a/esmvalcore/future/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py similarity index 100% rename from esmvalcore/future/config/_config_object.py rename to esmvalcore/experimental/config/_config_object.py diff --git a/esmvalcore/future/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py similarity index 100% rename from esmvalcore/future/config/_config_validators.py rename to esmvalcore/experimental/config/_config_validators.py diff --git a/esmvalcore/future/config/_validated_config.py b/esmvalcore/experimental/config/_validated_config.py similarity index 100% rename from esmvalcore/future/config/_validated_config.py rename to esmvalcore/experimental/config/_validated_config.py From 4d4860c139e201cb5db01ec8f07b13e0ba0119ca Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 11:22:48 +0100 Subject: [PATCH 22/45] Fix imports and rename some variables --- esmvalcore/experimental/__init__.py | 2 +- esmvalcore/experimental/config/_config_object.py | 8 ++++---- .../unit/{future => experimental}/test_config.py | 15 ++++++++------- 3 files changed, 13 insertions(+), 12 deletions(-) rename tests/unit/{future => experimental}/test_config.py (94%) diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py index 7ae0e69e05..cf46dd68fb 100644 --- a/esmvalcore/experimental/__init__.py +++ b/esmvalcore/experimental/__init__.py @@ -1,4 +1,4 @@ -"""ESMValCore future API module.""" +"""ESMValCore experimental API module.""" import warnings diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 2cdf239404..b5c770168d 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -12,7 +12,7 @@ from ._validated_config import ValidatedConfig -class ESMValCoreConfig(ValidatedConfig): +class Config(ValidatedConfig): """ESMValTool configuration object.""" validate = _validators @@ -143,7 +143,7 @@ def _load_user_config(filename: str, raise_exception: bool = True): raise_exception : bool Raise an exception if `filename` can not be found (default). Otherwise, silently pass and use the default configuration. This - setting is necessary for the case where `.esmvalcore/config-user.yml` + setting is necessary for the case where `.esmvaltool/config-user.yml` has not been defined (i.e. first start). """ try: @@ -182,8 +182,8 @@ def get_user_config_location(): USER_CONFIG = get_user_config_location() # initialize placeholders -CFG_DEFAULT = ESMValCoreConfig() -CFG = ESMValCoreConfig() +CFG_DEFAULT = Config() +CFG = Config() # update config objects _load_default_config(DEFAULT_CONFIG) diff --git a/tests/unit/future/test_config.py b/tests/unit/experimental/test_config.py similarity index 94% rename from tests/unit/future/test_config.py rename to tests/unit/experimental/test_config.py index ee815341b7..ac74b3b856 100644 --- a/tests/unit/future/test_config.py +++ b/tests/unit/experimental/test_config.py @@ -4,8 +4,8 @@ import pytest from esmvalcore import __version__ as current_version -from esmvalcore.future.config._config_object import ESMValCoreConfig -from esmvalcore.future.config._config_validators import ( +from esmvalcore.experimental.config._config_object import Config +from esmvalcore.experimental.config._config_validators import ( _listify_validator, deprecate, validate_bool, @@ -21,7 +21,8 @@ validate_string, validate_string_or_none, ) -from esmvalcore.future.config._validated_config import InvalidConfigParameter +from esmvalcore.experimental.config._validated_config import ( + InvalidConfigParameter, ) def generate_validator_testcases(valid): @@ -218,7 +219,7 @@ def test_config_class(): }, } - cfg = ESMValCoreConfig(config) + cfg = Config(config) assert isinstance(cfg['output_dir'], Path) assert isinstance(cfg['auxiliary_data_dir'], Path) @@ -228,7 +229,7 @@ def test_config_class(): def test_config_update(): - config = ESMValCoreConfig({'output_dir': 'directory'}) + config = Config({'output_dir': 'directory'}) fail_dict = {'output_dir': 123} with pytest.raises(InvalidConfigParameter): @@ -236,12 +237,12 @@ def test_config_update(): def test_config_init(): - config = ESMValCoreConfig() + config = Config() assert isinstance(config, dict) def test_session(): - config = ESMValCoreConfig({'output_dir': 'config'}) + config = Config({'output_dir': 'config'}) session = config.start_session('recipe_name') assert session == config From 63d0bfbd3fc43d03782e233fe33f0b8db6b871ed Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 11:27:03 +0100 Subject: [PATCH 23/45] Improve error message --- esmvalcore/experimental/config/_config_object.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index b5c770168d..2e17fb6b25 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -26,7 +26,8 @@ def load_from_file(filename): if try_path.exists(): path = try_path else: - raise FileNotFoundError(f'No such file: `{filename}`') + raise FileNotFoundError(f'Cannot find: `{filename}`' + f'locally or in `{try_path}`') _load_user_config(path) From 05d0be18a05f34e7929fc3fbf89a82af172dbfb9 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 13:12:27 +0100 Subject: [PATCH 24/45] Ignore flake8 import warning Warning for experimental API must come before the other imports --- esmvalcore/experimental/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py index cf46dd68fb..a0accd9a77 100644 --- a/esmvalcore/experimental/__init__.py +++ b/esmvalcore/experimental/__init__.py @@ -7,7 +7,7 @@ '\n Note that this API is experimental and may be subject to change.' '\n More info: https://github.com/ESMValGroup/ESMValCore/issues/498', ) -from .config import CFG +from .config import CFG # noqa: E402 __all__ = [ 'CFG', From 3089ac129e717cd07b4917f38fe8fb41730ba3a1 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 13:13:04 +0100 Subject: [PATCH 25/45] Format warning messages in a bit friendlier way --- esmvalcore/experimental/__init__.py | 2 +- esmvalcore/experimental/_warnings.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 esmvalcore/experimental/_warnings.py diff --git a/esmvalcore/experimental/__init__.py b/esmvalcore/experimental/__init__.py index a0accd9a77..5614ce7770 100644 --- a/esmvalcore/experimental/__init__.py +++ b/esmvalcore/experimental/__init__.py @@ -1,6 +1,6 @@ """ESMValCore experimental API module.""" -import warnings +from ._warnings import warnings warnings.warn( '\n Thank you for trying out the new ESMValCore API.' diff --git a/esmvalcore/experimental/_warnings.py b/esmvalcore/experimental/_warnings.py new file mode 100644 index 0000000000..d2e76acaf4 --- /dev/null +++ b/esmvalcore/experimental/_warnings.py @@ -0,0 +1,14 @@ +import warnings + + +def warning_formatter(message, + category, + filename, + lineno, + file=None, + line=None): + """Patch warning formatting to not mention itself.""" + return f'{filename}:{lineno}: {category.__name__}: {message}\n' + + +warnings.formatwarning = warning_formatter From 060078ce752ebe759c4b71fb7fdf26e32ce56e14 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 13:22:13 +0100 Subject: [PATCH 26/45] Add module docstring --- esmvalcore/experimental/_warnings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esmvalcore/experimental/_warnings.py b/esmvalcore/experimental/_warnings.py index d2e76acaf4..c1043481fc 100644 --- a/esmvalcore/experimental/_warnings.py +++ b/esmvalcore/experimental/_warnings.py @@ -1,3 +1,5 @@ +"""ESMValTool Warnings.""" + import warnings From dac9a409578bccd9765a401e1dd6add8d15c2a99 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 13:30:19 +0100 Subject: [PATCH 27/45] Fix linter warnings --- esmvalcore/experimental/_warnings.py | 16 ++++++++-------- .../experimental/config/_validated_config.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/esmvalcore/experimental/_warnings.py b/esmvalcore/experimental/_warnings.py index c1043481fc..0795667be1 100644 --- a/esmvalcore/experimental/_warnings.py +++ b/esmvalcore/experimental/_warnings.py @@ -1,16 +1,16 @@ -"""ESMValTool Warnings.""" +"""ESMValTool warnings.""" import warnings -def warning_formatter(message, - category, - filename, - lineno, - file=None, - line=None): +def _warning_formatter(message, + category, + filename, + lineno, + file=None, + line=None): """Patch warning formatting to not mention itself.""" return f'{filename}:{lineno}: {category.__name__}: {message}\n' -warnings.formatwarning = warning_formatter +warnings.formatwarning = _warning_formatter diff --git a/esmvalcore/experimental/config/_validated_config.py b/esmvalcore/experimental/config/_validated_config.py index 2bb3411e04..5a5ea83baf 100644 --- a/esmvalcore/experimental/config/_validated_config.py +++ b/esmvalcore/experimental/config/_validated_config.py @@ -25,6 +25,7 @@ def __init__(self, *args, **kwargs): self.update(*args, **kwargs) def __setitem__(self, key, val): + """Map key to value.""" try: cval = self.validate[key](val) except ValueError as verr: @@ -36,9 +37,11 @@ def __setitem__(self, key, val): dict.__setitem__(self, key, cval) def __getitem__(self, key): + """Return value mapped by key.""" return dict.__getitem__(self, key) def __repr__(self): + """Return canonical string representation.""" class_name = self.__class__.__name__ indent = len(class_name) + 1 repr_split = pprint.pformat(dict(self), indent=1, @@ -47,6 +50,7 @@ def __repr__(self): return '{}({})'.format(class_name, repr_indented) def __str__(self): + """Return string representation.""" return '\n'.join(map('{0[0]}: {0[1]}'.format, sorted(self.items()))) def __iter__(self): @@ -54,15 +58,17 @@ def __iter__(self): yield from sorted(dict.__iter__(self)) def __len__(self): + """Return number of config keys.""" return dict.__len__(self) def __delitem__(self, key): + """Delete key/value from config.""" dict.__delitem__(self, key) def copy(self): - """Copy the keys this object to a dict.""" + """Copy the keys/values of this object to a dict.""" return {k: dict.__getitem__(self, k) for k in self} def clear(self): - """Clear Config dictionary.""" + """Clear Config.""" dict.clear(self) From 2b9267b8786f783ab77139750b0cca110b06bd6c Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 3 Dec 2020 17:02:32 +0100 Subject: [PATCH 28/45] Add api entry page --- doc/api/esmvalcore.api.rst | 13 +++++++++++++ doc/api/esmvalcore.rst | 1 + 2 files changed, 14 insertions(+) create mode 100644 doc/api/esmvalcore.api.rst diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst new file mode 100644 index 0000000000..d63014ebdc --- /dev/null +++ b/doc/api/esmvalcore.api.rst @@ -0,0 +1,13 @@ +.. _experimental_api: + +Experimental API +================ + +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod +tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo +consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse +cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non +proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + +.. automodule:: esmvalcore.experimental diff --git a/doc/api/esmvalcore.rst b/doc/api/esmvalcore.rst index 7d1a3b3728..b9d2127688 100644 --- a/doc/api/esmvalcore.rst +++ b/doc/api/esmvalcore.rst @@ -8,3 +8,4 @@ library. This section documents the public API of ESMValCore. esmvalcore.cmor esmvalcore.preprocessor + esmvalcore.api From 9c6961999a4e2da8402c2a522caad9daa45293a4 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 4 Dec 2020 11:25:33 +0100 Subject: [PATCH 29/45] Add documentation for config/session --- doc/api/esmvalcore.api.rst | 142 ++++++++++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 8 deletions(-) diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst index d63014ebdc..dd5e97fbec 100644 --- a/doc/api/esmvalcore.api.rst +++ b/doc/api/esmvalcore.api.rst @@ -3,11 +3,137 @@ Experimental API ================ -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse -cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non -proident, sunt in culpa qui officia deserunt mollit anim id est laborum. - -.. automodule:: esmvalcore.experimental +This page describes the new ESMValCore API. +The API module is available in the submodule ``esmvalcore.experimental``. +The API is under development, so use at your own risk! + +Config +****** + +Configuration of ESMValCore/Tool is done via the ``Config`` object. +The global configuration can be imported from the ``esmvalcore.experimental`` module as ``CFG``: + +.. code-block:: python + + >>> from esmvalcore.experimental import CFG + >>> CFG + Config({'auxiliary_data_dir': PosixPath('/home/user/auxiliary_data'), + 'compress_netcdf': False, + 'config_developer_file': None, + 'config_file': PosixPath('/home/user/.esmvaltool/config-user.yml'), + 'drs': {'CMIP5': 'default', 'CMIP6': 'default'}, + 'exit_on_warning': False, + 'log_level': 'info', + 'max_parallel_tasks': None, + 'output_dir': PosixPath('/home/user/esmvaltool_output'), + 'output_file_type': 'png', + 'profile_diagnostic': False, + 'remove_preproc_dir': True, + 'rootpath': {'CMIP5': '~/default_inputpath', + 'CMIP6': '~/default_inputpath', + 'default': '~/default_inputpath'}, + 'save_intermediary_cubes': False, + 'write_netcdf': True, + 'write_plots': True}) + +The parameters for the user configuration file are listed `here `__. + +``CFG`` is essentially a python dictionary with a few extra functions, similar to ``matplotlib.rcParams``. +This means that values can be updated like this: + +.. code-block:: python + + >>> CFG['output_dir'] = '~/esmvaltool_output' + >>> CFG['output_dir'] + PosixPath('/home/user/esmvaltool_output') + +Notice that ``CFG`` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory. +All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key: + +.. code-block:: python + + >>> CFG['otoptu_dri'] = '~/esmvaltool_output' + InvalidConfigParameter: `otoptu_dri` is not a valid config parameter. + +Or, if the value entered cannot be converted to the expected type: + +.. code-block:: python + + >>> CFG['max_years'] = '🐜' + InvalidConfigParameter: Key `max_years`: Could not convert '🐜' to int + +``Config`` is also flexible, so it tries to correct the type of your input if possible: + +.. code-block:: python + + >>> CFG['max_years'] = '123' # str + >>> type(CFG['max_years']) + int + +By default, the config is loaded from the default location (``/home/user/.esmvaltool/config-user.yml``). +If it does not exist, it falls back to the default values. +to load a different file: + +.. code-block:: python + + >>> CFG.load_from_file('~/my-config.yml') + +Or to reload the current config: + +.. code-block:: python + + >>> CFG.reload() + + +Session +******* + +Recipes and diagnostics will be run in their own directories. +This behaviour can be controlled via the ``Session`` object. +A ``Session`` can be initiated from the global ``Config``. + +.. code-block:: python + + >>> session = CFG.start_session(name='my_session') + +A ``Session`` is very similar to the config. +It is also a dictionary, and copies all the keys from the ``Config``. +At this moment, ``session`` is essentially a copy of ``CFG``: + +.. code-block:: python + + >>> print(session == CFG) + True + >>> session['output_dir'] = '~/session_output' + >>> print(session == CFG) # False + False + +A ``Session`` also knows about the directories where the data will stored. +The session name is used to previx the directories. + +.. code-block:: python + + >>> session.session_dir + /home/user/esmvaltool_output/my_session_20201203_155821 + >>> session.run_dir + /home/user/esmvaltool_output/my_session_20201203_155821/run + >>> session.work_dir + /home/user/esmvaltool_output/my_session_20201203_155821/work + >>> session.preproc_dir + /home/user/esmvaltool_output/my_session_20201203_155821/preproc + >>> session.plot_dir + /home/user/esmvaltool_output/my_session_20201203_155821/plots + +``Session`` objects are persistent, so multiple sessions can be initiated from the ``Config``. + + +API reference +************* + +.. autoclass:: esmvalcore.experimental.config._config_object.Config + :no-inherited-members: + :no-show-inheritance: + +.. autoclass:: esmvalcore.experimental.config._config_object.Session + :no-inherited-members: + :no-show-inheritance: From f9a16fd213aa99d9baa29facba4ddd6b190e1d3c Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 09:40:00 +0000 Subject: [PATCH 30/45] Update doc/api/esmvalcore.api.rst Co-authored-by: Bouwe Andela --- doc/api/esmvalcore.api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst index dd5e97fbec..ede1795359 100644 --- a/doc/api/esmvalcore.api.rst +++ b/doc/api/esmvalcore.api.rst @@ -109,7 +109,7 @@ At this moment, ``session`` is essentially a copy of ``CFG``: False A ``Session`` also knows about the directories where the data will stored. -The session name is used to previx the directories. +The session name is used to prefix the directories. .. code-block:: python From ec2a5fa7f0d5382dfd6579c4803033e51cf24907 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 11:20:00 +0100 Subject: [PATCH 31/45] Clarify text/code in documentation --- doc/api/esmvalcore.api.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst index ede1795359..1012ddd9ef 100644 --- a/doc/api/esmvalcore.api.rst +++ b/doc/api/esmvalcore.api.rst @@ -104,7 +104,7 @@ At this moment, ``session`` is essentially a copy of ``CFG``: >>> print(session == CFG) True - >>> session['output_dir'] = '~/session_output' + >>> session['output_dir'] = '~/my_output_dir' >>> print(session == CFG) # False False @@ -114,17 +114,17 @@ The session name is used to prefix the directories. .. code-block:: python >>> session.session_dir - /home/user/esmvaltool_output/my_session_20201203_155821 + /home/user/my_output_dir/my_session_20201203_155821 >>> session.run_dir - /home/user/esmvaltool_output/my_session_20201203_155821/run + /home/user/my_output_dir/my_session_20201203_155821/run >>> session.work_dir - /home/user/esmvaltool_output/my_session_20201203_155821/work + /home/user/my_output_dir/my_session_20201203_155821/work >>> session.preproc_dir - /home/user/esmvaltool_output/my_session_20201203_155821/preproc + /home/user/my_output_dir/my_session_20201203_155821/preproc >>> session.plot_dir - /home/user/esmvaltool_output/my_session_20201203_155821/plots + /home/user/my_output_dir/my_session_20201203_155821/plots -``Session`` objects are persistent, so multiple sessions can be initiated from the ``Config``. +Unlike the global configuration, of which only one can exist, multiple sessions can be initiated from the ``Config``. API reference From 50215a3ee8a1c37db2d97d3a0aefe3f7ddab2ed4 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 11:20:55 +0100 Subject: [PATCH 32/45] Remove support for environment variables --- .../experimental/config/_config_object.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 2e17fb6b25..54128e3007 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -1,6 +1,5 @@ """Importable config object.""" -import os from datetime import datetime from pathlib import Path @@ -162,25 +161,11 @@ def _load_user_config(filename: str, raise_exception: bool = True): CFG.update(mapping) -def get_user_config_location(): - """Get the user config location by looking in the expected places. - - Check if environment variable `ESMVALTOOL_CONFIG` exists, otherwise - use the default location in the user config dir (`~/.esmvaltool`). - """ - try: - config_location = Path(os.environ['ESMVALTOOL_CONFIG']) - except KeyError: - config_location = USER_CONFIG_DIR / 'config-user.yml' - - return config_location - - DEFAULT_CONFIG_DIR = Path(esmvalcore.__file__).parent DEFAULT_CONFIG = DEFAULT_CONFIG_DIR / 'config-user.yml' USER_CONFIG_DIR = Path.home() / '.esmvaltool' -USER_CONFIG = get_user_config_location() +USER_CONFIG = USER_CONFIG_DIR / 'config-user.yml' # initialize placeholders CFG_DEFAULT = Config() From cbdcb1e8b78a5fcd6d42e47e09686d0e359b091a Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 11:21:12 +0100 Subject: [PATCH 33/45] Clarify deprecation message --- esmvalcore/experimental/config/_config_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index dc216ba434..1e2904460d 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -234,10 +234,10 @@ def deprecate(func, variable, version: str = None): version = 'a future version' if current_version >= version: - warnings.warn(f"`{variable}` has been deprecated in {version}", + warnings.warn(f"`{variable}` has been removed in {version}", ESMValToolDeprecationWarning) else: - warnings.warn(f"`{variable}` will be deprecated in {version}.", + warnings.warn(f"`{variable}` will be removed in {version}.", ESMValToolDeprecationWarning, stacklevel=2) From 4c82878f064bf0d2c8f851e4901c44a975d28f45 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 11:21:54 +0100 Subject: [PATCH 34/45] Make validate/config reader functions private --- esmvalcore/experimental/config/_config_object.py | 10 +++++----- esmvalcore/experimental/config/_validated_config.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 54128e3007..4c7f8b77d8 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -14,7 +14,7 @@ class Config(ValidatedConfig): """ESMValTool configuration object.""" - validate = _validators + _validate = _validators @staticmethod def load_from_file(filename): @@ -62,7 +62,7 @@ class Session(ValidatedConfig): recipe (default='session'). """ - validate = _validators + _validate = _validators def __init__(self, config: dict, name: str = 'session'): super().__init__(config) @@ -110,7 +110,7 @@ def config_dir(self): return USER_CONFIG_DIR -def read_config_file(config_file): +def _read_config_file(config_file): """Read config user file and store settings in a dictionary.""" config_file = Path(config_file) if not config_file.exists(): @@ -124,7 +124,7 @@ def read_config_file(config_file): def _load_default_config(filename: str): """Load the default configuration.""" - mapping = read_config_file(filename) + mapping = _read_config_file(filename) global CFG_DEFAULT @@ -147,7 +147,7 @@ def _load_user_config(filename: str, raise_exception: bool = True): has not been defined (i.e. first start). """ try: - mapping = read_config_file(filename) + mapping = _read_config_file(filename) mapping['config_file'] = filename except IOError: if raise_exception: diff --git a/esmvalcore/experimental/config/_validated_config.py b/esmvalcore/experimental/config/_validated_config.py index 5a5ea83baf..cbff837115 100644 --- a/esmvalcore/experimental/config/_validated_config.py +++ b/esmvalcore/experimental/config/_validated_config.py @@ -17,7 +17,7 @@ class InvalidConfigParameter(SuppressedError): class ValidatedConfig(MutableMapping, dict): """Based on `matplotlib.rcParams`.""" - validate = {} + _validate = {} # validate values on the way in def __init__(self, *args, **kwargs): @@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs): def __setitem__(self, key, val): """Map key to value.""" try: - cval = self.validate[key](val) + cval = self._validate[key](val) except ValueError as verr: raise InvalidConfigParameter(f"Key `{key}`: {verr}") from None except KeyError: From 953d2abf3895a82ced6c0fffcba3d659de82ab91 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 11:22:12 +0100 Subject: [PATCH 35/45] Fix bug with `session_dir` not updating automatically --- esmvalcore/experimental/config/_config_object.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 4c7f8b77d8..44c3efcbde 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -66,9 +66,9 @@ class Session(ValidatedConfig): def __init__(self, config: dict, name: str = 'session'): super().__init__(config) - self.init_session_dir(name) + self.set_session_name(name) - def init_session_dir(self, name: str = 'session'): + def set_session_name(self, name: str = 'session'): """Initialize session. The `name` is used to name the working directory, e.g. @@ -76,13 +76,12 @@ def init_session_dir(self, name: str = 'session'): interactive session, defaults to `session`. """ now = datetime.utcnow().strftime("%Y%m%d_%H%M%S") - session_name = f"{name}_{now}" - self._session_dir = self['output_dir'] / session_name + self.session_name = f"{name}_{now}" @property def session_dir(self): """Return session directory.""" - return self._session_dir + return self['output_dir'] / self.session_name @property def preproc_dir(self): From b6ba919a0949451a5d4fb0dcd7ab4e133d318d2e Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 12:11:10 +0100 Subject: [PATCH 36/45] Remove global statements --- esmvalcore/experimental/config/_config_object.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 44c3efcbde..3abc7d1907 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -124,9 +124,6 @@ def _read_config_file(config_file): def _load_default_config(filename: str): """Load the default configuration.""" mapping = _read_config_file(filename) - - global CFG_DEFAULT - CFG_DEFAULT.update(mapping) @@ -153,8 +150,6 @@ def _load_user_config(filename: str, raise_exception: bool = True): raise mapping = {} - global CFG - CFG.clear() CFG.update(CFG_DEFAULT) CFG.update(mapping) From 2191035eaa4e897b8e98690d84c33ef97e897c32 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 16:54:01 +0100 Subject: [PATCH 37/45] Warn if config parameters are missing By default, drs and rootpath are not defined, so we must have some mechanism to warn users early that their config is not complete. --- esmvalcore/config-user.yml | 20 +++++++++---------- .../experimental/config/_config_object.py | 7 +++++++ .../experimental/config/_validated_config.py | 15 ++++++++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index 8871dadee6..9f6526d3f5 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -41,19 +41,19 @@ profile_diagnostic: false # Rootpaths to the data from different projects (lists are also possible) # these are generic entries to better allow you to enter your own # For site-specific entries, see below -rootpath: - CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] - OBS: ~/obs_inputpath - RAWOBS: ~/rawobs_inputpath - default: ~/default_inputpath - CORDEX: ~/default_inputpath +# rootpath: +# CMIP5: [~/cmip5_inputpath1, ~/cmip5_inputpath2] +# OBS: ~/obs_inputpath +# RAWOBS: ~/rawobs_inputpath +# default: ~/default_inputpath +# CORDEX: ~/default_inputpath # Directory structure for input data: [default]/BADC/DKRZ/ETHZ/etc # See config-developer.yml for definitions. -drs: - CMIP5: default - CORDEX: default - OBS: default +# drs: +# CMIP5: default +# CORDEX: default +# OBS: default # Site-specific entries: Jasmin # Uncomment the lines below to locate data on JASMIN diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 3abc7d1907..7eaf4bea04 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -10,11 +10,18 @@ from ._config_validators import _validators from ._validated_config import ValidatedConfig +URL = ('https://docs.esmvaltool.org/projects/' + 'ESMValCore/en/latest/quickstart/configure.html') + class Config(ValidatedConfig): """ESMValTool configuration object.""" _validate = _validators + _warn_if_missing = ( + ('drs', URL), + ('rootpath', URL), + ) @staticmethod def load_from_file(filename): diff --git a/esmvalcore/experimental/config/_validated_config.py b/esmvalcore/experimental/config/_validated_config.py index cbff837115..06226287ae 100644 --- a/esmvalcore/experimental/config/_validated_config.py +++ b/esmvalcore/experimental/config/_validated_config.py @@ -1,6 +1,7 @@ """Config validation objects.""" import pprint +import warnings from collections.abc import MutableMapping from .._exceptions import SuppressedError @@ -10,6 +11,10 @@ class InvalidConfigParameter(SuppressedError): """Config parameter is invalid.""" +class MissingConfigParameter(UserWarning): + """Config parameter is missing.""" + + # The code for this class was take from matplotlib (v3.3) and modified to # fit the needs of ESMValCore. Matplotlib is licenced under the terms of # the the 'Python Software Foundation License' @@ -18,11 +23,13 @@ class ValidatedConfig(MutableMapping, dict): """Based on `matplotlib.rcParams`.""" _validate = {} + _warn_if_missing = () # validate values on the way in def __init__(self, *args, **kwargs): super().__init__() self.update(*args, **kwargs) + self.check_missing() def __setitem__(self, key, val): """Map key to value.""" @@ -65,6 +72,14 @@ def __delitem__(self, key): """Delete key/value from config.""" dict.__delitem__(self, key) + def check_missing(self): + """Check and warn for missing variables.""" + for (key, more_info) in self._warn_if_missing: + if key not in self: + more_info = f' ({more_info})' if more_info else '' + warnings.warn(f'`{key}` is not defined{more_info}', + MissingConfigParameter) + def copy(self): """Copy the keys/values of this object to a dict.""" return {k: dict.__getitem__(self, k) for k in self} From fe79f051f3c8f7c3bd646e15fa204a31a0c7b143 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 7 Dec 2020 17:08:05 +0100 Subject: [PATCH 38/45] Add custom validation error --- .../experimental/config/_config_validators.py | 30 +++++++++++-------- .../experimental/config/_validated_config.py | 3 +- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index 1e2904460d..0b5fecb77e 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -6,6 +6,15 @@ from pathlib import Path +class ValidationError(ValueError): + """Custom validation error.""" + + +# Custom warning, because DeprecationWarning is hidden by default +class ESMValToolDeprecationWarning(UserWarning): + """Configuration key has been deprecated.""" + + # The code for this function was taken from matplotlib (v3.3) and modified # to fit the needs of ESMValCore. Matplotlib is licenced under the terms of # the the 'Python Software Foundation License' @@ -24,7 +33,7 @@ def validator(inp): return cls(inp) except ValueError as err: if isinstance(cls, type): - raise ValueError( + raise ValidationError( f'Could not convert {repr(inp)} to {cls.__name__}' ) from err raise @@ -78,11 +87,11 @@ def func(inp): if not isinstance(val, str) or val ] else: - raise ValueError( + raise ValidationError( f"Expected str or other non-set iterable, but got {inp}") if n_items is not None and len(inp) != n_items: - raise ValueError(f"Expected {n_items} values, " - f"but there are {len(inp)} values in {inp}") + raise ValidationError(f"Expected {n_items} values, " + f"but there are {len(inp)} values in {inp}") return inp try: @@ -102,7 +111,7 @@ def validate_bool(value, allow_none=False): if (value is None) and allow_none: return value if not isinstance(value, bool): - raise ValueError(f"Could not convert `{value}` to `bool`") + raise ValidationError(f"Could not convert `{value}` to `bool`") return value @@ -113,7 +122,7 @@ def validate_path(value, allow_none=False): try: path = Path(value).expanduser().absolute() except TypeError as err: - raise ValueError(f"Expected a path, but got {value}") from err + raise ValidationError(f"Expected a path, but got {value}") from err else: return path @@ -121,7 +130,7 @@ def validate_path(value, allow_none=False): def validate_positive(value): """Check if number is positive.""" if value is not None and value <= 0: - raise ValueError(f'Expected a positive number, but got {value}') + raise ValidationError(f'Expected a positive number, but got {value}') return value @@ -187,7 +196,7 @@ def validate_check_level(value): try: value = CheckLevels[value.upper()] except KeyError: - raise ValueError( + raise ValidationError( f'`{value}` is not a valid strictness level') from None else: @@ -208,11 +217,6 @@ def validate_diagnostics(diagnostics): } -# Custom warning, because DeprecationWarning is hidden by default -class ESMValToolDeprecationWarning(UserWarning): - """Configuration key has been deprecated.""" - - def deprecate(func, variable, version: str = None): """Wrap function to mark variables to be deprecated. diff --git a/esmvalcore/experimental/config/_validated_config.py b/esmvalcore/experimental/config/_validated_config.py index 06226287ae..e3366d25a1 100644 --- a/esmvalcore/experimental/config/_validated_config.py +++ b/esmvalcore/experimental/config/_validated_config.py @@ -5,6 +5,7 @@ from collections.abc import MutableMapping from .._exceptions import SuppressedError +from ._config_validators import ValidationError class InvalidConfigParameter(SuppressedError): @@ -35,7 +36,7 @@ def __setitem__(self, key, val): """Map key to value.""" try: cval = self._validate[key](val) - except ValueError as verr: + except ValidationError as verr: raise InvalidConfigParameter(f"Key `{key}`: {verr}") from None except KeyError: raise InvalidConfigParameter( From a089e81b21dd306c61a6b295d108e123f8d70883 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 11:24:24 +0100 Subject: [PATCH 39/45] Make Config/Session available via public API --- doc/api/esmvalcore.api.rst | 8 ++++++-- esmvalcore/experimental/config/__init__.py | 4 +++- esmvalcore/experimental/config/_config_object.py | 9 ++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/doc/api/esmvalcore.api.rst b/doc/api/esmvalcore.api.rst index 1012ddd9ef..8870ef67e0 100644 --- a/doc/api/esmvalcore.api.rst +++ b/doc/api/esmvalcore.api.rst @@ -130,10 +130,14 @@ Unlike the global configuration, of which only one can exist, multiple sessions API reference ************* -.. autoclass:: esmvalcore.experimental.config._config_object.Config +.. autoclass:: esmvalcore.experimental.config.CFG :no-inherited-members: :no-show-inheritance: -.. autoclass:: esmvalcore.experimental.config._config_object.Session +.. autoclass:: esmvalcore.experimental.config.Config + :no-inherited-members: + :no-show-inheritance: + +.. autoclass:: esmvalcore.experimental.config.Session :no-inherited-members: :no-show-inheritance: diff --git a/esmvalcore/experimental/config/__init__.py b/esmvalcore/experimental/config/__init__.py index 7ac457d864..17998a0b60 100644 --- a/esmvalcore/experimental/config/__init__.py +++ b/esmvalcore/experimental/config/__init__.py @@ -1,7 +1,9 @@ """ESMValTool config module.""" -from ._config_object import CFG +from ._config_object import CFG, Config, Session __all__ = [ 'CFG', + 'Config', + 'Session', ] diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 7eaf4bea04..530eda53bf 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -15,7 +15,11 @@ class Config(ValidatedConfig): - """ESMValTool configuration object.""" + """ESMValTool configuration object. + + Do not instantiate this class directly, but use + :obj:`esmvalcore.experimental.CFG` instead. + """ _validate = _validators _warn_if_missing = ( @@ -60,6 +64,9 @@ def start_session(self, name: str): class Session(ValidatedConfig): """Container class for session configuration and directory information. + Do not instantiate this class directly, but use + :obj:`CFG.start_session` instead. + Parameters ---------- config : dict From 0bf4df759ced69c46ccebc0da4cba80760330ad2 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 11:48:13 +0100 Subject: [PATCH 40/45] Update docstring --- esmvalcore/experimental/config/_config_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 530eda53bf..f0100c337a 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -29,7 +29,7 @@ class Config(ValidatedConfig): @staticmethod def load_from_file(filename): - """Reload user configuration from the given file.""" + """Load user configuration from the given file.""" path = Path(filename).expanduser() if not path.exists(): try_path = USER_CONFIG_DIR / filename From abac115b0032d206bc08b36cea6ba37d7d684259 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 12:11:47 +0100 Subject: [PATCH 41/45] Re-organize class --- .../experimental/config/_config_object.py | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index f0100c337a..7450780911 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -27,8 +27,50 @@ class Config(ValidatedConfig): ('rootpath', URL), ) - @staticmethod - def load_from_file(filename): + @classmethod + def _load_user_config(cls, filename: str, raise_exception: bool = True): + """Load user configuration from the given file. + + The config is cleared and updated in-place. + + Parameters + ---------- + filename: pathlike + Name of the config file, must be yaml format + raise_exception : bool + Raise an exception if `filename` can not be found (default). + Otherwise, silently pass and use the default configuration. This + setting is necessary for the case where + `.esmvaltool/config-user.yml` has not been defined (i.e. first + start). + """ + new = cls() + + try: + mapping = _read_config_file(filename) + mapping['config_file'] = filename + except IOError: + if raise_exception: + raise + mapping = {} + + new.clear() + new.update(CFG_DEFAULT) + new.update(mapping) + + return new + + @classmethod + def _load_default_config(cls, filename: str): + """Load the default configuration.""" + new = cls() + + mapping = _read_config_file(filename) + new.update(mapping) + + return new + + def load_from_file(self, filename): """Load user configuration from the given file.""" path = Path(filename).expanduser() if not path.exists(): @@ -39,7 +81,8 @@ def load_from_file(filename): raise FileNotFoundError(f'Cannot find: `{filename}`' f'locally or in `{try_path}`') - _load_user_config(path) + self.clear() + self.update(Config._load_user_config(path)) def reload(self): """Reload the config file.""" @@ -135,40 +178,6 @@ def _read_config_file(config_file): return cfg -def _load_default_config(filename: str): - """Load the default configuration.""" - mapping = _read_config_file(filename) - CFG_DEFAULT.update(mapping) - - -def _load_user_config(filename: str, raise_exception: bool = True): - """Load user configuration from the given file. - - The config is cleared and updated in-place. - - Parameters - ---------- - filename: pathlike - Name of the config file, must be yaml format - raise_exception : bool - Raise an exception if `filename` can not be found (default). - Otherwise, silently pass and use the default configuration. This - setting is necessary for the case where `.esmvaltool/config-user.yml` - has not been defined (i.e. first start). - """ - try: - mapping = _read_config_file(filename) - mapping['config_file'] = filename - except IOError: - if raise_exception: - raise - mapping = {} - - CFG.clear() - CFG.update(CFG_DEFAULT) - CFG.update(mapping) - - DEFAULT_CONFIG_DIR = Path(esmvalcore.__file__).parent DEFAULT_CONFIG = DEFAULT_CONFIG_DIR / 'config-user.yml' @@ -176,9 +185,5 @@ def _load_user_config(filename: str, raise_exception: bool = True): USER_CONFIG = USER_CONFIG_DIR / 'config-user.yml' # initialize placeholders -CFG_DEFAULT = Config() -CFG = Config() - -# update config objects -_load_default_config(DEFAULT_CONFIG) -_load_user_config(USER_CONFIG, raise_exception=False) +CFG_DEFAULT = Config._load_default_config(DEFAULT_CONFIG) +CFG = Config._load_user_config(USER_CONFIG, raise_exception=False) From cc70b322535051722ed06b7b5da3ca54bf863418 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 12:12:02 +0100 Subject: [PATCH 42/45] Move imports to top --- esmvalcore/experimental/config/_config_validators.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index 0b5fecb77e..134438e8c9 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -5,6 +5,10 @@ from functools import lru_cache from pathlib import Path +from esmvalcore._config import load_config_developer +from esmvalcore._recipe import TASKSEP +from esmvalcore.cmor.check import CheckLevels + class ValidationError(ValueError): """Custom validation error.""" @@ -180,7 +184,6 @@ def validate_oldstyle_drs(value): def validate_config_developer(value): """Validate and load config developer path.""" - from esmvalcore._config import load_config_developer path = validate_path_or_none(value) load_config_developer(path) @@ -190,8 +193,6 @@ def validate_config_developer(value): def validate_check_level(value): """Validate CMOR level check.""" - from esmvalcore.cmor.check import CheckLevels - if isinstance(value, str): try: value = CheckLevels[value.upper()] @@ -207,8 +208,6 @@ def validate_check_level(value): def validate_diagnostics(diagnostics): """Validate diagnostic location.""" - from esmvalcore._recipe import TASKSEP - if isinstance(diagnostics, str): diagnostics = diagnostics.strip().split(' ') return { From dcae1859f213f5e388fd3d5798a9b158c9ef878f Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 12:12:46 +0100 Subject: [PATCH 43/45] Move import to top --- esmvalcore/experimental/config/_config_validators.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index 134438e8c9..bd55a6ffff 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -5,6 +5,7 @@ from functools import lru_cache from pathlib import Path +from esmvalcore import __version__ as current_version from esmvalcore._config import load_config_developer from esmvalcore._recipe import TASKSEP from esmvalcore.cmor.check import CheckLevels @@ -231,8 +232,6 @@ def deprecate(func, variable, version: str = None): Version to deprecate the variable in, should be something like '2.2.3' """ - from esmvalcore import __version__ as current_version - if not version: version = 'a future version' From 581085f92d6fbef90fba3910b9659b2bfd246dbe Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 14:32:24 +0100 Subject: [PATCH 44/45] Update docstring to reflect code changes --- esmvalcore/experimental/config/_config_object.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index 7450780911..ba4597f18e 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -126,11 +126,10 @@ def __init__(self, config: dict, name: str = 'session'): self.set_session_name(name) def set_session_name(self, name: str = 'session'): - """Initialize session. + """Set the name for the session. - The `name` is used to name the working directory, e.g. - `recipe_example_20200916/`. If no name is given, such as in an - interactive session, defaults to `session`. + The `name` is used to name the session directory, e.g. + `session_20201208_132800/`. The date is suffixed automatically. """ now = datetime.utcnow().strftime("%Y%m%d_%H%M%S") self.session_name = f"{name}_{now}" From a81a9708ab0caf787025976fb74433f3582e89a0 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 8 Dec 2020 14:36:19 +0100 Subject: [PATCH 45/45] Remove redundant clear statement --- esmvalcore/experimental/config/_config_object.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/experimental/config/_config_object.py b/esmvalcore/experimental/config/_config_object.py index ba4597f18e..0bd78259d7 100644 --- a/esmvalcore/experimental/config/_config_object.py +++ b/esmvalcore/experimental/config/_config_object.py @@ -54,7 +54,6 @@ def _load_user_config(cls, filename: str, raise_exception: bool = True): raise mapping = {} - new.clear() new.update(CFG_DEFAULT) new.update(mapping)