From 1b41892d3769c3496045e6f2229a35be661a7318 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 22 Sep 2023 22:15:55 +0200 Subject: [PATCH 1/3] Remove the deprecated option use_legacy_supplementaries --- doc/recipe/preprocessor.rst | 128 --- esmvalcore/_recipe/recipe.py | 219 +--- esmvalcore/_recipe/to_datasets.py | 15 +- esmvalcore/config/_config_validators.py | 27 - esmvalcore/preprocessor/__init__.py | 4 - .../preprocessor/_supplementary_vars.py | 148 --- tests/integration/conftest.py | 1 - .../test_add_fx_variables.py | 296 ------ .../test_add_supplementary_variables.py | 15 +- tests/integration/recipe/test_recipe.py | 944 +----------------- tests/integration/test_deprecated_config.py | 62 -- tests/unit/recipe/test_recipe.py | 50 - tests/unit/test_dataset.py | 3 - 13 files changed, 14 insertions(+), 1898 deletions(-) delete mode 100644 tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 19020ba8f8..26c50df715 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -300,134 +300,6 @@ and cell measure (``areacella``), but do not use ``areacella`` for dataset timerange: '1990/2000' scripts: null - -.. _`Fx variables as cell measures or ancillary variables`: - -Legacy method of specifying supplementary variables ---------------------------------------------------- - -.. deprecated:: 2.8.0 - The legacy method of specifying supplementary variables is deprecated and will - be removed in version 2.10.0. - To upgrade, remove all occurrences of ``fx_variables`` from your recipes and - rely on automatically defining the supplementary variables based on the - requirement of the preprocessor functions or specify them using the methods - described above. - To keep using the legacy behaviour until v2.10.0, set - ``use_legacy_supplementaries: true`` in the :ref:`user configuration file` or - run the tool with the flag ``--use-legacy-supplementaries=True``. - -Prior to version 2.8.0 of the tool, the supplementary variables could not be -defined at the variable or dataset level in the recipe, but could only be -defined in the preprocessor function that uses them using the ``fx_variables`` -argument. -This does not work well because in practice different datasets store their -supplementary variables under different facets. -For example, one dataset might only provide the ``areacella`` variable under the -``1pctCO2`` experiment while another one might only provide it for the -``historical`` experiment. -This forced the user to define a preprocessor per dataset, which was -inconvenient. - -============================================================== ===================== -Preprocessor Default fx variables -============================================================== ===================== -:ref:`area_statistics` ``areacella``, ``areacello`` -:ref:`mask_landsea` ``sftlf``, ``sftof`` -:ref:`mask_landseaice` ``sftgif`` -:ref:`volume_statistics` ``volcello`` -:ref:`weighting_landsea_fraction` ``sftlf``, ``sftof`` -============================================================== ===================== - -If the option ``fx_variables`` is not explicitly specified for these -preprocessors, the default fx variables in the second column are automatically -used. If given, the ``fx_variables`` argument specifies the fx variables that -the user wishes to input to the corresponding preprocessor function. The user -may specify these by simply adding the names of the variables, e.g., - -.. code-block:: yaml - - fx_variables: - areacello: - volcello: - -or by additionally specifying further keys that are used to define the fx -datasets, e.g., - -.. code-block:: yaml - - fx_variables: - areacello: - mip: Ofx - exp: piControl - volcello: - mip: Omon - -This might be useful to select fx files from a specific ``mip`` table or from a -specific ``exp`` in case not all experiments provide the fx variable. - -Alternatively, the ``fx_variables`` argument can also be specified as a list: - -.. code-block:: yaml - - fx_variables: ['areacello', 'volcello'] - -or as a list of dictionaries: - -.. code-block:: yaml - - fx_variables: [{'short_name': 'areacello', 'mip': 'Ofx', 'exp': 'piControl'}, {'short_name': 'volcello', 'mip': 'Omon'}] - -The recipe parser will automatically find the data files that are associated -with these variables and pass them to the function for loading and processing. - -If ``mip`` is not given, ESMValCore will search for the fx variable in all -available tables of the specified project. - -.. warning:: - Some fx variables exist in more than one table (e.g., ``volcello`` exists in - CMIP6's ``Odec``, ``Ofx``, ``Omon``, and ``Oyr`` tables; ``sftgif`` exists - in CMIP6's ``fx``, ``IyrAnt`` and ``IyrGre``, and ``LImon`` tables). If (for - a given dataset) fx files are found in more than one table, ``mip`` needs to - be specified, otherwise an error is raised. - -.. note:: - To explicitly **not** use any fx variables in a preprocessor, use - ``fx_variables: null``. While some of the preprocessors mentioned above do - work without fx variables (e.g., ``area_statistics`` or ``mask_landsea`` - with datasets that have regular latitude/longitude grids), using this option - is **not** recommended. - -Internally, the required ``fx_variables`` are automatically loaded by the -preprocessor step ``add_fx_variables`` which also checks them against CMOR -standards and adds them either as ``cell_measure`` (see `CF conventions on cell -measures -`_ -and :class:`iris.coords.CellMeasure`) or ``ancillary_variable`` (see `CF -conventions on ancillary variables -`_ -and :class:`iris.coords.AncillaryVariable`) inside the cube data. This ensures -that the defined preprocessor chain is applied to both ``variables`` and -``fx_variables``. - -Note that when calling steps that require ``fx_variables`` inside diagnostic -scripts, the variables are expected to contain the required ``cell_measures`` or -``Fx variables as cell measures or ancillary variables``. If missing, they can be added using the following functions: - -.. code-block:: - - from esmvalcore.preprocessor import (add_cell_measure, add_ancillary_variable) - - cube_with_area_measure = add_cell_measure(cube, area_cube, 'area') - - cube_with_volume_measure = add_cell_measure(cube, volume_cube, 'volume) - - cube_with_ancillary_sftlf = add_ancillary_variable(cube, sftlf_cube) - - cube_with_ancillary_sftgif = add_ancillary_variable(cube, sftgif_cube) - - Details on the arguments needed for each step can be found in the following sections. - .. _Vertical interpolation: Vertical interpolation diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index ed858e0c39..a990f6bd13 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -9,7 +9,6 @@ from copy import deepcopy from itertools import groupby from pathlib import Path -from pprint import pformat from typing import Any, Dict, Iterable, Sequence import yaml @@ -17,13 +16,10 @@ from esmvalcore import __version__, esgf from esmvalcore._provenance import get_recipe_provenance from esmvalcore._task import DiagnosticTask, ResumeTask, TaskSet -from esmvalcore.cmor.table import CMOR_TABLES, _update_cmor_facets -from esmvalcore.config import CFG -from esmvalcore.config._config import TASKSEP, get_project_config +from esmvalcore.config._config import TASKSEP from esmvalcore.config._diagnostics import TAGS from esmvalcore.dataset import Dataset from esmvalcore.exceptions import ( - ESMValCoreDeprecationWarning, InputFilesNotFound, RecipeError, ) @@ -50,10 +46,6 @@ get_reference_levels, parse_cell_spec, ) -from esmvalcore.preprocessor._supplementary_vars import ( - PREPROCESSOR_SUPPLEMENTARIES, -) -from esmvalcore.typing import Facets from . import check from .from_datasets import datasets_to_recipe @@ -235,177 +227,6 @@ def _get_default_settings(dataset): return settings -def _guess_fx_mip(facets: dict, dataset: Dataset): - """Search mip for fx variable.""" - project = facets.get('project', dataset.facets['project']) - # check if project in config-developer - get_project_config(project) - - tables = CMOR_TABLES[project].tables - - # Get all mips that offer that specific fx variable - mips_with_fx_var = [] - for mip in tables: - if facets['short_name'] in tables[mip]: - mips_with_fx_var.append(mip) - - # List is empty -> no table includes the fx variable - if not mips_with_fx_var: - raise RecipeError( - f"Requested fx variable '{facets['short_name']}' not available " - f"in any CMOR table for '{project}'") - - # Iterate through all possible mips and check if files are available; in - # case of ambiguity raise an error - fx_files_for_mips = {} - for mip in mips_with_fx_var: - logger.debug("For fx variable '%s', found table '%s'", - facets['short_name'], mip) - fx_dataset = dataset.copy(**facets) - fx_dataset.supplementaries = [] - fx_dataset.set_facet('mip', mip) - fx_dataset.facets.pop('timerange', None) - fx_files = fx_dataset.files - if fx_files: - logger.debug("Found fx variables '%s':\n%s", facets['short_name'], - pformat(fx_files)) - fx_files_for_mips[mip] = fx_files - - # Dict contains more than one element -> ambiguity - if len(fx_files_for_mips) > 1: - raise RecipeError( - f"Requested fx variable '{facets['short_name']}' for dataset " - f"'{dataset.facets['dataset']}' of project '{project}' is " - f"available in more than one CMOR MIP table for " - f"'{project}': {sorted(fx_files_for_mips)}") - - # Dict is empty -> no files found -> handled at later stage - if not fx_files_for_mips: - return mips_with_fx_var[0] - - # Dict contains one element -> ok - mip = list(fx_files_for_mips)[0] - return mip - - -def _set_default_preproc_fx_variables( - dataset: Dataset, - settings: PreprocessorSettings, -) -> None: - """Update `fx_variables` key in preprocessor settings with defaults.""" - default_fx = { - 'area_statistics': { - 'areacella': None, - }, - 'mask_landsea': { - 'sftlf': None, - }, - 'mask_landseaice': { - 'sftgif': None, - }, - 'volume_statistics': { - 'volcello': None, - }, - 'weighting_landsea_fraction': { - 'sftlf': None, - }, - } - if dataset.facets['project'] != 'obs4MIPs': - default_fx['area_statistics']['areacello'] = None - default_fx['mask_landsea']['sftof'] = None - default_fx['weighting_landsea_fraction']['sftof'] = None - - for step, fx_variables in default_fx.items(): - if step in settings and 'fx_variables' not in settings[step]: - settings[step]['fx_variables'] = fx_variables - - -def _get_supplementaries_from_fx_variables( - settings: PreprocessorSettings -) -> list[Facets]: - """Read supplementary facets from `fx_variables` in preprocessor.""" - supplementaries = [] - for step, kwargs in settings.items(): - allowed = PREPROCESSOR_SUPPLEMENTARIES.get(step, - {}).get('variables', []) - if fx_variables := kwargs.get('fx_variables'): - - if isinstance(fx_variables, list): - result: dict[str, Facets] = {} - for fx_variable in fx_variables: - if isinstance(fx_variable, str): - # Legacy legacy method of specifying fx variable - short_name = fx_variable - result[short_name] = {} - elif isinstance(fx_variable, dict): - short_name = fx_variable['short_name'] - result[short_name] = fx_variable - fx_variables = result - - for short_name, facets in fx_variables.items(): - if short_name not in allowed: - raise RecipeError( - f"Preprocessor function '{step}' does not support " - f"supplementary variable '{short_name}'") - if facets is None: - facets = {} - facets['short_name'] = short_name - supplementaries.append(facets) - - return supplementaries - - -def _get_legacy_supplementary_facets( - dataset: Dataset, - settings: PreprocessorSettings, -) -> list[Facets]: - """Load the supplementary dataset facets from the preprocessor settings.""" - # First update `fx_variables` in preprocessor settings with defaults - _set_default_preproc_fx_variables(dataset, settings) - - supplementaries = _get_supplementaries_from_fx_variables(settings) - - # Guess the ensemble and mip if they is not specified - for facets in supplementaries: - if 'ensemble' not in facets and dataset.facets['project'] == 'CMIP5': - facets['ensemble'] = 'r0i0p0' - if 'mip' not in facets: - facets['mip'] = _guess_fx_mip(facets, dataset) - return supplementaries - - -def _add_legacy_supplementary_datasets(dataset: Dataset, settings): - """Update fx settings depending on the needed method.""" - if not dataset.session['use_legacy_supplementaries']: - return - if dataset.supplementaries: - # Supplementaries have been defined in the recipe. - # Just remove any skipped supplementaries (they have been kept so we - # know that supplementaries have been defined in the recipe). - dataset.supplementaries = [ - ds for ds in dataset.supplementaries - if not ds.facets.get('skip', False) - ] - return - - logger.debug("Using legacy method to add supplementaries to %s", dataset) - - legacy_ds = dataset.copy() - for facets in _get_legacy_supplementary_facets(dataset, settings): - legacy_ds.add_supplementary(**facets) - - for supplementary_ds in legacy_ds.supplementaries: - _update_cmor_facets(supplementary_ds.facets, override=True) - if supplementary_ds.files: - dataset.supplementaries.append(supplementary_ds) - - dataset._fix_fx_exp() - - # Remove preprocessor keyword argument `fx_variables` - for kwargs in settings.values(): - kwargs.pop('fx_variables', None) - - def _exclude_dataset(settings, facets, step): """Exclude dataset from specific preprocessor step if requested.""" exclude = { @@ -687,7 +508,6 @@ def _get_preprocessor_products( _apply_preprocessor_profile(settings, profile) _update_multi_dataset_settings(dataset.facets, settings) _update_preproc_functions(settings, dataset, datasets, missing_vars) - _add_legacy_supplementary_datasets(dataset, settings) check.preprocessor_supplementaries(dataset, settings) input_datasets = _get_input_datasets(dataset) missing = _check_input_files(input_datasets) @@ -893,7 +713,6 @@ def __init__(self, raw_recipe, session, recipe_file: Path): self._preprocessors = raw_recipe.get('preprocessors', {}) if 'default' not in self._preprocessors: self._preprocessors['default'] = {} - self._set_use_legacy_supplementaries() self.datasets = Dataset.from_recipe(recipe_file, session) self.diagnostics = self._initialize_diagnostics( raw_recipe['diagnostics']) @@ -905,42 +724,6 @@ def __init__(self, raw_recipe, session, recipe_file: Path): self._log_recipe_errors(exc) raise - def _set_use_legacy_supplementaries(self): - """Automatically determine if legacy supplementaries are used.""" - names = set() - steps = set() - for name, profile in self._preprocessors.items(): - for step, kwargs in profile.items(): - if isinstance(kwargs, dict) and 'fx_variables' in kwargs: - names.add(name) - steps.add(step) - if self.session['use_legacy_supplementaries'] is False: - kwargs.pop('fx_variables') - if names: - warnings.warn( - ESMValCoreDeprecationWarning( - "Encountered 'fx_variables' argument in preprocessor(s) " - f"{sorted(names)}, function(s) {sorted(steps)}. The " - "'fx_variables' argument is deprecated and will stop " - "working in v2.10. Please remove it and if automatic " - "definition of supplementary variables does not work " - "correctly, specify the supplementary variables in the " - "recipe as described in https://docs.esmvaltool.org/" - "projects/esmvalcore/en/latest/recipe/preprocessor.html" - "#ancillary-variables-and-cell-measures")) - if self.session['use_legacy_supplementaries'] is None: - logger.info("Running with --use-legacy-supplementaries=True") - self.session['use_legacy_supplementaries'] = True - - # Also adapt the global config if necessary because it is used to check - # if mismatching shapes should be ignored when attaching - # supplementary variables in `esmvalcore.preprocessor. - # _supplementary_vars.add_supplementary_variables` to avoid having to - # introduce a new function argument that is immediately deprecated. - session_use_legacy_supp = self.session['use_legacy_supplementaries'] - if session_use_legacy_supp is not None: - CFG['use_legacy_supplementaries'] = session_use_legacy_supp - def _log_recipe_errors(self, exc): """Log a message with recipe errors.""" logger.error(exc.message) diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py index 06423cbed6..56d9d44221 100644 --- a/esmvalcore/_recipe/to_datasets.py +++ b/esmvalcore/_recipe/to_datasets.py @@ -288,14 +288,13 @@ def _get_dataset_facets_from_recipe( ), ) - if not session['use_legacy_supplementaries']: - preprocessor = facets.get('preprocessor', 'default') - settings = profiles.get(preprocessor, {}) - _append_missing_supplementaries(supplementaries, facets, settings) - supplementaries = [ - facets for facets in supplementaries - if not facets.pop('skip', False) - ] + preprocessor = facets.get('preprocessor', 'default') + settings = profiles.get(preprocessor, {}) + _append_missing_supplementaries(supplementaries, facets, settings) + supplementaries = [ + facets for facets in supplementaries + if not facets.pop('skip', False) + ] return facets, supplementaries diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 736a6ba689..8f5e47375a 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -297,7 +297,6 @@ def validate_diagnostics( 'run_diagnostic': validate_bool, 'save_intermediary_cubes': validate_bool, 'search_esgf': validate_search_esgf, - 'use_legacy_supplementaries': validate_bool_or_none, # From CLI 'check_level': validate_check_level, @@ -372,38 +371,12 @@ def deprecate_offline( validated_config['search_esgf'] = 'when_missing' -def deprecate_use_legacy_supplementaries( - validated_config: ValidatedConfig, - value: Any, - validated_value: Any, -) -> None: - """Deprecate ``use_legacy_supplementaries`` option. - - Parameters - ---------- - validated_config: ValidatedConfig - ``ValidatedConfig`` instance which will be modified in place. - value: Any - Raw input value for ``use_legacy_supplementaries`` option. - validated_value: Any - Validated value for ``use_legacy_supplementaries`` option. - - """ - option = 'use_legacy_supplementaries' - deprecated_version = '2.8.0' - remove_version = '2.10.0' - more_info = '' - _handle_deprecation(option, deprecated_version, remove_version, more_info) - - _deprecators: dict[str, Callable] = { 'offline': deprecate_offline, - 'use_legacy_supplementaries': deprecate_use_legacy_supplementaries, } # Default values for deprecated options _deprecated_options_defaults: dict[str, Any] = { 'offline': True, - 'use_legacy_supplementaries': None, } diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 736ed02156..3fb882b69c 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -57,9 +57,7 @@ ) from ._rolling_window import rolling_window_statistics from ._supplementary_vars import ( - add_fx_variables, add_supplementary_variables, - remove_fx_variables, remove_supplementary_variables, ) from ._time import ( @@ -110,7 +108,6 @@ 'fix_data', 'cmor_check_data', # Attach ancillary variables and cell measures - 'add_fx_variables', 'add_supplementary_variables', # Derive variable 'derive', @@ -189,7 +186,6 @@ 'bias', # Remove supplementary variables from cube 'remove_supplementary_variables', - 'remove_fx_variables', # Save to file 'save', 'cleanup', diff --git a/esmvalcore/preprocessor/_supplementary_vars.py b/esmvalcore/preprocessor/_supplementary_vars.py index 8813735098..d5cb0e2d31 100644 --- a/esmvalcore/preprocessor/_supplementary_vars.py +++ b/esmvalcore/preprocessor/_supplementary_vars.py @@ -1,22 +1,11 @@ """Preprocessor functions for ancillary variables and cell measures.""" import logging -import warnings -from pathlib import Path from typing import Iterable -import dask.array as da import iris.coords import iris.cube -from esmvalcore.cmor.check import cmor_check_data, cmor_check_metadata -from esmvalcore.cmor.fix import fix_data, fix_metadata -from esmvalcore.config import CFG -from esmvalcore.config._config import get_ignored_warnings -from esmvalcore.exceptions import ESMValCoreDeprecationWarning -from esmvalcore.preprocessor._io import concatenate, load -from esmvalcore.preprocessor._time import clip_timerange - logger = logging.getLogger(__name__) PREPROCESSOR_SUPPLEMENTARIES = {} @@ -50,62 +39,6 @@ def wrapper(func): return wrapper -def _load_fx(var_cube, fx_info, check_level): - """Load and CMOR-check fx variables.""" - fx_cubes = iris.cube.CubeList() - - project = fx_info['project'] - mip = fx_info['mip'] - short_name = fx_info['short_name'] - freq = fx_info['frequency'] - - for fx_file in fx_info['filename']: - ignored_warnings = get_ignored_warnings(project, 'load') - loaded_cube = load(fx_file, ignore_warnings=ignored_warnings) - loaded_cube = fix_metadata(loaded_cube, - check_level=check_level, - **fx_info) - fx_cubes.append(loaded_cube[0]) - - fx_cube = concatenate(fx_cubes) - - if freq != 'fx': - fx_cube = clip_timerange(fx_cube, fx_info['timerange']) - - if not _is_fx_broadcastable(fx_cube, var_cube): - return None - - fx_cube = cmor_check_metadata(fx_cube, - cmor_table=project, - mip=mip, - short_name=short_name, - frequency=freq, - check_level=check_level) - - fx_cube = fix_data(fx_cube, check_level=check_level, **fx_info) - - fx_cube = cmor_check_data(fx_cube, - cmor_table=project, - mip=mip, - short_name=fx_cube.var_name, - frequency=freq, - check_level=check_level) - - return fx_cube - - -def _is_fx_broadcastable(fx_cube, cube): - try: - da.broadcast_to(fx_cube.core_data(), cube.shape) - except ValueError as exc: - logger.debug( - "Dimensions of %s and %s cubes do not match. " - "Discarding use of fx_variable: %s", cube.var_name, - fx_cube.var_name, exc) - return False - return True - - def add_cell_measure(cube, cell_measure_cube, measure): """Add a cube as a cell_measure in the cube containing the data. @@ -172,56 +105,6 @@ def add_ancillary_variable(cube, ancillary_cube): ancillary_cube.var_name, cube.var_name) -def add_fx_variables(cube, fx_variables, check_level): - """Load requested fx files, check with CMOR standards and add the fx - variables as cell measures or ancillary variables in the cube containing - the data. - - .. deprecated:: 2.8.0 - This function is deprecated and will be removed in version 2.10.0. - Please use a :class:`esmvalcore.dataset.Dataset` or - :func:`esmvalcore.preprocessor.add_supplementary_variables` - instead. - - Parameters - ---------- - cube: iris.cube.Cube - Iris cube with input data. - fx_variables: dict - Dictionary with fx_variable information. - check_level: CheckLevels - Level of strictness of the checks. - - Returns - ------- - iris.cube.Cube - Cube with added cell measures or ancillary variables. - """ - msg = ( - "The function `add_fx_variables` has been deprecated in " - "ESMValCore version 2.8.0 and is scheduled for removal in " - "version 2.10.0. Use a `esmvalcore.dataset.Dataset` or the function " - "`add_supplementary_variables` instead.") - warnings.warn(msg, ESMValCoreDeprecationWarning) - if not fx_variables: - return cube - fx_cubes = [] - for fx_info in fx_variables.values(): - if not fx_info: - continue - if isinstance(fx_info['filename'], (str, Path)): - fx_info['filename'] = [fx_info['filename']] - fx_cube = _load_fx(cube, fx_info, check_level) - - if fx_cube is None: - continue - - fx_cubes.append(fx_cube) - - add_supplementary_variables(cube, fx_cubes) - return cube - - def add_supplementary_variables( cube: iris.cube.Cube, supplementary_cubes: Iterable[iris.cube.Cube], @@ -246,9 +129,6 @@ def add_supplementary_variables( 'volcello': 'volume' } for supplementary_cube in supplementary_cubes: - if (CFG['use_legacy_supplementaries'] - and not _is_fx_broadcastable(supplementary_cube, cube)): - continue if supplementary_cube.var_name in measure_names: measure_name = measure_names[supplementary_cube.var_name] add_cell_measure(cube, supplementary_cube, measure_name) @@ -280,31 +160,3 @@ def remove_supplementary_variables(cube: iris.cube.Cube): for variable in cube.ancillary_variables(): cube.remove_ancillary_variable(variable) return cube - - -def remove_fx_variables(cube): - """Remove fx variables present as cell measures or ancillary variables in - the cube containing the data. - - .. deprecated:: 2.8.0 - This function is deprecated and will be removed in version 2.10.0. - Please use - :func:`esmvalcore.preprocessor.remove_supplementary_variables` - instead. - - Parameters - ---------- - cube: iris.cube.Cube - Iris cube with data and cell measures or ancillary variables. - - Returns - ------- - iris.cube.Cube - Cube without cell measures or ancillary variables. - """ - msg = ("The function `remove_fx_variables` has been deprecated in " - "ESMValCore version 2.8.0 and is scheduled for removal in " - "version 2.10.0. Use the function `remove_supplementary_variables` " - "instead.") - warnings.warn(msg, ESMValCoreDeprecationWarning) - return remove_supplementary_variables(cube) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1dbb948940..612e72615e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,7 +23,6 @@ def session(tmp_path, monkeypatch): monkeypatch.setitem(_config.CFG[project]['input_dir'], 'default', '/') # The patched datafinder fixture does not return any facets, so automatic # supplementary definition does not work with it. - session['use_legacy_supplementaries'] = True return session diff --git a/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py b/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py deleted file mode 100644 index 415a5849c4..0000000000 --- a/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Test add_fx_variables. - -Integration tests for the -:func:`esmvalcore.preprocessor._supplementary_vars` module. -""" -import logging - -import iris -import numpy as np -import pytest - -from esmvalcore.cmor.check import CheckLevels -from esmvalcore.preprocessor._supplementary_vars import ( - _is_fx_broadcastable, - add_ancillary_variable, - add_cell_measure, - add_fx_variables, - remove_fx_variables, -) -from esmvalcore.preprocessor._time import clip_timerange - -logger = logging.getLogger(__name__) - -SHAPES_TO_BROADCAST = [ - ((), (1, ), True), - ((), (10, 10), True), - ((1, ), (10, ), True), - ((1, ), (10, 10), True), - ((2, ), (10, ), False), - ((10, ), (), False), - ((10, ), (1, ), False), - ((10, ), (10, ), True), - ((10, ), (10, 10), True), - ((10, ), (7, 1), False), - ((10, ), (10, 7), False), - ((10, ), (7, 1, 10), True), - ((10, ), (7, 1, 1), False), - ((10, ), (7, 1, 7), False), - ((10, ), (7, 10, 7), False), - ((10, 1), (1, 1), False), - ((10, 1), (1, 100), False), - ((10, 1), (10, 7), True), - ((10, 12), (10, 1), False), - ((10, 1), (10, 12), True), - ((10, 12), (), False), - ((), (10, 12), True), - ((10, 12), (1, ), False), - ((1, ), (10, 12), True), - ((10, 12), (12, ), False), - ((10, 12), (1, 1), False), - ((1, 1), (10, 12), True), - ((10, 12), (1, 12), False), - ((1, 12), (10, 12), True), - ((10, 12), (10, 10, 1), False), - ((10, 12), (10, 12, 1), False), - ((10, 12), (10, 12, 12), False), - ((10, 12), (10, 10, 12), True)] - - -@pytest.mark.parametrize('shape_1,shape_2,out', SHAPES_TO_BROADCAST) -def test_shape_is_broadcastable(shape_1, shape_2, out): - """Test check if two shapes are broadcastable.""" - fx_cube = iris.cube.Cube(np.ones(shape_1)) - cube = iris.cube.Cube(np.ones(shape_2)) - is_broadcastable = _is_fx_broadcastable(fx_cube, cube) - assert is_broadcastable == out - - -class Test: - """Test class.""" - @pytest.fixture(autouse=True) - def setUp(self): - """Assemble a stock cube.""" - fx_area_data = np.ones((3, 3)) - fx_volume_data = np.ones((3, 3, 3)) - self.new_cube_data = np.empty((3, 3)) - self.new_cube_data[:] = 200. - self.new_cube_3D_data = np.empty((3, 3, 3)) - self.new_cube_3D_data[:] = 200. - crd_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) - self.lons = iris.coords.DimCoord([0, 1.5, 3], - standard_name='longitude', - bounds=[[0, 1], [1, 2], [2, 3]], - units='degrees_east', - coord_system=crd_sys) - self.lats = iris.coords.DimCoord([0, 1.5, 3], - standard_name='latitude', - bounds=[[0, 1], [1, 2], [2, 3]], - units='degrees_north', - coord_system=crd_sys) - self.depth = iris.coords.DimCoord([0, 1.5, 3], - standard_name='depth', - bounds=[[0, 1], [1, 2], [2, 3]], - units='m', - long_name='ocean depth coordinate') - self.monthly_times = iris.coords.DimCoord( - [15.5, 45, 74.5, 105, 135.5, 166, - 196.5, 227.5, 258, 288.5, 319, 349.5], - standard_name='time', - var_name='time', - bounds=[[0, 31], [31, 59], [59, 90], - [90, 120], [120, 151], [151, 181], - [181, 212], [212, 243], [243, 273], - [273, 304], [304, 334], [334, 365]], - units='days since 1950-01-01 00:00:00') - self.yearly_times = iris.coords.DimCoord( - [182.5, 547.5], - standard_name='time', - bounds=[[0, 365], [365, 730]], - units='days since 1950-01-01 00:00') - self.coords_spec = [(self.lats, 0), (self.lons, 1)] - self.fx_area = iris.cube.Cube(fx_area_data, - dim_coords_and_dims=self.coords_spec) - self.fx_volume = iris.cube.Cube(fx_volume_data, - dim_coords_and_dims=[ - (self.depth, 0), - (self.lats, 1), - (self.lons, 2) - ]) - self.monthly_volume = iris.cube.Cube(np.ones((12, 3, 3, 3)), - dim_coords_and_dims=[ - (self.monthly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3) - ]) - - def test_add_cell_measure_area(self, tmp_path): - """Test add area fx variables as cell measures.""" - fx_vars = { - 'areacella': { - 'short_name': 'areacella', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx'}, - 'areacello': { - 'short_name': 'areacello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Ofx', - 'frequency': 'fx' - } - } - for fx_var in fx_vars: - self.fx_area.var_name = fx_var - self.fx_area.standard_name = 'cell_area' - self.fx_area.units = 'm2' - fx_file = str(tmp_path / f'{fx_var}.nc') - fx_vars[fx_var].update({'filename': fx_file}) - iris.save(self.fx_area, fx_file) - cube = iris.cube.Cube(self.new_cube_data, - dim_coords_and_dims=self.coords_spec) - cube = add_fx_variables( - cube, {fx_var: fx_vars[fx_var]}, CheckLevels.IGNORE) - assert cube.cell_measure(self.fx_area.standard_name) is not None - - def test_add_cell_measure_volume(self, tmp_path): - """Test add volume as cell measure.""" - fx_vars = { - 'volcello': { - 'short_name': 'volcello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Ofx', - 'frequency': 'fx'} - } - self.fx_volume.var_name = 'volcello' - self.fx_volume.standard_name = 'ocean_volume' - self.fx_volume.units = 'm3' - fx_file = str(tmp_path / 'volcello.nc') - iris.save(self.fx_volume, fx_file) - fx_vars['volcello'].update({'filename': fx_file}) - cube = iris.cube.Cube(self.new_cube_3D_data, - dim_coords_and_dims=[ - (self.depth, 0), - (self.lats, 1), - (self.lons, 2)]) - cube = add_fx_variables(cube, fx_vars, CheckLevels.IGNORE) - assert cube.cell_measure(self.fx_volume.standard_name) is not None - - def test_clip_volume_timerange(self, tmp_path): - """Test timerange is clipped in time dependent measures.""" - cell_measures = { - 'volcello': { - 'short_name': 'volcello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Omon', - 'frequency': 'mon', - 'timerange': '195001/195003'} - } - self.monthly_volume.var_name = 'volcello' - self.monthly_volume.standard_name = 'ocean_volume' - self.monthly_volume.units = 'm3' - cell_measure_file = str(tmp_path / 'volcello.nc') - iris.save(self.monthly_volume, cell_measure_file) - cell_measures['volcello'].update( - {'filename': cell_measure_file}) - cube = iris.cube.Cube(np.ones((12, 3, 3, 3)), - dim_coords_and_dims=[ - (self.monthly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3)]) - cube = clip_timerange(cube, '195001/195003') - cube = add_fx_variables(cube, cell_measures, CheckLevels.IGNORE) - cell_measure = cube.cell_measure(self.fx_volume.standard_name) - assert cell_measure is not None - assert cell_measure.shape == (3, 3, 3, 3) - - def test_no_cell_measure(self): - """Test no cell measure is added.""" - cube = iris.cube.Cube(self.new_cube_3D_data, - dim_coords_and_dims=[ - (self.depth, 0), - (self.lats, 1), - (self.lons, 2)]) - cube = add_fx_variables(cube, {'areacello': None}, CheckLevels.IGNORE) - assert cube.cell_measures() == [] - - def test_add_ancillary_variables(self, tmp_path): - """Test invalid variable is not added as cell measure.""" - self.fx_area.var_name = 'sftlf' - self.fx_area.standard_name = "land_area_fraction" - self.fx_area.units = '%' - fx_file = str(tmp_path / f'{self.fx_area.var_name}.nc') - iris.save(self.fx_area, fx_file) - fx_vars = { - 'sftlf': { - 'short_name': 'sftlf', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx', - 'filename': fx_file} - } - cube = iris.cube.Cube(self.new_cube_data, - dim_coords_and_dims=self.coords_spec) - cube = add_fx_variables(cube, fx_vars, CheckLevels.IGNORE) - assert cube.ancillary_variable(self.fx_area.standard_name) is not None - - def test_wrong_shape(self, tmp_path): - """Test fx_variable is not added if it's not broadcastable to cube.""" - volume_data = np.ones((2, 3, 3, 3)) - volume_cube = iris.cube.Cube( - volume_data, - dim_coords_and_dims=[(self.yearly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3)]) - volume_cube.standard_name = 'ocean_volume' - volume_cube.var_name = 'volcello' - volume_cube.units = 'm3' - fx_file = str(tmp_path / f'{volume_cube.var_name}.nc') - iris.save(volume_cube, fx_file) - fx_vars = { - 'volcello': { - 'short_name': 'volcello', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'Oyr', - 'frequency': 'yr', - 'filename': fx_file, - 'timerange': '1950/1951'} - } - data = np.ones((12, 3, 3, 3)) - cube = iris.cube.Cube( - data, - dim_coords_and_dims=[(self.monthly_times, 0), - (self.depth, 1), - (self.lats, 2), - (self.lons, 3)]) - cube.var_name = 'thetao' - cube = add_fx_variables(cube, fx_vars, CheckLevels.IGNORE) - assert cube.cell_measures() == [] - - def test_remove_fx_vars(self): - """Test fx_variables are removed from cube.""" - cube = iris.cube.Cube(self.new_cube_3D_data, - dim_coords_and_dims=[(self.depth, 0), - (self.lats, 1), - (self.lons, 2)]) - self.fx_area.var_name = 'areacella' - self.fx_area.standard_name = 'cell_area' - self.fx_area.units = 'm2' - add_cell_measure(cube, self.fx_area, measure='area') - assert cube.cell_measure(self.fx_area.standard_name) is not None - self.fx_area.var_name = 'sftlf' - self.fx_area.standard_name = "land_area_fraction" - self.fx_area.units = '%' - add_ancillary_variable(cube, self.fx_area) - assert cube.ancillary_variable(self.fx_area.standard_name) is not None - cube = remove_fx_variables(cube) - assert cube.cell_measures() == [] - assert cube.ancillary_variables() == [] diff --git a/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py b/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py index 9a3fe8169c..d4a9b0b217 100644 --- a/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py +++ b/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py @@ -8,7 +8,6 @@ import numpy as np import pytest -import esmvalcore.config from esmvalcore.preprocessor._supplementary_vars import ( add_ancillary_variable, add_cell_measure, @@ -120,14 +119,8 @@ def test_add_supplementary_vars(self): cube = add_supplementary_variables(cube, [self.fx_area]) assert cube.ancillary_variable(self.fx_area.standard_name) is not None - @pytest.mark.parametrize('use_legacy_supplementaries', [True, False]) - def test_wrong_shape(self, use_legacy_supplementaries, monkeypatch): + def test_wrong_shape(self, monkeypatch): """Test variable is not added if it's not broadcastable to cube.""" - monkeypatch.setitem( - esmvalcore.config.CFG, - 'use_legacy_supplementaries', - use_legacy_supplementaries, - ) volume_data = np.ones((2, 3, 3, 3)) volume_cube = iris.cube.Cube( volume_data, @@ -146,12 +139,8 @@ def test_wrong_shape(self, use_legacy_supplementaries, monkeypatch): (self.lats, 2), (self.lons, 3)]) cube.var_name = 'thetao' - if use_legacy_supplementaries: + with pytest.raises(iris.exceptions.CannotAddError): add_supplementary_variables(cube, [volume_cube]) - assert cube.cell_measures() == [] - else: - with pytest.raises(iris.exceptions.CannotAddError): - add_supplementary_variables(cube, [volume_cube]) def test_remove_supplementary_vars(self): """Test supplementary variables are removed from cube.""" diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index ba53388ddf..9655286872 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -1801,6 +1801,7 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, session): mm_products) == len(mm_statistics) * len(ens_statistics) * len(groupby) +@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -1886,6 +1887,7 @@ def test_weighting_landsea_fraction_no_fx(tmp_path, patched_failing_datafinder, get_recipe(tmp_path, content, session) +@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, session): content = dedent(""" @@ -1969,6 +1971,7 @@ def test_weighting_landsea_fraction_exclude_fail(tmp_path, patched_datafinder, "diagnostic 'diagnostic_name'.") +@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_area_statistics(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: @@ -2019,6 +2022,7 @@ def test_area_statistics(tmp_path, patched_datafinder, session): assert short_names == {'areacella', 'areacello'} +@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_landmask(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: @@ -2065,219 +2069,6 @@ def test_landmask(tmp_path, patched_datafinder, session): assert len(dataset.supplementaries) == 2 -def test_empty_fxvar_none(tmp_path, patched_datafinder, session): - """Test that no fx variables are added if explicitly specified.""" - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: null - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check that no custom fx variables are present - task = recipe.tasks.pop() - product = task.products.pop() - dataset = product.datasets[0] - assert dataset.supplementaries == [] - - -def test_empty_fxvar_list(tmp_path, patched_datafinder, session): - """Test that no fx variables are added if explicitly specified.""" - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: [] - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check that no custom fx variables are present - task = recipe.tasks.pop() - product = task.products.pop() - dataset = product.datasets[0] - assert dataset.supplementaries == [] - - -def test_empty_fxvar_dict(tmp_path, patched_datafinder, session): - """Test that no fx variables are added if explicitly specified.""" - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: {} - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check that no custom fx variables are present - task = recipe.tasks.pop() - product = task.products.pop() - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert dataset.supplementaries == [] - - -@pytest.mark.parametrize('content', [ - pytest.param(dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: - sftlf: - exp: piControl - mask_landseaice: - mask_out: sea - fx_variables: - sftgif: - exp: piControl - volume_statistics: - operator: mean - area_statistics: - operator: mean - fx_variables: - areacello: - mip: fx - exp: piControl - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """), - id='fx_variables_as_dict_of_dicts'), - pytest.param(dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: [{'short_name': 'sftlf', 'exp': 'piControl'}] - mask_landseaice: - mask_out: sea - fx_variables: [{'short_name': 'sftgif', 'exp': 'piControl'}] - volume_statistics: - operator: mean - area_statistics: - operator: mean - fx_variables: [{'short_name': 'areacello', 'mip': 'fx', - 'exp': 'piControl'}] - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """), - id='fx_variables_as_list_of_dicts'), -]) -def test_user_defined_fxvar(tmp_path, patched_datafinder, session, content): - recipe = get_recipe(tmp_path, content, session) - - # Check custom fx variables - task = recipe.tasks.pop() - product = task.products.pop() - - # landsea - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert isinstance(dataset.supplementaries, list) - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert len(list(supplementaries)) == 4 - sftlf_ds = supplementaries['sftlf'] - assert sftlf_ds.facets['mip'] == 'fx' - assert sftlf_ds.facets['exp'] == 'piControl' - - # landseaice - settings = product.settings['mask_landseaice'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - sftgif_ds = supplementaries['sftgif'] - assert sftgif_ds.facets['mip'] == 'fx' - assert sftgif_ds.facets['exp'] == 'piControl' - - # volume statistics - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert 'volcello' in supplementaries - - # area statistics - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - areacello_ds = supplementaries['areacello'] - assert areacello_ds.facets['mip'] == 'fx' - assert areacello_ds.facets['exp'] == 'piControl' - - def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): content = dedent(""" preprocessors: @@ -2323,733 +2114,6 @@ def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): assert dataset.supplementaries == [] -def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, session): - """Test fx variables with given mips.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - ensemble: r2i1p1f1 - mip: Ofx - mask_landseaice: - mask_out: ice - fx_variables: - sftgif: - mip: fx - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - - # Check legacy method of adding supplementary variables - assert len(product.datasets) == 1 - dataset = product.datasets[0] - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert len(list(supplementaries)) == 2 - sftgif_ds = supplementaries['sftgif'] - assert sftgif_ds.facets['mip'] == 'fx' - volcello_ds = supplementaries['volcello'] - assert volcello_ds.facets['ensemble'] == 'r2i1p1f1' - assert volcello_ds.facets['mip'] == 'Ofx' - - -def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, session): - """Test fx variables with invalid mip.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - mip: INVALID - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Unable to load CMOR table (project) 'CMIP6' for variable " - "'areacella' with mip 'INVALID'") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_fx_vars_invalid_mip_for_var_cmip6(tmp_path, patched_datafinder, - session): - """Test fx variables with invalid mip for variable.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - mip: Lmon - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Unable to load CMOR table (project) 'CMIP6' for variable " - "'areacella' with mip 'Lmon'") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, session): - """Test mip tables search for different fx variables.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - areacello: - mask_landsea: - mask_out: sea - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - - # Check mask_landsea - assert 'mask_landsea' in product.settings - settings = product.settings['mask_landsea'] - assert len(settings) == 1 - assert settings['mask_out'] == 'sea' - - # Check legacy method of adding supplementary variables - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 4 - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert supplementaries['areacella'].facets['mip'] == 'fx' - assert supplementaries['areacello'].facets['mip'] == 'Ofx' - assert supplementaries['sftlf'].facets['mip'] == 'fx' - assert supplementaries['sftof'].facets['mip'] == 'Ofx' - - -def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, session): - """Test mip tables search for list of different fx variables.""" - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: [ - 'areacella', - 'areacello', - ] - mask_landsea: - mask_out: sea - fx_variables: [ - 'sftlf', - 'sftof', - ] - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check area_statistics - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - - # Check legacy method of adding supplementary variables - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 4 - supplementaries = { - ds.facets['short_name']: ds - for ds in dataset.supplementaries - } - assert supplementaries['areacella'].facets['mip'] == 'fx' - assert supplementaries['areacello'].facets['mip'] == 'Ofx' - assert supplementaries['sftlf'].facets['mip'] == 'fx' - assert supplementaries['sftof'].facets['mip'] == 'Ofx' - - -def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, session): - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Ofx - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['mip'] == 'Ofx' - - -def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Oyr - exp: piControl - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['short_name'] == 'volcello' - assert volcello_ds.facets['mip'] == 'Oyr' - assert volcello_ds.facets['exp'] == 'piControl' - - -def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - regrid: - target_grid: 1x1 - scheme: linear - extract_volume: - z_min: 0 - z_max: 100 - annual_statistics: - operator: mean - convert_units: - units: K - area_statistics: - operator: mean - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.ancestors) == 0 - assert len(task.products) == 1 - product = task.products.pop() - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert product.attributes['short_name'] == 'tos' - assert dataset.files - assert 'area_statistics' in product.settings - settings = product.settings['area_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(dataset.supplementaries) == 2 - - -def test_fx_vars_volcello_in_omon_cmip6(tmp_path, patched_failing_datafinder, - session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Omon - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP6 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['mip'] == 'Omon' - - -def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, - session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - mip: Oyr - - diagnostics: - diagnostic_name: - variables: - o2: - preprocessor: preproc - project: CMIP6 - mip: Oyr - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'o2' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['short_name'] == 'volcello' - assert volcello_ds.facets['mip'] == 'Oyr' - - -def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP5 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tos' - assert len(task.products) == 1 - product = task.products.pop() - - # Check volume_statistics - assert 'volume_statistics' in product.settings - settings = product.settings['volume_statistics'] - assert len(settings) == 1 - assert settings['operator'] == 'mean' - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - volcello_ds = dataset.supplementaries[0] - assert volcello_ds.facets['short_name'] == 'volcello' - assert volcello_ds.facets['mip'] == 'fx' - - -def test_wrong_project(tmp_path, patched_datafinder, session): - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - - diagnostics: - diagnostic_name: - variables: - tos: - preprocessor: preproc - project: CMIP7 - mip: Omon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " - "with mip 'Omon'") - with pytest.raises(RecipeError) as wrong_proj: - get_recipe(tmp_path, content, session) - assert str(wrong_proj.value) == INITIALIZATION_ERROR_MSG - assert str(wrong_proj.value.failed_tasks[0].message) == msg - - -def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, session): - """Test that error is raised for invalid fx variable.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - area_statistics: - operator: mean - fx_variables: - areacella: - wrong_fx_variable: - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Preprocessor function 'area_statistics' does not support " - "supplementary variable 'wrong_fx_variable'") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_ambiguous_fx_var_cmip6(tmp_path, patched_datafinder, session): - """Test that error is raised for fx files available in multiple mips.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - volume_statistics: - operator: mean - fx_variables: - volcello: - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - msg = ("Requested fx variable 'volcello' for dataset 'CanESM5' of project " - "'CMIP6' is available in more than one CMOR MIP table for 'CMIP6': " - "['Odec', 'Ofx', 'Omon', 'Oyr']") - with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, session) - assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG - assert msg in rec_err_exp.value.failed_tasks[0].message - - -def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, - patched_failing_datafinder, - session): - """Test that no error is raised for fx files available in one mip.""" - TAGS.set_tag_values(TAGS_FOR_TESTING) - - content = dedent(""" - preprocessors: - preproc: - mask_landseaice: - mask_out: ice - fx_variables: - sftgif: - - diagnostics: - diagnostic_name: - variables: - tas: - preprocessor: preproc - project: CMIP6 - mip: Amon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1f1 - grid: gn - additional_datasets: - - {dataset: CanESM5} - scripts: null - """) - recipe = get_recipe(tmp_path, content, session) - - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'tas' - assert len(task.products) == 1 - product = task.products.pop() - - # Check mask_landseaice - assert 'mask_landseaice' in product.settings - settings = product.settings['mask_landseaice'] - assert len(settings) == 1 - assert settings['mask_out'] == 'ice' - - # Check legacy method of adding supplementary variables - # Due to failing datafinder, only files in LImon are found even though - # sftgif is available in the tables fx, IyrAnt, IyrGre and LImon - assert len(product.datasets) == 1 - dataset = product.datasets[0] - assert len(dataset.supplementaries) == 1 - sftgif_ds = dataset.supplementaries[0] - assert sftgif_ds.facets['short_name'] == 'sftgif' - assert sftgif_ds.facets['mip'] == 'LImon' - assert len(sftgif_ds.files) == 1 - - def test_multimodel_mask(tmp_path, patched_datafinder, session): """Test ``mask_multimodel``.""" content = dedent(""" diff --git a/tests/integration/test_deprecated_config.py b/tests/integration/test_deprecated_config.py index 8bcf246190..71905b53d2 100644 --- a/tests/integration/test_deprecated_config.py +++ b/tests/integration/test_deprecated_config.py @@ -98,65 +98,3 @@ def test_offline_false_deprecation_config(monkeypatch): monkeypatch.setitem(CFG, 'offline', False) assert CFG['offline'] is False assert CFG['search_esgf'] == 'when_missing' - - -def test_use_legacy_supplementaries_default_cfg(): - """Test that option is added for backwards-compatibility.""" - assert CFG['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_user_cfg(): - """Test that option is added for backwards-compatibility.""" - config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - assert cfg['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_default_session(): - """Test that option is added for backwards-compatibility.""" - session = CFG.start_session('my_session') - assert session['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_user_session(): - """Test that option is added for backwards-compatibility.""" - config_file = Path(esmvalcore.__file__).parent / 'config-user.yml' - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - session = cfg.start_session('my_session') - assert session['use_legacy_supplementaries'] is None - - -def test_use_legacy_supplementaries_deprecation_session_setitem(): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - session = CFG.start_session('my_session') - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - session['use_legacy_supplementaries'] = True - assert session['use_legacy_supplementaries'] is True - - -def test_use_legacy_supplementaries_deprecation_session_update(): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - session = CFG.start_session('my_session') - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - session.update({'use_legacy_supplementaries': False}) - assert session['use_legacy_supplementaries'] is False - - -def test_use_legacy_supplementaries_true_deprecation_config(monkeypatch): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - monkeypatch.setitem(CFG, 'use_legacy_supplementaries', True) - assert CFG['use_legacy_supplementaries'] is True - - -def test_use_legacy_supplementaries_false_deprecation_config(monkeypatch): - """Test that the usage of use_legacy_supplementaries is deprecated.""" - msg = "use_legacy_supplementaries" - with pytest.warns(ESMValCoreDeprecationWarning, match=msg): - monkeypatch.setitem(CFG, 'use_legacy_supplementaries', False) - assert CFG['use_legacy_supplementaries'] is False diff --git a/tests/unit/recipe/test_recipe.py b/tests/unit/recipe/test_recipe.py index a437221a0e..5aa466cfcf 100644 --- a/tests/unit/recipe/test_recipe.py +++ b/tests/unit/recipe/test_recipe.py @@ -547,56 +547,6 @@ def test_get_default_settings(mocker): } -def test_add_legacy_supplementaries_disabled(): - """Test that `_add_legacy_supplementaries` does nothing when disabled.""" - dataset = Dataset() - dataset.session = {'use_legacy_supplementaries': False} - _recipe._add_legacy_supplementary_datasets(dataset, settings={}) - - -def test_enable_legacy_supplementaries_when_used(mocker, session): - """Test that legacy supplementaries are enabled when used in the recipe.""" - recipe = mocker.create_autospec(_recipe.Recipe, instance=True) - recipe.session = session - recipe._preprocessors = { - 'preproc1': { - 'area_statistics': { - 'operator': 'mean', - 'fx_variables': 'areacella', - } - } - } - session['use_legacy_supplementaries'] = None - _recipe.Recipe._set_use_legacy_supplementaries(recipe) - - assert session['use_legacy_supplementaries'] is True - - -def test_strip_legacy_supplementaries_when_disabled(mocker, session): - """Test that legacy supplementaries are removed when disabled.""" - recipe = mocker.create_autospec(_recipe.Recipe, instance=True) - recipe.session = session - recipe._preprocessors = { - 'preproc1': { - 'area_statistics': { - 'operator': 'mean', - 'fx_variables': 'areacella', - } - } - } - session['use_legacy_supplementaries'] = False - _recipe.Recipe._set_use_legacy_supplementaries(recipe) - - assert session['use_legacy_supplementaries'] is False - assert recipe._preprocessors == { - 'preproc1': { - 'area_statistics': { - 'operator': 'mean', - } - } - } - - def test_set_version(mocker): dataset = Dataset(short_name='tas') diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 56baaa60d8..6ca98ee9dc 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -452,8 +452,6 @@ def test_from_recipe_with_supplementary(session, tmp_path): def test_from_recipe_with_skip_supplementary(session, tmp_path): - session['use_legacy_supplementaries'] = False - recipe_txt = textwrap.dedent(""" datasets: @@ -501,7 +499,6 @@ def test_from_recipe_with_skip_supplementary(session, tmp_path): def test_from_recipe_with_automatic_supplementary(session, tmp_path, monkeypatch): - session['use_legacy_supplementaries'] = False def _find_files(self): if self.facets['short_name'] == 'areacello': From 4b0cba193281f7351698a6eaf55d5b58fe2a0f8a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 10:55:51 +0200 Subject: [PATCH 2/3] Improve tests --- tests/integration/conftest.py | 88 +++++++++++++++---------- tests/integration/recipe/test_recipe.py | 74 +++++++++++++-------- 2 files changed, 98 insertions(+), 64 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 612e72615e..2771db32c0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,24 +5,24 @@ import pytest import esmvalcore.local -from esmvalcore.config import CFG, _config +from esmvalcore.config import CFG from esmvalcore.config._config_object import CFG_DEFAULT +from esmvalcore.local import ( + LocalFile, + _replace_tags, + _select_drs, + _select_files, +) @pytest.fixture -def session(tmp_path, monkeypatch): +def session(tmp_path: Path, monkeypatch): + CFG.clear() + CFG.update(CFG_DEFAULT) + monkeypatch.setitem(CFG, 'rootpath', {'default': str(tmp_path)}) + session = CFG.start_session('recipe_test') - session.clear() - session.update(CFG_DEFAULT) session['output_dir'] = tmp_path / 'esmvaltool_output' - - # The patched_datafinder fixture does not return the correct input - # directory structure, so make sure it is set to flat for every project - monkeypatch.setitem(CFG, 'drs', {}) - for project in _config.CFG: - monkeypatch.setitem(_config.CFG[project]['input_dir'], 'default', '/') - # The patched datafinder fixture does not return any facets, so automatic - # supplementary definition does not work with it. return session @@ -39,8 +39,10 @@ def create_test_file(filename, tracking_id=None): iris.save(cube, filename) -def _get_filenames(root_path, filename, tracking_id): - filename = Path(filename).name +def _get_files(root_path, facets, tracking_id): + file_template = _select_drs('input_file', facets['project']) + file_globs = _replace_tags(file_template, facets) + filename = Path(file_globs[0]).name filename = str(root_path / 'input' / filename) filenames = [] if filename.endswith('[_.]*nc'): @@ -49,23 +51,37 @@ def _get_filenames(root_path, filename, tracking_id): filename = filename.replace('[_.]*nc', '_*.nc') if filename.endswith('*.nc'): filename = filename[:-len('*.nc')] + '_' - intervals = [ - '1990_1999', - '2000_2009', - '2010_2019', - ] + if facets['frequency'] == 'fx': + intervals = [''] + else: + intervals = [ + '1990_1999', + '2000_2009', + '2010_2019', + ] for interval in intervals: filenames.append(filename + interval + '.nc') else: filenames.append(filename) + if 'timerange' in facets: + filenames = _select_files(filenames, facets['timerange']) + for filename in filenames: create_test_file(filename, next(tracking_id)) - return filenames + + files = [] + for filename in filenames: + file = LocalFile(filename) + file.facets = facets + files.append(file) + + return files, file_globs @pytest.fixture def patched_datafinder(tmp_path, monkeypatch): + def tracking_ids(i=0): while True: yield i @@ -73,14 +89,18 @@ def tracking_ids(i=0): tracking_id = tracking_ids() - def glob(file_glob): - return _get_filenames(tmp_path, file_glob, tracking_id) + def find_files(*, debug: bool = False, **facets): + files, file_globs = _get_files(tmp_path, facets, tracking_id) + if debug: + return files, file_globs + return files - monkeypatch.setattr(esmvalcore.local, 'glob', glob) + monkeypatch.setattr(esmvalcore.local, 'find_files', find_files) @pytest.fixture def patched_failing_datafinder(tmp_path, monkeypatch): + def tracking_ids(i=0): while True: yield i @@ -88,16 +108,12 @@ def tracking_ids(i=0): tracking_id = tracking_ids() - def glob(filename): - # Fail for specified fx variables - if 'fx_' in filename: - return [] - if 'sftlf' in filename: - return [] - if 'IyrAnt_' in filename: - return [] - if 'IyrGre_' in filename: - return [] - return _get_filenames(tmp_path, filename, tracking_id) - - monkeypatch.setattr(esmvalcore.local, 'glob', glob) + def find_files(*, debug: bool = False, **facets): + files, file_globs = _get_files(tmp_path, facets, tracking_id) + if 'fx' == facets['frequency']: + files = [] + if debug: + return files, file_globs + return files + + monkeypatch.setattr(esmvalcore.local, 'find_files', find_files) diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 9655286872..23f7dcf657 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -120,6 +120,7 @@ def _get_default_settings_for_chl(save_filename): @pytest.fixture def patched_tas_derivation(monkeypatch): + def get_required(short_name, _): if short_name != 'tas': assert False @@ -1801,7 +1802,6 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, session): mm_products) == len(mm_statistics) * len(ens_statistics) * len(groupby) -@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -1826,6 +1826,9 @@ def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): - {dataset: CanESM2} - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, tier: 1} + supplementary_variables: + - short_name: sftlf + mip: fx scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -1844,16 +1847,8 @@ def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): assert settings['area_type'] == 'land' assert len(product.datasets) == 1 dataset = product.datasets[0] - short_names = { - ds.facets['short_name'] - for ds in dataset.supplementaries - } - if dataset.facets['project'] == 'obs4MIPs': - assert len(dataset.supplementaries) == 1 - assert {'sftlf'} == short_names - else: - assert len(dataset.supplementaries) == 2 - assert {'sftlf', 'sftof'} == short_names + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0].facets['short_name'] == 'sftlf' def test_weighting_landsea_fraction_no_fx(tmp_path, patched_failing_datafinder, @@ -1887,7 +1882,6 @@ def test_weighting_landsea_fraction_no_fx(tmp_path, patched_failing_datafinder, get_recipe(tmp_path, content, session) -@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, session): content = dedent(""" @@ -1912,8 +1906,8 @@ def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, additional_datasets: - {dataset: CanESM2} - {dataset: GFDL-CM3} - - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, - tier: 1} + - {dataset: TEST, project: obs4MIPs, + supplementary_variables: [{short_name: sftlf, mip: fx}]} scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -1971,7 +1965,6 @@ def test_weighting_landsea_fraction_exclude_fail(tmp_path, patched_datafinder, "diagnostic 'diagnostic_name'.") -@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_area_statistics(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: @@ -1994,6 +1987,9 @@ def test_area_statistics(tmp_path, patched_datafinder, session): - {dataset: CanESM2} - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, tier: 1} + supplementary_variables: + - short_name: areacella + mip: fx scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -2012,17 +2008,10 @@ def test_area_statistics(tmp_path, patched_datafinder, session): assert settings['operator'] == 'mean' assert len(product.datasets) == 1 dataset = product.datasets[0] - short_names = { - ds.facets['short_name'] - for ds in dataset.supplementaries - } - if dataset.facets['project'] == 'obs4MIPs': - assert short_names == {'areacella'} - else: - assert short_names == {'areacella', 'areacello'} + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0].facets['short_name'] == 'areacella' -@pytest.mark.skip # TODO: check if this test is needed and if yes, fix it def test_landmask(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: @@ -2045,6 +2034,9 @@ def test_landmask(tmp_path, patched_datafinder, session): - {dataset: CanESM2} - {dataset: TEST, project: obs4MIPs, level: 1, version: 1, tier: 1} + supplementary_variables: + - short_name: sftlf + mip: fx scripts: null """) recipe = get_recipe(tmp_path, content, session) @@ -2063,10 +2055,8 @@ def test_landmask(tmp_path, patched_datafinder, session): assert settings['mask_out'] == 'sea' assert len(product.datasets) == 1 dataset = product.datasets[0] - if dataset.facets['project'] == 'obs4MIPs': - assert len(dataset.supplementaries) == 1 - else: - assert len(dataset.supplementaries) == 2 + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0].facets['short_name'] == 'sftlf' def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): @@ -2114,6 +2104,34 @@ def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): assert dataset.supplementaries == [] +def test_wrong_project(tmp_path, patched_datafinder, session): + content = dedent(""" + preprocessors: + preproc: + volume_statistics: + operator: mean + diagnostics: + diagnostic_name: + variables: + tos: + preprocessor: preproc + project: CMIP7 + mip: Omon + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1 + additional_datasets: + - {dataset: CanESM2} + scripts: null + """) + msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " + "with mip 'Omon'") + with pytest.raises(RecipeError) as wrong_proj: + get_recipe(tmp_path, content, session) + assert str(wrong_proj.value) == msg + + def test_multimodel_mask(tmp_path, patched_datafinder, session): """Test ``mask_multimodel``.""" content = dedent(""" From 9a358e5a0c39fc5643f16b7652fbe5a7cd52ade5 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Sep 2023 11:25:53 +0200 Subject: [PATCH 3/3] Add test --- tests/unit/test_dataset.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 6ca98ee9dc..853e3472dc 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -1162,6 +1162,24 @@ def test_remove_not_found_supplementaries(): assert len(dataset.supplementaries) == 0 +def test_concatenating_historical_and_future_exps(mocker): + mocker.patch.object(Dataset, 'files', True) + dataset = Dataset( + dataset='dataset1', + short_name='tas', + mip='Amon', + frequency='mon', + project='CMIP6', + exp=['historical', 'ssp585'], + ) + dataset.add_supplementary(short_name='areacella', mip='fx', frequency='fx') + dataset._fix_fx_exp() + + assert len(dataset.supplementaries) == 1 + assert dataset.facets['exp'] == ['historical', 'ssp585'] + assert dataset.supplementaries[0].facets['exp'] == 'historical' + + def test_from_recipe_with_glob(tmp_path, session, mocker): recipe_txt = textwrap.dedent("""