diff --git a/doc/recipe/overview.rst b/doc/recipe/overview.rst index 7798878620..3a04a0170f 100644 --- a/doc/recipe/overview.rst +++ b/doc/recipe/overview.rst @@ -340,6 +340,14 @@ while use ``sic`` as their short_name. If the recipe requested ``siconc``, the preprocessed files will be identical except that they will use the short_name ``siconc`` instead. +Another uncommon case that may be a problem sometimes is that, for some +variables, the ``out_name`` and the name of the entry in the CMOR tables do not +match, like in ``zg7h`` and ``zg27`` in CMIP6 ``6hrPlevPt``, two versions of +``zg`` that are mutually exclusive and have different requirements for the +levels. To allow ESMValCore to retrieve the correct information in those cases, +the ``cmor_name`` must be defined and ``short_name`` will be reserved for the +``out_name`` of the variable. + Diagnostic and variable specific datasets ----------------------------------------- The ``additional_datasets`` option can be used to add datasets beyond those diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 6db4df455c..9d1db507cb 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -71,7 +71,7 @@ def _add_cmor_info(variable, override=False): derive = variable.get('derive', False) table = CMOR_TABLES.get(project) if table: - table_entry = table.get_variable(mip, short_name, derive) + table_entry = table.get_variable(mip, variable['cmor_name'], derive) else: table_entry = None if table_entry is None: @@ -141,7 +141,7 @@ def _update_target_levels(variable, variables, settings, config_user): filename=filename, project=variable_data['project'], dataset=dataset, - short_name=variable_data['short_name'], + cmor_name=variable_data['cmor_name'], mip=variable_data['mip'], frequency=variable_data['frequency'], fix_dir=os.path.splitext(variable_data['filename'])[0] + @@ -201,7 +201,7 @@ def _dataset_to_file(variable, config_user): """Find the first file belonging to dataset from variable info.""" (files, dirnames, filenames) = _get_input_files(variable, config_user) if not files and variable.get('derive'): - required_vars = get_required(variable['short_name'], + required_vars = get_required(variable['cmor_name'], variable['project']) for required_var in required_vars: _augment(required_var, variable) @@ -264,7 +264,7 @@ def _get_default_settings(variable, config_user, derive=False): fix = { 'project': variable['project'], 'dataset': variable['dataset'], - 'short_name': variable['short_name'], + 'cmor_name': variable['cmor_name'], 'mip': variable['mip'], } # File fixes @@ -287,6 +287,7 @@ def _get_default_settings(variable, config_user, derive=False): if derive: settings['derive'] = { + 'cmor_name': variable['cmor_name'], 'short_name': variable['short_name'], 'standard_name': variable['standard_name'], 'long_name': variable['long_name'], @@ -297,7 +298,7 @@ def _get_default_settings(variable, config_user, derive=False): settings['cmor_check_metadata'] = { 'cmor_table': variable['project'], 'mip': variable['mip'], - 'short_name': variable['short_name'], + 'cmor_name': variable['cmor_name'], 'frequency': variable['frequency'], 'check_level': config_user.get('check_level', CheckLevels.DEFAULT) } @@ -330,6 +331,8 @@ def _add_fxvar_keys(fx_info, variable): fx_variable = deepcopy(variable) fx_variable.update(fx_info) fx_variable['variable_group'] = fx_info['short_name'] + if 'cmor_name' not in fx_variable: + fx_variable['cmor_name'] = fx_info['short_name'] # add special ensemble for CMIP5 only if fx_variable['project'] == 'CMIP5': @@ -344,7 +347,7 @@ def _add_fxvar_keys(fx_info, variable): def _search_fx_mip(tables, found_mip, variable, fx_info, config_user): fx_files = None for mip in tables: - fx_cmor = tables[mip].get(fx_info['short_name']) + fx_cmor = tables[mip].get(fx_info['cmor_name']) if fx_cmor: found_mip = True fx_info['mip'] = mip @@ -381,7 +384,7 @@ def _get_fx_files(variable, fx_info, config_user): found_mip, fx_info, fx_files = _search_fx_mip( project_tables, found_mip, variable, fx_info, config_user) else: - fx_cmor = project_tables[fx_info['mip']].get(fx_info['short_name']) + fx_cmor = project_tables[fx_info['mip']].get(fx_info['cmor_name']) if fx_cmor: found_mip = True fx_info = _add_fxvar_keys(fx_info, variable) @@ -436,6 +439,8 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars): fx_info.update({'mip': None}) if 'short_name' not in fx_info: fx_info.update({'short_name': fx_var}) + if 'cmor_name' not in fx_info: + fx_info.update({'cmor_name': fx_info['short_name']}) fx_files, fx_info = _get_fx_files(variable, fx_info, config_user) if fx_files: fx_info['filename'] = fx_files @@ -1097,6 +1102,8 @@ def _initialize_variables(self, raw_variable, raw_datasets): if activity: variable['activity'] = activity check.variable(variable, required_keys) + if 'cmor_name' not in variable: + variable['cmor_name'] = variable['short_name'] variables = self._expand_ensemble(variables) return variables diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index ba4175813b..2a29a90013 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -121,7 +121,7 @@ def __ne__(self, other): return not self.__eq__(other) @staticmethod - def get_fixes(project, dataset, mip, short_name): + def get_fixes(project, dataset, mip, cmor_name): """ Get the fixes that must be applied for a given dataset. @@ -141,7 +141,7 @@ def get_fixes(project, dataset, mip, short_name): project: str dataset: str mip: str - short_name: str + cmor_name: str Returns ------- @@ -149,11 +149,11 @@ def get_fixes(project, dataset, mip, short_name): Fixes to apply for the given data """ cmor_table = CMOR_TABLES[project] - vardef = cmor_table.get_variable(mip, short_name) + vardef = cmor_table.get_variable(mip, cmor_name) project = project.replace('-', '_').lower() dataset = dataset.replace('-', '_').lower() - short_name = short_name.replace('-', '_').lower() + cmor_name = cmor_name.replace('-', '_').lower() fixes = [] try: @@ -162,7 +162,7 @@ def get_fixes(project, dataset, mip, short_name): classes = inspect.getmembers(fixes_module, inspect.isclass) classes = dict((name.lower(), value) for name, value in classes) - for fix_name in (short_name, mip.lower(), 'allvars'): + for fix_name in (cmor_name, mip.lower(), 'allvars'): try: fixes.append(classes[fix_name](vardef)) except KeyError: diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 9fcbb5a62b..364280a701 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -939,7 +939,7 @@ def report_debug_message(self, message, *args): def _get_cmor_checker(table, mip, - short_name, + cmor_name, frequency, fail_on_error=False, check_level=CheckLevels.DEFAULT, @@ -952,9 +952,9 @@ def _get_cmor_checker(table, table, ', '.join(CMOR_TABLES))) cmor_table = CMOR_TABLES[table] - var_info = cmor_table.get_variable(mip, short_name) + var_info = cmor_table.get_variable(mip, cmor_name) if var_info is None: - var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) + var_info = CMOR_TABLES['custom'].get_variable(mip, cmor_name) def _checker(cube): return CMORCheck(cube, @@ -970,7 +970,7 @@ def _checker(cube): def cmor_check_metadata(cube, cmor_table, mip, - short_name, + cmor_name, frequency, check_level=CheckLevels.DEFAULT): """Check if metadata conforms to variable's CMOR definition. @@ -985,8 +985,8 @@ def cmor_check_metadata(cube, CMOR definitions to use. mip: Variable's mip. - short_name: basestring - Variable's short name. + cmor_name: str + Variable's cmor name. frequency: basestring Data frequency. check_level: CheckLevels @@ -994,7 +994,7 @@ def cmor_check_metadata(cube, """ checker = _get_cmor_checker(cmor_table, mip, - short_name, + cmor_name, frequency, check_level=check_level) checker(cube).check_metadata() @@ -1004,7 +1004,7 @@ def cmor_check_metadata(cube, def cmor_check_data(cube, cmor_table, mip, - short_name, + cmor_name, frequency, check_level=CheckLevels.DEFAULT): """Check if data conforms to variable's CMOR definition. @@ -1019,8 +1019,8 @@ def cmor_check_data(cube, CMOR definitions to use. mip: Variable's mip. - short_name: basestring - Variable's short name + cmor_name: basestring + Variable's cmor name frequency: basestring Data frequency check_level: CheckLevels @@ -1028,14 +1028,14 @@ def cmor_check_data(cube, """ checker = _get_cmor_checker(cmor_table, mip, - short_name, + cmor_name, frequency, check_level=check_level) checker(cube).check_data() return cube -def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): +def cmor_check(cube, cmor_table, mip, cmor_name, frequency, check_level): """Check if cube conforms to variable's CMOR definition. Equivalent to calling cmor_check_metadata and cmor_check_data @@ -1049,8 +1049,8 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): CMOR definitions to use. mip: Variable's mip. - short_name: basestring - Variable's short name. + cmor_name: basestring + Variable's cmor name. frequency: basestring Data frequency. check_level: enum.IntEnum @@ -1059,13 +1059,13 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level): cmor_check_metadata(cube, cmor_table, mip, - short_name, + cmor_name, frequency, check_level=check_level) cmor_check_data(cube, cmor_table, mip, - short_name, + cmor_name, frequency, check_level=check_level) return cube diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 23cc98e4f9..7071fa37bd 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -4,6 +4,7 @@ for the given dataset. Therefore is recommended to apply them to all variables to be sure that all known errors are fixed. """ +from esmvalcore.cmor.table import get_var_info import logging from collections import defaultdict @@ -15,7 +16,7 @@ logger = logging.getLogger(__name__) -def fix_file(file, short_name, project, dataset, mip, output_dir): +def fix_file(file, cmor_name, project, dataset, mip, output_dir): """Fix files before ESMValTool can load them. This fixes are only for issues that prevent iris from loading the cube or @@ -27,7 +28,7 @@ def fix_file(file, short_name, project, dataset, mip, output_dir): ---------- file: str Path to the original file - short_name: str + cmor_name: str Variable's short name project: str dataset:str @@ -42,13 +43,13 @@ def fix_file(file, short_name, project, dataset, mip, output_dir): for fix in Fix.get_fixes(project=project, dataset=dataset, mip=mip, - short_name=short_name): + cmor_name=cmor_name): file = fix.fix_file(file, output_dir) return file def fix_metadata(cubes, - short_name, + cmor_name, project, dataset, mip, @@ -65,8 +66,8 @@ def fix_metadata(cubes, ---------- cubes: iris.cube.CubeList Cubes to fix - short_name: str - Variable's short name + cmor_name: str + Variable's cmor name project: str dataset: str @@ -92,22 +93,24 @@ def fix_metadata(cubes, fixes = Fix.get_fixes(project=project, dataset=dataset, mip=mip, - short_name=short_name) + cmor_name=cmor_name) fixed_cubes = [] by_file = defaultdict(list) for cube in cubes: by_file[cube.attributes.get('source_file', '')].append(cube) + vardef = get_var_info(project, mip, cmor_name) + for cube_list in by_file.values(): cube_list = CubeList(cube_list) for fix in fixes: cube_list = fix.fix_metadata(cube_list) - cube = _get_single_cube(cube_list, short_name, project, dataset) + cube = _get_single_cube(cube_list, vardef.short_name, project, dataset) checker = _get_cmor_checker(frequency=frequency, table=project, mip=mip, - short_name=short_name, + cmor_name=cmor_name, check_level=check_level, fail_on_error=False, automatic_fixes=True) @@ -142,7 +145,7 @@ def _get_single_cube(cube_list, short_name, project, dataset): def fix_data(cube, - short_name, + cmor_name, project, dataset, mip, @@ -161,8 +164,8 @@ def fix_data(cube, ---------- cube: iris.cube.Cube Cube to fix - short_name: str - Variable's short name + cmor_name: str + Variable's cmor name project: str dataset: str mip: str @@ -185,12 +188,12 @@ def fix_data(cube, for fix in Fix.get_fixes(project=project, dataset=dataset, mip=mip, - short_name=short_name): + cmor_name=cmor_name): cube = fix.fix_data(cube) checker = _get_cmor_checker(frequency=frequency, table=project, mip=mip, - short_name=short_name, + cmor_name=cmor_name, fail_on_error=False, automatic_fixes=True, check_level=check_level) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 508ba47a8a..52a3710805 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -22,7 +22,7 @@ """dict of str, obj: CMOR info objects.""" -def get_var_info(project, mip, short_name): +def get_var_info(project, mip, cmor_name): """Get variable information. Parameters @@ -31,10 +31,10 @@ def get_var_info(project, mip, short_name): Dataset's project. mip : str Variable's cmor table. - short_name : str - Variable's short name. + cmor_name : str + Variable's CMOR name. """ - return CMOR_TABLES[project].get_variable(mip, short_name) + return CMOR_TABLES[project].get_variable(mip, cmor_name) def read_cmor_tables(cfg_developer=None): @@ -139,15 +139,15 @@ def get_table(self, table): """ return self.tables.get(table) - def get_variable(self, table_name, short_name, derived=False): + def get_variable(self, table_name, cmor_name, derived=False): """Search and return the variable info. Parameters ---------- table_name: basestring Table name - short_name: basestring - Variable's short name + cmor_name: basestring + Variable's CMOR name derived: bool, optional Variable is derived. Info retrieval for derived variables always look on the default tables if variable is not find in the @@ -159,7 +159,7 @@ def get_variable(self, table_name, short_name, derived=False): Return the VariableInfo object for the requested variable if found, returns None if not """ - alt_names_list = self._get_alt_names_list(short_name) + alt_names_list = self._get_alt_names_list(cmor_name) table = self.get_table(table_name) if table: @@ -197,10 +197,10 @@ def _look_in_all_tables(self, alt_names_list): break return var_info - def _get_alt_names_list(self, short_name): - alt_names_list = [short_name] + def _get_alt_names_list(self, cmor_name): + alt_names_list = [cmor_name] for alt_names in self.alt_names: - if short_name in alt_names: + if cmor_name in alt_names: alt_names_list.extend([ alt_name for alt_name in alt_names if alt_name not in alt_names_list @@ -458,19 +458,21 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, short_name): + def __init__(self, table_type, cmor_name): """Class to read and store variable information. Parameters ---------- - short_name: str - variable's short name + cmor_name: str + variable's cmor name """ super(VariableInfo, self).__init__() self.table_type = table_type self.modeling_realm = [] """Modeling realm""" - self.short_name = short_name + self.cmor_name = cmor_name + """CMOR name""" + self.short_name = cmor_name """Short name""" self.standard_name = '' """Standard name""" @@ -523,7 +525,7 @@ def read_json(self, json_data, default_freq): Default frequency to use if it is not defined at variable level """ self._json_data = json_data - + self.short_name = self._read_json_variable('out_name', self.cmor_name) self.standard_name = self._read_json_variable('standard_name') self.long_name = self._read_json_variable('long_name') self.units = self._read_json_variable('units') @@ -737,8 +739,8 @@ def _read_coordinate(self, value): setattr(coord, key, value) return coord - def _read_variable(self, short_name, frequency): - var = VariableInfo('CMIP5', short_name) + def _read_variable(self, cmor_name, frequency): + var = VariableInfo('CMIP5', cmor_name) var.frequency = frequency while self._read_line(): key, value = self._last_line_read @@ -748,6 +750,8 @@ def _read_variable(self, short_name, frequency): setattr(var, key, value.split()) elif hasattr(var, key): setattr(var, key, value) + elif key == 'out_name': + var.short_name = value for dim in var.dimensions: var.coordinates[dim] = self.coords[dim] return var diff --git a/esmvalcore/preprocessor/_ancillary_vars.py b/esmvalcore/preprocessor/_ancillary_vars.py index 5d6dbe32d1..cd6cf885e3 100644 --- a/esmvalcore/preprocessor/_ancillary_vars.py +++ b/esmvalcore/preprocessor/_ancillary_vars.py @@ -14,16 +14,18 @@ def _load_fx(fx_info, check_level): """Load and CMOR-check fx variables.""" + cmor_name = fx_info['cmor_name'] + # short_name = fx_info['short_name'] + project = fx_info['project'] + dataset = fx_info['dataset'] + mip = fx_info['mip'] + freq = fx_info['frequency'] + fx_cubes = iris.cube.CubeList() for fx_file in fx_info['filename']: loaded_cube = load(fx_file, callback=concatenate_callback) - short_name = fx_info['short_name'] - project = fx_info['project'] - dataset = fx_info['dataset'] - mip = fx_info['mip'] - freq = fx_info['frequency'] - loaded_cube = fix_metadata(loaded_cube, short_name=short_name, + loaded_cube = fix_metadata(loaded_cube, cmor_name=cmor_name, project=project, dataset=dataset, mip=mip, frequency=freq, check_level=check_level) @@ -32,15 +34,15 @@ def _load_fx(fx_info, check_level): fx_cube = concatenate(fx_cubes) fx_cube = cmor_check_metadata(fx_cube, cmor_table=project, mip=mip, - short_name=short_name, frequency=freq, + cmor_name=cmor_name, frequency=freq, check_level=check_level) - fx_cube = fix_data(fx_cube, short_name=short_name, project=project, + fx_cube = fix_data(fx_cube, cmor_name=cmor_name, project=project, dataset=dataset, mip=mip, frequency=freq, check_level=check_level) fx_cube = cmor_check_data(fx_cube, cmor_table=project, mip=mip, - short_name=fx_cube.var_name, frequency=freq, + cmor_name=cmor_name, frequency=freq, check_level=check_level) return fx_cube diff --git a/esmvalcore/preprocessor/_derive/__init__.py b/esmvalcore/preprocessor/_derive/__init__.py index 17209f0006..75ef0ffb45 100644 --- a/esmvalcore/preprocessor/_derive/__init__.py +++ b/esmvalcore/preprocessor/_derive/__init__.py @@ -34,7 +34,7 @@ def _get_all_derived_variables(): __all__ = list(ALL_DERIVED_VARIABLES) -def get_required(short_name, project): +def get_required(cmor_name, project): """Return all required variables for derivation. Get all information (at least `short_name`) required for derivation. @@ -52,16 +52,16 @@ def get_required(short_name, project): List of dictionaries (including at least the key `short_name`). """ - if short_name not in ALL_DERIVED_VARIABLES: + if cmor_name not in ALL_DERIVED_VARIABLES: raise NotImplementedError( - f"Cannot derive variable '{short_name}', no derivation script " + f"Cannot derive variable '{cmor_name}', no derivation script " f"available") - DerivedVariable = ALL_DERIVED_VARIABLES[short_name] # noqa: N806 + DerivedVariable = ALL_DERIVED_VARIABLES[cmor_name] # noqa: N806 variables = deepcopy(DerivedVariable().required(project)) return variables -def derive(cubes, short_name, long_name, units, standard_name=None): +def derive(cubes, cmor_name, short_name, long_name, units, standard_name=None): """Derive variable. Parameters @@ -90,11 +90,11 @@ def derive(cubes, short_name, long_name, units, standard_name=None): cubes = iris.cube.CubeList(cubes) # Derive variable - DerivedVariable = ALL_DERIVED_VARIABLES[short_name.lower()] # noqa: N806 + DerivedVariable = ALL_DERIVED_VARIABLES[cmor_name.lower()] # noqa: N806 try: cube = DerivedVariable().calculate(cubes) except Exception as exc: - msg = (f"Derivation of variable '{short_name}' failed. If you used " + msg = (f"Derivation of variable '{cmor_name}' failed. If you used " f"the option '--skip-nonexistent' for running your recipe, " f"this might be caused by missing input data for derivation") raise ValueError(msg) from exc @@ -114,13 +114,13 @@ def derive(cubes, short_name, long_name, units, standard_name=None): logger.warning( "Units of cube after executing derivation script of '%s' are " "'%s', automatically setting them to '%s'. This might lead to " - "incorrect data", short_name, cube.units, units) + "incorrect data", cmor_name, cube.units, units) cube.units = units elif cube.units.is_convertible(units): cube.convert_units(units) else: raise ValueError( f"Units '{cube.units}' after executing derivation script of " - f"'{short_name}' cannot be converted to target units '{units}'") + f"'{cmor_name}' cannot be converted to target units '{units}'") return cube diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 7ff8dde22d..e06a1d6655 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -733,7 +733,7 @@ def get_cmor_levels(cmor_table, coordinate): coordinate, cmor_table)) -def get_reference_levels(filename, project, dataset, short_name, mip, +def get_reference_levels(filename, project, dataset, cmor_name, mip, frequency, fix_dir): """Get level definition from a reference dataset. @@ -745,7 +745,7 @@ def get_reference_levels(filename, project, dataset, short_name, mip, Name of the project dataset : str Name of the dataset - short_name : str + cmor_name : str Name of the variable mip : str Name of the mip table @@ -766,7 +766,7 @@ def get_reference_levels(filename, project, dataset, short_name, mip, """ filename = fix_file( file=filename, - short_name=short_name, + cmor_name=cmor_name, project=project, dataset=dataset, mip=mip, @@ -775,7 +775,7 @@ def get_reference_levels(filename, project, dataset, short_name, mip, cubes = load(filename, callback=concatenate_callback) cubes = fix_metadata( cubes=cubes, - short_name=short_name, + cmor_name=cmor_name, project=project, dataset=dataset, mip=mip, diff --git a/tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py b/tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py index 7ac756d50a..39716bf58b 100644 --- a/tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py +++ b/tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py @@ -75,12 +75,14 @@ def test_add_cell_measure_area(self, tmp_path): fx_vars = { 'areacella': { 'short_name': 'areacella', + 'cmor_name': 'areacella', 'project': 'CMIP6', 'dataset': 'EC-Earth3', 'mip': 'fx', 'frequency': 'fx'}, 'areacello': { 'short_name': 'areacello', + 'cmor_name': 'areacello', 'project': 'CMIP6', 'dataset': 'EC-Earth3', 'mip': 'Ofx', @@ -105,6 +107,7 @@ def test_add_cell_measure_volume(self, tmp_path): fx_vars = { 'volcello': { 'short_name': 'volcello', + 'cmor_name': 'volcello', 'project': 'CMIP6', 'dataset': 'EC-Earth3', 'mip': 'Ofx', @@ -144,6 +147,7 @@ def test_add_ancillary_vars(self, tmp_path): fx_vars = { 'sftlf': { 'short_name': 'sftlf', + 'cmor_name': 'sftlf', 'project': 'CMIP6', 'dataset': 'EC-Earth3', 'mip': 'fx', @@ -175,6 +179,7 @@ def test_wrong_time_frequency(self, tmp_path): fx_vars = { 'volcello': { 'short_name': 'volcello', + 'cmor_name': 'volcello', 'project': 'CMIP6', 'dataset': 'EC-Earth3', 'mip': 'Oyr', diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index 14ea1d6c9a..5aa2a26b5f 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -47,7 +47,7 @@ def assert_derived_var_calc_called_once_with(*args): def test_check_units_none(mock_cubes): """Test units after derivation if derivation scripts returns None.""" mock_all_derived_variables(None) - cube = derive(mock_cubes, SHORT_NAME, mock.sentinel.long_name, + cube = derive(mock_cubes, SHORT_NAME, SHORT_NAME, mock.sentinel.long_name, mock.sentinel.units, standard_name=mock.sentinel.standard_name) assert_derived_var_calc_called_once_with(mock_cubes) @@ -63,8 +63,8 @@ def test_check_units_none(mock_cubes): def test_check_units_equal(mock_cubes): """Test units after derivation if derivation scripts returns None.""" mock_all_derived_variables(Unit('kg m2 s-2')) - cube = derive(mock_cubes, SHORT_NAME, mock.sentinel.long_name, 'J', - standard_name=mock.sentinel.standard_name) + cube = derive(mock_cubes, SHORT_NAME, SHORT_NAME, mock.sentinel.long_name, + 'J', standard_name=mock.sentinel.standard_name) assert_derived_var_calc_called_once_with(mock_cubes) assert cube.units == Unit('J') assert cube.var_name == SHORT_NAME @@ -78,8 +78,8 @@ def test_check_units_equal(mock_cubes): def test_check_units_nounit(mock_cubes): """Test units after derivation if derivation scripts returns None.""" mock_all_derived_variables(Unit('no unit')) - cube = derive(mock_cubes, SHORT_NAME, mock.sentinel.long_name, 'J', - standard_name=mock.sentinel.standard_name) + cube = derive(mock_cubes, SHORT_NAME, SHORT_NAME, mock.sentinel.long_name, + 'J', standard_name=mock.sentinel.standard_name) assert_derived_var_calc_called_once_with(mock_cubes) assert cube.units == Unit('J') assert cube.var_name == SHORT_NAME @@ -96,8 +96,8 @@ def test_check_units_nounit(mock_cubes): def test_check_units_unknown(mock_cubes): """Test units after derivation if derivation scripts returns None.""" mock_all_derived_variables(Unit('unknown')) - cube = derive(mock_cubes, SHORT_NAME, mock.sentinel.long_name, 'J', - standard_name=mock.sentinel.standard_name) + cube = derive(mock_cubes, SHORT_NAME, SHORT_NAME, mock.sentinel.long_name, + 'J', standard_name=mock.sentinel.standard_name) assert_derived_var_calc_called_once_with(mock_cubes) assert cube.units == Unit('J') assert cube.var_name == SHORT_NAME @@ -114,8 +114,8 @@ def test_check_units_unknown(mock_cubes): def test_check_units_convertible(mock_cubes): """Test units after derivation if derivation scripts returns None.""" mock_all_derived_variables(Unit('kg s-1')) - cube = derive(mock_cubes, SHORT_NAME, mock.sentinel.long_name, 'g yr-1', - standard_name=mock.sentinel.standard_name) + cube = derive(mock_cubes, SHORT_NAME, SHORT_NAME, mock.sentinel.long_name, + 'g yr-1', standard_name=mock.sentinel.standard_name) assert_derived_var_calc_called_once_with(mock_cubes) cube.convert_units.assert_called_once_with('g yr-1') assert cube.var_name == SHORT_NAME @@ -129,8 +129,8 @@ def test_check_units_fail(mock_cubes): """Test units after derivation if derivation scripts returns None.""" mock_all_derived_variables(Unit('kg')) with pytest.raises(ValueError) as err: - derive(mock_cubes, SHORT_NAME, mock.sentinel.long_name, 'm', - standard_name=mock.sentinel.standard_name) + derive(mock_cubes, SHORT_NAME, SHORT_NAME, mock.sentinel.long_name, + 'm', standard_name=mock.sentinel.standard_name) assert str(err.value) == ( "Units 'kg' after executing derivation script of 'short_name' cannot " "be converted to target units 'm'" @@ -183,7 +183,8 @@ def test_derive_nonstandard_nofx(): cubes = CubeList([rsdscs, rsuscs]) - alb = derive(cubes, short_name, long_name, units, standard_name) + alb = derive(cubes, short_name, short_name, long_name, units, + standard_name) assert alb.var_name == short_name assert alb.long_name == long_name @@ -198,7 +199,7 @@ def test_derive_noop(): alb.long_name = 'albedo at the surface' alb.units = 1 - cube = derive([alb], alb.var_name, alb.long_name, alb.units) + cube = derive([alb], alb.var_name, alb.var_name, alb.long_name, alb.units) assert cube is alb @@ -221,6 +222,7 @@ def mock_calculate(_, cubes): derive( [ohc_cube], short_name, + short_name, long_name, units, ) diff --git a/tests/integration/preprocessor/_mask/test_mask.py b/tests/integration/preprocessor/_mask/test_mask.py index 4e2ef513f8..4d800ac3b5 100644 --- a/tests/integration/preprocessor/_mask/test_mask.py +++ b/tests/integration/preprocessor/_mask/test_mask.py @@ -62,6 +62,7 @@ def test_components_fx_var(self, tmp_path): iris.save(self.fx_mask, sftlf_file) fx_vars = { 'sftlf': { + 'cmor_name': 'sftlf', 'short_name': 'sftlf', 'project': 'CMIP6', 'dataset': 'EC-Earth3', @@ -85,12 +86,13 @@ def test_components_fx_var(self, tmp_path): iris.save(self.fx_mask, sftgif_file) fx_vars = { 'sftgif': { + 'cmor_name': 'sftgif', 'short_name': 'sftgif', 'project': 'CMIP6', 'dataset': 'EC-Earth3', 'mip': 'fx', 'frequency': 'fx', - 'filename': sftlf_file} + 'filename': sftgif_file} } new_cube_ice = iris.cube.Cube(self.new_cube_data, dim_coords_and_dims=self.coords_spec) @@ -110,6 +112,7 @@ def test_mask_landsea(self, tmp_path): iris.save(self.fx_mask, sftlf_file) fx_vars = { 'sftlf': { + 'cmor_name': 'sftlf', 'short_name': 'sftlf', 'project': 'CMIP6', 'dataset': 'EC-Earth3', @@ -200,6 +203,7 @@ def test_mask_landseaice(self, tmp_path): iris.save(self.fx_mask, sftgif_file) fx_vars = { 'sftgif': { + 'cmor_name': 'sftgif', 'short_name': 'sftgif', 'project': 'CMIP6', 'dataset': 'EC-Earth3', diff --git a/tests/integration/preprocessor/_regrid/test_get_file_levels.py b/tests/integration/preprocessor/_regrid/test_get_file_levels.py index 128a074453..c2b119cdd6 100644 --- a/tests/integration/preprocessor/_regrid/test_get_file_levels.py +++ b/tests/integration/preprocessor/_regrid/test_get_file_levels.py @@ -49,7 +49,7 @@ def test_get_coord(self): filename=self.path, project='CMIP6', dataset='dataset', - short_name='short_name', + cmor_name='short_name', mip='mip', frequency='mon', fix_dir='output_dir', diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 81e3ef7720..80bdd5a989 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -125,7 +125,7 @@ def _get_default_settings_for_chl(fix_dir, save_filename): 'fix_file': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'chl', + 'cmor_name': 'chl', 'mip': 'Oyr', 'output_dir': fix_dir, }, @@ -133,7 +133,7 @@ def _get_default_settings_for_chl(fix_dir, save_filename): 'check_level': CheckLevels.DEFAULT, 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'chl', + 'cmor_name': 'chl', 'mip': 'Oyr', 'frequency': 'yr', }, @@ -141,7 +141,7 @@ def _get_default_settings_for_chl(fix_dir, save_filename): 'check_level': CheckLevels.DEFAULT, 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'chl', + 'cmor_name': 'chl', 'mip': 'Oyr', 'frequency': 'yr', }, @@ -153,14 +153,14 @@ def _get_default_settings_for_chl(fix_dir, save_filename): 'check_level': CheckLevels.DEFAULT, 'cmor_table': 'CMIP5', 'mip': 'Oyr', - 'short_name': 'chl', + 'cmor_name': 'chl', 'frequency': 'yr', }, 'cmor_check_data': { 'check_level': CheckLevels.DEFAULT, 'cmor_table': 'CMIP5', 'mip': 'Oyr', - 'short_name': 'chl', + 'cmor_name': 'chl', 'frequency': 'yr', }, 'add_fx_variables': { @@ -260,6 +260,7 @@ def get_required(short_name, _): }, { 'short_name': 'areacella', + 'cmor_name': 'areacella', 'mip': 'fx', 'optional': True }, @@ -555,7 +556,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_file': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'sftlf', + 'cmor_name': 'sftlf', 'mip': 'fx', 'output_dir': fix_dir, }, @@ -563,7 +564,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'check_level': CheckLevels.DEFAULT, 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'sftlf', + 'cmor_name': 'sftlf', 'mip': 'fx', 'frequency': 'fx', }, @@ -571,7 +572,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'check_level': CheckLevels.DEFAULT, 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'sftlf', + 'cmor_name': 'sftlf', 'mip': 'fx', 'frequency': 'fx', }, @@ -579,14 +580,14 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'check_level': CheckLevels.DEFAULT, 'cmor_table': 'CMIP5', 'mip': 'fx', - 'short_name': 'sftlf', + 'cmor_name': 'sftlf', 'frequency': 'fx', }, 'cmor_check_data': { 'check_level': CheckLevels.DEFAULT, 'cmor_table': 'CMIP5', 'mip': 'fx', - 'short_name': 'sftlf', + 'cmor_name': 'sftlf', 'frequency': 'fx', }, 'add_fx_variables': { @@ -817,6 +818,7 @@ def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): 'recipe_dataset_index': 0, 'rcm_version': 'v1', 'short_name': 'tas', + 'cmor_name': 'tas', 'original_short_name': 'tas', 'standard_name': 'air_temperature', 'start_year': 1991, @@ -899,7 +901,7 @@ def test_reference_dataset(tmp_path, patched_datafinder, config_user, filename=reference.files[0], project='CMIP5', dataset='MPI-ESM-LR', - short_name='ta', + cmor_name='ta', mip='Amon', frequency='mon', fix_dir=fix_dir, diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 341f549b23..a6a27292e7 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -21,7 +21,7 @@ def test_fix(self): return_value=[self.mock_fix]): file_returned = fix_file( file='filename', - short_name='short_name', + cmor_name='cmor_name', project='project', dataset='model', mip='mip', @@ -36,7 +36,7 @@ def test_nofix(self): return_value=[]): file_returned = fix_file( file='filename', - short_name='short_name', + cmor_name='cmor_name', project='project', dataset='model', mip='mip', @@ -54,9 +54,10 @@ def setUp(self): self.cube_2 = Mock() self.cube_2.var_name = 'cube2' self.cubes = [self.cube_1, self.cube_2] - vardef = Mock() - vardef.short_name = 'fix' - self.fix = Fix(vardef) + self.vardef = Mock() + self.vardef.cmor_name = 'fix' + self.vardef.short_name = 'fix' + self.fix = Fix(self.vardef) def test_get_first_cube(self): """Test selecting first cube.""" @@ -91,9 +92,12 @@ def setUp(self): self.mock_fix.fix_metadata.return_value = [self.intermediate_cube] self.checker = Mock() self.check_metadata = self.checker.return_value.check_metadata + self.vardef = Mock() + self.vardef.cmor_name = 'cmor_name' + self.vardef.short_name = 'cmor_name' @staticmethod - def _create_mock_cube(var_name='short_name'): + def _create_mock_cube(var_name='cmor_name'): cube = Mock() cube.var_name = var_name cube.attributes = {'source_file': 'source_file'} @@ -106,18 +110,21 @@ def test_fix(self): return_value=[self.mock_fix]): with patch('esmvalcore.cmor.fix._get_cmor_checker', return_value=self.checker): - cube_returned = fix_metadata( - cubes=[self.cube], - short_name='short_name', - project='project', - dataset='model', - mip='mip', - )[0] - self.checker.assert_called_once_with(self.intermediate_cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is not self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is self.fixed_cube + with patch('esmvalcore.cmor.fix.get_var_info', + return_value=self.vardef): + cube_returned = fix_metadata( + cubes=[self.cube], + cmor_name='cmor_name', + project='project', + dataset='model', + mip='mip', + )[0] + self.checker.assert_called_once_with( + self.intermediate_cube) + self.check_metadata.assert_called_once_with() + assert cube_returned is not self.cube + assert cube_returned is not self.intermediate_cube + assert cube_returned is self.fixed_cube def test_nofix(self): """Check that the same cube is returned if no fix is available.""" @@ -126,18 +133,20 @@ def test_nofix(self): return_value=[]): with patch('esmvalcore.cmor.fix._get_cmor_checker', return_value=self.checker): - cube_returned = fix_metadata( - cubes=[self.cube], - short_name='short_name', - project='project', - dataset='model', - mip='mip', - )[0] - self.checker.assert_called_once_with(self.cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is not self.fixed_cube + with patch('esmvalcore.cmor.fix.get_var_info', + return_value=self.vardef): + cube_returned = fix_metadata( + cubes=[self.cube], + cmor_name='cmor_name', + project='project', + dataset='model', + mip='mip', + )[0] + self.checker.assert_called_once_with(self.cube) + self.check_metadata.assert_called_once_with() + assert cube_returned is self.cube + assert cube_returned is not self.intermediate_cube + assert cube_returned is not self.fixed_cube def test_select_var(self): """Check that the same cube is returned if no fix is available.""" @@ -146,60 +155,65 @@ def test_select_var(self): return_value=[]): with patch('esmvalcore.cmor.fix._get_cmor_checker', return_value=self.checker): - cube_returned = fix_metadata( - cubes=[self.cube, - self._create_mock_cube('extra')], - short_name='short_name', - project='CMIP6', - dataset='model', - mip='mip', - )[0] - self.checker.assert_called_once_with(self.cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is self.cube + with patch('esmvalcore.cmor.fix.get_var_info', + return_value=self.vardef): + cube_returned = fix_metadata( + cubes=[self.cube, self._create_mock_cube('extra')], + cmor_name='cmor_name', + project='CMIP6', + dataset='model', + mip='mip', + )[0] + self.checker.assert_called_once_with(self.cube) + self.check_metadata.assert_called_once_with() + assert cube_returned is self.cube def test_select_var_failed_if_bad_var_name(self): """Check that the same cube is returned if no fix is available.""" with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - with self.assertRaises(ValueError): - fix_metadata( - cubes=[ - self._create_mock_cube('not_me'), - self._create_mock_cube('me_neither') - ], - short_name='short_name', - project='CMIP6', - dataset='model', - mip='mip', - ) + with patch('esmvalcore.cmor.fix.get_var_info', + return_value=self.vardef): + with self.assertRaises(ValueError): + fix_metadata( + cubes=[ + self._create_mock_cube('not_me'), + self._create_mock_cube('me_neither') + ], + cmor_name='cmor_name', + project='CMIP6', + dataset='model', + mip='mip', + ) def test_cmor_checker_called(self): """Check that the cmor check is done.""" - checker = Mock() - checker.return_value = Mock() + check = Mock() + check.return_value = Mock() with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): with patch('esmvalcore.cmor.fix._get_cmor_checker', - return_value=checker) as get_mock: - fix_metadata( - cubes=[self.cube], - short_name='short_name', - project='CMIP6', - dataset='dataset', - mip='mip', - frequency='frequency', - ) - get_mock.assert_called_once_with( - automatic_fixes=True, - fail_on_error=False, - frequency='frequency', - mip='mip', - short_name='short_name', - table='CMIP6', - check_level=CheckLevels.DEFAULT,) - checker.assert_called_once_with(self.cube) - checker.return_value.check_metadata.assert_called_once_with() + return_value=check) as get_mock: + with patch('esmvalcore.cmor.fix.get_var_info', + return_value=self.vardef): + fix_metadata( + cubes=[self.cube], + cmor_name='cmor_name', + project='CMIP6', + dataset='dataset', + mip='mip', + frequency='frequency', + ) + get_mock.assert_called_once_with( + automatic_fixes=True, + fail_on_error=False, + frequency='frequency', + mip='mip', + cmor_name='cmor_name', + table='CMIP6', + check_level=CheckLevels.DEFAULT,) + check.assert_called_once_with(self.cube) + check.return_value.check_metadata.assert_called_once_with() class TestFixData(TestCase): @@ -223,7 +237,7 @@ def test_fix(self): return_value=self.checker): cube_returned = fix_data( self.cube, - short_name='short_name', + cmor_name='cmor_name', project='project', dataset='model', mip='mip', @@ -243,7 +257,7 @@ def test_nofix(self): return_value=self.checker): cube_returned = fix_data( self.cube, - short_name='short_name', + cmor_name='cmor_name', project='CMIP6', dataset='model', mip='mip', @@ -262,7 +276,7 @@ def test_cmor_checker_called(self): return_value=[]): with patch('esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_data(self.cube, 'short_name', 'CMIP6', 'model', 'mip', + fix_data(self.cube, 'cmor_name', 'CMIP6', 'model', 'mip', 'frequency') get_mock.assert_called_once_with( table='CMIP6', @@ -271,7 +285,7 @@ def test_cmor_checker_called(self): fail_on_error=False, frequency='frequency', mip='mip', - short_name='short_name', + cmor_name='cmor_name', ) checker.assert_called_once_with(self.cube) checker.return_value.check_data.assert_called_once_with()