From 99451785fe4acc3370bce247656500023eb243b3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 9 Dec 2019 11:12:27 +0100 Subject: [PATCH 01/11] Diferentitate var_name and out_name --- esmvalcore/cmor/table.py | 17 +++++++++++------ tests/integration/cmor/test_table.py | 8 +++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index d876f9d062..b7c0ca0d1f 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -358,21 +358,23 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, short_name): + def __init__(self, table_type, var_name): """ Class to read and store variable information. Parameters ---------- - short_name: str - variable's short name + var_name: str + variable's CMOR name """ super(VariableInfo, self).__init__() self.table_type = table_type self.modeling_realm = [] """Modeling realm""" - self.short_name = short_name + self.var_name = var_name + """CMOR's table name""" + self.short_name = var_name """Short name""" self.standard_name = '' """Standard name""" @@ -425,6 +427,7 @@ def read_json(self, json_data, default_freq): """ self._json_data = json_data + self.short_name = self._read_json_variable('out_name') self.standard_name = self._read_json_variable('standard_name') self.long_name = self._read_json_variable('long_name') self.units = self._read_json_variable('units') @@ -635,13 +638,15 @@ def _read_coordinate(self, value): setattr(coord, key, value) return coord - def _read_variable(self, short_name, frequency): - var = VariableInfo('CMIP5', short_name) + def _read_variable(self, var_name, frequency): + var = VariableInfo('CMIP5', var_name) var.frequency = frequency while self._read_line(): key, value = self._last_line_read if key in ('variable_entry', 'axis_entry'): break + if key == 'out_name': + setattr(var, 'short_name', value) if key in ('dimensions', 'modeling_realm'): setattr(var, key, value.split()) elif hasattr(var, key): diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 22b84e936c..c2532e84d7 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -44,6 +44,12 @@ def test_get_variable_tas(self): var = self.variables_info.get_variable('Amon', 'tas') self.assertEqual(var.short_name, 'tas') + def test_get_variable_different_out_name(self): + """Get tas variable.""" + var = self.variables_info.get_variable('6hrPlevPt', 'zg7h') + self.assertEqual(var.short_name, 'zg') + self.assertEqual(var.var_name, 'zg7h') + def test_get_variable_from_alias(self): """Get a variable from a known alias.""" var = self.variables_info.get_variable('SImon', 'sic') @@ -220,7 +226,7 @@ def test_aermon_ta_fail_if_strict(self): """Get ta fails with AERMonZ if strict.""" self.assertIsNone(self.variables_info.get_variable('Omon', 'ta')) - def test_aermon_ta_succes_if_strict(self): + def test_aermon_ta_succes_if_not_strict(self): """Get ta does not fail with Omon if not strict.""" self.variables_info.strict = False var = self.variables_info.get_variable('Omon', 'ta') From 2c89beb5d66b49631298cef1788332c7a0162830 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 9 Dec 2019 12:37:18 +0100 Subject: [PATCH 02/11] Update recipe and derivation to use var_name --- esmvalcore/_recipe.py | 53 ++++++++-------- esmvalcore/_recipe_checks.py | 2 +- esmvalcore/cmor/check.py | 32 +++++----- esmvalcore/cmor/fix.py | 34 +++++------ esmvalcore/preprocessor/_derive/__init__.py | 22 +++---- esmvalcore/preprocessor/_derive/alb.py | 4 +- esmvalcore/preprocessor/_derive/amoc.py | 2 +- esmvalcore/preprocessor/_derive/asr.py | 2 +- esmvalcore/preprocessor/_derive/clhmtisccp.py | 2 +- esmvalcore/preprocessor/_derive/clhtkisccp.py | 2 +- esmvalcore/preprocessor/_derive/cllmtisccp.py | 2 +- esmvalcore/preprocessor/_derive/clltkisccp.py | 2 +- esmvalcore/preprocessor/_derive/clmmtisccp.py | 2 +- esmvalcore/preprocessor/_derive/clmtkisccp.py | 2 +- esmvalcore/preprocessor/_derive/ctotal.py | 4 +- esmvalcore/preprocessor/_derive/et.py | 2 +- esmvalcore/preprocessor/_derive/gtfgco2.py | 8 +-- esmvalcore/preprocessor/_derive/lvp.py | 6 +- esmvalcore/preprocessor/_derive/lwcre.py | 4 +- esmvalcore/preprocessor/_derive/lwp.py | 4 +- esmvalcore/preprocessor/_derive/netcre.py | 8 +-- esmvalcore/preprocessor/_derive/ohc.py | 8 +-- esmvalcore/preprocessor/_derive/rlns.py | 4 +- esmvalcore/preprocessor/_derive/rlnst.py | 6 +- esmvalcore/preprocessor/_derive/rlnstcs.py | 6 +- esmvalcore/preprocessor/_derive/rlntcs.py | 2 +- esmvalcore/preprocessor/_derive/rsns.py | 4 +- esmvalcore/preprocessor/_derive/rsnst.py | 8 +-- esmvalcore/preprocessor/_derive/rsnstcs.py | 8 +-- .../preprocessor/_derive/rsnstcsnorm.py | 8 +-- esmvalcore/preprocessor/_derive/rsnt.py | 4 +- esmvalcore/preprocessor/_derive/rsntcs.py | 4 +- esmvalcore/preprocessor/_derive/rtnt.py | 6 +- esmvalcore/preprocessor/_derive/sispeed.py | 4 +- esmvalcore/preprocessor/_derive/sithick.py | 4 +- esmvalcore/preprocessor/_derive/sm.py | 2 +- esmvalcore/preprocessor/_derive/swcre.py | 4 +- esmvalcore/preprocessor/_derive/toz.py | 4 +- esmvalcore/preprocessor/_derive/uajet.py | 2 +- esmvalcore/preprocessor/_derive/vegfrac.py | 2 +- esmvalcore/preprocessor/_io.py | 2 +- esmvalcore/preprocessor/_regrid.py | 6 +- tests/integration/test_recipe.py | 61 ++++++++++--------- tests/unit/cmor/test_cmor_check.py | 4 +- tests/unit/cmor/test_fix.py | 27 ++++---- 45 files changed, 198 insertions(+), 191 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 2277a57b39..68e422120b 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -111,17 +111,18 @@ def _add_cmor_info(variable, override=False): derive = variable.get('derive', False) # Copy the following keys from CMOR table cmor_keys = [ - 'standard_name', 'long_name', 'units', 'modeling_realm', 'frequency' + 'short_name', 'standard_name', 'long_name', 'units', 'modeling_realm', + 'frequency' ] cmor_table = variable['cmor_table'] mip = variable['mip'] - short_name = variable['short_name'] - table_entry = CMOR_TABLES[cmor_table].get_variable(mip, short_name, derive) + var_name = variable['var_name'] + table_entry = CMOR_TABLES[cmor_table].get_variable(mip, var_name, derive) if table_entry is None: raise RecipeError( "Unable to load CMOR table '{}' for variable '{}' with mip '{}'". - format(cmor_table, short_name, mip)) + format(cmor_table, var_name, mip)) for key in cmor_keys: if key not in variable or override: @@ -147,7 +148,7 @@ def _special_name_to_dataset(variable, special_name): variable['preprocessor'], special_name, special_name, - variable['short_name'], + variable['var_name'], variable['diagnostic'], )) special_name = variable[special_name] @@ -184,7 +185,7 @@ def _update_target_levels(variable, variables, settings, config_user): _dataset_to_file(variable_data, config_user) settings['extract_levels']['levels'] = get_reference_levels( filename, variable_data['project'], dataset, - variable_data['short_name'], + variable_data['var_name'], os.path.splitext(variable_data['filename'])[0] + '_fixed') @@ -235,7 +236,7 @@ def _dataset_to_file(variable, config_user): """Find the first file belonging to dataset from variable info.""" files = _get_input_files(variable, config_user) if not files and variable.get('derive'): - required_vars = get_required(variable['short_name'], + required_vars = get_required(variable['var_name'], variable['project']) for required_var in required_vars: _augment(required_var, variable) @@ -298,7 +299,7 @@ def _get_default_settings(variable, config_user, derive=False): fix = { 'project': variable['project'], 'dataset': variable['dataset'], - 'short_name': variable['short_name'], + 'var_name': variable['var_name'], } # File fixes fix_dir = os.path.splitext(variable['filename'])[0] + '_fixed' @@ -326,6 +327,7 @@ def _get_default_settings(variable, config_user, derive=False): if derive: settings['derive'] = { + 'var_name': variable['var_name'], 'short_name': variable['short_name'], 'standard_name': variable['standard_name'], 'long_name': variable['long_name'], @@ -337,7 +339,7 @@ def _get_default_settings(variable, config_user, derive=False): settings['cmor_check_metadata'] = { 'cmor_table': variable['cmor_table'], 'mip': variable['mip'], - 'short_name': variable['short_name'], + 'var_name': variable['var_name'], 'frequency': variable['frequency'], } # Configure final CMOR data check @@ -345,7 +347,7 @@ def _get_default_settings(variable, config_user, derive=False): settings['cmor_check_data'] = { 'cmor_table': variable['cmor_table'], 'mip': variable['mip'], - 'short_name': variable['short_name'], + 'var_name': variable['var_name'], 'frequency': variable['frequency'], } @@ -366,8 +368,8 @@ def _add_fxvar_keys(fx_var_dict, variable): fx_variable = dict(variable) # set variable names - fx_variable['variable_group'] = fx_var_dict['short_name'] - fx_variable['short_name'] = fx_var_dict['short_name'] + fx_variable['variable_group'] = fx_var_dict['var_name'] + fx_variable['var_name'] = fx_var_dict['var_name'] # specificities of project if fx_variable['project'] == 'CMIP5': @@ -388,17 +390,17 @@ def _get_correct_fx_file(variable, fx_varname, config_user): """Wrapper to standard file getter to recover the correct fx file.""" var = dict(variable) if var['project'] in ['CMIP5', 'OBS', 'OBS6', 'obs4mips']: - fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'fx'}, var) + fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'fx'}, var) elif var['project'] == 'CMIP6': if fx_varname == 'sftlf': - fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'fx'}, + fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'fx'}, var) elif fx_varname == 'sftof': - fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Ofx'}, + fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'Ofx'}, var) # TODO allow availability for multiple mip's for sftgif elif fx_varname == 'sftgif': - fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'fx'}, + fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'fx'}, var) else: raise RecipeError( @@ -512,7 +514,7 @@ def _get_ancestors(variable, config_user): input_files = _get_input_files(variable, config_user) logger.info("Using input files for variable %s of dataset %s:\n%s", - variable['short_name'], variable['dataset'], + variable['var_name'], variable['dataset'], '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') or variable['dataset'] == variable.get('reference_dataset')): @@ -800,7 +802,7 @@ def _get_derive_input_variables(variables, config_user): def append(group_prefix, var): """Append variable `var` to a derive input group.""" - group = group_prefix + var['short_name'] + group = group_prefix + var['var_name'] var['variable_group'] = group if group not in derive_input: derive_input[group] = [] @@ -816,10 +818,12 @@ def append(group_prefix, var): append(group_prefix, var) else: # Process input data needed to derive variable - required_vars = get_required(variable['short_name'], + required_vars = get_required(variable['var_name'], variable['project']) for var in required_vars: + print('Before augment: ', var) _augment(var, variable) + print('After augment: ', var) _add_cmor_info(var, override=True) files = _get_input_files(var, config_user) if var.get('optional') and not files: @@ -842,10 +846,10 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): if preproc_name not in profiles: raise RecipeError( "Unknown preprocessor {} in variable {} of diagnostic {}".format( - preproc_name, variable['short_name'], variable['diagnostic'])) + preproc_name, variable['var_name'], variable['diagnostic'])) profile = deepcopy(profiles[variable['preprocessor']]) logger.info("Creating preprocessor '%s' task for variable '%s'", - variable['preprocessor'], variable['short_name']) + variable['preprocessor'], variable['var_name']) variables = _limit_datasets(variables, profile, config_user.get('max_datasets')) for variable in variables: @@ -859,6 +863,7 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): for derive_variables in derive_input.values(): for derive_variable in derive_variables: + logger.info(derive_variable) _add_cmor_info(derive_variable, override=True) derive_name = task_name.split( TASKSEP)[0] + TASKSEP + derive_variables[0]['variable_group'] @@ -1015,7 +1020,7 @@ def _initialize_variables(self, raw_variable, raw_datasets): variables.append(variable) required_keys = { - 'short_name', + 'var_name', 'mip', 'dataset', 'project', @@ -1050,8 +1055,8 @@ def _initialize_preprocessor_output(self, diagnostic_name, raw_variables, else: raw_variable = deepcopy(raw_variable) raw_variable['variable_group'] = variable_group - if 'short_name' not in raw_variable: - raw_variable['short_name'] = variable_group + if 'var_name' not in raw_variable: + raw_variable['var_name'] = variable_group raw_variable['diagnostic'] = diagnostic_name raw_variable['preprocessor'] = str( raw_variable.get('preprocessor', 'default')) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 68a3064875..17a17441b3 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -87,7 +87,7 @@ def variable(var, required_keys): if missing: raise RecipeError( "Missing keys {} from variable {} in diagnostic {}".format( - missing, var.get('short_name'), var.get('diagnostic'))) + missing, var.get('var_name'), var.get('diagnostic'))) def data_availability(input_files, var): diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 735ec1f09b..73651f1ae0 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -665,7 +665,7 @@ def _add_auxiliar_time_coordinates(self): def _get_cmor_checker(table, mip, - short_name, + var_name, frequency, fail_on_error=True, automatic_fixes=False): @@ -677,9 +677,9 @@ def _get_cmor_checker(table, table, ', '.join(CMOR_TABLES))) cmor_table = CMOR_TABLES[table] - var_info = cmor_table.get_variable(mip, short_name) + var_info = cmor_table.get_variable(mip, var_name) if var_info is None: - var_info = CMOR_TABLES['custom'].get_variable(mip, short_name) + var_info = CMOR_TABLES['custom'].get_variable(mip, var_name) def _checker(cube): return CMORCheck( @@ -692,7 +692,7 @@ def _checker(cube): return _checker -def cmor_check_metadata(cube, cmor_table, mip, short_name, frequency): +def cmor_check_metadata(cube, cmor_table, mip, var_name, frequency): """Check if metadata conforms to variable's CMOR definiton. None of the checks at this step will force the cube to load the data. @@ -705,18 +705,18 @@ def cmor_check_metadata(cube, cmor_table, mip, short_name, frequency): CMOR definitions to use. mip: Variable's mip. - short_name: basestring - Variable's short name. + var_name: basestring + Variable's name. frequency: basestring Data frequency. """ - checker = _get_cmor_checker(cmor_table, mip, short_name, frequency) + checker = _get_cmor_checker(cmor_table, mip, var_name, frequency) checker(cube).check_metadata() return cube -def cmor_check_data(cube, cmor_table, mip, short_name, frequency): +def cmor_check_data(cube, cmor_table, mip, var_name, frequency): """Check if data conforms to variable's CMOR definiton. The checks performed at this step require the data in memory. @@ -729,18 +729,18 @@ def cmor_check_data(cube, cmor_table, mip, short_name, frequency): CMOR definitions to use. mip: Variable's mip. - short_name: basestring - Variable's short name + var_name: basestring + Variable's name frequency: basestring Data frequency """ - checker = _get_cmor_checker(cmor_table, mip, short_name, frequency) + checker = _get_cmor_checker(cmor_table, mip, var_name, frequency) checker(cube).check_data() return cube -def cmor_check(cube, cmor_table, mip, short_name, frequency): +def cmor_check(cube, cmor_table, mip, var_name, frequency): """Check if cube conforms to variable's CMOR definiton. Equivalent to calling cmor_check_metadata and cmor_check_data @@ -754,12 +754,12 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency): CMOR definitions to use. mip: Variable's mip. - short_name: basestring - Variable's short name. + var_name: basestring + Variable's name. frequency: basestring Data frequency. """ - cmor_check_metadata(cube, cmor_table, mip, short_name, frequency) - cmor_check_data(cube, cmor_table, mip, short_name, frequency) + cmor_check_metadata(cube, cmor_table, mip, var_name, frequency) + cmor_check_data(cube, cmor_table, mip, var_name, frequency) return cube diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 2881d49aa7..b5a6624745 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -def fix_file(file, short_name, project, dataset, output_dir): +def fix_file(file, var_name, project, dataset, output_dir): """ Fix files before ESMValTool can load them. @@ -31,8 +31,8 @@ def fix_file(file, short_name, project, dataset, output_dir): ---------- file: str Path to the original file - short_name: str - Variable's short name + var_name: str + Variable's cmor name project: str dataset:str output_dir: str @@ -45,13 +45,13 @@ def fix_file(file, short_name, project, dataset, output_dir): """ for fix in Fix.get_fixes( - project=project, dataset=dataset, variable=short_name): + project=project, dataset=dataset, variable=var_name): file = fix.fix_file(file, output_dir) return file def fix_metadata(cubes, - short_name, + var_name, project, dataset, cmor_table=None, @@ -69,8 +69,8 @@ def fix_metadata(cubes, ---------- cubes: iris.cube.CubeList Cubes to fix - short_name: str - Variable's short name + var_name: str + Variable's name project: str dataset: str @@ -96,7 +96,7 @@ def fix_metadata(cubes, """ fixes = Fix.get_fixes( - project=project, dataset=dataset, variable=short_name) + project=project, dataset=dataset, variable=var_name) fixed_cubes = [] by_file = defaultdict(list) for cube in cubes: @@ -110,7 +110,7 @@ def fix_metadata(cubes, if len(cube_list) != 1: cube = None for raw_cube in cube_list: - if raw_cube.var_name == short_name: + if raw_cube.short_name == var_name: cube = raw_cube break if not cube: @@ -118,7 +118,7 @@ def fix_metadata(cubes, 'More than one cube found for variable %s in %s:%s but ' 'none of their var_names match the expected. \n' 'Full list of cubes encountered: %s' % - (short_name, project, dataset, cube_list) + (var_name, project, dataset, cube_list) ) logger.warning( 'Found variable %s in %s:%s, but there were other present in ' @@ -126,7 +126,7 @@ def fix_metadata(cubes, '(cell area, latitude descriptions) that was not saved ' 'properly. It is possible that errors appear further on ' 'because of this. \nFull list of cubes encountered: %s', - short_name, + var_name, project, dataset, cube_list @@ -139,7 +139,7 @@ def fix_metadata(cubes, frequency=frequency, table=cmor_table, mip=mip, - short_name=short_name, + var_name=var_name, fail_on_error=False, automatic_fixes=True) cube = checker(cube).check_metadata() @@ -149,7 +149,7 @@ def fix_metadata(cubes, def fix_data(cube, - short_name, + var_name, project, dataset, cmor_table=None, @@ -169,8 +169,8 @@ def fix_data(cube, ---------- cube: iris.cube.Cube Cube to fix - short_name: str - Variable's short name + var_name: str + Variable's name project: str dataset: str @@ -196,14 +196,14 @@ def fix_data(cube, """ for fix in Fix.get_fixes( - project=project, dataset=dataset, variable=short_name): + project=project, dataset=dataset, variable=var_name): cube = fix.fix_data(cube) if cmor_table and mip: checker = _get_cmor_checker( frequency=frequency, table=cmor_table, mip=mip, - short_name=short_name, + var_name=var_name, fail_on_error=False, automatic_fixes=True) cube = checker(cube).check_data() diff --git a/esmvalcore/preprocessor/_derive/__init__.py b/esmvalcore/preprocessor/_derive/__init__.py index 3d94129fc7..75904576f8 100644 --- a/esmvalcore/preprocessor/_derive/__init__.py +++ b/esmvalcore/preprocessor/_derive/__init__.py @@ -34,15 +34,15 @@ def _get_all_derived_variables(): __all__ = list(ALL_DERIVED_VARIABLES) -def get_required(short_name, project): +def get_required(var_name, project): """Return all required variables for derivation. - Get all information (at least `short_name`) required for derivation. + Get all information (at least `var_name`) required for derivation. Parameters ---------- - short_name : str - `short_name` of the variable to derive. + var_name : str + name of the variable to derive. project : str `project` of the variable to derive. @@ -52,16 +52,16 @@ def get_required(short_name, project): List of dictionaries (including at least the key `short_name`). """ - if short_name not in ALL_DERIVED_VARIABLES: + if var_name not in ALL_DERIVED_VARIABLES: raise NotImplementedError( - f"Cannot derive variable '{short_name}', no derivation script " + f"Cannot derive variable '{var_name}', no derivation script " f"available") - DerivedVariable = ALL_DERIVED_VARIABLES[short_name] # noqa: N806 + DerivedVariable = ALL_DERIVED_VARIABLES[var_name] # noqa: N806 variables = deepcopy(DerivedVariable().required(project)) return variables -def derive(cubes, short_name, long_name, units, standard_name=None): +def derive(cubes, var_name, short_name, long_name, units, standard_name=None): """Derive variable. Parameters @@ -69,8 +69,8 @@ def derive(cubes, short_name, long_name, units, standard_name=None): cubes: iris.cube.CubeList Includes all the needed variables for derivation defined in :func:`get_required`. - short_name: str - short_name + var_name: str + var_name long_name: str long_name units: str @@ -90,7 +90,7 @@ def derive(cubes, short_name, long_name, units, standard_name=None): cubes = iris.cube.CubeList(cubes) # Derive variable - DerivedVariable = ALL_DERIVED_VARIABLES[short_name.lower()] # noqa: N806 + DerivedVariable = ALL_DERIVED_VARIABLES[var_name.lower()] # noqa: N806 cube = DerivedVariable().calculate(cubes) # Set standard attributes diff --git a/esmvalcore/preprocessor/_derive/alb.py b/esmvalcore/preprocessor/_derive/alb.py index 6b6fd4b930..5331d81d87 100644 --- a/esmvalcore/preprocessor/_derive/alb.py +++ b/esmvalcore/preprocessor/_derive/alb.py @@ -17,10 +17,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsdscs' + 'var_name': 'rsdscs' }, { - 'short_name': 'rsuscs' + 'var_name': 'rsuscs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/amoc.py b/esmvalcore/preprocessor/_derive/amoc.py index 52d4babf1b..4236219052 100644 --- a/esmvalcore/preprocessor/_derive/amoc.py +++ b/esmvalcore/preprocessor/_derive/amoc.py @@ -11,7 +11,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'msftmyz', 'mip': 'Omon'}] + required = [{'var_name': 'msftmyz', 'mip': 'Omon'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/asr.py b/esmvalcore/preprocessor/_derive/asr.py index 060c9d4253..cb58fa72d4 100644 --- a/esmvalcore/preprocessor/_derive/asr.py +++ b/esmvalcore/preprocessor/_derive/asr.py @@ -11,7 +11,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'rsdt'}, {'short_name': 'rsut'}] + required = [{'var_name': 'rsdt'}, {'var_name': 'rsut'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clhmtisccp.py b/esmvalcore/preprocessor/_derive/clhmtisccp.py index b20d1ee0ff..f76b81d51b 100644 --- a/esmvalcore/preprocessor/_derive/clhmtisccp.py +++ b/esmvalcore/preprocessor/_derive/clhmtisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'clisccp'}] + required = [{'var_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clhtkisccp.py b/esmvalcore/preprocessor/_derive/clhtkisccp.py index 9b9b5106e2..32cb2a3275 100644 --- a/esmvalcore/preprocessor/_derive/clhtkisccp.py +++ b/esmvalcore/preprocessor/_derive/clhtkisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'clisccp'}] + required = [{'var_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/cllmtisccp.py b/esmvalcore/preprocessor/_derive/cllmtisccp.py index 28b5104ea8..53efa2c26e 100644 --- a/esmvalcore/preprocessor/_derive/cllmtisccp.py +++ b/esmvalcore/preprocessor/_derive/cllmtisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'clisccp'}] + required = [{'var_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clltkisccp.py b/esmvalcore/preprocessor/_derive/clltkisccp.py index 58d2f9a2b5..a8006c63eb 100644 --- a/esmvalcore/preprocessor/_derive/clltkisccp.py +++ b/esmvalcore/preprocessor/_derive/clltkisccp.py @@ -9,7 +9,7 @@ class DerivedVariable(DerivedVariableBase): """Derivation of variable `clltkisccp`.""" - required = [{'short_name': 'clisccp'}] + required = [{'var_name': 'clisccp'}] @staticmethod def calculate(cubes): diff --git a/esmvalcore/preprocessor/_derive/clmmtisccp.py b/esmvalcore/preprocessor/_derive/clmmtisccp.py index 53129d065e..2916965c82 100644 --- a/esmvalcore/preprocessor/_derive/clmmtisccp.py +++ b/esmvalcore/preprocessor/_derive/clmmtisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'clisccp'}] + required = [{'var_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clmtkisccp.py b/esmvalcore/preprocessor/_derive/clmtkisccp.py index a61314a470..5b711c262e 100644 --- a/esmvalcore/preprocessor/_derive/clmtkisccp.py +++ b/esmvalcore/preprocessor/_derive/clmtkisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'clisccp'}] + required = [{'var_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/ctotal.py b/esmvalcore/preprocessor/_derive/ctotal.py index 76e539e898..e8cddb0671 100644 --- a/esmvalcore/preprocessor/_derive/ctotal.py +++ b/esmvalcore/preprocessor/_derive/ctotal.py @@ -11,10 +11,10 @@ class DerivedVariable(DerivedVariableBase): # Required variables required = [ { - 'short_name': 'cVeg' + 'var_name': 'cVeg' }, { - 'short_name': 'cSoil' + 'var_name': 'cSoil' }, ] diff --git a/esmvalcore/preprocessor/_derive/et.py b/esmvalcore/preprocessor/_derive/et.py index 013d77282b..930b295149 100644 --- a/esmvalcore/preprocessor/_derive/et.py +++ b/esmvalcore/preprocessor/_derive/et.py @@ -15,7 +15,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'hfls', 'mip': 'Amon'}] + required = [{'var_name': 'hfls', 'mip': 'Amon'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/gtfgco2.py b/esmvalcore/preprocessor/_derive/gtfgco2.py index 64efd558b1..92780d1c66 100644 --- a/esmvalcore/preprocessor/_derive/gtfgco2.py +++ b/esmvalcore/preprocessor/_derive/gtfgco2.py @@ -50,22 +50,22 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'fgco2', + 'var_name': 'fgco2', 'mip': 'Omon' }, { - 'short_name': 'areacello', + 'var_name': 'areacello', 'mip': 'fx' }, ] if project == 'CMIP6': required = [ { - 'short_name': 'fgco2', + 'var_name': 'fgco2', 'mip': 'Omon' }, { - 'short_name': 'areacello', + 'var_name': 'areacello', 'mip': 'Ofx' }, ] diff --git a/esmvalcore/preprocessor/_derive/lvp.py b/esmvalcore/preprocessor/_derive/lvp.py index e3f6d7d2c3..622b48345c 100644 --- a/esmvalcore/preprocessor/_derive/lvp.py +++ b/esmvalcore/preprocessor/_derive/lvp.py @@ -17,13 +17,13 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'hfls' + 'var_name': 'hfls' }, { - 'short_name': 'pr' + 'var_name': 'pr' }, { - 'short_name': 'evspsbl' + 'var_name': 'evspsbl' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/lwcre.py b/esmvalcore/preprocessor/_derive/lwcre.py index 9978adf243..851fc0fead 100644 --- a/esmvalcore/preprocessor/_derive/lwcre.py +++ b/esmvalcore/preprocessor/_derive/lwcre.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rlut' + 'var_name': 'rlut' }, { - 'short_name': 'rlutcs' + 'var_name': 'rlutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/lwp.py b/esmvalcore/preprocessor/_derive/lwp.py index 0208fc9d1f..0d75185199 100644 --- a/esmvalcore/preprocessor/_derive/lwp.py +++ b/esmvalcore/preprocessor/_derive/lwp.py @@ -16,10 +16,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'clwvi' + 'var_name': 'clwvi' }, { - 'short_name': 'clivi' + 'var_name': 'clivi' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/netcre.py b/esmvalcore/preprocessor/_derive/netcre.py index 806a79000b..5cf1039107 100644 --- a/esmvalcore/preprocessor/_derive/netcre.py +++ b/esmvalcore/preprocessor/_derive/netcre.py @@ -13,16 +13,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rlut' + 'var_name': 'rlut' }, { - 'short_name': 'rlutcs' + 'var_name': 'rlutcs' }, { - 'short_name': 'rsut' + 'var_name': 'rsut' }, { - 'short_name': 'rsutcs' + 'var_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/ohc.py b/esmvalcore/preprocessor/_derive/ohc.py index 5401a77017..fcbfb70f52 100644 --- a/esmvalcore/preprocessor/_derive/ohc.py +++ b/esmvalcore/preprocessor/_derive/ohc.py @@ -17,20 +17,20 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'thetao' + 'var_name': 'thetao' }, { - 'short_name': 'volcello', + 'var_name': 'volcello', 'mip': 'fx' }, ] if project == 'CMIP6': required = [ { - 'short_name': 'thetao' + 'var_name': 'thetao' }, { - 'short_name': 'volcello', + 'var_name': 'volcello', 'mip': 'Ofx' }, ] diff --git a/esmvalcore/preprocessor/_derive/rlns.py b/esmvalcore/preprocessor/_derive/rlns.py index d3e3f5330d..1131272e05 100644 --- a/esmvalcore/preprocessor/_derive/rlns.py +++ b/esmvalcore/preprocessor/_derive/rlns.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rlds' + 'var_name': 'rlds' }, { - 'short_name': 'rlus' + 'var_name': 'rlus' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rlnst.py b/esmvalcore/preprocessor/_derive/rlnst.py index 149f1e4cbb..57a7fc735b 100644 --- a/esmvalcore/preprocessor/_derive/rlnst.py +++ b/esmvalcore/preprocessor/_derive/rlnst.py @@ -17,13 +17,13 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rlds' + 'var_name': 'rlds' }, { - 'short_name': 'rlus' + 'var_name': 'rlus' }, { - 'short_name': 'rlut' + 'var_name': 'rlut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rlnstcs.py b/esmvalcore/preprocessor/_derive/rlnstcs.py index 454fbcbf55..41e059f227 100644 --- a/esmvalcore/preprocessor/_derive/rlnstcs.py +++ b/esmvalcore/preprocessor/_derive/rlnstcs.py @@ -14,13 +14,13 @@ class DerivedVariable(DerivedVariableBase): required = [ { - 'short_name': 'rldscs' + 'var_name': 'rldscs' }, { - 'short_name': 'rlus' + 'var_name': 'rlus' }, { - 'short_name': 'rlutcs' + 'var_name': 'rlutcs' }, ] diff --git a/esmvalcore/preprocessor/_derive/rlntcs.py b/esmvalcore/preprocessor/_derive/rlntcs.py index a53ed6a9a4..1672897a86 100644 --- a/esmvalcore/preprocessor/_derive/rlntcs.py +++ b/esmvalcore/preprocessor/_derive/rlntcs.py @@ -10,7 +10,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'rlutcs'}] + required = [{'var_name': 'rlutcs'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/rsns.py b/esmvalcore/preprocessor/_derive/rsns.py index 4b6cb9441b..1c145e9ab7 100644 --- a/esmvalcore/preprocessor/_derive/rsns.py +++ b/esmvalcore/preprocessor/_derive/rsns.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsds' + 'var_name': 'rsds' }, { - 'short_name': 'rsus' + 'var_name': 'rsus' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsnst.py b/esmvalcore/preprocessor/_derive/rsnst.py index 5f7154512b..21eda3e487 100644 --- a/esmvalcore/preprocessor/_derive/rsnst.py +++ b/esmvalcore/preprocessor/_derive/rsnst.py @@ -17,16 +17,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsds' + 'var_name': 'rsds' }, { - 'short_name': 'rsdt' + 'var_name': 'rsdt' }, { - 'short_name': 'rsus' + 'var_name': 'rsus' }, { - 'short_name': 'rsut' + 'var_name': 'rsut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsnstcs.py b/esmvalcore/preprocessor/_derive/rsnstcs.py index 65f8f5b6e4..8a1a9ebefa 100644 --- a/esmvalcore/preprocessor/_derive/rsnstcs.py +++ b/esmvalcore/preprocessor/_derive/rsnstcs.py @@ -17,16 +17,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsdscs' + 'var_name': 'rsdscs' }, { - 'short_name': 'rsdt' + 'var_name': 'rsdt' }, { - 'short_name': 'rsuscs' + 'var_name': 'rsuscs' }, { - 'short_name': 'rsutcs' + 'var_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsnstcsnorm.py b/esmvalcore/preprocessor/_derive/rsnstcsnorm.py index 2f4d2241e0..2abc2fab12 100644 --- a/esmvalcore/preprocessor/_derive/rsnstcsnorm.py +++ b/esmvalcore/preprocessor/_derive/rsnstcsnorm.py @@ -17,16 +17,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsdscs' + 'var_name': 'rsdscs' }, { - 'short_name': 'rsdt' + 'var_name': 'rsdt' }, { - 'short_name': 'rsuscs' + 'var_name': 'rsuscs' }, { - 'short_name': 'rsutcs' + 'var_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsnt.py b/esmvalcore/preprocessor/_derive/rsnt.py index 343a995b16..e6591bdb8f 100644 --- a/esmvalcore/preprocessor/_derive/rsnt.py +++ b/esmvalcore/preprocessor/_derive/rsnt.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsdt' + 'var_name': 'rsdt' }, { - 'short_name': 'rsut' + 'var_name': 'rsut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsntcs.py b/esmvalcore/preprocessor/_derive/rsntcs.py index c531dd0fd7..b603a8eee5 100644 --- a/esmvalcore/preprocessor/_derive/rsntcs.py +++ b/esmvalcore/preprocessor/_derive/rsntcs.py @@ -12,10 +12,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsdt' + 'var_name': 'rsdt' }, { - 'short_name': 'rsutcs' + 'var_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rtnt.py b/esmvalcore/preprocessor/_derive/rtnt.py index a6b9ed5a8a..5ff1ec50ed 100644 --- a/esmvalcore/preprocessor/_derive/rtnt.py +++ b/esmvalcore/preprocessor/_derive/rtnt.py @@ -13,13 +13,13 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsdt' + 'var_name': 'rsdt' }, { - 'short_name': 'rsut' + 'var_name': 'rsut' }, { - 'short_name': 'rlut' + 'var_name': 'rlut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/sispeed.py b/esmvalcore/preprocessor/_derive/sispeed.py index 54abae95ee..4bba38483c 100644 --- a/esmvalcore/preprocessor/_derive/sispeed.py +++ b/esmvalcore/preprocessor/_derive/sispeed.py @@ -17,9 +17,9 @@ class DerivedVariable(DerivedVariableBase): def required(project): """Declare the variables needed for derivation.""" required = [{ - 'short_name': 'usi', + 'var_name': 'usi', }, { - 'short_name': 'vsi', + 'var_name': 'vsi', }] return required diff --git a/esmvalcore/preprocessor/_derive/sithick.py b/esmvalcore/preprocessor/_derive/sithick.py index 365248af41..bee89f3495 100644 --- a/esmvalcore/preprocessor/_derive/sithick.py +++ b/esmvalcore/preprocessor/_derive/sithick.py @@ -12,9 +12,9 @@ class DerivedVariable(DerivedVariableBase): def required(project): """Declare the variables needed for derivation.""" required = [{ - 'short_name': 'sit', + 'var_name': 'sit', }, { - 'short_name': 'sic', + 'var_name': 'sic', }] return required diff --git a/esmvalcore/preprocessor/_derive/sm.py b/esmvalcore/preprocessor/_derive/sm.py index 45b3ddb074..c3714d285e 100644 --- a/esmvalcore/preprocessor/_derive/sm.py +++ b/esmvalcore/preprocessor/_derive/sm.py @@ -13,7 +13,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'mrsos'}] + required = [{'var_name': 'mrsos'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/swcre.py b/esmvalcore/preprocessor/_derive/swcre.py index 5108f6bfd7..a0ebb405ea 100644 --- a/esmvalcore/preprocessor/_derive/swcre.py +++ b/esmvalcore/preprocessor/_derive/swcre.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'rsut' + 'var_name': 'rsut' }, { - 'short_name': 'rsutcs' + 'var_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/toz.py b/esmvalcore/preprocessor/_derive/toz.py index 10a8adf388..6448e0c885 100644 --- a/esmvalcore/preprocessor/_derive/toz.py +++ b/esmvalcore/preprocessor/_derive/toz.py @@ -28,10 +28,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'short_name': 'tro3' + 'var_name': 'tro3' }, { - 'short_name': 'ps' + 'var_name': 'ps' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/uajet.py b/esmvalcore/preprocessor/_derive/uajet.py index 72af4193a9..a04caecea3 100644 --- a/esmvalcore/preprocessor/_derive/uajet.py +++ b/esmvalcore/preprocessor/_derive/uajet.py @@ -17,7 +17,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'short_name': 'ua'}] + required = [{'var_name': 'ua'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/vegfrac.py b/esmvalcore/preprocessor/_derive/vegfrac.py index 9402b48ad3..3446855a76 100644 --- a/esmvalcore/preprocessor/_derive/vegfrac.py +++ b/esmvalcore/preprocessor/_derive/vegfrac.py @@ -8,7 +8,7 @@ class DerivedVariable(DerivedVariableBase): """Derivation of variable `vegFrac`.""" required = [{ - 'short_name': 'baresoilFrac', + 'var_name': 'baresoilFrac', }] @staticmethod diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 22dc6b1c58..d05ad151a4 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -265,7 +265,7 @@ def _write_ncl_metadata(output_dir, metadata): variable_info[key] = variable[key] filename = os.path.join(output_dir, - variable_info['short_name'] + '_info.ncl') + variable_info['var_name'] + '_info.ncl') write_ncl_settings(info, filename) return filename diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 58a96491d4..91efeaa0d1 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -496,7 +496,7 @@ def get_cmor_levels(cmor_table, coordinate): def get_reference_levels(filename, project, dataset, - short_name, + var_name, fix_dir): """Get level definition from a CMOR coordinate. @@ -516,9 +516,9 @@ def get_reference_levels(filename, levels or the string is badly formatted. """ - filename = fix_file(filename, short_name, project, dataset, fix_dir) + filename = fix_file(filename, var_name, project, dataset, fix_dir) cubes = load(filename, callback=concatenate_callback) - cubes = fix_metadata(cubes, short_name, project, dataset) + cubes = fix_metadata(cubes, var_name, project, dataset) cube = cubes[0] try: coord = cube.coord(axis='Z') diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 326228e292..1ef8fe0714 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -31,6 +31,7 @@ 'modeling_realm', 'preprocessor', 'project', + 'var_name', 'short_name', 'standard_name', 'start_year', @@ -157,8 +158,8 @@ def get_required(short_name, _): if short_name != 'tas': assert False required = [ - {'short_name': 'pr'}, - {'short_name': 'areacella', 'mip': 'fx', 'optional': True}, + {'var_name': 'pr'}, + {'var_name': 'areacella', 'mip': 'fx', 'optional': True}, ] return required @@ -243,7 +244,7 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): for variable in variables: for key in MANDATORY_DATASET_KEYS: assert key in variable and variable[key] - assert variable_name == variable['short_name'] + assert variable_name == variable['var_name'] # Check that the correct tasks have been created variables = recipe.diagnostics['diagnostic_name']['preprocessor_output'][ @@ -347,13 +348,13 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_file': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'chl', + 'var_name': 'chl', 'output_dir': fix_dir, }, 'fix_data': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'chl', + 'var_name': 'chl', 'cmor_table': 'CMIP5', 'mip': 'Oyr', 'frequency': 'yr', @@ -361,7 +362,7 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_metadata': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'chl', + 'var_name': 'chl', 'cmor_table': 'CMIP5', 'mip': 'Oyr', 'frequency': 'yr', @@ -377,13 +378,13 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'cmor_check_metadata': { 'cmor_table': 'CMIP5', 'mip': 'Oyr', - 'short_name': 'chl', + 'var_name': 'chl', 'frequency': 'yr', }, 'cmor_check_data': { 'cmor_table': 'CMIP5', 'mip': 'Oyr', - 'short_name': 'chl', + 'var_name': 'chl', 'frequency': 'yr', }, 'cleanup': { @@ -434,13 +435,13 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_file': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'sftlf', + 'var_name': 'sftlf', 'output_dir': fix_dir, }, 'fix_data': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'sftlf', + 'var_name': 'sftlf', 'cmor_table': 'CMIP5', 'mip': 'fx', 'frequency': 'fx', @@ -448,7 +449,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_metadata': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'short_name': 'sftlf', + 'var_name': 'sftlf', 'cmor_table': 'CMIP5', 'mip': 'fx', 'frequency': 'fx', @@ -456,13 +457,13 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'cmor_check_metadata': { 'cmor_table': 'CMIP5', 'mip': 'fx', - 'short_name': 'sftlf', + 'var_name': 'sftlf', 'frequency': 'fx', }, 'cmor_check_data': { 'cmor_table': 'CMIP5', 'mip': 'fx', - 'short_name': 'sftlf', + 'var_name': 'sftlf', 'frequency': 'fx', }, 'cleanup': { @@ -499,7 +500,7 @@ def test_empty_variable(tmp_path, patched_datafinder, config_user): task = recipe.tasks.pop() assert len(task.products) == 1 product = task.products.pop() - assert product.attributes['short_name'] == 'pr' + assert product.attributes['var_name'] == 'pr' assert product.attributes['dataset'] == 'CanESM2' @@ -538,7 +539,7 @@ def test_cmip5_variable_autocomplete(tmp_path, patched_datafinder, 'modeling_realm': ['atmos'], 'preprocessor': 'default', 'project': 'CMIP5', - 'short_name': 'pr', + 'var_name': 'pr', 'standard_name': 'precipitation_flux', 'start_year': 2000, 'units': 'kg m-2 s-1', @@ -585,7 +586,7 @@ def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, 'modeling_realm': ['atmos'], 'preprocessor': 'default', 'project': 'CMIP6', - 'short_name': 'pr', + 'var_name': 'pr', 'standard_name': 'precipitation_flux', 'start_year': 2000, 'units': 'kg m-2 s-1', @@ -714,7 +715,7 @@ def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): diagnostic_name: variables: chl_default: &chl - short_name: chl + var_name: chl preprocessor: default project: CMIP5 mip: Oyr @@ -781,13 +782,13 @@ def test_derive(tmp_path, patched_datafinder, config_user): assert len(task.products) == 1 product = task.products.pop() assert 'derive' in product.settings - assert product.attributes['short_name'] == 'toz' + assert product.attributes['var_name'] == 'toz' assert product.files ps_product = next(p for a in task.ancestors for p in a.products - if p.attributes['short_name'] == 'ps') + if p.attributes['var_name'] == 'ps') tro3_product = next(p for a in task.ancestors for p in a.products - if p.attributes['short_name'] == 'tro3') + if p.attributes['var_name'] == 'tro3') assert ps_product.filename in product.files assert tro3_product.filename in product.files @@ -825,13 +826,13 @@ def test_derive_not_needed(tmp_path, patched_datafinder, config_user): # Check product content of tasks assert len(task.products) == 1 product = task.products.pop() - assert product.attributes['short_name'] == 'toz' + assert product.attributes['var_name'] == 'toz' assert 'derive' in product.settings assert len(ancestor.products) == 1 ancestor_product = ancestor.products.pop() assert ancestor_product.filename in product.files - assert ancestor_product.attributes['short_name'] == 'toz' + assert ancestor_product.attributes['var_name'] == 'toz' assert 'derive' not in ancestor_product.settings # Check that fixes are applied just once @@ -874,7 +875,7 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings - assert product.attributes['short_name'] == 'ohc' + assert product.attributes['var_name'] == 'ohc' all_product_files.extend(product.files) # Check ancestors @@ -884,10 +885,10 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert task.ancestors[1].name == ( 'diagnostic_name/ohc_derive_input_volcello') for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['short_name'] == 'thetao' + assert ancestor_product.attributes['var_name'] == 'thetao' assert ancestor_product.filename in all_product_files for ancestor_product in task.ancestors[1].products: - assert ancestor_product.attributes['short_name'] == 'volcello' + assert ancestor_product.attributes['var_name'] == 'volcello' if ancestor_product.attributes['project'] == 'CMIP6': assert ancestor_product.attributes['mip'] == 'Ofx' else: @@ -958,7 +959,7 @@ def test_derive_with_optional_var(tmp_path, assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings - assert product.attributes['short_name'] == 'tas' + assert product.attributes['var_name'] == 'tas' all_product_files.extend(product.files) # Check ancestors @@ -968,10 +969,10 @@ def test_derive_with_optional_var(tmp_path, assert task.ancestors[1].name == ( 'diagnostic_name/tas_derive_input_areacella') for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['short_name'] == 'pr' + assert ancestor_product.attributes['var_name'] == 'pr' assert ancestor_product.filename in all_product_files for ancestor_product in task.ancestors[1].products: - assert ancestor_product.attributes['short_name'] == 'areacella' + assert ancestor_product.attributes['var_name'] == 'areacella' assert ancestor_product.filename in all_product_files @@ -1011,7 +1012,7 @@ def test_derive_with_optional_var_nodata(tmp_path, assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings - assert product.attributes['short_name'] == 'tas' + assert product.attributes['var_name'] == 'tas' all_product_files.extend(product.files) # Check ancestors @@ -1019,7 +1020,7 @@ def test_derive_with_optional_var_nodata(tmp_path, assert task.ancestors[0].name == ( 'diagnostic_name/tas_derive_input_pr') for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['short_name'] == 'pr' + assert ancestor_product.attributes['var_name'] == 'pr' assert ancestor_product.filename in all_product_files diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 9caa5beea8..e0bcf57fd7 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -173,7 +173,7 @@ def test_check_bad_standard_name_auto_fix(self): self._check_cube() def test_check_bad_standard_name(self): - """Test check fails for a bad short_name.""" + """Test check fails for a bad var_name.""" self.cube = self.get_cube(self.var_info) self.cube.standard_name = 'wind_speed' self._check_fails_in_metadata(automatic_fixes=False) @@ -192,7 +192,7 @@ def test_check_bad_long_name_auto_fix_report_warning(self): self._check_warnings_on_metadata(automatic_fixes=True) def test_check_bad_long_name(self): - """Test check fails for a bad short_name.""" + """Test check fails for a bad var_name.""" self.cube = self.get_cube(self.var_info) self.cube.long_name = 'bad_name' self._check_fails_in_metadata() diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index a449c822b2..3f10775ce9 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -21,7 +21,7 @@ def test_fix(self): with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[self.mock_fix]): - file_returned = fix_file('filename', 'short_name', 'project', + file_returned = fix_file('filename', 'var_name', 'project', 'model', 'output_dir') self.assertNotEqual(file_returned, self.filename) self.assertEqual(file_returned, 'new_filename') @@ -30,7 +30,7 @@ def test_nofix(self): """Check that the same file is returned if no fix is available.""" with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - file_returned = fix_file('filename', 'short_name', 'project', + file_returned = fix_file('filename', 'var_name', 'project', 'model', 'output_dir') self.assertEqual(file_returned, self.filename) @@ -80,9 +80,10 @@ def setUp(self): self.mock_fix.fix_metadata.return_value = [self.fixed_cube] @staticmethod - def _create_mock_cube(var_name='short_name'): + def _create_mock_cube(var_name='var_name'): cube = mock.Mock() cube.var_name = var_name + cube.short_name = var_name cube.attributes = {'source_file': 'source_file'} return cube @@ -91,7 +92,7 @@ def test_fix(self): with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[self.mock_fix]): - cube_returned = fix_metadata([self.cube], 'short_name', 'project', + cube_returned = fix_metadata([self.cube], 'var_name', 'project', 'model')[0] self.assertTrue(cube_returned is not self.cube) self.assertTrue(cube_returned is self.fixed_cube) @@ -100,7 +101,7 @@ def test_nofix(self): """Check that the same cube is returned if no fix is available.""" with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_metadata([self.cube], 'short_name', 'project', + cube_returned = fix_metadata([self.cube], 'var_name', 'project', 'model')[0] self.assertTrue(cube_returned is self.cube) self.assertTrue(cube_returned is not self.fixed_cube) @@ -111,7 +112,7 @@ def test_select_var(self): 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): cube_returned = fix_metadata( [self.cube, self._create_mock_cube('extra')], - 'short_name', + 'var_name', 'project', 'model' )[0] @@ -127,7 +128,7 @@ def test_select_var_failed_if_bad_var_name(self): self._create_mock_cube('not_me'), self._create_mock_cube('me_neither') ], - 'short_name', + 'var_name', 'project', 'model' ) @@ -141,14 +142,14 @@ def test_cmor_checker_called(self): with mock.patch( 'esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_metadata([self.cube], 'short_name', 'project', 'model', + fix_metadata([self.cube], 'var_name', 'project', 'model', 'cmor_table', 'mip', 'frequency') get_mock.assert_called_once_with( automatic_fixes=True, fail_on_error=False, frequency='frequency', mip='mip', - short_name='short_name', + var_name='var_name', table='cmor_table') checker.assert_called_once_with(self.cube) checker.return_value.check_metadata.assert_called_once_with() @@ -169,7 +170,7 @@ def test_fix(self): with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[self.mock_fix]): - cube_returned = fix_data(self.cube, 'short_name', 'project', + cube_returned = fix_data(self.cube, 'var_name', 'project', 'model') self.assertTrue(cube_returned is not self.cube) self.assertTrue(cube_returned is self.fixed_cube) @@ -178,7 +179,7 @@ def test_nofix(self): """Check that the same cube is returned if no fix is available.""" with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_data(self.cube, 'short_name', 'project', + cube_returned = fix_data(self.cube, 'var_name', 'project', 'model') self.assertTrue(cube_returned is self.cube) self.assertTrue(cube_returned is not self.fixed_cube) @@ -192,14 +193,14 @@ def test_cmor_checker_called(self): with mock.patch( 'esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_data(self.cube, 'short_name', 'project', 'model', + fix_data(self.cube, 'var_name', 'project', 'model', 'cmor_table', 'mip', 'frequency') get_mock.assert_called_once_with( automatic_fixes=True, fail_on_error=False, frequency='frequency', mip='mip', - short_name='short_name', + var_name='var_name', table='cmor_table') checker.assert_called_once_with(self.cube) checker.return_value.check_data.assert_called_once_with() From 45d52e80f042c6863cf2dcd75964417eab10080b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 9 Dec 2019 12:39:35 +0100 Subject: [PATCH 03/11] Remove debug messages --- esmvalcore/_recipe.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 68e422120b..5ba9cb8e9e 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -821,9 +821,7 @@ def append(group_prefix, var): required_vars = get_required(variable['var_name'], variable['project']) for var in required_vars: - print('Before augment: ', var) _augment(var, variable) - print('After augment: ', var) _add_cmor_info(var, override=True) files = _get_input_files(var, config_user) if var.get('optional') and not files: From 5d811b29a917ee1ecfac3066a840297922a88d85 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 9 Dec 2019 12:46:19 +0100 Subject: [PATCH 04/11] Fix interface tests --- esmvalcore/_recipe.py | 1 - .../preprocessor/_derive/test_interface.py | 13 +++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 5ba9cb8e9e..f720bed662 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -861,7 +861,6 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): for derive_variables in derive_input.values(): for derive_variable in derive_variables: - logger.info(derive_variable) _add_cmor_info(derive_variable, override=True) derive_name = task_name.split( TASKSEP)[0] + TASKSEP + derive_variables[0]['variable_group'] diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index 7bc27858cc..17e9c640d1 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -11,10 +11,10 @@ def test_get_required(): reference = [ { - 'short_name': 'rsdscs', + 'var_name': 'rsdscs', }, { - 'short_name': 'rsuscs', + 'var_name': 'rsuscs', }, ] @@ -26,8 +26,8 @@ def test_get_required_with_fx(): variables = get_required('ohc', 'CMIP5') reference = [ - {'short_name': 'thetao'}, - {'short_name': 'volcello', 'mip': 'fx'}, + {'var_name': 'thetao'}, + {'var_name': 'volcello', 'mip': 'fx'}, ] assert variables == reference @@ -50,7 +50,7 @@ def test_derive_nonstandard_nofx(): cubes = CubeList([rsdscs, rsuscs]) - alb = derive(cubes, short_name, long_name, units, standard_name) + alb = derive(cubes, short_name, short_name, long_name, units, standard_name) print(alb) assert alb.var_name == short_name @@ -66,7 +66,7 @@ def test_derive_noop(): alb.long_name = 'albedo at the surface' alb.units = 1 - cube = derive([alb], alb.var_name, alb.long_name, alb.units) + cube = derive([alb], alb.var_name, alb.var_name, alb.long_name, alb.units) print(cube) assert cube is alb @@ -90,6 +90,7 @@ def mock_calculate(self, cubes): derive( [ohc_cube], short_name, + short_name, long_name, units, ) From 81ca461703a6679040a2c8b741ecf919e5fe9861 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 9 Dec 2019 13:00:44 +0100 Subject: [PATCH 05/11] Fix format --- tests/integration/preprocessor/_derive/test_interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index 17e9c640d1..1f117faf6a 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -50,7 +50,8 @@ def test_derive_nonstandard_nofx(): cubes = CubeList([rsdscs, rsuscs]) - alb = derive(cubes, short_name, short_name, long_name, units, standard_name) + alb = derive( + cubes, short_name, short_name, long_name, units, standard_name) print(alb) assert alb.var_name == short_name From bdbbf77c2b935a919053647baf7f01ac919a373d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 9 Jan 2020 12:37:04 +0100 Subject: [PATCH 06/11] Change var_name to cmor_name --- esmvalcore/_recipe.py | 48 +++++++------- esmvalcore/_recipe_checks.py | 2 +- esmvalcore/cmor/check.py | 26 ++++---- esmvalcore/cmor/fix.py | 28 ++++----- esmvalcore/cmor/table.py | 12 ++-- esmvalcore/preprocessor/_derive/__init__.py | 18 +++--- esmvalcore/preprocessor/_derive/alb.py | 4 +- esmvalcore/preprocessor/_derive/amoc.py | 2 +- esmvalcore/preprocessor/_derive/asr.py | 2 +- esmvalcore/preprocessor/_derive/clhmtisccp.py | 2 +- esmvalcore/preprocessor/_derive/clhtkisccp.py | 2 +- esmvalcore/preprocessor/_derive/clltkisccp.py | 2 +- esmvalcore/preprocessor/_derive/clmmtisccp.py | 2 +- esmvalcore/preprocessor/_derive/clmtkisccp.py | 2 +- esmvalcore/preprocessor/_derive/ctotal.py | 4 +- esmvalcore/preprocessor/_derive/et.py | 2 +- esmvalcore/preprocessor/_derive/gtfgco2.py | 8 +-- esmvalcore/preprocessor/_derive/lvp.py | 6 +- esmvalcore/preprocessor/_derive/lwcre.py | 4 +- esmvalcore/preprocessor/_derive/lwp.py | 4 +- esmvalcore/preprocessor/_derive/netcre.py | 8 +-- esmvalcore/preprocessor/_derive/ohc.py | 8 +-- esmvalcore/preprocessor/_derive/rlns.py | 4 +- esmvalcore/preprocessor/_derive/rlnst.py | 6 +- esmvalcore/preprocessor/_derive/rlnstcs.py | 6 +- esmvalcore/preprocessor/_derive/rlntcs.py | 2 +- esmvalcore/preprocessor/_derive/rsns.py | 4 +- esmvalcore/preprocessor/_derive/rsnst.py | 8 +-- esmvalcore/preprocessor/_derive/rsnstcs.py | 8 +-- esmvalcore/preprocessor/_derive/rtnt.py | 6 +- esmvalcore/preprocessor/_derive/sispeed.py | 4 +- esmvalcore/preprocessor/_derive/sithick.py | 4 +- esmvalcore/preprocessor/_derive/sm.py | 2 +- esmvalcore/preprocessor/_derive/toz.py | 4 +- esmvalcore/preprocessor/_derive/uajet.py | 2 +- esmvalcore/preprocessor/_derive/vegfrac.py | 2 +- esmvalcore/preprocessor/_io.py | 2 +- esmvalcore/preprocessor/_regrid.py | 6 +- tests/integration/cmor/test_table.py | 2 +- .../preprocessor/_derive/test_interface.py | 8 +-- tests/integration/test_recipe.py | 62 +++++++++---------- tests/unit/cmor/test_cmor_check.py | 4 +- tests/unit/cmor/test_fix.py | 29 +++++---- 43 files changed, 185 insertions(+), 186 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index f720bed662..355eea395b 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -116,13 +116,13 @@ def _add_cmor_info(variable, override=False): ] cmor_table = variable['cmor_table'] mip = variable['mip'] - var_name = variable['var_name'] - table_entry = CMOR_TABLES[cmor_table].get_variable(mip, var_name, derive) + cmor_name = variable['cmor_name'] + table_entry = CMOR_TABLES[cmor_table].get_variable(mip, cmor_name, derive) if table_entry is None: raise RecipeError( "Unable to load CMOR table '{}' for variable '{}' with mip '{}'". - format(cmor_table, var_name, mip)) + format(cmor_table, cmor_name, mip)) for key in cmor_keys: if key not in variable or override: @@ -148,7 +148,7 @@ def _special_name_to_dataset(variable, special_name): variable['preprocessor'], special_name, special_name, - variable['var_name'], + variable['cmor_name'], variable['diagnostic'], )) special_name = variable[special_name] @@ -185,7 +185,7 @@ def _update_target_levels(variable, variables, settings, config_user): _dataset_to_file(variable_data, config_user) settings['extract_levels']['levels'] = get_reference_levels( filename, variable_data['project'], dataset, - variable_data['var_name'], + variable_data['cmor_name'], os.path.splitext(variable_data['filename'])[0] + '_fixed') @@ -236,7 +236,7 @@ def _dataset_to_file(variable, config_user): """Find the first file belonging to dataset from variable info.""" files = _get_input_files(variable, config_user) if not files and variable.get('derive'): - required_vars = get_required(variable['var_name'], + required_vars = get_required(variable['cmor_name'], variable['project']) for required_var in required_vars: _augment(required_var, variable) @@ -299,7 +299,7 @@ def _get_default_settings(variable, config_user, derive=False): fix = { 'project': variable['project'], 'dataset': variable['dataset'], - 'var_name': variable['var_name'], + 'cmor_name': variable['cmor_name'], } # File fixes fix_dir = os.path.splitext(variable['filename'])[0] + '_fixed' @@ -327,7 +327,7 @@ def _get_default_settings(variable, config_user, derive=False): if derive: settings['derive'] = { - 'var_name': variable['var_name'], + 'cmor_name': variable['cmor_name'], 'short_name': variable['short_name'], 'standard_name': variable['standard_name'], 'long_name': variable['long_name'], @@ -339,7 +339,7 @@ def _get_default_settings(variable, config_user, derive=False): settings['cmor_check_metadata'] = { 'cmor_table': variable['cmor_table'], 'mip': variable['mip'], - 'var_name': variable['var_name'], + 'cmor_name': variable['cmor_name'], 'frequency': variable['frequency'], } # Configure final CMOR data check @@ -347,7 +347,7 @@ def _get_default_settings(variable, config_user, derive=False): settings['cmor_check_data'] = { 'cmor_table': variable['cmor_table'], 'mip': variable['mip'], - 'var_name': variable['var_name'], + 'cmor_name': variable['cmor_name'], 'frequency': variable['frequency'], } @@ -368,8 +368,8 @@ def _add_fxvar_keys(fx_var_dict, variable): fx_variable = dict(variable) # set variable names - fx_variable['variable_group'] = fx_var_dict['var_name'] - fx_variable['var_name'] = fx_var_dict['var_name'] + fx_variable['variable_group'] = fx_var_dict['cmor_name'] + fx_variable['cmor_name'] = fx_var_dict['cmor_name'] # specificities of project if fx_variable['project'] == 'CMIP5': @@ -390,17 +390,17 @@ def _get_correct_fx_file(variable, fx_varname, config_user): """Wrapper to standard file getter to recover the correct fx file.""" var = dict(variable) if var['project'] in ['CMIP5', 'OBS', 'OBS6', 'obs4mips']: - fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'fx'}, var) + fx_var = _add_fxvar_keys({'cmor_name': fx_varname, 'mip': 'fx'}, var) elif var['project'] == 'CMIP6': if fx_varname == 'sftlf': - fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'fx'}, + fx_var = _add_fxvar_keys({'cmor_name': fx_varname, 'mip': 'fx'}, var) elif fx_varname == 'sftof': - fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'Ofx'}, + fx_var = _add_fxvar_keys({'cmor_name': fx_varname, 'mip': 'Ofx'}, var) # TODO allow availability for multiple mip's for sftgif elif fx_varname == 'sftgif': - fx_var = _add_fxvar_keys({'var_name': fx_varname, 'mip': 'fx'}, + fx_var = _add_fxvar_keys({'cmor_name': fx_varname, 'mip': 'fx'}, var) else: raise RecipeError( @@ -514,7 +514,7 @@ def _get_ancestors(variable, config_user): input_files = _get_input_files(variable, config_user) logger.info("Using input files for variable %s of dataset %s:\n%s", - variable['var_name'], variable['dataset'], + variable['cmor_name'], variable['dataset'], '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') or variable['dataset'] == variable.get('reference_dataset')): @@ -802,7 +802,7 @@ def _get_derive_input_variables(variables, config_user): def append(group_prefix, var): """Append variable `var` to a derive input group.""" - group = group_prefix + var['var_name'] + group = group_prefix + var['cmor_name'] var['variable_group'] = group if group not in derive_input: derive_input[group] = [] @@ -818,7 +818,7 @@ def append(group_prefix, var): append(group_prefix, var) else: # Process input data needed to derive variable - required_vars = get_required(variable['var_name'], + required_vars = get_required(variable['cmor_name'], variable['project']) for var in required_vars: _augment(var, variable) @@ -844,10 +844,10 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): if preproc_name not in profiles: raise RecipeError( "Unknown preprocessor {} in variable {} of diagnostic {}".format( - preproc_name, variable['var_name'], variable['diagnostic'])) + preproc_name, variable['cmor_name'], variable['diagnostic'])) profile = deepcopy(profiles[variable['preprocessor']]) logger.info("Creating preprocessor '%s' task for variable '%s'", - variable['preprocessor'], variable['var_name']) + variable['preprocessor'], variable['cmor_name']) variables = _limit_datasets(variables, profile, config_user.get('max_datasets')) for variable in variables: @@ -1017,7 +1017,7 @@ def _initialize_variables(self, raw_variable, raw_datasets): variables.append(variable) required_keys = { - 'var_name', + 'cmor_name', 'mip', 'dataset', 'project', @@ -1052,8 +1052,8 @@ def _initialize_preprocessor_output(self, diagnostic_name, raw_variables, else: raw_variable = deepcopy(raw_variable) raw_variable['variable_group'] = variable_group - if 'var_name' not in raw_variable: - raw_variable['var_name'] = variable_group + if 'cmor_name' not in raw_variable: + raw_variable['cmor_name'] = variable_group raw_variable['diagnostic'] = diagnostic_name raw_variable['preprocessor'] = str( raw_variable.get('preprocessor', 'default')) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 17a17441b3..c6fedb397e 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -87,7 +87,7 @@ def variable(var, required_keys): if missing: raise RecipeError( "Missing keys {} from variable {} in diagnostic {}".format( - missing, var.get('var_name'), var.get('diagnostic'))) + missing, var.get('cmor_name'), var.get('diagnostic'))) def data_availability(input_files, var): diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 73651f1ae0..df30346d69 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -665,7 +665,7 @@ def _add_auxiliar_time_coordinates(self): def _get_cmor_checker(table, mip, - var_name, + cmor_name, frequency, fail_on_error=True, automatic_fixes=False): @@ -677,9 +677,9 @@ def _get_cmor_checker(table, table, ', '.join(CMOR_TABLES))) cmor_table = CMOR_TABLES[table] - var_info = cmor_table.get_variable(mip, var_name) + var_info = cmor_table.get_variable(mip, cmor_name) if var_info is None: - var_info = CMOR_TABLES['custom'].get_variable(mip, var_name) + var_info = CMOR_TABLES['custom'].get_variable(mip, cmor_name) def _checker(cube): return CMORCheck( @@ -692,7 +692,7 @@ def _checker(cube): return _checker -def cmor_check_metadata(cube, cmor_table, mip, var_name, frequency): +def cmor_check_metadata(cube, cmor_table, mip, cmor_name, frequency): """Check if metadata conforms to variable's CMOR definiton. None of the checks at this step will force the cube to load the data. @@ -705,18 +705,18 @@ def cmor_check_metadata(cube, cmor_table, mip, var_name, frequency): CMOR definitions to use. mip: Variable's mip. - var_name: basestring + cmor_name: basestring Variable's name. frequency: basestring Data frequency. """ - checker = _get_cmor_checker(cmor_table, mip, var_name, frequency) + checker = _get_cmor_checker(cmor_table, mip, cmor_name, frequency) checker(cube).check_metadata() return cube -def cmor_check_data(cube, cmor_table, mip, var_name, frequency): +def cmor_check_data(cube, cmor_table, mip, cmor_name, frequency): """Check if data conforms to variable's CMOR definiton. The checks performed at this step require the data in memory. @@ -729,18 +729,18 @@ def cmor_check_data(cube, cmor_table, mip, var_name, frequency): CMOR definitions to use. mip: Variable's mip. - var_name: basestring + cmor_name: basestring Variable's name frequency: basestring Data frequency """ - checker = _get_cmor_checker(cmor_table, mip, var_name, frequency) + checker = _get_cmor_checker(cmor_table, mip, cmor_name, frequency) checker(cube).check_data() return cube -def cmor_check(cube, cmor_table, mip, var_name, frequency): +def cmor_check(cube, cmor_table, mip, cmor_name, frequency): """Check if cube conforms to variable's CMOR definiton. Equivalent to calling cmor_check_metadata and cmor_check_data @@ -754,12 +754,12 @@ def cmor_check(cube, cmor_table, mip, var_name, frequency): CMOR definitions to use. mip: Variable's mip. - var_name: basestring + cmor_name: basestring Variable's name. frequency: basestring Data frequency. """ - cmor_check_metadata(cube, cmor_table, mip, var_name, frequency) - cmor_check_data(cube, cmor_table, mip, var_name, frequency) + cmor_check_metadata(cube, cmor_table, mip, cmor_name, frequency) + cmor_check_data(cube, cmor_table, mip, cmor_name, frequency) return cube diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index b5a6624745..837860e266 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -def fix_file(file, var_name, project, dataset, output_dir): +def fix_file(file, cmor_name, project, dataset, output_dir): """ Fix files before ESMValTool can load them. @@ -31,7 +31,7 @@ def fix_file(file, var_name, project, dataset, output_dir): ---------- file: str Path to the original file - var_name: str + cmor_name: str Variable's cmor name project: str dataset:str @@ -45,13 +45,13 @@ def fix_file(file, var_name, project, dataset, output_dir): """ for fix in Fix.get_fixes( - project=project, dataset=dataset, variable=var_name): + project=project, dataset=dataset, variable=cmor_name): file = fix.fix_file(file, output_dir) return file def fix_metadata(cubes, - var_name, + cmor_name, project, dataset, cmor_table=None, @@ -69,7 +69,7 @@ def fix_metadata(cubes, ---------- cubes: iris.cube.CubeList Cubes to fix - var_name: str + cmor_name: str Variable's name project: str @@ -96,7 +96,7 @@ def fix_metadata(cubes, """ fixes = Fix.get_fixes( - project=project, dataset=dataset, variable=var_name) + project=project, dataset=dataset, variable=cmor_name) fixed_cubes = [] by_file = defaultdict(list) for cube in cubes: @@ -110,7 +110,7 @@ def fix_metadata(cubes, if len(cube_list) != 1: cube = None for raw_cube in cube_list: - if raw_cube.short_name == var_name: + if raw_cube.var_name == cmor_name: cube = raw_cube break if not cube: @@ -118,7 +118,7 @@ def fix_metadata(cubes, 'More than one cube found for variable %s in %s:%s but ' 'none of their var_names match the expected. \n' 'Full list of cubes encountered: %s' % - (var_name, project, dataset, cube_list) + (cmor_name, project, dataset, cube_list) ) logger.warning( 'Found variable %s in %s:%s, but there were other present in ' @@ -126,7 +126,7 @@ def fix_metadata(cubes, '(cell area, latitude descriptions) that was not saved ' 'properly. It is possible that errors appear further on ' 'because of this. \nFull list of cubes encountered: %s', - var_name, + cmor_name, project, dataset, cube_list @@ -139,7 +139,7 @@ def fix_metadata(cubes, frequency=frequency, table=cmor_table, mip=mip, - var_name=var_name, + cmor_name=cmor_name, fail_on_error=False, automatic_fixes=True) cube = checker(cube).check_metadata() @@ -149,7 +149,7 @@ def fix_metadata(cubes, def fix_data(cube, - var_name, + cmor_name, project, dataset, cmor_table=None, @@ -169,7 +169,7 @@ def fix_data(cube, ---------- cube: iris.cube.Cube Cube to fix - var_name: str + cmor_name: str Variable's name project: str @@ -196,14 +196,14 @@ def fix_data(cube, """ for fix in Fix.get_fixes( - project=project, dataset=dataset, variable=var_name): + project=project, dataset=dataset, variable=cmor_name): cube = fix.fix_data(cube) if cmor_table and mip: checker = _get_cmor_checker( frequency=frequency, table=cmor_table, mip=mip, - var_name=var_name, + cmor_name=cmor_name, fail_on_error=False, automatic_fixes=True) cube = checker(cube).check_data() diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index b7c0ca0d1f..a35b0073da 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -358,13 +358,13 @@ def _read_json_list_variable(self, parameter): class VariableInfo(JsonInfo): """Class to read and store variable information.""" - def __init__(self, table_type, var_name): + def __init__(self, table_type, cmor_name): """ Class to read and store variable information. Parameters ---------- - var_name: str + cmor_name: str variable's CMOR name """ @@ -372,9 +372,9 @@ def __init__(self, table_type, var_name): self.table_type = table_type self.modeling_realm = [] """Modeling realm""" - self.var_name = var_name + self.cmor_name = cmor_name """CMOR's table name""" - self.short_name = var_name + self.short_name = cmor_name """Short name""" self.standard_name = '' """Standard name""" @@ -638,8 +638,8 @@ def _read_coordinate(self, value): setattr(coord, key, value) return coord - def _read_variable(self, var_name, frequency): - var = VariableInfo('CMIP5', var_name) + def _read_variable(self, cmor_name, frequency): + var = VariableInfo('CMIP5', cmor_name) var.frequency = frequency while self._read_line(): key, value = self._last_line_read diff --git a/esmvalcore/preprocessor/_derive/__init__.py b/esmvalcore/preprocessor/_derive/__init__.py index 75904576f8..2d3e3a5806 100644 --- a/esmvalcore/preprocessor/_derive/__init__.py +++ b/esmvalcore/preprocessor/_derive/__init__.py @@ -34,14 +34,14 @@ def _get_all_derived_variables(): __all__ = list(ALL_DERIVED_VARIABLES) -def get_required(var_name, project): +def get_required(cmor_name, project): """Return all required variables for derivation. Get all information (at least `var_name`) required for derivation. Parameters ---------- - var_name : str + cmor_name : str name of the variable to derive. project : str `project` of the variable to derive. @@ -52,16 +52,16 @@ def get_required(var_name, project): List of dictionaries (including at least the key `short_name`). """ - if var_name not in ALL_DERIVED_VARIABLES: + if cmor_name not in ALL_DERIVED_VARIABLES: raise NotImplementedError( - f"Cannot derive variable '{var_name}', no derivation script " + f"Cannot derive variable '{cmor_name}', no derivation script " f"available") - DerivedVariable = ALL_DERIVED_VARIABLES[var_name] # noqa: N806 + DerivedVariable = ALL_DERIVED_VARIABLES[cmor_name] # noqa: N806 variables = deepcopy(DerivedVariable().required(project)) return variables -def derive(cubes, var_name, short_name, long_name, units, standard_name=None): +def derive(cubes, cmor_name, short_name, long_name, units, standard_name=None): """Derive variable. Parameters @@ -69,8 +69,8 @@ def derive(cubes, var_name, short_name, long_name, units, standard_name=None): cubes: iris.cube.CubeList Includes all the needed variables for derivation defined in :func:`get_required`. - var_name: str - var_name + cmor_name: str + cmor_name long_name: str long_name units: str @@ -90,7 +90,7 @@ def derive(cubes, var_name, short_name, long_name, units, standard_name=None): cubes = iris.cube.CubeList(cubes) # Derive variable - DerivedVariable = ALL_DERIVED_VARIABLES[var_name.lower()] # noqa: N806 + DerivedVariable = ALL_DERIVED_VARIABLES[cmor_name.lower()] # noqa: N806 cube = DerivedVariable().calculate(cubes) # Set standard attributes diff --git a/esmvalcore/preprocessor/_derive/alb.py b/esmvalcore/preprocessor/_derive/alb.py index 5331d81d87..c408b602ce 100644 --- a/esmvalcore/preprocessor/_derive/alb.py +++ b/esmvalcore/preprocessor/_derive/alb.py @@ -17,10 +17,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rsdscs' + 'cmor_name': 'rsdscs' }, { - 'var_name': 'rsuscs' + 'cmor_name': 'rsuscs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/amoc.py b/esmvalcore/preprocessor/_derive/amoc.py index 4236219052..5baf0fe8ec 100644 --- a/esmvalcore/preprocessor/_derive/amoc.py +++ b/esmvalcore/preprocessor/_derive/amoc.py @@ -11,7 +11,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'msftmyz', 'mip': 'Omon'}] + required = [{'cmor_name': 'msftmyz', 'mip': 'Omon'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/asr.py b/esmvalcore/preprocessor/_derive/asr.py index cb58fa72d4..67b9bed97f 100644 --- a/esmvalcore/preprocessor/_derive/asr.py +++ b/esmvalcore/preprocessor/_derive/asr.py @@ -11,7 +11,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'rsdt'}, {'var_name': 'rsut'}] + required = [{'cmor_name': 'rsdt'}, {'cmor_name': 'rsut'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clhmtisccp.py b/esmvalcore/preprocessor/_derive/clhmtisccp.py index f76b81d51b..b5cbaa723d 100644 --- a/esmvalcore/preprocessor/_derive/clhmtisccp.py +++ b/esmvalcore/preprocessor/_derive/clhmtisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'clisccp'}] + required = [{'cmor_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clhtkisccp.py b/esmvalcore/preprocessor/_derive/clhtkisccp.py index 32cb2a3275..57dd1fceab 100644 --- a/esmvalcore/preprocessor/_derive/clhtkisccp.py +++ b/esmvalcore/preprocessor/_derive/clhtkisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'clisccp'}] + required = [{'cmor_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clltkisccp.py b/esmvalcore/preprocessor/_derive/clltkisccp.py index a8006c63eb..84cd9e8990 100644 --- a/esmvalcore/preprocessor/_derive/clltkisccp.py +++ b/esmvalcore/preprocessor/_derive/clltkisccp.py @@ -9,7 +9,7 @@ class DerivedVariable(DerivedVariableBase): """Derivation of variable `clltkisccp`.""" - required = [{'var_name': 'clisccp'}] + required = [{'cmor_name': 'clisccp'}] @staticmethod def calculate(cubes): diff --git a/esmvalcore/preprocessor/_derive/clmmtisccp.py b/esmvalcore/preprocessor/_derive/clmmtisccp.py index 2916965c82..51ce0aee8e 100644 --- a/esmvalcore/preprocessor/_derive/clmmtisccp.py +++ b/esmvalcore/preprocessor/_derive/clmmtisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'clisccp'}] + required = [{'cmor_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/clmtkisccp.py b/esmvalcore/preprocessor/_derive/clmtkisccp.py index 5b711c262e..0ce38c0045 100644 --- a/esmvalcore/preprocessor/_derive/clmtkisccp.py +++ b/esmvalcore/preprocessor/_derive/clmtkisccp.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'clisccp'}] + required = [{'cmor_name': 'clisccp'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/ctotal.py b/esmvalcore/preprocessor/_derive/ctotal.py index e8cddb0671..542a43d52c 100644 --- a/esmvalcore/preprocessor/_derive/ctotal.py +++ b/esmvalcore/preprocessor/_derive/ctotal.py @@ -11,10 +11,10 @@ class DerivedVariable(DerivedVariableBase): # Required variables required = [ { - 'var_name': 'cVeg' + 'cmor_name': 'cVeg' }, { - 'var_name': 'cSoil' + 'cmor_name': 'cSoil' }, ] diff --git a/esmvalcore/preprocessor/_derive/et.py b/esmvalcore/preprocessor/_derive/et.py index 930b295149..ac6b3ef7cb 100644 --- a/esmvalcore/preprocessor/_derive/et.py +++ b/esmvalcore/preprocessor/_derive/et.py @@ -15,7 +15,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'hfls', 'mip': 'Amon'}] + required = [{'cmor_name': 'hfls', 'mip': 'Amon'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/gtfgco2.py b/esmvalcore/preprocessor/_derive/gtfgco2.py index 92780d1c66..bd18457bfb 100644 --- a/esmvalcore/preprocessor/_derive/gtfgco2.py +++ b/esmvalcore/preprocessor/_derive/gtfgco2.py @@ -50,22 +50,22 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'fgco2', + 'cmor_name': 'fgco2', 'mip': 'Omon' }, { - 'var_name': 'areacello', + 'cmor_name': 'areacello', 'mip': 'fx' }, ] if project == 'CMIP6': required = [ { - 'var_name': 'fgco2', + 'cmor_name': 'fgco2', 'mip': 'Omon' }, { - 'var_name': 'areacello', + 'cmor_name': 'areacello', 'mip': 'Ofx' }, ] diff --git a/esmvalcore/preprocessor/_derive/lvp.py b/esmvalcore/preprocessor/_derive/lvp.py index 622b48345c..da02a81875 100644 --- a/esmvalcore/preprocessor/_derive/lvp.py +++ b/esmvalcore/preprocessor/_derive/lvp.py @@ -17,13 +17,13 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'hfls' + 'cmor_name': 'hfls' }, { - 'var_name': 'pr' + 'cmor_name': 'pr' }, { - 'var_name': 'evspsbl' + 'cmor_name': 'evspsbl' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/lwcre.py b/esmvalcore/preprocessor/_derive/lwcre.py index 851fc0fead..92c424dca4 100644 --- a/esmvalcore/preprocessor/_derive/lwcre.py +++ b/esmvalcore/preprocessor/_derive/lwcre.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rlut' + 'cmor_name': 'rlut' }, { - 'var_name': 'rlutcs' + 'cmor_name': 'rlutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/lwp.py b/esmvalcore/preprocessor/_derive/lwp.py index 0d75185199..aee55432b7 100644 --- a/esmvalcore/preprocessor/_derive/lwp.py +++ b/esmvalcore/preprocessor/_derive/lwp.py @@ -16,10 +16,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'clwvi' + 'cmor_name': 'clwvi' }, { - 'var_name': 'clivi' + 'cmor_name': 'clivi' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/netcre.py b/esmvalcore/preprocessor/_derive/netcre.py index 5cf1039107..60ee25b104 100644 --- a/esmvalcore/preprocessor/_derive/netcre.py +++ b/esmvalcore/preprocessor/_derive/netcre.py @@ -13,16 +13,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rlut' + 'cmor_name': 'rlut' }, { - 'var_name': 'rlutcs' + 'cmor_name': 'rlutcs' }, { - 'var_name': 'rsut' + 'cmor_name': 'rsut' }, { - 'var_name': 'rsutcs' + 'cmor_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/ohc.py b/esmvalcore/preprocessor/_derive/ohc.py index fcbfb70f52..0d5ba5e8a2 100644 --- a/esmvalcore/preprocessor/_derive/ohc.py +++ b/esmvalcore/preprocessor/_derive/ohc.py @@ -17,20 +17,20 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'thetao' + 'cmor_name': 'thetao' }, { - 'var_name': 'volcello', + 'cmor_name': 'volcello', 'mip': 'fx' }, ] if project == 'CMIP6': required = [ { - 'var_name': 'thetao' + 'cmor_name': 'thetao' }, { - 'var_name': 'volcello', + 'cmor_name': 'volcello', 'mip': 'Ofx' }, ] diff --git a/esmvalcore/preprocessor/_derive/rlns.py b/esmvalcore/preprocessor/_derive/rlns.py index 1131272e05..326349f02d 100644 --- a/esmvalcore/preprocessor/_derive/rlns.py +++ b/esmvalcore/preprocessor/_derive/rlns.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rlds' + 'cmor_name': 'rlds' }, { - 'var_name': 'rlus' + 'cmor_name': 'rlus' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rlnst.py b/esmvalcore/preprocessor/_derive/rlnst.py index 57a7fc735b..5677da0a53 100644 --- a/esmvalcore/preprocessor/_derive/rlnst.py +++ b/esmvalcore/preprocessor/_derive/rlnst.py @@ -17,13 +17,13 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rlds' + 'cmor_name': 'rlds' }, { - 'var_name': 'rlus' + 'cmor_name': 'rlus' }, { - 'var_name': 'rlut' + 'cmor_name': 'rlut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rlnstcs.py b/esmvalcore/preprocessor/_derive/rlnstcs.py index 41e059f227..81b419af47 100644 --- a/esmvalcore/preprocessor/_derive/rlnstcs.py +++ b/esmvalcore/preprocessor/_derive/rlnstcs.py @@ -14,13 +14,13 @@ class DerivedVariable(DerivedVariableBase): required = [ { - 'var_name': 'rldscs' + 'cmor_name': 'rldscs' }, { - 'var_name': 'rlus' + 'cmor_name': 'rlus' }, { - 'var_name': 'rlutcs' + 'cmor_name': 'rlutcs' }, ] diff --git a/esmvalcore/preprocessor/_derive/rlntcs.py b/esmvalcore/preprocessor/_derive/rlntcs.py index 1672897a86..2f4b96f632 100644 --- a/esmvalcore/preprocessor/_derive/rlntcs.py +++ b/esmvalcore/preprocessor/_derive/rlntcs.py @@ -10,7 +10,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'rlutcs'}] + required = [{'cmor_name': 'rlutcs'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/rsns.py b/esmvalcore/preprocessor/_derive/rsns.py index 1c145e9ab7..f7870f6a48 100644 --- a/esmvalcore/preprocessor/_derive/rsns.py +++ b/esmvalcore/preprocessor/_derive/rsns.py @@ -13,10 +13,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rsds' + 'cmor_name': 'rsds' }, { - 'var_name': 'rsus' + 'cmor_name': 'rsus' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsnst.py b/esmvalcore/preprocessor/_derive/rsnst.py index 21eda3e487..c537a0edbc 100644 --- a/esmvalcore/preprocessor/_derive/rsnst.py +++ b/esmvalcore/preprocessor/_derive/rsnst.py @@ -17,16 +17,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rsds' + 'cmor_name': 'rsds' }, { - 'var_name': 'rsdt' + 'cmor_name': 'rsdt' }, { - 'var_name': 'rsus' + 'cmor_name': 'rsus' }, { - 'var_name': 'rsut' + 'cmor_name': 'rsut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rsnstcs.py b/esmvalcore/preprocessor/_derive/rsnstcs.py index 8a1a9ebefa..071182321f 100644 --- a/esmvalcore/preprocessor/_derive/rsnstcs.py +++ b/esmvalcore/preprocessor/_derive/rsnstcs.py @@ -17,16 +17,16 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rsdscs' + 'cmor_name': 'rsdscs' }, { - 'var_name': 'rsdt' + 'cmor_name': 'rsdt' }, { - 'var_name': 'rsuscs' + 'cmor_name': 'rsuscs' }, { - 'var_name': 'rsutcs' + 'cmor_name': 'rsutcs' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/rtnt.py b/esmvalcore/preprocessor/_derive/rtnt.py index 5ff1ec50ed..ba68bd5c8a 100644 --- a/esmvalcore/preprocessor/_derive/rtnt.py +++ b/esmvalcore/preprocessor/_derive/rtnt.py @@ -13,13 +13,13 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'rsdt' + 'cmor_name': 'rsdt' }, { - 'var_name': 'rsut' + 'cmor_name': 'rsut' }, { - 'var_name': 'rlut' + 'cmor_name': 'rlut' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/sispeed.py b/esmvalcore/preprocessor/_derive/sispeed.py index 4bba38483c..82800ee1b4 100644 --- a/esmvalcore/preprocessor/_derive/sispeed.py +++ b/esmvalcore/preprocessor/_derive/sispeed.py @@ -17,9 +17,9 @@ class DerivedVariable(DerivedVariableBase): def required(project): """Declare the variables needed for derivation.""" required = [{ - 'var_name': 'usi', + 'cmor_name': 'usi', }, { - 'var_name': 'vsi', + 'cmor_name': 'vsi', }] return required diff --git a/esmvalcore/preprocessor/_derive/sithick.py b/esmvalcore/preprocessor/_derive/sithick.py index bee89f3495..71d2dfb140 100644 --- a/esmvalcore/preprocessor/_derive/sithick.py +++ b/esmvalcore/preprocessor/_derive/sithick.py @@ -12,9 +12,9 @@ class DerivedVariable(DerivedVariableBase): def required(project): """Declare the variables needed for derivation.""" required = [{ - 'var_name': 'sit', + 'cmor_name': 'sit', }, { - 'var_name': 'sic', + 'cmor_name': 'sic', }] return required diff --git a/esmvalcore/preprocessor/_derive/sm.py b/esmvalcore/preprocessor/_derive/sm.py index c3714d285e..24f79a12f9 100644 --- a/esmvalcore/preprocessor/_derive/sm.py +++ b/esmvalcore/preprocessor/_derive/sm.py @@ -13,7 +13,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'mrsos'}] + required = [{'cmor_name': 'mrsos'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/toz.py b/esmvalcore/preprocessor/_derive/toz.py index 6448e0c885..9d94957097 100644 --- a/esmvalcore/preprocessor/_derive/toz.py +++ b/esmvalcore/preprocessor/_derive/toz.py @@ -28,10 +28,10 @@ def required(project): """Declare the variables needed for derivation.""" required = [ { - 'var_name': 'tro3' + 'cmor_name': 'tro3' }, { - 'var_name': 'ps' + 'cmor_name': 'ps' }, ] return required diff --git a/esmvalcore/preprocessor/_derive/uajet.py b/esmvalcore/preprocessor/_derive/uajet.py index a04caecea3..ec2ab63508 100644 --- a/esmvalcore/preprocessor/_derive/uajet.py +++ b/esmvalcore/preprocessor/_derive/uajet.py @@ -17,7 +17,7 @@ class DerivedVariable(DerivedVariableBase): @staticmethod def required(project): """Declare the variables needed for derivation.""" - required = [{'var_name': 'ua'}] + required = [{'cmor_name': 'ua'}] return required @staticmethod diff --git a/esmvalcore/preprocessor/_derive/vegfrac.py b/esmvalcore/preprocessor/_derive/vegfrac.py index 3446855a76..7418d11c04 100644 --- a/esmvalcore/preprocessor/_derive/vegfrac.py +++ b/esmvalcore/preprocessor/_derive/vegfrac.py @@ -8,7 +8,7 @@ class DerivedVariable(DerivedVariableBase): """Derivation of variable `vegFrac`.""" required = [{ - 'var_name': 'baresoilFrac', + 'cmor_name': 'baresoilFrac', }] @staticmethod diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index d05ad151a4..8f24f9975b 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -265,7 +265,7 @@ def _write_ncl_metadata(output_dir, metadata): variable_info[key] = variable[key] filename = os.path.join(output_dir, - variable_info['var_name'] + '_info.ncl') + variable_info['cmor_name'] + '_info.ncl') write_ncl_settings(info, filename) return filename diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 91efeaa0d1..c3fce9c07b 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -496,7 +496,7 @@ def get_cmor_levels(cmor_table, coordinate): def get_reference_levels(filename, project, dataset, - var_name, + cmor_name, fix_dir): """Get level definition from a CMOR coordinate. @@ -516,9 +516,9 @@ def get_reference_levels(filename, levels or the string is badly formatted. """ - filename = fix_file(filename, var_name, project, dataset, fix_dir) + filename = fix_file(filename, cmor_name, project, dataset, fix_dir) cubes = load(filename, callback=concatenate_callback) - cubes = fix_metadata(cubes, var_name, project, dataset) + cubes = fix_metadata(cubes, cmor_name, project, dataset) cube = cubes[0] try: coord = cube.coord(axis='Z') diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index c2532e84d7..db90533e1a 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -48,7 +48,7 @@ def test_get_variable_different_out_name(self): """Get tas variable.""" var = self.variables_info.get_variable('6hrPlevPt', 'zg7h') self.assertEqual(var.short_name, 'zg') - self.assertEqual(var.var_name, 'zg7h') + self.assertEqual(var.cmor_name, 'zg7h') def test_get_variable_from_alias(self): """Get a variable from a known alias.""" diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index 1f117faf6a..a6e1d33444 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -11,10 +11,10 @@ def test_get_required(): reference = [ { - 'var_name': 'rsdscs', + 'cmor_name': 'rsdscs', }, { - 'var_name': 'rsuscs', + 'cmor_name': 'rsuscs', }, ] @@ -26,8 +26,8 @@ def test_get_required_with_fx(): variables = get_required('ohc', 'CMIP5') reference = [ - {'var_name': 'thetao'}, - {'var_name': 'volcello', 'mip': 'fx'}, + {'cmor_name': 'thetao'}, + {'cmor_name': 'volcello', 'mip': 'fx'}, ] assert variables == reference diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 1ef8fe0714..bae7a49044 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -31,7 +31,7 @@ 'modeling_realm', 'preprocessor', 'project', - 'var_name', + 'cmor_name', 'short_name', 'standard_name', 'start_year', @@ -158,8 +158,8 @@ def get_required(short_name, _): if short_name != 'tas': assert False required = [ - {'var_name': 'pr'}, - {'var_name': 'areacella', 'mip': 'fx', 'optional': True}, + {'cmor_name': 'pr'}, + {'cmor_name': 'areacella', 'mip': 'fx', 'optional': True}, ] return required @@ -244,7 +244,7 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): for variable in variables: for key in MANDATORY_DATASET_KEYS: assert key in variable and variable[key] - assert variable_name == variable['var_name'] + assert variable_name == variable['cmor_name'] # Check that the correct tasks have been created variables = recipe.diagnostics['diagnostic_name']['preprocessor_output'][ @@ -348,13 +348,13 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_file': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'var_name': 'chl', + 'cmor_name': 'chl', 'output_dir': fix_dir, }, 'fix_data': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'var_name': 'chl', + 'cmor_name': 'chl', 'cmor_table': 'CMIP5', 'mip': 'Oyr', 'frequency': 'yr', @@ -362,7 +362,7 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_metadata': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'var_name': 'chl', + 'cmor_name': 'chl', 'cmor_table': 'CMIP5', 'mip': 'Oyr', 'frequency': 'yr', @@ -378,13 +378,13 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): 'cmor_check_metadata': { 'cmor_table': 'CMIP5', 'mip': 'Oyr', - 'var_name': 'chl', + 'cmor_name': 'chl', 'frequency': 'yr', }, 'cmor_check_data': { 'cmor_table': 'CMIP5', 'mip': 'Oyr', - 'var_name': 'chl', + 'cmor_name': 'chl', 'frequency': 'yr', }, 'cleanup': { @@ -435,13 +435,13 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_file': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'var_name': 'sftlf', + 'cmor_name': 'sftlf', 'output_dir': fix_dir, }, 'fix_data': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'var_name': 'sftlf', + 'cmor_name': 'sftlf', 'cmor_table': 'CMIP5', 'mip': 'fx', 'frequency': 'fx', @@ -449,7 +449,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'fix_metadata': { 'project': 'CMIP5', 'dataset': 'CanESM2', - 'var_name': 'sftlf', + 'cmor_name': 'sftlf', 'cmor_table': 'CMIP5', 'mip': 'fx', 'frequency': 'fx', @@ -457,13 +457,13 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): 'cmor_check_metadata': { 'cmor_table': 'CMIP5', 'mip': 'fx', - 'var_name': 'sftlf', + 'cmor_name': 'sftlf', 'frequency': 'fx', }, 'cmor_check_data': { 'cmor_table': 'CMIP5', 'mip': 'fx', - 'var_name': 'sftlf', + 'cmor_name': 'sftlf', 'frequency': 'fx', }, 'cleanup': { @@ -500,7 +500,7 @@ def test_empty_variable(tmp_path, patched_datafinder, config_user): task = recipe.tasks.pop() assert len(task.products) == 1 product = task.products.pop() - assert product.attributes['var_name'] == 'pr' + assert product.attributes['cmor_name'] == 'pr' assert product.attributes['dataset'] == 'CanESM2' @@ -539,7 +539,7 @@ def test_cmip5_variable_autocomplete(tmp_path, patched_datafinder, 'modeling_realm': ['atmos'], 'preprocessor': 'default', 'project': 'CMIP5', - 'var_name': 'pr', + 'cmor_name': 'pr', 'standard_name': 'precipitation_flux', 'start_year': 2000, 'units': 'kg m-2 s-1', @@ -586,7 +586,7 @@ def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, 'modeling_realm': ['atmos'], 'preprocessor': 'default', 'project': 'CMIP6', - 'var_name': 'pr', + 'cmor_name': 'pr', 'standard_name': 'precipitation_flux', 'start_year': 2000, 'units': 'kg m-2 s-1', @@ -715,7 +715,7 @@ def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): diagnostic_name: variables: chl_default: &chl - var_name: chl + cmor_name: chl preprocessor: default project: CMIP5 mip: Oyr @@ -782,13 +782,13 @@ def test_derive(tmp_path, patched_datafinder, config_user): assert len(task.products) == 1 product = task.products.pop() assert 'derive' in product.settings - assert product.attributes['var_name'] == 'toz' + assert product.attributes['cmor_name'] == 'toz' assert product.files ps_product = next(p for a in task.ancestors for p in a.products - if p.attributes['var_name'] == 'ps') + if p.attributes['cmor_name'] == 'ps') tro3_product = next(p for a in task.ancestors for p in a.products - if p.attributes['var_name'] == 'tro3') + if p.attributes['cmor_name'] == 'tro3') assert ps_product.filename in product.files assert tro3_product.filename in product.files @@ -826,13 +826,13 @@ def test_derive_not_needed(tmp_path, patched_datafinder, config_user): # Check product content of tasks assert len(task.products) == 1 product = task.products.pop() - assert product.attributes['var_name'] == 'toz' + assert product.attributes['cmor_name'] == 'toz' assert 'derive' in product.settings assert len(ancestor.products) == 1 ancestor_product = ancestor.products.pop() assert ancestor_product.filename in product.files - assert ancestor_product.attributes['var_name'] == 'toz' + assert ancestor_product.attributes['cmor_name'] == 'toz' assert 'derive' not in ancestor_product.settings # Check that fixes are applied just once @@ -875,7 +875,7 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings - assert product.attributes['var_name'] == 'ohc' + assert product.attributes['cmor_name'] == 'ohc' all_product_files.extend(product.files) # Check ancestors @@ -885,10 +885,10 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert task.ancestors[1].name == ( 'diagnostic_name/ohc_derive_input_volcello') for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['var_name'] == 'thetao' + assert ancestor_product.attributes['cmor_name'] == 'thetao' assert ancestor_product.filename in all_product_files for ancestor_product in task.ancestors[1].products: - assert ancestor_product.attributes['var_name'] == 'volcello' + assert ancestor_product.attributes['cmor_name'] == 'volcello' if ancestor_product.attributes['project'] == 'CMIP6': assert ancestor_product.attributes['mip'] == 'Ofx' else: @@ -959,7 +959,7 @@ def test_derive_with_optional_var(tmp_path, assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings - assert product.attributes['var_name'] == 'tas' + assert product.attributes['cmor_name'] == 'tas' all_product_files.extend(product.files) # Check ancestors @@ -969,10 +969,10 @@ def test_derive_with_optional_var(tmp_path, assert task.ancestors[1].name == ( 'diagnostic_name/tas_derive_input_areacella') for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['var_name'] == 'pr' + assert ancestor_product.attributes['cmor_name'] == 'pr' assert ancestor_product.filename in all_product_files for ancestor_product in task.ancestors[1].products: - assert ancestor_product.attributes['var_name'] == 'areacella' + assert ancestor_product.attributes['cmor_name'] == 'areacella' assert ancestor_product.filename in all_product_files @@ -1012,7 +1012,7 @@ def test_derive_with_optional_var_nodata(tmp_path, assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings - assert product.attributes['var_name'] == 'tas' + assert product.attributes['cmor_name'] == 'tas' all_product_files.extend(product.files) # Check ancestors @@ -1020,7 +1020,7 @@ def test_derive_with_optional_var_nodata(tmp_path, assert task.ancestors[0].name == ( 'diagnostic_name/tas_derive_input_pr') for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['var_name'] == 'pr' + assert ancestor_product.attributes['cmor_name'] == 'pr' assert ancestor_product.filename in all_product_files diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index e0bcf57fd7..ad08037492 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -173,7 +173,7 @@ def test_check_bad_standard_name_auto_fix(self): self._check_cube() def test_check_bad_standard_name(self): - """Test check fails for a bad var_name.""" + """Test check fails for a bad cmor_name.""" self.cube = self.get_cube(self.var_info) self.cube.standard_name = 'wind_speed' self._check_fails_in_metadata(automatic_fixes=False) @@ -192,7 +192,7 @@ def test_check_bad_long_name_auto_fix_report_warning(self): self._check_warnings_on_metadata(automatic_fixes=True) def test_check_bad_long_name(self): - """Test check fails for a bad var_name.""" + """Test check fails for a bad cmor_name.""" self.cube = self.get_cube(self.var_info) self.cube.long_name = 'bad_name' self._check_fails_in_metadata() diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 3f10775ce9..8900b6856f 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -21,7 +21,7 @@ def test_fix(self): with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[self.mock_fix]): - file_returned = fix_file('filename', 'var_name', 'project', + file_returned = fix_file('filename', 'cmor_name', 'project', 'model', 'output_dir') self.assertNotEqual(file_returned, self.filename) self.assertEqual(file_returned, 'new_filename') @@ -30,7 +30,7 @@ def test_nofix(self): """Check that the same file is returned if no fix is available.""" with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - file_returned = fix_file('filename', 'var_name', 'project', + file_returned = fix_file('filename', 'cmor_name', 'project', 'model', 'output_dir') self.assertEqual(file_returned, self.filename) @@ -80,10 +80,9 @@ def setUp(self): self.mock_fix.fix_metadata.return_value = [self.fixed_cube] @staticmethod - def _create_mock_cube(var_name='var_name'): + def _create_mock_cube(cmor_name='cmor_name'): cube = mock.Mock() - cube.var_name = var_name - cube.short_name = var_name + cube.var_name = cmor_name cube.attributes = {'source_file': 'source_file'} return cube @@ -92,7 +91,7 @@ def test_fix(self): with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[self.mock_fix]): - cube_returned = fix_metadata([self.cube], 'var_name', 'project', + cube_returned = fix_metadata([self.cube], 'cmor_name', 'project', 'model')[0] self.assertTrue(cube_returned is not self.cube) self.assertTrue(cube_returned is self.fixed_cube) @@ -101,7 +100,7 @@ def test_nofix(self): """Check that the same cube is returned if no fix is available.""" with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_metadata([self.cube], 'var_name', 'project', + cube_returned = fix_metadata([self.cube], 'cmor_name', 'project', 'model')[0] self.assertTrue(cube_returned is self.cube) self.assertTrue(cube_returned is not self.fixed_cube) @@ -112,7 +111,7 @@ def test_select_var(self): 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): cube_returned = fix_metadata( [self.cube, self._create_mock_cube('extra')], - 'var_name', + 'cmor_name', 'project', 'model' )[0] @@ -128,7 +127,7 @@ def test_select_var_failed_if_bad_var_name(self): self._create_mock_cube('not_me'), self._create_mock_cube('me_neither') ], - 'var_name', + 'cmor_name', 'project', 'model' ) @@ -142,14 +141,14 @@ def test_cmor_checker_called(self): with mock.patch( 'esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_metadata([self.cube], 'var_name', 'project', 'model', + fix_metadata([self.cube], 'cmor_name', 'project', 'model', 'cmor_table', 'mip', 'frequency') get_mock.assert_called_once_with( automatic_fixes=True, fail_on_error=False, frequency='frequency', mip='mip', - var_name='var_name', + cmor_name='cmor_name', table='cmor_table') checker.assert_called_once_with(self.cube) checker.return_value.check_metadata.assert_called_once_with() @@ -170,7 +169,7 @@ def test_fix(self): with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[self.mock_fix]): - cube_returned = fix_data(self.cube, 'var_name', 'project', + cube_returned = fix_data(self.cube, 'cmor_name', 'project', 'model') self.assertTrue(cube_returned is not self.cube) self.assertTrue(cube_returned is self.fixed_cube) @@ -179,7 +178,7 @@ def test_nofix(self): """Check that the same cube is returned if no fix is available.""" with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - cube_returned = fix_data(self.cube, 'var_name', 'project', + cube_returned = fix_data(self.cube, 'cmor_name', 'project', 'model') self.assertTrue(cube_returned is self.cube) self.assertTrue(cube_returned is not self.fixed_cube) @@ -193,14 +192,14 @@ def test_cmor_checker_called(self): with mock.patch( 'esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_data(self.cube, 'var_name', 'project', 'model', + fix_data(self.cube, 'cmor_name', 'project', 'model', 'cmor_table', 'mip', 'frequency') get_mock.assert_called_once_with( automatic_fixes=True, fail_on_error=False, frequency='frequency', mip='mip', - var_name='var_name', + cmor_name='cmor_name', table='cmor_table') checker.assert_called_once_with(self.cube) checker.return_value.check_data.assert_called_once_with() From d6cbe02be81f900e1290aaad3454fe82bb59cca5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 4 Feb 2020 10:00:13 +0100 Subject: [PATCH 07/11] Fix tests --- esmvalcore/cmor/fix.py | 8 +++++++- tests/unit/cmor/test_fix.py | 26 ++++++++++++++------------ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 837860e266..a08fb3ddf1 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -13,6 +13,7 @@ from iris.cube import CubeList from ._fixes.fix import Fix +from .table import CMOR_TABLES from .check import _get_cmor_checker logger = logging.getLogger(__name__) @@ -97,6 +98,11 @@ def fix_metadata(cubes, """ fixes = Fix.get_fixes( project=project, dataset=dataset, variable=cmor_name) + if cmor_table and mip: + short_name = CMOR_TABLES[cmor_table].get_variable( + mip, cmor_name).short_name + else: + short_name = cmor_name fixed_cubes = [] by_file = defaultdict(list) for cube in cubes: @@ -110,7 +116,7 @@ def fix_metadata(cubes, if len(cube_list) != 1: cube = None for raw_cube in cube_list: - if raw_cube.var_name == cmor_name: + if raw_cube.var_name == short_name: cube = raw_cube break if not cube: diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 8900b6856f..fdfa0ec855 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -138,20 +138,22 @@ def test_cmor_checker_called(self): checker.return_value = mock.Mock() with mock.patch( 'esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): - with mock.patch( + with mock.patch( 'esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_metadata([self.cube], 'cmor_name', 'project', 'model', - 'cmor_table', 'mip', 'frequency') - get_mock.assert_called_once_with( - automatic_fixes=True, - fail_on_error=False, - frequency='frequency', - mip='mip', - cmor_name='cmor_name', - table='cmor_table') - checker.assert_called_once_with(self.cube) - checker.return_value.check_metadata.assert_called_once_with() + with mock.patch( + 'esmvalcore.cmor.table.CMOR_TABLES') as tables_mock: + fix_metadata([self.cube], 'cmor_name', 'project', 'model', + 'cmor_table', 'mip', 'frequency') + get_mock.assert_called_once_with( + automatic_fixes=True, + fail_on_error=False, + frequency='frequency', + mip='mip', + cmor_name='cmor_name', + table='cmor_table') + checker.assert_called_once_with(self.cube) + checker.return_value.check_metadata.assert_called_once_with() class TestFixData(unittest.TestCase): From b9b227fe527dd339474ae8609d7c9fec533bae25 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 18 Feb 2020 10:31:49 +0100 Subject: [PATCH 08/11] Fix codacy warnigns --- esmvalcore/_recipe.py | 16 ---------------- esmvalcore/cmor/table.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 6d32e6fc47..1fefa6aad0 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -673,22 +673,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, for variable in variables: variable['filename'] = get_output_file(variable, config_user['preproc_dir']) - """[summary] - - Returns - ------- - [type] - [description] - - Raises - ------ - RecipeError - [description] - RecipeError - [description] - RecipeError - [description] - """ if ancestor_products: grouped_ancestors = _match_products(ancestor_products, variables) else: diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 3694c45f60..f53929d628 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -21,6 +21,18 @@ def get_var_info(project, mip, cmor_name): + """ + Get variable information + + Parameters + ---------- + project: of str + Dataset's project + mip: str + Variable's cmor table + cmor_name: str + Variable's cmor name + """ return CMOR_TABLES[project].get_variable(mip, cmor_name) From ac97e1dfbfedb95432ac39824523bc13533d17cd Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 26 Feb 2020 17:01:13 +0100 Subject: [PATCH 09/11] Replace many more short_name with cmor_name --- doc/esmvalcore/derivation.rst | 4 +- doc/esmvalcore/fixing_data.rst | 6 +-- doc/esmvalcore/recipe.rst | 6 +-- esmvalcore/cmor/_fixes/fix.py | 2 +- esmvalcore/cmor/check.py | 1 - esmvalcore/cmor/fix.py | 5 +- esmvalcore/cmor/table.py | 54 +++++++++---------- esmvalcore/preprocessor/_derive/__init__.py | 10 ++-- .../cmor/_fixes/cmip5/test_access1_0.py | 4 +- .../cmor/_fixes/cmip5/test_access1_3.py | 4 +- .../cmor/_fixes/cmip5/test_bcc_csm1_1.py | 2 +- .../cmor/_fixes/cmip5/test_bcc_csm1_1_m.py | 2 +- .../cmor/_fixes/cmip5/test_bnu_esm.py | 22 +++++--- .../cmor/_fixes/cmip5/test_canesm2.py | 2 +- .../cmor/_fixes/cmip5/test_ccsm4.py | 10 ++-- .../cmor/_fixes/cmip5/test_cnrm_cm5.py | 2 +- .../cmor/_fixes/cmip5/test_fgoals_g2.py | 2 +- .../cmor/_fixes/cmip5/test_fio_esm.py | 2 +- .../cmor/_fixes/cmip5/test_gfdl_cm2p1.py | 13 +++-- .../cmor/_fixes/cmip5/test_gfdl_cm3.py | 16 +++--- .../cmor/_fixes/cmip5/test_gfdl_esm2g.py | 23 ++++---- .../cmor/_fixes/cmip5/test_gfdl_esm2m.py | 15 +++--- .../cmor/_fixes/cmip5/test_hadgem2_cc.py | 2 +- .../cmor/_fixes/cmip5/test_hadgem2_es.py | 2 +- .../cmor/_fixes/cmip5/test_inmcm4.py | 4 +- .../cmor/_fixes/cmip5/test_miroc_esm.py | 8 +-- .../cmor/_fixes/cmip5/test_miroc_esm_chem.py | 2 +- .../cmor/_fixes/cmip5/test_mpi_esm_lr.py | 2 +- .../cmor/_fixes/cmip5/test_mri_cgcm3.py | 2 +- .../cmor/_fixes/cmip5/test_mri_esm1.py | 2 +- .../cmor/_fixes/cmip5/test_noresm1_me.py | 4 +- .../cmor/_fixes/cmip6/test_cesm2.py | 4 +- .../cmor/_fixes/cmip6/test_cesm2_waccm.py | 4 +- .../cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py | 3 +- .../cmor/_fixes/cmip6/test_mcm_ua_1_0.py | 4 +- .../preprocessor/_derive/test_interface.py | 2 +- tests/integration/test_recipe.py | 9 ++-- tests/unit/cmor/test_fix.py | 6 ++- tests/unit/cmor/test_table.py | 8 ++- 39 files changed, 154 insertions(+), 121 deletions(-) diff --git a/doc/esmvalcore/derivation.rst b/doc/esmvalcore/derivation.rst index fcc317db26..5161286a92 100644 --- a/doc/esmvalcore/derivation.rst +++ b/doc/esmvalcore/derivation.rst @@ -27,8 +27,8 @@ A typical example looks like this: if project == 'CMIP6': mip = 'Ofx' required = [ - {'short_name': 'var_a'}, - {'short_name': 'var_b', 'mip': mip, 'optional': True}, + {'cmor_name': 'var_a'}, + {'cmor_name': 'var_b', 'mip': mip, 'optional': True}, ] return required diff --git a/doc/esmvalcore/fixing_data.rst b/doc/esmvalcore/fixing_data.rst index bd58e6e979..bf51604009 100644 --- a/doc/esmvalcore/fixing_data.rst +++ b/doc/esmvalcore/fixing_data.rst @@ -66,7 +66,7 @@ coordinates. In our example it looks like this: month_number x - - year x - - Attributes: - {'cmor_table': 'CMIPX', 'mip': 'Amon', 'short_name': 'tas', 'frequency': 'mon'}) + {'cmor_table': 'CMIPX', 'mip': 'Amon', 'cmor_name': 'tas', 'frequency': 'mon'}) So now the mistake is clear: the latitude coordinate is badly named and the @@ -132,7 +132,7 @@ so we will implement the ``fix_metadata`` method: latitude = tas_cube.coord('altitude') # Fix the names. Latitude values, units and - latitude.short_name = 'lat' + latitude.var_name = 'lat' latitude.standard_name = 'latitude' latitude.long_name = 'latitude' return cubes @@ -185,7 +185,7 @@ from the one you just created: latitude = tas_cube.coord('altitude') # Fix the names. Latitude values, units and - latitude.short_name = 'lat' + latitude.var_name = 'lat' latitude.standard_name = 'latitude' latitude.long_name = 'latitude' return cubes diff --git a/doc/esmvalcore/recipe.rst b/doc/esmvalcore/recipe.rst index db270c22d6..c7a6c7d3be 100644 --- a/doc/esmvalcore/recipe.rst +++ b/doc/esmvalcore/recipe.rst @@ -215,7 +215,7 @@ A (simplified) example diagnostics section could look like - atmos variables: variable_name: - short_name: ta + cmor_name: ta preprocessor: preprocessor_name mip: Amon scripts: @@ -249,7 +249,7 @@ that wildcard expansion can be used to define ancestors. diagnostic_1: variables: airtemp: - short_name: ta + cmor_name: ta preprocessor: preprocessor_name mip: Amon scripts: @@ -258,7 +258,7 @@ that wildcard expansion can be used to define ancestors. diagnostic_2: variables: precip: - short_name: pr + cmor_name: pr preprocessor: preprocessor_name mip: Amon scripts: diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 1fd4e1f5f5..c32972abae 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -89,7 +89,7 @@ def get_cube_from_list(self, cubes, short_name=None): Variable's cube """ if short_name is None: - short_name = self.__class__.__name__.lower() + short_name = self.vardef.short_name for cube in cubes: if cube.var_name == short_name: return cube diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 92822ec667..91b750dbd2 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -2,7 +2,6 @@ import logging import cf_units -import iris.coord_categorisation import iris.coords import iris.exceptions import iris.util diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 5831403dbb..7b429a8945 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -95,10 +95,7 @@ def fix_metadata(cubes, """ fixes = Fix.get_fixes( project=project, dataset=dataset, mip=mip, cmor_name=cmor_name) - if project and mip: - short_name = get_var_info(project, mip, cmor_name).short_name - else: - short_name = cmor_name + short_name = get_var_info(project, mip, cmor_name).short_name fixed_cubes = [] by_file = defaultdict(list) for cube in cubes: diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index f53929d628..c7420cbd85 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -91,7 +91,7 @@ class CMIP6Info(object): Parameters ---------- - cmor_tables_path: basestring + cmor_tables_path: str Path to the folder containing the Tables folder with the json files default: object @@ -240,7 +240,7 @@ def get_table(self, table): Parameters ---------- - table: basestring + table: str Table name Returns @@ -261,10 +261,10 @@ def get_variable(self, table_name, cmor_name, derived=False): Parameters ---------- - table_name: basestring + table_name: str Table name - short_name: basestring - Variable's short name + cmor_name: str + Variable's cmor name derived: bool, optional Variable is derived. Info retrieval is less strict @@ -283,8 +283,8 @@ def get_variable(self, table_name, cmor_name, derived=False): pass if cmor_name in CMIP6Info._CMIP_5to6_varname: - new_short_name = CMIP6Info._CMIP_5to6_varname[cmor_name] - return self.get_variable(table_name, new_short_name, derived) + new_cmor_name = CMIP6Info._CMIP_5to6_varname[cmor_name] + return self.get_variable(table_name, new_cmor_name, derived) var_info = None if not self.strict: @@ -550,7 +550,7 @@ class CMIP5Info(object): Parameters ---------- - cmor_tables_path: basestring + cmor_tables_path: str Path to the folder containing the Tables folder with the json files default: object @@ -692,7 +692,7 @@ def get_table(self, table): Parameters ---------- - table: basestring + table: str Table name Returns @@ -704,16 +704,16 @@ def get_table(self, table): """ return self.tables.get(table) - def get_variable(self, table, short_name, derived=False): + def get_variable(self, table, cmor_name, derived=False): """ Search and return the variable info. Parameters ---------- - table: basestring + table: str Table name - short_name: basestring - Variable's short name + cmor_name: str + Variable's cmor name derived: bool, optional Variable is derived. Info retrieval is less strict @@ -724,16 +724,16 @@ def get_variable(self, table, short_name, derived=False): found, returns None if not """ - var_info = self.tables.get(table, {}).get(short_name, None) + var_info = self.tables.get(table, {}).get(cmor_name, None) if var_info: return var_info if not self.strict: for table_vars in sorted(self.tables.values()): - if short_name in table_vars: - var_info = table_vars[short_name] + if cmor_name in table_vars: + var_info = table_vars[cmor_name] break if not var_info and (derived or not self.strict): - var_info = self.default.get_variable(table, short_name) + var_info = self.default.get_variable(table, cmor_name) if var_info: mip_info = self.get_table(table) @@ -749,7 +749,7 @@ class CMIP3Info(CMIP5Info): Parameters ---------- - cmor_tables_path: basestring + cmor_tables_path: str Path to the folder containing the Tables folder with the json files default: object @@ -775,8 +775,8 @@ def _read_coordinate(self, value): coord.var_name = coord.name return coord - def _read_variable(self, short_name, frequency): - var = super()._read_variable(short_name, frequency) + def _read_variable(self, cmor_name, frequency): + var = super()._read_variable(cmor_name, frequency) var.modeling_realm = None var.frequency = None return var @@ -788,7 +788,7 @@ class CustomInfo(CMIP5Info): Parameters ---------- - cmor_tables_path: basestring or None + cmor_tables_path: str or None Full path to the table or name for the table if it is present in ESMValTool repository @@ -827,7 +827,7 @@ def get_table(self, table): Parameters ---------- - table: basestring + table: str Table name Returns @@ -839,16 +839,16 @@ def get_table(self, table): """ return self.tables.get(table) - def get_variable(self, table, short_name, derived=False): + def get_variable(self, table, cmor_name, derived=False): """ Search and return the variable info. Parameters ---------- - table: basestring + table: str Table name - short_name: basestring - Variable's short name + cmor_name: str + Variable's cmor name derived: bool, optional Variable is derived. Info retrieval is less strict @@ -859,7 +859,7 @@ def get_variable(self, table, short_name, derived=False): found, returns None if not """ - return self.tables['custom'].get(short_name, None) + return self.tables['custom'].get(cmor_name, None) def _read_table_file(self, table_file, table=None): with open(table_file) as self._current_table: diff --git a/esmvalcore/preprocessor/_derive/__init__.py b/esmvalcore/preprocessor/_derive/__init__.py index f5c82ce39e..5de65e498f 100644 --- a/esmvalcore/preprocessor/_derive/__init__.py +++ b/esmvalcore/preprocessor/_derive/__init__.py @@ -16,16 +16,16 @@ def _get_all_derived_variables(): Returns ------- dict - All derived variables with `short_name` (keys) and the associated + All derived variables with `cmor_name` (keys) and the associated python classes (values). """ derivers = {} for path in Path(__file__).parent.glob('[a-z]*.py'): - short_name = path.stem + cmor_name = path.stem module = importlib.import_module( - f'esmvalcore.preprocessor._derive.{short_name}') - derivers[short_name] = getattr(module, 'DerivedVariable') + f'esmvalcore.preprocessor._derive.{cmor_name}') + derivers[cmor_name] = getattr(module, 'DerivedVariable') return derivers @@ -49,7 +49,7 @@ def get_required(cmor_name, project): Returns ------- list - List of dictionaries (including at least the key `short_name`). + List of dictionaries (including at least the key `cmor_name`). """ if cmor_name not in ALL_DERIVED_VARIABLES: diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py index 3669a2b380..40d29769ee 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py @@ -2,11 +2,11 @@ import unittest from cf_units import Unit -from iris.cube import Cube from iris.coords import AuxCoord +from iris.cube import Cube -from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor._fixes.cmip5.access1_0 import AllVars +from esmvalcore.cmor._fixes.fix import Fix class TestAllVars(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py index a09b79f266..37cdc4ed8b 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py @@ -2,11 +2,11 @@ import unittest from cf_units import Unit -from iris.cube import Cube from iris.coords import AuxCoord +from iris.cube import Cube -from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor._fixes.cmip5.access1_3 import AllVars +from esmvalcore.cmor._fixes.fix import Fix class TestAllVars(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py index 30c0c44466..2767d2484f 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1.py @@ -1,8 +1,8 @@ """Test Access1-0 fixes.""" import unittest -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.bcc_csm1_1 import Tos +from esmvalcore.cmor.fix import Fix class TestTos(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py index 01f427f478..9ed1d26d08 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bcc_csm1_1_m.py @@ -1,8 +1,8 @@ """Test Access1-0 fixes.""" import unittest -from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor._fixes.cmip5.bcc_csm1_1_m import Tos +from esmvalcore.cmor._fixes.fix import Fix class TestTos(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py index 6fd29d47a2..bb309b635a 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py @@ -5,9 +5,10 @@ from cf_units import Unit from iris.cube import Cube +from esmvalcore.cmor._fixes.cmip5.bnu_esm import (Ch4, Co2, FgCo2, Od550Aer, + SpCo2) from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.bnu_esm import (Ch4, Co2, FgCo2, SpCo2, - Od550Aer) +from esmvalcore.cmor.table import get_var_info class TestCo2(unittest.TestCase): @@ -15,15 +16,17 @@ class TestCo2(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2(None) + self.vardef = get_var_info('CMIP5', 'Amon', self.cube.var_name) + self.fix = Co2(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'co2'), - [Co2(None)]) + [Co2(self.vardef)]) def test_fix_metadata(self): """Test unit change.""" + cube = self.fix.fix_metadata([self.cube])[0] self.assertEqual(cube.units, Unit('1e-6')) self.assertEqual(cube.data[0], 1.0) @@ -40,12 +43,14 @@ class Testfgco2(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='fgco2', units='J') - self.fix = FgCo2(None) + self.vardef = get_var_info('CMIP5', 'Omon', self.cube.var_name) + self.fix = FgCo2(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'fgco2'), [FgCo2(None)]) + Fix.get_fixes('CMIP5', 'BNU-ESM', 'Omon', 'fgco2'), + [FgCo2(self.vardef)]) def test_fix_metadata(self): """Test unit fix.""" @@ -65,12 +70,13 @@ class TestCh4(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='ch4', units='J') - self.fix = Ch4(None) + self.vardef = get_var_info('CMIP5', 'Amon', self.cube.var_name) + self.fix = Ch4(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual(Fix.get_fixes('CMIP5', 'BNU-ESM', 'Amon', 'ch4'), - [Ch4(None)]) + [Ch4(self.vardef)]) def test_fix_metadata(self): """Test unit fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py index ff9d75082e..68f0b1da77 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_canesm2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_canesm2.py @@ -4,8 +4,8 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.canesm2 import FgCo2 +from esmvalcore.cmor.fix import Fix class TestCanESM2Fgco2(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py index 1d326c5a03..a2ae2c0801 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ccsm4.py @@ -5,8 +5,9 @@ from iris.coords import DimCoord from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.ccsm4 import Rlut, Rlutcs, So +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info class TestsRlut(unittest.TestCase): @@ -74,12 +75,13 @@ class TestSo(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0, 2.0], var_name='so', units='1.0') - self.fix = So(None) + self.vardef = get_var_info('CMIP5', 'Omon', self.cube.var_name) + self.fix = So(self.vardef) def test_get(self): """Test fix get""" - self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Amon', 'so'), - [So(None)]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'CCSM4', 'Omon', 'so'), + [So(self.vardef)]) def test_fix_metadata(self): """Checks that units are changed to the correct value.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py index ae9da2ef04..deaacf5e84 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py @@ -4,8 +4,8 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.cnrm_cm5 import Msftmyz, Msftmyzba +from esmvalcore.cmor.fix import Fix class TestMsftmyz(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py index 8488632db6..906f1f89fc 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py @@ -5,8 +5,8 @@ from iris.coords import DimCoord from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.fgoals_g2 import AllVars +from esmvalcore.cmor.fix import Fix class TestAll(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py index 709f78ce24..f3f4eaca26 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py @@ -4,8 +4,8 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.fio_esm import Ch4, Co2 +from esmvalcore.cmor.fix import Fix class TestCh4(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py index 4540645c98..181b2c26ef 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py @@ -4,8 +4,9 @@ from cf_units import Unit from iris.cube import Cube +from esmvalcore.cmor._fixes.cmip5.gfdl_cm2p1 import AllVars, Areacello, Sftof from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.gfdl_cm2p1 import Sftof, AllVars, Areacello +from esmvalcore.cmor.table import get_var_info class TestSftof(unittest.TestCase): @@ -18,7 +19,7 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'sftof'), + Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'fx', 'sftof'), [Sftof(None), AllVars(None)]) def test_fix_data(self): @@ -33,13 +34,15 @@ class TestAreacello(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello(None) + self.vardef = get_var_info('CMIP5', 'fx', self.cube.var_name) + self.fix = Areacello(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'Amon', 'areacello'), - [Areacello(None), AllVars(None)]) + Fix.get_fixes('CMIP5', 'GFDL-CM2P1', 'fx', 'areacello'), + [Areacello(self.vardef), + AllVars(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py index fdd2f4c4c2..ccce289019 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py @@ -4,8 +4,9 @@ from cf_units import Unit from iris.cube import Cube +from esmvalcore.cmor._fixes.cmip5.gfdl_cm3 import AllVars, Areacello, Sftof from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.gfdl_cm3 import Sftof, AllVars, Areacello +from esmvalcore.cmor.table import get_var_info class TestSftof(unittest.TestCase): @@ -17,9 +18,8 @@ def setUp(self): def test_get(self): """Test fix get""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'sftof'), - [Sftof(None), AllVars(None)]) + self.assertListEqual(Fix.get_fixes('CMIP5', 'GFDL-CM3', 'fx', 'sftof'), + [Sftof(None), AllVars(None)]) def test_fix_data(self): """Test data fix.""" @@ -33,13 +33,15 @@ class TestAreacello(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello(None) + self.vardef = get_var_info('CMIP5', 'fx', self.cube.var_name) + self.fix = Areacello(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-CM3', 'Amon', 'areacello'), - [Areacello(None), AllVars(None)]) + Fix.get_fixes('CMIP5', 'GFDL-CM3', 'fx', 'areacello'), + [Areacello(self.vardef), + AllVars(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py index 8c24e35077..6d4ceaa796 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py @@ -11,6 +11,7 @@ FgCo2, Usi, Vsi, _get_and_remove) from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info CUBE_1 = iris.cube.Cube([1.0], long_name='to_be_rm') CUBE_2 = iris.cube.Cube([1.0], long_name='not_to_be_rm') @@ -84,13 +85,14 @@ class TestUsi(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = iris.cube.Cube([1.0], var_name='usi', units='J') - self.fix = Usi(None) + self.vardef = get_var_info('CMIP5', 'day', self.cube.var_name) + self.fix = Usi(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'usi'), - [Usi(None), AllVars(None)]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'day', 'usi'), + [Usi(self.vardef), AllVars(self.vardef)]) def test_fix_data(self): """Test metadata fix.""" @@ -103,13 +105,14 @@ class TestVsi(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = iris.cube.Cube([1.0], var_name='vsi', units='J') - self.fix = Vsi(None) + self.vardef = get_var_info('CMIP5', 'day', self.cube.var_name) + self.fix = Vsi(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'vsi'), - [Vsi(None), AllVars(None)]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'day', 'vsi'), + [Vsi(self.vardef), AllVars(self.vardef)]) def test_fix_data(self): """Test metadata fix.""" @@ -122,13 +125,15 @@ class TestAreacello(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello(None) + self.vardef = get_var_info('CMIP5', 'fx', self.cube.var_name) + self.fix = Areacello(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'Amon', 'areacello'), - [Areacello(None), AllVars(None)]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2G', 'fx', 'areacello'), + [Areacello(self.vardef), + AllVars(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py index 900da08a97..77c2e4f359 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py @@ -4,9 +4,10 @@ from cf_units import Unit from iris.cube import Cube +from esmvalcore.cmor._fixes.cmip5.gfdl_esm2m import (AllVars, Areacello, Co2, + Sftof) from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.gfdl_esm2m import Co2, Sftof, AllVars, \ - Areacello +from esmvalcore.cmor.table import get_var_info class TestSftof(unittest.TestCase): @@ -19,7 +20,7 @@ def setUp(self): def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'sftof'), + Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'fx', 'sftof'), [Sftof(None), AllVars(None)]) def test_fix_data(self): @@ -54,13 +55,15 @@ class TestAreacello(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='areacello', units='m-2') - self.fix = Areacello(None) + self.vardef = get_var_info('CMIP5', 'fx', self.cube.var_name) + self.fix = Areacello(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( - Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'Amon', 'areacello'), - [Areacello(None), AllVars(None)]) + Fix.get_fixes('CMIP5', 'GFDL-ESM2M', 'fx', 'areacello'), + [Areacello(self.vardef), + AllVars(self.vardef)]) def test_fix_metadata(self): """Test data fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py index ed2f7c0d1f..8d339bb276 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py @@ -1,8 +1,8 @@ """Test HADGEM2-CC fixes.""" import unittest +from esmvalcore.cmor._fixes.cmip5.hadgem2_cc import O2, AllVars from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.hadgem2_cc import AllVars, O2 class TestAllVars(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py index b6eea1cd25..65421a302d 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_es.py @@ -1,8 +1,8 @@ """Test HADGEM2-ES fixes.""" import unittest +from esmvalcore.cmor._fixes.cmip5.hadgem2_es import O2, AllVars from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor._fixes.cmip5.hadgem2_es import AllVars, O2 class TestAllVars(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py index 9da7f16103..daa0128168 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py +++ b/tests/integration/cmor/_fixes/cmip5/test_inmcm4.py @@ -1,11 +1,11 @@ """Tests for inmcm4 fixes.""" import unittest -from iris.cube import Cube from cf_units import Unit +from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.inmcm4 import Gpp, Lai, Nbp +from esmvalcore.cmor.fix import Fix class TestGpp(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py index 51d887c597..1bd24632b1 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm.py @@ -6,8 +6,9 @@ from iris.cube import Cube from iris.exceptions import CoordinateNotFoundError -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.miroc_esm import AllVars, Co2, Tro3 +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info class TestCo2(unittest.TestCase): @@ -15,13 +16,14 @@ class TestCo2(unittest.TestCase): def setUp(self): """Prepare tests.""" self.cube = Cube([1.0], var_name='co2', units='J') - self.fix = Co2(None) + self.vardef = get_var_info('CMIP5', 'Amon', self.cube.var_name) + self.fix = Co2(self.vardef) def test_get(self): """Test fix get""" self.assertListEqual( Fix.get_fixes('CMIP5', 'MIROC-ESM', 'Amon', 'co2'), - [Co2(None), AllVars(None)]) + [Co2(self.vardef), AllVars(self.vardef)]) def test_fix_metadata(self): """Test unit fix.""" diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py index 186f321710..a98318f9e4 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py @@ -5,8 +5,8 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.miroc_esm_chem import Tro3 +from esmvalcore.cmor.fix import Fix class TestTro3(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py index f8c90ec3e2..0654b22db2 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mpi_esm_lr.py @@ -4,8 +4,8 @@ from cf_units import Unit from iris.cube import Cube -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.mpi_esm_lr import Pctisccp +from esmvalcore.cmor.fix import Fix class TestPctisccp2(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py index 1b93fb3004..bf697dec95 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_cgcm3.py @@ -1,8 +1,8 @@ """Test MRI-GCM3 fixes.""" import unittest -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.mri_cgcm3 import Msftmyz, ThetaO +from esmvalcore.cmor.fix import Fix class TestMsftmyz(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py index c00af1d6fe..e30fceebc0 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py @@ -1,8 +1,8 @@ """Test MRI-ESM1 fixes.""" import unittest -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.mri_esm1 import Msftmyz +from esmvalcore.cmor.fix import Fix class TestMsftmyz(unittest.TestCase): diff --git a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py index ab286d78bd..8e57166c85 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py +++ b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py @@ -1,10 +1,10 @@ """Tests for fixes of NorESM1-ME (CMIP5).""" -import pytest import iris +import pytest from iris.cube import CubeList -from esmvalcore.cmor.fix import Fix from esmvalcore.cmor._fixes.cmip5.noresm1_me import Tas +from esmvalcore.cmor.fix import Fix DIM_COORD_SHORT = iris.coords.DimCoord( [1.0, 2.0, 3.0], diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 826d2f6118..997f94acaa 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -5,6 +5,7 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info @pytest.fixture @@ -29,7 +30,8 @@ def test_tas_fix_metadata(tas_cubes): long_name='height', units=Unit('m'), attributes={'positive': 'up'}) - fix = Tas(None) + vardef = get_var_info('CMIP6', 'Amon', 'tas') + fix = Tas(vardef) out_cubes = fix.fix_metadata(tas_cubes) assert out_cubes is tas_cubes for cube in out_cubes: diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index f5410a8393..f479416425 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -5,6 +5,7 @@ from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import Tas from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info @pytest.fixture @@ -29,7 +30,8 @@ def test_tas_fix_metadata(tas_cubes): long_name='height', units=Unit('m'), attributes={'positive': 'up'}) - fix = Tas(None) + vardef = get_var_info('CMIP6', 'Amon', 'tas') + fix = Tas(vardef) out_cubes = fix.fix_metadata(tas_cubes) assert out_cubes is tas_cubes for cube in out_cubes: diff --git a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py index 5edc8b8554..d47b065537 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_ipsl_cm6a_lr.py @@ -1,9 +1,8 @@ import unittest import numpy as np - -from iris.cube import Cube, CubeList from iris.coords import AuxCoord +from iris.cube import Cube, CubeList from iris.exceptions import CoordinateNotFoundError from esmvalcore.cmor._fixes.cmip6.ipsl_cm6a_lr import AllVars diff --git a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py index 2239576ec5..58e1cc592f 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py +++ b/tests/integration/cmor/_fixes/cmip6/test_mcm_ua_1_0.py @@ -5,6 +5,7 @@ from esmvalcore.cmor._fixes.cmip6.mcm_ua_1_0 import AllVars, Tas from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info @pytest.fixture @@ -80,7 +81,8 @@ def test_tas_fix_metadata(cubes): long_name='height', units=Unit('m'), attributes={'positive': 'up'}) - fix = Tas(None) + vardef = get_var_info('CMIP6', 'Amon', 'tas') + fix = Tas(vardef) # Check fix out_cubes = fix.fix_metadata(cubes) diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index a6e1d33444..586a68a0e3 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -73,7 +73,7 @@ def test_derive_noop(): assert cube is alb -def test_derive_mixed_case_with_fx(tmp_path, monkeypatch): +def test_derive_mixed_case_with_fx(monkeypatch): short_name = 'ohc' long_name = 'Heat content in grid cell' diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 969c63b03a..2a3eff8f46 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -155,8 +155,8 @@ def find_files(_, filenames): @pytest.fixture def patched_tas_derivation(monkeypatch): - def get_required(short_name, _): - if short_name != 'tas': + def get_required(cmor_name, _): + if cmor_name != 'tas': assert False required = [ {'cmor_name': 'pr'}, @@ -527,6 +527,7 @@ def test_cmip3_variable_autocomplete(tmp_path, patched_datafinder, variable = recipe.diagnostics['test']['preprocessor_output']['zg'][0] reference = { + 'cmor_name': 'zg', 'dataset': 'bccr_bcm2_0', 'diagnostic': 'test', 'end_year': 2001, @@ -571,6 +572,7 @@ def test_cmip5_variable_autocomplete(tmp_path, patched_datafinder, variable = recipe.diagnostics['test']['preprocessor_output']['pr'][0] reference = { + 'cmor_name': 'pr', 'dataset': 'CanESM2', 'diagnostic': 'test', 'end_year': 2001, @@ -583,7 +585,6 @@ def test_cmip5_variable_autocomplete(tmp_path, patched_datafinder, 'modeling_realm': ['atmos'], 'preprocessor': 'default', 'project': 'CMIP5', - 'cmor_name': 'pr', 'standard_name': 'precipitation_flux', 'start_year': 2000, 'units': 'kg m-2 s-1', @@ -617,6 +618,7 @@ def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, reference = { 'activity': 'CMIP', + 'cmor_name': 'pr', 'dataset': 'HadGEM3-GC31-LL', 'diagnostic': 'test', 'end_year': 2001, @@ -630,7 +632,6 @@ def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, 'modeling_realm': ['atmos'], 'preprocessor': 'default', 'project': 'CMIP6', - 'cmor_name': 'pr', 'standard_name': 'precipitation_flux', 'start_year': 2000, 'units': 'kg m-2 s-1', diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index a016636df0..d4e80c43e7 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -53,7 +53,8 @@ def setUp(self): self.cube_2 = Mock() self.cube_2.var_name = 'cube2' self.cubes = [self.cube_1, self.cube_2] - self.fix = Fix(None) + self.vardef = Mock() + self.fix = Fix(self.vardef) def test_get_first_cube(self): """Test selecting first cube.""" @@ -73,6 +74,7 @@ def test_get_default_raises(self): def test_get_default(self): """Check that the default return the cube (fix is a cube).""" self.cube_1.var_name = 'fix' + self.vardef.short_name = self.cube_1.var_name self.assertIs(self.cube_1, self.fix.get_cube_from_list(self.cubes)) @@ -189,7 +191,7 @@ def test_cmor_checker_called(self): checker = Mock() checker.return_value = Mock() var_info = Mock() - var_info.shoart_name = 'short_name' + var_info.short_name = 'short_name' with patch('esmvalcore.cmor._fixes.fix.Fix.get_fixes', return_value=[]): with patch('esmvalcore.cmor.fix._get_cmor_checker', diff --git a/tests/unit/cmor/test_table.py b/tests/unit/cmor/test_table.py index 32239f7a96..424c25faf8 100644 --- a/tests/unit/cmor/test_table.py +++ b/tests/unit/cmor/test_table.py @@ -16,7 +16,7 @@ def test_constructor(self): """Test basic constructor.""" info = VariableInfo('table_type', 'var') self.assertEqual('table_type', info.table_type) - self.assertEqual('var', info.short_name) + self.assertEqual('var', info.cmor_name) def test_read_empty_dictionary(self): """Test read empty dict.""" @@ -24,6 +24,12 @@ def test_read_empty_dictionary(self): info.read_json({}, '') self.assertEqual('', info.standard_name) + def test_read_short_name(self): + """Test out_name.""" + info = VariableInfo('table_type', 'var') + info.read_json({'out_name': self.value}, '') + self.assertEqual(info.short_name, self.value) + def test_read_standard_name(self): """Test standard_name.""" info = VariableInfo('table_type', 'var') From aac179d5197a6a993234ade3a3d942c8c99bf7e2 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 26 Feb 2020 17:13:19 +0100 Subject: [PATCH 10/11] Minor docstring improvements --- esmvalcore/cmor/_fixes/fix.py | 8 ++++---- esmvalcore/cmor/check.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index c32972abae..a5a0cd7ae8 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -15,7 +15,7 @@ def __init__(self, vardef): Parameters ---------- - vardef: basestring + vardef: esmvalcore.cmor.table.VariableInfo CMOR table entry """ @@ -31,14 +31,14 @@ def fix_file(self, filepath, output_dir): Parameters ---------- - filepath: basestring + filepath: str file to fix - output_dir: basestring + output_dir: str path to the folder to store the fixe files, if required Returns ------- - basestring + str Path to the corrected file. It can be different from the original filepath if a fix has been applied, but if not it should be the original filepath diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 91b750dbd2..bd66616bf3 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -742,13 +742,13 @@ def cmor_check_metadata(cube, cmor_table, mip, cmor_name, frequency): ---------- cube: iris.cube.Cube Data cube to check. - cmor_table: basestring + cmor_table: str CMOR definitions to use. mip: Variable's mip. - cmor_name: basestring + cmor_name: str Variable's name. - frequency: basestring + frequency: str Data frequency. """ @@ -766,13 +766,13 @@ def cmor_check_data(cube, cmor_table, mip, cmor_name, frequency): ---------- cube: iris.cube.Cube Data cube to check. - cmor_table: basestring + cmor_table: str CMOR definitions to use. mip: Variable's mip. - cmor_name: basestring + cmor_name: str Variable's name - frequency: basestring + frequency: str Data frequency """ @@ -791,13 +791,13 @@ def cmor_check(cube, cmor_table, mip, cmor_name, frequency): ---------- cube: iris.cube.Cube Data cube to check. - cmor_table: basestring + cmor_table: str CMOR definitions to use. mip: Variable's mip. - cmor_name: basestring + cmor_name: str Variable's name. - frequency: basestring + frequency: str Data frequency. """ From ed24b2423dccf531cd2567259cc8a32cddb9fa5b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 1 Jul 2020 17:46:54 +0200 Subject: [PATCH 11/11] Fixing tests --- esmvalcore/cmor/fix.py | 64 +++++++++---------- .../cmor/_fixes/cmip6/test_cesm2_waccm.py | 10 ++- tests/unit/cmor/test_fix.py | 41 ++++++------ 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 88b74b1355..50ddb6724e 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -45,9 +45,10 @@ def fix_file(file, cmor_name, project, dataset, mip, output_dir): Path to the fixed file """ - for fix in Fix.get_fixes( - project=project, dataset=dataset, mip=mip, - cmor_name=cmor_name): + for fix in Fix.get_fixes(project=project, + dataset=dataset, + mip=mip, + cmor_name=cmor_name): file = fix.fix_file(file, output_dir) return file @@ -96,8 +97,10 @@ def fix_metadata(cubes, If the checker detects errors in the metadata that it can not fix. """ - fixes = Fix.get_fixes( - project=project, dataset=dataset, mip=mip, cmor_name=cmor_name) + fixes = Fix.get_fixes(project=project, + dataset=dataset, + mip=mip, + cmor_name=cmor_name) short_name = get_var_info(project, mip, cmor_name).short_name fixed_cubes = [] by_file = defaultdict(list) @@ -109,27 +112,26 @@ def fix_metadata(cubes, for fix in fixes: cube_list = fix.fix_metadata(cube_list) - cube = _get_single_cube(cube_list, cmor_name, project, dataset) - checker = _get_cmor_checker( - frequency=frequency, - table=project, - mip=mip, - cmor_name=cmor_name, - check_level=check_level, - fail_on_error=False, - automatic_fixes=True) + cube = _get_single_cube(cube_list, short_name, project, dataset) + checker = _get_cmor_checker(frequency=frequency, + table=project, + mip=mip, + cmor_name=cmor_name, + check_level=check_level, + fail_on_error=False, + automatic_fixes=True) cube = checker(cube).check_metadata() cube.attributes.pop('source_file', None) fixed_cubes.append(cube) return fixed_cubes -def _get_single_cube(cube_list, cmor_name, project, dataset): +def _get_single_cube(cube_list, short_name, project, dataset): if len(cube_list) == 1: return cube_list[0] cube = None for raw_cube in cube_list: - if raw_cube.var_name == cmor_name: + if raw_cube.var_name == short_name: cube = raw_cube break if not cube: @@ -137,19 +139,14 @@ def _get_single_cube(cube_list, cmor_name, project, dataset): 'More than one cube found for variable %s in %s:%s but ' 'none of their var_names match the expected. \n' 'Full list of cubes encountered: %s' % - (cmor_name, project, dataset, cube_list) - ) + (short_name, project, dataset, cube_list)) logger.warning( 'Found variable %s in %s:%s, but there were other present in ' 'the file. Those extra variables are usually metadata ' '(cell area, latitude descriptions) that was not saved ' 'according to CF-conventions. It is possible that errors appear ' 'further on because of this. \nFull list of cubes encountered: %s', - cmor_name, - project, - dataset, - cube_list - ) + short_name, project, dataset, cube_list) return cube @@ -196,16 +193,17 @@ def fix_data(cube, If the checker detects errors in the data that it can not fix. """ - for fix in Fix.get_fixes( - project=project, dataset=dataset, mip=mip, cmor_name=cmor_name): + for fix in Fix.get_fixes(project=project, + dataset=dataset, + mip=mip, + cmor_name=cmor_name): cube = fix.fix_data(cube) - checker = _get_cmor_checker( - frequency=frequency, - table=project, - mip=mip, - cmor_name=cmor_name, - fail_on_error=False, - automatic_fixes=True, - check_level=check_level) + checker = _get_cmor_checker(frequency=frequency, + table=project, + mip=mip, + cmor_name=cmor_name, + fail_on_error=False, + automatic_fixes=True, + check_level=check_level) cube = checker(cube).check_data() return cube diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index 2e885f4308..0b4173396a 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -10,7 +10,6 @@ from esmvalcore.cmor._fixes.cmip6.cesm2 import Tas as BaseTas from esmvalcore.cmor._fixes.cmip6.cesm2_waccm import Cl, Cli, Clw, Tas from esmvalcore.cmor.fix import Fix -from esmvalcore.cmor.table import get_var_info @pytest.fixture @@ -22,7 +21,7 @@ def cl_file(tmp_path): dataset.createDimension('bnds', size=2) # Dimensional variables - dataset.createVariable('lev', np.float64, dimensions=('lev',)) + dataset.createVariable('lev', np.float64, dimensions=('lev', )) dataset.createVariable('lev_bnds', np.float64, dimensions=('lev', 'bnds')) dataset.variables['lev'][:] = [1.0, 2.0] dataset.variables['lev'].bounds = 'lev_bnds' @@ -35,9 +34,9 @@ def cl_file(tmp_path): 'p0: p0 a: a_bnds b: b_bnds ps: ps') # Coordinates for derivation of pressure coordinate - dataset.createVariable('a', np.float64, dimensions=('lev',)) + dataset.createVariable('a', np.float64, dimensions=('lev', )) dataset.createVariable('a_bnds', np.float64, dimensions=('lev', 'bnds')) - dataset.createVariable('b', np.float64, dimensions=('lev',)) + dataset.createVariable('b', np.float64, dimensions=('lev', )) dataset.createVariable('b_bnds', np.float64, dimensions=('lev', 'bnds')) dataset.variables['a'][:] = [1.0, 2.0] dataset.variables['a'].bounds = 'a_bnds' @@ -57,8 +56,7 @@ def test_get_cl_fix(): @unittest.mock.patch( - 'esmvalcore.cmor._fixes.cmip6.cesm2.Fix.get_fixed_filepath', - autospec=True) + 'esmvalcore.cmor._fixes.cmip6.cesm2.Fix.get_fixed_filepath', autospec=True) def test_cl_fix_file(mock_get_filepath, cl_file, tmp_path): """Test ``fix_file`` for ``cl``.""" mock_get_filepath.return_value = os.path.join(tmp_path, diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index c68344c6f1..3a8fd0baa2 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -76,7 +76,6 @@ def test_get_default_raises(self): def test_get_default(self): """Check that the default return the cube (fix is a cube).""" self.cube_1.var_name = 'fix' - self.vardef.short_name = self.cube_1.var_name self.assertIs(self.cube_1, self.fix.get_cube_from_list(self.cubes)) @@ -156,9 +155,8 @@ def test_select_var(self): with patch('esmvalcore.cmor.fix.get_var_info', return_value=var_info): cube_returned = fix_metadata( - cubes=[ - self.cube, - self._create_mock_cube('extra')], + cubes=[self.cube, + self._create_mock_cube('extra')], cmor_name='cmor_name', project='CMIP6', dataset='model', @@ -198,22 +196,25 @@ def test_cmor_checker_called(self): return_value=[]): with patch('esmvalcore.cmor.fix._get_cmor_checker', return_value=checker) as get_mock: - fix_metadata( - cubes=[self.cube], - cmor_name='cmor_name', - project='CMIP6', - dataset='dataset', - mip='mip', - frequency='frequency', - ) - get_mock.assert_called_once_with( - automatic_fixes=True, - fail_on_error=False, - frequency='frequency', - mip='mip', - cmor_name='cmor_name', - table='CMIP6', - check_level=CheckLevels.DEFAULT,) + with patch('esmvalcore.cmor.table.get_var_info', + return_value=var_info): + fix_metadata( + cubes=[self.cube], + cmor_name='cmor_name', + project='CMIP6', + dataset='dataset', + mip='mip', + frequency='frequency', + ) + get_mock.assert_called_once_with( + automatic_fixes=True, + fail_on_error=False, + frequency='frequency', + mip='mip', + cmor_name='cmor_name', + table='CMIP6', + check_level=CheckLevels.DEFAULT, + ) checker.assert_called_once_with(self.cube) checker.return_value.check_metadata.assert_called_once_with()