Skip to content

Add cmor_name as an extra tag for getting variable info #1099

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jvegreg wants to merge 4 commits into from
Closed

Add cmor_name as an extra tag for getting variable info #1099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
716f744
Add cmor_name as an extra tag for getting variable info
May 5, 2021
ce605df
Revert unwanted change
May 5, 2021
3f99450
Update doc
May 5, 2021
b479909
Update esmvalcore/cmor/check.py
May 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions doc/recipe/overview.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,14 @@ while use ``sic`` as their short_name. If the recipe requested ``siconc``, the
preprocessed files will be identical except that they will use the short_name
``siconc`` instead.

Another uncommon case that may be a problem sometimes is that, for some
variables, the ``out_name`` and the name of the entry in the CMOR tables do not
match, like in ``zg7h`` and ``zg27`` in CMIP6 ``6hrPlevPt``, two versions of
``zg`` that are mutually exclusive and have different requirements for the
levels. To allow ESMValCore to retrieve the correct information in those cases,
the ``cmor_name`` must be defined and ``short_name`` will be reserved for the
``out_name`` of the variable.

Diagnostic and variable specific datasets
-----------------------------------------
The ``additional_datasets`` option can be used to add datasets beyond those
Expand Down
21 changes: 14 additions & 7 deletions esmvalcore/_recipe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def _add_cmor_info(variable, override=False):
derive = variable.get('derive', False)
table = CMOR_TABLES.get(project)
if table:
table_entry = table.get_variable(mip, short_name, derive)
table_entry = table.get_variable(mip, variable['cmor_name'], derive)
else:
table_entry = None
if table_entry is None:
Expand Down Expand Up @@ -141,7 +141,7 @@ def _update_target_levels(variable, variables, settings, config_user):
filename=filename,
project=variable_data['project'],
dataset=dataset,
short_name=variable_data['short_name'],
cmor_name=variable_data['cmor_name'],
mip=variable_data['mip'],
frequency=variable_data['frequency'],
fix_dir=os.path.splitext(variable_data['filename'])[0] +
Expand Down Expand Up @@ -201,7 +201,7 @@ def _dataset_to_file(variable, config_user):
"""Find the first file belonging to dataset from variable info."""
(files, dirnames, filenames) = _get_input_files(variable, config_user)
if not files and variable.get('derive'):
required_vars = get_required(variable['short_name'],
required_vars = get_required(variable['cmor_name'],
variable['project'])
for required_var in required_vars:
_augment(required_var, variable)
Expand Down Expand Up @@ -264,7 +264,7 @@ def _get_default_settings(variable, config_user, derive=False):
fix = {
'project': variable['project'],
'dataset': variable['dataset'],
'short_name': variable['short_name'],
'cmor_name': variable['cmor_name'],
'mip': variable['mip'],
}
# File fixes
Expand All @@ -287,6 +287,7 @@ def _get_default_settings(variable, config_user, derive=False):

if derive:
settings['derive'] = {
'cmor_name': variable['cmor_name'],
'short_name': variable['short_name'],
'standard_name': variable['standard_name'],
'long_name': variable['long_name'],
Expand All @@ -297,7 +298,7 @@ def _get_default_settings(variable, config_user, derive=False):
settings['cmor_check_metadata'] = {
'cmor_table': variable['project'],
'mip': variable['mip'],
'short_name': variable['short_name'],
'cmor_name': variable['cmor_name'],
'frequency': variable['frequency'],
'check_level': config_user.get('check_level', CheckLevels.DEFAULT)
}
Expand Down Expand Up @@ -330,6 +331,8 @@ def _add_fxvar_keys(fx_info, variable):
fx_variable = deepcopy(variable)
fx_variable.update(fx_info)
fx_variable['variable_group'] = fx_info['short_name']
if 'cmor_name' not in fx_variable:
fx_variable['cmor_name'] = fx_info['short_name']

# add special ensemble for CMIP5 only
if fx_variable['project'] == 'CMIP5':
Expand All @@ -344,7 +347,7 @@ def _add_fxvar_keys(fx_info, variable):
def _search_fx_mip(tables, found_mip, variable, fx_info, config_user):
fx_files = None
for mip in tables:
fx_cmor = tables[mip].get(fx_info['short_name'])
fx_cmor = tables[mip].get(fx_info['cmor_name'])
if fx_cmor:
found_mip = True
fx_info['mip'] = mip
Expand Down Expand Up @@ -381,7 +384,7 @@ def _get_fx_files(variable, fx_info, config_user):
found_mip, fx_info, fx_files = _search_fx_mip(
project_tables, found_mip, variable, fx_info, config_user)
else:
fx_cmor = project_tables[fx_info['mip']].get(fx_info['short_name'])
fx_cmor = project_tables[fx_info['mip']].get(fx_info['cmor_name'])
if fx_cmor:
found_mip = True
fx_info = _add_fxvar_keys(fx_info, variable)
Expand Down Expand Up @@ -436,6 +439,8 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars):
fx_info.update({'mip': None})
if 'short_name' not in fx_info:
fx_info.update({'short_name': fx_var})
if 'cmor_name' not in fx_info:
fx_info.update({'cmor_name': fx_info['short_name']})
fx_files, fx_info = _get_fx_files(variable, fx_info, config_user)
if fx_files:
fx_info['filename'] = fx_files
Expand Down Expand Up @@ -1097,6 +1102,8 @@ def _initialize_variables(self, raw_variable, raw_datasets):
if activity:
variable['activity'] = activity
check.variable(variable, required_keys)
if 'cmor_name' not in variable:
variable['cmor_name'] = variable['short_name']
variables = self._expand_ensemble(variables)
return variables

Expand Down
10 changes: 5 additions & 5 deletions esmvalcore/cmor/_fixes/fix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def __ne__(self, other):
return not self.__eq__(other)

@staticmethod
def get_fixes(project, dataset, mip, short_name):
def get_fixes(project, dataset, mip, cmor_name):
"""
Get the fixes that must be applied for a given dataset.

Expand All @@ -141,19 +141,19 @@ def get_fixes(project, dataset, mip, short_name):
project: str
dataset: str
mip: str
short_name: str
cmor_name: str

Returns
-------
list(Fix)
Fixes to apply for the given data
"""
cmor_table = CMOR_TABLES[project]
vardef = cmor_table.get_variable(mip, short_name)
vardef = cmor_table.get_variable(mip, cmor_name)

project = project.replace('-', '_').lower()
dataset = dataset.replace('-', '_').lower()
short_name = short_name.replace('-', '_').lower()
cmor_name = cmor_name.replace('-', '_').lower()

fixes = []
try:
Expand All @@ -162,7 +162,7 @@ def get_fixes(project, dataset, mip, short_name):

classes = inspect.getmembers(fixes_module, inspect.isclass)
classes = dict((name.lower(), value) for name, value in classes)
for fix_name in (short_name, mip.lower(), 'allvars'):
for fix_name in (cmor_name, mip.lower(), 'allvars'):
try:
fixes.append(classes[fix_name](vardef))
except KeyError:
Expand Down
32 changes: 16 additions & 16 deletions esmvalcore/cmor/check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -939,7 +939,7 @@ def report_debug_message(self, message, *args):

def _get_cmor_checker(table,
mip,
short_name,
cmor_name,
frequency,
fail_on_error=False,
check_level=CheckLevels.DEFAULT,
Expand All @@ -952,9 +952,9 @@ def _get_cmor_checker(table,
table, ', '.join(CMOR_TABLES)))

cmor_table = CMOR_TABLES[table]
var_info = cmor_table.get_variable(mip, short_name)
var_info = cmor_table.get_variable(mip, cmor_name)
if var_info is None:
var_info = CMOR_TABLES['custom'].get_variable(mip, short_name)
var_info = CMOR_TABLES['custom'].get_variable(mip, cmor_name)

def _checker(cube):
return CMORCheck(cube,
Expand All @@ -970,7 +970,7 @@ def _checker(cube):
def cmor_check_metadata(cube,
cmor_table,
mip,
short_name,
cmor_name,
frequency,
check_level=CheckLevels.DEFAULT):
"""Check if metadata conforms to variable's CMOR definition.
Expand All @@ -985,16 +985,16 @@ def cmor_check_metadata(cube,
CMOR definitions to use.
mip:
Variable's mip.
short_name: basestring
Variable's short name.
cmor_name: str
Variable's cmor name.
frequency: basestring
Data frequency.
check_level: CheckLevels
Level of strictness of the checks.
"""
checker = _get_cmor_checker(cmor_table,
mip,
short_name,
cmor_name,
frequency,
check_level=check_level)
checker(cube).check_metadata()
Expand All @@ -1004,7 +1004,7 @@ def cmor_check_metadata(cube,
def cmor_check_data(cube,
cmor_table,
mip,
short_name,
cmor_name,
frequency,
check_level=CheckLevels.DEFAULT):
"""Check if data conforms to variable's CMOR definition.
Expand All @@ -1019,23 +1019,23 @@ def cmor_check_data(cube,
CMOR definitions to use.
mip:
Variable's mip.
short_name: basestring
Variable's short name
cmor_name: basestring
Variable's cmor name
frequency: basestring
Data frequency
check_level: CheckLevels
Level of strictness of the checks.
"""
checker = _get_cmor_checker(cmor_table,
mip,
short_name,
cmor_name,
frequency,
check_level=check_level)
checker(cube).check_data()
return cube


def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level):
def cmor_check(cube, cmor_table, mip, cmor_name, frequency, check_level):
"""Check if cube conforms to variable's CMOR definition.

Equivalent to calling cmor_check_metadata and cmor_check_data
Expand All @@ -1049,8 +1049,8 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level):
CMOR definitions to use.
mip:
Variable's mip.
short_name: basestring
Variable's short name.
cmor_name: basestring
Variable's cmor name.
frequency: basestring
Data frequency.
check_level: enum.IntEnum
Expand All @@ -1059,13 +1059,13 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level):
cmor_check_metadata(cube,
cmor_table,
mip,
short_name,
cmor_name,
frequency,
check_level=check_level)
cmor_check_data(cube,
cmor_table,
mip,
short_name,
cmor_name,
frequency,
check_level=check_level)
return cube
31 changes: 17 additions & 14 deletions esmvalcore/cmor/fix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
for the given dataset. Therefore is recommended to apply them to all
variables to be sure that all known errors are fixed.
"""
from esmvalcore.cmor.table import get_var_info
import logging
from collections import defaultdict

Expand All @@ -15,7 +16,7 @@
logger = logging.getLogger(__name__)


def fix_file(file, short_name, project, dataset, mip, output_dir):
def fix_file(file, cmor_name, project, dataset, mip, output_dir):
"""Fix files before ESMValTool can load them.

This fixes are only for issues that prevent iris from loading the cube or
Expand All @@ -27,7 +28,7 @@ def fix_file(file, short_name, project, dataset, mip, output_dir):
----------
file: str
Path to the original file
short_name: str
cmor_name: str
Variable's short name
project: str
dataset:str
Expand All @@ -42,13 +43,13 @@ def fix_file(file, short_name, project, dataset, mip, output_dir):
for fix in Fix.get_fixes(project=project,
dataset=dataset,
mip=mip,
short_name=short_name):
cmor_name=cmor_name):
file = fix.fix_file(file, output_dir)
return file


def fix_metadata(cubes,
short_name,
cmor_name,
project,
dataset,
mip,
Expand All @@ -65,8 +66,8 @@ def fix_metadata(cubes,
----------
cubes: iris.cube.CubeList
Cubes to fix
short_name: str
Variable's short name
cmor_name: str
Variable's cmor name
project: str

dataset: str
Expand All @@ -92,22 +93,24 @@ def fix_metadata(cubes,
fixes = Fix.get_fixes(project=project,
dataset=dataset,
mip=mip,
short_name=short_name)
cmor_name=cmor_name)
fixed_cubes = []
by_file = defaultdict(list)
for cube in cubes:
by_file[cube.attributes.get('source_file', '')].append(cube)

vardef = get_var_info(project, mip, cmor_name)

for cube_list in by_file.values():
cube_list = CubeList(cube_list)
for fix in fixes:
cube_list = fix.fix_metadata(cube_list)

cube = _get_single_cube(cube_list, short_name, project, dataset)
cube = _get_single_cube(cube_list, vardef.short_name, project, dataset)
checker = _get_cmor_checker(frequency=frequency,
table=project,
mip=mip,
short_name=short_name,
cmor_name=cmor_name,
check_level=check_level,
fail_on_error=False,
automatic_fixes=True)
Expand Down Expand Up @@ -142,7 +145,7 @@ def _get_single_cube(cube_list, short_name, project, dataset):


def fix_data(cube,
short_name,
cmor_name,
project,
dataset,
mip,
Expand All @@ -161,8 +164,8 @@ def fix_data(cube,
----------
cube: iris.cube.Cube
Cube to fix
short_name: str
Variable's short name
cmor_name: str
Variable's cmor name
project: str
dataset: str
mip: str
Expand All @@ -185,12 +188,12 @@ def fix_data(cube,
for fix in Fix.get_fixes(project=project,
dataset=dataset,
mip=mip,
short_name=short_name):
cmor_name=cmor_name):
cube = fix.fix_data(cube)
checker = _get_cmor_checker(frequency=frequency,
table=project,
mip=mip,
short_name=short_name,
cmor_name=cmor_name,
fail_on_error=False,
automatic_fixes=True,
check_level=check_level)
Expand Down
Loading