Distinguish between out_name and var_name

doc/develop/derivation.rst

@@ -27,8 +27,8 @@ A typical example looks like this:
            if project == 'CMIP6':
                mip = 'Ofx'
            required = [
-               {'short_name': 'var_a'},
-               {'short_name': 'var_b', 'mip': mip, 'optional': True},
+               {'cmor_name': 'var_a'},
+               {'cmor_name': 'var_b', 'mip': mip, 'optional': True},
            ]
            return required
 

doc/develop/fixing_data.rst

@@ -74,7 +74,7 @@ coordinates. In our example it looks like this:
             month_number                             x              -              -
             year                                     x              -              -
         Attributes:
-            {'cmor_table': 'CMIPX', 'mip': 'Amon', 'short_name': 'tas', 'frequency': 'mon'})
+            {'cmor_table': 'CMIPX', 'mip': 'Amon', 'cmor_name': 'tas', 'frequency': 'mon'})
 
 
 So now the mistake is clear: the latitude coordinate is badly named and the
@@ -140,7 +140,7 @@ so we will implement the ``fix_metadata`` method:
             latitude = tas_cube.coord('altitude')
 
             # Fix the names. Latitude values, units and
-            latitude.short_name = 'lat'
+            latitude.var_name = 'lat'
             latitude.standard_name = 'latitude'
             latitude.long_name = 'latitude'
             return cubes
@@ -200,7 +200,7 @@ from the one you just created:
             latitude = tas_cube.coord('altitude')
 
             # Fix the names. Latitude values, units and
-            latitude.short_name = 'lat'
+            latitude.var_name = 'lat'
             latitude.standard_name = 'latitude'
             latitude.long_name = 'latitude'
             return cubes

doc/recipe/overview.rst

@@ -227,7 +227,7 @@ A (simplified) example diagnostics section could look like
         - atmos
       variables:
         variable_name:
-          short_name: ta
+          cmor_name: ta
           preprocessor: preprocessor_name
           mip: Amon
       scripts:
@@ -271,7 +271,7 @@ that wildcard expansion can be used to define ancestors.
     diagnostic_1:
       variables:
         airtemp:
-          short_name: ta
+          cmor_name: ta
           preprocessor: preprocessor_name
           mip: Amon
       scripts:
@@ -280,7 +280,7 @@ that wildcard expansion can be used to define ancestors.
     diagnostic_2:
       variables:
         precip:
-          short_name: pr
+          cmor_name: pr
           preprocessor: preprocessor_name
           mip: Amon
       scripts:

esmvalcore/_recipe.py

@@ -63,21 +63,22 @@ def _add_cmor_info(variable, override=False):
     logger.debug("If not present: adding keys from CMOR table to %s", variable)
     # Copy the following keys from CMOR table
     cmor_keys = [
-        'standard_name', 'long_name', 'units', 'modeling_realm', 'frequency'
+        'cmor_name', 'short_name', 'standard_name', 'long_name', 'units',
+        'modeling_realm', 'frequency'
     ]
     project = variable['project']
     mip = variable['mip']
-    short_name = variable['short_name']
+    cmor_name = variable['cmor_name']
     derive = variable.get('derive', False)
     table = CMOR_TABLES.get(project)
     if table:
-        table_entry = table.get_variable(mip, short_name, derive)
+        table_entry = table.get_variable(mip, cmor_name, derive)
     else:
         table_entry = None
     if table_entry is None:
         raise RecipeError(
             f"Unable to load CMOR table (project) '{project}' for variable "
-            f"'{short_name}' with mip '{mip}'")
+            f"'{cmor_name}' with mip '{mip}'")
     variable['original_short_name'] = table_entry.short_name
     for key in cmor_keys:
         if key not in variable or override:
@@ -99,11 +100,11 @@ def _special_name_to_dataset(variable, special_name):
         if special_name not in variable:
             raise RecipeError(
                 "Preprocessor {preproc} uses {name}, but {name} is not "
-                "defined for variable {short_name} of diagnostic "
+                "defined for variable {cmor_name} of diagnostic "
                 "{diagnostic}".format(
                     preproc=variable['preprocessor'],
                     name=special_name,
-                    short_name=variable['short_name'],
+                    cmor_name=variable['cmor_name'],
                     diagnostic=variable['diagnostic'],
                 ))
         special_name = variable[special_name]
@@ -141,7 +142,7 @@ def _update_target_levels(variable, variables, settings, config_user):
                 filename=filename,
                 project=variable_data['project'],
                 dataset=dataset,
-                short_name=variable_data['short_name'],
+                cmor_name=variable_data['cmor_name'],
                 mip=variable_data['mip'],
                 frequency=variable_data['frequency'],
                 fix_dir=os.path.splitext(variable_data['filename'])[0] +
@@ -201,7 +202,7 @@ def _dataset_to_file(variable, config_user):
     """Find the first file belonging to dataset from variable info."""
     (files, dirnames, filenames) = _get_input_files(variable, config_user)
     if not files and variable.get('derive'):
-        required_vars = get_required(variable['short_name'],
+        required_vars = get_required(variable['cmor_name'],
                                      variable['project'])
         for required_var in required_vars:
             _augment(required_var, variable)
@@ -264,8 +265,8 @@ def _get_default_settings(variable, config_user, derive=False):
     fix = {
         'project': variable['project'],
         'dataset': variable['dataset'],
-        'short_name': variable['short_name'],
         'mip': variable['mip'],
+        'cmor_name': variable['cmor_name'],
     }
     # File fixes
     fix_dir = os.path.splitext(variable['filename'])[0] + '_fixed'
@@ -287,6 +288,7 @@ def _get_default_settings(variable, config_user, derive=False):
 
     if derive:
         settings['derive'] = {
+            'cmor_name': variable['cmor_name'],
             'short_name': variable['short_name'],
             'standard_name': variable['standard_name'],
             'long_name': variable['long_name'],
@@ -297,7 +299,7 @@ def _get_default_settings(variable, config_user, derive=False):
     settings['cmor_check_metadata'] = {
         'cmor_table': variable['project'],
         'mip': variable['mip'],
-        'short_name': variable['short_name'],
+        'cmor_name': variable['cmor_name'],
         'frequency': variable['frequency'],
         'check_level': config_user.get('check_level', CheckLevels.DEFAULT)
     }
@@ -329,7 +331,8 @@ def _add_fxvar_keys(fx_info, variable):
     """Add keys specific to fx variable to use get_input_filelist."""
     fx_variable = deepcopy(variable)
     fx_variable.update(fx_info)
-    fx_variable['variable_group'] = fx_info['short_name']
+    fx_variable['variable_group'] = fx_info['cmor_name']
+    fx_variable['cmor_name'] = fx_info['cmor_name']
 
     # add special ensemble for CMIP5 only
     if fx_variable['project'] == 'CMIP5':
@@ -344,19 +347,19 @@ def _add_fxvar_keys(fx_info, variable):
 def _search_fx_mip(tables, found_mip, variable, fx_info, config_user):
     fx_files = None
     for mip in tables:
-        fx_cmor = tables[mip].get(fx_info['short_name'])
+        fx_cmor = tables[mip].get(fx_info['cmor_name'])
         if fx_cmor:
             found_mip = True
             fx_info['mip'] = mip
             fx_info = _add_fxvar_keys(fx_info, variable)
             logger.debug(
                 "For fx variable '%s', found table '%s'",
-                fx_info['short_name'], mip)
+                fx_info['cmor_name'], mip)
             fx_files = _get_input_files(fx_info, config_user)[0]
             if fx_files:
                 logger.debug(
                     "Found fx variables '%s':\n%s",
-                    fx_info['short_name'], pformat(fx_files))
+                    fx_info['cmor_name'], pformat(fx_files))
     return found_mip, fx_info, fx_files
 
 
@@ -370,7 +373,7 @@ def _get_fx_files(variable, fx_info, config_user):
         get_project_config(var_project)
     except ValueError:
         raise RecipeError(
-            f"Requested fx variable '{fx_info['short_name']}' "
+            f"Requested fx variable '{fx_info['cmor_name']}' "
             f"with parent variable '{variable}' does not have "
             f"a '{var_project}' project in config-developer.")
     project_tables = CMOR_TABLES[var_project].tables
@@ -381,7 +384,7 @@ def _get_fx_files(variable, fx_info, config_user):
         found_mip, fx_info, fx_files = _search_fx_mip(
             project_tables, found_mip, variable, fx_info, config_user)
     else:
-        fx_cmor = project_tables[fx_info['mip']].get(fx_info['short_name'])
+        fx_cmor = project_tables[fx_info['mip']].get(fx_info['cmor_name'])
         if fx_cmor:
             found_mip = True
             fx_info = _add_fxvar_keys(fx_info, variable)
@@ -390,13 +393,13 @@ def _get_fx_files(variable, fx_info, config_user):
     # If fx variable was not found in any table, raise exception
     if not found_mip:
         raise RecipeError(
-            f"Requested fx variable '{fx_info['short_name']}' "
+            f"Requested fx variable '{fx_info['cmor_name']}' "
             f"not available in any CMOR table for '{var_project}'")
 
     # flag a warning
     if not fx_files:
         logger.warning(
-            "Missing data for fx variable '%s'", fx_info['short_name'])
+            "Missing data for fx variable '%s'", fx_info['cmor_name'])
 
     # allow for empty lists corrected for by NE masks
     if fx_files:
@@ -434,8 +437,8 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars):
             fx_info = {}
         if 'mip' not in fx_info:
             fx_info.update({'mip': None})
-        if 'short_name' not in fx_info:
-            fx_info.update({'short_name': fx_var})
+        if 'cmor_name' not in fx_info:
+            fx_info.update({'cmor_name': fx_var})
         fx_files, fx_info = _get_fx_files(variable, fx_info, config_user)
         if fx_files:
             fx_info['filename'] = fx_files
@@ -445,23 +448,24 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars):
 
     logger.info('Using fx_files: %s for variable %s during step %s',
                 pformat(settings['add_fx_variables']['fx_variables']),
-                variable['short_name'], step_name)
+                variable['cmor_name'], step_name)
 
 
 def _fx_list_to_dict(fx_vars):
     """Convert fx list to dictionary. To be deprecated at some point."""
     user_fx_vars = {}
     for fx_var in fx_vars:
         if isinstance(fx_var, dict):
-            short_name = fx_var['short_name']
-            user_fx_vars.update({short_name: fx_var})
+            cmor_name = fx_var['cmor_name']
+            user_fx_vars.update({cmor_name: fx_var})
             continue
         user_fx_vars.update({fx_var: None})
     return user_fx_vars
 
 
 def _update_fx_settings(settings, variable, config_user):
     """Update fx settings depending on the needed method."""
+
     # get fx variables either from user defined attribute or fixed
     def _get_fx_vars_from_attribute(step_settings, step_name):
         user_fx_vars = step_settings.get('fx_variables')
@@ -530,7 +534,7 @@ def _get_ancestors(variable, config_user):
      filenames) = _get_input_files(variable, config_user)
 
     logger.info("Using input files for variable %s of dataset %s:\n%s",
-                variable['short_name'], variable['dataset'],
+                variable['cmor_name'], variable['dataset'],
                 '\n'.join(input_files))
     if (not config_user.get('skip-nonexistent')
             or variable['dataset'] == variable.get('reference_dataset')):
@@ -684,7 +688,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products,
     for variable in variables:
         variable['filename'] = get_output_file(variable,
                                                config_user['preproc_dir'])
-
     if ancestor_products:
         grouped_ancestors = _match_products(ancestor_products, variables)
     else:
@@ -845,7 +848,7 @@ def _get_derive_input_variables(variables, config_user):
 
     def append(group_prefix, var):
         """Append variable `var` to a derive input group."""
-        group = group_prefix + var['short_name']
+        group = group_prefix + var['cmor_name']
         var['variable_group'] = group
         if group not in derive_input:
             derive_input[group] = []
@@ -860,7 +863,7 @@ def append(group_prefix, var):
             append(group_prefix, var)
         else:
             # Process input data needed to derive variable
-            required_vars = get_required(variable['short_name'],
+            required_vars = get_required(variable['cmor_name'],
                                          variable['project'])
             for var in required_vars:
                 _augment(var, variable)
@@ -886,10 +889,10 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name):
     if preproc_name not in profiles:
         raise RecipeError(
             "Unknown preprocessor {} in variable {} of diagnostic {}".format(
-                preproc_name, variable['short_name'], variable['diagnostic']))
+                preproc_name, variable['cmor_name'], variable['diagnostic']))
     profile = deepcopy(profiles[variable['preprocessor']])
     logger.info("Creating preprocessor '%s' task for variable '%s'",
-                variable['preprocessor'], variable['short_name'])
+                variable['preprocessor'], variable['cmor_name'])
     variables = _limit_datasets(variables, profile,
                                 config_user.get('max_datasets'))
     for variable in variables:
@@ -1078,7 +1081,7 @@ def _initialize_variables(self, raw_variable, raw_datasets):
             variables.append(variable)
 
         required_keys = {
-            'short_name',
+            'cmor_name',
             'mip',
             'dataset',
             'project',
@@ -1114,8 +1117,8 @@ def _initialize_preprocessor_output(self, diagnostic_name, raw_variables,
             else:
                 raw_variable = deepcopy(raw_variable)
             raw_variable['variable_group'] = variable_group
-            if 'short_name' not in raw_variable:
-                raw_variable['short_name'] = variable_group
+            if 'cmor_name' not in raw_variable:
+                raw_variable['cmor_name'] = variable_group
             raw_variable['diagnostic'] = diagnostic_name
             raw_variable['preprocessor'] = str(
                 raw_variable.get('preprocessor', 'default'))

esmvalcore/_recipe_checks.py

@@ -97,7 +97,7 @@ def variable(var, required_keys):
     if missing:
         raise RecipeError(
             "Missing keys {} from variable {} in diagnostic {}".format(
-                missing, var.get('short_name'), var.get('diagnostic')))
+                missing, var.get('cmor_name'), var.get('diagnostic')))
 
 
 def data_availability(input_files, var, dirnames, filenames):