From 506859440197bb5b84ac55202cd63c0998e8887d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Sun, 12 May 2019 13:16:53 +0100 Subject: [PATCH 01/53] removed all traces of specific fx file retrieval --- esmvaltool/_data_finder.py | 53 ++++++++++++-------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/esmvaltool/_data_finder.py b/esmvaltool/_data_finder.py index ba6e21d532..1f9702dcbd 100644 --- a/esmvaltool/_data_finder.py +++ b/esmvaltool/_data_finder.py @@ -11,8 +11,7 @@ import six -from ._config import get_project_config, replace_mip_fx -from .cmor.table import CMOR_TABLES +from ._config import get_project_config logger = logging.getLogger(__name__) @@ -95,7 +94,7 @@ def select_files(filenames, start_year, end_year): return selection -def _replace_tags(path, variable, fx_var=None): +def _replace_tags(path, variable): """Replace tags in the config-developer's file with actual values.""" path = path.strip('/') @@ -106,9 +105,7 @@ def _replace_tags(path, variable, fx_var=None): original_tag = tag tag, _, _ = _get_caps_options(tag) - if tag == 'fx_var': - replacewith = fx_var - elif tag == 'latestversion': # handled separately later + if tag == 'latestversion': # handled separately later continue elif tag in variable: replacewith = variable[tag] @@ -198,16 +195,16 @@ def get_rootpath(rootpath, project): raise KeyError('default rootpath must be specified in config-user file') -def _find_input_dirs(variable, rootpath, drs, fx_var=None): +def _find_input_dirs(variable, rootpath, drs): """Return a the full paths to input directories.""" project = variable['project'] root = get_rootpath(rootpath, project) - input_type = 'input_{}dir'.format('fx_' if fx_var else '') + input_type = 'input_dir' path_template = _select_drs(input_type, drs, project) dirnames = [] - for dirname_template in _replace_tags(path_template, variable, fx_var): + for dirname_template in _replace_tags(path_template, variable): for base_path in root: dirname = os.path.join(base_path, dirname_template) dirname = _resolve_latestversion(dirname) @@ -220,21 +217,17 @@ def _find_input_dirs(variable, rootpath, drs, fx_var=None): return dirnames -def _get_filenames_glob(variable, drs, fx_var=None): +def _get_filenames_glob(variable, drs): """Return patterns that can be used to look for input files.""" - input_type = 'input_{}file'.format('fx_' if fx_var else '') + input_type = 'input_file' path_template = _select_drs(input_type, drs, variable['project']) - filenames_glob = _replace_tags(path_template, variable, fx_var) + filenames_glob = _replace_tags(path_template, variable) return filenames_glob -def _find_input_files(variable, rootpath, drs, fx_var=None): - logger.debug("Looking for input %sfiles for variable %s of dataset %s", - fx_var + ' fx ' if fx_var else '', variable['short_name'], - variable['dataset']) - - input_dirs = _find_input_dirs(variable, rootpath, drs, fx_var) - filenames_glob = _get_filenames_glob(variable, drs, fx_var) +def _find_input_files(variable, rootpath, drs): + input_dirs = _find_input_dirs(variable, rootpath, drs) + filenames_glob = _get_filenames_glob(variable, drs) files = find_files(input_dirs, filenames_glob) return files @@ -243,27 +236,13 @@ def _find_input_files(variable, rootpath, drs, fx_var=None): def get_input_filelist(variable, rootpath, drs): """Return the full path to input files.""" files = _find_input_files(variable, rootpath, drs) - files = select_files(files, variable['start_year'], variable['end_year']) + # do time gating only for non-fx variables + if variable['frequency'] != 'fx': + files = select_files(files, variable['start_year'], + variable['end_year']) return files -def get_input_fx_filelist(variable, rootpath, drs): - """Return a dict with the full path to fx input files.""" - fx_files = {} - for fx_var in variable['fx_files']: - var = dict(variable) - var['mip'] = replace_mip_fx(fx_var) - table = CMOR_TABLES[var['cmor_table']].get_table(var['mip']) - var['frequency'] = table.frequency - realm = getattr(table.get(var['short_name']), 'modeling_realm', None) - var['modeling_realm'] = realm if realm else table.realm - - files = _find_input_files(var, rootpath, drs, fx_var) - fx_files[fx_var] = files[0] if files else None - - return fx_files - - def get_output_file(variable, preproc_dir): """Return the full path to the output (preprocessed) file.""" cfg = get_project_config(variable['project']) From 909e34f6495734080387a5fffaee1aac3a53311b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Sun, 12 May 2019 13:17:50 +0100 Subject: [PATCH 02/53] do time checks only if not fx variable --- esmvaltool/cmor/check.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esmvaltool/cmor/check.py b/esmvaltool/cmor/check.py index 7f9399d545..8a9a9eec35 100644 --- a/esmvaltool/cmor/check.py +++ b/esmvaltool/cmor/check.py @@ -95,13 +95,15 @@ def check_metadata(self, logger=None): self._check_fill_value() self._check_dim_names() self._check_coords() - self._check_time_coord() + if self.frequency != 'fx': + self._check_time_coord() self._check_rank() self.report_warnings(logger) self.report_errors() - self._add_auxiliar_time_coordinates() + if self.frequency != 'fx': + self._add_auxiliar_time_coordinates() return self._cube def report_errors(self): From 1bbf364ff4490ee6544662de1b3bd942566c7219 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Sun, 12 May 2019 13:18:43 +0100 Subject: [PATCH 03/53] removed references to fx files and dirs --- esmvaltool/config-developer.yml | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/esmvaltool/config-developer.yml b/esmvaltool/config-developer.yml index 06433da1bf..eb8e3464f6 100644 --- a/esmvaltool/config-developer.yml +++ b/esmvaltool/config-developer.yml @@ -24,7 +24,7 @@ CMIP6: BADC: '[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' DKRZ: '[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/[grid]/' - input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]_*.nc' + input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]*.nc' output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP6' institutes: @@ -147,26 +147,7 @@ CMIP5: ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' SMHI: '[dataset]/[ensemble]/[exp]/[frequency]' BSC: '[type]/[project]/[exp]/[dataset.lower]' - input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_*.nc' - input_fx_dir: - default: '/' - BADC: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[latestversion]/[fx_var]' - CP4CDS: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[fx_var]/latest/' - DKRZ: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[latestversion]/[fx_var]' - ETHZ: '[exp]/fx/[fx_var]/[dataset]/r0i0p0' - input_fx_file: '[fx_var]_fx_[dataset]_[exp]_r0i0p0.nc' - fx_mip_change: - 'areacella': 'Amon' - 'areacello': 'Omon' - 'basin': 'Omon' - 'deptho': 'Omon' - 'mrsofc': 'Lmon' - 'orog': 'Amon' - 'rootd': 'Lmon' - 'sftgif': 'Lmon' - 'sftlf': 'Amon' - 'sftof': 'Omon' - 'volcello': 'Omon' + input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]*.nc' output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' institutes: 'ACCESS1-0': ['CSIRO-BOM'] @@ -239,10 +220,6 @@ OBS: input_file: default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' BSC: '[short_name]_*.nc' - input_fx_dir: - default: 'Tier[tier]/[dataset]' - input_fx_file: - default: '[project]_[dataset]_[type]_[version]_fx_[fx_var].nc' output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP5' @@ -251,10 +228,6 @@ obs4mips: input_dir: default: 'Tier[tier]/[dataset]' input_file: '[short_name]_[dataset]_[level]_[version]_*.nc' - input_fx_dir: - default: 'Tier[tier]/[dataset]' - input_fx_file: - default: '[project]_[dataset]_fx_[fx_var].nc' output_file: '[project]_[dataset]_[level]_[version]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP6' cmor_path: 'obs4mips' From 3e6342442ae95dc5b3bdb9fc0f769a94cbb1321e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Sun, 12 May 2019 13:19:32 +0100 Subject: [PATCH 04/53] removed fx files function --- esmvaltool/_config.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/esmvaltool/_config.py b/esmvaltool/_config.py index 221c06d026..cc4ed79e4e 100644 --- a/esmvaltool/_config.py +++ b/esmvaltool/_config.py @@ -175,18 +175,6 @@ def get_institutes(variable): return CFG.get(project, {}).get('institutes', {}).get(dataset, []) -def replace_mip_fx(fx_file): - """Replace MIP so to retrieve correct fx files.""" - default_mip = 'Amon' - if fx_file not in CFG['CMIP5']['fx_mip_change']: - logger.warning( - 'mip for fx variable %s is not specified in ' - 'config_developer.yml, using default (%s)', fx_file, default_mip) - new_mip = CFG['CMIP5']['fx_mip_change'].get(fx_file, default_mip) - logger.debug("Switching mip for fx file finding to %s", new_mip) - return new_mip - - TAGS_CONFIG_FILE = os.path.join( os.path.dirname(__file__), 'config-references.yml') From 33b906604a3e743b033b21bec8cdb32dda516b90 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Sun, 12 May 2019 13:21:15 +0100 Subject: [PATCH 05/53] restructuring to allow for fx vars treated like any other var --- esmvaltool/_recipe.py | 113 ++++++++++++++++++++++++++++++------------ 1 file changed, 80 insertions(+), 33 deletions(-) diff --git a/esmvaltool/_recipe.py b/esmvaltool/_recipe.py index 6ab0236f55..c0a7c26e12 100644 --- a/esmvaltool/_recipe.py +++ b/esmvaltool/_recipe.py @@ -11,8 +11,8 @@ from . import __version__ from . import _recipe_checks as check from ._config import TAGS, get_institutes, replace_tags -from ._data_finder import (get_input_filelist, get_input_fx_filelist, - get_output_file, get_statistic_output_file) +from ._data_finder import (get_input_filelist, get_output_file, + get_statistic_output_file) from ._provenance import TrackedFile, get_recipe_provenance from ._recipe_checks import RecipeError from ._task import (DiagnosticTask, get_flattened_tasks, get_independent_tasks, @@ -361,6 +361,26 @@ def _get_default_settings(variable, config_user, derive=False): return settings +def _add_fxvar_keys(fx_var_dict, variable): + """Add a couple keys specific to fx variable.""" + fx_variable = deepcopy(variable) + + # add internal recognition flag + fx_variable['variable_group'] = fx_var_dict['short_name'] + fx_variable['short_name'] = fx_var_dict['short_name'] + + # specificities of project + if fx_variable['project'] == 'CMIP5': + fx_variable['mip'] = 'fx' + fx_variable['ensemble'] = 'r0i0p0' + elif fx_variable['project'] == 'CMIP6': + fx_variable['grid'] = variable['grid'] + if 'mip' in fx_var_dict: + fx_variable['mip'] = fx_var_dict['mip'] + + return fx_variable + + def _update_fx_settings(settings, variable, config_user): """Find and set the FX derive/mask settings.""" # update for derive @@ -369,11 +389,13 @@ def _update_fx_settings(settings, variable, config_user): for var in get_required(variable['short_name']): if 'fx_files' in var: _augment(var, variable) - fx_files.update( - get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs'])) + for fx_var_dict in var['fx_files']: + fx_var = _add_fxvar_keys(fx_var_dict, variable) + fx_files.update( + get_input_filelist( + variable=fx_var, + rootpath=config_user['rootpath'], + drs=config_user['drs'])) settings['derive']['fx_files'] = fx_files # update for landsea @@ -384,11 +406,14 @@ def _update_fx_settings(settings, variable, config_user): settings['mask_landsea']['fx_files'] = [] var = dict(variable) - var['fx_files'] = ['sftlf', 'sftof'] - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + var['fx_files'] = [{'short_name': 'sftlf'}, {'short_name': 'sftof'}] + fx_files_dict = {} + for fx_var_dict in var['fx_files']: + fx_var = _add_fxvar_keys(fx_var_dict, var) + fx_files_dict[fx_var['short_name']] = get_input_filelist( + variable=fx_var, + rootpath=config_user['rootpath'], + drs=config_user['drs']) # allow both sftlf and sftof if fx_files_dict['sftlf']: @@ -402,11 +427,14 @@ def _update_fx_settings(settings, variable, config_user): settings['mask_landseaice']['fx_files'] = [] var = dict(variable) - var['fx_files'] = ['sftgif'] - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + var['fx_files'] = [{'short_name': 'sftgif'}] + fx_files_dict = {} + for fx_var_dict in var['fx_files']: + fx_var = _add_fxvar_keys(fx_var_dict, var) + fx_files_dict[fx_var['short_name']] = get_input_filelist( + variable=fx_var, + rootpath=config_user['rootpath'], + drs=config_user['drs']) # allow sftgif (only, for now) if fx_files_dict['sftgif']: @@ -415,11 +443,16 @@ def _update_fx_settings(settings, variable, config_user): for step in ('average_region', 'average_volume'): if settings.get(step, {}).get('fx_files'): - settings[step]['fx_files'] = get_input_fx_filelist( - variable=variable, - rootpath=config_user['rootpath'], - drs=config_user['drs'], - ) + var = dict(variable) + var['fx_files'] = settings.get(step, {}).get('fx_files') + fx_files_dict = {} + for fx_var_dict in var['fx_files']: + fx_var = _add_fxvar_keys(fx_var_dict, var) + fx_files_dict[fx_var['short_name']] = get_input_filelist( + variable=fx_var, + rootpath=config_user['rootpath'], + drs=config_user['drs']) + settings[step]['fx_files'] = fx_files_dict def _read_attributes(filename): @@ -438,22 +471,27 @@ def _read_attributes(filename): def _get_input_files(variable, config_user): """Get the input files for a single dataset.""" # Find input files locally. + var = dict(variable) + # change ensemble to fixed r0i0p0 + if var['project'] == 'CMIP5': + if var['frequency'] == 'fx': + var['ensemble'] = 'r0i0p0' input_files = get_input_filelist( - variable=variable, + variable=var, rootpath=config_user['rootpath'], drs=config_user['drs']) # Set up downloading using synda if requested. # Do not download if files are already available locally. if config_user['synda_download'] and not input_files: - input_files = synda_search(variable) + input_files = synda_search(var) logger.info("Using input files for variable %s of dataset %s:\n%s", - variable['short_name'], variable['dataset'], + var['short_name'], var['dataset'], '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') - or variable['dataset'] == variable.get('reference_dataset')): - check.data_availability(input_files, variable) + or var['dataset'] == var.get('reference_dataset')): + check.data_availability(input_files, var) # Set up provenance tracking for i, filename in enumerate(input_files): @@ -772,6 +810,10 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): name=derive_name) derive_tasks.append(task) + # don't do time gating for fx variables + if variables[0]['frequency'] == 'fx': + profile['extract_time'] = False + # Create (final) preprocessor task task = _get_single_preprocessor_task( variables, @@ -905,13 +947,18 @@ def _initialize_variables(self, raw_variable, raw_datasets): variable['institute'] = institute check.variable(variable, required_keys) if 'fx_files' in variable: - for fx_file in variable['fx_files']: - DATASET_KEYS.add(fx_file) + for fx_file_dict in variable['fx_files']: + DATASET_KEYS.add(fx_file_dict['short_name']) # Get the fx files - variable['fx_files'] = get_input_fx_filelist( - variable=variable, - rootpath=self._cfg['rootpath'], - drs=self._cfg['drs']) + fx_files_dict = {} + for fx_var_dict in variable['fx_files']: + fx_var = _add_fxvar_keys(fx_var_dict, variable) + _add_cmor_info(fx_var) + fx_files_dict[fx_var['short_name']] = get_input_filelist( + variable=fx_var, + rootpath=self._cfg['rootpath'], + drs=self._cfg['drs']) + variable['fx_files'] = fx_files_dict logger.info("Using fx files for var %s of dataset %s:\n%s", variable['short_name'], variable['dataset'], variable['fx_files']) From 5614f7c44c1db4ff355ec52661e62eddc13de8bc Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 15:30:26 +0100 Subject: [PATCH 06/53] cleaning up --- esmvaltool/_data_finder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esmvaltool/_data_finder.py b/esmvaltool/_data_finder.py index 1f9702dcbd..158272daa7 100644 --- a/esmvaltool/_data_finder.py +++ b/esmvaltool/_data_finder.py @@ -235,6 +235,9 @@ def _find_input_files(variable, rootpath, drs): def get_input_filelist(variable, rootpath, drs): """Return the full path to input files.""" + # change ensemble to fixed r0i0p0 for fx variables + if variable['project'] == 'CMIP5'and variable['frequency'] == 'fx': + variable['ensemble'] = 'r0i0p0' files = _find_input_files(variable, rootpath, drs) # do time gating only for non-fx variables if variable['frequency'] != 'fx': From 475cd8f9e897b0a4a87ad54cffaf70f84d6cfa20 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 15:31:23 +0100 Subject: [PATCH 07/53] cleaning up --- esmvaltool/_recipe.py | 101 +++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/esmvaltool/_recipe.py b/esmvaltool/_recipe.py index c0a7c26e12..79524c7970 100644 --- a/esmvaltool/_recipe.py +++ b/esmvaltool/_recipe.py @@ -6,10 +6,11 @@ from copy import deepcopy import yaml + from netCDF4 import Dataset -from . import __version__ from . import _recipe_checks as check +from . import __version__ from ._config import TAGS, get_institutes, replace_tags from ._data_finder import (get_input_filelist, get_output_file, get_statistic_output_file) @@ -361,11 +362,28 @@ def _get_default_settings(variable, config_user, derive=False): return settings +def get_input_fx_filelist(variable, rootpath, drs): + """Return a dict with fx vars keys and full file paths values.""" + fx_files_dict = {} + for fx_var_dict in variable['fx_files']: + fx_var = _add_fxvar_keys(fx_var_dict, variable) + fx_files = get_input_filelist( + variable=fx_var, + rootpath=rootpath, + drs=drs) + if fx_files: + fx_files_dict[fx_var['short_name']] = fx_files[0] + else: + fx_files_dict[fx_var['short_name']] = None + + return fx_files_dict + + def _add_fxvar_keys(fx_var_dict, variable): - """Add a couple keys specific to fx variable.""" - fx_variable = deepcopy(variable) + """Add keys specific to fx variable to use get_input_filelist.""" + fx_variable = dict(variable) - # add internal recognition flag + # set variable names fx_variable['variable_group'] = fx_var_dict['short_name'] fx_variable['short_name'] = fx_var_dict['short_name'] @@ -377,6 +395,8 @@ def _add_fxvar_keys(fx_var_dict, variable): fx_variable['grid'] = variable['grid'] if 'mip' in fx_var_dict: fx_variable['mip'] = fx_var_dict['mip'] + # add missing cmor info + _add_cmor_info(fx_variable, override=True) return fx_variable @@ -389,13 +409,11 @@ def _update_fx_settings(settings, variable, config_user): for var in get_required(variable['short_name']): if 'fx_files' in var: _augment(var, variable) - for fx_var_dict in var['fx_files']: - fx_var = _add_fxvar_keys(fx_var_dict, variable) - fx_files.update( - get_input_filelist( - variable=fx_var, - rootpath=config_user['rootpath'], - drs=config_user['drs'])) + fx_files.update( + get_input_fx_filelist( + variable=var, + rootpath=config_user['rootpath'], + drs=config_user['drs'])) settings['derive']['fx_files'] = fx_files # update for landsea @@ -407,13 +425,10 @@ def _update_fx_settings(settings, variable, config_user): var = dict(variable) var['fx_files'] = [{'short_name': 'sftlf'}, {'short_name': 'sftof'}] - fx_files_dict = {} - for fx_var_dict in var['fx_files']: - fx_var = _add_fxvar_keys(fx_var_dict, var) - fx_files_dict[fx_var['short_name']] = get_input_filelist( - variable=fx_var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + fx_files_dict = get_input_fx_filelist( + variable=var, + rootpath=config_user['rootpath'], + drs=config_user['drs']) # allow both sftlf and sftof if fx_files_dict['sftlf']: @@ -428,13 +443,10 @@ def _update_fx_settings(settings, variable, config_user): var = dict(variable) var['fx_files'] = [{'short_name': 'sftgif'}] - fx_files_dict = {} - for fx_var_dict in var['fx_files']: - fx_var = _add_fxvar_keys(fx_var_dict, var) - fx_files_dict[fx_var['short_name']] = get_input_filelist( - variable=fx_var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + fx_files_dict = get_input_fx_filelist( + variable=var, + rootpath=config_user['rootpath'], + drs=config_user['drs']) # allow sftgif (only, for now) if fx_files_dict['sftgif']: @@ -445,13 +457,10 @@ def _update_fx_settings(settings, variable, config_user): if settings.get(step, {}).get('fx_files'): var = dict(variable) var['fx_files'] = settings.get(step, {}).get('fx_files') - fx_files_dict = {} - for fx_var_dict in var['fx_files']: - fx_var = _add_fxvar_keys(fx_var_dict, var) - fx_files_dict[fx_var['short_name']] = get_input_filelist( - variable=fx_var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + fx_files_dict = get_input_fx_filelist( + variable=var, + rootpath=config_user['rootpath'], + drs=config_user['drs']) settings[step]['fx_files'] = fx_files_dict @@ -471,27 +480,22 @@ def _read_attributes(filename): def _get_input_files(variable, config_user): """Get the input files for a single dataset.""" # Find input files locally. - var = dict(variable) - # change ensemble to fixed r0i0p0 - if var['project'] == 'CMIP5': - if var['frequency'] == 'fx': - var['ensemble'] = 'r0i0p0' input_files = get_input_filelist( - variable=var, + variable=variable, rootpath=config_user['rootpath'], drs=config_user['drs']) # Set up downloading using synda if requested. # Do not download if files are already available locally. if config_user['synda_download'] and not input_files: - input_files = synda_search(var) + input_files = synda_search(variable) logger.info("Using input files for variable %s of dataset %s:\n%s", - var['short_name'], var['dataset'], + variable['short_name'], variable['dataset'], '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') - or var['dataset'] == var.get('reference_dataset')): - check.data_availability(input_files, var) + or var['dataset'] == variable.get('reference_dataset')): + check.data_availability(input_files, variable) # Set up provenance tracking for i, filename in enumerate(input_files): @@ -950,15 +954,10 @@ def _initialize_variables(self, raw_variable, raw_datasets): for fx_file_dict in variable['fx_files']: DATASET_KEYS.add(fx_file_dict['short_name']) # Get the fx files - fx_files_dict = {} - for fx_var_dict in variable['fx_files']: - fx_var = _add_fxvar_keys(fx_var_dict, variable) - _add_cmor_info(fx_var) - fx_files_dict[fx_var['short_name']] = get_input_filelist( - variable=fx_var, - rootpath=self._cfg['rootpath'], - drs=self._cfg['drs']) - variable['fx_files'] = fx_files_dict + variable['fx_files'] = get_input_fx_filelist( + variable=variable, + rootpath=self._cfg['rootpath'], + drs=self._cfg['drs']) logger.info("Using fx files for var %s of dataset %s:\n%s", variable['short_name'], variable['dataset'], variable['fx_files']) From 0ee4a729fb59803f3704bebb7bab1eb1188faa5b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 15:32:36 +0100 Subject: [PATCH 08/53] fixed test for new fx file retrieval --- tests/integration/test_data_finder.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index 484fdc99b7..ef8826cb44 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -3,12 +3,11 @@ import shutil import tempfile +import esmvaltool._config import pytest import yaml - -import esmvaltool._config -from esmvaltool._data_finder import (get_input_filelist, get_input_fx_filelist, - get_output_file) +from esmvaltool._data_finder import get_input_filelist, get_output_file +from esmvaltool._recipe import get_input_fx_filelist from esmvaltool.cmor.table import read_cmor_tables # Initialize with standard config developer file @@ -103,11 +102,18 @@ def test_get_input_fx_filelist(root, cfg): # Find files rootpath = {cfg['variable']['project']: [root]} drs = {cfg['variable']['project']: cfg['drs']} - fx_files = get_input_fx_filelist(cfg['variable'], rootpath, drs) + cfg['variable']['fx_files'] = [ + {'short_name': short_name} for short_name + in cfg['variable']['fx_files'] + ] + fx_files_dict = get_input_fx_filelist( + variable=cfg['variable'], + rootpath=rootpath, + drs=drs) # Test result reference = { fx_var: os.path.join(root, filename) if filename else None for fx_var, filename in cfg['found_files'].items() } - assert fx_files == reference + assert fx_files_dict == reference From 6989951c63f67f9fe66beb7d4f2e475afe91eec1 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 15:33:15 +0100 Subject: [PATCH 09/53] added checks for no time gating --- esmvaltool/_recipe_checks.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/esmvaltool/_recipe_checks.py b/esmvaltool/_recipe_checks.py index a6ab607df0..ee0e569304 100644 --- a/esmvaltool/_recipe_checks.py +++ b/esmvaltool/_recipe_checks.py @@ -97,15 +97,18 @@ def data_availability(input_files, var): required_years = set(range(var['start_year'], var['end_year'] + 1)) available_years = set() - for filename in input_files: - start, end = get_start_end_year(filename) - available_years.update(range(start, end + 1)) - - missing_years = required_years - available_years - if missing_years: - raise RecipeError( - "No input data available for years {} in files {}".format( - ", ".join(str(year) for year in missing_years), input_files)) + # check time avail only for non-fx variables + if var['frequency'] != 'fx': + for filename in input_files: + start, end = get_start_end_year(filename) + available_years.update(range(start, end + 1)) + + missing_years = required_years - available_years + if missing_years: + raise RecipeError( + "No input data available for years {} in files {}".format( + ", ".join(str(year) for year in missing_years), + input_files)) def tasks_valid(tasks): From 988d09cdba42e75fa62a03fc47baf01d1a64362a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 16:23:30 +0100 Subject: [PATCH 10/53] fixed the test to reflect the new standards --- tests/integration/test_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index aeaaedabd5..885eecc5d3 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -100,7 +100,7 @@ def find_files(_, filenames): filename = str(tmp_path / 'input' / filename) filenames = [] if filename.endswith('*.nc'): - filename = filename[:-len('*.nc')] + filename = filename[:-len('*.nc')] + '_' intervals = [ '1990_1999', '2000_2009', From 775715c801768aeccf65de296e3fc987cab0bb0e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 16:23:51 +0100 Subject: [PATCH 11/53] fixed the test to reflect the new standards --- tests/integration/preprocessor/_derive/test_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index cef8ea93a4..8608cccf18 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -26,7 +26,7 @@ def test_get_required_with_fx(): reference = [{ 'short_name': 'nbp', - 'fx_files': ['sftlf'], + 'fx_files': [{'short_name': 'sftlf'}], }] assert variables == reference From 32b6376fbd1533c7e34f278fc2cc717792908d89 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 16:24:14 +0100 Subject: [PATCH 12/53] fixed the data structure standard to reflect new standards --- esmvaltool/preprocessor/_derive/nbp_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/preprocessor/_derive/nbp_grid.py b/esmvaltool/preprocessor/_derive/nbp_grid.py index 5a07fcaed7..44047c4f7f 100644 --- a/esmvaltool/preprocessor/_derive/nbp_grid.py +++ b/esmvaltool/preprocessor/_derive/nbp_grid.py @@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase): # Required variables required = [{ 'short_name': 'nbp', - 'fx_files': ['sftlf'], + 'fx_files': [{'short_name': 'sftlf'}], }] @staticmethod From 2ff537d6023c6b1591d73085770f342979fbec27 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 13 May 2019 18:11:03 +0100 Subject: [PATCH 13/53] fixed little bug --- esmvaltool/_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/_recipe.py b/esmvaltool/_recipe.py index 79524c7970..d0cbc2b195 100644 --- a/esmvaltool/_recipe.py +++ b/esmvaltool/_recipe.py @@ -494,7 +494,7 @@ def _get_input_files(variable, config_user): variable['short_name'], variable['dataset'], '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') - or var['dataset'] == variable.get('reference_dataset')): + or variable['dataset'] == variable.get('reference_dataset')): check.data_availability(input_files, variable) # Set up provenance tracking From b36f31223d47a623a090e38bd5070e772ea2f188 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 10 Jun 2019 16:00:44 +0100 Subject: [PATCH 14/53] removed stray import from conflict --- tests/integration/test_data_finder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index 4609b87c91..73df055167 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -3,7 +3,6 @@ import shutil import tempfile -import esmvaltool._config import pytest import yaml From cf05187194ac103a5e71eddb6e894fb04afdfac9 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 10 Jun 2019 17:04:26 +0100 Subject: [PATCH 15/53] fixed stray import mistakes introduced by merge --- tests/integration/test_data_finder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index 73df055167..71e49ae53d 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -7,8 +7,8 @@ import yaml import esmvalcore._config -from esmvalcore._data_finder import (get_input_filelist, get_input_fx_filelist, - get_output_file) +from esmvalcore._data_finder import get_input_filelist, get_output_file +from esmvalcore._recipe import get_input_fx_filelist from esmvalcore.cmor.table import read_cmor_tables # Initialize with standard config developer file From e799ec83014454d55786fbf4a88eb8f83a5776b8 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 10 Jun 2019 17:04:38 +0100 Subject: [PATCH 16/53] removed unused import --- esmvalcore/preprocessor/_mask.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/preprocessor/_mask.py b/esmvalcore/preprocessor/_mask.py index 491d537c56..efba23d2e0 100644 --- a/esmvalcore/preprocessor/_mask.py +++ b/esmvalcore/preprocessor/_mask.py @@ -12,7 +12,6 @@ import numpy as np -import dask.array as da import cartopy.io.shapereader as shpreader import iris import shapely.vectorized as shp_vect From 7ab878fcbbc12145552b2964037d0e87952b04dc Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 14 Jun 2019 15:06:48 +0100 Subject: [PATCH 17/53] removed unused import --- esmvalcore/_data_finder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 67894a7568..3ea5bcc2ad 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -10,7 +10,6 @@ import re from ._config import get_project_config -from .cmor.table import CMOR_TABLES logger = logging.getLogger(__name__) From 0fc4392fd5f8660c7f00c9d4044e0dd0aedd1558 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 14 Jun 2019 16:55:40 +0100 Subject: [PATCH 18/53] removed useless variable --- esmvalcore/_data_finder.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 3ea5bcc2ad..0bc9450f70 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -198,8 +198,7 @@ def _find_input_dirs(variable, rootpath, drs): project = variable['project'] root = get_rootpath(rootpath, project) - input_type = 'input_dir' - path_template = _select_drs(input_type, drs, project) + path_template = _select_drs('input_dir', drs, project) dirnames = [] for dirname_template in _replace_tags(path_template, variable): @@ -217,8 +216,7 @@ def _find_input_dirs(variable, rootpath, drs): def _get_filenames_glob(variable, drs): """Return patterns that can be used to look for input files.""" - input_type = 'input_file' - path_template = _select_drs(input_type, drs, variable['project']) + path_template = _select_drs('input_file', drs, variable['project']) filenames_glob = _replace_tags(path_template, variable) return filenames_glob From 4fa9c4fbde0c5bc0536d710cc2accea07091f703 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 15 Jul 2019 15:26:23 +0100 Subject: [PATCH 19/53] start work on full integration of fx variables --- esmvalcore/_recipe.py | 53 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index a5b8d81495..5c4933b13b 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -779,6 +779,37 @@ def append(group_prefix, var): return derive_input +def _get_fx_input_variables(variables): + """Determine the input sets of fx vars needed for masking or others.""" + fx_input = {} + + def append(group_prefix, var): + """Append variable `var` to an fx input group.""" + group = group_prefix + var['short_name'] + var['variable_group'] = group + if group not in fx_input: + fx_input[group] = [] + fx_input[group].append(var) + + # figure out which of variables is fx variable + fx_variables = [ + var for var in variables if 'fxvar' in var and var['fxvar'] + ] + # figure out which variable needs fx variables + needs_fx_variables = [ + var for var in variables if 'needs_fxvar' in var and var['need_fxvar'] + ] + + for variable in needs_fx_variables: + group_prefix = variable['variable_group'] + '_fx_input_' + # Process fx input data needed by variable + for var in fx_variables: + _augment(var, variable) + append(group_prefix, var) + + return fx_input + + def _get_preprocessor_task(variables, profiles, config_user, task_name): """Create preprocessor task(s) for a set of datasets.""" # First set up the preprocessor profile @@ -814,9 +845,25 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): name=derive_name) derive_tasks.append(task) - # don't do time gating for fx variables - if variables[0]['frequency'] == 'fx': - profile['extract_time'] = False + # treat fx variables and the variables they need them + fx_tasks = [] + if variable.get('needs_fxvar'): + # Create tasks to prepare the input data for the fx var addition + fx_input = _get_fx_input_variables(variables, config_user) + fx_profile = profile + fx_profile['extract_time'] = False + + for fx_variables in fx_input.values(): + for fx_variable in fx_variables: + _add_cmor_info(fx_variable, override=True) + fx_name = task_name.split( + TASKSEP)[0] + TASKSEP + fx_variables[0]['variable_group'] + task = _get_single_preprocessor_task( + fx_variables, + fx_profile, + config_user, + name=derive_name) + fx_tasks.append(task) # Create (final) preprocessor task task = _get_single_preprocessor_task( From c75f6f8b1f5cac8ca74ccb031c0c803a3f4b5b85 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 16 Jul 2019 17:56:29 +0100 Subject: [PATCH 20/53] working version but masking funcs need to be fixed --- esmvalcore/_recipe.py | 164 ++++++++++++++---------------------------- 1 file changed, 55 insertions(+), 109 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 5c4933b13b..6301461e7b 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -362,23 +362,6 @@ def _get_default_settings(variable, config_user, derive=False): return settings -def get_input_fx_filelist(variable, rootpath, drs): - """Return a dict with fx vars keys and full file paths values.""" - fx_files_dict = {} - for fx_var_dict in variable['fx_files']: - fx_var = _add_fxvar_keys(fx_var_dict, variable) - fx_files = get_input_filelist( - variable=fx_var, - rootpath=rootpath, - drs=drs) - if fx_files: - fx_files_dict[fx_var['short_name']] = fx_files[0] - else: - fx_files_dict[fx_var['short_name']] = None - - return fx_files_dict - - def _add_fxvar_keys(fx_var_dict, variable): """Add keys specific to fx variable to use get_input_filelist.""" fx_variable = dict(variable) @@ -410,7 +393,7 @@ def _update_fx_settings(settings, variable, config_user): if 'fx_files' in var: _augment(var, variable) fx_files.update( - get_input_fx_filelist( + get_input_filelist( variable=var, rootpath=config_user['rootpath'], drs=config_user['drs'])) @@ -418,18 +401,21 @@ def _update_fx_settings(settings, variable, config_user): # update for landsea if 'mask_landsea' in settings: + fx_files_dict = {} # Configure ingestion of land/sea masks logger.debug('Getting fx mask settings now...') - settings['mask_landsea']['fx_files'] = [] - var = dict(variable) - var['fx_files'] = [{'short_name': 'sftlf'}, {'short_name': 'sftof'}] - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) - + if var['project'] == 'CMIP5': + fx_var = _add_fxvar_keys({'short_name': 'sftlf', 'mip': 'fx'}, var) + fx_files_dict['sftlf'] = get_output_file(fx_var, + config_user['preproc_dir'], + masking=True) + if var['project'] == 'CMIP5': + fx_var = _add_fxvar_keys({'short_name': 'sftof', 'mip': 'fx'}, var) + fx_files_dict['sftof'] = get_output_file(fx_var, + config_user['preproc_dir'], + masking=True) # allow both sftlf and sftof if fx_files_dict['sftlf']: settings['mask_landsea']['fx_files'].append(fx_files_dict['sftlf']) @@ -438,17 +424,11 @@ def _update_fx_settings(settings, variable, config_user): if 'mask_landseaice' in settings: logger.debug('Getting fx mask settings now...') - settings['mask_landseaice']['fx_files'] = [] - var = dict(variable) - var['fx_files'] = [{'short_name': 'sftgif'}] - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) - # allow sftgif (only, for now) + fx_files_dict['sftgif'] = get_output_file(var, + config_user['preproc_dir']) if fx_files_dict['sftgif']: settings['mask_landseaice']['fx_files'].append( fx_files_dict['sftgif']) @@ -457,10 +437,8 @@ def _update_fx_settings(settings, variable, config_user): if settings.get(step, {}).get('fx_files'): var = dict(variable) var['fx_files'] = settings.get(step, {}).get('fx_files') - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs']) + fx_files_dict = get_output_file(var, + config_user['preproc_dir']) settings[step]['fx_files'] = fx_files_dict @@ -649,7 +627,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, settings=settings, config_user=config_user) _update_fx_settings( - settings=settings, variable=variable, config_user=config_user) + settings=settings, variable=variable, + config_user=config_user) _update_target_grid( variable=variable, variables=variables, @@ -674,28 +653,57 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, return products +def _remove_temporal_preprocs(profile): + """Remove all temporal operations on fx variables.""" + fx_profile = profile + if 'mask_landsea' in profile: + fx_profile['mask_landsea'] = False + if 'mask_landseaice' in profile: + fx_profile['mask_landseaice'] = False + if 'seasonal_mean' in profile: + fx_profile['seasonal_mean'] = False + return fx_profile + + def _get_single_preprocessor_task(variables, profile, config_user, name, ancestor_tasks=None): - """Create preprocessor tasks for a set of datasets.""" + """Create preprocessor tasks for a set of datasets w/ special case fx.""" if ancestor_tasks is None: ancestor_tasks = [] order = _extract_preprocessor_order(profile) ancestor_products = [p for task in ancestor_tasks for p in task.products] - products = _get_preprocessor_products( - variables=variables, - profile=profile, - order=order, - ancestor_products=ancestor_products, - config_user=config_user, - ) + + products = [] + for variable in variables: + if variable['frequency'] != 'fx': + product = list(_get_preprocessor_products( + variables=[variable], + profile=profile, + order=order, + ancestor_products=ancestor_products, + config_user=config_user, + ))[0] + else: + fx_profile = profile + fx_profile['extract_time'] = False + fx_profile = _remove_temporal_preprocs(profile) + product = list(_get_preprocessor_products( + variables=[variable], + profile=fx_profile, + order=order, + ancestor_products=None, + config_user=config_user, + ))[0] + products.append(product) if not products: raise RecipeError( "Did not find any input data for task {}".format(name)) + # fx variable task task = PreprocessingTask( products=products, ancestors=ancestor_tasks, @@ -779,37 +787,6 @@ def append(group_prefix, var): return derive_input -def _get_fx_input_variables(variables): - """Determine the input sets of fx vars needed for masking or others.""" - fx_input = {} - - def append(group_prefix, var): - """Append variable `var` to an fx input group.""" - group = group_prefix + var['short_name'] - var['variable_group'] = group - if group not in fx_input: - fx_input[group] = [] - fx_input[group].append(var) - - # figure out which of variables is fx variable - fx_variables = [ - var for var in variables if 'fxvar' in var and var['fxvar'] - ] - # figure out which variable needs fx variables - needs_fx_variables = [ - var for var in variables if 'needs_fxvar' in var and var['need_fxvar'] - ] - - for variable in needs_fx_variables: - group_prefix = variable['variable_group'] + '_fx_input_' - # Process fx input data needed by variable - for var in fx_variables: - _augment(var, variable) - append(group_prefix, var) - - return fx_input - - def _get_preprocessor_task(variables, profiles, config_user, task_name): """Create preprocessor task(s) for a set of datasets.""" # First set up the preprocessor profile @@ -845,26 +822,6 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name): name=derive_name) derive_tasks.append(task) - # treat fx variables and the variables they need them - fx_tasks = [] - if variable.get('needs_fxvar'): - # Create tasks to prepare the input data for the fx var addition - fx_input = _get_fx_input_variables(variables, config_user) - fx_profile = profile - fx_profile['extract_time'] = False - - for fx_variables in fx_input.values(): - for fx_variable in fx_variables: - _add_cmor_info(fx_variable, override=True) - fx_name = task_name.split( - TASKSEP)[0] + TASKSEP + fx_variables[0]['variable_group'] - task = _get_single_preprocessor_task( - fx_variables, - fx_profile, - config_user, - name=derive_name) - fx_tasks.append(task) - # Create (final) preprocessor task task = _get_single_preprocessor_task( variables, @@ -997,17 +954,6 @@ def _initialize_variables(self, raw_variable, raw_datasets): if institute: variable['institute'] = institute check.variable(variable, required_keys) - if 'fx_files' in variable: - for fx_file_dict in variable['fx_files']: - DATASET_KEYS.add(fx_file_dict['short_name']) - # Get the fx files - variable['fx_files'] = get_input_fx_filelist( - variable=variable, - rootpath=self._cfg['rootpath'], - drs=self._cfg['drs']) - logger.info("Using fx files for var %s of dataset %s:\n%s", - variable['short_name'], variable['dataset'], - variable['fx_files']) return variables From 25d6fc07844c2b096d765dd942f0805277983719 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 12:36:38 +0100 Subject: [PATCH 21/53] changes to allow correct fx files treatment --- esmvalcore/_recipe.py | 77 ++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 6301461e7b..66144afb9e 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -384,6 +384,28 @@ def _add_fxvar_keys(fx_var_dict, variable): return fx_variable +def _get_correct_fx_file(variable, fx_varname, config_user): + """Wrapper to standard file getter to recover the correct fx file.""" + var = dict(variable) + if var['project'] == 'CMIP5': + fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'fx'}, var) + elif var['project'] == 'CMIP6': + if fx_varname == 'sftlf': + fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'fx'}, + var) + elif fx_varname == 'sftof': + fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Ofx'}, + var) + elif fx_varname == 'sftgif': + fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Efx'}, + var) + fx_files = get_input_filelist(variable=fx_var, + rootpath=config_user['rootpath'], + drs=config_user['drs'])[0] + + return fx_files + + def _update_fx_settings(settings, variable, config_user): """Find and set the FX derive/mask settings.""" # update for derive @@ -392,11 +414,9 @@ def _update_fx_settings(settings, variable, config_user): for var in get_required(variable['short_name']): if 'fx_files' in var: _augment(var, variable) - fx_files.update( - get_input_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs'])) + for fxvar in var['fx_files']: + fx_files[fxvar] = _get_correct_fx_file(var, fxvar, + config_user) settings['derive']['fx_files'] = fx_files # update for landsea @@ -405,17 +425,9 @@ def _update_fx_settings(settings, variable, config_user): # Configure ingestion of land/sea masks logger.debug('Getting fx mask settings now...') settings['mask_landsea']['fx_files'] = [] - var = dict(variable) - if var['project'] == 'CMIP5': - fx_var = _add_fxvar_keys({'short_name': 'sftlf', 'mip': 'fx'}, var) - fx_files_dict['sftlf'] = get_output_file(fx_var, - config_user['preproc_dir'], - masking=True) - if var['project'] == 'CMIP5': - fx_var = _add_fxvar_keys({'short_name': 'sftof', 'mip': 'fx'}, var) - fx_files_dict['sftof'] = get_output_file(fx_var, - config_user['preproc_dir'], - masking=True) + fx_files_dict = { + 'sftlf': _get_correct_fx_file(variable, 'sftlf', config_user), + 'sftof': _get_correct_fx_file(variable, 'sftof', config_user)} # allow both sftlf and sftof if fx_files_dict['sftlf']: settings['mask_landsea']['fx_files'].append(fx_files_dict['sftlf']) @@ -425,10 +437,8 @@ def _update_fx_settings(settings, variable, config_user): if 'mask_landseaice' in settings: logger.debug('Getting fx mask settings now...') settings['mask_landseaice']['fx_files'] = [] - var = dict(variable) - # allow sftgif (only, for now) - fx_files_dict['sftgif'] = get_output_file(var, - config_user['preproc_dir']) + fx_files_dict = { + 'sftgif': _get_correct_fx_file(variable, 'sftgif', config_user)} if fx_files_dict['sftgif']: settings['mask_landseaice']['fx_files'].append( fx_files_dict['sftgif']) @@ -437,8 +447,9 @@ def _update_fx_settings(settings, variable, config_user): if settings.get(step, {}).get('fx_files'): var = dict(variable) var['fx_files'] = settings.get(step, {}).get('fx_files') - fx_files_dict = get_output_file(var, - config_user['preproc_dir']) + fx_files_dict = { + fxvar: _get_correct_fx_file(variable, fxvar, config_user) + for fxvar in var['fx_files']} settings[step]['fx_files'] = fx_files_dict @@ -656,12 +667,18 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, def _remove_temporal_preprocs(profile): """Remove all temporal operations on fx variables.""" fx_profile = profile - if 'mask_landsea' in profile: - fx_profile['mask_landsea'] = False - if 'mask_landseaice' in profile: - fx_profile['mask_landseaice'] = False - if 'seasonal_mean' in profile: - fx_profile['seasonal_mean'] = False + temporal_preprocs = [ + 'extract_season', + 'extract_month', + 'annual_mean', + 'seasonal_mean', + 'time_average', + 'regrid_time', + ] + for temporal_preproc in temporal_preprocs: + if temporal_preproc in profile: + fx_profile[temporal_preproc] = False + return fx_profile @@ -687,9 +704,9 @@ def _get_single_preprocessor_task(variables, config_user=config_user, ))[0] else: - fx_profile = profile - fx_profile['extract_time'] = False fx_profile = _remove_temporal_preprocs(profile) + # not do time extraction; this is a default preproc + fx_profile['extract_time'] = False product = list(_get_preprocessor_products( variables=[variable], profile=fx_profile, From 4102f96c829cac6c66333754e2281fd04fda6ca9 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 12:37:03 +0100 Subject: [PATCH 22/53] revertinguseless changes --- esmvalcore/preprocessor/_derive/nbp_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_derive/nbp_grid.py b/esmvalcore/preprocessor/_derive/nbp_grid.py index ff6169fda6..54a6b95ce6 100644 --- a/esmvalcore/preprocessor/_derive/nbp_grid.py +++ b/esmvalcore/preprocessor/_derive/nbp_grid.py @@ -9,7 +9,7 @@ class DerivedVariable(DerivedVariableBase): # Required variables required = [{ 'short_name': 'nbp', - 'fx_files': [{'short_name': 'sftlf'}], + 'fx_files': ['sftlf'], }] @staticmethod From 19536ed6150fb8313170a441fa17c507c483d481 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 12:37:31 +0100 Subject: [PATCH 23/53] reverting to trunk --- tests/integration/test_data_finder.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index 71e49ae53d..d931f5730c 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -8,7 +8,6 @@ import esmvalcore._config from esmvalcore._data_finder import get_input_filelist, get_output_file -from esmvalcore._recipe import get_input_fx_filelist from esmvalcore.cmor.table import read_cmor_tables # Initialize with standard config developer file @@ -92,29 +91,3 @@ def test_get_input_filelist(root, cfg): # Test result reference = [os.path.join(root, file) for file in cfg['found_files']] assert sorted(input_filelist) == sorted(reference) - - -@pytest.mark.parametrize('cfg', CONFIG['get_input_fx_filelist']) -def test_get_input_fx_filelist(root, cfg): - """Test retrieving fx filelist.""" - create_tree(root, cfg.get('available_files'), - cfg.get('available_symlinks')) - - # Find files - rootpath = {cfg['variable']['project']: [root]} - drs = {cfg['variable']['project']: cfg['drs']} - cfg['variable']['fx_files'] = [ - {'short_name': short_name} for short_name - in cfg['variable']['fx_files'] - ] - fx_files_dict = get_input_fx_filelist( - variable=cfg['variable'], - rootpath=rootpath, - drs=drs) - - # Test result - reference = { - fx_var: os.path.join(root, filename) if filename else None - for fx_var, filename in cfg['found_files'].items() - } - assert fx_files_dict == reference From ed4702cce42486871953cdfc90bdf9c4989802a0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 12:38:00 +0100 Subject: [PATCH 24/53] reverting to trunk --- tests/integration/preprocessor/_derive/test_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py index c0854ed529..460d3f5341 100644 --- a/tests/integration/preprocessor/_derive/test_interface.py +++ b/tests/integration/preprocessor/_derive/test_interface.py @@ -26,7 +26,7 @@ def test_get_required_with_fx(): reference = [{ 'short_name': 'nbp', - 'fx_files': [{'short_name': 'sftlf'}], + 'fx_files': ['sftlf'], }] assert variables == reference From d70cb162941ef1751352927a2e5fcd5c0cd1e552 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 14:07:26 +0100 Subject: [PATCH 25/53] fixed erroneous implementation --- esmvalcore/_recipe.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 66144afb9e..9b98ca8310 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -396,6 +396,7 @@ def _get_correct_fx_file(variable, fx_varname, config_user): elif fx_varname == 'sftof': fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Ofx'}, var) + # TODO check this is the correct mip elif fx_varname == 'sftgif': fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Efx'}, var) @@ -693,28 +694,25 @@ def _get_single_preprocessor_task(variables, order = _extract_preprocessor_order(profile) ancestor_products = [p for task in ancestor_tasks for p in task.products] - products = [] - for variable in variables: - if variable['frequency'] != 'fx': - product = list(_get_preprocessor_products( - variables=[variable], - profile=profile, - order=order, - ancestor_products=ancestor_products, - config_user=config_user, - ))[0] - else: - fx_profile = _remove_temporal_preprocs(profile) - # not do time extraction; this is a default preproc - fx_profile['extract_time'] = False - product = list(_get_preprocessor_products( - variables=[variable], - profile=fx_profile, - order=order, - ancestor_products=None, - config_user=config_user, - ))[0] - products.append(product) + if variables[0]['frequency'] != 'fx': + products = list(_get_preprocessor_products( + variables=variables, + profile=profile, + order=order, + ancestor_products=ancestor_products, + config_user=config_user, + )) + else: + fx_profile = _remove_temporal_preprocs(profile) + # not do time extraction; this is a default preproc + fx_profile['extract_time'] = False + products = list(_get_preprocessor_products( + variables=variables, + profile=fx_profile, + order=order, + ancestor_products=None, + config_user=config_user, + )) if not products: raise RecipeError( From 0abbdc5abd04121501e62480440d6eff6957b218 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 14:25:19 +0100 Subject: [PATCH 26/53] fixed typo and added comment --- esmvalcore/_data_finder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 0bc9450f70..8610f9a0a4 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -232,7 +232,8 @@ def _find_input_files(variable, rootpath, drs): def get_input_filelist(variable, rootpath, drs): """Return the full path to input files.""" # change ensemble to fixed r0i0p0 for fx variables - if variable['project'] == 'CMIP5'and variable['frequency'] == 'fx': + # this is needed and is not a duplicate effort + if variable['project'] == 'CMIP5' and variable['frequency'] == 'fx': variable['ensemble'] = 'r0i0p0' files = _find_input_files(variable, rootpath, drs) # do time gating only for non-fx variables From 55886894cd9f9f5a4dffe6da45af4a1f40cb78aa Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 17 Jul 2019 14:28:16 +0100 Subject: [PATCH 27/53] removed wrong inline comment --- esmvalcore/_recipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 9b98ca8310..301ded6586 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -718,7 +718,6 @@ def _get_single_preprocessor_task(variables, raise RecipeError( "Did not find any input data for task {}".format(name)) - # fx variable task task = PreprocessingTask( products=products, ancestors=ancestor_tasks, From fc1bf65fa0f576f659e51c2955ecbdadc1a28795 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 22 Jul 2019 16:47:36 +0100 Subject: [PATCH 28/53] Added several changes needed to run area_statistics. --- esmvalcore/preprocessor/_area.py | 56 +++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index 9b5cf74bbe..8c048b13db 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -8,6 +8,7 @@ import iris from dask import array as da +import numpy as np logger = logging.getLogger(__name__) @@ -56,20 +57,24 @@ def extract_region(cube, start_longitude, end_longitude, start_latitude, start_latitude = float(start_latitude) end_latitude = float(end_latitude) + # Regular grid if cube.coord('latitude').ndim == 1: region_subset = cube.intersection( longitude=(start_longitude, end_longitude), latitude=(start_latitude, end_latitude)) region_subset = region_subset.intersection(longitude=(0., 360.)) return region_subset - # irregular grids + + # Irregular grids - not lazy. lats = cube.coord('latitude').points lons = cube.coord('longitude').points - select_lats = start_latitude < lats < end_latitude - select_lons = start_longitude < lons < end_longitude - selection = select_lats & select_lons - data = da.ma.masked_where(~selection, cube.core_data()) - return cube.copy(data) + mask = np.ma.array(cube.data).mask + mask += np.ma.masked_where(lats < start_latitude, lats).mask + mask += np.ma.masked_where(lats > end_latitude, lats).mask + mask += np.ma.masked_where(lons > start_longitude, lons).mask + mask += np.ma.masked_where(lons > end_longitude, lons).mask + cube.data = np.ma.masked_where(mask, cube.data) + return cube def get_iris_analysis_operation(operator): @@ -156,19 +161,32 @@ def tile_grid_areas(cube, fx_files): continue logger.info('Attempting to load %s from file: %s', key, fx_file) fx_cube = iris.load_cube(fx_file) - grid_areas = fx_cube.core_data() - if cube.ndim == 4 and grid_areas.ndim == 2: - grid_areas = da.tile(grid_areas, - [cube.shape[0], cube.shape[1], 1, 1]) - elif cube.ndim == 4 and grid_areas.ndim == 3: - grid_areas = da.tile(grid_areas, [cube.shape[0], 1, 1, 1]) - elif cube.ndim == 3 and grid_areas.ndim == 2: - grid_areas = da.tile(grid_areas, [cube.shape[0], 1, 1]) - else: - raise ValueError('Grid and dataset number of dimensions not ' - 'recognised: {} and {}.' - ''.format(cube.ndim, grid_areas.ndim)) + else: + return None + + if grid_areas.shape != cube.shape[-2:]: + raise ValueError('Fx area {} and dataset {} shapes do not match.' + ''.format(grid_areas.shape, cube.shape)) + + if cube.ndim == grid_areas.ndim: + return grid_areas + + # Use dash.array.stack to tile areacello. + elif cube.ndim == 4 and grid_areas.ndim == 2: + for shape in [1, 0]: + ga = [grid_areas for itr in range(cube.shape[shape])] + grid_areas = da.stack(ga, axis=0) + elif cube.ndim == 4 and grid_areas.ndim == 3: + ga = [grid_areas for itr in range(cube.shape[0])] + grid_areas = da.stack(ga, axis=0) + elif cube.ndim == 3 and grid_areas.ndim == 2: + ga = [grid_areas for itr in range(cube.shape[0])] + grid_areas = da.stack(ga, axis=0) + else: + raise ValueError('Grid and dataset number of dimensions not ' + 'recognised: {} and {}.' + ''.format(cube.ndim, grid_areas.ndim)) return grid_areas @@ -246,7 +264,7 @@ def area_statistics(cube, operator, fx_files=None): if operator == 'mean': return cube.collapsed(coord_names, operation, - weights=grid_areas) + weights=np.array(grid_areas)) # Many IRIS analysis functions do not accept weights arguments. return cube.collapsed(coord_names, operation) From b231b65dd443d041237db5d3c4ef6ddce0190f11 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Wed, 24 Jul 2019 10:03:28 +0100 Subject: [PATCH 29/53] Bug fix and added dask array back into the mask. --- esmvalcore/preprocessor/_area.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index 8c048b13db..b9408995c4 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -71,9 +71,9 @@ def extract_region(cube, start_longitude, end_longitude, start_latitude, mask = np.ma.array(cube.data).mask mask += np.ma.masked_where(lats < start_latitude, lats).mask mask += np.ma.masked_where(lats > end_latitude, lats).mask - mask += np.ma.masked_where(lons > start_longitude, lons).mask + mask += np.ma.masked_where(lons < start_longitude, lons).mask mask += np.ma.masked_where(lons > end_longitude, lons).mask - cube.data = np.ma.masked_where(mask, cube.data) + cube.data = da.ma.masked_array(data=cube.data, mask=mask) return cube From 14e29fcbd305266fd410a0006d86e47002c2af1b Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Wed, 24 Jul 2019 12:07:08 +0100 Subject: [PATCH 30/53] Added bugifx to volume statistics _volume.py --- esmvalcore/preprocessor/_volume.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index 4859b73639..73f43933bb 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -260,12 +260,13 @@ def volume_statistics( layer_vol = np.ma.masked_where( cube[time_itr, z_itr].data.mask, grid_volume[time_itr, z_itr]).sum() - except AttributeError: # #### # No mask in the cube data. layer_vol = grid_volume.sum() depth_volume.append(layer_vol) + + column = np.ma.masked_invalid(column) # #### # Calculate weighted mean over the water volumn result.append(np.average(column, weights=depth_volume)) From e0c97a72b6a918b1162269ad51139e296df1a317 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Wed, 24 Jul 2019 12:13:37 +0100 Subject: [PATCH 31/53] minor cosmetic change to appeal the codacy elves. --- esmvalcore/preprocessor/_area.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index b9408995c4..b6a8512b80 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -175,14 +175,14 @@ def tile_grid_areas(cube, fx_files): # Use dash.array.stack to tile areacello. elif cube.ndim == 4 and grid_areas.ndim == 2: for shape in [1, 0]: - ga = [grid_areas for itr in range(cube.shape[shape])] - grid_areas = da.stack(ga, axis=0) + grida = [grid_areas for itr in range(cube.shape[shape])] + grid_areas = da.stack(grida, axis=0) elif cube.ndim == 4 and grid_areas.ndim == 3: - ga = [grid_areas for itr in range(cube.shape[0])] - grid_areas = da.stack(ga, axis=0) + grida = [grid_areas for itr in range(cube.shape[0])] + grid_areas = da.stack(grida, axis=0) elif cube.ndim == 3 and grid_areas.ndim == 2: - ga = [grid_areas for itr in range(cube.shape[0])] - grid_areas = da.stack(ga, axis=0) + grida = [grid_areas for itr in range(cube.shape[0])] + grid_areas = da.stack(grida, axis=0) else: raise ValueError('Grid and dataset number of dimensions not ' 'recognised: {} and {}.' From f54d166217f43f04ca2e5264e7a5abc30e831da4 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 29 Jul 2019 15:36:47 +0100 Subject: [PATCH 32/53] isorted imports --- esmvalcore/_recipe.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 301ded6586..d0435ceed8 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -6,11 +6,10 @@ from copy import deepcopy import yaml - from netCDF4 import Dataset -from . import _recipe_checks as check from . import __version__ +from . import _recipe_checks as check from ._config import TAGS, get_institutes, replace_tags from ._data_finder import (get_input_filelist, get_output_file, get_statistic_output_file) From 7a1f4fd6839bd67de50999f739bda879acbf6347 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 29 Jul 2019 15:43:36 +0100 Subject: [PATCH 33/53] changed hardcoded mip for sftgif --- esmvalcore/_recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index d0435ceed8..d405c20be9 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -395,9 +395,9 @@ def _get_correct_fx_file(variable, fx_varname, config_user): elif fx_varname == 'sftof': fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Ofx'}, var) - # TODO check this is the correct mip + # TODO allow availability for multiple mip's for sftgif elif fx_varname == 'sftgif': - fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'Efx'}, + fx_var = _add_fxvar_keys({'short_name': fx_varname, 'mip': 'fx'}, var) fx_files = get_input_filelist(variable=fx_var, rootpath=config_user['rootpath'], From 6b00da21ad89d3caae83f8187db03b0b812e4cbd Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 29 Jul 2019 15:49:34 +0100 Subject: [PATCH 34/53] removed making a list for products --- esmvalcore/_recipe.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index d405c20be9..4bf7057bc4 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -694,24 +694,24 @@ def _get_single_preprocessor_task(variables, ancestor_products = [p for task in ancestor_tasks for p in task.products] if variables[0]['frequency'] != 'fx': - products = list(_get_preprocessor_products( + products = _get_preprocessor_products( variables=variables, profile=profile, order=order, ancestor_products=ancestor_products, config_user=config_user, - )) + ) else: fx_profile = _remove_temporal_preprocs(profile) # not do time extraction; this is a default preproc fx_profile['extract_time'] = False - products = list(_get_preprocessor_products( + products = _get_preprocessor_products( variables=variables, profile=fx_profile, order=order, ancestor_products=None, config_user=config_user, - )) + ) if not products: raise RecipeError( From 2ef1072c6838213606705ebfbfebf1f99635b135 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 29 Jul 2019 15:58:58 +0100 Subject: [PATCH 35/53] simplified code --- esmvalcore/_recipe.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 4bf7057bc4..f7ff21e686 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -693,25 +693,18 @@ def _get_single_preprocessor_task(variables, order = _extract_preprocessor_order(profile) ancestor_products = [p for task in ancestor_tasks for p in task.products] - if variables[0]['frequency'] != 'fx': - products = _get_preprocessor_products( - variables=variables, - profile=profile, - order=order, - ancestor_products=ancestor_products, - config_user=config_user, - ) - else: - fx_profile = _remove_temporal_preprocs(profile) - # not do time extraction; this is a default preproc - fx_profile['extract_time'] = False - products = _get_preprocessor_products( - variables=variables, - profile=fx_profile, - order=order, - ancestor_products=None, - config_user=config_user, - ) + if variables[0]['frequency'] == 'fx': + profile = _remove_temporal_preprocs(profile) + profile['extract_time'] = False + ancestor_products = None + + products = _get_preprocessor_products( + variables=variables, + profile=profile, + order=order, + ancestor_products=ancestor_products, + config_user=config_user, + ) if not products: raise RecipeError( From 55780cd687f959509579e01c4acbb389a5cb2a40 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 29 Jul 2019 16:19:26 +0100 Subject: [PATCH 36/53] re added the rcast line --- esmvalcore/config-developer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 7aea5531db..ba9219b28f 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -222,6 +222,7 @@ OBS: input_file: default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' BSC: '[short_name]_*.nc' + RCAST: '[short_name]_[mip]_[type]_[dataset]_*.nc' output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP5' From 34528b1e9fdf9b2fe61cd5cb54bc2f16ebe3685a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 27 Aug 2019 11:54:37 +0100 Subject: [PATCH 37/53] removed the temp preproc fx func and replaced it with recipe check --- esmvalcore/_recipe.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 5c414d4bb7..a744e588ca 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -664,24 +664,6 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, return products -def _remove_temporal_preprocs(profile): - """Remove all temporal operations on fx variables.""" - fx_profile = profile - temporal_preprocs = [ - 'extract_season', - 'extract_month', - 'annual_mean', - 'seasonal_mean', - 'time_average', - 'regrid_time', - ] - for temporal_preproc in temporal_preprocs: - if temporal_preproc in profile: - fx_profile[temporal_preproc] = False - - return fx_profile - - def _get_single_preprocessor_task(variables, profile, config_user, @@ -694,7 +676,7 @@ def _get_single_preprocessor_task(variables, ancestor_products = [p for task in ancestor_tasks for p in task.products] if variables[0]['frequency'] == 'fx': - profile = _remove_temporal_preprocs(profile) + check.check_for_temporal_preprocs(profile) profile['extract_time'] = False ancestor_products = None @@ -1012,7 +994,7 @@ def _set_alias(self, preprocessor_output): but it will use the dataset info to compute the others Examples: - + -------- - {project: CMIP5, model: EC-Earth, ensemble: r1i1p1} - {project: CMIP6, model: EC-Earth, ensemble: r1i1p1f1} will generate alias 'CMIP5' and 'CMIP6' @@ -1035,7 +1017,7 @@ def _set_alias(self, preprocessor_output): - {project: CMIP5, model: EC-Earth, experiment: historical} will generate alias 'EC-Earth' - Parameters + Parameters: ---------- preprocessor_output : dict preprocessor output dictionary From 747299c75eba788a7ea6e69c7ea3bcf91a2849c4 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 27 Aug 2019 11:55:03 +0100 Subject: [PATCH 38/53] added recipe error if a temporal preproc step applied for fx vars --- esmvalcore/_recipe_checks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index ee0e569304..4e9a65f889 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -121,3 +121,20 @@ def tasks_valid(tasks): if product.filename in filenames: raise ValueError(msg.format(product.filename)) filenames.add(product.filename) + + +def check_for_temporal_preprocs(profile): + """Check for temporal operations on fx variables.""" + temporal_preprocs = [ + 'extract_season', + 'extract_month', + 'annual_mean', + 'seasonal_mean', + 'time_average', + 'regrid_time', + ] + temp_preprocs = [ + preproc for preproc in profile if preproc in temporal_preprocs] + raise RecipeError( + "Time coordinate preprocessor step {} not permitted on fx vars \ + please remove them from recipe.".format(", ".join(temp_preprocs))) From 7589a713ec3613cda7530a4895f35a65440a613e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 27 Aug 2019 12:48:12 +0100 Subject: [PATCH 39/53] put the check after setting time prproc --- esmvalcore/_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index a744e588ca..1889a550d8 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -676,8 +676,8 @@ def _get_single_preprocessor_task(variables, ancestor_products = [p for task in ancestor_tasks for p in task.products] if variables[0]['frequency'] == 'fx': - check.check_for_temporal_preprocs(profile) profile['extract_time'] = False + check.check_for_temporal_preprocs(profile) ancestor_products = None products = _get_preprocessor_products( From 08f57c26dd7a7eddf5ac2ecfb09c90304ced5f55 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 27 Aug 2019 12:48:28 +0100 Subject: [PATCH 40/53] fixed bug --- esmvalcore/_recipe_checks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe_checks.py b/esmvalcore/_recipe_checks.py index 4e9a65f889..d0456e9ebb 100644 --- a/esmvalcore/_recipe_checks.py +++ b/esmvalcore/_recipe_checks.py @@ -135,6 +135,7 @@ def check_for_temporal_preprocs(profile): ] temp_preprocs = [ preproc for preproc in profile if preproc in temporal_preprocs] - raise RecipeError( - "Time coordinate preprocessor step {} not permitted on fx vars \ - please remove them from recipe.".format(", ".join(temp_preprocs))) + if temp_preprocs: + raise RecipeError( + "Time coordinate preprocessor step {} not permitted on fx vars \ + please remove them from recipe.".format(", ".join(temp_preprocs))) From 63fa525740096e154804fd95d7cc80a30c85a781 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 27 Aug 2019 12:48:45 +0100 Subject: [PATCH 41/53] added test for fx variable --- tests/integration/test_recipe.py | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 60e29e8484..9a641f8a39 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -311,6 +311,87 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): assert product.settings == defaults +def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): + + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + sftlf: + project: CMIP5 + mip: fx + exp: historical + ensemble: r0i0p0 + start_year: 2000 + end_year: 2005 + additional_datasets: + - {dataset: CanESM2} + scripts: null + """) + + recipe = get_recipe(tmp_path, content, config_user) + + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + assert len(task.products) == 1 + product = task.products.pop() + preproc_dir = os.path.dirname(product.filename) + assert preproc_dir.startswith(str(tmp_path)) + + fix_dir = os.path.join( + preproc_dir, + 'CMIP5_CanESM2_fx_historical_r0i0p0_sftlf_2000-2005_fixed') + + defaults = { + 'load': { + 'callback': concatenate_callback, + }, + 'concatenate': {}, + 'fix_file': { + 'project': 'CMIP5', + 'dataset': 'CanESM2', + 'short_name': 'sftlf', + 'output_dir': fix_dir, + }, + 'fix_data': { + 'project': 'CMIP5', + 'dataset': 'CanESM2', + 'short_name': 'sftlf', + 'cmor_table': 'CMIP5', + 'mip': 'fx', + 'frequency': 'fx', + }, + 'fix_metadata': { + 'project': 'CMIP5', + 'dataset': 'CanESM2', + 'short_name': 'sftlf', + 'cmor_table': 'CMIP5', + 'mip': 'fx', + 'frequency': 'fx', + }, + 'cmor_check_metadata': { + 'cmor_table': 'CMIP5', + 'mip': 'fx', + 'short_name': 'sftlf', + 'frequency': 'fx', + }, + 'cmor_check_data': { + 'cmor_table': 'CMIP5', + 'mip': 'fx', + 'short_name': 'sftlf', + 'frequency': 'fx', + }, + 'cleanup': { + 'remove': [fix_dir] + }, + 'save': { + 'compress': False, + 'filename': product.filename, + } + } + assert product.settings == defaults + + def test_empty_variable(tmp_path, patched_datafinder, config_user): """Test that it is possible to specify all information in the dataset.""" content = dedent(""" From 9a265bc95d90684470ca5acfe16c59e0932ef403 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 27 Aug 2019 15:25:10 +0100 Subject: [PATCH 42/53] added sanity check for time invarian preproc steps for fx --- tests/integration/test_recipe.py | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 9a641f8a39..07578b1673 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -224,6 +224,41 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): assert task.settings['custom_setting'] == 1 +def test_fx_preproc_error(tmp_path, patched_datafinder, config_user): + script = tmp_path / 'diagnostic.py' + script.write_text('') + content = dedent(""" + datasets: + - dataset: bcc-csm1-1 + + preprocessors: + preprocessor_name: + extract_season: + season: MAM + + diagnostics: + diagnostic_name: + variables: + sftlf: + preprocessor: preprocessor_name + project: CMIP5 + mip: fx + exp: historical + ensemble: r0i0p0 + start_year: 1999 + end_year: 2002 + additional_datasets: + - dataset: MPI-ESM-LR + scripts: null + """) + rec_err = "Time coordinate preprocessor step extract_season \ + not permitted on fx vars \ + please remove them from recipe." + with pytest.raises(Exception) as rec_err_exp: + get_recipe(tmp_path, content, config_user) + assert rec_err == rec_err_exp + + def test_default_preprocessor(tmp_path, patched_datafinder, config_user): content = dedent(""" From 5bd6d6274b4db4bc2c0694fa66f4dd0a5d8a0cd0 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 28 Aug 2019 16:09:02 +0100 Subject: [PATCH 43/53] removed old fx code for ncl settings --- esmvalcore/preprocessor/_io.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 654903cb5d..cd81c61b67 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -239,11 +239,6 @@ def _write_ncl_metadata(output_dir, metadata): """Write NCL metadata files to output_dir.""" variables = [copy.deepcopy(v) for v in metadata.values()] - for variable in variables: - fx_files = variable.pop('fx_files', {}) - for fx_type in fx_files: - variable[fx_type] = fx_files[fx_type] - info = {'input_file_info': variables} # Split input_file_info into dataset and variable properties From bb262c5796dfcc90ceae3408775908fecbe502dc Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 29 Aug 2019 11:44:04 +0100 Subject: [PATCH 44/53] reverted to developmnet branch version --- esmvalcore/preprocessor/_area.py | 56 +++++++++++--------------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index b6a8512b80..9b5cf74bbe 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -8,7 +8,6 @@ import iris from dask import array as da -import numpy as np logger = logging.getLogger(__name__) @@ -57,24 +56,20 @@ def extract_region(cube, start_longitude, end_longitude, start_latitude, start_latitude = float(start_latitude) end_latitude = float(end_latitude) - # Regular grid if cube.coord('latitude').ndim == 1: region_subset = cube.intersection( longitude=(start_longitude, end_longitude), latitude=(start_latitude, end_latitude)) region_subset = region_subset.intersection(longitude=(0., 360.)) return region_subset - - # Irregular grids - not lazy. + # irregular grids lats = cube.coord('latitude').points lons = cube.coord('longitude').points - mask = np.ma.array(cube.data).mask - mask += np.ma.masked_where(lats < start_latitude, lats).mask - mask += np.ma.masked_where(lats > end_latitude, lats).mask - mask += np.ma.masked_where(lons < start_longitude, lons).mask - mask += np.ma.masked_where(lons > end_longitude, lons).mask - cube.data = da.ma.masked_array(data=cube.data, mask=mask) - return cube + select_lats = start_latitude < lats < end_latitude + select_lons = start_longitude < lons < end_longitude + selection = select_lats & select_lons + data = da.ma.masked_where(~selection, cube.core_data()) + return cube.copy(data) def get_iris_analysis_operation(operator): @@ -161,32 +156,19 @@ def tile_grid_areas(cube, fx_files): continue logger.info('Attempting to load %s from file: %s', key, fx_file) fx_cube = iris.load_cube(fx_file) + grid_areas = fx_cube.core_data() - else: - return None - - if grid_areas.shape != cube.shape[-2:]: - raise ValueError('Fx area {} and dataset {} shapes do not match.' - ''.format(grid_areas.shape, cube.shape)) - - if cube.ndim == grid_areas.ndim: - return grid_areas - - # Use dash.array.stack to tile areacello. - elif cube.ndim == 4 and grid_areas.ndim == 2: - for shape in [1, 0]: - grida = [grid_areas for itr in range(cube.shape[shape])] - grid_areas = da.stack(grida, axis=0) - elif cube.ndim == 4 and grid_areas.ndim == 3: - grida = [grid_areas for itr in range(cube.shape[0])] - grid_areas = da.stack(grida, axis=0) - elif cube.ndim == 3 and grid_areas.ndim == 2: - grida = [grid_areas for itr in range(cube.shape[0])] - grid_areas = da.stack(grida, axis=0) - else: - raise ValueError('Grid and dataset number of dimensions not ' - 'recognised: {} and {}.' - ''.format(cube.ndim, grid_areas.ndim)) + if cube.ndim == 4 and grid_areas.ndim == 2: + grid_areas = da.tile(grid_areas, + [cube.shape[0], cube.shape[1], 1, 1]) + elif cube.ndim == 4 and grid_areas.ndim == 3: + grid_areas = da.tile(grid_areas, [cube.shape[0], 1, 1, 1]) + elif cube.ndim == 3 and grid_areas.ndim == 2: + grid_areas = da.tile(grid_areas, [cube.shape[0], 1, 1]) + else: + raise ValueError('Grid and dataset number of dimensions not ' + 'recognised: {} and {}.' + ''.format(cube.ndim, grid_areas.ndim)) return grid_areas @@ -264,7 +246,7 @@ def area_statistics(cube, operator, fx_files=None): if operator == 'mean': return cube.collapsed(coord_names, operation, - weights=np.array(grid_areas)) + weights=grid_areas) # Many IRIS analysis functions do not accept weights arguments. return cube.collapsed(coord_names, operation) From 95fca8a274d19a42fee2f35594cf92c84d4e21fb Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 29 Aug 2019 11:44:15 +0100 Subject: [PATCH 45/53] reverted to developmnet branch version --- esmvalcore/preprocessor/_volume.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index 73f43933bb..4859b73639 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -260,13 +260,12 @@ def volume_statistics( layer_vol = np.ma.masked_where( cube[time_itr, z_itr].data.mask, grid_volume[time_itr, z_itr]).sum() + except AttributeError: # #### # No mask in the cube data. layer_vol = grid_volume.sum() depth_volume.append(layer_vol) - - column = np.ma.masked_invalid(column) # #### # Calculate weighted mean over the water volumn result.append(np.average(column, weights=depth_volume)) From 4eba1799a47b7a83dc9bb1559fb3abe5ae1f1506 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 30 Aug 2019 15:12:25 +0100 Subject: [PATCH 46/53] check for data files only if not fx --- esmvalcore/_recipe.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 1889a550d8..c12345eaf9 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -484,7 +484,10 @@ def _get_input_files(variable, config_user): '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') or variable['dataset'] == variable.get('reference_dataset')): - check.data_availability(input_files, variable) + if variable['frequency'] != 'fx': + # check only if we are not dealing with fx variables; + # it is ok to have an empty list for fx vars + check.data_availability(input_files, variable) # Set up provenance tracking for i, filename in enumerate(input_files): From 3a46300fedc1d455a0bf6f9ec48cc4a52cf3a6f3 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 9 Sep 2019 12:50:53 +0100 Subject: [PATCH 47/53] fixed bad import resulted from bad fixing of conflict --- esmvalcore/_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 53dc09e3f2..f5f10ade00 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -10,7 +10,7 @@ from . import __version__ from . import _recipe_checks as check -from ._config import TAGS, get_institutes, replace_tags +from ._config import TAGS, get_institutes, get_activity, replace_tags from ._data_finder import (get_input_filelist, get_output_file, get_statistic_output_file) from ._provenance import TrackedFile, get_recipe_provenance From d3830ba60938dbe6006622057c6cace53261e38c Mon Sep 17 00:00:00 2001 From: Mattia Righi Date: Fri, 13 Sep 2019 10:53:21 +0200 Subject: [PATCH 48/53] Fix default filename for OBS --- esmvalcore/config-developer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 969ae2389e..3a71ddbdfe 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -111,7 +111,7 @@ OBS: BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' RCAST: '[dataset]' input_file: - default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' + default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]*.nc' BSC: '[short_name]_*.nc' RCAST: '[short_name]_[mip]_[type]_[dataset]_*.nc' output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' From 255b887f352af52633d1ca9a544eb188f8d263bd Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 13 Sep 2019 12:43:28 +0100 Subject: [PATCH 49/53] readeed underscore so we dont analyze all the vars that start with a pattern name --- esmvalcore/config-developer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 3a71ddbdfe..969ae2389e 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -111,7 +111,7 @@ OBS: BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' RCAST: '[dataset]' input_file: - default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]*.nc' + default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' BSC: '[short_name]_*.nc' RCAST: '[short_name]_[mip]_[type]_[dataset]_*.nc' output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' From d2898db4ce85f7cb0a01811e4a3ab48f936942fa Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 13 Sep 2019 12:44:42 +0100 Subject: [PATCH 50/53] preserve the mip from raw variable --- esmvalcore/_recipe.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index f5f10ade00..0f0394faa8 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -914,6 +914,9 @@ def _initialize_variables(self, raw_variable, raw_datasets): variable = deepcopy(raw_variable) variable.update(dataset) variable['recipe_dataset_index'] = index + # preserve the MIP from RAW in variable object + if 'mip' in raw_variable: + variable['mip'] = raw_variable['mip'] if ('cmor_table' not in variable and variable.get('project') in CMOR_TABLES): variable['cmor_table'] = variable['project'] From 50614d4dcc4a6ceacad6ba71241a31ef17de93ab Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 13 Sep 2019 13:34:44 +0100 Subject: [PATCH 51/53] reinstated data availability checks on all variables --- esmvalcore/_recipe.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 0f0394faa8..637f08aefb 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -476,10 +476,7 @@ def _get_input_files(variable, config_user): '\n'.join(input_files)) if (not config_user.get('skip-nonexistent') or variable['dataset'] == variable.get('reference_dataset')): - if variable['frequency'] != 'fx': - # check only if we are not dealing with fx variables; - # it is ok to have an empty list for fx vars - check.data_availability(input_files, variable) + check.data_availability(input_files, variable) # Set up provenance tracking for i, filename in enumerate(input_files): From f086543619474600dab90562f64f1dec573baf04 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 13 Sep 2019 13:53:01 +0100 Subject: [PATCH 52/53] fixed shitton of missed conflicts gaah --- esmvalcore/_recipe.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 25310ac59a..4fefcaa394 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -413,18 +413,9 @@ def _update_fx_settings(settings, variable, config_user): for var in get_required(variable['short_name']): if 'fx_files' in var: _augment(var, variable) -<<<<<<< HEAD for fxvar in var['fx_files']: fx_files[fxvar] = _get_correct_fx_file(var, fxvar, config_user) -======= - fx_files.update( - get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs'], - )) ->>>>>>> development settings['derive']['fx_files'] = fx_files # update for landsea @@ -433,21 +424,10 @@ def _update_fx_settings(settings, variable, config_user): # Configure ingestion of land/sea masks logger.debug('Getting fx mask settings now...') settings['mask_landsea']['fx_files'] = [] -<<<<<<< HEAD fx_files_dict = { 'sftlf': _get_correct_fx_file(variable, 'sftlf', config_user), 'sftof': _get_correct_fx_file(variable, 'sftof', config_user)} -======= - var = dict(variable) - var['fx_files'] = ['sftlf', 'sftof'] - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs'], - ) - ->>>>>>> development # allow both sftlf and sftof if fx_files_dict['sftlf']: settings['mask_landsea']['fx_files'].append(fx_files_dict['sftlf']) @@ -457,21 +437,8 @@ def _update_fx_settings(settings, variable, config_user): if 'mask_landseaice' in settings: logger.debug('Getting fx mask settings now...') settings['mask_landseaice']['fx_files'] = [] -<<<<<<< HEAD fx_files_dict = { 'sftgif': _get_correct_fx_file(variable, 'sftgif', config_user)} -======= - - var = dict(variable) - var['fx_files'] = ['sftgif'] - fx_files_dict = get_input_fx_filelist( - variable=var, - rootpath=config_user['rootpath'], - drs=config_user['drs'], - ) - - # allow sftgif (only, for now) ->>>>>>> development if fx_files_dict['sftgif']: settings['mask_landseaice']['fx_files'].append( fx_files_dict['sftgif']) @@ -690,15 +657,8 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, ) _update_extract_shape(settings, config_user) _update_fx_settings( -<<<<<<< HEAD settings=settings, variable=variable, config_user=config_user) -======= - settings=settings, - variable=variable, - config_user=config_user, - ) ->>>>>>> development _update_target_grid( variable=variable, variables=variables, From 076b672a828dc4210813071448889c887a052181 Mon Sep 17 00:00:00 2001 From: Mattia Righi Date: Mon, 30 Sep 2019 16:05:05 +0200 Subject: [PATCH 53/53] Remove change in mip definition --- esmvalcore/_recipe.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 4fefcaa394..81a76a6077 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -943,9 +943,6 @@ def _initialize_variables(self, raw_variable, raw_datasets): variable = deepcopy(raw_variable) variable.update(dataset) variable['recipe_dataset_index'] = index - # preserve the MIP from RAW in variable object - if 'mip' in raw_variable: - variable['mip'] = raw_variable['mip'] if ('cmor_table' not in variable and variable.get('project') in CMOR_TABLES): variable['cmor_table'] = variable['project']