From 6be6f66c940be2d08f5caa73220ada752f8b842e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 26 Aug 2019 11:57:48 +0200 Subject: [PATCH 1/7] Use glob in datafinder --- esmvalcore/_data_finder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index 258b315c7d..a94b0cb693 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -8,6 +8,7 @@ import logging import os import re +import glob from ._config import get_project_config, replace_mip_fx from .cmor.table import CMOR_TABLES @@ -209,9 +210,12 @@ def _find_input_dirs(variable, rootpath, drs, fx_var=None): for base_path in root: dirname = os.path.join(base_path, dirname_template) dirname = _resolve_latestversion(dirname) - if os.path.exists(dirname): - logger.debug("Found %s", dirname) - dirnames.append(dirname) + matches = glob.glob(dirname) + matches = [match for match in matches if os.path.isdir(match)] + if matches: + for match in matches: + logger.debug("Found %s", match) + dirnames.append(match) else: logger.debug("Skipping non-existent %s", dirname) From 7de0b338079decb0ddacc2812a31f7e09b8f8aad Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 17 Sep 2019 16:11:13 +0200 Subject: [PATCH 2/7] Change tag brackets --- esmvalcore/_data_finder.py | 12 ++--- esmvalcore/config-developer.yml | 90 ++++++++++++++++----------------- 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index a94b0cb693..e51a15d9a8 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -97,9 +97,7 @@ def select_files(filenames, start_year, end_year): def _replace_tags(path, variable, fx_var=None): """Replace tags in the config-developer's file with actual values.""" path = path.strip('/') - - tlist = re.findall(r'\[([^]]*)\]', path) - + tlist = re.findall(r'{([^}]*)}', path) paths = [path] for tag in tlist: original_tag = tag @@ -116,7 +114,7 @@ def _replace_tags(path, variable, fx_var=None): "your recipe entry".format(tag, variable)) paths = _replace_tag(paths, original_tag, replacewith) - + print(paths) return paths @@ -129,7 +127,7 @@ def _replace_tag(paths, tag, replacewith): result.extend(_replace_tag(paths, tag, item)) else: text = _apply_caps(str(replacewith), lower, upper) - result.extend(p.replace('[' + tag + ']', text) for p in paths) + result.extend(p.replace('{' + tag + '}', text) for p in paths) return result @@ -155,11 +153,11 @@ def _apply_caps(original, lower, upper): def _resolve_latestversion(dirname_template): """Resolve the 'latestversion' tag.""" - if '[latestversion]' not in dirname_template: + if '{latestversion}' not in dirname_template: return dirname_template # Find latest version - part1, part2 = dirname_template.split('[latestversion]') + part1, part2 = dirname_template.split('{latestversion}') part2 = part2.lstrip(os.sep) if os.path.exists(part1): versions = os.listdir(part1) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 10a265778e..c04221ee55 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -22,32 +22,32 @@ CMIP6: cmor_strict: true input_dir: default: '/' - BADC: '[activity]/[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' - DKRZ: '[activity]/[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' - ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/[grid]/' - input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]_*.nc' - output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' + BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' + input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}_*.nc' + output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' CMIP5: cmor_strict: true input_dir: default: '/' - BADC: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/latest/[short_name]' - CP4CDS: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[short_name]/latest/' - DKRZ: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[latestversion]/[short_name]' - ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' - SMHI: '[dataset]/[ensemble]/[exp]/[frequency]' - RCAST: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' - BSC: '[type]/[project]/[exp]/[dataset.lower]' - input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_*.nc' + BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/latest/{short_name}' + CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/' + DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}/{short_name}' + ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' + SMHI: '{dataset}/{ensemble}/{exp}/{frequency}' + RCAST: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' + BSC: '{type}/{project}/{exp}/{dataset.lower}' + input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_*.nc' input_fx_dir: default: '/' - BADC: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[latestversion]/[fx_var]' - CP4CDS: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[fx_var]/latest/' - DKRZ: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[latestversion]/[fx_var]' - ETHZ: '[exp]/fx/[fx_var]/[dataset]/r0i0p0' - input_fx_file: '[fx_var]_fx_[dataset]_[exp]_r0i0p0.nc' + BADC: '{institute}/{dataset}/{exp}/fx/{modeling_realm}/fx/r0i0p0/{latestversion}/{fx_var}' + CP4CDS: '{institute}/{dataset}/{exp}/fx/{modeling_realm}/fx/r0i0p0/{fx_var}/latest/' + DKRZ: '{institute}/{dataset}/{exp}/fx/{modeling_realm}/fx/r0i0p0/{latestversion}/{fx_var}' + ETHZ: '{exp}/fx/{fx_var}/{dataset}/r0i0p0' + input_fx_file: '{fx_var}_fx_{dataset}_{exp}_r0i0p0.nc' fx_mip_change: 'areacella': 'Amon' 'areacello': 'Omon' @@ -60,7 +60,7 @@ CMIP5: 'sftlf': 'Amon' 'sftof': 'Omon' 'volcello': 'Omon' - output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' + output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' institutes: 'ACCESS1-0': ['CSIRO-BOM'] 'ACCESS1-3': ['CSIRO-BOM'] @@ -127,46 +127,46 @@ CMIP5: OBS: cmor_strict: false input_dir: - default: 'Tier[tier]/[dataset]' - BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' - RCAST: '[dataset]' + default: 'Tier{tier}/{dataset}' + BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' + RCAST: '{dataset}' input_file: - default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' - BSC: '[short_name]_*.nc' - RCAST: '[short_name]_[mip]_[type]_[dataset]_*.nc' + default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_*.nc' + BSC: '{short_name}_*.nc' + RCAST: '{short_name}_{mip}_{type}_{dataset}_*.nc' input_fx_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' input_fx_file: - default: '[project]_[dataset]_[type]_[version]_fx_[fx_var].nc' - output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' + default: '{project}_{dataset}_{type}_{version}_fx_{fx_var}.nc' + output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP5' OBS6: cmor_strict: false input_dir: - default: 'Tier[tier]/[dataset]' - BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' + default: 'Tier{tier}/{dataset}' + BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' input_file: - default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' - BSC: '[short_name]_*.nc' + default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_*.nc' + BSC: '{short_name}_*.nc' input_fx_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' input_fx_file: - default: '[project]_[dataset]_[type]_[version]_fx_[fx_var].nc' - output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' + default: '{project}_{dataset}_{type}_{version}_fx_{fx_var}.nc' + output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' obs4mips: cmor_strict: true input_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' RCAST: '/' - input_file: '[short_name]_[dataset]_[level]_[version]_*.nc' + input_file: '{short_name}_{dataset}_{level}_{version}_*.nc' input_fx_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' input_fx_file: - default: '[project]_[dataset]_fx_[fx_var].nc' - output_file: '[project]_[dataset]_[level]_[version]_[short_name]_[start_year]-[end_year]' + default: '{project}_{dataset}_fx_{fx_var}.nc' + output_file: '{project}_{dataset}_{level}_{version}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' cmor_path: 'obs4mips' cmor_default_table_prefix: 'obs4MIPs_' @@ -174,16 +174,16 @@ obs4mips: ana4mips: cmor_strict: false input_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' RCAST: '/' - input_file: '[short_name]_[mip]_[type]_[dataset]_*.nc' - output_file: '[project]_[mip]_[type]_[dataset]_[short_name]_[start_year]-[end_year]' + input_file: '{short_name}_{mip}_{type}_{dataset}_*.nc' + output_file: '{project}_{mip}_{type}_{dataset}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP5' # TODO: add cmor_path and table and set cmor_strict to true EMAC: input_dir: - default: '[dataset]' + default: '{dataset}' input_file: '' - output_file: '[dataset]_[ensemble]_[short_name]_[start_year]-[end_year]' + output_file: '{dataset}_{ensemble}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP5' From 68a3c07e76738215d5f4d08762f26af2eab90f43 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 20 Sep 2019 16:35:32 +0200 Subject: [PATCH 3/7] Address suggestions --- esmvalcore/_data_finder.py | 1 - esmvalcore/config-developer.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index e51a15d9a8..dd251a9a11 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -114,7 +114,6 @@ def _replace_tags(path, variable, fx_var=None): "your recipe entry".format(tag, variable)) paths = _replace_tag(paths, original_tag, replacewith) - print(paths) return paths diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index c04221ee55..0b4cd634ac 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -131,7 +131,7 @@ OBS: BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' RCAST: '{dataset}' input_file: - default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_*.nc' + default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc' BSC: '{short_name}_*.nc' RCAST: '{short_name}_{mip}_{type}_{dataset}_*.nc' input_fx_dir: From 492903453a5ad40b46981b4e853dcdd9872725d0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 26 Sep 2019 16:34:18 +0200 Subject: [PATCH 4/7] Fix test --- tests/integration/test_recipe.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index 31d2d78538..b8a28061a4 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -93,11 +93,15 @@ def find_files(_, filenames): # Any occurrence of [something] in filename should have # been replaced before this function is called. for filename in filenames: - assert '[' not in filename + assert '{' not in filename filename = filenames[0] filename = str(tmp_path / 'input' / filename) filenames = [] + if filename.endswith('[_.]*nc'): + # Restore when we support filenames with no dates + # filenames.append(filename.replace('[_.]*nc', '.nc')) + filename = filename.replace('[_.]*nc', '_*.nc') if filename.endswith('*.nc'): filename = filename[:-len('*.nc')] intervals = [ @@ -110,8 +114,8 @@ def find_files(_, filenames): else: filenames.append(filename) - for file in filenames: - create_test_file(file, next(tracking_id)) + for filename in filenames: + create_test_file(filename, next(tracking_id)) return filenames monkeypatch.setattr(esmvalcore._data_finder, 'find_files', find_files) From 7fdaaf745f42ffd7bfc3d63e3158a93532b10baa Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 8 Oct 2019 11:59:36 +0100 Subject: [PATCH 5/7] align config developer to development branch --- esmvalcore/config-developer.yml | 72 ++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index e18795ea89..1b3f8c975a 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -22,26 +22,26 @@ CMIP6: cmor_strict: true input_dir: default: '/' - BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' - DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' - ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' - input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}_*.nc' - output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' + BADC: '[activity]/[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' + DKRZ: '[activity]/[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' + ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/[grid]/' + input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]*.nc' + output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP6' CMIP5: cmor_strict: true input_dir: default: '/' - BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/latest/{short_name}' - CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/' - DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}/{short_name}' - ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' - SMHI: '{dataset}/{ensemble}/{exp}/{frequency}' - RCAST: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' - BSC: '{type}/{project}/{exp}/{dataset.lower}' - input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_*.nc' - output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' + BADC: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[latestversion]/[short_name]' + CP4CDS: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[short_name]/latest/' + DKRZ: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[latestversion]/[short_name]' + ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' + SMHI: '[dataset]/[ensemble]/[exp]/[frequency]' + RCAST: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' + BSC: '[type]/[project]/[exp]/[dataset.lower]' + input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]*.nc' + output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' institutes: 'ACCESS1-0': ['CSIRO-BOM'] 'ACCESS1-3': ['CSIRO-BOM'] @@ -108,38 +108,38 @@ CMIP5: OBS: cmor_strict: false input_dir: - default: 'Tier{tier}/{dataset}' - BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' - RCAST: '{dataset}' + default: 'Tier[tier]/[dataset]' + BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' + RCAST: '[dataset]' input_file: - default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc' - BSC: '{short_name}_*.nc' - RCAST: '{short_name}_{mip}_{type}_{dataset}_*.nc' - output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' + default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' + BSC: '[short_name]_*.nc' + RCAST: '[short_name]_[mip]_[type]_[dataset]_*.nc' + output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP5' OBS6: cmor_strict: false input_dir: - default: 'Tier{tier}/{dataset}' - BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' + default: 'Tier[tier]/[dataset]' + BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' input_file: - default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_*.nc' - BSC: '{short_name}_*.nc' + default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' + BSC: '[short_name]_*.nc' input_fx_dir: - default: 'Tier{tier}/{dataset}' + default: 'Tier[tier]/[dataset]' input_fx_file: - default: '{project}_{dataset}_{type}_{version}_fx_{fx_var}.nc' - output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' + default: '[project]_[dataset]_[type]_[version]_fx_[fx_var].nc' + output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP6' obs4mips: cmor_strict: true input_dir: - default: 'Tier{tier}/{dataset}' + default: 'Tier[tier]/[dataset]' RCAST: '/' - input_file: '{short_name}_{dataset}_{level}_{version}_*.nc' - output_file: '{project}_{dataset}_{level}_{version}_{short_name}_{start_year}-{end_year}' + input_file: '[short_name]_[dataset]_[level]_[version]_*.nc' + output_file: '[project]_[dataset]_[level]_[version]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP6' cmor_path: 'obs4mips' cmor_default_table_prefix: 'obs4MIPs_' @@ -147,16 +147,16 @@ obs4mips: ana4mips: cmor_strict: false input_dir: - default: 'Tier{tier}/{dataset}' + default: 'Tier[tier]/[dataset]' RCAST: '/' - input_file: '{short_name}_{mip}_{type}_{dataset}_*.nc' - output_file: '{project}_{mip}_{type}_{dataset}_{short_name}_{start_year}-{end_year}' + input_file: '[short_name]_[mip]_[type]_[dataset]_*.nc' + output_file: '[project]_[mip]_[type]_[dataset]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP5' # TODO: add cmor_path and table and set cmor_strict to true EMAC: input_dir: - default: '{dataset}' + default: '[dataset]' input_file: '' - output_file: '{dataset}_{ensemble}_{short_name}_{start_year}-{end_year}' + output_file: '[dataset]_[ensemble]_[short_name]_[start_year]-[end_year]' cmor_type: 'CMIP5' From b2a01ae9ab81ce5203f85cc8e5d7cb8831f9148a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 8 Oct 2019 12:09:24 +0100 Subject: [PATCH 6/7] Revert "align config developer to development branch" This reverts commit 7fdaaf745f42ffd7bfc3d63e3158a93532b10baa. --- esmvalcore/config-developer.yml | 72 ++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 1b3f8c975a..e18795ea89 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -22,26 +22,26 @@ CMIP6: cmor_strict: true input_dir: default: '/' - BADC: '[activity]/[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' - DKRZ: '[activity]/[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]' - ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/[grid]/' - input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]*.nc' - output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' + BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' + ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' + input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}_*.nc' + output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' CMIP5: cmor_strict: true input_dir: default: '/' - BADC: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[latestversion]/[short_name]' - CP4CDS: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[short_name]/latest/' - DKRZ: '[institute]/[dataset]/[exp]/[frequency]/[modeling_realm]/[mip]/[ensemble]/[latestversion]/[short_name]' - ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' - SMHI: '[dataset]/[ensemble]/[exp]/[frequency]' - RCAST: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/' - BSC: '[type]/[project]/[exp]/[dataset.lower]' - input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]*.nc' - output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]' + BADC: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/latest/{short_name}' + CP4CDS: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{short_name}/latest/' + DKRZ: '{institute}/{dataset}/{exp}/{frequency}/{modeling_realm}/{mip}/{ensemble}/{latestversion}/{short_name}' + ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' + SMHI: '{dataset}/{ensemble}/{exp}/{frequency}' + RCAST: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' + BSC: '{type}/{project}/{exp}/{dataset.lower}' + input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_*.nc' + output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' institutes: 'ACCESS1-0': ['CSIRO-BOM'] 'ACCESS1-3': ['CSIRO-BOM'] @@ -108,38 +108,38 @@ CMIP5: OBS: cmor_strict: false input_dir: - default: 'Tier[tier]/[dataset]' - BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' - RCAST: '[dataset]' + default: 'Tier{tier}/{dataset}' + BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' + RCAST: '{dataset}' input_file: - default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' - BSC: '[short_name]_*.nc' - RCAST: '[short_name]_[mip]_[type]_[dataset]_*.nc' - output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' + default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc' + BSC: '{short_name}_*.nc' + RCAST: '{short_name}_{mip}_{type}_{dataset}_*.nc' + output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP5' OBS6: cmor_strict: false input_dir: - default: 'Tier[tier]/[dataset]' - BSC: '[type]/[institute.lower]/[dataset.lower]/[freq_folder]/[short_name][freq_base]' + default: 'Tier{tier}/{dataset}' + BSC: '{type}/{institute.lower}/{dataset.lower}/{freq_folder}/{short_name}{freq_base}' input_file: - default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc' - BSC: '[short_name]_*.nc' + default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_*.nc' + BSC: '{short_name}_*.nc' input_fx_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' input_fx_file: - default: '[project]_[dataset]_[type]_[version]_fx_[fx_var].nc' - output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]' + default: '{project}_{dataset}_{type}_{version}_fx_{fx_var}.nc' + output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' obs4mips: cmor_strict: true input_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' RCAST: '/' - input_file: '[short_name]_[dataset]_[level]_[version]_*.nc' - output_file: '[project]_[dataset]_[level]_[version]_[short_name]_[start_year]-[end_year]' + input_file: '{short_name}_{dataset}_{level}_{version}_*.nc' + output_file: '{project}_{dataset}_{level}_{version}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' cmor_path: 'obs4mips' cmor_default_table_prefix: 'obs4MIPs_' @@ -147,16 +147,16 @@ obs4mips: ana4mips: cmor_strict: false input_dir: - default: 'Tier[tier]/[dataset]' + default: 'Tier{tier}/{dataset}' RCAST: '/' - input_file: '[short_name]_[mip]_[type]_[dataset]_*.nc' - output_file: '[project]_[mip]_[type]_[dataset]_[short_name]_[start_year]-[end_year]' + input_file: '{short_name}_{mip}_{type}_{dataset}_*.nc' + output_file: '{project}_{mip}_{type}_{dataset}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP5' # TODO: add cmor_path and table and set cmor_strict to true EMAC: input_dir: - default: '[dataset]' + default: '{dataset}' input_file: '' - output_file: '[dataset]_[ensemble]_[short_name]_[start_year]-[end_year]' + output_file: '{dataset}_{ensemble}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP5' From f25dd06299b74c4cea16d3478588a63e0c7a562d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Tue, 8 Oct 2019 12:19:11 +0100 Subject: [PATCH 7/7] bring config developer to standards --- esmvalcore/config-developer.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index e18795ea89..82f0d347b7 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -25,7 +25,7 @@ CMIP6: BADC: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' DKRZ: '{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{latestversion}' ETHZ: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/{grid}/' - input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}_*.nc' + input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc' output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6' @@ -40,7 +40,7 @@ CMIP5: SMHI: '{dataset}/{ensemble}/{exp}/{frequency}' RCAST: '{exp}/{mip}/{short_name}/{dataset}/{ensemble}/' BSC: '{type}/{project}/{exp}/{dataset.lower}' - input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}_*.nc' + input_file: '{short_name}_{mip}_{dataset}_{exp}_{ensemble}*.nc' output_file: '{project}_{dataset}_{mip}_{exp}_{ensemble}_{short_name}_{start_year}-{end_year}' institutes: 'ACCESS1-0': ['CSIRO-BOM'] @@ -126,10 +126,6 @@ OBS6: input_file: default: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_*.nc' BSC: '{short_name}_*.nc' - input_fx_dir: - default: 'Tier{tier}/{dataset}' - input_fx_file: - default: '{project}_{dataset}_{type}_{version}_fx_{fx_var}.nc' output_file: '{project}_{dataset}_{type}_{version}_{mip}_{short_name}_{start_year}-{end_year}' cmor_type: 'CMIP6'