From 506859440197bb5b84ac55202cd63c0998e8887d Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Sun, 12 May 2019 13:16:53 +0100
Subject: [PATCH 01/18] removed all traces of specific fx file retrieval

---
 esmvaltool/_data_finder.py | 53 ++++++++++++--------------------------
 1 file changed, 16 insertions(+), 37 deletions(-)

diff --git a/esmvaltool/_data_finder.py b/esmvaltool/_data_finder.py
index ba6e21d532..1f9702dcbd 100644
--- a/esmvaltool/_data_finder.py
+++ b/esmvaltool/_data_finder.py
@@ -11,8 +11,7 @@
 
 import six
 
-from ._config import get_project_config, replace_mip_fx
-from .cmor.table import CMOR_TABLES
+from ._config import get_project_config
 
 logger = logging.getLogger(__name__)
 
@@ -95,7 +94,7 @@ def select_files(filenames, start_year, end_year):
     return selection
 
 
-def _replace_tags(path, variable, fx_var=None):
+def _replace_tags(path, variable):
     """Replace tags in the config-developer's file with actual values."""
     path = path.strip('/')
 
@@ -106,9 +105,7 @@ def _replace_tags(path, variable, fx_var=None):
         original_tag = tag
         tag, _, _ = _get_caps_options(tag)
 
-        if tag == 'fx_var':
-            replacewith = fx_var
-        elif tag == 'latestversion':  # handled separately later
+        if tag == 'latestversion':  # handled separately later
             continue
         elif tag in variable:
             replacewith = variable[tag]
@@ -198,16 +195,16 @@ def get_rootpath(rootpath, project):
     raise KeyError('default rootpath must be specified in config-user file')
 
 
-def _find_input_dirs(variable, rootpath, drs, fx_var=None):
+def _find_input_dirs(variable, rootpath, drs):
     """Return a the full paths to input directories."""
     project = variable['project']
 
     root = get_rootpath(rootpath, project)
-    input_type = 'input_{}dir'.format('fx_' if fx_var else '')
+    input_type = 'input_dir'
     path_template = _select_drs(input_type, drs, project)
 
     dirnames = []
-    for dirname_template in _replace_tags(path_template, variable, fx_var):
+    for dirname_template in _replace_tags(path_template, variable):
         for base_path in root:
             dirname = os.path.join(base_path, dirname_template)
             dirname = _resolve_latestversion(dirname)
@@ -220,21 +217,17 @@ def _find_input_dirs(variable, rootpath, drs, fx_var=None):
     return dirnames
 
 
-def _get_filenames_glob(variable, drs, fx_var=None):
+def _get_filenames_glob(variable, drs):
     """Return patterns that can be used to look for input files."""
-    input_type = 'input_{}file'.format('fx_' if fx_var else '')
+    input_type = 'input_file'
     path_template = _select_drs(input_type, drs, variable['project'])
-    filenames_glob = _replace_tags(path_template, variable, fx_var)
+    filenames_glob = _replace_tags(path_template, variable)
     return filenames_glob
 
 
-def _find_input_files(variable, rootpath, drs, fx_var=None):
-    logger.debug("Looking for input %sfiles for variable %s of dataset %s",
-                 fx_var + ' fx ' if fx_var else '', variable['short_name'],
-                 variable['dataset'])
-
-    input_dirs = _find_input_dirs(variable, rootpath, drs, fx_var)
-    filenames_glob = _get_filenames_glob(variable, drs, fx_var)
+def _find_input_files(variable, rootpath, drs):
+    input_dirs = _find_input_dirs(variable, rootpath, drs)
+    filenames_glob = _get_filenames_glob(variable, drs)
     files = find_files(input_dirs, filenames_glob)
 
     return files
@@ -243,27 +236,13 @@ def _find_input_files(variable, rootpath, drs, fx_var=None):
 def get_input_filelist(variable, rootpath, drs):
     """Return the full path to input files."""
     files = _find_input_files(variable, rootpath, drs)
-    files = select_files(files, variable['start_year'], variable['end_year'])
+    # do time gating only for non-fx variables
+    if variable['frequency'] != 'fx':
+        files = select_files(files, variable['start_year'],
+                             variable['end_year'])
     return files
 
 
-def get_input_fx_filelist(variable, rootpath, drs):
-    """Return a dict with the full path to fx input files."""
-    fx_files = {}
-    for fx_var in variable['fx_files']:
-        var = dict(variable)
-        var['mip'] = replace_mip_fx(fx_var)
-        table = CMOR_TABLES[var['cmor_table']].get_table(var['mip'])
-        var['frequency'] = table.frequency
-        realm = getattr(table.get(var['short_name']), 'modeling_realm', None)
-        var['modeling_realm'] = realm if realm else table.realm
-
-        files = _find_input_files(var, rootpath, drs, fx_var)
-        fx_files[fx_var] = files[0] if files else None
-
-    return fx_files
-
-
 def get_output_file(variable, preproc_dir):
     """Return the full path to the output (preprocessed) file."""
     cfg = get_project_config(variable['project'])

From 909e34f6495734080387a5fffaee1aac3a53311b Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Sun, 12 May 2019 13:17:50 +0100
Subject: [PATCH 02/18] do time checks only if not fx variable

---
 esmvaltool/cmor/check.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/esmvaltool/cmor/check.py b/esmvaltool/cmor/check.py
index 7f9399d545..8a9a9eec35 100644
--- a/esmvaltool/cmor/check.py
+++ b/esmvaltool/cmor/check.py
@@ -95,13 +95,15 @@ def check_metadata(self, logger=None):
         self._check_fill_value()
         self._check_dim_names()
         self._check_coords()
-        self._check_time_coord()
+        if self.frequency != 'fx':
+            self._check_time_coord()
         self._check_rank()
 
         self.report_warnings(logger)
         self.report_errors()
 
-        self._add_auxiliar_time_coordinates()
+        if self.frequency != 'fx':
+            self._add_auxiliar_time_coordinates()
         return self._cube
 
     def report_errors(self):

From 1bbf364ff4490ee6544662de1b3bd942566c7219 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Sun, 12 May 2019 13:18:43 +0100
Subject: [PATCH 03/18] removed references to fx files and dirs

---
 esmvaltool/config-developer.yml | 31 ++-----------------------------
 1 file changed, 2 insertions(+), 29 deletions(-)

diff --git a/esmvaltool/config-developer.yml b/esmvaltool/config-developer.yml
index 06433da1bf..eb8e3464f6 100644
--- a/esmvaltool/config-developer.yml
+++ b/esmvaltool/config-developer.yml
@@ -24,7 +24,7 @@ CMIP6:
     BADC: '[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]'
     DKRZ: '[institute]/[dataset]/[exp]/[ensemble]/[mip]/[short_name]/[grid]/[latestversion]'
     ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/[grid]/'
-  input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]_*.nc'
+  input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_[grid]*.nc'
   output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]'
   cmor_type: 'CMIP6'
   institutes:
@@ -147,26 +147,7 @@ CMIP5:
     ETHZ: '[exp]/[mip]/[short_name]/[dataset]/[ensemble]/'
     SMHI: '[dataset]/[ensemble]/[exp]/[frequency]'
     BSC: '[type]/[project]/[exp]/[dataset.lower]'
-  input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]_*.nc'
-  input_fx_dir:
-    default: '/'
-    BADC: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[latestversion]/[fx_var]'
-    CP4CDS: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[fx_var]/latest/'
-    DKRZ: '[institute]/[dataset]/[exp]/fx/[modeling_realm]/fx/r0i0p0/[latestversion]/[fx_var]'
-    ETHZ: '[exp]/fx/[fx_var]/[dataset]/r0i0p0'
-  input_fx_file: '[fx_var]_fx_[dataset]_[exp]_r0i0p0.nc'
-  fx_mip_change:
-    'areacella': 'Amon'
-    'areacello': 'Omon'
-    'basin': 'Omon'
-    'deptho': 'Omon'
-    'mrsofc': 'Lmon'
-    'orog': 'Amon'
-    'rootd': 'Lmon'
-    'sftgif': 'Lmon'
-    'sftlf': 'Amon'
-    'sftof': 'Omon'
-    'volcello': 'Omon'
+  input_file: '[short_name]_[mip]_[dataset]_[exp]_[ensemble]*.nc'
   output_file: '[project]_[dataset]_[mip]_[exp]_[ensemble]_[short_name]_[start_year]-[end_year]'
   institutes:
     'ACCESS1-0': ['CSIRO-BOM']
@@ -239,10 +220,6 @@ OBS:
   input_file:
     default: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_*.nc'
     BSC: '[short_name]_*.nc'
-  input_fx_dir:
-    default: 'Tier[tier]/[dataset]'
-  input_fx_file:
-    default: '[project]_[dataset]_[type]_[version]_fx_[fx_var].nc'
   output_file: '[project]_[dataset]_[type]_[version]_[mip]_[short_name]_[start_year]-[end_year]'
   cmor_type: 'CMIP5'
 
@@ -251,10 +228,6 @@ obs4mips:
   input_dir:
     default: 'Tier[tier]/[dataset]'
   input_file: '[short_name]_[dataset]_[level]_[version]_*.nc'
-  input_fx_dir:
-    default: 'Tier[tier]/[dataset]'
-  input_fx_file:
-    default: '[project]_[dataset]_fx_[fx_var].nc'
   output_file: '[project]_[dataset]_[level]_[version]_[short_name]_[start_year]-[end_year]'
   cmor_type: 'CMIP6'
   cmor_path: 'obs4mips'

From 3e6342442ae95dc5b3bdb9fc0f769a94cbb1321e Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Sun, 12 May 2019 13:19:32 +0100
Subject: [PATCH 04/18] removed fx files function

---
 esmvaltool/_config.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/esmvaltool/_config.py b/esmvaltool/_config.py
index 221c06d026..cc4ed79e4e 100644
--- a/esmvaltool/_config.py
+++ b/esmvaltool/_config.py
@@ -175,18 +175,6 @@ def get_institutes(variable):
     return CFG.get(project, {}).get('institutes', {}).get(dataset, [])
 
 
-def replace_mip_fx(fx_file):
-    """Replace MIP so to retrieve correct fx files."""
-    default_mip = 'Amon'
-    if fx_file not in CFG['CMIP5']['fx_mip_change']:
-        logger.warning(
-            'mip for fx variable %s is not specified in '
-            'config_developer.yml, using default (%s)', fx_file, default_mip)
-    new_mip = CFG['CMIP5']['fx_mip_change'].get(fx_file, default_mip)
-    logger.debug("Switching mip for fx file finding to %s", new_mip)
-    return new_mip
-
-
 TAGS_CONFIG_FILE = os.path.join(
     os.path.dirname(__file__), 'config-references.yml')
 

From 33b906604a3e743b033b21bec8cdb32dda516b90 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Sun, 12 May 2019 13:21:15 +0100
Subject: [PATCH 05/18] restructuring to allow for fx vars treated like any
 other var

---
 esmvaltool/_recipe.py | 113 ++++++++++++++++++++++++++++++------------
 1 file changed, 80 insertions(+), 33 deletions(-)

diff --git a/esmvaltool/_recipe.py b/esmvaltool/_recipe.py
index 6ab0236f55..c0a7c26e12 100644
--- a/esmvaltool/_recipe.py
+++ b/esmvaltool/_recipe.py
@@ -11,8 +11,8 @@
 from . import __version__
 from . import _recipe_checks as check
 from ._config import TAGS, get_institutes, replace_tags
-from ._data_finder import (get_input_filelist, get_input_fx_filelist,
-                           get_output_file, get_statistic_output_file)
+from ._data_finder import (get_input_filelist, get_output_file,
+                           get_statistic_output_file)
 from ._provenance import TrackedFile, get_recipe_provenance
 from ._recipe_checks import RecipeError
 from ._task import (DiagnosticTask, get_flattened_tasks, get_independent_tasks,
@@ -361,6 +361,26 @@ def _get_default_settings(variable, config_user, derive=False):
     return settings
 
 
+def _add_fxvar_keys(fx_var_dict, variable):
+    """Add a couple keys specific to fx variable."""
+    fx_variable = deepcopy(variable)
+
+    # add internal recognition flag
+    fx_variable['variable_group'] = fx_var_dict['short_name']
+    fx_variable['short_name'] = fx_var_dict['short_name']
+
+    # specificities of project
+    if fx_variable['project'] == 'CMIP5':
+        fx_variable['mip'] = 'fx'
+        fx_variable['ensemble'] = 'r0i0p0'
+    elif fx_variable['project'] == 'CMIP6':
+        fx_variable['grid'] = variable['grid']
+        if 'mip' in fx_var_dict:
+            fx_variable['mip'] = fx_var_dict['mip']
+
+    return fx_variable
+
+
 def _update_fx_settings(settings, variable, config_user):
     """Find and set the FX derive/mask settings."""
     # update for derive
@@ -369,11 +389,13 @@ def _update_fx_settings(settings, variable, config_user):
         for var in get_required(variable['short_name']):
             if 'fx_files' in var:
                 _augment(var, variable)
-                fx_files.update(
-                    get_input_fx_filelist(
-                        variable=var,
-                        rootpath=config_user['rootpath'],
-                        drs=config_user['drs']))
+                for fx_var_dict in var['fx_files']:
+                    fx_var = _add_fxvar_keys(fx_var_dict, variable)
+                    fx_files.update(
+                        get_input_filelist(
+                            variable=fx_var,
+                            rootpath=config_user['rootpath'],
+                            drs=config_user['drs']))
         settings['derive']['fx_files'] = fx_files
 
     # update for landsea
@@ -384,11 +406,14 @@ def _update_fx_settings(settings, variable, config_user):
         settings['mask_landsea']['fx_files'] = []
 
         var = dict(variable)
-        var['fx_files'] = ['sftlf', 'sftof']
-        fx_files_dict = get_input_fx_filelist(
-            variable=var,
-            rootpath=config_user['rootpath'],
-            drs=config_user['drs'])
+        var['fx_files'] = [{'short_name': 'sftlf'}, {'short_name': 'sftof'}]
+        fx_files_dict = {}
+        for fx_var_dict in var['fx_files']:
+            fx_var = _add_fxvar_keys(fx_var_dict, var)
+            fx_files_dict[fx_var['short_name']] = get_input_filelist(
+                variable=fx_var,
+                rootpath=config_user['rootpath'],
+                drs=config_user['drs'])
 
         # allow both sftlf and sftof
         if fx_files_dict['sftlf']:
@@ -402,11 +427,14 @@ def _update_fx_settings(settings, variable, config_user):
         settings['mask_landseaice']['fx_files'] = []
 
         var = dict(variable)
-        var['fx_files'] = ['sftgif']
-        fx_files_dict = get_input_fx_filelist(
-            variable=var,
-            rootpath=config_user['rootpath'],
-            drs=config_user['drs'])
+        var['fx_files'] = [{'short_name': 'sftgif'}]
+        fx_files_dict = {}
+        for fx_var_dict in var['fx_files']:
+            fx_var = _add_fxvar_keys(fx_var_dict, var)
+            fx_files_dict[fx_var['short_name']] = get_input_filelist(
+                variable=fx_var,
+                rootpath=config_user['rootpath'],
+                drs=config_user['drs'])
 
         # allow sftgif (only, for now)
         if fx_files_dict['sftgif']:
@@ -415,11 +443,16 @@ def _update_fx_settings(settings, variable, config_user):
 
     for step in ('average_region', 'average_volume'):
         if settings.get(step, {}).get('fx_files'):
-            settings[step]['fx_files'] = get_input_fx_filelist(
-                variable=variable,
-                rootpath=config_user['rootpath'],
-                drs=config_user['drs'],
-            )
+            var = dict(variable)
+            var['fx_files'] = settings.get(step, {}).get('fx_files')
+            fx_files_dict = {}
+            for fx_var_dict in var['fx_files']:
+                fx_var = _add_fxvar_keys(fx_var_dict, var)
+                fx_files_dict[fx_var['short_name']] = get_input_filelist(
+                    variable=fx_var,
+                    rootpath=config_user['rootpath'],
+                    drs=config_user['drs'])
+            settings[step]['fx_files'] = fx_files_dict
 
 
 def _read_attributes(filename):
@@ -438,22 +471,27 @@ def _read_attributes(filename):
 def _get_input_files(variable, config_user):
     """Get the input files for a single dataset."""
     # Find input files locally.
+    var = dict(variable)
+    # change ensemble to fixed r0i0p0
+    if var['project'] == 'CMIP5':
+        if var['frequency'] == 'fx':
+            var['ensemble'] = 'r0i0p0'
     input_files = get_input_filelist(
-        variable=variable,
+        variable=var,
         rootpath=config_user['rootpath'],
         drs=config_user['drs'])
 
     # Set up downloading using synda if requested.
     # Do not download if files are already available locally.
     if config_user['synda_download'] and not input_files:
-        input_files = synda_search(variable)
+        input_files = synda_search(var)
 
     logger.info("Using input files for variable %s of dataset %s:\n%s",
-                variable['short_name'], variable['dataset'],
+                var['short_name'], var['dataset'],
                 '\n'.join(input_files))
     if (not config_user.get('skip-nonexistent')
-            or variable['dataset'] == variable.get('reference_dataset')):
-        check.data_availability(input_files, variable)
+            or var['dataset'] == var.get('reference_dataset')):
+        check.data_availability(input_files, var)
 
     # Set up provenance tracking
     for i, filename in enumerate(input_files):
@@ -772,6 +810,10 @@ def _get_preprocessor_task(variables, profiles, config_user, task_name):
                 name=derive_name)
             derive_tasks.append(task)
 
+    # don't do time gating for fx variables
+    if variables[0]['frequency'] == 'fx':
+        profile['extract_time'] = False
+
     # Create (final) preprocessor task
     task = _get_single_preprocessor_task(
         variables,
@@ -905,13 +947,18 @@ def _initialize_variables(self, raw_variable, raw_datasets):
                 variable['institute'] = institute
             check.variable(variable, required_keys)
             if 'fx_files' in variable:
-                for fx_file in variable['fx_files']:
-                    DATASET_KEYS.add(fx_file)
+                for fx_file_dict in variable['fx_files']:
+                    DATASET_KEYS.add(fx_file_dict['short_name'])
                 # Get the fx files
-                variable['fx_files'] = get_input_fx_filelist(
-                    variable=variable,
-                    rootpath=self._cfg['rootpath'],
-                    drs=self._cfg['drs'])
+                fx_files_dict = {}
+                for fx_var_dict in variable['fx_files']:
+                    fx_var = _add_fxvar_keys(fx_var_dict, variable)
+                    _add_cmor_info(fx_var)
+                    fx_files_dict[fx_var['short_name']] = get_input_filelist(
+                        variable=fx_var,
+                        rootpath=self._cfg['rootpath'],
+                        drs=self._cfg['drs'])
+                variable['fx_files'] = fx_files_dict
                 logger.info("Using fx files for var %s of dataset %s:\n%s",
                             variable['short_name'], variable['dataset'],
                             variable['fx_files'])

From 5614f7c44c1db4ff355ec52661e62eddc13de8bc Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 15:30:26 +0100
Subject: [PATCH 06/18] cleaning up

---
 esmvaltool/_data_finder.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/esmvaltool/_data_finder.py b/esmvaltool/_data_finder.py
index 1f9702dcbd..158272daa7 100644
--- a/esmvaltool/_data_finder.py
+++ b/esmvaltool/_data_finder.py
@@ -235,6 +235,9 @@ def _find_input_files(variable, rootpath, drs):
 
 def get_input_filelist(variable, rootpath, drs):
     """Return the full path to input files."""
+    # change ensemble to fixed r0i0p0 for fx variables
+    if variable['project'] == 'CMIP5'and variable['frequency'] == 'fx':
+        variable['ensemble'] = 'r0i0p0'
     files = _find_input_files(variable, rootpath, drs)
     # do time gating only for non-fx variables
     if variable['frequency'] != 'fx':

From 475cd8f9e897b0a4a87ad54cffaf70f84d6cfa20 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 15:31:23 +0100
Subject: [PATCH 07/18] cleaning up

---
 esmvaltool/_recipe.py | 101 +++++++++++++++++++++---------------------
 1 file changed, 50 insertions(+), 51 deletions(-)

diff --git a/esmvaltool/_recipe.py b/esmvaltool/_recipe.py
index c0a7c26e12..79524c7970 100644
--- a/esmvaltool/_recipe.py
+++ b/esmvaltool/_recipe.py
@@ -6,10 +6,11 @@
 from copy import deepcopy
 
 import yaml
+
 from netCDF4 import Dataset
 
-from . import __version__
 from . import _recipe_checks as check
+from . import __version__
 from ._config import TAGS, get_institutes, replace_tags
 from ._data_finder import (get_input_filelist, get_output_file,
                            get_statistic_output_file)
@@ -361,11 +362,28 @@ def _get_default_settings(variable, config_user, derive=False):
     return settings
 
 
+def get_input_fx_filelist(variable, rootpath, drs):
+    """Return a dict with fx vars keys and full file paths values."""
+    fx_files_dict = {}
+    for fx_var_dict in variable['fx_files']:
+        fx_var = _add_fxvar_keys(fx_var_dict, variable)
+        fx_files = get_input_filelist(
+            variable=fx_var,
+            rootpath=rootpath,
+            drs=drs)
+        if fx_files:
+            fx_files_dict[fx_var['short_name']] = fx_files[0]
+        else:
+            fx_files_dict[fx_var['short_name']] = None
+
+    return fx_files_dict
+
+
 def _add_fxvar_keys(fx_var_dict, variable):
-    """Add a couple keys specific to fx variable."""
-    fx_variable = deepcopy(variable)
+    """Add keys specific to fx variable to use get_input_filelist."""
+    fx_variable = dict(variable)
 
-    # add internal recognition flag
+    # set variable names
     fx_variable['variable_group'] = fx_var_dict['short_name']
     fx_variable['short_name'] = fx_var_dict['short_name']
 
@@ -377,6 +395,8 @@ def _add_fxvar_keys(fx_var_dict, variable):
         fx_variable['grid'] = variable['grid']
         if 'mip' in fx_var_dict:
             fx_variable['mip'] = fx_var_dict['mip']
+    # add missing cmor info
+    _add_cmor_info(fx_variable, override=True)
 
     return fx_variable
 
@@ -389,13 +409,11 @@ def _update_fx_settings(settings, variable, config_user):
         for var in get_required(variable['short_name']):
             if 'fx_files' in var:
                 _augment(var, variable)
-                for fx_var_dict in var['fx_files']:
-                    fx_var = _add_fxvar_keys(fx_var_dict, variable)
-                    fx_files.update(
-                        get_input_filelist(
-                            variable=fx_var,
-                            rootpath=config_user['rootpath'],
-                            drs=config_user['drs']))
+                fx_files.update(
+                    get_input_fx_filelist(
+                        variable=var,
+                        rootpath=config_user['rootpath'],
+                        drs=config_user['drs']))
         settings['derive']['fx_files'] = fx_files
 
     # update for landsea
@@ -407,13 +425,10 @@ def _update_fx_settings(settings, variable, config_user):
 
         var = dict(variable)
         var['fx_files'] = [{'short_name': 'sftlf'}, {'short_name': 'sftof'}]
-        fx_files_dict = {}
-        for fx_var_dict in var['fx_files']:
-            fx_var = _add_fxvar_keys(fx_var_dict, var)
-            fx_files_dict[fx_var['short_name']] = get_input_filelist(
-                variable=fx_var,
-                rootpath=config_user['rootpath'],
-                drs=config_user['drs'])
+        fx_files_dict = get_input_fx_filelist(
+            variable=var,
+            rootpath=config_user['rootpath'],
+            drs=config_user['drs'])
 
         # allow both sftlf and sftof
         if fx_files_dict['sftlf']:
@@ -428,13 +443,10 @@ def _update_fx_settings(settings, variable, config_user):
 
         var = dict(variable)
         var['fx_files'] = [{'short_name': 'sftgif'}]
-        fx_files_dict = {}
-        for fx_var_dict in var['fx_files']:
-            fx_var = _add_fxvar_keys(fx_var_dict, var)
-            fx_files_dict[fx_var['short_name']] = get_input_filelist(
-                variable=fx_var,
-                rootpath=config_user['rootpath'],
-                drs=config_user['drs'])
+        fx_files_dict = get_input_fx_filelist(
+            variable=var,
+            rootpath=config_user['rootpath'],
+            drs=config_user['drs'])
 
         # allow sftgif (only, for now)
         if fx_files_dict['sftgif']:
@@ -445,13 +457,10 @@ def _update_fx_settings(settings, variable, config_user):
         if settings.get(step, {}).get('fx_files'):
             var = dict(variable)
             var['fx_files'] = settings.get(step, {}).get('fx_files')
-            fx_files_dict = {}
-            for fx_var_dict in var['fx_files']:
-                fx_var = _add_fxvar_keys(fx_var_dict, var)
-                fx_files_dict[fx_var['short_name']] = get_input_filelist(
-                    variable=fx_var,
-                    rootpath=config_user['rootpath'],
-                    drs=config_user['drs'])
+            fx_files_dict = get_input_fx_filelist(
+                variable=var,
+                rootpath=config_user['rootpath'],
+                drs=config_user['drs'])
             settings[step]['fx_files'] = fx_files_dict
 
 
@@ -471,27 +480,22 @@ def _read_attributes(filename):
 def _get_input_files(variable, config_user):
     """Get the input files for a single dataset."""
     # Find input files locally.
-    var = dict(variable)
-    # change ensemble to fixed r0i0p0
-    if var['project'] == 'CMIP5':
-        if var['frequency'] == 'fx':
-            var['ensemble'] = 'r0i0p0'
     input_files = get_input_filelist(
-        variable=var,
+        variable=variable,
         rootpath=config_user['rootpath'],
         drs=config_user['drs'])
 
     # Set up downloading using synda if requested.
     # Do not download if files are already available locally.
     if config_user['synda_download'] and not input_files:
-        input_files = synda_search(var)
+        input_files = synda_search(variable)
 
     logger.info("Using input files for variable %s of dataset %s:\n%s",
-                var['short_name'], var['dataset'],
+                variable['short_name'], variable['dataset'],
                 '\n'.join(input_files))
     if (not config_user.get('skip-nonexistent')
-            or var['dataset'] == var.get('reference_dataset')):
-        check.data_availability(input_files, var)
+            or var['dataset'] == variable.get('reference_dataset')):
+        check.data_availability(input_files, variable)
 
     # Set up provenance tracking
     for i, filename in enumerate(input_files):
@@ -950,15 +954,10 @@ def _initialize_variables(self, raw_variable, raw_datasets):
                 for fx_file_dict in variable['fx_files']:
                     DATASET_KEYS.add(fx_file_dict['short_name'])
                 # Get the fx files
-                fx_files_dict = {}
-                for fx_var_dict in variable['fx_files']:
-                    fx_var = _add_fxvar_keys(fx_var_dict, variable)
-                    _add_cmor_info(fx_var)
-                    fx_files_dict[fx_var['short_name']] = get_input_filelist(
-                        variable=fx_var,
-                        rootpath=self._cfg['rootpath'],
-                        drs=self._cfg['drs'])
-                variable['fx_files'] = fx_files_dict
+                variable['fx_files'] = get_input_fx_filelist(
+                    variable=variable,
+                    rootpath=self._cfg['rootpath'],
+                    drs=self._cfg['drs'])
                 logger.info("Using fx files for var %s of dataset %s:\n%s",
                             variable['short_name'], variable['dataset'],
                             variable['fx_files'])

From 0ee4a729fb59803f3704bebb7bab1eb1188faa5b Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 15:32:36 +0100
Subject: [PATCH 08/18] fixed test for new fx file retrieval

---
 tests/integration/test_data_finder.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py
index 484fdc99b7..ef8826cb44 100644
--- a/tests/integration/test_data_finder.py
+++ b/tests/integration/test_data_finder.py
@@ -3,12 +3,11 @@
 import shutil
 import tempfile
 
+import esmvaltool._config
 import pytest
 import yaml
-
-import esmvaltool._config
-from esmvaltool._data_finder import (get_input_filelist, get_input_fx_filelist,
-                                     get_output_file)
+from esmvaltool._data_finder import get_input_filelist, get_output_file
+from esmvaltool._recipe import get_input_fx_filelist
 from esmvaltool.cmor.table import read_cmor_tables
 
 # Initialize with standard config developer file
@@ -103,11 +102,18 @@ def test_get_input_fx_filelist(root, cfg):
     # Find files
     rootpath = {cfg['variable']['project']: [root]}
     drs = {cfg['variable']['project']: cfg['drs']}
-    fx_files = get_input_fx_filelist(cfg['variable'], rootpath, drs)
+    cfg['variable']['fx_files'] = [
+        {'short_name': short_name} for short_name
+        in cfg['variable']['fx_files']
+    ]
+    fx_files_dict = get_input_fx_filelist(
+        variable=cfg['variable'],
+        rootpath=rootpath,
+        drs=drs)
 
     # Test result
     reference = {
         fx_var: os.path.join(root, filename) if filename else None
         for fx_var, filename in cfg['found_files'].items()
     }
-    assert fx_files == reference
+    assert fx_files_dict == reference

From 6989951c63f67f9fe66beb7d4f2e475afe91eec1 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 15:33:15 +0100
Subject: [PATCH 09/18] added checks for no time gating

---
 esmvaltool/_recipe_checks.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/esmvaltool/_recipe_checks.py b/esmvaltool/_recipe_checks.py
index a6ab607df0..ee0e569304 100644
--- a/esmvaltool/_recipe_checks.py
+++ b/esmvaltool/_recipe_checks.py
@@ -97,15 +97,18 @@ def data_availability(input_files, var):
 
     required_years = set(range(var['start_year'], var['end_year'] + 1))
     available_years = set()
-    for filename in input_files:
-        start, end = get_start_end_year(filename)
-        available_years.update(range(start, end + 1))
-
-    missing_years = required_years - available_years
-    if missing_years:
-        raise RecipeError(
-            "No input data available for years {} in files {}".format(
-                ", ".join(str(year) for year in missing_years), input_files))
+    # check time avail only for non-fx variables
+    if var['frequency'] != 'fx':
+        for filename in input_files:
+            start, end = get_start_end_year(filename)
+            available_years.update(range(start, end + 1))
+
+        missing_years = required_years - available_years
+        if missing_years:
+            raise RecipeError(
+                "No input data available for years {} in files {}".format(
+                    ", ".join(str(year) for year in missing_years),
+                    input_files))
 
 
 def tasks_valid(tasks):

From 988d09cdba42e75fa62a03fc47baf01d1a64362a Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 16:23:30 +0100
Subject: [PATCH 10/18] fixed the test to reflect the new standards

---
 tests/integration/test_recipe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py
index aeaaedabd5..885eecc5d3 100644
--- a/tests/integration/test_recipe.py
+++ b/tests/integration/test_recipe.py
@@ -100,7 +100,7 @@ def find_files(_, filenames):
         filename = str(tmp_path / 'input' / filename)
         filenames = []
         if filename.endswith('*.nc'):
-            filename = filename[:-len('*.nc')]
+            filename = filename[:-len('*.nc')] + '_'
             intervals = [
                 '1990_1999',
                 '2000_2009',

From 775715c801768aeccf65de296e3fc987cab0bb0e Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 16:23:51 +0100
Subject: [PATCH 11/18] fixed the test to reflect the new standards

---
 tests/integration/preprocessor/_derive/test_interface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/preprocessor/_derive/test_interface.py b/tests/integration/preprocessor/_derive/test_interface.py
index cef8ea93a4..8608cccf18 100644
--- a/tests/integration/preprocessor/_derive/test_interface.py
+++ b/tests/integration/preprocessor/_derive/test_interface.py
@@ -26,7 +26,7 @@ def test_get_required_with_fx():
 
     reference = [{
         'short_name': 'nbp',
-        'fx_files': ['sftlf'],
+        'fx_files': [{'short_name': 'sftlf'}],
     }]
 
     assert variables == reference

From 32b6376fbd1533c7e34f278fc2cc717792908d89 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 16:24:14 +0100
Subject: [PATCH 12/18] fixed the data structure standard to reflect new
 standards

---
 esmvaltool/preprocessor/_derive/nbp_grid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esmvaltool/preprocessor/_derive/nbp_grid.py b/esmvaltool/preprocessor/_derive/nbp_grid.py
index 5a07fcaed7..44047c4f7f 100644
--- a/esmvaltool/preprocessor/_derive/nbp_grid.py
+++ b/esmvaltool/preprocessor/_derive/nbp_grid.py
@@ -12,7 +12,7 @@ class DerivedVariable(DerivedVariableBase):
     # Required variables
     required = [{
         'short_name': 'nbp',
-        'fx_files': ['sftlf'],
+        'fx_files': [{'short_name': 'sftlf'}],
     }]
 
     @staticmethod

From 2ff537d6023c6b1591d73085770f342979fbec27 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 13 May 2019 18:11:03 +0100
Subject: [PATCH 13/18] fixed little bug

---
 esmvaltool/_recipe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/esmvaltool/_recipe.py b/esmvaltool/_recipe.py
index 79524c7970..d0cbc2b195 100644
--- a/esmvaltool/_recipe.py
+++ b/esmvaltool/_recipe.py
@@ -494,7 +494,7 @@ def _get_input_files(variable, config_user):
                 variable['short_name'], variable['dataset'],
                 '\n'.join(input_files))
     if (not config_user.get('skip-nonexistent')
-            or var['dataset'] == variable.get('reference_dataset')):
+            or variable['dataset'] == variable.get('reference_dataset')):
         check.data_availability(input_files, variable)
 
     # Set up provenance tracking

From b36f31223d47a623a090e38bd5070e772ea2f188 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 10 Jun 2019 16:00:44 +0100
Subject: [PATCH 14/18] removed stray import from conflict

---
 tests/integration/test_data_finder.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py
index 4609b87c91..73df055167 100644
--- a/tests/integration/test_data_finder.py
+++ b/tests/integration/test_data_finder.py
@@ -3,7 +3,6 @@
 import shutil
 import tempfile
 
-import esmvaltool._config
 import pytest
 import yaml
 

From cf05187194ac103a5e71eddb6e894fb04afdfac9 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 10 Jun 2019 17:04:26 +0100
Subject: [PATCH 15/18] fixed stray import mistakes introduced by merge

---
 tests/integration/test_data_finder.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py
index 73df055167..71e49ae53d 100644
--- a/tests/integration/test_data_finder.py
+++ b/tests/integration/test_data_finder.py
@@ -7,8 +7,8 @@
 import yaml
 
 import esmvalcore._config
-from esmvalcore._data_finder import (get_input_filelist, get_input_fx_filelist,
-                                     get_output_file)
+from esmvalcore._data_finder import get_input_filelist, get_output_file
+from esmvalcore._recipe import get_input_fx_filelist
 from esmvalcore.cmor.table import read_cmor_tables
 
 # Initialize with standard config developer file

From e799ec83014454d55786fbf4a88eb8f83a5776b8 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Mon, 10 Jun 2019 17:04:38 +0100
Subject: [PATCH 16/18] removed unused import

---
 esmvalcore/preprocessor/_mask.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/esmvalcore/preprocessor/_mask.py b/esmvalcore/preprocessor/_mask.py
index 491d537c56..efba23d2e0 100644
--- a/esmvalcore/preprocessor/_mask.py
+++ b/esmvalcore/preprocessor/_mask.py
@@ -12,7 +12,6 @@
 
 import numpy as np
 
-import dask.array as da
 import cartopy.io.shapereader as shpreader
 import iris
 import shapely.vectorized as shp_vect

From 7ab878fcbbc12145552b2964037d0e87952b04dc Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Fri, 14 Jun 2019 15:06:48 +0100
Subject: [PATCH 17/18] removed unused import

---
 esmvalcore/_data_finder.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 67894a7568..3ea5bcc2ad 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -10,7 +10,6 @@
 import re
 
 from ._config import get_project_config
-from .cmor.table import CMOR_TABLES
 
 logger = logging.getLogger(__name__)
 

From 0fc4392fd5f8660c7f00c9d4044e0dd0aedd1558 Mon Sep 17 00:00:00 2001
From: Valeriu Predoi <valeriu.predoi@gmail.com>
Date: Fri, 14 Jun 2019 16:55:40 +0100
Subject: [PATCH 18/18] removed useless variable

---
 esmvalcore/_data_finder.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py
index 3ea5bcc2ad..0bc9450f70 100644
--- a/esmvalcore/_data_finder.py
+++ b/esmvalcore/_data_finder.py
@@ -198,8 +198,7 @@ def _find_input_dirs(variable, rootpath, drs):
     project = variable['project']
 
     root = get_rootpath(rootpath, project)
-    input_type = 'input_dir'
-    path_template = _select_drs(input_type, drs, project)
+    path_template = _select_drs('input_dir', drs, project)
 
     dirnames = []
     for dirname_template in _replace_tags(path_template, variable):
@@ -217,8 +216,7 @@ def _find_input_dirs(variable, rootpath, drs):
 
 def _get_filenames_glob(variable, drs):
     """Return patterns that can be used to look for input files."""
-    input_type = 'input_file'
-    path_template = _select_drs(input_type, drs, variable['project'])
+    path_template = _select_drs('input_file', drs, variable['project'])
     filenames_glob = _replace_tags(path_template, variable)
     return filenames_glob