Extended fix for cl-like variables of CESM2 models

esmvalcore/cmor/_fixes/cmip6/cesm2.py

@@ -11,11 +11,42 @@
 class Cl(Fix):
     """Fixes for ``cl``."""
 
+    def _fix_formula_terms(self, filepath, output_dir):
+        """Fix ``formula_terms`` attribute."""
+        new_path = self.get_fixed_filepath(output_dir, filepath)
+        copyfile(filepath, new_path)
+        dataset = Dataset(new_path, mode='a')
+        dataset.variables['lev'].formula_terms = 'p0: p0 a: a b: b ps: ps'
+        dataset.variables['lev'].standard_name = (
+            'atmosphere_hybrid_sigma_pressure_coordinate')
+        dataset.close()
+        return new_path
+
+    def fix_data(self, cube):
+        """Fix data.
+
+        Fixed ordering of vertical coordinate.
+
+        Parameters
+        ----------
+        cube: iris.cube.Cube
+            Input cube to fix.
+
+        Returns
+        -------
+        iris.cube.Cube
+
+        """
+        (z_axis,) = cube.coord_dims(cube.coord(axis='Z', dim_coords=True))
+        indices = [slice(None)] * cube.ndim
+        indices[z_axis] = slice(None, None, -1)
+        cube = cube[tuple(indices)]
+        return cube
+
     def fix_file(self, filepath, output_dir):
         """Fix hybrid pressure coordinate.
 
-        Adds missing ``formula_terms`` attribute to file and fix ordering
-        of auxiliary coordinates ``a`` and ``b``.
+        Adds missing ``formula_terms`` attribute to file.
 
         Note
         ----
@@ -37,21 +68,10 @@ def fix_file(self, filepath, output_dir):
             Path to the fixed file.
 
         """
-        new_path = self.get_fixed_filepath(output_dir, filepath)
-        copyfile(filepath, new_path)
+        new_path = self._fix_formula_terms(filepath, output_dir)
         dataset = Dataset(new_path, mode='a')
-
-        # Fix hybrid sigma pressure coordinate
-        dataset.variables['lev'].formula_terms = 'p0: p0 a: a b: b ps: ps'
-        dataset.variables['lev'].standard_name = (
-            'atmosphere_hybrid_sigma_pressure_coordinate')
-        dataset.variables['lev'].units = '1'
-
-        # Fix auxiliary coordinates
-        dataset.variables['a'][:] = dataset.variables['a'][::-1]
-        dataset.variables['b'][:] = dataset.variables['b'][::-1]
-
-        # Save
+        dataset.variables['a_bnds'][:] = dataset.variables['a_bnds'][::-1, :]
+        dataset.variables['b_bnds'][:] = dataset.variables['b_bnds'][::-1, :]
         dataset.close()
         return new_path
 

esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py

@@ -11,8 +11,7 @@ class Cl(BaseCl):
     def fix_file(self, filepath, output_dir):
         """Fix hybrid pressure coordinate.
 
-        Adds missing ``formula_terms`` attribute to file and fix ordering
-        of auxiliary coordinates ``a``, ``b``, ``a_bnds`` and ``b_bnds``.
+        Adds missing ``formula_terms`` attribute to file.
 
         Note
         ----
@@ -34,10 +33,10 @@ def fix_file(self, filepath, output_dir):
             Path to the fixed file.
 
         """
-        new_path = super().fix_file(filepath, output_dir)
+        new_path = self._fix_formula_terms(filepath, output_dir)
         dataset = Dataset(new_path, mode='a')
-        dataset.variables['a_bnds'][:] = dataset.variables['a_bnds'][::-1]
-        dataset.variables['b_bnds'][:] = dataset.variables['b_bnds'][::-1]
+        dataset.variables['a_bnds'][:] = dataset.variables['a_bnds'][:, ::-1]
+        dataset.variables['b_bnds'][:] = dataset.variables['b_bnds'][:, ::-1]
         dataset.close()
         return new_path
 

esmvalcore/cmor/_fixes/shared.py

@@ -9,7 +9,7 @@
 from cf_units import Unit
 from scipy.interpolate import interp1d
 
-from esmvalcore.preprocessor._derive._shared import var_name_constraint
+from esmvalcore.iris_helpers import var_name_constraint
 
 logger = logging.getLogger(__name__)
 

esmvalcore/iris_helpers.py

@@ -0,0 +1,7 @@
+"""Auxiliary functions for :mod:`iris`."""
+import iris
+
+
+def var_name_constraint(var_name):
+    """:mod:`iris.Constraint` using `var_name` of a :mod:`iris.cube.Cube`."""
+    return iris.Constraint(cube_func=lambda c: c.var_name == var_name)

esmvalcore/preprocessor/_derive/_shared.py

@@ -3,16 +3,10 @@
 import logging
 
 import iris
-from iris import Constraint
 
 logger = logging.getLogger(__name__)
 
 
-def var_name_constraint(var_name):
-    """:mod:`iris.Constraint` using `var_name` of a :mod:`iris.cube.Cube`."""
-    return Constraint(cube_func=lambda c: c.var_name == var_name)
-
-
 def cloud_area_fraction(cubes, tau_constraint, plev_constraint):
     """Calculate cloud area fraction for different parameters."""
     clisccp_cube = cubes.extract_strict(

esmvalcore/preprocessor/_derive/alb.py

@@ -5,8 +5,9 @@
 
 """
 
+from esmvalcore.iris_helpers import var_name_constraint
+
 from ._baseclass import DerivedVariableBase
-from ._shared import var_name_constraint
 
 
 class DerivedVariable(DerivedVariableBase):

esmvalcore/preprocessor/_derive/lwp.py

@@ -2,8 +2,9 @@
 
 import logging
 
+from esmvalcore.iris_helpers import var_name_constraint
+
 from ._baseclass import DerivedVariableBase
-from ._shared import var_name_constraint
 
 logger = logging.getLogger(__name__)
 

tests/integration/cmor/_fixes/cmip6/test_cesm2.py

@@ -35,7 +35,7 @@ def cl_file(tmp_path):
     dataset.variables['time'].units = 'days since 6543-2-1'
     dataset.variables['lev'][:] = [1.0, 2.0]
     dataset.variables['lev'].bounds = 'lev_bnds'
-    dataset.variables['lev'].units = '1'
+    dataset.variables['lev'].units = 'hPa'
     dataset.variables['lev_bnds'][:] = [[0.5, 1.5], [1.5, 3.0]]
     dataset.variables['lev_bnds'].standard_name = (
         'atmosphere_hybrid_sigma_pressure_coordinate')
@@ -57,12 +57,12 @@ def cl_file(tmp_path):
     dataset.createVariable('p0', np.float64, dimensions=())
     dataset.createVariable('ps', np.float64,
                            dimensions=('time', 'lat', 'lon'))
-    dataset.variables['a'][:] = [2.0, 1.0]  # Wrong order intended
+    dataset.variables['a'][:] = [1.0, 2.0]
     dataset.variables['a'].bounds = 'a_bnds'
-    dataset.variables['a_bnds'][:] = [[0.0, 1.5], [1.5, 3.0]]
-    dataset.variables['b'][:] = [1.0, 0.0]  # Wrong order intended
+    dataset.variables['a_bnds'][:] = [[1.5, 3.0], [0.0, 1.5]]  # intended
+    dataset.variables['b'][:] = [0.0, 1.0]
     dataset.variables['b'].bounds = 'b_bnds'
-    dataset.variables['b_bnds'][:] = [[-1.0, 0.5], [0.5, 2.0]]
+    dataset.variables['b_bnds'][:] = [[0.5, 2.0], [-1.0, 0.5]]  # intended
     dataset.variables['p0'][:] = 1.0
     dataset.variables['p0'].units = 'Pa'
     dataset.variables['ps'][:] = np.arange(1 * 3 * 4).reshape(1, 3, 4)
@@ -136,8 +136,8 @@ def test_cl_fix_file(mock_get_filepath, cl_file, tmp_path):
     assert 'ps' in var_names
 
     # Raw cl cube
-    cl_cube = cubes.extract_strict('cloud_area_fraction_in_atmosphere_layer')
-    assert not cl_cube.coords('air_pressure')
+    raw_cube = cubes.extract_strict('cloud_area_fraction_in_atmosphere_layer')
+    assert not raw_cube.coords('air_pressure')
 
     # Apply fix
     mock_get_filepath.return_value = os.path.join(tmp_path,
@@ -161,6 +161,58 @@ def test_cl_fix_file(mock_get_filepath, cl_file, tmp_path):
                                AIR_PRESSURE_BOUNDS)
 
 
+@pytest.fixture
+def cl_cube():
+    """``cl`` cube."""
+    time_coord = iris.coords.DimCoord(
+        [0.0, 1.0], var_name='time', standard_name='time',
+        units='days since 1850-01-01 00:00:00')
+    lev_coord = iris.coords.DimCoord(
+        [0.0, 1.0, 2.0], var_name='lev',
+        standard_name='atmosphere_hybrid_sigma_pressure_coordinate', units='1',
+        attributes={'positive': 'up'})
+    lat_coord = iris.coords.DimCoord(
+        [0.0, 1.0], var_name='lat', standard_name='latitude', units='degrees')
+    lon_coord = iris.coords.DimCoord(
+        [0.0, 1.0], var_name='lon', standard_name='longitude', units='degrees')
+    coord_specs = [
+        (time_coord, 0),
+        (lev_coord, 1),
+        (lat_coord, 2),
+        (lon_coord, 3),
+    ]
+    cube = iris.cube.Cube(
+        np.arange(2 * 3 * 2 * 2).reshape(2, 3, 2, 2),
+        var_name='cl',
+        standard_name='cloud_area_fraction_in_atmosphere_layer',
+        units='%',
+        dim_coords_and_dims=coord_specs,
+    )
+    return cube
+
+
+def test_cl_fix_data(cl_cube):
+    """Test ``fix_data`` for ``cl``."""
+    fix = Cl(None)
+    out_cube = fix.fix_data(cl_cube)
+    assert out_cube.shape == cl_cube.shape
+    np.testing.assert_allclose(out_cube.data,
+                               [[[[8, 9],
+                                  [10, 11]],
+                                 [[4, 5],
+                                  [6, 7]],
+                                 [[0, 1],
+                                  [2, 3]]],
+                                [[[20, 21],
+                                  [22, 23]],
+                                 [[16, 17],
+                                  [18, 19]],
+                                 [[12, 13],
+                                  [14, 15]]]])
+    np.testing.assert_allclose(out_cube.coord(var_name='lev').points,
+                               [2.0, 1.0, 0.0])
+
+
 def test_get_cli_fix():
     """Test getting of fix."""
     fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'cli')

tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py

@@ -38,12 +38,12 @@ def cl_file(tmp_path):
     dataset.createVariable('a_bnds', np.float64, dimensions=('lev', 'bnds'))
     dataset.createVariable('b', np.float64, dimensions=('lev',))
     dataset.createVariable('b_bnds', np.float64, dimensions=('lev', 'bnds'))
-    dataset.variables['a'][:] = [2.0, 1.0]
+    dataset.variables['a'][:] = [1.0, 2.0]
     dataset.variables['a'].bounds = 'a_bnds'
-    dataset.variables['a_bnds'][:] = [[1.5, 3.0], [0.0, 1.5]]
-    dataset.variables['b'][:] = [1.0, 0.0]
+    dataset.variables['a_bnds'][:] = [[1.5, 0.0], [3.0, 1.5]]
+    dataset.variables['b'][:] = [0.0, 1.0]
     dataset.variables['b'].bounds = 'b_bnds'
-    dataset.variables['b_bnds'][:] = [[0.5, 2.0], [-1.0, 0.5]]
+    dataset.variables['b_bnds'][:] = [[0.5, -1.0], [2.0, 0.5]]
 
     dataset.close()
     return nc_path

tests/integration/cmor/_fixes/test_common.py

@@ -9,7 +9,7 @@
 from esmvalcore.cmor._fixes.common import (ClFixHybridHeightCoord,
                                            ClFixHybridPressureCoord)
 from esmvalcore.cmor.table import get_var_info
-from esmvalcore.preprocessor._derive._shared import var_name_constraint
+from esmvalcore.iris_helpers import var_name_constraint
 
 
 def create_hybrid_pressure_file_without_ap(dataset, short_name):

tests/integration/cmor/_fixes/test_shared.py

@@ -4,18 +4,18 @@
 import pytest
 from cf_units import Unit
 
-from esmvalcore.cmor._fixes.shared import (altitude_to_pressure,
-                                           _get_altitude_to_pressure_func,
+from esmvalcore.cmor._fixes.shared import (_get_altitude_to_pressure_func,
                                            add_aux_coords_from_cubes,
                                            add_plev_from_altitude,
                                            add_scalar_depth_coord,
                                            add_scalar_height_coord,
                                            add_scalar_typeland_coord,
                                            add_scalar_typesea_coord,
                                            add_sigma_factory,
+                                           altitude_to_pressure,
                                            cube_to_aux_coord, fix_bounds,
                                            get_bounds_cube, round_coordinates)
-from esmvalcore.preprocessor._derive._shared import var_name_constraint
+from esmvalcore.iris_helpers import var_name_constraint
 
 
 @pytest.mark.parametrize('func', [altitude_to_pressure,

tests/unit/test_iris_helpers.py

@@ -0,0 +1,37 @@
+"""Tests for :mod:`esmvalcore.iris_helpers`."""
+import iris
+import pytest
+
+from esmvalcore.iris_helpers import var_name_constraint
+
+
+@pytest.fixture
+def cubes():
+    """Test cubes."""
+    cubes = iris.cube.CubeList([
+        iris.cube.Cube(0.0, var_name='a', long_name='a'),
+        iris.cube.Cube(0.0, var_name='a', long_name='b'),
+        iris.cube.Cube(0.0, var_name='c', long_name='d'),
+    ])
+    return cubes
+
+
+def test_var_name_constraint(cubes):
+    """Test :func:`esmvalcore.iris_helpers.var_name_constraint`."""
+    out_cubes = cubes.extract(var_name_constraint('a'))
+    assert out_cubes == iris.cube.CubeList([
+        iris.cube.Cube(0.0, var_name='a', long_name='a'),
+        iris.cube.Cube(0.0, var_name='a', long_name='b'),
+    ])
+    out_cubes = cubes.extract(var_name_constraint('b'))
+    assert out_cubes == iris.cube.CubeList([])
+    out_cubes = cubes.extract(var_name_constraint('c'))
+    assert out_cubes == iris.cube.CubeList([
+        iris.cube.Cube(0.0, var_name='c', long_name='d'),
+    ])
+    with pytest.raises(iris.exceptions.ConstraintMismatchError):
+        cubes.extract_strict(var_name_constraint('a'))
+    with pytest.raises(iris.exceptions.ConstraintMismatchError):
+        cubes.extract_strict(var_name_constraint('b'))
+    out_cube = cubes.extract_strict(var_name_constraint('c'))
+    assert out_cube == iris.cube.Cube(0.0, var_name='c', long_name='d')