Impro-1121: Rename probability cubes - 1 (#813)

* initial changes for probability_of.... names in wxcodes and tests

* further changes to wxcodes stuff, threshold.py and unit tests for both

* separate naming of cubes now required for above/below in threshold.py

* applied naming changes to set_up_test_cubes.py and associated unit test

* formatting fixes/reverts

* formatting fixes

* fix incorrect renaming of cloud diagnostic sfc-1000ft

* correction to wxcode bats test, further minor updates

* update diagnostic probability names

* update lightning nowcast files

* fix missed update for cloud diagnostic name

* update wxcode bats tests to work with new inpute file/cube probability names

* some formatting fixes

* changes made in response to reviewers comments

* remove uneccessary np.flipud from unit test

* doc changes in response to reviewers comments

Author: markysparks

bin/improver-wxcode

@@ -92,9 +92,8 @@ def interrogate_decision_tree(wxtree):
                                   return_index=True)
         output.append('{}; thresholds: {}'.format(
             requirement, ', '.join([
-                '{} {} ({})'.format(str(relation),
-                                    str(threshold.points.item()),
-                                    str(threshold.units))
+                '{} ({})'.format(str(threshold.points.item()),
+                                 str(threshold.units))
                 for threshold, relation in
                 zip(entries[thresholds], relations[thresholds])])))
     return output

lib/improver/nowcasting/lightning.py

@@ -186,7 +186,7 @@ def _update_metadata(cube):
         Modify the meta data of input cube to resemble a Nowcast of lightning
         probability.
 
-        1. Rename to "probability_of_lightning"
+        1. Rename to "probability_of_lightning_rate_above_threshold"
 
         2. Remove "threshold" coord
         (or causes iris.exceptions.CoordinateNotFoundError)
@@ -204,7 +204,7 @@ def _update_metadata(cube):
                 The data array will be a copy of the input cube.data
         """
         new_cube = cube.copy()
-        new_cube.rename("probability_of_lightning")
+        new_cube.rename("probability_of_lightning_rate_above_threshold")
         new_cube.remove_coord('threshold')
         new_cube.cell_methods = None
         return new_cube
@@ -465,16 +465,16 @@ def process(self, cubelist):
                 If cubelist does not contain the expected cubes.
         """
         first_guess_lightning_cube = cubelist.extract(
-            "probability_of_lightning", strict=True)
+            "probability_of_lightning_rate_above_threshold", strict=True)
         lightning_rate_cube = cubelist.extract(
             "rate_of_lightning", strict=True)
         lightning_rate_cube.convert_units("min^-1")  # Ensure units are correct
         prob_precip_cube = cubelist.extract(
-            "probability_of_precipitation", strict=True)
+            "probability_of_precipitation_rate_above_threshold", strict=True)
         # Now find prob_vii_cube. Can't use strict=True here as cube may not be
         # present, so will use a normal extract and then merge_cube if needed.
         prob_vii_cube = cubelist.extract(
-            "probability_of_vertical_integral_of_ice")
+            "probability_of_vertical_integral_of_ice_above_threshold")
         if prob_vii_cube:
             prob_vii_cube = prob_vii_cube.merge_cube()
         prob_precip_cube.coord('threshold').convert_units('mm hr-1')

lib/improver/tests/blending/weighted_blend/test_WeightedBlendAcrossWholeDimension.py

@@ -183,8 +183,9 @@ def setUp(self):
         data_threshold[1, 1, :, :] = 0.6
         data_threshold[1, 2, :, :] = 0.8
 
-        cube_threshold = Cube(data_threshold,
-                              long_name="probability_of_precipitation_amount")
+        cube_threshold = Cube(
+            data_threshold,
+            long_name="probability_of_precipitation_amount_below_threshold")
         cube_threshold.add_dim_coord(threshold, 0)
         cube_threshold.add_dim_coord(frt_coord, 1)
         cube_threshold.add_dim_coord(lat_coord, 2)

lib/improver/tests/blending/weighted_blend/test_rationalise_blend_time_coords.py

@@ -54,16 +54,16 @@ def setUp(self):
         # coming from the UKV
         data = np.full((3, 3), 0.6, dtype=np.float32)
         self.ukv_cube = set_up_variable_cube(
-            data, name='probability_of_air_temperature', units='1',
-            time=datetime(2017, 1, 10, 3, 0), frt=datetime(2017, 1, 9, 23, 0),
-            standard_grid_metadata='uk_det')
+            data, name='probability_of_air_temperature_above_threshold',
+            units='1', time=datetime(2017, 1, 10, 3, 0),
+            frt=datetime(2017, 1, 9, 23, 0), standard_grid_metadata='uk_det')
 
         # make a cube labelled as coming from MOGREPS-UK, with a different
         # forecast reference time from the UKV cube
         self.enuk_cube = set_up_variable_cube(
-            data, name='probability_of_air_temperature', units='1',
-            time=datetime(2017, 1, 10, 3, 0), frt=datetime(2017, 1, 10, 0, 0),
-            standard_grid_metadata='uk_ens')
+            data, name='probability_of_air_temperature_above_threshold',
+            units='1', time=datetime(2017, 1, 10, 3, 0),
+            frt=datetime(2017, 1, 10, 0, 0), standard_grid_metadata='uk_ens')
 
         # make a cube list and merged cube containing the two model cubes, for
         # use in defining reference coordinates for tests below

lib/improver/tests/nowcasting/lightning/test_NowcastLightning.py

@@ -115,7 +115,8 @@ def test_basic(self):
         cell methods to be discarded."""
         result = self.plugin._update_metadata(self.cube)
         self.assertIsInstance(result, Cube)
-        self.assertEqual(result.name(), "probability_of_lightning")
+        self.assertEqual(
+            result.name(), "probability_of_lightning_rate_above_threshold")
         msg = ("Expected to find exactly 1 threshold coordinate, but found "
                "none.")
         with self.assertRaisesRegex(CoordinateNotFoundError, msg):
@@ -427,21 +428,21 @@ def setUp(self):
         self.fg_cube:
             forecast_period (on time coord): 0.0 hours (simulates nowcast data)
             All points contain float(1.)
-            Cube name is "probability_of_lightning".
+            Cube name is "probability_of_lightning_rate_above_threshold".
         self.precip_cube:
             With extra coordinate of length(3) "threshold" containing
             points [0.5, 7., 35.] mm hr-1.
             All points contain float(1.) except the
             zero point [0, 0, 1, 1] which is float(0.)
             and [1:, 0, ...] which are float(0.)
-            Cube name is "probability_of_precipitation".
+            Cube name is "probability_of_precipitation_rate_above_threshold".
             Cube has added attribute {'relative_to_threshold': 'above'}
         """
         self.fg_cube = add_forecast_reference_time_and_forecast_period(
             set_up_cube_with_no_realizations(zero_point_indices=[],
                                              num_grid_points=3),
             fp_point=0.0)
-        self.fg_cube.rename("probability_of_lightning")
+        self.fg_cube.rename("probability_of_lightning_rate_above_threshold")
         self.precip_cube = (
             add_forecast_reference_time_and_forecast_period(
                 set_up_cube(num_realization_points=3,
@@ -451,7 +452,8 @@ def setUp(self):
         threshold_coord.points = [0.5, 7.0, 35.0]
         threshold_coord.rename('threshold')
         threshold_coord.units = cf_units.Unit('mm hr-1')
-        self.precip_cube.rename("probability_of_precipitation")
+        self.precip_cube.rename(
+            "probability_of_precipitation_rate_above_threshold")
         self.precip_cube.attributes.update({'relative_to_threshold': 'above'})
         self.precip_cube.data[1:, 0, ...] = 0.
         self.plugin = Plugin()
@@ -598,19 +600,20 @@ def setUp(self):
         Data:
         self.fg_cube:
             All points contain float(1.)
-            Cube name is "probability_of_lightning".
+            Cube name is "probability_of_lightning_rate_above_threshold".
         self.ice_cube:
             With extra coordinate of length(3) "threshold" containing
             points [0.5, 1., 2.] kg m^-2.
             Time and forecast_period dimensions "sqeezed" to be Scalar coords.
             All points contain float(0.)
-            Cube name is "probability_of_vertical_integral_of_ice".
+            Cube name is
+            "probability_of_vertical_integral_of_ice_above_threshold".
         """
         self.fg_cube = add_forecast_reference_time_and_forecast_period(
             set_up_cube_with_no_realizations(zero_point_indices=[],
                                              num_grid_points=3),
             fp_point=0.0)
-        self.fg_cube.rename("probability_of_lightning")
+        self.fg_cube.rename("probability_of_lightning_rate_above_threshold")
         self.ice_cube = squeeze(
             add_forecast_reference_time_and_forecast_period(
                 set_up_cube(num_realization_points=3,
@@ -622,7 +625,8 @@ def setUp(self):
         threshold_coord.rename('threshold')
         threshold_coord.units = cf_units.Unit('kg m^-2')
         self.ice_cube.data = np.zeros_like(self.ice_cube.data)
-        self.ice_cube.rename("probability_of_vertical_integral_of_ice")
+        self.ice_cube.rename(
+            "probability_of_vertical_integral_of_ice_above_threshold")
         self.plugin = Plugin()
 
     def test_basic(self):
@@ -748,7 +752,7 @@ def setUp(self):
         Data:
         self.fg_cube:
             All points contain float(1.)
-            Cube name is "probability_of_lightning".
+            Cube name is "probability_of_lightning_rate_above_threshold".
         self.ltng_cube:
             forecast_period (on time coord): 0.0 hours (simulates nowcast data)
             All points contain float(1.)
@@ -760,7 +764,7 @@ def setUp(self):
             All points contain float(1.) except the
             zero point [0, 0, 7, 7] which is float(0.)
             and [1:, 0, ...] which are float(0.)
-            Cube name is "probability_of_precipitation".
+            Cube name is "probability_of_precipitation_rate_above_threshold".
             Cube has added attribute {'relative_to_threshold': 'above'}
         self.vii_cube:
             forecast_period (on time coord): 0.0 hours (simulates nowcast data)
@@ -769,11 +773,12 @@ def setUp(self):
             forecast_period (on time coord): 0.0 hours (simulates nowcast data)
             Time and forecast_period dimensions "sqeezed" to be Scalar coords.
             All points contain float(0.)
-            Cube name is "probability_of_vertical_integral_of_ice".
+            Cube name is
+            "probability_of_vertical_integral_of_ice_above_threshold".
         """
         self.fg_cube = add_forecast_reference_time_and_forecast_period(
             set_up_cube_with_no_realizations(zero_point_indices=[]))
-        self.fg_cube.rename("probability_of_lightning")
+        self.fg_cube.rename("probability_of_lightning_rate_above_threshold")
         self.ltng_cube = add_forecast_reference_time_and_forecast_period(
             set_up_cube_with_no_realizations(zero_point_indices=[]),
             fp_point=0.0)
@@ -786,7 +791,8 @@ def setUp(self):
         threshold_coord.points = [0.5, 7.0, 35.0]
         threshold_coord.rename('threshold')
         threshold_coord.units = cf_units.Unit('mm hr-1')
-        self.precip_cube.rename("probability_of_precipitation")
+        self.precip_cube.rename(
+            "probability_of_precipitation_rate_above_threshold")
         self.precip_cube.attributes.update({'relative_to_threshold': 'above'})
         self.precip_cube.data[1:, 0, ...] = 0.
         self.vii_cube = squeeze(
@@ -799,7 +805,8 @@ def setUp(self):
         threshold_coord.rename('threshold')
         threshold_coord.units = cf_units.Unit('kg m^-2')
         self.vii_cube.data = np.zeros_like(self.vii_cube.data)
-        self.vii_cube.rename("probability_of_vertical_integral_of_ice")
+        self.vii_cube.rename(
+            "probability_of_vertical_integral_of_ice_above_threshold")
         self.plugin = Plugin()
 
     def set_up_vii_input_output(self):
@@ -860,7 +867,8 @@ def test_basic(self):
         # We expect the threshold coordinate to have been removed.
         self.assertCountEqual(find_dimension_coordinate_mismatch(
                 result, self.precip_cube), ['threshold'])
-        self.assertTrue(result.name() == 'probability_of_lightning')
+        self.assertTrue(
+            result.name() == 'probability_of_lightning_rate_above_threshold')
 
     def test_basic_with_vii(self):
         """Test that the method returns the expected cube type when vii is
@@ -874,13 +882,14 @@ def test_basic_with_vii(self):
         # We expect the threshold coordinate to have been removed.
         self.assertCountEqual(find_dimension_coordinate_mismatch(
                 result, self.precip_cube), ['threshold'])
-        self.assertTrue(result.name() == 'probability_of_lightning')
+        self.assertTrue(
+            result.name() == 'probability_of_lightning_rate_above_threshold')
 
     def test_no_first_guess_cube(self):
         """Test that the method raises an error if the first_guess cube is
         omitted from the cubelist"""
         msg = (r"Got 0 cubes for constraint Constraint\(name=\'probability_of_"
-               r"lightning\'\), expecting 1.")
+               r"lightning_rate_above_threshold\'\), expecting 1.")
         with self.assertRaisesRegex(ConstraintMismatchError, msg):
             self.plugin.process(CubeList([
                 self.ltng_cube,
@@ -900,7 +909,7 @@ def test_no_precip_cube(self):
         """Test that the method raises an error if the precip cube is
         omitted from the cubelist"""
         msg = (r"Got 0 cubes for constraint Constraint\(name=\'probability_of_"
-               r"precipitation\'\), expecting 1.")
+               r"precipitation_rate_above_threshold\'\), expecting 1.")
         with self.assertRaisesRegex(ConstraintMismatchError, msg):
             self.plugin.process(CubeList([
                 self.fg_cube,

lib/improver/tests/set_up_test_cubes.py

@@ -323,7 +323,8 @@ def set_up_probability_cube(data, thresholds, variable_name='air_temperature',
     - option to specify additional scalar coordinates
     - "relative_to_threshold" attribute (default "above")
     - default or configurable attributes
-    - configurable cube data, name conforms to "probability_of_X" convention
+    - configurable cube data, name conforms to
+    "probability_of_X_above(or below)_threshold" convention
 
     Args:
         data (np.ndarray):
@@ -366,9 +367,19 @@ def set_up_probability_cube(data, thresholds, variable_name='air_temperature',
         attributes['relative_to_threshold'] = relative_to_threshold
 
     if variable_name.startswith('probability_of_'):
-        name = variable_name
+        name_prefix = ''
     else:
-        name = 'probability_of_{}'.format(variable_name)
+        name_prefix = 'probability_of_'
+
+    if relative_to_threshold == 'above':
+        name = name_prefix + '{}_above_threshold'.format(variable_name)
+    elif relative_to_threshold == 'below':
+        name = name_prefix + '{}_below_threshold'.format(variable_name)
+    else:
+        msg = 'The relative_to_threshold attribute MUST be set for ' \
+              'IMPROVER probability cubes'
+        raise ValueError(msg)
+
     cube = set_up_variable_cube(
         data, name=name, units='1', spatial_grid=spatial_grid,
         time=time, frt=frt, time_bounds=time_bounds,

lib/improver/tests/test_set_up_test_cubes.py

@@ -337,7 +337,8 @@ def test_defaults(self):
         and metadata"""
         result = set_up_probability_cube(self.data, self.thresholds)
         thresh_coord = result.coord("threshold")
-        self.assertEqual(result.name(), 'probability_of_air_temperature')
+        self.assertEqual(
+            result.name(), 'probability_of_air_temperature_above_threshold')
         self.assertEqual(result.units, '1')
         self.assertArrayEqual(thresh_coord.points, self.thresholds)
         self.assertEqual(thresh_coord.units, 'K')
@@ -349,7 +350,8 @@ def test_probability_of_name(self):
         result = set_up_probability_cube(
             self.data, self.thresholds,
             variable_name='probability_of_air_temperature')
-        self.assertEqual(result.name(), 'probability_of_air_temperature')
+        self.assertEqual(
+            result.name(), 'probability_of_air_temperature_above_threshold')
 
     def test_relative_to_threshold(self):
         """Test ability to reset the "relative_to_threshold" attribute"""
@@ -359,6 +361,14 @@ def test_relative_to_threshold(self):
         self.assertEqual(len(result.attributes), 1)
         self.assertEqual(result.attributes['relative_to_threshold'], 'below')
 
+    def test_relative_to_threshold_set(self):
+        """Test that an error is raised if the "relative_to_threshold"
+        attribute has not been set when setting up a probability cube"""
+        msg = 'The relative_to_threshold attribute MUST be set'
+        with self.assertRaisesRegex(ValueError, msg):
+            set_up_probability_cube(self.data, self.thresholds,
+                                    relative_to_threshold=None)
+
     def test_standard_grid_metadata(self):
         """Test standard grid metadata"""
         result = set_up_probability_cube(self.data, self.thresholds,

lib/improver/tests/threshold/test_BasicThreshold.py

@@ -190,7 +190,7 @@ def test_metadata_changes(self):
         plugin = Threshold(0.1)
         result = plugin.process(cube)
         # The single 0.5-valued point => 1.0, so cheat by * 2.0 vs orig data.
-        name = "probability_of_{}"
+        name = "probability_of_{}_above_threshold"
         expected_name = name.format(self.cube.name())
         expected_attribute = "above"
         expected_units = 1

lib/improver/tests/utilities/test_cube_extraction.py

@@ -58,15 +58,16 @@ def set_up_precip_probability_cube():
                       [0.02, 0.02, 0.00],
                       [0.01, 0.00, 0.00]]])
 
-    MMH_TO_MS = 0.001/3600.
-    threshold = DimCoord(MMH_TO_MS*np.array([0.03, 0.1, 1.0]),
+    MMH_TO_MS = 0.001 / 3600.
+    threshold = DimCoord(MMH_TO_MS * np.array([0.03, 0.1, 1.0]),
                          long_name="threshold", units="m s-1")
     ycoord = DimCoord(np.arange(3), "projection_y_coordinate")
     xcoord = DimCoord(np.arange(3), "projection_x_coordinate")
 
-    cube = iris.cube.Cube(data, long_name="probability_of_precipitation",
-                          dim_coords_and_dims=[(threshold, 0), (ycoord, 1),
-                                               (xcoord, 2)], units="1")
+    cube = iris.cube.Cube(
+        data, long_name="probability_of_precipitation_rate_above_threshold",
+        dim_coords_and_dims=[(threshold, 0), (ycoord, 1),
+                             (xcoord, 2)], units="1")
     return cube
 
 
@@ -189,7 +190,8 @@ def setUp(self):
 
     def test_basic_no_units(self):
         """ Test cube extraction for single constraint without units """
-        constraint_dict = {"name": "probability_of_precipitation"}
+        constraint_dict = {
+            "name": "probability_of_precipitation_rate_above_threshold"}
         constr = iris.Constraint(**constraint_dict)
         cube = apply_extraction(self.precip_cube, constr)
         self.assertIsInstance(cube, iris.cube.Cube)
@@ -208,8 +210,9 @@ def test_basic_with_units(self):
 
     def test_multiple_constraints_with_units(self):
         """ Test behaviour with a list of constraints and units """
-        constraint_dict = {"name": "probability_of_precipitation",
-                           "threshold": 0.03}
+        constraint_dict = {
+            "name": "probability_of_precipitation_rate_above_threshold",
+                    "threshold": 0.03}
         constr = iris.Constraint(**constraint_dict)
         cube = apply_extraction(self.precip_cube, constr, self.units_dict)
         self.assertIsInstance(cube, iris.cube.Cube)
@@ -219,7 +222,8 @@ def test_multiple_constraints_with_units(self):
     def test_error_non_coord_units(self):
         """ Test error raised if units are provided for a non-coordinate
         constraint """
-        constraint_dict = {"name": "probability_of_precipitation"}
+        constraint_dict = {
+            "name": "probability_of_precipitation_rate_above_threshold"}
         units_dict = {"name": "1"}
         with self.assertRaises(CoordinateNotFoundError):
             apply_extraction(self.precip_cube, constraint_dict, units_dict)
@@ -228,8 +232,9 @@ def test_return_none(self):
         """ Test function returns None rather than raising an error where
         no subcubes match the required constraints, when unit conversion is
         required """
-        constraint_dict = {"name": "probability_of_precipitation",
-                           "threshold": 5}
+        constraint_dict = {
+            "name": "probability_of_precipitation_rate_above_threshold",
+                    "threshold": 5}
         constr = iris.Constraint(**constraint_dict)
         cube = apply_extraction(self.precip_cube, constr, self.units_dict)
         self.assertFalse(cube)