From 977290d7a34d51851d5c608156d512a1a6c860ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:00:13 +0100 Subject: [PATCH 01/22] Check horizontal grid before regridding Check if horizontal grid of cube is same as of target grid before regridding. Exits on first difference. Note: No warning or feedback, if regridding is skipped as there is no logging functionality implemented in file yet. --- esmvalcore/preprocessor/_regrid.py | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 2ac3b2580e..4cc30e9380 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -251,6 +251,10 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): if coords: [coord] = coords cube.remove_coord(coord) + + # Return cube if hotizontal grid is the same. + if _check_horizontal_grid_closeness(cube, target_grid): + return cube # Perform the horizontal regridding. if _attempt_irregular_regridding(cube, scheme): @@ -260,6 +264,50 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): return cube +def _check_horizontal_grid_closeness(cube1, cube2): + """ + Check if two cubes have the same horizontal grid definition. + + The result of the function is a boolean answer, if both cubes have the + same horizontal grid definition. The function checks both longitude and + latitude, based on extent and resolution. + + Parameters + ---------- + cube1 : cube + The first of the cubes to be checked. + cube2 : cube + The second of the cubes to be checked. + + Returns + ------- + bool + + .. note:: + + The current implementation checks if the bounds and the + grid shapes are the same. + Exits on first difference. + """ + # Go through the 2 expected horizontal coordinates longitude and latitude. + for coord in ['latitude', 'longitude']: + # Compare shapes. + if cube1.coord(coord).shape != cube2.coord(coord).shape: + # Different shapes detected. + return False + else: + # Iterate through the combined bounds. + for c1bndval, c2bndval in np.nditer( + [cube1.coord(coord).bounds, + cube2.coord(coord).bounds] + ): + # Check if bound value is same or similar. + if (c1bndval != c2bndval + and not(np.allclose(c1bndval, c2bndval))): + # Different bounds values detected. + return False + # Returns default value. + return True def _create_cube(src_cube, data, src_levels, levels, ): """ From 031fc6a07aa86330d281e641e3809e524c2cbbfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:09:35 +0100 Subject: [PATCH 02/22] Update _regrid.py --- esmvalcore/preprocessor/_regrid.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 4cc30e9380..28ac444975 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -251,7 +251,7 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): if coords: [coord] = coords cube.remove_coord(coord) - + # Return cube if hotizontal grid is the same. if _check_horizontal_grid_closeness(cube, target_grid): return cube @@ -264,11 +264,12 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): return cube + def _check_horizontal_grid_closeness(cube1, cube2): """ Check if two cubes have the same horizontal grid definition. - The result of the function is a boolean answer, if both cubes have the + The result of the function is a boolean answer, if both cubes have the same horizontal grid definition. The function checks both longitude and latitude, based on extent and resolution. @@ -285,7 +286,7 @@ def _check_horizontal_grid_closeness(cube1, cube2): .. note:: - The current implementation checks if the bounds and the + The current implementation checks if the bounds and the grid shapes are the same. Exits on first difference. """ @@ -309,6 +310,7 @@ def _check_horizontal_grid_closeness(cube1, cube2): # Returns default value. return True + def _create_cube(src_cube, data, src_levels, levels, ): """ Generate a new cube with the interpolated data. From 2d8166911b479a85fb32c4bc4264e433802b34db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:16:45 +0100 Subject: [PATCH 03/22] Formatting issues --- esmvalcore/preprocessor/_regrid.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 28ac444975..90a185ab6c 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -253,7 +253,7 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): cube.remove_coord(coord) # Return cube if hotizontal grid is the same. - if _check_horizontal_grid_closeness(cube, target_grid): + if _check_horiz_grid_closeness(cube, target_grid): return cube # Perform the horizontal regridding. @@ -265,7 +265,7 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): return cube -def _check_horizontal_grid_closeness(cube1, cube2): +def _check_horiz_grid_closeness(cube1, cube2): """ Check if two cubes have the same horizontal grid definition. @@ -279,13 +279,13 @@ def _check_horizontal_grid_closeness(cube1, cube2): The first of the cubes to be checked. cube2 : cube The second of the cubes to be checked. - + Returns ------- bool .. note:: - + The current implementation checks if the bounds and the grid shapes are the same. Exits on first difference. @@ -301,10 +301,10 @@ def _check_horizontal_grid_closeness(cube1, cube2): for c1bndval, c2bndval in np.nditer( [cube1.coord(coord).bounds, cube2.coord(coord).bounds] - ): + ): # Check if bound value is same or similar. if (c1bndval != c2bndval - and not(np.allclose(c1bndval, c2bndval))): + and not np.allclose(c1bndval, c2bndval)): # Different bounds values detected. return False # Returns default value. From 4a0d217110d85139a6b14cd397fe282bf5bf45c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:21:19 +0100 Subject: [PATCH 04/22] Whitespace issue --- esmvalcore/preprocessor/_regrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 90a185ab6c..a9f788281b 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -285,7 +285,7 @@ def _check_horiz_grid_closeness(cube1, cube2): bool .. note:: - + The current implementation checks if the bounds and the grid shapes are the same. Exits on first difference. From c8c4a98de9f2d716df57af629f8c7143528d6061 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:27:41 +0100 Subject: [PATCH 05/22] Update horizontal regridding information --- doc/esmvalcore/preprocessor.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/esmvalcore/preprocessor.rst b/doc/esmvalcore/preprocessor.rst index 20096dfe2d..9f24fad248 100644 --- a/doc/esmvalcore/preprocessor.rst +++ b/doc/esmvalcore/preprocessor.rst @@ -44,7 +44,7 @@ Overview * ease of maintenance (including unit and integration testing) of existing routines; * a straightforward manner of importing and using the preprocessing routines as - part of the overall usage of the software and, as a special case, the use + part of the overall usage of the software and, as a special case, the useFH during diagnostic execution; * shifting the effort for the scientific diagnostic developer from implementing both standard and diagnostic-specific functionalities to allowing them to @@ -513,7 +513,8 @@ conceptually a very similar process to interpolation (in fact, the regridder engine uses interpolation and extrapolation, with various schemes). The primary difference is that interpolation is based on sample data points, while regridding is based on the horizontal grid of another cube (the reference -grid). +grid). If the horizontal grids of a cube and its reference grid are sufficiently +the same, regridding is automatically and silently skipped for performance reasons. The underlying regridding mechanism in ESMValTool uses the `cube.regrid() `_ From ab63b0f345d047c84ecd7e4090ba013370270f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:29:44 +0100 Subject: [PATCH 06/22] More whitespace issues --- esmvalcore/preprocessor/_regrid.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index a9f788281b..6ed87b13c7 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -268,22 +268,22 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): def _check_horiz_grid_closeness(cube1, cube2): """ Check if two cubes have the same horizontal grid definition. - + The result of the function is a boolean answer, if both cubes have the same horizontal grid definition. The function checks both longitude and latitude, based on extent and resolution. - + Parameters ---------- cube1 : cube The first of the cubes to be checked. cube2 : cube The second of the cubes to be checked. - + Returns ------- bool - + .. note:: The current implementation checks if the bounds and the From 77a255600752c45ecff26e9a9c43b78c5b93bfad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 21 Feb 2020 09:44:44 +0100 Subject: [PATCH 07/22] No extra return --- esmvalcore/preprocessor/_regrid.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 6ed87b13c7..3b553e3782 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -252,15 +252,14 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): [coord] = coords cube.remove_coord(coord) - # Return cube if hotizontal grid is the same. - if _check_horiz_grid_closeness(cube, target_grid): - return cube + # Return unaltered cube if hotizontal grid is the same. + if not _check_horiz_grid_closeness(cube, target_grid): - # Perform the horizontal regridding. - if _attempt_irregular_regridding(cube, scheme): - cube = esmpy_regrid(cube, target_grid, scheme) - else: - cube = cube.regrid(target_grid, HORIZONTAL_SCHEMES[scheme]) + # Perform the horizontal regridding. + if _attempt_irregular_regridding(cube, scheme): + cube = esmpy_regrid(cube, target_grid, scheme) + else: + cube = cube.regrid(target_grid, HORIZONTAL_SCHEMES[scheme]) return cube From aee41028f977d69a63719e7935b5c30da049272f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Tue, 25 Feb 2020 07:11:07 +0100 Subject: [PATCH 08/22] Update doc/esmvalcore/preprocessor.rst Co-Authored-By: Bouwe Andela --- doc/esmvalcore/preprocessor.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/esmvalcore/preprocessor.rst b/doc/esmvalcore/preprocessor.rst index 9f24fad248..b2fbade2ac 100644 --- a/doc/esmvalcore/preprocessor.rst +++ b/doc/esmvalcore/preprocessor.rst @@ -44,7 +44,7 @@ Overview * ease of maintenance (including unit and integration testing) of existing routines; * a straightforward manner of importing and using the preprocessing routines as - part of the overall usage of the software and, as a special case, the useFH + part of the overall usage of the software and, as a special case, the use during diagnostic execution; * shifting the effort for the scientific diagnostic developer from implementing both standard and diagnostic-specific functionalities to allowing them to From 06c08d69b19c87acac1a0b36922b2dd46e0ecff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Tue, 25 Feb 2020 07:25:45 +0100 Subject: [PATCH 09/22] aggregate shape and bounds checks --- esmvalcore/preprocessor/_regrid.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index 3b553e3782..ee5171b0cc 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -252,7 +252,7 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): [coord] = coords cube.remove_coord(coord) - # Return unaltered cube if hotizontal grid is the same. + # Return non-regridded cube if hotizontal grid is the same. if not _check_horiz_grid_closeness(cube, target_grid): # Perform the horizontal regridding. @@ -291,21 +291,13 @@ def _check_horiz_grid_closeness(cube1, cube2): """ # Go through the 2 expected horizontal coordinates longitude and latitude. for coord in ['latitude', 'longitude']: - # Compare shapes. - if cube1.coord(coord).shape != cube2.coord(coord).shape: - # Different shapes detected. + coord1 = cube1.coord(coord) + coord2 = cube2.coord(coord) + # Compare shapes and bounds + shp_n_bnds = (coord1.shape == coord2.shape + and np.allclose(coord1.bounds, coord2.bounds)) + if not shp_n_bnds: return False - else: - # Iterate through the combined bounds. - for c1bndval, c2bndval in np.nditer( - [cube1.coord(coord).bounds, - cube2.coord(coord).bounds] - ): - # Check if bound value is same or similar. - if (c1bndval != c2bndval - and not np.allclose(c1bndval, c2bndval)): - # Different bounds values detected. - return False # Returns default value. return True From 17d273939377f91be43386fc1e0f6ecf1f53748f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Tue, 25 Feb 2020 07:34:33 +0100 Subject: [PATCH 10/22] whitespace edits --- esmvalcore/preprocessor/_regrid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index ee5171b0cc..da710ac03f 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -293,9 +293,9 @@ def _check_horiz_grid_closeness(cube1, cube2): for coord in ['latitude', 'longitude']: coord1 = cube1.coord(coord) coord2 = cube2.coord(coord) - # Compare shapes and bounds - shp_n_bnds = (coord1.shape == coord2.shape - and np.allclose(coord1.bounds, coord2.bounds)) + # Compare shapes and bounds. + shp_n_bnds = (coord1.shape == coord2.shape + and np.allclose(coord1.bounds, coord2.bounds)) if not shp_n_bnds: return False # Returns default value. From 9ec0062ddb9ec24c9d97b4b3b31aae840db403c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Thu, 27 Feb 2020 15:25:14 +0100 Subject: [PATCH 11/22] extend mock calls new grid checker for avoiding horizontal regriding needs additional calls --- tests/unit/preprocessor/_regrid/test_regrid.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index b1f951fb55..8762c2f8ce 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -24,7 +24,9 @@ def _check(self, tgt_grid, scheme, spec=False): self.coord_system.asset_called_once() expected_calls = [ mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True) + mock.call(axis='y', dim_coords=True), + mock.call(axis='latitude', dim_coords=True), + mock.call(axis='longitude', dim_coords=True) ] self.assertEqual(self.tgt_grid_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(self.tgt_grid, expected_scheme) @@ -32,7 +34,9 @@ def _check(self, tgt_grid, scheme, spec=False): if scheme == 'unstructured_nearest': expected_calls = [ mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True) + mock.call(axis='y', dim_coords=True), + mock.call(axis='latitude', dim_coords=True), + mock.call(axis='longitude', dim_coords=True) ] self.assertEqual(self.coords.mock_calls, expected_calls) expected_calls = [mock.call(self.coord), mock.call(self.coord)] From ca53d2ded9fe490564a2e02357aa0ed0ffc4d796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Thu, 27 Feb 2020 15:55:32 +0100 Subject: [PATCH 12/22] correct mocked call --- tests/unit/preprocessor/_regrid/test_regrid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 8762c2f8ce..0fc4d90261 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -35,8 +35,8 @@ def _check(self, tgt_grid, scheme, spec=False): expected_calls = [ mock.call(axis='x', dim_coords=True), mock.call(axis='y', dim_coords=True), - mock.call(axis='latitude', dim_coords=True), - mock.call(axis='longitude', dim_coords=True) + mock.call(axis='latitude'), + mock.call(axis='longitude') ] self.assertEqual(self.coords.mock_calls, expected_calls) expected_calls = [mock.call(self.coord), mock.call(self.coord)] From 232ff8725955e8d63ec62ed6ca3e5b40f6a676d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20M=C3=BCller?= Date: Fri, 28 Feb 2020 09:21:09 +0100 Subject: [PATCH 13/22] correct mock calls --- tests/unit/preprocessor/_regrid/test_regrid.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 0fc4d90261..7017abc185 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -25,8 +25,7 @@ def _check(self, tgt_grid, scheme, spec=False): expected_calls = [ mock.call(axis='x', dim_coords=True), mock.call(axis='y', dim_coords=True), - mock.call(axis='latitude', dim_coords=True), - mock.call(axis='longitude', dim_coords=True) + mock.call('latitude') ] self.assertEqual(self.tgt_grid_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(self.tgt_grid, expected_scheme) @@ -34,9 +33,7 @@ def _check(self, tgt_grid, scheme, spec=False): if scheme == 'unstructured_nearest': expected_calls = [ mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True), - mock.call(axis='latitude'), - mock.call(axis='longitude') + mock.call(axis='y', dim_coords=True) ] self.assertEqual(self.coords.mock_calls, expected_calls) expected_calls = [mock.call(self.coord), mock.call(self.coord)] From d280611d5061f8c9d187d68803f4e034b564e5f8 Mon Sep 17 00:00:00 2001 From: BenMGeo Date: Mon, 9 Mar 2020 13:53:41 +0100 Subject: [PATCH 14/22] add test for closeness --- .../unit/preprocessor/_regrid/test_regrid.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 7017abc185..0e8cb295a7 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -7,10 +7,12 @@ from unittest import mock import iris +import numpy as np import tests from esmvalcore.preprocessor import regrid from esmvalcore.preprocessor._regrid import _CACHE, HORIZONTAL_SCHEMES +from esmvalcore.preprocessor._regrid import _check_horiz_grid_closeness class Test(tests.Test): @@ -109,5 +111,38 @@ def test_regrid__cell_specification(self): self.assertEqual(set(_CACHE.keys()), set(specs)) +class TestCloseness(tests.Test): + def test_regrid__closeness_cell_specification(self): + latitude = iris.coords.DimCoord(np.linspace(-85, 85, 18), + standard_name='latitude', + units='degrees') + longitude = iris.coords.DimCoord(np.linspace(5, 355, 36), + standard_name='longitude', + units='degrees') + latitude.guess_bounds() + longitude.guess_bounds() + loc_cube = iris.cube.Cube(np.empty([18, 36]), + standard_name=None, + long_name=None, + var_name=None, + units=None, + attributes=None, + cell_methods=None, + dim_coords_and_dims=[(latitude, 0), + (longitude, 1)], + aux_coords_and_dims=None, + aux_factories=None, + cell_measures_and_dims=None) + scheme = 'area_weighted' + tgt_cubes = [loc_cube, + regrid(loc_cube, '10x10', scheme), + regrid(loc_cube, '10x20', scheme), + regrid(loc_cube, '20x10', scheme)] + closeness = [] + for tgt in tgt_cubes: + closeness.append(_check_horiz_grid_closeness(loc_cube, tgt)) + self.assertEqual(closeness, [True, True, False, False]) + + if __name__ == '__main__': unittest.main() From cef7b0da282d20109cbfd8258631b768bd0e4cdb Mon Sep 17 00:00:00 2001 From: BenMGeo Date: Mon, 9 Mar 2020 15:11:12 +0100 Subject: [PATCH 15/22] improved tests to not test with the assumption that the tests succeed. --- .../unit/preprocessor/_regrid/test_regrid.py | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 0e8cb295a7..26f199f9ff 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -119,29 +119,37 @@ def test_regrid__closeness_cell_specification(self): longitude = iris.coords.DimCoord(np.linspace(5, 355, 36), standard_name='longitude', units='degrees') + latitude2 = iris.coords.DimCoord(np.linspace(-85, 85, 17), + standard_name='latitude', + units='degrees') + longitude2 = iris.coords.DimCoord(np.linspace(5, 355, 35), + standard_name='longitude', + units='degrees') latitude.guess_bounds() longitude.guess_bounds() loc_cube = iris.cube.Cube(np.empty([18, 36]), - standard_name=None, - long_name=None, - var_name=None, - units=None, - attributes=None, - cell_methods=None, dim_coords_and_dims=[(latitude, 0), (longitude, 1)], - aux_coords_and_dims=None, - aux_factories=None, - cell_measures_and_dims=None) - scheme = 'area_weighted' + ) tgt_cubes = [loc_cube, - regrid(loc_cube, '10x10', scheme), - regrid(loc_cube, '10x20', scheme), - regrid(loc_cube, '20x10', scheme)] - closeness = [] - for tgt in tgt_cubes: - closeness.append(_check_horiz_grid_closeness(loc_cube, tgt)) - self.assertEqual(closeness, [True, True, False, False]) + iris.cube.Cube(np.empty([18, 36]), + dim_coords_and_dims=[(latitude, 0), + (longitude, 1)], + ), + iris.cube.Cube(np.empty([17, 36]), + dim_coords_and_dims=[(latitude2, 0), + (longitude, 1)], + ), + iris.cube.Cube(np.empty([18, 35]), + dim_coords_and_dims=[(latitude, 0), + (longitude2, 1)], + ) + ] + + tgt_closeness = [True, True, False, False] + for tgt in zip(tgt_cubes, tgt_closeness): + self.assertEqual(_check_horiz_grid_closeness(loc_cube, tgt[0]), + tgt[1]) if __name__ == '__main__': From 48d200ab6434412964f6bb623621b49b98b8ddb3 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 23 Mar 2020 16:01:24 +0100 Subject: [PATCH 16/22] Avoid calling _check_horiz_grid_closeness in unrelated test --- tests/unit/preprocessor/_regrid/test_regrid.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 26f199f9ff..03c8bf17c4 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -26,8 +26,7 @@ def _check(self, tgt_grid, scheme, spec=False): self.coord_system.asset_called_once() expected_calls = [ mock.call(axis='x', dim_coords=True), - mock.call(axis='y', dim_coords=True), - mock.call('latitude') + mock.call(axis='y', dim_coords=True) ] self.assertEqual(self.tgt_grid_coord.mock_calls, expected_calls) self.regrid.assert_called_once_with(self.tgt_grid, expected_scheme) @@ -67,6 +66,13 @@ def setUp(self): 'unstructured_nearest' ] + def _mock_check_horiz_grid_closeness(src, tgt): + return False + + self.patch( + 'esmvalcore.preprocessor._regrid._check_horiz_grid_closeness', + side_effect=_mock_check_horiz_grid_closeness) + def _return_mock_stock_cube(spec, lat_offset=True, lon_offset=True): return self.tgt_grid From 766f1fd4f908fa772a792efa35d607188308c793 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 26 Apr 2021 12:24:16 +0200 Subject: [PATCH 17/22] Fix flake8 error --- tests/unit/preprocessor/_regrid/test_regrid.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 3a0da88b75..3446d49717 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -73,7 +73,11 @@ def _mock_check_horiz_grid_closeness(src, tgt): 'esmvalcore.preprocessor._regrid._check_horiz_grid_closeness', side_effect=_mock_check_horiz_grid_closeness) - def _return_mock_stock_cube(spec, lat_offset=True, lon_offset=True): + def _return_mock_global_stock_cube( + spec, + lat_offset=True, + lon_offset=True, + ): return self.tgt_grid self.mock_stock = self.patch( From d60d88937dea9d9f96179995041c335259eb5ea4 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 26 Apr 2021 15:42:15 +0200 Subject: [PATCH 18/22] Clean up function --- esmvalcore/preprocessor/_regrid.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index ce9a5c41d6..b35c4b3a25 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -467,8 +467,8 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): [coord] = coords cube.remove_coord(coord) - # Return non-regridded cube if hotizontal grid is the same. - if not _check_horiz_grid_closeness(cube, target_grid): + # Return non-regridded cube if horizontal grid is the same. + if not _horizontal_grid_is_close(cube, target_grid): # Perform the horizontal regridding. if _attempt_irregular_regridding(cube, scheme): @@ -479,9 +479,8 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): return cube -def _check_horiz_grid_closeness(cube1, cube2): - """ - Check if two cubes have the same horizontal grid definition. +def _horizontal_grid_is_close(cube1, cube2): + """Check if two cubes have the same horizontal grid definition. The result of the function is a boolean answer, if both cubes have the same horizontal grid definition. The function checks both longitude and @@ -508,12 +507,13 @@ def _check_horiz_grid_closeness(cube1, cube2): for coord in ['latitude', 'longitude']: coord1 = cube1.coord(coord) coord2 = cube2.coord(coord) - # Compare shapes and bounds. - shp_n_bnds = (coord1.shape == coord2.shape - and np.allclose(coord1.bounds, coord2.bounds)) - if not shp_n_bnds: + + if not coord1.shape == coord2.shape: return False - # Returns default value. + + if not np.allclose(coord1.bounds, coord2.bounds): + return False + return True From 0b26859f2a0b81de8af04d07e3b6d253cf21aa2f Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 26 Apr 2021 15:44:08 +0200 Subject: [PATCH 19/22] Refactor unit tests using pytest --- .../unit/preprocessor/_regrid/test_regrid.py | 146 ++++++++++++------ 1 file changed, 99 insertions(+), 47 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 3446d49717..b4a9d6a883 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -1,18 +1,20 @@ -""" -Unit tests for the :func:`esmvalcore.preprocessor.regrid.regrid` function. - -""" +"""Unit tests for the :func:`esmvalcore.preprocessor.regrid.regrid` +function.""" import unittest from unittest import mock import iris import numpy as np +import pytest import tests from esmvalcore.preprocessor import regrid -from esmvalcore.preprocessor._regrid import _CACHE, HORIZONTAL_SCHEMES -from esmvalcore.preprocessor._regrid import _check_horiz_grid_closeness +from esmvalcore.preprocessor._regrid import ( + _CACHE, + HORIZONTAL_SCHEMES, + _horizontal_grid_is_close, +) class Test(tests.Test): @@ -74,8 +76,8 @@ def _mock_check_horiz_grid_closeness(src, tgt): side_effect=_mock_check_horiz_grid_closeness) def _return_mock_global_stock_cube( - spec, - lat_offset=True, + spec, + lat_offset=True, lon_offset=True, ): return self.tgt_grid @@ -128,45 +130,95 @@ def test_regrid__cell_specification(self): _CACHE.clear() -class TestCloseness(tests.Test): - def test_regrid__closeness_cell_specification(self): - latitude = iris.coords.DimCoord(np.linspace(-85, 85, 18), - standard_name='latitude', - units='degrees') - longitude = iris.coords.DimCoord(np.linspace(5, 355, 36), - standard_name='longitude', - units='degrees') - latitude2 = iris.coords.DimCoord(np.linspace(-85, 85, 17), - standard_name='latitude', - units='degrees') - longitude2 = iris.coords.DimCoord(np.linspace(5, 355, 35), - standard_name='longitude', - units='degrees') - latitude.guess_bounds() - longitude.guess_bounds() - loc_cube = iris.cube.Cube(np.empty([18, 36]), - dim_coords_and_dims=[(latitude, 0), - (longitude, 1)], - ) - tgt_cubes = [loc_cube, - iris.cube.Cube(np.empty([18, 36]), - dim_coords_and_dims=[(latitude, 0), - (longitude, 1)], - ), - iris.cube.Cube(np.empty([17, 36]), - dim_coords_and_dims=[(latitude2, 0), - (longitude, 1)], - ), - iris.cube.Cube(np.empty([18, 35]), - dim_coords_and_dims=[(latitude, 0), - (longitude2, 1)], - ) - ] - - tgt_closeness = [True, True, False, False] - for tgt in zip(tgt_cubes, tgt_closeness): - self.assertEqual(_check_horiz_grid_closeness(loc_cube, tgt[0]), - tgt[1]) +def _make_coord(start: float, stop: float, step: int, *, name: str): + """Helper function for creating a coord.""" + coord = iris.coords.DimCoord( + np.linspace(start, stop, step), + standard_name=name, + units='degrees', + ) + coord.guess_bounds() + return coord + + +def _make_cube(*, lat: tuple, lon: tuple): + """Helper function for creating a cube.""" + lat_coord = _make_coord(*lat, name='latitude') + lon_coord = _make_coord(*lon, name='longitude') + + return iris.cube.Cube( + np.empty([len(lat_coord.points), + len(lon_coord.points)]), + dim_coords_and_dims=[(lat_coord, 0), (lon_coord, 1)], + ) + + +LAT_SPEC1 = (-85, 85, 18) +LAT_SPEC2 = (-85, 85, 17) +LON_SPEC1 = (5, 355, 36) +LON_SPEC2 = (5, 355, 35) + + +@pytest.mark.parametrize( + 'hor_spec1, hor_spec2, expected', + ( + # equal lat/equal lon + ( + { + 'lat': LAT_SPEC1, + 'lon': LON_SPEC1 + }, + { + 'lat': LAT_SPEC1, + 'lon': LON_SPEC1 + }, + True, + ), + # equal lat/different lon + ( + { + 'lat': LAT_SPEC1, + 'lon': LON_SPEC1 + }, + { + 'lat': LAT_SPEC1, + 'lon': LON_SPEC2 + }, + False, + ), + # different lat/equal lon + ( + { + 'lat': LAT_SPEC1, + 'lon': LON_SPEC1 + }, + { + 'lat': LAT_SPEC2, + 'lon': LON_SPEC1 + }, + False, + ), + # different lat/different lon + ( + { + 'lat': LAT_SPEC1, + 'lon': LON_SPEC1 + }, + { + 'lat': LAT_SPEC2, + 'lon': LON_SPEC2 + }, + False, + ), + ), +) +def test_horizontal_grid_is_close(hor_spec1: dict, hor_spec2: dict, + expected: bool): + """Test for `_horizontal_grid_is_close`.""" + cube1 = _make_cube(**hor_spec1) + cube2 = _make_cube(**hor_spec2) + + assert _horizontal_grid_is_close(cube1, cube2) == expected if __name__ == '__main__': From 7b336656aa47c93e6191b661a7e55844fb908c75 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 26 Apr 2021 16:00:23 +0200 Subject: [PATCH 20/22] Update mock variable names --- tests/unit/preprocessor/_regrid/test_regrid.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index b4a9d6a883..47a625edac 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -68,12 +68,11 @@ def setUp(self): 'unstructured_nearest' ] - def _mock_check_horiz_grid_closeness(src, tgt): + def _mock_horizontal_grid_is_close(src, tgt): return False - self.patch( - 'esmvalcore.preprocessor._regrid._check_horiz_grid_closeness', - side_effect=_mock_check_horiz_grid_closeness) + self.patch('esmvalcore.preprocessor._regrid._horizontal_grid_is_close', + side_effect=_mock_horizontal_grid_is_close) def _return_mock_global_stock_cube( spec, From 96e65e4d86f24bd2f175444a44193b214c867a05 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 26 Apr 2021 16:25:39 +0200 Subject: [PATCH 21/22] Add test that regrid is skipped if grids are the same --- .../unit/preprocessor/_regrid/test_regrid.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 47a625edac..84442671b2 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -152,9 +152,12 @@ def _make_cube(*, lat: tuple, lon: tuple): ) +# 10x10 LAT_SPEC1 = (-85, 85, 18) -LAT_SPEC2 = (-85, 85, 17) LON_SPEC1 = (5, 355, 36) + +# almost 10x10 +LAT_SPEC2 = (-85, 85, 17) LON_SPEC2 = (5, 355, 35) @@ -220,5 +223,19 @@ def test_horizontal_grid_is_close(hor_spec1: dict, hor_spec2: dict, assert _horizontal_grid_is_close(cube1, cube2) == expected +def test_regrid_is_skipped_if_grids_are_the_same(): + """Test that regridding is skipped if the grids are the same.""" + cube = _make_cube(lat=LAT_SPEC1, lon=LON_SPEC1) + scheme = 'linear' + + # regridding to the same spec returns the same cube + expected_same_cube = regrid(cube, target_grid='10x10', scheme=scheme) + assert expected_same_cube is cube + + # regridding to a different spec returns a different cube + expected_different_cube = regrid(cube, target_grid='5x5', scheme=scheme) + assert expected_different_cube is not cube + + if __name__ == '__main__': unittest.main() From 9ba5471da1fb70d5636ad1a0488885d4e23f3aad Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 26 Apr 2021 16:37:27 +0200 Subject: [PATCH 22/22] Simplify tests and add check for different values --- .../unit/preprocessor/_regrid/test_regrid.py | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 84442671b2..b7beaca442 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -156,69 +156,64 @@ def _make_cube(*, lat: tuple, lon: tuple): LAT_SPEC1 = (-85, 85, 18) LON_SPEC1 = (5, 355, 36) -# almost 10x10 +# almost 10x10, but different shape LAT_SPEC2 = (-85, 85, 17) LON_SPEC2 = (5, 355, 35) +# 10x10, but different coords +LAT_SPEC3 = (-90, 90, 18) +LON_SPEC3 = (0, 360, 36) + @pytest.mark.parametrize( - 'hor_spec1, hor_spec2, expected', + 'cube2_spec, expected', ( - # equal lat/equal lon + # equal lat/lon ( { 'lat': LAT_SPEC1, - 'lon': LON_SPEC1 - }, - { - 'lat': LAT_SPEC1, - 'lon': LON_SPEC1 + 'lon': LON_SPEC1, }, True, ), - # equal lat/different lon + # different lon shape ( { 'lat': LAT_SPEC1, - 'lon': LON_SPEC1 - }, - { - 'lat': LAT_SPEC1, - 'lon': LON_SPEC2 + 'lon': LON_SPEC2, }, False, ), - # different lat/equal lon + # different lat shape ( - { - 'lat': LAT_SPEC1, - 'lon': LON_SPEC1 - }, { 'lat': LAT_SPEC2, - 'lon': LON_SPEC1 + 'lon': LON_SPEC1, }, False, ), - # different lat/different lon + # different lon values ( { 'lat': LAT_SPEC1, - 'lon': LON_SPEC1 + 'lon': LON_SPEC3, }, + False, + ), + # different lat values + ( { - 'lat': LAT_SPEC2, - 'lon': LON_SPEC2 + 'lat': LAT_SPEC3, + 'lon': LON_SPEC1, }, False, ), ), ) -def test_horizontal_grid_is_close(hor_spec1: dict, hor_spec2: dict, - expected: bool): +def test_horizontal_grid_is_close(cube2_spec: dict, expected: bool): """Test for `_horizontal_grid_is_close`.""" - cube1 = _make_cube(**hor_spec1) - cube2 = _make_cube(**hor_spec2) + cube1 = _make_cube(lat=LAT_SPEC1, lon=LON_SPEC1) + cube2 = _make_cube(**cube2_spec) assert _horizontal_grid_is_close(cube1, cube2) == expected