From 0eae8c686ea5218977676b5ceeb8e14c211add09 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Fri, 24 Apr 2020 11:38:13 -0700 Subject: [PATCH 01/20] Updates to CMIP6 fixes - tos and tas tested --- esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py | 28 ++++++- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 40 ++++++++++ esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py | 8 ++ esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py | 32 ++++++++ esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py | 31 ++++++++ esmvalcore/cmor/_fixes/common.py | 83 +++++++++++++++++++++ 6 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py create mode 100644 esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py diff --git a/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py b/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py index 2787e282e4..472789dad6 100644 --- a/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py +++ b/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py @@ -17,16 +17,40 @@ class Tos(BaseTos): def fix_metadata(self, cubes): """Rename ``var_name`` of 1D-``latitude`` and 1D-``longitude``. - Parameters ---------- cubes : iris.cube.CubeList Input cubes. - Returns ------- iris.cube.CubeList + """ + cube = self.get_cube_from_list(cubes) + lat_coord = cube.coord('latitude', dimensions=(1, )) + lon_coord = cube.coord('longitude', dimensions=(2, )) + lat_coord.standard_name = None + lat_coord.long_name = 'grid_latitude' + lat_coord.var_name = 'i' + lat_coord.units = '1' + lon_coord.standard_name = None + lon_coord.long_name = 'grid_longitude' + lon_coord.var_name = 'j' + lon_coord.units = '1' + lon_coord.circular = False + return cubes + +class Siconc(BaseTos): + """Fixes for siconc.""" + def fix_metadata(self, cubes): + """Rename ``var_name`` of 1D-``latitude`` and 1D-``longitude``. + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + Returns + ------- + iris.cube.CubeList """ cube = self.get_cube_from_list(cubes) lat_coord = cube.coord('latitude', dimensions=(1, )) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index d763e04244..d0bfaf0b78 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -6,6 +6,8 @@ from ..fix import Fix from ..shared import (add_scalar_depth_coord, add_scalar_height_coord, add_scalar_typeland_coord, add_scalar_typesea_coord) +import numpy as np +from .gfdl_esm4 import Siconc class Cl(Fix): @@ -88,6 +90,7 @@ class Tas(Fix): def fix_metadata(self, cubes): """Add height (2m) coordinate. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. Parameters ---------- @@ -101,6 +104,15 @@ def fix_metadata(self, cubes): """ cube = self.get_cube_from_list(cubes) add_scalar_height_coord(cube) + + for cube in cubes: + latitude = cube.coord('latitude') + latitude.bounds = latitude.bounds.astype(np.float64) + latitude.bounds=np.round(latitude.bounds,4) + longitude = cube.coord('longitude') + longitude.bounds = longitude.bounds.astype(np.float64) + longitude.bounds=np.round(longitude.bounds,4) + return cubes @@ -144,3 +156,31 @@ def fix_metadata(self, cubes): cube = self.get_cube_from_list(cubes) add_scalar_typesea_coord(cube) return cubes + +class Tos(Fix): + """Fixes for tos.""" + + def fix_metadata(self, cubes): + """Round times to 1 d.p. for monthly means. + Required to get hist-GHG and ssp245-GHG Omon tos to concatenate. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + + for cube in cubes: + if cube.attributes['mipTable']=='Omon': + cube.coord('time').points=np.round(cube.coord('time').points,1) + + return cubes + + +Siconc = Siconc diff --git a/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py b/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py index df6de94284..c752ca2076 100644 --- a/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py +++ b/esmvalcore/cmor/_fixes/cmip6/fgoals_g3.py @@ -1,5 +1,7 @@ """Fixes for FGOALS-g3 model.""" from ..cmip5.fgoals_g2 import Cl as BaseCl +from ..common import OceanFixGrid + Cl = BaseCl @@ -8,3 +10,9 @@ Clw = BaseCl + + +Tos = OceanFixGrid + + +Siconc = OceanFixGrid diff --git a/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py b/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py new file mode 100644 index 0000000000..e7ccfa40b5 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py @@ -0,0 +1,32 @@ +"""Fixes for GFDL-ESM4 model.""" +from ..fix import Fix +import iris + + +class Siconc(Fix): + """Fixes for siconc.""" + + def fix_metadata(self, cubelist): + """ + Fix missing type. + + Parameters + ---------- + cubelist: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + typesi = iris.coords.AuxCoord( + 'siconc', + standard_name='area_type', + long_name='Sea Ice area type', + var_name='type', + units='1', + bounds=None) + for cube in cubelist: + cube.add_aux_coord(typesi) + return cubelist diff --git a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py new file mode 100644 index 0000000000..ddc5291be7 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py @@ -0,0 +1,31 @@ +"""Fixes for NorESM2-LM model.""" +from ..fix import Fix +import numpy as np + + +class Siconc(Fix): + """Fixes for tas.""" + + def fix_metadata(self, cubes): + """Fix metadata. + + Some coordinate points vary for different files of this dataset (for + different time range). This fix removes these inaccuracies by rounding + the coordinates. + + Parameters + ---------- + cubes: iris.cube.CubeList + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + latitude = cube.coord('latitude') + latitude.bounds=np.round(latitude.bounds,4) + longitude = cube.coord('longitude') + longitude.bounds=np.round(longitude.bounds,4) + + return (cubes) diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index d60b2eb7f0..be9cc44b7e 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -3,6 +3,8 @@ from .fix import Fix from .shared import add_plev_from_altitude, fix_bounds +import numpy as np +from scipy.ndimage import map_coordinates class ClFixHybridHeightCoord(Fix): @@ -101,3 +103,84 @@ def fix_metadata(self, cubes): cube.coord(var_name='ps').attributes = {} return iris.cube.CubeList([cube]) + + +class OceanFixGrid(Fix): + + """Fixes for tos, siconc in FGOALS-g3.""" + + def fix_data(self, cube): + """Fix data. + Calculate missing latitude/longitude boundaries using interpolation. + Based on a similar fix for BCC-CSM2-MR + Parameters + ---------- + cube: iris.cube.Cube + Input cube to fix. + Returns + ------- + iris.cube.Cube + """ + + rlat = cube.coord('grid_latitude').points + rlon = cube.coord('grid_longitude').points + + #Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). + rlat_idx_bnds = np.zeros((len(rlat),2)) + rlat_idx_bnds[:,0]=np.arange(len(rlat))-0.5 + rlat_idx_bnds[:,1]=np.arange(len(rlat))+0.5 + rlat_idx_bnds[0,0]=0. + rlat_idx_bnds[len(rlat)-1,1]=len(rlat) + rlon_idx_bnds = np.zeros((len(rlon),2)) + rlon_idx_bnds[:,0]=np.arange(len(rlon))-0.5 + rlon_idx_bnds[:,1]=np.arange(len(rlon))+0.5 + + # Calculate latitude/longitude vertices by interpolation + lat_vertices = [] + lon_vertices = [] + for (i, j) in [(0, 0), (0, 1), (1, 1), (1, 0)]: + (rlat_v, rlon_v) = np.meshgrid(rlat_idx_bnds[:, i], + rlon_idx_bnds[:, j], + indexing='ij') + lat_vertices.append( + map_coordinates(cube.coord('latitude').points, + [rlat_v, rlon_v], + mode='nearest')) + lon_vertices.append( + map_coordinates(cube.coord('longitude').points, + [rlat_v, rlon_v], + mode='wrap')) + lat_vertices = np.array(lat_vertices) + lon_vertices = np.array(lon_vertices) + lat_vertices = np.moveaxis(lat_vertices, 0, -1) + lon_vertices = np.moveaxis(lon_vertices, 0, -1) + + # Copy vertices to cube + cube.coord('latitude').bounds = lat_vertices + cube.coord('longitude').bounds = lon_vertices + + return cube + + def fix_metadata(self, cubes): + """Rename ``var_name`` of 1D-``latitude`` and 1D-``longitude``. + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes. + Returns + ------- + iris.cube.CubeList + """ + cube = self.get_cube_from_list(cubes) + lat_coord = cube.coord('cell index along second dimension', dimensions=(1, )) + lon_coord = cube.coord('cell index along first dimension', dimensions=(2, )) + lat_coord.standard_name = None + lat_coord.long_name = 'grid_latitude' + lat_coord.var_name = 'i' + lat_coord.units = '1' + lon_coord.standard_name = None + lon_coord.long_name = 'grid_longitude' + lon_coord.var_name = 'j' + lon_coord.units = '1' + lon_coord.circular = False + return cubes From 382222388d62989f233a2a2ed19b4d675e64b5c7 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Wed, 29 Apr 2020 11:06:33 -0700 Subject: [PATCH 02/20] fix added for FGOALS-g3 longitudes and latitudes > 1e30 --- esmvalcore/cmor/_fixes/common.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index be9cc44b7e..ceaf97f599 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -5,7 +5,7 @@ from .shared import add_plev_from_altitude, fix_bounds import numpy as np from scipy.ndimage import map_coordinates - +import matplotlib.pyplot as plt class ClFixHybridHeightCoord(Fix): """Fixes for ``cl`` regarding hybrid sigma height coordinates.""" @@ -158,7 +158,7 @@ def fix_data(self, cube): # Copy vertices to cube cube.coord('latitude').bounds = lat_vertices cube.coord('longitude').bounds = lon_vertices - + print ('************ fix_data,rlon,rlat',rlon,rlat) return cube def fix_metadata(self, cubes): @@ -183,4 +183,8 @@ def fix_metadata(self, cubes): lon_coord.var_name = 'j' lon_coord.units = '1' lon_coord.circular = False + #FGOALS-g3 data contain latitude and longitude data set to >1e30 in some + #places. Set to 0. to avoid problem in check.py. + cube.coord('latitude').points[cube.coord('latitude').points > 1000.]=0. + cube.coord('longitude').points[cube.coord('longitude').points > 1000.]=0. return cubes From f1558cd5640560b847d5860312a087ba72c7b61e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 18 May 2020 14:26:30 +0100 Subject: [PATCH 03/20] wrote test for gfdl esm4 siconc --- .../cmor/_fixes/cmip6/test_gfdl_esm4.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py new file mode 100644 index 0000000000..573faf3c51 --- /dev/null +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py @@ -0,0 +1,79 @@ +"""Tests for the fixes of GFDL-ESM4.""" +import os + +import iris +import numpy as np +import pytest +from cf_units import Unit +from netCDF4 import Dataset + +from esmvalcore.cmor._fixes.cmip6.gfdl_esm4 import Siconc +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info + + +@pytest.fixture +def siconc_file(tmp_path): + """Create netcdf file with similar issues as ``cl``.""" + nc_path = os.path.join(tmp_path, 'gfdl_esm4_siconc.nc') + dataset = Dataset(nc_path, mode='w') + dataset.createDimension('time', size=1) + dataset.createDimension('lat', size=1) + dataset.createDimension('lon', size=1) + + # Dimensional variables + dataset.createVariable('time', np.float64, dimensions=('time',)) + dataset.createVariable('lat', np.float64, dimensions=('lat',)) + dataset.createVariable('lon', np.float64, dimensions=('lon',)) + dataset.variables['time'][:] = [0.0] + dataset.variables['time'].standard_name = 'time' + dataset.variables['time'].units = 'days since 6543-2-1' + dataset.variables['lat'][:] = [-30.0] + dataset.variables['lat'].standard_name = 'latitude' + dataset.variables['lat'].units = 'degrees_north' + dataset.variables['lon'][:] = [30.0] + dataset.variables['lon'].standard_name = 'longitude' + dataset.variables['lon'].units = 'degrees_east' + dataset.createVariable('siconc', np.float64, + dimensions=('time', 'lat', 'lon')) + dataset.variables['siconc'][:] = 22. + dataset.variables['siconc'].standard_name = 'sea_ice_area_fraction' + dataset.variables['siconc'].units = '%' + + dataset.close() + return nc_path + + +def test_get_siconc_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CMIP6', 'GFDL-ESM4', 'SImon', 'siconc') + assert fix == [Siconc(None)] + + +def test_siconc_fix_metadata(siconc_file): + """Test ``fix_metadata`` for ``cl``.""" + cubes = iris.load(siconc_file) + + # Raw cubes + assert len(cubes) == 1 + siconc_cube = cubes[0] + assert siconc_cube.var_name == "siconc" + + # Extract siconc cube + siconc_cube = cubes.extract_strict('sea_ice_area_fraction') + assert not siconc_cube.coords('typesi') + + # Apply fix + vardef = get_var_info('CMIP6', 'SImon', 'siconc') + fix = Siconc(vardef) + fixed_cubes = fix.fix_metadata(cubes) + assert len(fixed_cubes) == 1 + fixed_siconc_cube = fixed_cubes.extract_strict( + 'sea_ice_area_fraction') + fixed_typesi_coord = fixed_siconc_cube.coord('area_type') + assert fixed_typesi_coord.points is not None + assert fixed_typesi_coord.bounds is None + np.testing.assert_equal(fixed_typesi_coord.points, + ['siconc']) + np.testing.assert_equal(fixed_typesi_coord.units, + Unit('1')) From 6e71f6c53fb2e5181a0ae6309279513b9dfd6722 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Thu, 21 May 2020 11:56:37 -0700 Subject: [PATCH 04/20] Updates to code to fix issues identified by Codacy --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 15 ++++---- esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py | 10 +++--- esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py | 10 +++--- esmvalcore/cmor/_fixes/common.py | 42 ++++++++++++---------- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index d0bfaf0b78..bc757639ca 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -2,12 +2,12 @@ from shutil import copyfile from netCDF4 import Dataset +import numpy as np from ..fix import Fix from ..shared import (add_scalar_depth_coord, add_scalar_height_coord, add_scalar_typeland_coord, add_scalar_typesea_coord) -import numpy as np -from .gfdl_esm4 import Siconc +from .gfdl_esm4 import Siconc as Addtypesi class Cl(Fix): @@ -108,10 +108,10 @@ def fix_metadata(self, cubes): for cube in cubes: latitude = cube.coord('latitude') latitude.bounds = latitude.bounds.astype(np.float64) - latitude.bounds=np.round(latitude.bounds,4) + latitude.bounds=np.round(latitude.bounds, 4) longitude = cube.coord('longitude') longitude.bounds = longitude.bounds.astype(np.float64) - longitude.bounds=np.round(longitude.bounds,4) + longitude.bounds=np.round(longitude.bounds, 4) return cubes @@ -177,10 +177,9 @@ def fix_metadata(self, cubes): cube = self.get_cube_from_list(cubes) for cube in cubes: - if cube.attributes['mipTable']=='Omon': - cube.coord('time').points=np.round(cube.coord('time').points,1) - + if cube.attributes['mipTable'] == 'Omon': + cube.coord('time').points=np.round(cube.coord('time').points, 1) return cubes -Siconc = Siconc +Siconc = Addtypesi diff --git a/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py b/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py index e7ccfa40b5..a4e937842e 100644 --- a/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py +++ b/esmvalcore/cmor/_fixes/cmip6/gfdl_esm4.py @@ -1,18 +1,18 @@ """Fixes for GFDL-ESM4 model.""" -from ..fix import Fix import iris +from ..fix import Fix class Siconc(Fix): """Fixes for siconc.""" - def fix_metadata(self, cubelist): + def fix_metadata(self, cubes): """ Fix missing type. Parameters ---------- - cubelist: iris CubeList + cubes: iris CubeList List of cubes to fix Returns @@ -27,6 +27,6 @@ def fix_metadata(self, cubelist): var_name='type', units='1', bounds=None) - for cube in cubelist: + for cube in cubes: cube.add_aux_coord(typesi) - return cubelist + return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py index ddc5291be7..3525a7236b 100644 --- a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py +++ b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py @@ -1,10 +1,10 @@ """Fixes for NorESM2-LM model.""" -from ..fix import Fix import numpy as np +from ..fix import Fix class Siconc(Fix): - """Fixes for tas.""" + """Fixes for siconc.""" def fix_metadata(self, cubes): """Fix metadata. @@ -24,8 +24,8 @@ def fix_metadata(self, cubes): """ for cube in cubes: latitude = cube.coord('latitude') - latitude.bounds=np.round(latitude.bounds,4) + latitude.bounds=np.round(latitude.bounds, 4) longitude = cube.coord('longitude') - longitude.bounds=np.round(longitude.bounds,4) + longitude.bounds=np.round(longitude.bounds, 4) - return (cubes) + return cubes diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index ceaf97f599..7a659e0e52 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -1,11 +1,10 @@ """Common fixes used for multiple datasets.""" import iris +import numpy as np +from scipy.ndimage import map_coordinates from .fix import Fix from .shared import add_plev_from_altitude, fix_bounds -import numpy as np -from scipy.ndimage import map_coordinates -import matplotlib.pyplot as plt class ClFixHybridHeightCoord(Fix): """Fixes for ``cl`` regarding hybrid sigma height coordinates.""" @@ -126,15 +125,15 @@ def fix_data(self, cube): rlon = cube.coord('grid_longitude').points #Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). - rlat_idx_bnds = np.zeros((len(rlat),2)) - rlat_idx_bnds[:,0]=np.arange(len(rlat))-0.5 - rlat_idx_bnds[:,1]=np.arange(len(rlat))+0.5 - rlat_idx_bnds[0,0]=0. - rlat_idx_bnds[len(rlat)-1,1]=len(rlat) - rlon_idx_bnds = np.zeros((len(rlon),2)) - rlon_idx_bnds[:,0]=np.arange(len(rlon))-0.5 - rlon_idx_bnds[:,1]=np.arange(len(rlon))+0.5 - + rlat_idx_bnds = np.zeros((len(rlat), 2)) + rlat_idx_bnds[:,0] = np.arange(len(rlat))-0.5 + rlat_idx_bnds[:,1] = np.arange(len(rlat))+0.5 + rlat_idx_bnds[0,0] = 0. + rlat_idx_bnds[len(rlat)-1,1] = len(rlat) + rlon_idx_bnds = np.zeros((len(rlon), 2)) + rlon_idx_bnds[:,0] = np.arange(len(rlon))-0.5 + rlon_idx_bnds[:,1] = np.arange(len(rlon))+0.5 + # Calculate latitude/longitude vertices by interpolation lat_vertices = [] lon_vertices = [] @@ -158,7 +157,6 @@ def fix_data(self, cube): # Copy vertices to cube cube.coord('latitude').bounds = lat_vertices cube.coord('longitude').bounds = lon_vertices - print ('************ fix_data,rlon,rlat',rlon,rlat) return cube def fix_metadata(self, cubes): @@ -172,8 +170,10 @@ def fix_metadata(self, cubes): iris.cube.CubeList """ cube = self.get_cube_from_list(cubes) - lat_coord = cube.coord('cell index along second dimension', dimensions=(1, )) - lon_coord = cube.coord('cell index along first dimension', dimensions=(2, )) + lat_coord = cube.coord('cell index along second dimension', + dimensions=(1, )) + lon_coord = cube.coord('cell index along first dimension', + dimensions=(2, )) lat_coord.standard_name = None lat_coord.long_name = 'grid_latitude' lat_coord.var_name = 'i' @@ -183,8 +183,12 @@ def fix_metadata(self, cubes): lon_coord.var_name = 'j' lon_coord.units = '1' lon_coord.circular = False - #FGOALS-g3 data contain latitude and longitude data set to >1e30 in some - #places. Set to 0. to avoid problem in check.py. - cube.coord('latitude').points[cube.coord('latitude').points > 1000.]=0. - cube.coord('longitude').points[cube.coord('longitude').points > 1000.]=0. + #FGOALS-g3 data contain latitude and longitude data set to + #>1e30 in some places. Set to 0. to avoid problem in check.py. + cube.coord('latitude').points[cube.coord('latitude').points > 1000.]\ + = 0. + cube.coord('longitude').points[cube.coord('longitude').points > 1000.]\ + = 0. return cubes + + From 25d63f1b9f50c525bd8ccf6a82bbf818e7dddc40 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Thu, 21 May 2020 12:16:05 -0700 Subject: [PATCH 05/20] Further modifications to address Codacy issues --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 3 ++- esmvalcore/cmor/_fixes/common.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index bc757639ca..528d908e80 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -178,7 +178,8 @@ def fix_metadata(self, cubes): for cube in cubes: if cube.attributes['mipTable'] == 'Omon': - cube.coord('time').points=np.round(cube.coord('time').points, 1) + cube.coord('time').points = \ + np.round(cube.coord('time').points, 1) return cubes diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index 7a659e0e52..09c28b7ae7 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -190,5 +190,3 @@ def fix_metadata(self, cubes): cube.coord('longitude').points[cube.coord('longitude').points > 1000.]\ = 0. return cubes - - From 9cc050c37c060accab5ded8a6a5c23e7d0551a89 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Thu, 21 May 2020 13:47:09 -0700 Subject: [PATCH 06/20] More syntax changes --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 4 ++-- esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py | 4 ++-- esmvalcore/cmor/_fixes/common.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index 528d908e80..cabc0b83b2 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -108,10 +108,10 @@ def fix_metadata(self, cubes): for cube in cubes: latitude = cube.coord('latitude') latitude.bounds = latitude.bounds.astype(np.float64) - latitude.bounds=np.round(latitude.bounds, 4) + latitude.bounds = np.round(latitude.bounds, 4) longitude = cube.coord('longitude') longitude.bounds = longitude.bounds.astype(np.float64) - longitude.bounds=np.round(longitude.bounds, 4) + longitude.bounds = np.round(longitude.bounds, 4) return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py index 3525a7236b..5077a1a843 100644 --- a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py +++ b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py @@ -24,8 +24,8 @@ def fix_metadata(self, cubes): """ for cube in cubes: latitude = cube.coord('latitude') - latitude.bounds=np.round(latitude.bounds, 4) + latitude.bounds = np.round(latitude.bounds, 4) longitude = cube.coord('longitude') - longitude.bounds=np.round(longitude.bounds, 4) + longitude.bounds = np.round(longitude.bounds, 4) return cubes diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index 09c28b7ae7..76a2f9cea7 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -126,13 +126,13 @@ def fix_data(self, cube): #Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). rlat_idx_bnds = np.zeros((len(rlat), 2)) - rlat_idx_bnds[:,0] = np.arange(len(rlat))-0.5 - rlat_idx_bnds[:,1] = np.arange(len(rlat))+0.5 + rlat_idx_bnds[:,0] = np.arange(len(rlat)) - 0.5 + rlat_idx_bnds[:,1] = np.arange(len(rlat)) + 0.5 rlat_idx_bnds[0,0] = 0. - rlat_idx_bnds[len(rlat)-1,1] = len(rlat) + rlat_idx_bnds[len(rlat) - 1,1] = len(rlat) rlon_idx_bnds = np.zeros((len(rlon), 2)) - rlon_idx_bnds[:,0] = np.arange(len(rlon))-0.5 - rlon_idx_bnds[:,1] = np.arange(len(rlon))+0.5 + rlon_idx_bnds[:,0] = np.arange(len(rlon)) - 0.5 + rlon_idx_bnds[:,1] = np.arange(len(rlon)) + 0.5 # Calculate latitude/longitude vertices by interpolation lat_vertices = [] From 9cc9c78b2cb91a4b0706b80e9956f222d4df35f5 Mon Sep 17 00:00:00 2001 From: Nathan Gillett Date: Thu, 21 May 2020 14:13:48 -0700 Subject: [PATCH 07/20] Further syntax corrections to cmor/_fixes/common.py --- esmvalcore/cmor/_fixes/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index 76a2f9cea7..98e5dd5f5c 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -126,13 +126,13 @@ def fix_data(self, cube): #Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). rlat_idx_bnds = np.zeros((len(rlat), 2)) - rlat_idx_bnds[:,0] = np.arange(len(rlat)) - 0.5 - rlat_idx_bnds[:,1] = np.arange(len(rlat)) + 0.5 - rlat_idx_bnds[0,0] = 0. - rlat_idx_bnds[len(rlat) - 1,1] = len(rlat) + rlat_idx_bnds[:, 0] = np.arange(len(rlat))-0.5 + rlat_idx_bnds[:, 1] = np.arange(len(rlat))+0.5 + rlat_idx_bnds[0, 0] = 0. + rlat_idx_bnds[len(rlat)-1, 1] = len(rlat) rlon_idx_bnds = np.zeros((len(rlon), 2)) - rlon_idx_bnds[:,0] = np.arange(len(rlon)) - 0.5 - rlon_idx_bnds[:,1] = np.arange(len(rlon)) + 0.5 + rlon_idx_bnds[:, 0] = np.arange(len(rlon))-0.5 + rlon_idx_bnds[:, 1] = np.arange(len(rlon))+0.5 # Calculate latitude/longitude vertices by interpolation lat_vertices = [] From 502c12ab06608b45d5828bd8cae4d68b53d5485d Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2020 12:09:13 +0100 Subject: [PATCH 08/20] fixed test --- .../cmor/_fixes/cmip6/test_cesm2.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 0473cba025..6d3b53e6f3 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -238,8 +238,29 @@ def test_clw_fix(): @pytest.fixture def tas_cubes(): """Cubes to test fixes for ``tas``.""" - ta_cube = iris.cube.Cube([1.0], var_name='ta') - tas_cube = iris.cube.Cube([3.0], var_name='tas') + time_coord = iris.coords.DimCoord( + [0.0, 1.0], var_name='time', standard_name='time', + units='days since 1850-01-01 00:00:00') + lat_coord = iris.coords.DimCoord( + [0.0, 1.0], var_name='lat', standard_name='latitude', units='degrees') + lon_coord = iris.coords.DimCoord( + [0.0, 1.0], var_name='lon', standard_name='longitude', units='degrees') + coord_specs = [ + (time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ] + ta_cube = iris.cube.Cube( + np.ones((2, 2, 2)), + var_name='ta', + dim_coords_and_dims=coord_specs, + ) + tas_cube = iris.cube.Cube( + np.ones((2, 2, 2)), + var_name='tas', + dim_coords_and_dims=coord_specs, + ) + return iris.cube.CubeList([ta_cube, tas_cube]) @@ -265,6 +286,8 @@ def test_tas_fix_metadata(tas_cubes): out_cubes = fix.fix_metadata(tas_cubes) assert out_cubes is tas_cubes for cube in out_cubes: + assert cube.coord("longitude").has_bounds() + assert cube.coord("latitude").has_bounds() if cube.var_name == 'tas': coord = cube.coord('height') assert coord == height_coord From 210dc6cc59c689452992f93b780659253ae13aa9 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2020 12:09:41 +0100 Subject: [PATCH 09/20] fixed style --- esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py b/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py index 472789dad6..e084fbebb5 100644 --- a/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py +++ b/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py @@ -39,6 +39,7 @@ def fix_metadata(self, cubes): lon_coord.circular = False return cubes + class Siconc(BaseTos): """Fixes for siconc.""" From d0037b90d3a87716b7353e3c60a719d4ad9f1961 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2020 12:10:00 +0100 Subject: [PATCH 10/20] fixed style --- esmvalcore/cmor/_fixes/common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index 98e5dd5f5c..1a81ea8aa8 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -6,6 +6,7 @@ from .fix import Fix from .shared import add_plev_from_altitude, fix_bounds + class ClFixHybridHeightCoord(Fix): """Fixes for ``cl`` regarding hybrid sigma height coordinates.""" @@ -124,7 +125,7 @@ def fix_data(self, cube): rlat = cube.coord('grid_latitude').points rlon = cube.coord('grid_longitude').points - #Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). + # Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). rlat_idx_bnds = np.zeros((len(rlat), 2)) rlat_idx_bnds[:, 0] = np.arange(len(rlat))-0.5 rlat_idx_bnds[:, 1] = np.arange(len(rlat))+0.5 @@ -183,8 +184,8 @@ def fix_metadata(self, cubes): lon_coord.var_name = 'j' lon_coord.units = '1' lon_coord.circular = False - #FGOALS-g3 data contain latitude and longitude data set to - #>1e30 in some places. Set to 0. to avoid problem in check.py. + # FGOALS-g3 data contain latitude and longitude data set to + # >1e30 in some places. Set to 0. to avoid problem in check.py. cube.coord('latitude').points[cube.coord('latitude').points > 1000.]\ = 0. cube.coord('longitude').points[cube.coord('longitude').points > 1000.]\ From 662464c162a8f53d589d5a6c92b874930e4d401b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 27 May 2020 12:10:21 +0100 Subject: [PATCH 11/20] fixed fix --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index e30cb8c01e..a87311e40e 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -127,9 +127,13 @@ def fix_metadata(self, cubes): for cube in cubes: latitude = cube.coord('latitude') + if latitude.bounds is None: + latitude.guess_bounds() latitude.bounds = latitude.bounds.astype(np.float64) latitude.bounds = np.round(latitude.bounds, 4) longitude = cube.coord('longitude') + if longitude.bounds is None: + longitude.guess_bounds() longitude.bounds = longitude.bounds.astype(np.float64) longitude.bounds = np.round(longitude.bounds, 4) @@ -177,6 +181,7 @@ def fix_metadata(self, cubes): add_scalar_typesea_coord(cube) return cubes + class Tos(Fix): """Fixes for tos.""" From 8d7b29f0d625a3d20f0842f0ed3df2d7d5be8e1a Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 11:15:22 +0100 Subject: [PATCH 12/20] Bouwe suggestion Co-authored-by: Bouwe Andela --- tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py index 573faf3c51..b9523e0d4e 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py @@ -16,7 +16,7 @@ def siconc_file(tmp_path): """Create netcdf file with similar issues as ``cl``.""" nc_path = os.path.join(tmp_path, 'gfdl_esm4_siconc.nc') - dataset = Dataset(nc_path, mode='w') +with Dataset(nc_path, mode='w') as dataset: dataset.createDimension('time', size=1) dataset.createDimension('lat', size=1) dataset.createDimension('lon', size=1) @@ -39,8 +39,6 @@ def siconc_file(tmp_path): dataset.variables['siconc'][:] = 22. dataset.variables['siconc'].standard_name = 'sea_ice_area_fraction' dataset.variables['siconc'].units = '%' - - dataset.close() return nc_path From e0406288e562b9bad8228a3ac45a3e36779e99bd Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 11:19:45 +0100 Subject: [PATCH 13/20] correct syntax --- .../cmor/_fixes/cmip6/test_gfdl_esm4.py | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py index b9523e0d4e..ecc8f01794 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_esm4.py @@ -16,29 +16,30 @@ def siconc_file(tmp_path): """Create netcdf file with similar issues as ``cl``.""" nc_path = os.path.join(tmp_path, 'gfdl_esm4_siconc.nc') -with Dataset(nc_path, mode='w') as dataset: - dataset.createDimension('time', size=1) - dataset.createDimension('lat', size=1) - dataset.createDimension('lon', size=1) + with Dataset(nc_path, mode='w') as dataset: + dataset.createDimension('time', size=1) + dataset.createDimension('lat', size=1) + dataset.createDimension('lon', size=1) + + # Dimensional variables + dataset.createVariable('time', np.float64, dimensions=('time',)) + dataset.createVariable('lat', np.float64, dimensions=('lat',)) + dataset.createVariable('lon', np.float64, dimensions=('lon',)) + dataset.variables['time'][:] = [0.0] + dataset.variables['time'].standard_name = 'time' + dataset.variables['time'].units = 'days since 6543-2-1' + dataset.variables['lat'][:] = [-30.0] + dataset.variables['lat'].standard_name = 'latitude' + dataset.variables['lat'].units = 'degrees_north' + dataset.variables['lon'][:] = [30.0] + dataset.variables['lon'].standard_name = 'longitude' + dataset.variables['lon'].units = 'degrees_east' + dataset.createVariable('siconc', np.float64, + dimensions=('time', 'lat', 'lon')) + dataset.variables['siconc'][:] = 22. + dataset.variables['siconc'].standard_name = 'sea_ice_area_fraction' + dataset.variables['siconc'].units = '%' - # Dimensional variables - dataset.createVariable('time', np.float64, dimensions=('time',)) - dataset.createVariable('lat', np.float64, dimensions=('lat',)) - dataset.createVariable('lon', np.float64, dimensions=('lon',)) - dataset.variables['time'][:] = [0.0] - dataset.variables['time'].standard_name = 'time' - dataset.variables['time'].units = 'days since 6543-2-1' - dataset.variables['lat'][:] = [-30.0] - dataset.variables['lat'].standard_name = 'latitude' - dataset.variables['lat'].units = 'degrees_north' - dataset.variables['lon'][:] = [30.0] - dataset.variables['lon'].standard_name = 'longitude' - dataset.variables['lon'].units = 'degrees_east' - dataset.createVariable('siconc', np.float64, - dimensions=('time', 'lat', 'lon')) - dataset.variables['siconc'][:] = 22. - dataset.variables['siconc'].standard_name = 'sea_ice_area_fraction' - dataset.variables['siconc'].units = '%' return nc_path From dd8979490f287fa686329eb712921cae93855d99 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 11:43:52 +0100 Subject: [PATCH 14/20] siconc test --- .../cmor/_fixes/cmip6/test_bcc_csm2_mr.py | 79 ++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py b/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py index d6081042ef..4316bbc17a 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py +++ b/tests/integration/cmor/_fixes/cmip6/test_bcc_csm2_mr.py @@ -3,7 +3,8 @@ import iris -from esmvalcore.cmor._fixes.cmip6.bcc_csm2_mr import Cl, Cli, Clw, Tos +from esmvalcore.cmor._fixes.cmip6.bcc_csm2_mr import (Cl, Cli, + Clw, Tos, Siconc) from esmvalcore.cmor._fixes.common import ClFixHybridPressureCoord from esmvalcore.cmor._fixes.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -48,6 +49,12 @@ def test_get_tos_fix(): assert fix == [Tos(None)] +def test_get_siconc_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CMIP6', 'BCC-CSM2-MR', 'Omon', 'siconc') + assert fix == [Siconc(None)] + + @unittest.mock.patch( 'esmvalcore.cmor._fixes.cmip6.bcc_csm2_mr.BaseTos.fix_data', autospec=True) @@ -58,6 +65,16 @@ def test_tos_fix_data(mock_base_fix_data): mock_base_fix_data.assert_called_once_with(fix, 'cubes') +@unittest.mock.patch( + 'esmvalcore.cmor._fixes.cmip6.bcc_csm2_mr.BaseTos.fix_data', + autospec=True) +def test_siconc_fix_data(mock_base_fix_data): + """Test ``fix_data`` for ``siconc``.""" + fix = Siconc(None) + fix.fix_data('cubes') + mock_base_fix_data.assert_called_once_with(fix, 'cubes') + + def test_tos_fix_metadata(): """Test ``fix_metadata`` for ``tos``.""" grid_lat = iris.coords.DimCoord([1.0], @@ -115,3 +132,63 @@ def test_tos_fix_metadata(): assert grid_lon.standard_name is None assert grid_lon.units == '1' assert not grid_lon.circular + + +def test_siconc_fix_metadata(): + """Test ``fix_metadata`` for ``tos``.""" + grid_lat = iris.coords.DimCoord([1.0], + var_name='lat', + standard_name='latitude', + long_name='latitude', + units='degrees_north', + attributes={'1D': '1'}) + grid_lon = iris.coords.DimCoord([1.0], + var_name='lon', + standard_name='longitude', + long_name='longitude', + units='degrees_east', + circular=True, + attributes={'1D': '1'}) + latitude = iris.coords.AuxCoord([[0.0]], + var_name='lat', + standard_name='latitude', + long_name='latitude', + units='degrees_north') + longitude = iris.coords.AuxCoord([[0]], + var_name='lon', + standard_name='longitude', + long_name='longitude', + units='degrees_east') + + cube = iris.cube.Cube( + [[[0.0]]], + var_name='siconc', + long_name='sea_ice_area_fraction', + dim_coords_and_dims=[(grid_lat.copy(), 1), (grid_lon.copy(), 2)], + aux_coords_and_dims=[(latitude.copy(), (1, 2)), + (longitude.copy(), (1, 2))], + ) + cubes = iris.cube.CubeList([cube, iris.cube.Cube(0.0)]) + vardef = get_var_info('CMIP6', 'SImon', 'siconc') + fix = Siconc(vardef) + fixed_cubes = fix.fix_metadata(cubes) + siconc_cube = fixed_cubes.extract_strict('sea_ice_area_fraction') + + # No duplicates anymore + assert len(siconc_cube.coords('latitude')) == 1 + assert len(siconc_cube.coords('longitude')) == 1 + + # Latitude + grid_lat = siconc_cube.coord('grid_latitude') + assert grid_lat.var_name == 'i' + assert grid_lat.long_name == 'grid_latitude' + assert grid_lat.standard_name is None + assert grid_lat.units == '1' + + # Longitude + grid_lon = siconc_cube.coord('grid_longitude') + assert grid_lon.var_name == 'j' + assert grid_lon.long_name == 'grid_longitude' + assert grid_lon.standard_name is None + assert grid_lon.units == '1' + assert not grid_lon.circular From 0d265881d6ebadfac1cd460922fd2f0a16a3703c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 12:06:14 +0100 Subject: [PATCH 15/20] tos test --- .../cmor/_fixes/cmip6/test_cesm2.py | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index c238e815b7..3aabe46416 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -9,7 +9,7 @@ from cf_units import Unit from netCDF4 import Dataset -from esmvalcore.cmor._fixes.cmip6.cesm2 import Cl, Cli, Clw, Tas +from esmvalcore.cmor._fixes.cmip6.cesm2 import Cl, Cli, Clw, Tas, Tos from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info @@ -267,12 +267,44 @@ def tas_cubes(): return iris.cube.CubeList([ta_cube, tas_cube]) +@pytest.fixture +def tos_cubes(): + """Cubes to test fixes for ``tos``.""" + time_coord = iris.coords.DimCoord( + [0.0004, 1.09776], var_name='time', standard_name='time', + units='days since 1850-01-01 00:00:00') + lat_coord = iris.coords.DimCoord( + [0.0, 1.0], var_name='lat', standard_name='latitude', units='degrees') + lon_coord = iris.coords.DimCoord( + [0.0, 1.0], var_name='lon', standard_name='longitude', units='degrees') + coord_specs = [ + (time_coord, 0), + (lat_coord, 1), + (lon_coord, 2), + ] + tos_cube = iris.cube.Cube( + np.ones((2, 2, 2)), + var_name='tos', + dim_coords_and_dims=coord_specs, + ) + tos_cube.attributes = {} + tos_cube.attributes['mipTable'] = 'Omon' + + return iris.cube.CubeList([tos_cube]) + + def test_get_tas_fix(): """Test getting of fix.""" fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'tas') assert fix == [Tas(None)] +def test_get_tos_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CMIP6', 'CESM2', 'Amon', 'tos') + assert fix == [Tos(None)] + + def test_tas_fix_metadata(tas_cubes): """Test ``fix_metadata`` for ``tas``.""" for cube in tas_cubes: @@ -297,3 +329,13 @@ def test_tas_fix_metadata(tas_cubes): else: with pytest.raises(iris.exceptions.CoordinateNotFoundError): cube.coord('height') + + +def test_tos_fix_metadata(tos_cubes): + """Test ``fix_metadata`` for ``tos``.""" + vardef = get_var_info('CMIP6', 'Omon', 'tos') + fix = Tos(vardef) + out_cubes = fix.fix_metadata(tos_cubes) + assert out_cubes is tos_cubes + for cube in out_cubes: + np.testing.assert_equal(cube.coord("time").points, [0., 1.1]) From b20c11ffb236d54146f360f03da0cf5f85b55344 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 12:20:16 +0100 Subject: [PATCH 16/20] context manager --- .../cmor/_fixes/cmip6/test_cesm2.py | 123 +++++++++--------- 1 file changed, 63 insertions(+), 60 deletions(-) diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 3aabe46416..20d2a351ab 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -18,67 +18,70 @@ def cl_file(tmp_path): """Create netcdf file with similar issues as ``cl``.""" nc_path = os.path.join(tmp_path, 'cesm2_cl.nc') - dataset = Dataset(nc_path, mode='w') - dataset.createDimension('time', size=1) - dataset.createDimension('lev', size=2) - dataset.createDimension('lat', size=3) - dataset.createDimension('lon', size=4) - dataset.createDimension('bnds', size=2) - - # Dimensional variables - dataset.createVariable('time', np.float64, dimensions=('time',)) - dataset.createVariable('lev', np.float64, dimensions=('lev',)) - dataset.createVariable('lev_bnds', np.float64, dimensions=('lev', 'bnds')) - dataset.createVariable('lat', np.float64, dimensions=('lat',)) - dataset.createVariable('lon', np.float64, dimensions=('lon',)) - dataset.variables['time'][:] = [0.0] - dataset.variables['time'].standard_name = 'time' - dataset.variables['time'].units = 'days since 6543-2-1' - dataset.variables['lev'][:] = [1.0, 2.0] - dataset.variables['lev'].bounds = 'lev_bnds' - dataset.variables['lev'].units = 'hPa' - dataset.variables['lev_bnds'][:] = [[0.5, 1.5], [1.5, 3.0]] - dataset.variables['lev_bnds'].standard_name = ( - 'atmosphere_hybrid_sigma_pressure_coordinate') - dataset.variables['lev_bnds'].units = '1' - dataset.variables['lev_bnds'].formula_terms = ( - 'p0: p0 a: a_bnds b: b_bnds ps: ps') - dataset.variables['lat'][:] = [-30.0, 0.0, 30.0] - dataset.variables['lat'].standard_name = 'latitude' - dataset.variables['lat'].units = 'degrees_north' - dataset.variables['lon'][:] = [30.0, 60.0, 90.0, 120.0] - dataset.variables['lon'].standard_name = 'longitude' - dataset.variables['lon'].units = 'degrees_east' - - # Coordinates for derivation of pressure coordinate - dataset.createVariable('a', np.float64, dimensions=('lev',)) - dataset.createVariable('a_bnds', np.float64, dimensions=('lev', 'bnds')) - dataset.createVariable('b', np.float64, dimensions=('lev',)) - dataset.createVariable('b_bnds', np.float64, dimensions=('lev', 'bnds')) - dataset.createVariable('p0', np.float64, dimensions=()) - dataset.createVariable('ps', np.float64, - dimensions=('time', 'lat', 'lon')) - dataset.variables['a'][:] = [1.0, 2.0] - dataset.variables['a'].bounds = 'a_bnds' - dataset.variables['a_bnds'][:] = [[1.5, 3.0], [0.0, 1.5]] # intended - dataset.variables['b'][:] = [0.0, 1.0] - dataset.variables['b'].bounds = 'b_bnds' - dataset.variables['b_bnds'][:] = [[0.5, 2.0], [-1.0, 0.5]] # intended - dataset.variables['p0'][:] = 1.0 - dataset.variables['p0'].units = 'Pa' - dataset.variables['ps'][:] = np.arange(1 * 3 * 4).reshape(1, 3, 4) - dataset.variables['ps'].standard_name = 'surface_air_pressure' - dataset.variables['ps'].units = 'Pa' - - # Cl variable - dataset.createVariable('cl', np.float32, - dimensions=('time', 'lev', 'lat', 'lon')) - dataset.variables['cl'][:] = np.full((1, 2, 3, 4), 0.0, dtype=np.float32) - dataset.variables['cl'].standard_name = ( - 'cloud_area_fraction_in_atmosphere_layer') - dataset.variables['cl'].units = '%' + with Dataset(nc_path, mode='w') as dataset: + dataset.createDimension('time', size=1) + dataset.createDimension('lev', size=2) + dataset.createDimension('lat', size=3) + dataset.createDimension('lon', size=4) + dataset.createDimension('bnds', size=2) + + # Dimensional variables + dataset.createVariable('time', np.float64, dimensions=('time',)) + dataset.createVariable('lev', np.float64, dimensions=('lev',)) + dataset.createVariable('lev_bnds', np.float64, dimensions=('lev', + 'bnds')) + dataset.createVariable('lat', np.float64, dimensions=('lat',)) + dataset.createVariable('lon', np.float64, dimensions=('lon',)) + dataset.variables['time'][:] = [0.0] + dataset.variables['time'].standard_name = 'time' + dataset.variables['time'].units = 'days since 6543-2-1' + dataset.variables['lev'][:] = [1.0, 2.0] + dataset.variables['lev'].bounds = 'lev_bnds' + dataset.variables['lev'].units = 'hPa' + dataset.variables['lev_bnds'][:] = [[0.5, 1.5], [1.5, 3.0]] + dataset.variables['lev_bnds'].standard_name = ( + 'atmosphere_hybrid_sigma_pressure_coordinate') + dataset.variables['lev_bnds'].units = '1' + dataset.variables['lev_bnds'].formula_terms = ( + 'p0: p0 a: a_bnds b: b_bnds ps: ps') + dataset.variables['lat'][:] = [-30.0, 0.0, 30.0] + dataset.variables['lat'].standard_name = 'latitude' + dataset.variables['lat'].units = 'degrees_north' + dataset.variables['lon'][:] = [30.0, 60.0, 90.0, 120.0] + dataset.variables['lon'].standard_name = 'longitude' + dataset.variables['lon'].units = 'degrees_east' + + # Coordinates for derivation of pressure coordinate + dataset.createVariable('a', np.float64, dimensions=('lev',)) + dataset.createVariable('a_bnds', np.float64, dimensions=('lev', + 'bnds')) + dataset.createVariable('b', np.float64, dimensions=('lev',)) + dataset.createVariable('b_bnds', np.float64, dimensions=('lev', + 'bnds')) + dataset.createVariable('p0', np.float64, dimensions=()) + dataset.createVariable('ps', np.float64, + dimensions=('time', 'lat', 'lon')) + dataset.variables['a'][:] = [1.0, 2.0] + dataset.variables['a'].bounds = 'a_bnds' + dataset.variables['a_bnds'][:] = [[1.5, 3.0], [0.0, 1.5]] # intended + dataset.variables['b'][:] = [0.0, 1.0] + dataset.variables['b'].bounds = 'b_bnds' + dataset.variables['b_bnds'][:] = [[0.5, 2.0], [-1.0, 0.5]] # intended + dataset.variables['p0'][:] = 1.0 + dataset.variables['p0'].units = 'Pa' + dataset.variables['ps'][:] = np.arange(1 * 3 * 4).reshape(1, 3, 4) + dataset.variables['ps'].standard_name = 'surface_air_pressure' + dataset.variables['ps'].units = 'Pa' + + # Cl variable + dataset.createVariable('cl', np.float32, + dimensions=('time', 'lev', 'lat', 'lon')) + dataset.variables['cl'][:] = np.full((1, 2, 3, 4), + 0.0, dtype=np.float32) + dataset.variables['cl'].standard_name = ( + 'cloud_area_fraction_in_atmosphere_layer') + dataset.variables['cl'].units = '%' - dataset.close() return nc_path From f13b09a670eddbaa39be0d099f1590e72f5f495e Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 12:44:23 +0100 Subject: [PATCH 17/20] fixed codacy --- esmvalcore/cmor/_fixes/common.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/esmvalcore/cmor/_fixes/common.py b/esmvalcore/cmor/_fixes/common.py index 1a81ea8aa8..2e3165dc33 100644 --- a/esmvalcore/cmor/_fixes/common.py +++ b/esmvalcore/cmor/_fixes/common.py @@ -106,34 +106,36 @@ def fix_metadata(self, cubes): class OceanFixGrid(Fix): - """Fixes for tos, siconc in FGOALS-g3.""" def fix_data(self, cube): - """Fix data. + """ + Fix data. + Calculate missing latitude/longitude boundaries using interpolation. - Based on a similar fix for BCC-CSM2-MR + Based on a similar fix for BCC-CSM2-MR. + Parameters ---------- cube: iris.cube.Cube Input cube to fix. + Returns ------- iris.cube.Cube """ - rlat = cube.coord('grid_latitude').points rlon = cube.coord('grid_longitude').points # Guess coordinate bounds in rlat, rlon (following BCC-CSM2-MR-1). rlat_idx_bnds = np.zeros((len(rlat), 2)) - rlat_idx_bnds[:, 0] = np.arange(len(rlat))-0.5 - rlat_idx_bnds[:, 1] = np.arange(len(rlat))+0.5 + rlat_idx_bnds[:, 0] = np.arange(len(rlat)) - 0.5 + rlat_idx_bnds[:, 1] = np.arange(len(rlat)) + 0.5 rlat_idx_bnds[0, 0] = 0. - rlat_idx_bnds[len(rlat)-1, 1] = len(rlat) + rlat_idx_bnds[len(rlat) - 1, 1] = len(rlat) rlon_idx_bnds = np.zeros((len(rlon), 2)) - rlon_idx_bnds[:, 0] = np.arange(len(rlon))-0.5 - rlon_idx_bnds[:, 1] = np.arange(len(rlon))+0.5 + rlon_idx_bnds[:, 0] = np.arange(len(rlon)) - 0.5 + rlon_idx_bnds[:, 1] = np.arange(len(rlon)) + 0.5 # Calculate latitude/longitude vertices by interpolation lat_vertices = [] @@ -161,11 +163,14 @@ def fix_data(self, cube): return cube def fix_metadata(self, cubes): - """Rename ``var_name`` of 1D-``latitude`` and 1D-``longitude``. + """ + Rename ``var_name`` of 1D-``latitude`` and 1D-``longitude``. + Parameters ---------- cubes : iris.cube.CubeList Input cubes. + Returns ------- iris.cube.CubeList From 88f159489aaabcc9cdc8feb2a9466ceb874b0ada Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 12:44:30 +0100 Subject: [PATCH 18/20] fixed codacy --- esmvalcore/cmor/_fixes/cmip6/cesm2.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2.py index a87311e40e..e358354fd3 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2.py @@ -109,7 +109,9 @@ class Tas(Fix): """Fixes for tas.""" def fix_metadata(self, cubes): - """Add height (2m) coordinate. + """ + Add height (2m) coordinate. + Fix latitude_bounds and longitude_bounds data type and round to 4 d.p. Parameters @@ -186,7 +188,9 @@ class Tos(Fix): """Fixes for tos.""" def fix_metadata(self, cubes): - """Round times to 1 d.p. for monthly means. + """ + Round times to 1 d.p. for monthly means. + Required to get hist-GHG and ssp245-GHG Omon tos to concatenate. Parameters @@ -204,7 +208,7 @@ def fix_metadata(self, cubes): for cube in cubes: if cube.attributes['mipTable'] == 'Omon': cube.coord('time').points = \ - np.round(cube.coord('time').points, 1) + np.round(cube.coord('time').points, 1) return cubes From bde29a868391d53d451abfc86c6418372242bfb8 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 12:44:40 +0100 Subject: [PATCH 19/20] fixed codacy --- esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py index 5077a1a843..0e25c90fbc 100644 --- a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py +++ b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py @@ -7,7 +7,8 @@ class Siconc(Fix): """Fixes for siconc.""" def fix_metadata(self, cubes): - """Fix metadata. + """ + Fix metadata. Some coordinate points vary for different files of this dataset (for different time range). This fix removes these inaccuracies by rounding @@ -16,6 +17,7 @@ def fix_metadata(self, cubes): Parameters ---------- cubes: iris.cube.CubeList + Input cubes to fix. Returns ------- From 2f437c30bbd904c571753edc9524f1dc448f0794 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Sep 2020 14:20:20 +0100 Subject: [PATCH 20/20] added test for noresm --- .../cmor/_fixes/cmip6/test_noresm2_lm.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py diff --git a/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py b/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py new file mode 100644 index 0000000000..54a87dd849 --- /dev/null +++ b/tests/integration/cmor/_fixes/cmip6/test_noresm2_lm.py @@ -0,0 +1,80 @@ +"""Tests for the fixes of GFDL-ESM4.""" +import os + +import iris +import numpy as np +import pytest +from netCDF4 import Dataset + +from esmvalcore.cmor._fixes.cmip6.noresm2_lm import Siconc +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info + + +@pytest.fixture +def siconc_file(tmp_path): + """Create netcdf file.""" + nc_path = os.path.join(tmp_path, 'noresm2_lm_siconc.nc') + with Dataset(nc_path, mode='w') as dataset: + dataset.createDimension('time', size=1) + dataset.createDimension('lat', size=1) + dataset.createDimension('lon', size=1) + + # Dimensional variables + dataset.createVariable('time', np.float64, dimensions=('time',)) + dataset.createVariable('lat', np.float64, dimensions=('lat',)) + dataset.createVariable('lon', np.float64, dimensions=('lon',)) + dataset.variables['time'][:] = [0.2] + dataset.variables['time'].standard_name = 'time' + dataset.variables['time'].units = 'days since 1850-01-01' + dataset.variables['lat'][:] = [30.0] + dataset.variables['lat'].standard_name = 'latitude' + dataset.variables['lat'].units = 'degrees_north' + dataset.variables['lon'][:] = [30.0] + dataset.variables['lon'].standard_name = 'longitude' + dataset.variables['lon'].units = 'degrees_east' + dataset.createVariable('siconc', np.float64, + dimensions=('time', 'lat', 'lon')) + dataset.variables['siconc'][:] = 22. + dataset.variables['siconc'].standard_name = 'sea_ice_area_fraction' + dataset.variables['siconc'].units = '%' + + return nc_path + + +def test_get_siconc_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CMIP6', 'NorESM2-LM', 'SImon', 'siconc') + assert fix == [Siconc(None)] + + +def test_siconc_fix_metadata(siconc_file): + """Test ``fix_metadata``.""" + print(siconc_file) + cubes = iris.load(siconc_file) + for cube in cubes: + cube.coord("latitude").bounds = [28.9956255, 32.3445677] + cube.coord("longitude").bounds = [28.9956255, 32.3445677] + + # Raw cubes + assert len(cubes) == 1 + siconc_cube = cubes[0] + assert siconc_cube.var_name == "siconc" + + # Extract siconc cube + siconc_cube = cubes.extract_strict('sea_ice_area_fraction') + assert not siconc_cube.coords('typesi') + + # Apply fix + vardef = get_var_info('CMIP6', 'SImon', 'siconc') + fix = Siconc(vardef) + fixed_cubes = fix.fix_metadata(cubes) + assert len(fixed_cubes) == 1 + fixed_siconc_cube = fixed_cubes.extract_strict( + 'sea_ice_area_fraction') + fixed_lon = fixed_siconc_cube.coord('longitude') + fixed_lat = fixed_siconc_cube.coord('latitude') + assert fixed_lon.bounds is not None + assert fixed_lat.bounds is not None + np.testing.assert_equal(fixed_lon.bounds, [[28.9956, 32.3446]]) + np.testing.assert_equal(fixed_lat.bounds, [[28.9956, 32.3446]])