From c12a02da229908929a9e156faae4b8977544bcf7 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 28 Jan 2021 16:59:12 +0100 Subject: [PATCH 01/14] Implement fixes for monthly MSWEP data --- esmvalcore/cmor/_fixes/native6/mswep.py | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 esmvalcore/cmor/_fixes/native6/mswep.py diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py new file mode 100644 index 0000000000..d5128d225c --- /dev/null +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -0,0 +1,54 @@ +"""Fixes for MSWEP.""" +from datetime import datetime + +import cf_units +from cf_units import Unit + +from ..fix import Fix + + +def fix_time(cube): + """Fix time coordinates. + + Convert from months since 1899 to days since 1850 as per CMOR + standard. + """ + time_coord = cube.coord('time') + origin = time_coord.units.origin + + origin_year, origin_month = [ + int(val) for val in origin.split()[2].split('-') + ] + + dates = [] + + for time_point in time_coord.points: + new_year = origin_year + (origin_month - 1 + time_point) // 12 + new_month = (origin_month - 1 + time_point) % 12 + 1 + dates.append(datetime(int(new_year), int(new_month), 15)) + + t_unit = cf_units.Unit("days since 1850-01-01", calendar="standard") + + cube.coord('time').points = t_unit.date2num(dates) + cube.coord('time').units = t_unit + + +class Pr(Fix): + """Fixes for pr.""" + def fix_metadata(self, cubes): + """Fix metadata.""" + for cube in cubes: + cube.var_name = self.vardef.short_name + cube.standard_name = self.vardef.standard_name + cube.long_name = self.vardef.long_name + + fix_time(cube) + self._fix_units(cube) + + return cubes + + def _fix_units(self, cube): + """Convert units from mm/month to kg m-3 s-1 units.""" + cube.units = Unit(self.vardef.units) + # divide by number of seconds in a month + cube.data /= 60 * 60 * 24 * 30 From 725c94f471955dc25e95345422632c4e4e17a588 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Thu, 28 Jan 2021 17:18:15 +0100 Subject: [PATCH 02/14] Use lazy operation for faster loading of data --- esmvalcore/cmor/_fixes/native6/mswep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index d5128d225c..6947ca1d88 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -51,4 +51,4 @@ def _fix_units(self, cube): """Convert units from mm/month to kg m-3 s-1 units.""" cube.units = Unit(self.vardef.units) # divide by number of seconds in a month - cube.data /= 60 * 60 * 24 * 30 + cube.data = cube.core_data() / 60 * 60 * 24 * 30 From 8bb4f73515251671ee049d21db62fe0898855a21 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 29 Jan 2021 09:00:28 +0100 Subject: [PATCH 03/14] Add fixes for daily data --- esmvalcore/cmor/_fixes/native6/mswep.py | 37 +++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index 6947ca1d88..ca5bed4be7 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -7,10 +7,10 @@ from ..fix import Fix -def fix_time(cube): - """Fix time coordinates. +def fix_time_month(cube): + """Fix time coordinates for monthly values. - Convert from months since 1899 to days since 1850 as per CMOR + Convert from months since 1899-12 to days since 1850 as per CMOR standard. """ time_coord = cube.coord('time') @@ -33,22 +33,49 @@ def fix_time(cube): cube.coord('time').units = t_unit +def fix_time_day(cube): + """Fix time coordinates for monthly values. + + Convert from days since 1899-12-31 to days since 1850 as per CMOR + standard. + """ + time_coord = cube.coord('time') + time_coord.convert_units('days since 1850-1-1 00:00:00.0') + + class Pr(Fix): """Fixes for pr.""" def fix_metadata(self, cubes): """Fix metadata.""" + frequency = self.vardef.frequency + + if frequency == 'day': + fix_time = fix_time_day + fix_units = self._fix_units_day + elif frequency == 'mon': + fix_units = self._fix_units_month + fix_time = fix_time_month + else: + raise ValueError(f'Cannot fix frequency: {frequency!r}') + for cube in cubes: cube.var_name = self.vardef.short_name cube.standard_name = self.vardef.standard_name cube.long_name = self.vardef.long_name + fix_units(cube) fix_time(cube) - self._fix_units(cube) return cubes - def _fix_units(self, cube): + def _fix_units_month(self, cube): """Convert units from mm/month to kg m-3 s-1 units.""" cube.units = Unit(self.vardef.units) # divide by number of seconds in a month cube.data = cube.core_data() / 60 * 60 * 24 * 30 + + def _fix_units_day(self, cube): + """Convert units from mm/month to kg m-3 s-1 units.""" + cube.units = Unit(self.vardef.units) + # divide by number of seconds in a day + cube.data = cube.core_data() / 60 * 60 * 24 From 380b96db59164b8948ddee7cd250cb7b43ce7954 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 29 Jan 2021 11:17:13 +0100 Subject: [PATCH 04/14] Fix longitude, bounds, and re-organize code --- esmvalcore/cmor/_fixes/native6/mswep.py | 75 +++++++++++++++++++------ 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index ca5bed4be7..427f6ebda0 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -2,6 +2,7 @@ from datetime import datetime import cf_units +import numpy as np from cf_units import Unit from ..fix import Fix @@ -47,35 +48,77 @@ class Pr(Fix): """Fixes for pr.""" def fix_metadata(self, cubes): """Fix metadata.""" + self._init_frequency_specific_fixes() + + for cube in cubes: + self._fix_names(cube) + self._fix_units(cube) + self._fix_time(cube) + self._fix_longitude(cube) + self._fix_bounds(cube) + + return cubes + + def _init_frequency_specific_fixes(self): + """Set frequency specific fixes (day/mon).""" frequency = self.vardef.frequency if frequency == 'day': - fix_time = fix_time_day - fix_units = self._fix_units_day + self._fix_units = self._fix_units_day + self._fix_time = fix_time_day elif frequency == 'mon': - fix_units = self._fix_units_month - fix_time = fix_time_month + self._fix_units = self._fix_units_month + self._fix_time = fix_time_month else: raise ValueError(f'Cannot fix frequency: {frequency!r}') - for cube in cubes: - cube.var_name = self.vardef.short_name - cube.standard_name = self.vardef.standard_name - cube.long_name = self.vardef.long_name - - fix_units(cube) - fix_time(cube) - - return cubes - def _fix_units_month(self, cube): - """Convert units from mm/month to kg m-3 s-1 units.""" + """Convert units from mm/month to kg m-2 s-1 units.""" cube.units = Unit(self.vardef.units) # divide by number of seconds in a month cube.data = cube.core_data() / 60 * 60 * 24 * 30 def _fix_units_day(self, cube): - """Convert units from mm/month to kg m-3 s-1 units.""" + """Convert units from mm/month to kg m-2 s-1 units.""" cube.units = Unit(self.vardef.units) # divide by number of seconds in a day cube.data = cube.core_data() / 60 * 60 * 24 + + def _fix_longitude(self, cube): + """Fix longitude coordinate from -180:180 to 0:360.""" + lon = cube.coord(axis='X') + + # roll data because iris forces `lon.points` to be strictly monotonic. + shift = np.sum(lon.points < 0) + points = np.roll(lon.points, shift) % 360 + cube.data = np.roll(cube.core_data(), shift, axis=-1) + + lon.points = points + + def _fix_bounds(self, cube): + """Add bounds to coords.""" + coord_defs = tuple(coord_def + for coord_def in self.vardef.coordinates.values()) + + for coord_def in coord_defs: + if not coord_def.must_have_bounds == 'yes': + continue + + coord = cube.coord(axis=coord_def.axis) + + if coord.bounds is None: + coord.guess_bounds() + + def _fix_names(self, cube): + """Fix miscellaneous.""" + cube.var_name = self.vardef.short_name + cube.standard_name = self.vardef.standard_name + cube.long_name = self.vardef.long_name + + coord_defs = tuple(coord_def + for coord_def in self.vardef.coordinates.values()) + + for coord_def in coord_defs: + coord = cube.coord(axis=coord_def.axis) + if not coord.long_name: + coord.long_name = coord_def.long_name From 44128b1e4ad1f6eb1c3c9e9c6374606abc15b35d Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Fri, 29 Jan 2021 12:14:58 +0100 Subject: [PATCH 05/14] Add tests for MSWEP data [WIP] --- .../cmor/_fixes/native6/mswep_day.nc | Bin 0 -> 8596 bytes .../cmor/_fixes/native6/mswep_month.nc | Bin 0 -> 8596 bytes .../cmor/_fixes/native6/test_mswep.py | 89 ++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 tests/integration/cmor/_fixes/native6/mswep_day.nc create mode 100644 tests/integration/cmor/_fixes/native6/mswep_month.nc create mode 100644 tests/integration/cmor/_fixes/native6/test_mswep.py diff --git a/tests/integration/cmor/_fixes/native6/mswep_day.nc b/tests/integration/cmor/_fixes/native6/mswep_day.nc new file mode 100644 index 0000000000000000000000000000000000000000..bbdbcee1cb922c50fd9198e6edd1864584252cd0 GIT binary patch literal 8596 zcmeGhZERCj^uD#*N_k^L2L{D`8ofjVzr$de#a(O*%wR1?sXnWUsTqMloO}?Ld8B2pOX|YswSRI~8-7xhbx2ec zA&=G-aWi_j+8bsrzBa)ACB8%*pP8cZgl-Tg79K8Fvd#~3#>R03Xzkb7aYv=pD!5Q-zyCQZXC02P%y z4qsBDr4ALm1;b41-oVyv+u8yhZJhycd;50&QCk_fhT74+f|WA0R9;J(DX>h)LhU3A z4ORn;)dOV(Uy)8_bX)>V98f#%Gp!iS>!aDckuvwhKmK+o+S?~~>~~Q-?3!40k3m2s zY76EWMLKoRz!>l#;4GQ(@^|Of{w9*`qq1bf4M8C6_=jY>yi)L!>D7}h1EwKt6sVYN z_5XIVJ%x@^2uhQ!bnh}*^B%)o1NPrtm|wzls+^M88)!E)9nwNCB=uCm(zbhkteoPg z7E-w>epGDSFN(4iDT+hLq3%E%N4P3oZtN4qg7UbS=WUQ!%lk!5O^qT-?Z#oJ3yq}_ zbr0r6{fxCrN2q&L@0#jf-wwr+-%hbgl;u>`&OVu@6@3tlSY~1i;cQ9XFSa*TDEbniV7|)~V>DrOm1|Nyo?i zTIlOTxus6gSl%gG?xx=No?E>6he(_Beb)OnnNOG~2-WMo2TViwkJNk0VrpBpa%=!Q z`DO#Aax}AfYYfl3J0A!VA=xc$B$7a3P&Ha(L0;+?Zi- z+7%TKOlB;jEs>qf!{M|~H%7-SU2BV(1%3QLM)&S6y{ULN4kY%)c5iB!JqZ>vMmHDH zB^X=${+nKIK_AZ2ftDPM?#^d(dfqa4t)T@Q_&jnkHl^naU|q1I)#T@Hu*W2r-H}dtZXHTC;-{k!fs{!Bzf!P0Z->4EcO}&2&eWS!Jo{LbdpUIoDdO%z{MV%IS z1tv#~jLsu~Rk9tv6fOT)c3k%tEPcd}$?D&0==n}RuSY)!)eDxaBbYwq&*l9c?Sakh z0iT&xeWF_nJ<+T=rJL9e3Pn9wR-`Aovn}A;a>V0`-y!ed`g!O6RC@h{S{>yLs^PKW zTU@98V@KdVBM^@c_4AaK7bi+zYQ*$bOa)m^r^AEj=Ov@KZZ5sJk{l}Fa(+6Ih~6j# z&>swlPV#9?nE2&;Ilm}kXh^TTc@^m-N4@|R=_E(K{uk*aM|RgDo%mPDeIR>?zp;!? zO-qJ*nVi*S{3M6J^TQm;Cpp;O5Qt9vc;Q7LI`QMh7lG)-(_fI;EpkwfZDdElj({Bj zI|6nD>a_&0XqT@ZUk;_ycW8?=VUnOu8+8{ZrD-D(<`&HL8R0CmD&B> zIEJ<1k&#bvUFgl`6A`3Ug5LQP-P7@R!xKSwn6X@>63_KL7ec-Biy`;bv)qRL$1cKu zi5UyJJ>k=*)^~SyegSLe!pAOtS#kYze4f+i-#84q=7YWS&oARUc5z|HrG;O^q@#KI zGS|^O{Q>y#4C)C@1doO;E$ra_z4PABT*fC7&q&O2FACXnQpZd{j3XU048hwc@WBy* TZ@eP#T(`j9=OPTD>D9>JB8yrU literal 0 HcmV?d00001 diff --git a/tests/integration/cmor/_fixes/native6/mswep_month.nc b/tests/integration/cmor/_fixes/native6/mswep_month.nc new file mode 100644 index 0000000000000000000000000000000000000000..827c8b428d1deee17819253035c587ffe0b50237 GIT binary patch literal 8596 zcmeGhZERCj^uDbdU3q0g2MiP_A_*kh`mq^4bnf+aWy5{JhJw1Jl=gLdrh6^z8^Ng& zH3TG*#e@J`41eGU3G(4D=LZP^6Js)BNK7UKll6zj07m|3CV%jpd*A7rLXm)pk@RG* z=iPJ9Jzw{nbMMPOY-?HXbS!c(1iPKtSQUSzpF0*DTqYfftC`vPT7@;vW{r1XT1f7E zdl{PrlnTBg5{_%Q1eiFWa@=NG(Vx=#6Dd7xTsal3=?=8Diw(OWYKL7Ds~$22s6<|d zX-0t#@6s^@JO(&RX1x0S$@$kst{uqBHQWyb!lnO^Yv0Zm{A7CJT#JKg2rC83=Gv@( zJJ+5?M=1okxt6 zTT<1!U+mm_0(K-RcD)k% zP)$z%>DbTVMoRFg&`We(t(cmGo%chBN*lQ4&(+Ixo%IC0P9K9QMtoY_HeI+D_p zc-i0q!7PIlFRQq7C~oR?y@{a|98X&`y??;eqIE$dtqpFEYp#vCw-wjQ&fYD-jjNm! zM?7?M5nY0@+n2v<`nyP(QV9%JN=u z=@d1Z=G7O9>2Zxm0IMVf4@Jv8kQmh5X;X{2FIj>+JDGBK ztGyvU9ZG3oBOHG-XQOp- zqRga5Okc%Rkoj~vJcxeEViecSnGaWzLj|1APbU)5D=h(5jq{5idO`Auk1+9D-{o0; z(rewivUHMTeF0_ZB**&v&(cYb)m^i6;-6!MnCv0`$~-zXEi2r2$(ftSPjdJ-Kg^MQ zl7sCHf#}4K7hVLS6F**j5s02W{RNp#lY`c{h>8&?MxYphVg!m2C`Oba{h~QDe6V@) zSh5-A+}t$Y+?Ls+F5UB+f5*9sz{rJ+|I$R2e@D~)7afzwUa(ZK@g*yi$zu-n=B7jb z@g*(ls%wV>4Gkar+rlTnzA?3-;Rx6lQ8UB)K;Jy&(!@9L$RMjj85^xon)ZLB_KXjM z?Zax{NR7(axZ;>x4*zMMfwvBxQOB;#5A=<^5Bhzo<8TGMmq08&Qyr66LHBm`;?Nz? z@dK;}IB`JX_^gG;Eu4Hs(5GG!_`FwO#dn?|xY`BY6%+W@YXVRC1$L Date: Mon, 1 Feb 2021 13:31:56 +0100 Subject: [PATCH 06/14] Make data rolling more robust --- esmvalcore/cmor/_fixes/native6/mswep.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index 427f6ebda0..4c63053ad7 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -86,12 +86,15 @@ def _fix_units_day(self, cube): def _fix_longitude(self, cube): """Fix longitude coordinate from -180:180 to 0:360.""" + lon_axis = cube.coord_dims('longitude') lon = cube.coord(axis='X') + assert lon.is_monotonic(), "Data must be monotonic to fix longitude." + # roll data because iris forces `lon.points` to be strictly monotonic. shift = np.sum(lon.points < 0) - points = np.roll(lon.points, shift) % 360 - cube.data = np.roll(cube.core_data(), shift, axis=-1) + points = np.roll(lon.points, -shift) % 360 + cube.data = np.roll(cube.core_data(), -shift, axis=lon_axis) lon.points = points From dd1cab4fb64af30d2e9d63a60fa9f75acbc3befc Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 1 Feb 2021 13:33:13 +0100 Subject: [PATCH 07/14] Complete tests for daily/monthly data --- .../cmor/_fixes/native6/test_mswep.py | 63 ++++++++++++++++--- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/tests/integration/cmor/_fixes/native6/test_mswep.py b/tests/integration/cmor/_fixes/native6/test_mswep.py index 4c515789a2..2208c04f86 100644 --- a/tests/integration/cmor/_fixes/native6/test_mswep.py +++ b/tests/integration/cmor/_fixes/native6/test_mswep.py @@ -2,6 +2,7 @@ from pathlib import Path import iris +import numpy as np import pytest from esmvalcore.cmor._fixes.native6.mswep import ( @@ -54,36 +55,78 @@ def fix_day(): return Pr(vardef) -def test_fix_time_month(cube_month): - """Test `fix_time_month`.""" - fix_time_month(cube_month) +def test_fix_names(fix_month, cube_month): + """Test `Pr._fix_names`.""" + fix_month._fix_names(cube_month) + vardef = fix_month.vardef -def test_fix_time_day(cube_day): - """Test `fix_time_day`.""" - fix_time_day(cube_day) + assert cube_month.var_name == vardef.short_name + assert cube_month.long_name == vardef.long_name + assert cube_month.standard_name == vardef.standard_name + + coord_defs = tuple(coord_def for coord_def in vardef.coordinates.values()) + + for coord_def in coord_defs: + coord = cube_month.coord(axis=coord_def.axis) + assert coord.long_name == coord_def.long_name def test_fix_units_month(fix_month, cube_month): """Test `Pr._fix_units_month`.""" fix_month._fix_units_month(cube_month) + assert cube_month.units == fix_month.vardef.units def test_fix_units_day(fix_day, cube_day): """Test `Pr._fix_units_day`.""" fix_day._fix_units_day(cube_day) + assert cube_day.units == fix_day.vardef.units + + +def test_fix_time_month(cube_month): + """Test `fix_time_month`.""" + fix_time_month(cube_month) + + time = cube_month.coord('time') + assert time.units == 'days since 1850-01-01' + + +def test_fix_time_day(cube_day): + """Test `fix_time_day`.""" + fix_time_day(cube_day) + + time = cube_day.coord('time') + assert time.units == 'days since 1850-01-01' def test_fix_longitude(fix_month, cube_month): """Test `Pr._fix_longitude`.""" + unfixed_data = cube_month.data.copy() + unfixed_lon = cube_month.coord(axis='X') + shift = (unfixed_lon.points < 0).sum() + fix_month._fix_longitude(cube_month) + lon = cube_month.coord(axis='X') + + assert lon.is_monotonic + + coord_def = fix_month.vardef.coordinates['longitude'] + valid_min = float(coord_def.valid_min) + valid_max = float(coord_def.valid_max) + + assert lon.points.min() >= valid_min + assert lon.points.max() <= valid_max + + # make sure that data are rolled correctly along lon axis + assert np.all(unfixed_data[:, :, 0] == cube_month.data[:, :, -shift]) + def test_fix_bounds(fix_month, cube_month): """Test `Pr._fix_bounds`.""" fix_month._fix_bounds(cube_month) - -def test_fix_names(fix_month, cube_month): - """Test `Pr._fix_names`.""" - fix_month._fix_names(cube_month) + for axis in 'XYT': + coord = cube_month.coord(axis=axis) + assert coord.has_bounds() From a604dd50d7a100d4a17d8e6f6f6f1f952c5db986 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 1 Feb 2021 13:51:39 +0100 Subject: [PATCH 08/14] Add support for `3hr` frequency (same fixes as daily) --- esmvalcore/cmor/_fixes/native6/mswep.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index 4c63053ad7..35461ca355 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -63,14 +63,14 @@ def _init_frequency_specific_fixes(self): """Set frequency specific fixes (day/mon).""" frequency = self.vardef.frequency - if frequency == 'day': + if frequency in ('day', '3hr'): self._fix_units = self._fix_units_day self._fix_time = fix_time_day elif frequency == 'mon': self._fix_units = self._fix_units_month self._fix_time = fix_time_month else: - raise ValueError(f'Cannot fix frequency: {frequency!r}') + raise ValueError(f'No fixes for frequency: {frequency!r}') def _fix_units_month(self, cube): """Convert units from mm/month to kg m-2 s-1 units.""" From 4e54a7fbd5c1860a926d3f0f82f13c50c291280e Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 1 Feb 2021 14:12:30 +0100 Subject: [PATCH 09/14] Fix some codacy issues --- esmvalcore/cmor/_fixes/native6/mswep.py | 4 +++- tests/integration/cmor/_fixes/native6/test_mswep.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index 35461ca355..f972c75208 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -46,6 +46,7 @@ def fix_time_day(cube): class Pr(Fix): """Fixes for pr.""" + def fix_metadata(self, cubes): """Fix metadata.""" self._init_frequency_specific_fixes() @@ -89,7 +90,8 @@ def _fix_longitude(self, cube): lon_axis = cube.coord_dims('longitude') lon = cube.coord(axis='X') - assert lon.is_monotonic(), "Data must be monotonic to fix longitude." + if not lon.is_monotonic(): + raise ValueError("Data must be monotonic to fix longitude.") # roll data because iris forces `lon.points` to be strictly monotonic. shift = np.sum(lon.points < 0) diff --git a/tests/integration/cmor/_fixes/native6/test_mswep.py b/tests/integration/cmor/_fixes/native6/test_mswep.py index 2208c04f86..e37134a036 100644 --- a/tests/integration/cmor/_fixes/native6/test_mswep.py +++ b/tests/integration/cmor/_fixes/native6/test_mswep.py @@ -18,7 +18,7 @@ def test_get_pr_fix(mip_table): """Test whether the right fix gets found.""" fix = Fix.get_fixes('native6', 'MSWEP', mip_table, 'pr') - assert fix == [Pr(None)] + assert isinstance(fix[0], Pr) @pytest.fixture From cedc5ecf0980a00390534a80507063477631cfc9 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 2 Feb 2021 08:28:32 +0000 Subject: [PATCH 10/14] Update tests/integration/cmor/_fixes/native6/test_mswep.py Co-authored-by: Peter Kalverla --- tests/integration/cmor/_fixes/native6/test_mswep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cmor/_fixes/native6/test_mswep.py b/tests/integration/cmor/_fixes/native6/test_mswep.py index e37134a036..5fa2118b28 100644 --- a/tests/integration/cmor/_fixes/native6/test_mswep.py +++ b/tests/integration/cmor/_fixes/native6/test_mswep.py @@ -1,4 +1,4 @@ -"""Tests for the fixes of ERA5.""" +"""Tests for the fixes of MSWEP.""" from pathlib import Path import iris From 270818eef21e391ac1b781199ee823ed5deed1f0 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 2 Feb 2021 11:22:34 +0100 Subject: [PATCH 11/14] Add documentation for MSWEP support --- doc/develop/fixing_data.rst | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index f6b7ae8acd..5169598545 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -305,3 +305,30 @@ strictness to the highest: - ``strict``: fail if there are any warnings, this is the highest level of strictness. Mostly useful for checking datasets that you have produced, to be sure that future users will not be distracted by inoffensive warnings. + + +Natively supported non-CMOR datasets +==================================== + +Fixed datasets are supported through the ``native6`` project. Below is a list of +datasets currently supported. + +ERA5 +---- + +- Supported variables: ``clt``, ``evspsbl``, ``evspsblpot``, ``mrro``, ``pr``, ``prsn``, ``ps``, ``psl``, ``ptype``, ``rls``, ``rlds``, ``rsds``, ``rsdt``, ``rss``, ``uas``, ``vas``, ``tas``, ``tasmax``, ``tasmin``, ``tdps``, ``ts``, ``tsn`` (``E1hr``/``Amon``), ``orog`` (``fx``) +- Tier: 3 + +MSWEP +----- + +- Supported variables: `pr` +- Supported frequencies: `mon`, `day`, `3hr`. +- Tier: 3 + +The data must be placed in the ``/Tier3/MSWEP/latestversion/mon/pr`` subdirectory of your ``native6`` project location. + +.. note:: + For monthly data (V220), the data must be postfixed with the date, i.e. rename ``global_monthly_050deg.nc`` to ``global_monthly_050deg_197901-201710.nc`` + +For more info: http://www.gloh2o.org/ From c4a811c7a126a1811159b092f89c9fbfe22de19a Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 2 Feb 2021 11:25:39 +0100 Subject: [PATCH 12/14] Tweak text --- doc/develop/fixing_data.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 5169598545..55af3f752a 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -322,11 +322,11 @@ ERA5 MSWEP ----- -- Supported variables: `pr` -- Supported frequencies: `mon`, `day`, `3hr`. +- Supported variables: ``pr`` +- Supported frequencies: ``mon``, ``day``, `3hr``. - Tier: 3 -The data must be placed in the ``/Tier3/MSWEP/latestversion/mon/pr`` subdirectory of your ``native6`` project location. +For example for monthly data, place the files in the ``/Tier3/MSWEP/latestversion/mon/pr`` subdirectory of your ``native6`` project location. .. note:: For monthly data (V220), the data must be postfixed with the date, i.e. rename ``global_monthly_050deg.nc`` to ``global_monthly_050deg_197901-201710.nc`` From d909a9acf7f14226e681d43bd0c09b85cc48bd02 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 2 Feb 2021 15:24:43 +0100 Subject: [PATCH 13/14] 'Fix' Codacy issues --- esmvalcore/cmor/_fixes/native6/mswep.py | 65 ++++++++++--------- .../cmor/_fixes/native6/test_mswep.py | 7 +- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index f972c75208..f38a88209b 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -44,9 +44,24 @@ def fix_time_day(cube): time_coord.convert_units('days since 1850-1-1 00:00:00.0') +def fix_longitude(cube): + """Fix longitude coordinate from -180:180 to 0:360.""" + lon_axis = cube.coord_dims('longitude') + lon = cube.coord(axis='X') + + if not lon.is_monotonic(): + raise ValueError("Data must be monotonic to fix longitude.") + + # roll data because iris forces `lon.points` to be strictly monotonic. + shift = np.sum(lon.points < 0) + points = np.roll(lon.points, -shift) % 360 + cube.data = np.roll(cube.core_data(), -shift, axis=lon_axis) + + lon.points = points + + class Pr(Fix): """Fixes for pr.""" - def fix_metadata(self, cubes): """Fix metadata.""" self._init_frequency_specific_fixes() @@ -55,50 +70,36 @@ def fix_metadata(self, cubes): self._fix_names(cube) self._fix_units(cube) self._fix_time(cube) - self._fix_longitude(cube) + fix_longitude(cube) self._fix_bounds(cube) return cubes - def _init_frequency_specific_fixes(self): - """Set frequency specific fixes (day/mon).""" + def _fix_time(self, cube): + """Fix time.""" frequency = self.vardef.frequency if frequency in ('day', '3hr'): - self._fix_units = self._fix_units_day - self._fix_time = fix_time_day + fix_time_day(cube) elif frequency == 'mon': - self._fix_units = self._fix_units_month - self._fix_time = fix_time_month + fix_time_month(cube) else: - raise ValueError(f'No fixes for frequency: {frequency!r}') + raise ValueError(f'Cannot fix time for frequency: {frequency!r}') - def _fix_units_month(self, cube): - """Convert units from mm/month to kg m-2 s-1 units.""" - cube.units = Unit(self.vardef.units) - # divide by number of seconds in a month - cube.data = cube.core_data() / 60 * 60 * 24 * 30 + def _fix_units(self, cube): + """Convert units from mm/[t] to kg m-2 s-1 units.""" + frequency = self.vardef.frequency - def _fix_units_day(self, cube): - """Convert units from mm/month to kg m-2 s-1 units.""" cube.units = Unit(self.vardef.units) - # divide by number of seconds in a day - cube.data = cube.core_data() / 60 * 60 * 24 - - def _fix_longitude(self, cube): - """Fix longitude coordinate from -180:180 to 0:360.""" - lon_axis = cube.coord_dims('longitude') - lon = cube.coord(axis='X') - - if not lon.is_monotonic(): - raise ValueError("Data must be monotonic to fix longitude.") - - # roll data because iris forces `lon.points` to be strictly monotonic. - shift = np.sum(lon.points < 0) - points = np.roll(lon.points, -shift) % 360 - cube.data = np.roll(cube.core_data(), -shift, axis=lon_axis) - lon.points = points + if frequency in ('day', '3hr'): + # divide by number of seconds in a month + cube.data = cube.core_data() / 60 * 60 * 24 * 30 + elif frequency == 'mon': + # divide by number of seconds in a day + cube.data = cube.core_data() / 60 * 60 * 24 + else: + raise ValueError(f'Cannot fix units for frequency: {frequency!r}') def _fix_bounds(self, cube): """Add bounds to coords.""" diff --git a/tests/integration/cmor/_fixes/native6/test_mswep.py b/tests/integration/cmor/_fixes/native6/test_mswep.py index 5fa2118b28..70418c748c 100644 --- a/tests/integration/cmor/_fixes/native6/test_mswep.py +++ b/tests/integration/cmor/_fixes/native6/test_mswep.py @@ -7,6 +7,7 @@ from esmvalcore.cmor._fixes.native6.mswep import ( Pr, + fix_longitude, fix_time_day, fix_time_month, ) @@ -74,13 +75,13 @@ def test_fix_names(fix_month, cube_month): def test_fix_units_month(fix_month, cube_month): """Test `Pr._fix_units_month`.""" - fix_month._fix_units_month(cube_month) + fix_month._fix_units(cube_month) assert cube_month.units == fix_month.vardef.units def test_fix_units_day(fix_day, cube_day): """Test `Pr._fix_units_day`.""" - fix_day._fix_units_day(cube_day) + fix_day._fix_units(cube_day) assert cube_day.units == fix_day.vardef.units @@ -106,7 +107,7 @@ def test_fix_longitude(fix_month, cube_month): unfixed_lon = cube_month.coord(axis='X') shift = (unfixed_lon.points < 0).sum() - fix_month._fix_longitude(cube_month) + fix_longitude(cube_month) lon = cube_month.coord(axis='X') From 9bbca2fc8652494d99cdb2e44522c17b6d800bfe Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 2 Feb 2021 15:29:09 +0100 Subject: [PATCH 14/14] Remove unused function call --- esmvalcore/cmor/_fixes/native6/mswep.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/esmvalcore/cmor/_fixes/native6/mswep.py b/esmvalcore/cmor/_fixes/native6/mswep.py index f38a88209b..34563ec5dc 100644 --- a/esmvalcore/cmor/_fixes/native6/mswep.py +++ b/esmvalcore/cmor/_fixes/native6/mswep.py @@ -62,10 +62,9 @@ def fix_longitude(cube): class Pr(Fix): """Fixes for pr.""" + def fix_metadata(self, cubes): """Fix metadata.""" - self._init_frequency_specific_fixes() - for cube in cubes: self._fix_names(cube) self._fix_units(cube)