diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..fe5fa218 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,31 @@ +name: build + +on: + push: + branches: + - main + pull_request: + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@main + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 black pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements_dev.txt ]; then pip install -r requirements_dev.txt; fi + - name: Test with pytest + run: | + python -m pytest -v tests/test_readers.py diff --git a/checksit/check.py b/checksit/check.py index aea1ffc9..199495cb 100644 --- a/checksit/check.py +++ b/checksit/check.py @@ -159,7 +159,8 @@ def _check_file(self, file_content, template, mappings=None, extra_rules=None, s if log_mode == "compact": highest = "ERROR" if len(errors) > 0 else "NONE" - print(f"{highest} | {len(errors)} ", end="") + endstr = "" if len(errors) > 0 else "\n" + print(f"{highest} | {len(errors)} ", end=endstr) err_string = " | ".join([err.replace("|", "__VERTICAL_BAR_REPLACED__") for err in errors]) if err_string: print(f"| {err_string}") diff --git a/checksit/cli.py b/checksit/cli.py index 2d001cc7..ca14f882 100644 --- a/checksit/cli.py +++ b/checksit/cli.py @@ -62,9 +62,28 @@ def check(file_path, mappings=None, rules=None, specs=None, ignore_attrs=None, i @main.command() @click.argument("log_files", nargs=-1, default=None) @click.option("-d", "--log-directory", default=None) +@click.option("--show-files/--no-show-files", default=False) +@click.option("-x", "--exclude", default=None) +@click.option("-e", "--exclude-file", default=None) @click.option("--verbose/--no-verbose", default=False) -def summary(log_files=None, log_directory=None, verbose=False): - return summarise(log_files, log_directory=log_directory, verbose=verbose) +def summary(log_files=None, log_directory=None, show_files=False, + exclude=None, exclude_file=None, + verbose=False): + + if exclude: + exclude = string_to_list(exclude) + else: + exclude = [] + + if exclude_file: + if not os.path.isfile(exclude_file): + raise Exception(f"'--exclude-file' does not point to a valid file") + + with open(exclude_file) as exfile: + exclude.extend([exclude_pattern for exclude_pattern in exfile if exclude_pattern.strip()]) + + return summarise(log_files, log_directory=log_directory, show_files=show_files, + exclude=exclude, verbose=verbose) @main.command() diff --git a/checksit/readers/cdl.py b/checksit/readers/cdl.py index 3c761999..5093d65d 100644 --- a/checksit/readers/cdl.py +++ b/checksit/readers/cdl.py @@ -55,7 +55,7 @@ def _check_format(self): min_chars = 10 if len(source) < min_chars: - self.fmt_errors.append(f"[FORMAT:global_attributes:source] Must be at least {min_chars} characters, not {source}") + self.fmt_errors.append(f"[FORMAT:global_attributes:source] Must be at least {min_chars} characters, not {source}") def _get_sections(self, lines, split_patterns, start_at): split_patterns = deque(split_patterns) @@ -114,12 +114,16 @@ def _construct_variables(self, content): var_id = None current = None + # Set defaults for key and value so they can be sent to multiline parser even if not set + key = None + value = None + for line in content: if re.match(f"^{vocabs_prefix}:[0-9a-zA-Z_-]+:variables:", line): vocab_var_id = line.split(":")[3] vocab_lookup = line.split(":", 1)[-1] variables[vocab_var_id] = vocabs.lookup(vocab_lookup) - elif not var_id or not line.startswith(f"{var_id}:"): + elif not var_id or not line.startswith(f"{var_id}:") and last_line.strip()[-1] != ",": # Add current collected variable to list if it exists if current: variables[var_id] = current.copy() @@ -127,18 +131,52 @@ def _construct_variables(self, content): var_id, dtype, dimensions = self._parse_var_dtype_dims(line) current = {"type": dtype, "dimension": ', '.join(dimensions)} else: - key, value = [x.strip() for x in line.split(":", 1)[1].split("=", 1)] +# key, value = [x.strip() for x in line.split(":", 1)[1].split("=", 1)] + # Send last key and last value (from last iteration of loop) and line to get new value + key, value = self._parse_key_value_multiline_safe(line, key, value, variable_attr=True) current[key] = self._safe_parse_value(value) + + last_line = line else: variables[var_id] = current.copy() return variables + def _parse_key_value_multiline_safe(self, line, last_key, last_value, variable_attr=False): + # Caters for continuation lines for arrays of strings, etc + if "=" in line: + # A new (key, value) pair is found + if variable_attr: # var attr + key, value = [x.strip() for x in line.split(":", 1)[1].split("=", 1)] + else: # global attr + key, value = [x.strip() for x in line.lstrip(":").split("=", 1)] + else: + # Assume a continuation of th last value, so set key to None + key, value = last_key, last_value + " " + line.strip().rstrip(";") + + return key, value + + def _ordered_dict(self, content): resp = {} + key = None + value = None + for line in content: if self.verbose: print(f"WORKING ON LINE: {line}") - key, value = [x.strip() for x in line.lstrip(":").split("=", 1)] + + # Cater for continuation lines for arrays of strings, etc +# if "=" in line: + # A new (key, value) pair is found +# key, value = [x.strip() for x in line.lstrip(":").split("=", 1)] +# else: + # Assume a continuation of th last value +# value += " " + line.strip() + # Send last key and last value (from last iteration of loop) and line to get new value + key, value = self._parse_key_value_multiline_safe(line, key, value) + + # This will overwrite the previous value - which is safe if a continuation happened + # as the key is the same as last time resp[key] = self._safe_parse_value(value) return resp diff --git a/checksit/summary.py b/checksit/summary.py index 48de4f48..a59ebb52 100755 --- a/checksit/summary.py +++ b/checksit/summary.py @@ -1,6 +1,8 @@ import os import re import glob +from collections import defaultdict, OrderedDict as OD + import pandas as pd @@ -20,8 +22,18 @@ def get_max_column_count(files, sep): return count -def summarise(log_files=None, log_directory=None, verbose=False): +def do_exclude(err, exclude_patterns): + for exclude_pattern in exclude_patterns: + if exclude_pattern in err: + return True + + return False + + +def summarise(log_files=None, log_directory=None, show_files=False, + exclude=None, verbose=False): log_files = log_files or find_log_files(log_directory) + exclude_patterns = exclude or [] if len(log_files) == 0: print("[ERROR] No log files found!") @@ -58,16 +70,32 @@ def summarise(log_files=None, log_directory=None, verbose=False): fatals = len(df[df["highest_error"].str.contains("FATAL")]) print(f"[INFO] Found {fatals} FATAL errors.") - all_errors = [] + errors_by_type = defaultdict(list) + for err_col in err_cols: - all_errors.extend(list(set( - [f"{err} [found in {int(df[df[err_col] == err][err_col].value_counts())} file(s)]" - for err in df[err_col].unique() if err.strip()]))) + for err in df[err_col].unique(): + err = err.strip() + if not err or do_exclude(err, exclude_patterns): continue + + filepaths = sorted(df[df[err_col] == err]["filepath"]) + errors_by_type[err].extend(filepaths) + + all_errors = OD() + for err in sorted(errors_by_type): + filepaths = errors_by_type[err] + all_errors[err] = sorted(filepaths) - all_errors = sorted(set(all_errors)) print(f"[INFO] {len(all_errors)} found. They are...") for err in all_errors: - print(f"\t\t{err}") - + filepaths = all_errors[err] + print(f"\t\t{err} [found in {len(filepaths)} file(s)]") + + if show_files: + print("\n------- File paths --------\n") + + for err in all_errors: + print(f"\t\t{err}") + for filepath in all_errors[err]: + print(f"\t\t\t{filepath}") diff --git a/requirements_dev.txt b/requirements_dev.txt index 74da827f..6c3a449a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,13 +1,14 @@ -pip==19.2.3 -bump2version==0.5.11 -wheel==0.33.6 -watchdog==0.9.0 -flake8==3.7.8 -tox==3.14.0 -coverage==4.5.4 -Sphinx==1.8.5 -twine==1.14.0 -pre-commit==2.8.2 -Click==7.0 -pytest==4.6.5 -pytest-runner==5.1 +pip +bump2version +wheel +watchdog +flake8 +tox +coverage +Sphinx +twine +pre-commit +Click +pytest +pytest-runner +deepdiff diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 00000000..a29199b0 --- /dev/null +++ b/tests/common.py @@ -0,0 +1,5 @@ +import os + +TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "testdata") + + diff --git a/tests/sample-fixture/make-sample-fixture.py b/tests/sample-fixture/make-sample-fixture.py new file mode 100644 index 00000000..81f63440 --- /dev/null +++ b/tests/sample-fixture/make-sample-fixture.py @@ -0,0 +1,23 @@ +import os +import glob +import json +import shelve + +from checksit.readers.cdl import read as read_cdl + + +d = {} + +for cdl in glob.glob("sample-ncs/*.cdl"): + fname = os.path.basename(cdl) + + d[fname] = read_cdl(cdl).to_dict() + + +with shelve.open("sample-fixture") as db: + for fname in d: + db[fname] = d[fname] + + + + diff --git a/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.bak b/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.bak new file mode 100644 index 00000000..1dcdc347 --- /dev/null +++ b/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.bak @@ -0,0 +1,6 @@ +'rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl', (0, 3754) +'tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl', (4096, 3789) +'cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl', (8192, 3890) +'20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl', (12288, 6368) +'summer_rainfall_2001.cdl', (18944, 2068) +'ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl', (21504, 7238) diff --git a/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.dat b/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.dat new file mode 100644 index 00000000..127ee555 Binary files /dev/null and b/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.dat differ diff --git a/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.dir b/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.dir new file mode 100644 index 00000000..1dcdc347 --- /dev/null +++ b/tests/sample-fixture/previous-sample-fixtures/20221104/sample-fixture.dir @@ -0,0 +1,6 @@ +'rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl', (0, 3754) +'tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl', (4096, 3789) +'cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl', (8192, 3890) +'20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl', (12288, 6368) +'summer_rainfall_2001.cdl', (18944, 2068) +'ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl', (21504, 7238) diff --git a/tests/sample-fixture/sample-fixture.bak b/tests/sample-fixture/sample-fixture.bak new file mode 100644 index 00000000..fd9edab2 --- /dev/null +++ b/tests/sample-fixture/sample-fixture.bak @@ -0,0 +1,6 @@ +'rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl', (0, 3754) +'tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl', (4096, 3789) +'cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl', (8192, 3890) +'20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl', (12288, 6168) +'summer_rainfall_2001.cdl', (18944, 2068) +'ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl', (21504, 7238) diff --git a/tests/sample-fixture/sample-fixture.dat b/tests/sample-fixture/sample-fixture.dat new file mode 100644 index 00000000..1e473b3d Binary files /dev/null and b/tests/sample-fixture/sample-fixture.dat differ diff --git a/tests/sample-fixture/sample-fixture.dir b/tests/sample-fixture/sample-fixture.dir new file mode 100644 index 00000000..fd9edab2 --- /dev/null +++ b/tests/sample-fixture/sample-fixture.dir @@ -0,0 +1,6 @@ +'rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl', (0, 3754) +'tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl', (4096, 3789) +'cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl', (8192, 3890) +'20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl', (12288, 6168) +'summer_rainfall_2001.cdl', (18944, 2068) +'ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl', (21504, 7238) diff --git a/tests/sample-fixture/sample-ncs/20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl b/tests/sample-fixture/sample-ncs/20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl new file mode 100644 index 00000000..788a7f54 --- /dev/null +++ b/tests/sample-fixture/sample-ncs/20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.cdl @@ -0,0 +1,115 @@ +netcdf \20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0 { +dimensions: + vegetation_class = 18 ; + lat = 720 ; + lon = 1440 ; + bounds = 2 ; + strlen = 150 ; + time = UNLIMITED ; // (1 currently) +variables: + double lat(lat) ; + lat:units = "degree_north" ; + lat:standard_name = "latitude" ; + lat:long_name = "latitude" ; + lat:bounds = "lat_bounds" ; + double lat_bounds(lat, bounds) ; + double lon(lon) ; + lon:units = "degree_east" ; + lon:standard_name = "longitude" ; + lon:long_name = "longitude" ; + lon:bounds = "lon_bounds" ; + double lon_bounds(lon, bounds) ; + double time(time) ; + time:units = "days since 1970-01-01 00:00:00" ; + time:standard_name = "time" ; + time:long_name = "time" ; + time:bounds = "time_bounds" ; + time:calendar = "standard" ; + float time_bounds(time, bounds) ; + int vegetation_class(vegetation_class) ; + vegetation_class:units = "1" ; + vegetation_class:long_name = "vegetation class number" ; + char vegetation_class_name(vegetation_class, strlen) ; + vegetation_class_name:units = "1" ; + vegetation_class_name:long_name = "vegetation class name" ; + float burned_area(time, lat, lon) ; + burned_area:units = "m2" ; + burned_area:standard_name = "burned_area" ; + burned_area:long_name = "total burned_area" ; + burned_area:valid_range = 0.f, 7.693146e+08f ; + burned_area:cell_methods = "time: sum" ; + float standard_error(time, lat, lon) ; + standard_error:units = "m2" ; + standard_error:long_name = "standard error of the estimation of burned area" ; + standard_error:valid_range = 0.f, 7.693146e+08f ; + float fraction_of_burnable_area(time, lat, lon) ; + fraction_of_burnable_area:units = "1" ; + fraction_of_burnable_area:long_name = "fraction of burnable area" ; + fraction_of_burnable_area:comment = "The fraction of burnable area is the fraction of the cell that corresponds to vegetated land covers that could burn. The land cover classes are those from C3S Land Cover, https://cds.climate.copernicus.eu/cdsapp#!/dataset/satellite-land-cover?tab=overview" ; + fraction_of_burnable_area:valid_range = 0.f, 1.f ; + float fraction_of_observed_area(time, lat, lon) ; + fraction_of_observed_area:units = "1" ; + fraction_of_observed_area:long_name = "fraction of observed area" ; + fraction_of_observed_area:comment = "The fraction of observed area is the fraction of the total burnable area in the cell (fraction_of_burnable_area variable of this file) that was observed during the time interval, and was not marked as unsuitable/not observable. The latter refers to the area where it was not possible to obtain observational burned area information for the whole time interval because of the lack of input data (non-existing data for that location and period)." ; + fraction_of_observed_area:valid_range = 0.f, 1.f ; + float burned_area_in_vegetation_class(time, vegetation_class, lat, lon) ; + burned_area_in_vegetation_class:units = "m2" ; + burned_area_in_vegetation_class:long_name = "burned area in vegetation class" ; + burned_area_in_vegetation_class:cell_methods = "time: sum" ; + burned_area_in_vegetation_class:comment = "Burned area by land cover classes; land cover classes are from C3S Land Cover, https://cds.climate.copernicus.eu/cdsapp#!/dataset/satellite-land-cover?tab=overview" ; + burned_area_in_vegetation_class:valid_range = 0.f, 7.693146e+08f ; + int crs ; + crs:wkt = "GEOGCS[\"WGS84(DD)\", \n", + " DATUM[\"WGS84\", \n", + " SPHEROID[\"WGS84\", 6378137.0, 298.257223563]], \n", + " PRIMEM[\"Greenwich\", 0.0], \n", + " UNIT[\"degree\", 0.017453292519943295], \n", + " AXIS[\"Geodetic longitude\", EAST], \n", + " AXIS[\"Geodetic latitude\", NORTH]]" ; + crs:i2m = "0.25,0.0,0.0,-0.25,-180.0,90.0" ; + +// global attributes: + :title = "Sentinel-3 SYN Burned Area Grid product, version 1.0" ; + :institution = "University of Alcala" ; + :source = "Sentinel-3 Synergy (SYN) product, derived from OLCI+SLSTR Surface Reflectance, VIIRS VNP14IMGML thermal anomalies, C3S Land Cover dataset v2.1.1" ; + :history = "Created on 2022-03-01 17:20:33" ; + :references = "See https://climate.esa.int/en/projects/fire/" ; + :tracking_id = "d978331b-be86-46d3-887c-3525f652b830" ; + :Conventions = "CF-1.7" ; + :product_version = "v1.0" ; + :format_version = "CCI Data Standards v2.3" ; + :summary = "The grid product is the result of summing burned area pixels and their attributes within each cell of 0.25x0.25 degrees in a regular grid covering the whole Earth in monthly composites. The attributes stored are sum of burned area, standard error, fraction of burnable area, fraction of observed area, and the burned area for 18 land cover classes of C3S Land Cover." ; + :keywords = "Burned Area, Fire Disturbance, Climate Change, ESA, GCOS" ; + :id = "20190101-ESACCI-L4_FIRE-BA-SYN-fv1.0.nc" ; + :naming_authority = "int.esa.climate" ; + :doi = "10.5285/3aaaaf94813e48f18f2b83242a8dacbe" ; + :keywords_vocabulary = "none" ; + :cdm_data_type = "Grid" ; + :comment = "These data were produced as part of the Climate Change Initiative Programme, Fire Disturbance ECV." ; + :date_created = "20220301T172033Z" ; + :creator_name = "University of Alcala" ; + :creator_url = "https://geogra.uah.es/gita/en/" ; + :creator_email = "emilio.chuvieco@uah.es" ; + :contact = "mlucrecia.pettinari@uah.es" ; + :project = "Climate Change Initiative - European Space Agency" ; + :geospatial_lat_min = "-90" ; + :geospatial_lat_max = "90" ; + :geospatial_lon_min = "-180" ; + :geospatial_lon_max = "180" ; + :geospatial_vertical_min = "0" ; + :geospatial_vertical_max = "0" ; + :time_coverage_start = "20190101T000000Z" ; + :time_coverage_end = "20190131T235959Z" ; + :time_coverage_duration = "P1M" ; + :time_coverage_resolution = "P1M" ; + :standard_name_vocabulary = "NetCDF Climate and Forecast (CF) Metadata Convention" ; + :license = "ESA CCI Data Policy: free and open access" ; + :platform = "Sentinel-3A, Sentinel-3B" ; + :sensor = "OLCI, SLSTR" ; + :spatial_resolution = "0.25 degrees" ; + :key_variables = "burned_area" ; + :geospatial_lon_units = "degrees_east" ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_resolution = "0.25" ; + :geospatial_lat_resolution = "0.25" ; +} diff --git a/tests/sample-fixture/sample-ncs/ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl b/tests/sample-fixture/sample-ncs/ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl new file mode 100644 index 00000000..66268e73 --- /dev/null +++ b/tests/sample-fixture/sample-ncs/ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0.cdl @@ -0,0 +1,134 @@ +netcdf ESACCI-SEAICE-L3C-SITHICK-RA2_ENVISAT-SH50KMEASE2-201202-fv2.0 { +dimensions: + time = UNLIMITED ; // (1 currently) + yc = 216 ; + xc = 216 ; + nv = 2 ; +variables: + float freeboard(time, yc, xc) ; + freeboard:coordinates = "time lon lat" ; + freeboard:grid_mapping = "Lambert_Azimuthal_Grid" ; + freeboard:long_name = "elevation of retracked point above instantaneous sea surface height (with snow range corrections)" ; + freeboard:standard_name = "sea_ice_freeboard" ; + freeboard:units = "m" ; + float freeboard_uncertainty(time, yc, xc) ; + freeboard_uncertainty:coordinates = "time lon lat" ; + freeboard_uncertainty:grid_mapping = "Lambert_Azimuthal_Grid" ; + freeboard_uncertainty:long_name = "freeboard uncertainty" ; + freeboard_uncertainty:units = "m" ; + double lat(yc, xc) ; + lat:long_name = "latitude of grid cell center" ; + lat:standard_name = "latitude" ; + lat:units = "degrees_north" ; + double lon(yc, xc) ; + lon:long_name = "longitude of grid cell center" ; + lon:standard_name = "longitude" ; + lon:units = "degrees_east" ; + float radar_freeboard(time, yc, xc) ; + radar_freeboard:coordinates = "time lon lat" ; + radar_freeboard:grid_mapping = "Lambert_Azimuthal_Grid" ; + radar_freeboard:long_name = "elevation of retracked point above instantaneous sea surface height (no snow range corrections)" ; + radar_freeboard:units = "m" ; + float radar_freeboard_uncertainty(time, yc, xc) ; + radar_freeboard_uncertainty:coordinates = "time lon lat" ; + radar_freeboard_uncertainty:grid_mapping = "Lambert_Azimuthal_Grid" ; + radar_freeboard_uncertainty:long_name = "uncertainty of radar freeboard" ; + radar_freeboard_uncertainty:units = "m" ; + float sea_ice_concentration(time, yc, xc) ; + sea_ice_concentration:comment = "Average grid cell sea ice concentration during days with altimetry data coverage (not monthly mean)" ; + sea_ice_concentration:coordinates = "time lon lat" ; + sea_ice_concentration:grid_mapping = "Lambert_Azimuthal_Grid" ; + sea_ice_concentration:long_name = "sea ice contration" ; + sea_ice_concentration:standard_name = "sea_ice_area_fraction" ; + sea_ice_concentration:units = "percent" ; + sea_ice_concentration:valid_max = 100. ; + sea_ice_concentration:valid_min = 0. ; + float sea_ice_thickness(time, yc, xc) ; + sea_ice_thickness:ancillary_variables = "sea_ice_thickness_uncertainty status_flag" ; + sea_ice_thickness:coordinates = "time lon lat" ; + sea_ice_thickness:grid_mapping = "Lambert_Azimuthal_Grid" ; + sea_ice_thickness:long_name = "thickness of the sea ice layer" ; + sea_ice_thickness:standard_name = "sea_ice_thickness" ; + sea_ice_thickness:units = "m" ; + float sea_ice_thickness_uncertainty(time, yc, xc) ; + sea_ice_thickness_uncertainty:coordinates = "time lon lat" ; + sea_ice_thickness_uncertainty:grid_mapping = "Lambert_Azimuthal_Grid" ; + sea_ice_thickness_uncertainty:long_name = "uncertainty of the sea ice layer thickness" ; + sea_ice_thickness_uncertainty:units = "m" ; + byte status_flag(time, yc, xc) ; + status_flag:comment = "Describes the status of the sea-ice thickness retrieval" ; + status_flag:coordinates = "time lat lon" ; + status_flag:flag_meaning = "0: nominal sea ice thickness retrieval; 1: no input data; 2: outside sea ice concentration mask; 3: latitude above orbit inclination; 4: land, lake or land ice; 5: sea ice thickness retrieval failed;" ; + status_flag:flag_values = 0, 1, 2, 3, 4, 5 ; + status_flag:grid_mapping = "Lambert_Azimuthal_Grid" ; + status_flag:long_name = "status flag for the sea ice thickness retrieval" ; + status_flag:standard_name = "sea_ice_thickness status_flag" ; + status_flag:unit = 1 ; + status_flag:valid_max = 5 ; + status_flag:valid_min = 0 ; + double time(time) ; + time:standard_name = "time" ; + time:units = "seconds since 1970-01-01" ; + time:long_name = "reference time of product" ; + time:axis = "T" ; + time:calendar = "standard" ; + time:bounds = "time_bnds" ; + double time_bnds(time, nv) ; + time_bnds:units = "seconds since 1970-01-01" ; + double xc(xc) ; + xc:standard_name = "projection_x_coordinate" ; + xc:units = "km" ; + xc:long_name = "x coordinate of projection (eastings)" ; + double yc(yc) ; + yc:standard_name = "projection_y_coordinate" ; + yc:units = "km" ; + yc:long_name = "y coordinate of projection (eastings)" ; + byte Lambert_Azimuthal_Grid ; + Lambert_Azimuthal_Grid:false_easting = 0. ; + Lambert_Azimuthal_Grid:false_northing = 0. ; + Lambert_Azimuthal_Grid:grid_mapping_name = "lambert_azimuthal_equal_area" ; + Lambert_Azimuthal_Grid:inverse_flattening = 298.257223563 ; + Lambert_Azimuthal_Grid:latitude_of_projection_origin = -90. ; + Lambert_Azimuthal_Grid:longitude_of_projection_origin = 0. ; + Lambert_Azimuthal_Grid:proj4_string = "+proj=laea +lon_0=0 +datum=WGS84 +ellps=WGS84 +lat_0=-90.0" ; + Lambert_Azimuthal_Grid:semi_major_axis = 6378137. ; + +// global attributes: + :title = "ESA Climate Change Initiative Sea Ice: Experimental Southern Hemisphere Sea Ice Thickness Climate Data Record" ; + :institution = "Alfred-Wegener-Institut Helmholtz Zentrum für Polar und Meeresforschung" ; + :source = "Altimetry: envisat, Snow depth: ESA-SICCI AMSR-E/AMSR2 snow depth on sea ice climatology, Mean Sea Surface: DTU15 global mean sea surface, Sea ice Concentration: OSI-SAF Global Sea Ice Concentration (OSI-409), Sea ice type: First-year sea ice only" ; + :platform = "Envisat" ; + :sensor = "RA-2" ; + :history = "20180417T184311Z (created)" ; + :references = "Algorithm Theoretical Baseline Document, Sea Ice Climate Change Initiative: Phase 2 (version 2.2)" ; + :tracking_id = "1a6cb716-a7ba-4c8b-89a3-b601c96520e1" ; + :conventions = "CF-1.6" ; + :product_version = "2.0" ; + :processing_level = "Level-3 Collated (l3c)" ; + :summary = "Monthly gridded Southern Hemisphere Sea Ice Thickness Climate Data Record from Envisat and CryoSat-2 satellite radar altimetry for the period June 2002 - April 2017." ; + :keywords = "Sea Ice, Ice Depth/Thickness, Radar Altimeters" ; + :id = "esacci-seaice-l3c-sit-RA-2-envisat-sh50kmEASE2-20120201-fv2.0" ; + :naming_authority = "de.awi" ; + :keywords_vocabulary = "GCMD Science Keywords" ; + :doi = "10.5285/b1f1ac03077b4aa784c5a413a2210bf5" ; + :cdm_data_type = "Grid" ; + :comment = "Southern hemisphere sea ice thickness is an experimental climate data record, as the algorithm does not properly considers the impact of the complex snow morphology in the freeboard retrieval. Sea ice thickness is provided for all month but needs to be considered biased high in areas with high snow depth and during the southern summer month. Please consult the Product User Guide (PUG) for more information." ; + :date_created = "20180417T184311Z" ; + :creator_name = "Stefan Hendricks, Stephan Paul (Alfred Wegener Institute Helmholtz Centre for Polar and Marine Research, Bremerhaven, Germany); Eero Rine (Finnish Meteorological Institute, Helsinki, Finland)" ; + :creator_url = "http://www.awi.de" ; + :creator_email = "stefan.hendricks@awi.de, stephan.paul@awi.de, eero.rinne@fmi.fi" ; + :project = "Climate Change Initiative - European Space Agency" ; + :geospatial_lat_min = "-89.6835" ; + :geospatial_lat_max = "-16.8229" ; + :geospatial_lon_min = "-179.7335" ; + :geospatial_lon_max = "179.7335" ; + :geospatial_vertical_min = "0.0" ; + :geospatial_vertical_max = "0.0" ; + :time_coverage_start = "20120201T000000Z" ; + :time_coverage_end = "20120229T235959Z" ; + :time_coverage_duration = "P1M" ; + :time_coverage_resolution = "P1M" ; + :spatial_resolution = "50 km" ; + :standard_name_vocabulary = "CF" ; + :license = "Creative Commons Attribution 4.0 International (CC BY 4.0)" ; +} diff --git a/tests/sample-fixture/sample-ncs/cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl b/tests/sample-fixture/sample-ncs/cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl new file mode 100644 index 00000000..3216c98e --- /dev/null +++ b/tests/sample-fixture/sample-ncs/cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130.cdl @@ -0,0 +1,91 @@ +netcdf cltAnom_rcp85_land-prob_uk_25km_sample_b8110_30y_mon_20091201-20991130 { +dimensions: + time = 84 ; + projection_y_coordinate = 52 ; + projection_x_coordinate = 39 ; + sample = 3000 ; + bnds = 2 ; + string64 = 64 ; +variables: + float cloud_area_fraction(time, projection_y_coordinate, projection_x_coordinate, sample) ; + cloud_area_fraction:long_name = "Cloud area fraction" ; + cloud_area_fraction:units = "%" ; + cloud_area_fraction:anomaly_type = "percentage_change" ; + cloud_area_fraction:description = "Total cloud" ; + cloud_area_fraction:label_units = "%" ; + cloud_area_fraction:plot_label = "Total cloud anomaly (%)" ; + cloud_area_fraction:cell_methods = "time: mean" ; + cloud_area_fraction:grid_mapping = "transverse_mercator" ; + cloud_area_fraction:coordinates = "latitude longitude month_number season season_year year" ; + int transverse_mercator ; + transverse_mercator:grid_mapping_name = "transverse_mercator" ; + transverse_mercator:longitude_of_prime_meridian = 0. ; + transverse_mercator:semi_major_axis = 6377563.396 ; + transverse_mercator:semi_minor_axis = 6356256.909 ; + transverse_mercator:longitude_of_central_meridian = -2. ; + transverse_mercator:latitude_of_projection_origin = 49. ; + transverse_mercator:false_easting = 400000. ; + transverse_mercator:false_northing = -100000. ; + transverse_mercator:scale_factor_at_central_meridian = 0.9996012717 ; + double time(time) ; + time:axis = "T" ; + time:bounds = "time_bnds" ; + time:units = "hours since 1970-01-01 00:00:00" ; + time:standard_name = "time" ; + time:calendar = "360_day" ; + double time_bnds(time, bnds) ; + double projection_y_coordinate(projection_y_coordinate) ; + projection_y_coordinate:axis = "Y" ; + projection_y_coordinate:bounds = "projection_y_coordinate_bnds" ; + projection_y_coordinate:units = "m" ; + projection_y_coordinate:standard_name = "projection_y_coordinate" ; + double projection_y_coordinate_bnds(projection_y_coordinate, bnds) ; + double projection_x_coordinate(projection_x_coordinate) ; + projection_x_coordinate:axis = "X" ; + projection_x_coordinate:bounds = "projection_x_coordinate_bnds" ; + projection_x_coordinate:units = "m" ; + projection_x_coordinate:standard_name = "projection_x_coordinate" ; + double projection_x_coordinate_bnds(projection_x_coordinate, bnds) ; + int sample(sample) ; + sample:long_name = "sample" ; + double latitude(projection_y_coordinate, projection_x_coordinate) ; + latitude:units = "degrees_north" ; + latitude:standard_name = "latitude" ; + latitude:long_name = "latitude" ; + double longitude(projection_y_coordinate, projection_x_coordinate) ; + longitude:units = "degrees_east" ; + longitude:standard_name = "longitude" ; + longitude:long_name = "longitude" ; + int month_number(time) ; + month_number:units = "1" ; + month_number:long_name = "month_number" ; + char season(time, string64) ; + season:units = "1" ; + season:long_name = "season" ; + int season_year(time) ; + season_year:units = "1" ; + season_year:long_name = "season_year" ; + int year(time) ; + year:units = "1" ; + year:long_name = "year" ; + +// global attributes: + :baseline_period = "b8110" ; + :collection = "land-prob" ; + :contact = "ukcpproject@metoffice.gov.uk, UKCP Team, Met Office Hadley Centre" ; + :creation_date = "2021-12-03T17:54:30" ; + :domain = "uk" ; + :frequency = "mon" ; + :institution = "Met Office Hadley Centre (MOHC), FitzRoy Road, Exeter, Devon, EX1 3PB, UK." ; + :institution_id = "MOHC" ; + :prob_data_type = "sample" ; + :project = "UKCP18" ; + :references = "http://ukclimateprojections.metoffice.gov.uk/" ; + :resolution = "25km" ; + :scenario = "rcp85" ; + :source = "Probabilistic climate prediction based on family of Met Office Hadley Centre climate models HadCM3, HadRM3 and HadSM3, plus climate models from other climate centres contributing to IPCC AR5 and CFMIP." ; + :time_slice_type = "30y" ; + :title = "UKCP18 probabilistic projections for total cloud anomaly (%) for UK land points, for the RCP 8.5 scenario with a 1981-2010 baseline." ; + :version = "v20211110" ; + :Conventions = "CF-1.7" ; +} diff --git a/tests/sample-fixture/sample-ncs/rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl b/tests/sample-fixture/sample-ncs/rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl new file mode 100644 index 00000000..4f6eafc9 --- /dev/null +++ b/tests/sample-fixture/sample-ncs/rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130.cdl @@ -0,0 +1,86 @@ +netcdf rss_rcp85_land-cpm_uk_2.2km_01_day_20671201-20681130 { +dimensions: + ensemble_member = 1 ; + time = 360 ; + grid_latitude = 606 ; + grid_longitude = 484 ; + bnds = 2 ; + string27 = 27 ; + string64 = 64 ; +variables: + float rss(ensemble_member, time, grid_latitude, grid_longitude) ; + rss:_FillValue = 1.e+20f ; + rss:standard_name = "surface_net_downward_shortwave_flux" ; + rss:long_name = "Net Surface short wave flux" ; + rss:units = "W m-2" ; + rss:description = "Net Surface short wave flux" ; + rss:label_units = "W m-2" ; + rss:plot_label = "Net Surface short wave flux (W m-2)" ; + rss:cell_methods = "time: mean" ; + rss:grid_mapping = "rotated_latitude_longitude" ; + rss:coordinates = "ensemble_member_id latitude longitude month_number year yyyymmdd" ; + int rotated_latitude_longitude ; + rotated_latitude_longitude:grid_mapping_name = "rotated_latitude_longitude" ; + rotated_latitude_longitude:longitude_of_prime_meridian = 0. ; + rotated_latitude_longitude:earth_radius = 6371229. ; + rotated_latitude_longitude:grid_north_pole_latitude = 37.5 ; + rotated_latitude_longitude:grid_north_pole_longitude = 177.5 ; + rotated_latitude_longitude:north_pole_grid_longitude = 0. ; + int ensemble_member(ensemble_member) ; + ensemble_member:units = "1" ; + ensemble_member:long_name = "ensemble_member" ; + double time(time) ; + time:axis = "T" ; + time:bounds = "time_bnds" ; + time:units = "hours since 1970-01-01 00:00:00" ; + time:standard_name = "time" ; + time:calendar = "360_day" ; + double time_bnds(time, bnds) ; + double grid_latitude(grid_latitude) ; + grid_latitude:axis = "Y" ; + grid_latitude:bounds = "grid_latitude_bnds" ; + grid_latitude:units = "degrees" ; + grid_latitude:standard_name = "grid_latitude" ; + double grid_latitude_bnds(grid_latitude, bnds) ; + double grid_longitude(grid_longitude) ; + grid_longitude:axis = "X" ; + grid_longitude:bounds = "grid_longitude_bnds" ; + grid_longitude:units = "degrees" ; + grid_longitude:standard_name = "grid_longitude" ; + double grid_longitude_bnds(grid_longitude, bnds) ; + char ensemble_member_id(ensemble_member, string27) ; + ensemble_member_id:units = "1" ; + ensemble_member_id:long_name = "ensemble_member_id" ; + double latitude(grid_latitude, grid_longitude) ; + latitude:units = "degrees_north" ; + latitude:standard_name = "latitude" ; + double longitude(grid_latitude, grid_longitude) ; + longitude:units = "degrees_east" ; + longitude:standard_name = "longitude" ; + int month_number(time) ; + month_number:units = "1" ; + month_number:long_name = "month_number" ; + int year(time) ; + year:units = "1" ; + year:long_name = "year" ; + char yyyymmdd(time, string64) ; + yyyymmdd:units = "1" ; + yyyymmdd:long_name = "yyyymmdd" ; + +// global attributes: + :collection = "land-cpm" ; + :contact = "ukcpproject@metoffice.gov.uk" ; + :creation_date = "2021-05-11T14:15:06" ; + :domain = "uk" ; + :frequency = "day" ; + :institution = "Met Office Hadley Centre (MOHC), FitzRoy Road, Exeter, Devon, EX1 3PB, UK." ; + :institution_id = "MOHC" ; + :project = "UKCP18" ; + :references = "https://ukclimateprojections.metoffice.gov.uk" ; + :resolution = "2.2km" ; + :scenario = "rcp85" ; + :source = "UKCP18 realisation from a set of 12 convection-permitting models (HadREM3-RA11M) driven by perturbed variants of the Met Office Unified Model Global Atmosphere GA7 model (HadREM3-GA705) at 12km resolution. The HadREM3-GA705 models were driven by perturbed variants of the global HadGEM3-GC3.05" ; + :title = "UKCP18 land projections - 2.2km convection-permitting climate model, Net Surface short wave flux over the UK for the RCP8.5 scenario" ; + :version = "v20210615" ; + :Conventions = "CF-1.7" ; +} diff --git a/tests/sample-fixture/sample-ncs/summer_rainfall_2001.cdl b/tests/sample-fixture/sample-ncs/summer_rainfall_2001.cdl new file mode 100644 index 00000000..e9d006ad --- /dev/null +++ b/tests/sample-fixture/sample-ncs/summer_rainfall_2001.cdl @@ -0,0 +1,47 @@ +netcdf summer_rainfall_2001 { +dimensions: + projection_y_coordinate = UNLIMITED ; // (290 currently) + projection_x_coordinate = 180 ; + bnds = 2 ; +variables: + float seasonal_rainfall(projection_y_coordinate, projection_x_coordinate) ; + seasonal_rainfall:_FillValue = 1.e+20f ; + seasonal_rainfall:standard_name = "precipitation_amount" ; + seasonal_rainfall:long_name = "Seasonal total precipitation amount" ; + seasonal_rainfall:units = "mm" ; + seasonal_rainfall:cell_methods = "time: sum" ; + seasonal_rainfall:coordinates = "lat lon time" ; + double projection_y_coordinate(projection_y_coordinate) ; + projection_y_coordinate:axis = "Y" ; + projection_y_coordinate:bounds = "projection_y_coordinate_bnds" ; + projection_y_coordinate:units = "m" ; + projection_y_coordinate:standard_name = "projection_y_coordinate" ; + double projection_y_coordinate_bnds(projection_y_coordinate, bnds) ; + double projection_x_coordinate(projection_x_coordinate) ; + projection_x_coordinate:axis = "X" ; + projection_x_coordinate:bounds = "projection_x_coordinate_bnds" ; + projection_x_coordinate:units = "m" ; + projection_x_coordinate:standard_name = "projection_x_coordinate" ; + double projection_x_coordinate_bnds(projection_x_coordinate, bnds) ; + double lat(projection_y_coordinate, projection_x_coordinate) ; + lat:units = "degree_north" ; + lat:standard_name = "latitude" ; + double lon(projection_y_coordinate, projection_x_coordinate) ; + lon:units = "degree_east" ; + lon:standard_name = "longitude" ; + double time ; + time:bounds = "time_bnds" ; + time:units = "hours since 1800-01-01 00:00:00" ; + time:standard_name = "time" ; + time:calendar = "standard" ; + double time_bnds(bnds) ; + +// global attributes: + :comment = "These data are part of the Met Office Gridded Observation and Derived Data Sets in support of UKCP09" ; + :institution = "Met Office" ; + :references = "" ; + :short_name = "seasonal_rainfall" ; + :source = "UKCP09" ; + :title = "Gridded surface climate observations data for the UK" ; + :Conventions = "CF-1.5" ; +} diff --git a/tests/sample-fixture/sample-ncs/tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl b/tests/sample-fixture/sample-ncs/tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl new file mode 100644 index 00000000..f83879b7 --- /dev/null +++ b/tests/sample-fixture/sample-ncs/tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130.cdl @@ -0,0 +1,93 @@ +netcdf tasmax_rcp85_land-rcm_uk_12km_EC-EARTH_r12i1p1_HIRHAM5_day_19801201-19901130 { +dimensions: + ensemble_member = 1 ; + projection_y_coordinate = 112 ; + projection_x_coordinate = 82 ; + time = 3652 ; + bnds = 2 ; + string46 = 46 ; + string64 = 64 ; +variables: + string ensemble_member(ensemble_member) ; + double projection_y_coordinate(projection_y_coordinate) ; + projection_y_coordinate:_FillValue = NaN ; + projection_y_coordinate:axis = "Y" ; + projection_y_coordinate:bounds = "projection_y_coordinate_bnds" ; + projection_y_coordinate:units = "m" ; + projection_y_coordinate:standard_name = "projection_y_coordinate" ; + double projection_x_coordinate(projection_x_coordinate) ; + projection_x_coordinate:_FillValue = NaN ; + projection_x_coordinate:axis = "X" ; + projection_x_coordinate:bounds = "projection_x_coordinate_bnds" ; + projection_x_coordinate:units = "m" ; + projection_x_coordinate:standard_name = "projection_x_coordinate" ; + double time(time) ; + time:_FillValue = NaN ; + time:standard_name = "time" ; + time:bounds = "time_bnds" ; + time:long_name = "time" ; + time:cell_methods = "time: mean" ; + time:units = "days since 1949-12-01" ; + time:calendar = "proleptic_gregorian" ; + double tasmax(ensemble_member, time, projection_y_coordinate, projection_x_coordinate) ; + tasmax:_FillValue = NaN ; + tasmax:regrid_method = "conservative_normed" ; + tasmax:grid_mapping = "transverse_mercator" ; + tasmax:coordinates = "grid_longitude month_number grid_latitude ensemble_member_id year yyyymmdd" ; + int transverse_mercator ; + transverse_mercator:grid_mapping_name = "transverse_mercator" ; + transverse_mercator:longitude_of_prime_meridian = 0. ; + transverse_mercator:semi_major_axis = 6377563.396 ; + transverse_mercator:semi_minor_axis = 6356256.909 ; + transverse_mercator:longitude_of_central_meridian = -2. ; + transverse_mercator:latitude_of_projection_origin = 49. ; + transverse_mercator:false_easting = 400000. ; + transverse_mercator:false_northing = -100000. ; + transverse_mercator:scale_factor_at_central_meridian = 0.9996012717 ; + int64 time_bnds(time, bnds) ; + time_bnds:coordinates = "year yyyymmdd month_number" ; + double projection_y_coordinate_bnds(projection_y_coordinate, bnds) ; + projection_y_coordinate_bnds:_FillValue = NaN ; + double projection_x_coordinate_bnds(projection_x_coordinate, bnds) ; + projection_x_coordinate_bnds:_FillValue = NaN ; + char ensemble_member_id(ensemble_member, string46) ; + ensemble_member_id:units = "1" ; + ensemble_member_id:long_name = "ensemble_member_id" ; + double grid_longitude(projection_y_coordinate, projection_x_coordinate) ; + grid_longitude:_FillValue = NaN ; + grid_longitude:units = "degrees_east" ; + grid_longitude:standard_name = "grid_longitude" ; + double grid_latitude(projection_y_coordinate, projection_x_coordinate) ; + grid_latitude:_FillValue = NaN ; + grid_latitude:units = "degrees_north" ; + grid_latitude:standard_name = "grid_latitude" ; + int64 month_number(time) ; + month_number:units = "1" ; + month_number:long_name = "month_number" ; + int64 year(time) ; + year:units = "1" ; + year:long_name = "year" ; + char yyyymmdd(time, string64) ; + yyyymmdd:units = "1" ; + yyyymmdd:long_name = "yyyymmdd" ; + +// global attributes: + :collection = "EuroCORDEX" ; + :contact = "clair.barnes.16@ucl.ac.uk" ; + :creation_date = "2022-02-08T16:25:09" ; + :domain = "uk" ; + :frequency = "day" ; + :institution = "Danish Meteorological Institute" ; + :institution_id = "DMI" ; + :project = "CORDEX" ; + :resolution = "12km" ; + :scenario = "rcp85" ; + :source = "EuroCORDEX downscaled climate projections based on historical + rcp85 scenarios." ; + string :title = "EuroCORDEX regional projections for maximum air temperature at 1.5m (°C) for the UK, for the historical + RCP 8.5 scenarios." ; + :version = "" ; + :Conventions = "CF-1.6" ; + :driving_model_id = "ICHEC-EC-EARTH" ; + :model_id = "DMI-HIRHAM5" ; + :driving_model_ensemble_member = "r12i1p1" ; + :rcm_version_id = "v1" ; +} diff --git a/tests/test_cli.py b/tests/test_cli.py index 86b65d9d..d377b42d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,82 +14,3 @@ """ -import os -import time -import datetime as dt - -from wflogger.credentials import user_id, hostname, creds -from wflogger.wflogger import INSERT_SQL - -v_1 = {"workflow": "my-model-1", - "tag": "idl-version", - "stages": [(1, "start", 0), (2, "read", 3), (3, "process", 10), (4, "summarise", 4)], - "n_iterations": 200} - -v_2 = {"workflow": "my-model-1", - "tag": "python-version", - "stages": [(1, "start", 0), (2, "read", 8), (3, "process", 12), (4, "summarise", 2)], - "n_iterations": 300} - -host_tmpl = "host{n:03d}.jc.rl.ac.uk" -BAD_HOSTS = [host_tmpl.format(n=n) for n in range(101, 130)] - - -def _random_duration(n): - # Add some randomness - if randint(0, 1000) > 995: - return randint(44, 75) - - secs = randint(-20, 40) * 0.1 + n - if secs < 0: secs = 0.01 - return secs - - -def SLOW_test_load_workflows_simulating_realtime_interactions(): - for wf in (v_1, v_2): - max_iter = wf["n_iterations"] + 1 - - for iteration in range(1, max_iter): - for stage_number, stage, wait in wf["stages"]: - secs = _random_duration(wait) - time.sleep(secs) - insert_record(wf["workflow"], wf["tag"], stage_number, stage, iteration) - - -def _get_random_host(): - n = randint(70, 700) - return host_tmpl.format(n=n) - - -def test_load_workflows_faking_date_times(): - - date_time = dt.datetime.now() - - conn = psycopg2.connect(creds) - curs = conn.cursor() - - for wf in (v_1, v_2): - max_iter = wf["n_iterations"] + 1 - - for iteration in range(1, max_iter): - - hostname = _get_random_host() - - for stage_number, stage, wait in wf["stages"]: - - secs = _random_duration(wait) - if hostname in BAD_HOSTS: - secs += 100 - - date_time += dt.timedelta(seconds=secs) - - workflow, tag = wf["workflow"], wf["tag"] - comment, flag = "", -999 - curs.execute(INSERT_SQL, - (user_id, hostname, workflow, tag, stage_number, - stage, iteration, date_time, comment, flag)) - - conn.commit() - curs.close() - conn.close() - diff --git a/tests/test_readers.py b/tests/test_readers.py new file mode 100644 index 00000000..b6f41d8e --- /dev/null +++ b/tests/test_readers.py @@ -0,0 +1,38 @@ +import os +import pytest + +from checksit.readers.cdl import read as read_cdl + +from .common import TESTDATA_DIR + + +""" +def check(file_path, mappings=None, rules=None, ignore_attrs=None, ignore_all_globals=False, + ignore_all_dimensions=False, ignore_all_variables=False, ignore_all_variable_attrs=False, + auto_cache=False, log_mode="standard", verbose=False, template="auto"): + +""" + +def test_cdl_reader_multiline_parser_1(): + cci_file = os.path.join(TESTDATA_DIR, "esacci/ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2.cdl") + resp = read_cdl(cci_file) + + d = resp.to_dict() + assert d["variables"]["pressure_levels"]["comment"] == \ + ("Pressure levels define the boundaries of the averaging kernel and a priori profile layers.\n", + "Levels are ordered from surface to top of atmosphere.") + + assert d["global_attributes"]["summary"] == \ + ("Weighting Function Modified DOAS (WFMD) was adjusted to simultaneously retrieve column-averaged dry air\n", + "mole fractions of atmospheric methane and carbon monoxide from the shortwave-infrared (SWIR) nadir spectra\n", + "of the TROPOMI instrument onboard Sentinel-5 Precursor.") + + +@pytest.mark.xfail(reason="File contains badly defined number attributes in strings - so let it fail for now.") +def test_cdl_reader_multiline_parser_2(): + cci_file = os.path.join(TESTDATA_DIR, "esacci/ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2.cdl") + resp = read_cdl(cci_file) + + d = resp.to_dict() + + diff --git a/tests/test_sample_data_matches.py b/tests/test_sample_data_matches.py new file mode 100644 index 00000000..4ba6b7e0 --- /dev/null +++ b/tests/test_sample_data_matches.py @@ -0,0 +1,35 @@ +""" +Tests to see that changes have not caused the contents of sample data files to be +parsed into different content. +""" + +import shelve +import os +import pytest + +from deepdiff import DeepDiff +from pprint import pprint + +from checksit.readers.cdl import read as read_cdl + +from .common import TESTDATA_DIR + +sample_fixture_dir = os.path.join(TESTDATA_DIR, "../sample-fixture") +SAMPLE_NCS_DIR = os.path.join(sample_fixture_dir, "sample-ncs") +sample_fixture_file = os.path.join(sample_fixture_dir, "sample-fixture") + + +sample_files = tuple(os.listdir(SAMPLE_NCS_DIR)) + + +@pytest.mark.parametrize('fname', sample_files) +def test_regression_cdl_reader_matches_sample_fixture(fname): + with shelve.open(sample_fixture_file) as sample_dict: + fpath = os.path.join(SAMPLE_NCS_DIR, fname) + content = read_cdl(fpath).to_dict() + + d1, d2 = sample_dict[fname], content + assert DeepDiff(d1, d2) == {}, pprint(DeepDiff(d1, d2), indent=2) + + + diff --git a/tests/testdata/esacci/ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2.cdl b/tests/testdata/esacci/ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2.cdl new file mode 100644 index 00000000..a57286ec --- /dev/null +++ b/tests/testdata/esacci/ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2.cdl @@ -0,0 +1,196 @@ +netcdf ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2 { +dimensions: + sounding_dim = 11254 ; + level_dim = 21 ; + layer_dim = 20 ; + corners_dim = 4 ; +variables: + double time(sounding_dim) ; + time:standard_name = "time" ; + time:long_name = "time" ; + time:units = "seconds since 1970-01-01 00:00:00" ; + time:calendar = "standard" ; + float latitude(sounding_dim) ; + latitude:standard_name = "latitude" ; + latitude:long_name = "latitude" ; + latitude:units = "degree_north" ; + latitude:valid_range = -90.f, 90.f ; + latitude:comment = "Center latitude of the measurement" ; + float longitude(sounding_dim) ; + longitude:standard_name = "longitude" ; + longitude:long_name = "longitude" ; + longitude:units = "degree_east" ; + longitude:valid_range = -180.f, 180.f ; + longitude:comment = "Center longitude of the measurement" ; + float solar_zenith_angle(sounding_dim) ; + solar_zenith_angle:standard_name = "solar_zenith_angle" ; + solar_zenith_angle:long_name = "solar_zenith_angle" ; + solar_zenith_angle:units = "degree" ; + solar_zenith_angle:comment = "Solar zenith angle is the the angle between the line of sight to the sun and the local vertical." ; + float sensor_zenith_angle(sounding_dim) ; + sensor_zenith_angle:standard_name = "sensor_zenith_angle" ; + sensor_zenith_angle:long_name = "sensor_zenith_angle" ; + sensor_zenith_angle:units = "degree" ; + sensor_zenith_angle:comment = "Sensor zenith angle is the the angle between the line of sight to the sensor and the local vertical." ; + float azimuth_difference(sounding_dim) ; + azimuth_difference:long_name = "azimuth difference" ; + azimuth_difference:units = "degree" ; + azimuth_difference:comment = "Relative azimuth angle between sun and sensor direction." ; + float xch4(sounding_dim) ; + xch4:standard_name = "dry_atmosphere_mole_fraction_of_methane" ; + xch4:long_name = "column-averaged dry air mole fraction of atmospheric methane" ; + xch4:units = "1e-9" ; + xch4:comment = "Retrieved column-averaged dry air mole fraction of atmospheric methane (XCH4) in ppb" ; + float xch4_uncertainty(sounding_dim) ; + xch4_uncertainty:long_name = "1-sigma uncertainty of the retrieved column-averaged dry air mole fraction of atmospheric methane" ; + xch4_uncertainty:units = "1e-9" ; + xch4_uncertainty:comment = "1-sigma uncertainty of the retrieved column-averaged dry air mole fraction of atmospheric methane (XCH4) in ppb" ; + int xch4_quality_flag(sounding_dim) ; + xch4_quality_flag:long_name = "quality flag for the retrieved column-averaged dry air mole fraction of atmospheric methane" ; + xch4_quality_flag:flag_values = 0, 1 ; + xch4_quality_flag:flag_meanings = "good_quality potentially_bad_quality" ; + xch4_quality_flag:comment = "0=good, 1=bad" ; + float xco(sounding_dim) ; + xco:long_name = "column-averaged dry air mole fraction of atmospheric carbon monoxide" ; + xco:units = "1e-9" ; + xco:comment = "Retrieved column-averaged dry air mole fraction of atmospheric carbon monoxide (XCO) in ppb" ; + float xco_uncertainty(sounding_dim) ; + xco_uncertainty:long_name = "1-sigma uncertainty of the retrieved column-averaged dry air mole fraction of atmospheric carbon monoxide" ; + xco_uncertainty:units = "1e-9" ; + xco_uncertainty:comment = "1-sigma uncertainty of the retrieved column-averaged dry air mole fraction of atmospheric carbon monoxide (XCO) in ppb" ; + int xco_quality_flag(sounding_dim) ; + xco_quality_flag:long_name = "quality flag for the retrieved column-averaged dry air mole fraction of atmospheric carbon monoxide" ; + xco_quality_flag:flag_values = 0, 1 ; + xco_quality_flag:flag_meanings = "good_quality potentially_bad_quality" ; + xco_quality_flag:comment = "0=good, 1=bad" ; + float pressure_levels(sounding_dim, level_dim) ; + pressure_levels:long_name = "pressure levels" ; + pressure_levels:units = "hPa" ; + pressure_levels:comment = "Pressure levels define the boundaries of the averaging kernel and a priori profile layers.\n", + "Levels are ordered from surface to top of atmosphere." ; + float pressure_weight(sounding_dim, layer_dim) ; + pressure_weight:long_name = "pressure weight" ; + pressure_weight:units = "1" ; + pressure_weight:comment = "Layer dependent weights needed to apply the averaging kernels." ; + float ch4_profile_apriori(sounding_dim, layer_dim) ; + ch4_profile_apriori:long_name = "a priori dry air mole fraction profile of atmospheric methane" ; + ch4_profile_apriori:units = "1e-9" ; + ch4_profile_apriori:comment = "A priori dry-air mole fraction profile of atmospheric methane in ppb.\n", + "All values represent layer averages within the corresponding pressure levels.\n", + "Profiles are ordered from surface to top of atmosphere." ; + float xch4_averaging_kernel(sounding_dim, layer_dim) ; + xch4_averaging_kernel:long_name = "xch4 averaging kernel" ; + xch4_averaging_kernel:units = "1" ; + xch4_averaging_kernel:comment = "Represents the altitude sensitivity of the retrieval as a function of pressure.\n", + "All values represent layer averages within the corresponding pressure levels.\n", + "Profiles are ordered from surface to top of atmosphere." ; + float co_profile_apriori(sounding_dim, layer_dim) ; + co_profile_apriori:long_name = "a priori dry air mole fraction profile of atmospheric carbon monoxide" ; + co_profile_apriori:units = "1e-9" ; + co_profile_apriori:comment = "A priori dry-air mole fraction profile of atmospheric carbon monoxide in ppb.\n", + "All values represent layer averages within the corresponding pressure levels.\n", + "Profiles are ordered from surface to top of atmosphere." ; + float xco_averaging_kernel(sounding_dim, layer_dim) ; + xco_averaging_kernel:long_name = "xco averaging kernel" ; + xco_averaging_kernel:units = "1" ; + xco_averaging_kernel:comment = "Represents the altitude sensitivity of the retrieval as a function of pressure.\n", + "All values represent layer averages within the corresponding pressure levels.\n", + "Profiles are ordered from surface to top of atmosphere." ; + int orbit_number(sounding_dim) ; + orbit_number:long_name = "orbit number" ; + orbit_number:units = "1" ; + orbit_number:comment = "Orbit number" ; + int scanline(sounding_dim) ; + scanline:long_name = "along track dimension index" ; + scanline:units = "1" ; + scanline:comment = "This dimension variable defines the indices along track" ; + int ground_pixel(sounding_dim) ; + ground_pixel:long_name = "across track dimension index" ; + ground_pixel:units = "1" ; + ground_pixel:comment = "This dimension variable defines the indices across track" ; + float latitude_corners(sounding_dim, corners_dim) ; + latitude_corners:long_name = "latitude_corners" ; + latitude_corners:units = "degree_north" ; + latitude_corners:valid_range = -90.f, 90.f ; + latitude_corners:comment = "Corner latitudes of the measurement" ; + float longitude_corners(sounding_dim, corners_dim) ; + longitude_corners:long_name = "longitude_corners" ; + longitude_corners:units = "degree_east" ; + longitude_corners:valid_range = -180.f, 180.f ; + longitude_corners:comment = "Corner longitudes of the measurement" ; + float altitude(sounding_dim) ; + altitude:standard_name = "altitude" ; + altitude:long_name = "altitude" ; + altitude:units = "m" ; + altitude:comment = "Average surface altitude" ; + float surface_roughness(sounding_dim) ; + surface_roughness:long_name = "surface roughness" ; + surface_roughness:units = "m" ; + surface_roughness:comment = "Surface roughness" ; + float apparent_albedo(sounding_dim) ; + apparent_albedo:long_name = "apparent surface albedo" ; + apparent_albedo:units = "1" ; + apparent_albedo:comment = "Retrieved surface albedo at 2313nm" ; + int land_fraction(sounding_dim) ; + land_fraction:long_name = "land fraction" ; + land_fraction:units = "1e-2" ; + land_fraction:valid_range = 0, 100 ; + land_fraction:comment = "Land fraction of the observed scene in percent" ; + float cloud_parameter(sounding_dim) ; + cloud_parameter:long_name = "cloud parameter from strong water vapour absorption" ; + cloud_parameter:units = "1" ; + cloud_parameter:comment = "Ratio of measured to cloud-free reference radiance for selected strong water vapour lines" ; + float co_column(sounding_dim) ; + co_column:long_name = "vertical column of carbon monoxide" ; + co_column:units = "mol m-2" ; + co_column:multiplication_factor_to_convert_to_molecules_percm2 = 6.022141e+19 ; + co_column:comment = "Retrieved vertical column amount of carbon monoxide" ; + float h2o_column(sounding_dim) ; + h2o_column:long_name = "vertical column of water vapour" ; + h2o_column:units = "g cm-2" ; + h2o_column:comment = "Retrieved vertical column amount of water vapour" ; + float h2o_column_uncertainty(sounding_dim) ; + h2o_column_uncertainty:long_name = "1-sigma uncertainty of the retrieved vertical column of atmospheric water vapour" ; + h2o_column_uncertainty:units = "g cm-2" ; + h2o_column_uncertainty:comment = "1-sigma uncertainty of the retrieved vertical column of atmospheric water vapour" ; + +// global attributes: + :_NCProperties = "version=1|netcdflibversion=4.4.1.1|hdf5libversion=1.8.18" ; + :title = "TROPOMI/WFMD XCH4 and XCO" ; + :institution = "University of Bremen" ; + :source = "TROPOMI L1B version 01.00.00" ; + :history = "2021 - product generated with WFMD" ; + :tracking_id = "6c455a82-fd17-4eee-8e00-46d9ed36a40f" ; + :Conventions = "CF-1.6" ; + :product_version = "v1.5" ; + :summary = "Weighting Function Modified DOAS (WFMD) was adjusted to simultaneously retrieve column-averaged dry air\n", + "mole fractions of atmospheric methane and carbon monoxide from the shortwave-infrared (SWIR) nadir spectra\n", + "of the TROPOMI instrument onboard Sentinel-5 Precursor." ; + :keywords = "satellite, Sentinel-5 Precursor, TROPOMI, atmosphere, methane, carbon monoxide" ; + :id = "ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2.nc" ; + :naming_authority = "iup.uni-bremen.de" ; + :keywords_vocabulary = "NASA Global Change Master Directory (GCMD)" ; + :cdm_data_type = "point" ; + :comment = "These data were produced at the University of Bremen in the framework of the ESA GHG-CCI project" ; + :date_created = "20210713T015023Z" ; + :creator_name = "University of Bremen, IUP, Oliver Schneising" ; + :creator_email = "schneising@iup.physik.uni-bremen.de" ; + :project = "Climate Change Initiative - European Space Agency" ; + :geospatial_lat_min = -90 ; + :geospatial_lat_max = 90 ; + :geospatial_lat_units = "degree_north" ; + :geospatial_lon_min = -180 ; + :geospatial_lon_max = 180 ; + :geospatial_lon_units = "degree_east" ; + :geospatial_vertical_min = 0 ; + :geospatial_vertical_max = 100000 ; + :time_coverage_start = "20171110T000000Z" ; + :time_coverage_end = "20171110T235959Z" ; + :time_coverage_duration = "P1D" ; + :time_coverage_resolution = "P1D" ; + :standard_name_vocabulary = "NetCDF Climate and Forecast (CF) Metadata Conventions Version 1.6" ; + :license = "ESA CCI Data Policy: free and open access" ; + :platform = "Sentinel-5 Precursor" ; + :sensor = "TROPOMI" ; + :spatial_resolution = "7km x 7km at nadir (typically)" ; +} diff --git a/tests/testdata/esacci/ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2.cdl b/tests/testdata/esacci/ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2.cdl new file mode 100644 index 00000000..f08aea16 --- /dev/null +++ b/tests/testdata/esacci/ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2.cdl @@ -0,0 +1,203 @@ +netcdf ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2 { +dimensions: + sounding_dim = 3393 ; + polarization_dim = 2 ; + level_dim = 13 ; + layer_dim = 12 ; + window_dim = 4 ; + char_l1bname = 28 ; + gain_dim = 6 ; +variables: + float solar_zenith_angle(sounding_dim) ; + solar_zenith_angle:long_name = "solar zenith angle" ; + solar_zenith_angle:units = "degrees" ; + solar_zenith_angle:comment = "Solar zenith angle is the angle between the line of sight to the sun and the local vertical." ; + float sensor_zenith_angle(sounding_dim) ; + sensor_zenith_angle:long_name = "sensor zenith angle" ; + sensor_zenith_angle:units = "degrees" ; + sensor_zenith_angle:comment = "Sensor zenith angle is the angle between the line of sight to the sensor and the local vertical." ; + double time(sounding_dim) ; + time:long_name = "time" ; + time:units = "seconds since 1970-01-01 00:00:00" ; + time:calender = "standard" ; + float longitude(sounding_dim) ; + longitude:long_name = "longitude" ; + longitude:units = "degrees_east" ; + longitude:comment = "Center longitude of the measurement." ; + longitude:valid_range = -180., 180. ; + float latitude(sounding_dim) ; + latitude:long_name = "latitude" ; + latitude:units = "degrees_north" ; + latitude:comment = "Center latitude of the measurement." ; + latitude:valid_range = -90., 90. ; + float pressure_levels(sounding_dim, level_dim) ; + pressure_levels:long_name = "pressure_levels" ; + pressure_levels:units = "hPa" ; + pressure_levels:comment = "Pressure levels define the boundaries of the averaging kernel and mole fraction profile layers. Surface pressure is represented by the 1st element, i.e., profiles are ordered from surface to top of atmosphere." ; + float pressure_weight(sounding_dim, layer_dim) ; + pressure_weight:long_name = "pressure weight" ; + pressure_weight:units = "1" ; + pressure_weight:comment = "Pressure weights are the layer dependent weights needed to apply the averaging kernels." ; + float xco2(sounding_dim) ; + xco2:long_name = "column-average dry-air mole fraction of atmospheric carbon dioxide" ; + xco2:units = "1e-9" ; + xco2:comment = "Retrieved column-average dry-air mole fraction of atmospheric carbon dioxide (XCO2) in ppm." ; + float xco2_uncertainty(sounding_dim) ; + xco2_uncertainty:long_name = "1-sigma uncertainty of the retrieved column-average dry-air mole fraction of atmospheric carbon dioxide" ; + xco2_uncertainty:units = "1e-9" ; + xco2_uncertainty:comment = "1-sigma uncertainty of the retrieved column-average dry-air mole fraction of atmospheric carbon dioxide (XCO2) in ppm." ; + float xco2_averaging_kernel(sounding_dim, layer_dim) ; + xco2_averaging_kernel:long_name = "normalized column averaging kernel" ; + xco2_averaging_kernel:units = "1" ; + xco2_averaging_kernel:comment = "The normalized column-averaging kernel represents the sensitivity of the retrieved XCO2 to the atmospheric carbon dioxide mole fraction depending on pressure (height). All values represent layer averages within the corresponding pressure levels. Values near one are ideal and indicate that the influence of the a priori is minimal. Profiles are ordered from surface to top of atmosphere." ; + float co2_profile_apriori(sounding_dim, layer_dim) ; + co2_profile_apriori:long_name = "a priori dry-air mole fraction profile of atmospheric carbon dioxide" ; + co2_profile_apriori:units = "1e-9" ; + co2_profile_apriori:comment = "A priori dry-air mole fraction profile of atmospheric carbon dioxide in ppm (http://www.atmos-meas-tech.net/5/1349/2012/amt-5-1349-2012.html). All values represent layer averages within the corresponding pressure levels. Profiles are ordered from surface to top of atmosphere." ; + int xco2_quality_flag(sounding_dim) ; + xco2_quality_flag:long_name = "xco2_quality flag" ; + xco2_quality_flag:units = "1" ; + xco2_quality_flag:values = "0B, 1B; // byte" ; + xco2_quality_flag:meanings = "Quality flag for XCO2 retrieval" ; + xco2_quality_flag:comment = "0=good, 1=bad" ; + int flag_landtype(sounding_dim) ; + flag_landtype:long_name = "flag for land / ocean soundings" ; + flag_landtype:units = "1" ; + flag_landtype:comment = "0 = no glint, 1 = glint" ; + int flag_sunglint(sounding_dim) ; + flag_sunglint:long_name = "flag for normal / sunglint soundings" ; + flag_sunglint:units = "1" ; + flag_sunglint:comment = "0 = no glint, 1 = glint" ; + char gain(sounding_dim, gain_dim) ; + gain:long_name = "gain" ; + gain:units = "1" ; + gain:comment = "Number of gain coefficient is stored for each band. The gain coefficient of each band is calculated from solar calibration mode data. The order is 1P, 1S, 2P, 2S, 3P, 3S." ; + int exposure_id(sounding_dim) ; + exposure_id:long_name = "exposure id" ; + exposure_id:units = "1" ; + exposure_id:comment = "Exposure identification number of the sounding" ; + char l1b_name(sounding_dim, char_l1bname) ; + l1b_name:long_name = "level 1B name" ; + l1b_name:units = "1" ; + l1b_name:comment = "Name of the Level 1B file of the sounding" ; + float signal_to_noise_window(sounding_dim, polarization_dim, window_dim) ; + signal_to_noise_window:long_name = "signal to noise ratio" ; + signal_to_noise_window:units = "1" ; + signal_to_noise_window:comment = "The signal to noise ratio per retrieval window for both polarization directions. Window 1 ranges between 757 nm and 773 nm, Window 2 ranges between 1.59 um and 1.62 um, Window 3 ranges between 1.63 um and 1.65 um and Window 4 ranges between 2.04 um and 2.08 um" ; + float dry_airmass_layer(sounding_dim, layer_dim) ; + dry_airmass_layer:long_name = "Dry airmass per layer" ; + dry_airmass_layer:units = "m-2" ; + dry_airmass_layer:comment = "The dry airmass per layer, units are molecules m-2" ; + float altitude(sounding_dim) ; + altitude:long_name = "Altitude" ; + altitude:units = "m" ; + altitude:comment = "Altitude is the surface elevation in meters" ; + float air_temperature(sounding_dim, level_dim) ; + air_temperature:long_name = "Air temperature at each level" ; + air_temperature:units = "K" ; + air_temperature:comment = "" ; + float surface_altitude_stdv(sounding_dim) ; + surface_altitude_stdv:long_name = "Standard deviation of the surface elevation" ; + surface_altitude_stdv:units = "m" ; + surface_altitude_stdv:comment = "Standard deviation of the surface elevation within the area of the GOSAT-2 sounding, as derived from the SRTM database" ; + float x_wind(sounding_dim, level_dim) ; + x_wind:long_name = "grid_eastward_wind " ; + x_wind:units = "m s-1" ; + x_wind:comment = "\'x\' indicates a vector component along the grid x-axis, positive with increasing x. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. (Vertical motion in the atmosphere has the standard name upward_air_velocity.)" ; + float y_wind(sounding_dim, level_dim) ; + y_wind:long_name = "grid_northward_wind" ; + y_wind:units = "m s-1" ; + y_wind:comment = "\'y\' indicates a vector component along the grid y-axis, positive with increasing y. Wind is defined as a two-dimensional (horizontal) air velocity vector, with no vertical component. (Vertical motion in the atmosphere has the standard name upward_air_velocity.) " ; + float chi2(sounding_dim) ; + chi2:long_name = "Chi-squared" ; + chi2:unit = "1" ; + chi2:comment = "Chi_squared value of the sounding" ; + float optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol(sounding_dim, window_dim) ; + optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol:long_name = "Aerosol optical thickness per retrieval window" ; + optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol:unit = "1" ; + optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol:comment = "\'Layer\' means any layer with upper and lower boundaries that have constant values in some vertical coordinate. There must be a vertical coordinate variable indicating the extent of the layer(s). If the layers are model layers, the vertical coordinate can be model_level_number, but it is recommended to specify a physical coordinate (in a scalar or auxiliary coordinate variable) as well. The optical thickness is the integral along the path of radiation of a volume scattering/absorption/attenuation coefficient. The radiative flux is reduced by a factor exp(-optical_thickness) on traversing the path. A coordinate variable of radiation_wavelength or radiation_frequency can be specified to indicate that the optical thickness applies at specific wavelengths or frequencies. \'Aerosol\' means the suspended liquid or solid particles in air (except cloud droplets). \'Ambient aerosol\' is aerosol that has taken up ambient water through hygroscopic growth. The extent of hygroscopic growth depends on the relative humidity and the composition of the aerosol. The specification of a physical process by the phrase due_to_process means that the quantity named is a single term in a sum of terms which together compose the general quantity named by omitting the phrase. Window 1 ranges between 757 nm and 773 nm, window 2 ranges from 1.59 um and 1.62 um, window 3 ranges between 1.63 um and 1.65 um and window 4 ranges between 2.04 um and 2.08 um" ; + float raw_xco2(sounding_dim) ; + raw_xco2:long_name = "Raw retrieved XCO2 column" ; + raw_xco2:units = "1e-9" ; + raw_xco2:comment = "The retrieved raw XCO2 total column before bias correction" ; + float raw_xco2_err(sounding_dim) ; + raw_xco2_err:long_name = "Raw uncertainty on the XCO2 total column" ; + raw_xco2_err:units = "1e-9" ; + raw_xco2_err:commment = "The raw uncertainty due to measurement noise on the XCO2 total column" ; + float h2o_column(sounding_dim) ; + h2o_column:long_name = "H2O total column" ; + h2o_column:units = "m-2" ; + h2o_column:comment = "The retrieved H2O total column in units of molecules m-2" ; + float surface_albedo_758(sounding_dim) ; + surface_albedo_758:long_name = "Surface albedo at 758 nm" ; + surface_albedo_758:units = "1" ; + surface_albedo_758:comment = "The retrieved surface albedo for window 1, at 758 nm" ; + float surface_albedo_1593(sounding_dim) ; + surface_albedo_1593:long_name = "Surface albedo at 1593 nm" ; + surface_albedo_1593:units = "1" ; + surface_albedo_1593:comment = "The retrieved surface albedo for window 2, at 1593 nm" ; + float surface_albedo_1629(sounding_dim) ; + surface_albedo_1629:long_name = "Surface albedo at 1629 nm" ; + surface_albedo_1629:units = "1" ; + surface_albedo_1629:comment = "The retrieved surface albedo for window 3, at 1629 nm" ; + float surface_albedo_2042(sounding_dim) ; + surface_albedo_2042:long_name = "Surface albedo at 2042 nm" ; + surface_albedo_2042:units = "1" ; + surface_albedo_2042:comment = "The retrieved surface albedo for window 4, at 2042 nm" ; + float intensity_offset_o2a(sounding_dim) ; + intensity_offset_o2a:long_name = "Intensity offset in the O2A-band" ; + intensity_offset_o2a:units = "W cm-2" ; + intensity_offset_o2a:comment = "The retrieved intensity offset in the O2A-band" ; + float aerosol_size(sounding_dim) ; + aerosol_size:long_name = "Retrieved size parameter of the aerosol distribution" ; + aerosol_size:units = "1" ; + aerosol_size:comment = "The size parameter is the exponent of the power-law slope of the aerosol size distribution" ; + float aerosol_central_height(sounding_dim) ; + aerosol_central_height:long_name = "Retrieved aerosol peak height" ; + aerosol_central_height:units = "m" ; + aerosol_central_height:comment = "Peak height of the aerosol Gaussian height distribution" ; + float aerosol_total_column(sounding_dim) ; + aerosol_total_column:long_name = "Retrieved aerosol total column" ; + aerosol_total_column:units = "m-2" ; + aerosol_total_column:comment = "The retrieved total aerosol column in units of particles m-2" ; + +// global attributes: + :title = "ESA CCI GOSAT2 SRFP XCO2" ; + :institution = "SRON Netherlands Institute for Space Research" ; + :source = "GOSAT2 L1B version v210" ; + :history = "02.11.2022 - product generated with RemoTeC v2.0.0" ; + :references = "http://www.esa-ghg-cii.org/ \n http://onlinelibrary.wiley.com/doi/10.1002/jgrd.50332/abstract \n http://onlinelibrary.wiley.com/doi/10.1029/2011GL047888/abstract" ; + :tracking_id = "ea60bd02-6c38-11e5-82fb-d3dc42d9b150" ; + :Conventions = "CF-1.6" ; + :product_version = "v2.0.0" ; + :summary = "The RemoTeC algorithm was designed to analyze GOSAT2 TANSO-FTS2 data, both for CO2 and CH4. It utilizes both a PROXY approach (for CH4) where the scattering due to clouds and aerosols is taken into account by multiplying the observed ratio of CH4/CO2 with a prior model CO2, as well as a Full Physics approach (for CH4 and CO2) where the CO2 and CH4 spectra are simultaneously fitted with aerosol parameters to retrieve the scattering information due to clouds and aerosols. This product uses the Full Physics approach for CH4 " ; + :keywords = "satellite, GOSAT2, TANSO-FTS2, atmosphere, carbondioxide" ; + :id = "ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2.nc" ; + :naming_authority = "home.sron.nl" ; + :keywords_vocabulary = "NASA Global Change Master Directory (GCMD)" ; + :cdm_data_type = "point" ; + :comment = "These data were produced at SRON Netherlands Institute for Space Research in the frame of the ESA GHG CCI project" ; + :date_created = "CreatedWed Nov 2 09:32:11 2022" ; + :creator_name = "SRON Netherlands Institute for Space Research, Andrew Barr" ; + :creator_url = "ftp://ftp.sron.nl/pub/pub/RemoTeC/" ; + :creator_email = "a.g.barr@sron.nl" ; + :project = "Climate Change Initiative - European Space Agency" ; + :geospatial_lat_min = "-90.0f; // float" ; + :geospatial_lat_max = "90.0f; // float" ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_min = "-180.0f; // float" ; + :geospatial_lon_max = "180.0f; // float" ; + :geospatial_lon_units = "degrees_east" ; + :geospatial_vertical_min = "0.0f; // float" ; + :geospatial_vertical_max = "100000.0; // float" ; + :time_coverage_start = "20191231T000000Z" ; + :time_coverage_end = "20191231T235959Z" ; + :time_coverage_duration = "P1D" ; + :time_coverage_resolution = "P1D" ; + :standard_name_vocabulary = "CF Standard Name Table v79" ; + :license = "ESA CCI Data Policy: free and open access" ; + :platform = "GOSAT2" ; + :sensor = "TANSO-FTS2" ; + :spatial_resolution = "10.5km x 10.5km at nadir (typically)" ; + :_CoordSysBuilder = "ucar.nc2.dataset.conv.CF1Convention" ; +}