From e25f741dc88e946aeb125804daeb349676c7fd57 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Mon, 4 Jan 2021 12:10:26 +0100 Subject: [PATCH 1/4] Add function to generate rootpath entry for user config --- README.md | 5 +++-- esmvaltool_sample_data/__init__.py | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 01f9b38..b6c6a7e 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,14 @@ If you want to use this dataset in your ESMValTool projects, add the following l ```yaml rootpath: - CMIP6: /esmvaltool_sample_data/data/timeseries/ + CMIP6: + - /esmvaltool_sample_data/data/timeseries/CMIP6 drs: CMIP6: default ``` -If you have installed `esmvaltool_sample_data`, you can find the `rootpath` using `python -c 'from esmvaltool_sample_data import base_dir; print(base_dir.parent)'` +If you have installed `esmvaltool_sample_data`, you can find the `rootpath` using `python -c 'import esmvaltool_sample_data, yaml; print(yaml.dump(esmvaltool_sample_data.get_rootpaths()))'` ## License diff --git a/esmvaltool_sample_data/__init__.py b/esmvaltool_sample_data/__init__.py index 9c38c91..0baf3f8 100644 --- a/esmvaltool_sample_data/__init__.py +++ b/esmvaltool_sample_data/__init__.py @@ -7,7 +7,9 @@ VERBOSE = False -# This ignore list is used to help with debugging +# This ignore list is used to help with debugging only +# for a more permanent solution, add +# problematic datasets the `ignore` section in `../datasets.yml`. ignore_list = [ # 'CMIP6/CMIP/AWI/AWI-ESM-1-1-LR/historical/r1i1p1f1/Amon/ta/gn/v20200212', ] @@ -50,6 +52,7 @@ def load_cubes_from_input_dirs(input_dirs: list) -> 'iris.Cube': def filter_ignored_datasets(dirs, root): + """Filter datasets defined in the global `ignore` list.""" for drc in dirs: test_drc = str(drc.relative_to(root)) if test_drc not in ignore_list: @@ -87,6 +90,22 @@ def load_timeseries_cubes(mip_table: str = 'Amon') -> list: return list(cubes) +def get_rootpaths() -> dict: + """Return a dict with rootpaths to update the user config in ESMValTool.""" + rootpath = { + 'rootpath': { + 'CMIP6': [ + str(base_dir / 'data' / 'timeseries' / 'CMIP6'), + ] + }, + 'drs': { + 'CMIP6': 'default', + }, + } + + return rootpath + + if __name__ == '__main__': VERBOSE = True From 552c3e9818522751a83b547eec899fe3fd3c3f05 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Tue, 5 Jan 2021 13:49:27 +0000 Subject: [PATCH 2/4] Update README.md Co-authored-by: Bouwe Andela --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6c6a7e..806d549 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you want to use this dataset in your ESMValTool projects, add the following l ```yaml rootpath: CMIP6: - - /esmvaltool_sample_data/data/timeseries/CMIP6 + - /esmvaltool_sample_data/data/timeseries/CMIP6 drs: CMIP6: default From 4f5e18acbdbd5327f19bd9df0ffcf703e11a3659 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Wed, 6 Jan 2021 09:20:48 +0100 Subject: [PATCH 3/4] Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6c6a7e..3e00115 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ drs: CMIP6: default ``` -If you have installed `esmvaltool_sample_data`, you can find the `rootpath` using `python -c 'import esmvaltool_sample_data, yaml; print(yaml.dump(esmvaltool_sample_data.get_rootpaths()))'` +If you have installed `esmvaltool_sample_data`, you can find the `rootpath` settings using `python -c 'import esmvaltool_sample_data, yaml; print(yaml.dump(esmvaltool_sample_data.get_rootpaths()))'` ## License From ce471e898f02590e07677b1bb3254a13e6303be3 Mon Sep 17 00:00:00 2001 From: Stef Smeets Date: Wed, 6 Jan 2021 09:29:20 +0100 Subject: [PATCH 4/4] Specify that the data are stored using the DKRZ syntax --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 31401e6..eef97f6 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ The goal is to keep the repository size small (~ 100 MB), so it can be easily do At present, the repository contains 44 monthly, and 35 daily timeseries datasets. The specifications can be found in [`datasets.yml`](esmvaltool_sample_data/datasets.yml). +The data are stored using the data reference syntax structure from DKRZ. + ## Usage The functionality of this repository is kept simple. @@ -31,7 +33,7 @@ rootpath: - /esmvaltool_sample_data/data/timeseries/CMIP6 drs: - CMIP6: default + CMIP6: DKRZ ``` If you have installed `esmvaltool_sample_data`, you can find the `rootpath` settings using `python -c 'import esmvaltool_sample_data, yaml; print(yaml.dump(esmvaltool_sample_data.get_rootpaths()))'`