Updated cordex inventories and migrated files to gitlab

Author: zequihg50

ESGF-inventory/CORDEX/CORDEX_day.csv

@@ -19,16 +19,6 @@
 "cordex.output.AFR-22.GERICS.NCC-NorESM1-M.historical.r0i0p0.REMO2015.v1.fx",671561,true,true
 "cordex.output.AFR-22.GERICS.NCC-NorESM1-M.rcp26.r0i0p0.REMO2015.v1.fx",671561,true,true
 "cordex.output.AFR-22.GERICS.NCC-NorESM1-M.rcp85.r0i0p0.REMO2015.v1.fx",671561,true,true
-"cordex.output.AFR-22.ICTP.ECMWF-ERAINT.evaluation.r1i1p1.RegCM4-7.v0.fx",5170334,true,true
-"cordex.output.AFR-22.ICTP.MOHC-HadGEM2-ES.historical.r1i1p1.RegCM4-7.v0.fx",5164800,true,true
-"cordex.output.AFR-22.ICTP.MOHC-HadGEM2-ES.rcp26.r1i1p1.RegCM4-7.v0.fx",5164805,true,true
-"cordex.output.AFR-22.ICTP.MOHC-HadGEM2-ES.rcp85.r1i1p1.RegCM4-7.v0.fx",5164805,true,true
-"cordex.output.AFR-22.ICTP.MPI-M-MPI-ESM-MR.historical.r1i1p1.RegCM4-7.v0.fx",5164796,true,true
-"cordex.output.AFR-22.ICTP.MPI-M-MPI-ESM-MR.rcp26.r1i1p1.RegCM4-7.v0.fx",5168899,true,true
-"cordex.output.AFR-22.ICTP.MPI-M-MPI-ESM-MR.rcp85.r1i1p1.RegCM4-7.v0.fx",5168899,true,true
-"cordex.output.AFR-22.ICTP.NCC-NorESM1-M.historical.r1i1p1.RegCM4-7.v0.fx",5164790,true,true
-"cordex.output.AFR-22.ICTP.NCC-NorESM1-M.rcp26.r1i1p1.RegCM4-7.v0.fx",5168889,true,true
-"cordex.output.AFR-22.ICTP.NCC-NorESM1-M.rcp85.r1i1p1.RegCM4-7.v0.fx",5168889,true,true
 "cordex.output.AFR-44.CLMcom.CNRM-CERFACS-CNRM-CM5.historical.r0i0p0.CCLM4-8-17.v1.fx",463899,true,true
 "cordex.output.AFR-44.CLMcom.CNRM-CERFACS-CNRM-CM5.rcp45.r0i0p0.CCLM4-8-17.v1.fx",463879,true,true
 "cordex.output.AFR-44.CLMcom.CNRM-CERFACS-CNRM-CM5.rcp85.r0i0p0.CCLM4-8-17.v1.fx",463879,true,true
@@ -343,16 +333,21 @@
 "cordex.output.EUR-11.DMI.NCC-NorESM1-M.historical.r1i1p1.HIRHAM5.v3.fx",2062028,true,true
 "cordex.output.EUR-11.DMI.NCC-NorESM1-M.rcp45.r1i1p1.HIRHAM5.v3.fx",2062034,true,true
 "cordex.output.EUR-11.DMI.NCC-NorESM1-M.rcp85.r1i1p1.HIRHAM5.v3.fx",2062062,true,true
+"cordex.output.EUR-11.GERICS.CNRM-CERFACS-CNRM-CM5.historical.r0i0p0.REMO2015.v2.fx",7233319,true,true
 "cordex.output.EUR-11.GERICS.ECMWF-ERAINT.evaluation.r0i0p0.REMO2015.v1.fx",7381763,true,true
 "cordex.output.EUR-11.GERICS.ICHEC-EC-EARTH.rcp26.r0i0p0.REMO2015.v1.fx",7382767,true,true
+"cordex.output.EUR-11.GERICS.ICHEC-EC-EARTH.rcp45.r0i0p0.REMO2015.v1.fx",7229267,true,true
 "cordex.output.EUR-11.GERICS.IPSL-IPSL-CM5A-LR.historical.r0i0p0.REMO2015.v1.fx",7382787,true,true
 "cordex.output.EUR-11.GERICS.IPSL-IPSL-CM5A-LR.rcp26.r0i0p0.REMO2015.v1.fx",7382767,true,true
+"cordex.output.EUR-11.GERICS.IPSL-IPSL-CM5A-MR.historical.r0i0p0.REMO2015.v1.fx",7229275,true,true
 "cordex.output.EUR-11.GERICS.MIROC-MIROC5.rcp26.r0i0p0.REMO2015.v1.fx",7382767,true,true
 "cordex.output.EUR-11.GERICS.MOHC-HadGEM2-ES.rcp26.r0i0p0.REMO2015.v1.fx",7382767,true,true
+"cordex.output.EUR-11.GERICS.MOHC-HadGEM2-ES.rcp45.r0i0p0.REMO2015.v1.fx",7229267,true,true
 "cordex.output.EUR-11.GERICS.MPI-M-MPI-ESM-LR.historical.r0i0p0.REMO2015.v1.fx",7229275,true,true
 "cordex.output.EUR-11.GERICS.MPI-M-MPI-ESM-LR.rcp85.r0i0p0.REMO2015.v1.fx",7229275,true,true
 "cordex.output.EUR-11.GERICS.NCC-NorESM1-M.historical.r0i0p0.REMO2015.v1.fx",7229275,true,true
 "cordex.output.EUR-11.GERICS.NCC-NorESM1-M.rcp26.r0i0p0.REMO2015.v1.fx",7229275,true,true
+"cordex.output.EUR-11.GERICS.NCC-NorESM1-M.rcp45.r0i0p0.REMO2015.v1.fx",7229267,true,true
 "cordex.output.EUR-11.GERICS.NCC-NorESM1-M.rcp85.r0i0p0.REMO2015.v1.fx",7229275,true,true
 "cordex.output.EUR-11.GERICS.NOAA-GFDL-GFDL-ESM2G.historical.r0i0p0.REMO2015.v1.fx",7382787,true,true
 "cordex.output.EUR-11.GERICS.NOAA-GFDL-GFDL-ESM2G.rcp26.r0i0p0.REMO2015.v1.fx",7382767,true,true
@@ -384,6 +379,7 @@
 "cordex.output.EUR-11.KNMI.MOHC-HadGEM2-ES.rcp45.r1i1p1.RACMO22E.v2.fx",4445835,true,true
 "cordex.output.EUR-11.KNMI.MOHC-HadGEM2-ES.rcp85.r1i1p1.RACMO22E.v2.fx",4445835,true,true
 "cordex.output.EUR-11.KNMI.MPI-M-MPI-ESM-LR.historical.r1i1p1.RACMO22E.v1.fx",4449681,true,true
+"cordex.output.EUR-11.KNMI.MPI-M-MPI-ESM-LR.rcp26.r1i1p1.RACMO22E.v1.fx",4451741,true,true
 "cordex.output.EUR-11.KNMI.MPI-M-MPI-ESM-LR.rcp85.r1i1p1.RACMO22E.v1.fx",4449653,true,true
 "cordex.output.EUR-11.KNMI.NCC-NorESM1-M.historical.r1i1p1.RACMO22E.v1.fx",4449675,true,true
 "cordex.output.EUR-11.KNMI.NCC-NorESM1-M.rcp26.r1i1p1.RACMO22E.v1.fx",4451735,true,true
@@ -414,6 +410,7 @@
 "cordex.output.EUR-11.SMHI.MPI-M-MPI-ESM-LR.rcp45.r0i0p0.RCA4.v1a.fx",4490512,true,true
 "cordex.output.EUR-11.SMHI.MPI-M-MPI-ESM-LR.rcp85.r0i0p0.RCA4.v1a.fx",4490512,true,true
 "cordex.output.EUR-11.SMHI.NCC-NorESM1-M.historical.r0i0p0.RCA4.v1.fx",4497330,true,true
+"cordex.output.EUR-11.SMHI.NCC-NorESM1-M.rcp45.r0i0p0.RCA4.v1.fx",4497312,true,true
 "cordex.output.EUR-11.SMHI.NCC-NorESM1-M.rcp85.r0i0p0.RCA4.v1.fx",4497312,true,true
 "cordex.output.EUR-11.UHOH.MPI-M-MPI-ESM-LR.rcp85.r0i0p0.WRF361H.v1.fx",7132643,true,true
 "cordex.output.EUR-22.GERICS.ECMWF-ERAINT.evaluation.r0i0p0.REMO2015.v1.fx",2015051,true,true

ESGF-inventory/CORDEX/CORDEX_fx.csv

@@ -1,58 +1,3 @@
 # CORDEX ATLAS dataset
 
-The management of the dataset involves the following processes:
-
-- Discovering available datasets in ESGF and generation of inventories in csv format.
-- Downloading available datasets from ESGF and reporting status of the download.
-- Generation on NcMLs.
-
-This is a long process so, when a new update is required, I suggest to work in a specific branch and commit changes after every step of the process. I also suggest to merge with a commit message including the date of the merge and tag the merge commit (ex. `cordex-2020-06-23`).
-
-## Generation of inventories of datasets available in ESGF
-
-```bash
-scripts/CORDEX.sh
-```
-
-This creates the inventories `CORDEX_day.json` and `CORDEX_fx.json` and the metalink files in the `metalink` directory.
-
-## Downloading available datasets from ESGF
-
-Download the selected files from ESGF using aria2c and metalink files. You can check the status of the download in `download-report`, generated using `../esgf-check -m -r /oceano/gmeteo/WORK/PROYECTOS/2020_C3S_34d/synda/data/ metalinks/* > download-report`.
-
-## Generation of NcMLs
-
-See `scripts/publisher.sh`.
-
-## Unique inventory
-
-jq can be used to create an unique inventory combining daily and fixed `time_frequency` values. Here we show how (just for reference).
-
-```
-jq -r --slurp --arg variables "${variables}" '
-	map(. + map_values(arrays|first)) | 
-	map(select(.replica == false)) |
-
-	map(. + { dataset_all_runs: (.master_id|split(".")|del(.[6,9,10])|join(".")) }) |
-	group_by(.dataset_all_runs) |
-	map( {	dataset_all_runs: .[0].dataset_all_runs,
-			time: map(select(.time_frequency != "fx")),
-			fixed: map(select(.time_frequency == "fx")) }) |
-
-	map(.time |= group_by(.ensemble)) |
-	map(.time |= map({ensemble: .[0].ensemble, size: map(.size)|add, variables: map(.variable)})) |
-	map(.fixed |= {ensemble: .[0].ensemble, size: map(.size)|add, variables: map(.variable)}) |
-
-	map({dataset_all_runs, variables: [.time, [.fixed]]|combinations}) |
-	map({dataset_id:
-			((.dataset_all_runs|split(".")[:6]) +
-			[(.variables|map(select(.ensemble != "r0i0p0"))|map(.ensemble)|first)] +
-			(.dataset_all_runs|split(".")[6:])) | join("."),
-		size: .variables|map(.size)|add,
-		variables: .variables|map(.variables)|add}) |
-
-	(reduce ($variables|split(","))[] as $v ({}; . + {($v): false})) as $false_variables |
-	map(reduce .variables[] as $v ({dataset_id, size} + $false_variables; . + {($v): true})) |
-
-	(["dataset_id", "size"] + ($variables|split(",")|sort)) as $keys | $keys, map([.[ $keys[] ]])[] | @csv'
-```
+See issue in [gitlab](https://gitlab.com/scds/data-management/-/issues/4).