Migrated CMIP6 to gitlab

Author: zequihg50

.gitmodules

@@ -1,5 +1,10 @@
 "dataset_id","size","sftlf"
 "CMIP6.CMIP.AS-RCEC.TaiESM1.historical.r1i1p1f1.fx.gn",75766,true
+"CMIP6.CMIP.AWI.AWI-CM-1-1-MR.historical.r1i1p1f1.fx.gn",58553,true
+"CMIP6.CMIP.AWI.AWI-CM-1-1-MR.historical.r2i1p1f1.fx.gn",58553,true
+"CMIP6.CMIP.AWI.AWI-CM-1-1-MR.historical.r3i1p1f1.fx.gn",58553,true
+"CMIP6.CMIP.AWI.AWI-CM-1-1-MR.historical.r4i1p1f1.fx.gn",58553,true
+"CMIP6.CMIP.AWI.AWI-CM-1-1-MR.historical.r5i1p1f1.fx.gn",58553,true
 "CMIP6.CMIP.AWI.AWI-ESM-1-1-LR.historical.r1i1p1f1.fx.gn",52215,true
 "CMIP6.CMIP.CAS.FGOALS-f3-L.historical.r1i1p1f1.fx.gr",231812,true
 "CMIP6.CMIP.CAS.FGOALS-g3.historical.r1i1p1f1.fx.gn",76916,true
@@ -45,10 +50,25 @@
 "CMIP6.CMIP.CCCma.CanESM5.historical.r24i1p2f1.fx.gn",57597,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r25i1p1f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r25i1p2f1.fx.gn",57597,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r26i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r27i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r28i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r29i1p2f1.fx.gn",57837,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r2i1p1f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r2i1p2f1.fx.gn",59765,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r30i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r31i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r32i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r33i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r34i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r35i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r36i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r37i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r38i1p2f1.fx.gn",57837,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r39i1p2f1.fx.gn",57837,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r3i1p1f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r3i1p2f1.fx.gn",59765,true
+"CMIP6.CMIP.CCCma.CanESM5.historical.r40i1p2f1.fx.gn",57837,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r4i1p1f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r4i1p2f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r5i1p1f1.fx.gn",59765,true
@@ -61,6 +81,7 @@
 "CMIP6.CMIP.CCCma.CanESM5.historical.r8i1p2f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r9i1p1f1.fx.gn",59765,true
 "CMIP6.CMIP.CCCma.CanESM5.historical.r9i1p2f1.fx.gn",59765,true
+"CMIP6.CMIP.CMCC.CMCC-CM2-SR5.historical.r1i1p1f1.fx.gn",77031,true
 "CMIP6.CMIP.CNRM-CERFACS.CNRM-CM6-1-HR.historical.r1i1p1f2.fx.gr",121611,true
 "CMIP6.CMIP.CNRM-CERFACS.CNRM-CM6-1.historical.r10i1p1f2.fx.gr",53801,true
 "CMIP6.CMIP.CNRM-CERFACS.CNRM-CM6-1.historical.r11i1p1f2.fx.gr",53797,true
@@ -104,9 +125,15 @@
 "CMIP6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.historical.r1i1p1f1.fx.gn",61347,true
 "CMIP6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.historical.r2i1p1f1.fx.gn",61347,true
 "CMIP6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.historical.r3i1p1f1.fx.gn",59299,true
+"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r10i1p1f1.fx.gn",58938,true
 "CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r1i1p1f1.fx.gn",60986,true
 "CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r2i1p1f1.fx.gn",60986,true
 "CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r3i1p1f1.fx.gn",60986,true
+"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r4i1p1f1.fx.gn",58938,true
+"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r5i1p1f1.fx.gn",58938,true
+"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r6i1p1f1.fx.gn",58938,true
+"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r8i1p1f1.fx.gn",58938,true
+"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r9i1p1f1.fx.gn",58938,true
 "CMIP6.CMIP.E3SM-Project.E3SM-1-0.historical.r1i1p1f1.fx.gr",93496,true
 "CMIP6.CMIP.E3SM-Project.E3SM-1-0.historical.r2i1p1f1.fx.gr",93496,true
 "CMIP6.CMIP.E3SM-Project.E3SM-1-0.historical.r3i1p1f1.fx.gr",93496,true

ESGF-inventory/CMIP6/CMIP6_day.csv

@@ -1,28 +1,3 @@
 # CMIP6 ATLAS dataset
 
-The management of the dataset involves the following processes:
-
-- Discovering available datasets in ESGF and generation of inventories in csv format.
-- Downloading available datasets from ESGF and reporting status of the download.
-- Generation on NcMLs.
-
-This is a long process so, when a new update is required, I suggest to work in a specific branch and commit changes after every step of the process. I also suggest to merge with a commit message including the date of the merge and tag the merge commit (ex. `cmip6-2020-06-23`).
-
-## Generation of inventories of datasets available in ESGF
-
-```bash
-scripts/CMIP6.sh
-```
-
-This creates the inventories `CMIP6_day.json`, `CMIP6_mon.json` and `CMIP6_fx.json`.
-Manual revision of these inventories is required and the results should be saved in the file `CMIP6_day_1run.csv`.
-
-The script `scripts/inventory-to-metalink` inspects `CMIP6_day_1run.csv` and generates the corresponding metalink file, used for data download.
-
-## Downloading available datasets from ESGF
-
-Download the selected files from ESGF using aria2c and metalink file. You can check the status of the download in `download-report`, generated by `../esgf-check -m metalink > download-report`.
-
-## Generation of NcMLs
-
-See `scripts/publisher.sh`.
+See issue in [gitlab](https://gitlab.com/scds/data-management/-/issues/8).