Skip to content

Commit d3f5c33

Browse files
chantreuxchantreux
committed
add observation to catalogue production conditions
1 parent 5d63dcf commit d3f5c33

9 files changed

+31
-96
lines changed

‎catalogues/catalogues/all_catalogues.csv‎

Lines changed: 20 additions & 45 deletions
Large diffs are not rendered by default.

‎catalogues/catalogues/derived-era5-single-levels-daily-statistics_raw_catalogue.csv‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
<<<<<<< HEAD
21
variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date,script
32
t2m,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/daily/native/t2m,CDS,True,True,1940,2024,scripts/download/derived-era5-single-levels-daily-statistics.py
43
t2mx,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/daily/native/t2mx,CDS,True,True,1940,2024,scripts/download/derived-era5-single-levels-daily-statistics.py
@@ -8,14 +7,3 @@ tp,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,daily,na
87
d2m,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/daily/native/d2m,CDS,True,True,1940,2024,scripts/download/derived-era5-single-levels-daily-statistics.py
98
sp,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/daily/native/sp,CDS,True,True,1940,2024,scripts/download/derived-era5-single-levels-daily-statistics.py
109
ssrd,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/daily/native/ssrd,CDS,True,True,1940,2024,scripts/download/derived-era5-single-levels-daily-statistics.py
11-
=======
12-
variable,model,experiment,dataset,dataset_type,product_type,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date
13-
t2m,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/t2m,CDS,False,False,,
14-
t2mx,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/t2mx,CDS,False,False,,
15-
t2mn,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/t2mn,CDS,False,False,,
16-
e,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/e,CDS,False,False,,
17-
tp,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/tp,CDS,False,False,,
18-
d2m,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/d2m,CDS,False,False,,
19-
sp,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/sp,CDS,False,False,,
20-
ssrd,None,None,derived-era5-single-levels-daily-statistics,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/derived-era5-single-levels-daily-statistics/ssrd,CDS,False,False,,
21-
>>>>>>> 78ace9e181fc202949816aef1d8a9505b407cf13

‎catalogues/catalogues/reanalysis-cerra-land_derived_catalogue.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
variable,model,experiment,dataset,dataset_type,product_type,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date
2-
eva,None,None,reanalysis-cerra-land,reanalysis,derived,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-land/eva,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/eva,False,False,,
3-
ssrd,None,None,reanalysis-cerra-land,reanalysis,derived,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-land/ssrd,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/ssrd,False,False,,
4-
strd,None,None,reanalysis-cerra-land,reanalysis,derived,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-land/strd,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/strd,False,False,,
1+
variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date,script
2+
eva,None,None,reanalysis-cerra-land,reanalysis,derived,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-land/daily/native/eva,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/daily/native/eva,True,True,,,scripts/derived/reanalysis-cerra-land_accumulation.py
3+
ssrd,None,None,reanalysis-cerra-land,reanalysis,derived,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-land/daily/native/ssrd,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/daily/native/ssrd,True,False,,,scripts/derived/reanalysis-cerra-land_accumulation.py
4+
strd,None,None,reanalysis-cerra-land,reanalysis,derived,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-land/daily/native/strd,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/daily/native/strd,False,False,,,scripts/derived/reanalysis-cerra-land_accumulation.py

‎catalogues/catalogues/reanalysis-cerra-land_raw_catalogue.csv‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,5 @@
1-
<<<<<<< HEAD
21
variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date,script
3-
tp,None,None,reanalysis-cerra-land,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/daily/native/tp,CDS,True,True,198408.0,202106.0,scripts/reanalysis-cerra-land-single-levels.py
4-
eva,None,None,reanalysis-cerra-land,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/hourly/native/eva,CDS,True,True,198501.0,202109.0,scripts/reanalysis-cerra-land-single-levels.py
5-
ssrd,None,None,reanalysis-cerra-land,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/hourly/native/ssrd,CDS,True,False,198501.0,201006.0,scripts/reanalysis-cerra-land-single-levels.py
6-
strd,None,None,reanalysis-cerra-land,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/hourly/native/strd,CDS,False,False,,,scripts/reanalysis-cerra-land-single-levels.py
7-
=======
8-
variable,model,experiment,dataset,dataset_type,product_type,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date
9-
tp,None,None,reanalysis-cerra-land,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/tp,CDS,False,False,,
10-
eva,None,None,reanalysis-cerra-land,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/eva,CDS,False,False,,
11-
ssrd,None,None,reanalysis-cerra-land,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/ssrd,CDS,False,False,,
12-
strd,None,None,reanalysis-cerra-land,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/strd,CDS,False,False,,
13-
>>>>>>> 78ace9e181fc202949816aef1d8a9505b407cf13
2+
tp,None,None,reanalysis-cerra-land,reanalysis,raw,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/daily/native/tp,CDS,True,True,198408.0,202106.0,scripts/download/reanalysis-cerra-land-single-levels.py
3+
eva,None,None,reanalysis-cerra-land,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/hourly/native/eva,CDS,True,True,198501.0,202109.0,scripts/download/reanalysis-cerra-land-single-levels.py
4+
ssrd,None,None,reanalysis-cerra-land,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/hourly/native/ssrd,CDS,True,False,198501.0,201908.0,scripts/download/reanalysis-cerra-land-single-levels.py
5+
strd,None,None,reanalysis-cerra-land,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-land/hourly/native/strd,CDS,False,False,,,scripts/download/reanalysis-cerra-land-single-levels.py

‎catalogues/catalogues/reanalysis-cerra-single-levels_derived_catalogue.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,
22
t2m,None,None,reanalysis-cerra-single-levels,reanalysis,derived,hourly,gr006,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-single-levels/hourly/gr006/t2m,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/t2m,True,True,198801,201712,scripts/interpolation/reanalysis-cerra-single-levels.py
33
r2,None,None,reanalysis-cerra-single-levels,reanalysis,derived,hourly,gr006,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-single-levels/hourly/gr006/r2,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/r2,True,True,198801,201712,scripts/interpolation/reanalysis-cerra-single-levels.py
44
si10,None,None,reanalysis-cerra-single-levels,reanalysis,derived,hourly,gr006,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-single-levels/hourly/gr006/si10,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/si10,True,True,198801,201712,scripts/interpolation/reanalysis-cerra-single-levels.py
5-
ssrd,None,None,reanalysis-cerra-single-levels,reanalysis,derived,hourly,gr006,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-single-levels/hourly/gr006/ssrd,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/ssrd,True,False,198801,200703,scripts/interpolation/reanalysis-cerra-single-levels.py
5+
ssrd,None,None,reanalysis-cerra-single-levels,reanalysis,derived,hourly,gr006,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-cerra-single-levels/hourly/gr006/ssrd,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/ssrd,True,False,198801,201304,scripts/interpolation/reanalysis-cerra-single-levels.py

‎catalogues/catalogues/reanalysis-cerra-single-levels_raw_catalogue.csv‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,5 @@
1-
<<<<<<< HEAD
21
variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date,script
32
t2m,None,None,reanalysis-cerra-single-levels,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/t2m,CDS,True,True,198801,201712,scripts/download/reanalysis-cerra-single-levels.py
43
r2,None,None,reanalysis-cerra-single-levels,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/r2,CDS,True,True,198801,201712,scripts/download/reanalysis-cerra-single-levels.py
54
si10,None,None,reanalysis-cerra-single-levels,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/si10,CDS,True,True,198801,201712,scripts/download/reanalysis-cerra-single-levels.py
65
ssrd,None,None,reanalysis-cerra-single-levels,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/hourly/native/ssrd,CDS,True,True,198801,201712,scripts/download/reanalysis-cerra-single-levels.py
7-
=======
8-
variable,model,experiment,dataset,dataset_type,product_type,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date
9-
t2m,None,None,reanalysis-cerra-single-levels,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/t2m,CDS,False,False,,
10-
r2,None,None,reanalysis-cerra-single-levels,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/r2,CDS,False,False,,
11-
si10,None,None,reanalysis-cerra-single-levels,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/si10,CDS,False,False,,
12-
ssrd,None,None,reanalysis-cerra-single-levels,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-cerra-single-levels/ssrd,CDS,False,False,,
13-
>>>>>>> 78ace9e181fc202949816aef1d8a9505b407cf13

‎catalogues/catalogues/reanalysis-era5-single-levels_derived_catalogue.csv‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,2 @@
1-
<<<<<<< HEAD
21
variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date,script
32
sfcwind,None,None,reanalysis-era5-single-levels,reanalysis,derived,daily,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-era5-single-levels/daily/native/sfcwind,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-era5-single-levels/daily/native/sfcwind,True,True,2022,2024,scripts/derived/reanalysis-era5-single-levels.py
4-
=======
5-
variable,model,experiment,dataset,dataset_type,product_type,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date
6-
sfcwind,None,None,reanalysis-era5-single-levels,reanalysis,derived,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/derived/reanalysis-era5-single-levels/sfcwind,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-era5-single-levels/sfcwind,False,False,,
7-
>>>>>>> 78ace9e181fc202949816aef1d8a9505b407cf13

‎catalogues/catalogues/reanalysis-era5-single-levels_raw_catalogue.csv‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
1-
<<<<<<< HEAD
21
variable,model,experiment,dataset,dataset_type,product_type,temporal_resolution,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date,script
32
u10,None,None,reanalysis-era5-single-levels,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-era5-single-levels/hourly/native/u10,CDS,True,True,2022,2024,scripts/download/reanalysis-era5-single-levels.py
43
v10,None,None,reanalysis-era5-single-levels,reanalysis,raw,hourly,native,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-era5-single-levels/hourly/native/v10,CDS,True,True,2022,2024,scripts/download/reanalysis-era5-single-levels.py
5-
=======
6-
variable,model,experiment,dataset,dataset_type,product_type,interpolation,data_path,origin_path,start_file_exists,final_file_exists,earliest_date,latest_date
7-
u10,None,None,reanalysis-era5-single-levels,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-era5-single-levels/u10,CDS,False,False,,
8-
v10,None,None,reanalysis-era5-single-levels,reanalysis,raw,,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/raw/reanalysis-era5-single-levels/v10,CDS,False,False,,
9-
>>>>>>> 78ace9e181fc202949816aef1d8a9505b407cf13

‎scripts/catalogue/produce_catalog.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ def process_csv_file(file_path, type_data):
166166
aux_df, dataset_type = create_auxiliar_df(data)
167167
project = os.path.basename(file_path).split('.')[0]
168168

169-
if dataset_type in ["reanalysis"]:
169+
if dataset_type in ["reanalysis","observation"]:
170170
simss = [project]
171171
scess = ["historical"]
172172
else:
@@ -178,9 +178,8 @@ def process_csv_file(file_path, type_data):
178178

179179
for ind in df_final.index:
180180
for col in df_final.columns:
181-
if dataset_type in ["reanalysis"]:
181+
if dataset_type in ["reanalysis","observation"]:
182182
logging.info(f"Processing variable {col[0]} for reanalysis dataset {project}")
183-
logging.info(f"aux_df: {aux_df}")
184183
if aux_df.loc[aux_df['variable']==col[0]]['start_file_exists'].squeeze()==True and \
185184
aux_df.loc[aux_df['variable']==col[0]]['final_file_exists'].squeeze()==True:
186185
value=0

0 commit comments

Comments
 (0)