update E-OBS naming

Author: chantreux

requests/insitu-gridded-observations-europe.csv

@@ -1,2 +1,2 @@
 dataset,dataset_type,filename_variable,cds_request_variable,cds_years_start,cds_years_end,cds_product_type,cds_grid_resolution,cds_version,input_path,output_path,script,product_type,temporal_resolution,interpolation,interpolation_file
-insitu-gridded-observations-europe,observation,tp,mean_temperature,1950,2024,ensemble_mean,0_1deg,31_0e,CDS,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/,scripts/download/insitu-gridded-observations-europe.py,raw,daily,native,None
+insitu-gridded-observations-europe,observation,tg,mean_temperature,1950,2024,ensemble_mean,0_1deg,31_0e,CDS,/lustre/gmeteo/WORK/DATA/C3S-CDS/ERA5_temp/,scripts/download/insitu-gridded-observations-europe.py,raw,daily,native,None

scripts/download/insitu-gridded-observations-europe.py

@@ -6,10 +6,10 @@
 
 
 def get_output_filename(row,dataset):
-
+    version=row["cds_version"]
     var=row["filename_variable"]
-    date=f"{row["cds_years_start"]}-{row["cds_years_end"]}"
-    return f"{var}_{dataset}_{date}.nc"
+    date=f"{row['cds_years_start']}-{row['cds_years_end']}"
+    return f"{var}_{dataset}_{date}_{version}.zip"
 
 def create_request(row):
     var=row["cds_request_variable"]
@@ -31,7 +31,6 @@ def create_request(row):
 def main():
     dataset = "insitu-gridded-observations-europe"
     variables_file_path = f"../../requests/{dataset}.csv"
-    download_files(dataset, variables_file_path, create_request, get_output_filename)
-
+    download_files(dataset, variables_file_path, create_request, get_output_filename,year_request=False)
 if __name__ == "__main__":
     main()

scripts/utilities/utils.py

@@ -7,6 +7,9 @@
 import yaml
 from concurrent.futures import ThreadPoolExecutor
 import pandas as pd
+from c3s_atlas.utils import (
+    extract_zip_and_delete
+)
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 warnings.filterwarnings("ignore")
@@ -252,6 +255,7 @@ def download_files(dataset, variables_file_path, create_request_func, get_output
                             logging.info(f"{path_file} already exists, skipping")
                             continue
                         futures.append(executor.submit(download_single_file, dataset, request, path_file))
+
             elif year_request:
                 for year in year_list:
                     request = create_request_func(row, year)
@@ -265,11 +269,12 @@ def download_files(dataset, variables_file_path, create_request_func, get_output
                 request = create_request_func(row)
                 file = get_output_filename_func(row, dataset)
                 path_file = dest_dir / file
-                if path_file.exists():
+                if Path(str(path_file).replace('zip','nc')).exists():
                     logging.info(f"{path_file} already exists, skipping")
                     continue
                 futures.append(executor.submit(download_single_file, dataset, request, path_file))
-
+                if path_file.suffix == '.zip':
+                            futures.append(executor.submit(extract_zip_and_delete, path_file))
             for future in futures:
                 try:
                     future.result()