From 9d40f740b2389e4af6494c28825a0a28d28158f7 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 2 Oct 2020 15:29:00 +0200 Subject: [PATCH 01/10] Deprecate use of write_plots, write_netcdf, output_file_type from config-user.yml Please define those settings in the diagnostic script setting of the recipe instead, for those diagnostic scripts that support these settings --- doc/quickstart/configure.rst | 24 ------------------------ doc/quickstart/output.rst | 11 +++-------- esmvalcore/_config.py | 18 ++++++++++++++---- esmvalcore/_recipe.py | 13 ++++++++++--- esmvalcore/_task.py | 13 +++++++------ esmvalcore/config-user.yml | 11 ++--------- tests/integration/test_recipe.py | 3 +-- 7 files changed, 37 insertions(+), 56 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index d2d9b6498f..3163ebf41d 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -47,14 +47,6 @@ with explanations in a commented line above each option: .. code-block:: yaml - # Diagnostics create plots? [true]/false - # turning it off will turn off graphical output from diagnostic - write_plots: true - - # Diagnostics write NetCDF files? [true]/false - # turning it off will turn off netCDF output from diagnostic - write_netcdf: true - # Set the console log level debug, [info], warning, error # for much more information printed to screen set log_level: debug log_level: info @@ -62,9 +54,6 @@ with explanations in a commented line above each option: # Exit on warning (only for NCL diagnostic scripts)? true/[false] exit_on_warning: false - # Plot file format? [png]/pdf/ps/eps/epsi - output_file_type: png - # Destination directory where all output will be written # including log files and performance stats output_dir: ./esmvaltool_output @@ -117,19 +106,6 @@ with explanations in a commented line above each option: drs: CMIP5: default -Most of these settings are fairly self-explanatory, e.g.: - -.. code-block:: yaml - - # Diagnostics create plots? [true]/false - write_plots: true - # Diagnostics write NetCDF files? [true]/false - write_netcdf: true - -The ``write_plots`` setting is used to inform ESMValTool diagnostics about your -preference for creating figures. Similarly, the ``write_netcdf`` setting is a -boolean which turns on or off the writing of netCDF files by the diagnostic -scripts. .. code-block:: yaml diff --git a/doc/quickstart/output.rst b/doc/quickstart/output.rst index fbe55086c1..5806b8659c 100644 --- a/doc/quickstart/output.rst +++ b/doc/quickstart/output.rst @@ -68,9 +68,7 @@ the results should be saved to the work directory. Plots ===== -The plots directory is where diagnostics save their output figures. These -plots are saved in the format requested by the option `output_file_type` in the -config-user.yml file. +The plots directory is where diagnostics save their output figures. Settings.yml @@ -81,7 +79,7 @@ will produce a unique settings.yml file. The settings.yml file passes several global level keys to diagnostic scripts. This includes several flags from the config-user.yml file (such as -'write_netcdf', 'write_plots', etc...), several paths which are specific to the +'log_level'), several paths which are specific to the diagnostic being run (such as 'plot_dir' and 'run_dir') and the location on disk of the metadata.yml file (described below). @@ -89,7 +87,6 @@ disk of the metadata.yml file (described below). input_files:[[...]recipe_ocean_bgc_20190118_134855/preproc/diag_timeseries_scalars/mfo/metadata.yml] log_level: debug - output_file_type: png plot_dir: [...]recipe_ocean_bgc_20190118_134855/plots/diag_timeseries_scalars/Scalar_timeseries profile_diagnostic: false recipe: recipe_ocean_bgc.yml @@ -97,8 +94,6 @@ disk of the metadata.yml file (described below). script: Scalar_timeseries version: 2.0a1 work_dir: [...]recipe_ocean_bgc_20190118_134855/work/diag_timeseries_scalars/Scalar_timeseries - write_netcdf: true - write_plots: true The first item in the settings file will be a list of `Metadata.yml`_ files. There is a metadata.yml file generated for each field in each diagnostic. @@ -146,5 +141,5 @@ As you can see, this is effectively a dictionary with several items including data paths, metadata and other information. There are several tools available in python which are built to read and parse -these files. The tools are avaialbe in the shared directory in the diagnostics +these files. The tools are available in the shared directory in the diagnostics directory. diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 9f433e5d6c..c7620a79d7 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -4,6 +4,7 @@ import logging.config import os import time +import warnings from pathlib import Path import yaml @@ -44,6 +45,19 @@ def read_config_user_file(config_file, folder_name, options=None): with open(config_file, 'r') as file: cfg = yaml.safe_load(file) + # TODO: remove in v2.3 + for setting in ('write_plots', 'write_netcdf', 'output_file_type'): + if setting in cfg: + msg = ( + f"Using '{setting}' in {config_file} is deprecated and will " + "be removed in ESMValCore version 2.3. For diagnostics " + "that support this setting, it should be set in the " + "diagnostic script section of the recipe instead. " + f"Remove the setting from {config_file} to get rid of this " + "warning message.") + print(f"Warning: {msg}") + warnings.warn(DeprecationWarning(msg)) + if options is None: options = dict() for key, value in options.items(): @@ -51,11 +65,8 @@ def read_config_user_file(config_file, folder_name, options=None): # set defaults defaults = { - 'write_plots': True, - 'write_netcdf': True, 'compress_netcdf': False, 'exit_on_warning': False, - 'output_file_type': 'png', 'output_dir': 'esmvaltool_output', 'auxiliary_data_dir': 'auxiliary_data', 'save_intermediary_cubes': False, @@ -121,7 +132,6 @@ def _normalize_path(path): ------- str: Normalized path - """ if path is None: return None diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index b48f935f49..826da5605f 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1242,15 +1242,22 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, if self._cfg['write_ncl_interface']: settings['exit_on_ncl_warning'] = self._cfg['exit_on_warning'] for key in ( - 'output_file_type', 'log_level', - 'write_plots', - 'write_netcdf', 'profile_diagnostic', 'auxiliary_data_dir', ): settings[key] = self._cfg[key] + # Add deprecated settings from configuration file + # TODO: remove in v2.3 + for key in ( + 'output_file_type', + 'write_plots', + 'write_netcdf', + ): + if key not in settings and key in self._cfg: + settings[key] = self._cfg[key] + scripts[script_name] = { 'script': script, 'output_dir': settings['work_dir'], diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 637c8cb5d5..228bf2018a 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -367,10 +367,7 @@ def _write_ncl_settings(self): 'run_dir', 'plot_dir', 'work_dir', - 'output_file_type', 'log_level', - 'write_plots', - 'write_netcdf', } settings = {'diag_script_info': {}, 'config_user_info': {}} for key, value in self.settings.items(): @@ -381,6 +378,13 @@ def _write_ncl_settings(self): else: settings[key] = value + # Still add deprecated keys to config_user_info to avoid + # crashing the diagnostic script that need this. + # TODO: remove in v2.3 + for key in ('write_plots', 'write_netcdf', 'output_file_type'): + if key in self.settings: + settings['config_user_info'][key] = self.settings[key] + write_ncl_settings(settings, filename) return filename @@ -549,15 +553,12 @@ def _collect_provenance(self): 'exit_on_ncl_warning', 'input_files', 'log_level', - 'output_file_type', 'plot_dir', 'profile_diagnostic', 'recipe', 'run_dir', 'version', - 'write_netcdf', 'write_ncl_interface', - 'write_plots', 'work_dir', ) attrs = { diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index 7129aeaaa1..00020845a9 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -3,16 +3,9 @@ ############################################################################### --- -# Diagnostics create plots? [true]/false -write_plots: true -# Diagnostics write NetCDF files? [true]/false -write_netcdf: true -# Set the console log level debug, [info], warning, error log_level: info # Exit on warning (only for NCL diagnostic scripts)? true/[false] exit_on_warning: false -# Plot file format? [png]/pdf/ps/eps/epsi -output_file_type: png # Destination directory output_dir: ./esmvaltool_output # Auxiliary data directory (used for some additional datasets) @@ -85,8 +78,8 @@ profile_diagnostic: false # CORDEX: /mnt/lustre02/work/ik1017/C3SCORDEX/data/c3s-cordex/output # OBS: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS # OBS6: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS -# obs4mips: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS -# ana4mips: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS +# obs4mips: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS +# ana4mips: /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS #drs: # CMIP6: DKRZ # CMIP5: DKRZ diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index bf1864eaec..732b97e7cf 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -65,7 +65,6 @@ def config_user(tmp_path): filename = write_config_user_file(tmp_path) cfg = esmvalcore._config.read_config_user_file(filename, 'recipe_test', {}) cfg['synda_download'] = False - cfg['output_file_type'] = 'png' cfg['check_level'] = CheckLevels.DEFAULT return cfg @@ -1141,7 +1140,7 @@ def test_derive_with_optional_var_nodata(tmp_path, def create_test_image(basename, cfg): """Get a valid path for saving a diagnostic plot.""" - image = Path(cfg['plot_dir']) / (basename + '.' + cfg['output_file_type']) + image = Path(cfg['plot_dir']) / (basename + '.png') image.parent.mkdir(parents=True) Image.new('RGB', (1, 1)).save(image) return str(image) From 5d9f66c20e02fa1a88d5b9fd285ae641d5709a8e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 2 Oct 2020 15:36:20 +0200 Subject: [PATCH 02/10] Add a note in documentation --- doc/quickstart/configure.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 3163ebf41d..4a64ca2863 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -106,6 +106,11 @@ with explanations in a commented line above each option: drs: CMIP5: default +There used to be a setting ``write_plots``, ``write_netcdf``, and ``output_file_type`` +in the config user file, but these have been deprecated since ESMValCore v2.1 and +will be removed in v2.3, because only some diagnostic scripts supported these settings. +For those diagnostic scripts that do support these settings, they can now be configured +in the diagnostic script section of the recipe. .. code-block:: yaml From 0c2dccffe791c0dfeefc6b00faf56eddf5c567ad Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 2 Oct 2020 15:44:40 +0200 Subject: [PATCH 03/10] Fix indentation --- esmvalcore/_recipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 826da5605f..765c9e1fc9 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1251,9 +1251,9 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, # Add deprecated settings from configuration file # TODO: remove in v2.3 for key in ( - 'output_file_type', - 'write_plots', - 'write_netcdf', + 'output_file_type', + 'write_plots', + 'write_netcdf', ): if key not in settings and key in self._cfg: settings[key] = self._cfg[key] From ddd6e17dd43781c2b323ef2032b365959a8610e1 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Fri, 2 Oct 2020 15:51:08 +0200 Subject: [PATCH 04/10] Use DEPRECATED instead of TODO to mark deprecated stuff --- doc/quickstart/configure.rst | 3 +++ esmvalcore/_config.py | 2 +- esmvalcore/_recipe.py | 2 +- esmvalcore/_task.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 4a64ca2863..3cd9dc707e 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -106,6 +106,9 @@ with explanations in a commented line above each option: drs: CMIP5: default +.. + DEPRECATED: remove in v2.3 + There used to be a setting ``write_plots``, ``write_netcdf``, and ``output_file_type`` in the config user file, but these have been deprecated since ESMValCore v2.1 and will be removed in v2.3, because only some diagnostic scripts supported these settings. diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index c7620a79d7..a8e0903e35 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -45,7 +45,7 @@ def read_config_user_file(config_file, folder_name, options=None): with open(config_file, 'r') as file: cfg = yaml.safe_load(file) - # TODO: remove in v2.3 + # DEPRECATED: remove in v2.3 for setting in ('write_plots', 'write_netcdf', 'output_file_type'): if setting in cfg: msg = ( diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 765c9e1fc9..2ff9b206ff 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1249,7 +1249,7 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, settings[key] = self._cfg[key] # Add deprecated settings from configuration file - # TODO: remove in v2.3 + # DEPRECATED: remove in v2.3 for key in ( 'output_file_type', 'write_plots', diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 228bf2018a..aa9fb0ea56 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -380,7 +380,7 @@ def _write_ncl_settings(self): # Still add deprecated keys to config_user_info to avoid # crashing the diagnostic script that need this. - # TODO: remove in v2.3 + # DEPRECATED: remove in v2.3 for key in ('write_plots', 'write_netcdf', 'output_file_type'): if key in self.settings: settings['config_user_info'][key] = self.settings[key] From 1a9d6dce39a25ae0ec7a0d21383f227b9770135a Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 5 Oct 2020 11:10:40 +0200 Subject: [PATCH 05/10] Restore default values and add warning about use from cli --- esmvalcore/_config.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index a8e0903e35..60465851a0 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -62,6 +62,15 @@ def read_config_user_file(config_file, folder_name, options=None): options = dict() for key, value in options.items(): cfg[key] = value + # DEPRECATED: remove in v2.3 + if key in ('write_plots', 'write_netcdf', 'output_file_type'): + msg = ( + f"Setting '{key}' from the command line is deprecated and " + "will be removed in ESMValCore version 2.3. For diagnostics " + "that support this setting, it should be set in the " + "diagnostic script section of the recipe instead.") + print(f"Warning: {msg}") + warnings.warn(DeprecationWarning(msg)) # set defaults defaults = { @@ -76,6 +85,10 @@ def read_config_user_file(config_file, folder_name, options=None): 'profile_diagnostic': False, 'config_developer_file': None, 'drs': {}, + # DEPRECATED: remove default settings below in v2.3 + 'write_plots': True, + 'write_netcdf': True, + 'output_file_type': 'png', } for key in defaults: From 822afd57f0e57bcfc6795adee2b4f066a6bb3ffb Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 8 Dec 2020 16:26:06 +0100 Subject: [PATCH 06/10] Increase version in which feature will be removed to v2.4 --- doc/quickstart/configure.rst | 4 ++-- esmvalcore/_config.py | 6 +++--- esmvalcore/_recipe.py | 2 +- esmvalcore/_task.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 3cd9dc707e..a952277dfe 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -107,11 +107,11 @@ with explanations in a commented line above each option: CMIP5: default .. - DEPRECATED: remove in v2.3 + DEPRECATED: remove in v2.4 There used to be a setting ``write_plots``, ``write_netcdf``, and ``output_file_type`` in the config user file, but these have been deprecated since ESMValCore v2.1 and -will be removed in v2.3, because only some diagnostic scripts supported these settings. +will be removed in v2.4, because only some diagnostic scripts supported these settings. For those diagnostic scripts that do support these settings, they can now be configured in the diagnostic script section of the recipe. diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 60465851a0..4773ac4ece 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -45,7 +45,7 @@ def read_config_user_file(config_file, folder_name, options=None): with open(config_file, 'r') as file: cfg = yaml.safe_load(file) - # DEPRECATED: remove in v2.3 + # DEPRECATED: remove in v2.4 for setting in ('write_plots', 'write_netcdf', 'output_file_type'): if setting in cfg: msg = ( @@ -62,7 +62,7 @@ def read_config_user_file(config_file, folder_name, options=None): options = dict() for key, value in options.items(): cfg[key] = value - # DEPRECATED: remove in v2.3 + # DEPRECATED: remove in v2.4 if key in ('write_plots', 'write_netcdf', 'output_file_type'): msg = ( f"Setting '{key}' from the command line is deprecated and " @@ -85,7 +85,7 @@ def read_config_user_file(config_file, folder_name, options=None): 'profile_diagnostic': False, 'config_developer_file': None, 'drs': {}, - # DEPRECATED: remove default settings below in v2.3 + # DEPRECATED: remove default settings below in v2.4 'write_plots': True, 'write_netcdf': True, 'output_file_type': 'png', diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 2ff9b206ff..a33f8a6bc5 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1249,7 +1249,7 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, settings[key] = self._cfg[key] # Add deprecated settings from configuration file - # DEPRECATED: remove in v2.3 + # DEPRECATED: remove in v2.4 for key in ( 'output_file_type', 'write_plots', diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index aa9fb0ea56..17337ea0ed 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -380,7 +380,7 @@ def _write_ncl_settings(self): # Still add deprecated keys to config_user_info to avoid # crashing the diagnostic script that need this. - # DEPRECATED: remove in v2.3 + # DEPRECATED: remove in v2.4 for key in ('write_plots', 'write_netcdf', 'output_file_type'): if key in self.settings: settings['config_user_info'][key] = self.settings[key] From 053e0973f36da74ed56050ee636018f760f55cd5 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Tue, 5 Jan 2021 15:50:56 +0100 Subject: [PATCH 07/10] Update text to v2.2 --- doc/quickstart/configure.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index a952277dfe..f62be610cc 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -110,7 +110,7 @@ with explanations in a commented line above each option: DEPRECATED: remove in v2.4 There used to be a setting ``write_plots``, ``write_netcdf``, and ``output_file_type`` -in the config user file, but these have been deprecated since ESMValCore v2.1 and +in the config user file, but these have been deprecated since ESMValCore v2.2 and will be removed in v2.4, because only some diagnostic scripts supported these settings. For those diagnostic scripts that do support these settings, they can now be configured in the diagnostic script section of the recipe. From 0e8dba5d314c293c47d1ea12d76c19d3fa373d81 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 27 Jan 2021 13:27:25 +0100 Subject: [PATCH 08/10] Correct version in which feature will be removed Co-authored-by: Javier Vegas-Regidor --- esmvalcore/_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 29e5799c44..e015e91312 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -50,7 +50,7 @@ def read_config_user_file(config_file, folder_name, options=None): if setting in cfg: msg = ( f"Using '{setting}' in {config_file} is deprecated and will " - "be removed in ESMValCore version 2.3. For diagnostics " + "be removed in ESMValCore version 2.4. For diagnostics " "that support this setting, it should be set in the " "diagnostic script section of the recipe instead. " f"Remove the setting from {config_file} to get rid of this " @@ -66,7 +66,7 @@ def read_config_user_file(config_file, folder_name, options=None): if key in ('write_plots', 'write_netcdf', 'output_file_type'): msg = ( f"Setting '{key}' from the command line is deprecated and " - "will be removed in ESMValCore version 2.3. For diagnostics " + "will be removed in ESMValCore version 2.4. For diagnostics " "that support this setting, it should be set in the " "diagnostic script section of the recipe instead.") print(f"Warning: {msg}") From e9c00f1a80eecd89d55790601838b6095f5f45e9 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 27 Jan 2021 17:11:42 +0100 Subject: [PATCH 09/10] Do not deprecate output_file_type just yet --- doc/quickstart/configure.rst | 5 ++++- doc/quickstart/output.rst | 5 ++++- esmvalcore/_config.py | 6 +++--- esmvalcore/_recipe.py | 2 +- esmvalcore/_task.py | 4 +++- esmvalcore/config-user.yml | 2 ++ esmvalcore/experimental/config/_config_validators.py | 3 +-- tests/integration/test_recipe.py | 2 +- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index f62be610cc..c95c62c55b 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -54,6 +54,9 @@ with explanations in a commented line above each option: # Exit on warning (only for NCL diagnostic scripts)? true/[false] exit_on_warning: false + # Plot file format? [png]/pdf/ps/eps/epsi + output_file_type: png + # Destination directory where all output will be written # including log files and performance stats output_dir: ./esmvaltool_output @@ -109,7 +112,7 @@ with explanations in a commented line above each option: .. DEPRECATED: remove in v2.4 -There used to be a setting ``write_plots``, ``write_netcdf``, and ``output_file_type`` +There used to be a setting ``write_plots`` and ``write_netcdf`` in the config user file, but these have been deprecated since ESMValCore v2.2 and will be removed in v2.4, because only some diagnostic scripts supported these settings. For those diagnostic scripts that do support these settings, they can now be configured diff --git a/doc/quickstart/output.rst b/doc/quickstart/output.rst index 5806b8659c..dbc62a1946 100644 --- a/doc/quickstart/output.rst +++ b/doc/quickstart/output.rst @@ -68,7 +68,9 @@ the results should be saved to the work directory. Plots ===== -The plots directory is where diagnostics save their output figures. +The plots directory is where diagnostics save their output figures. These +plots are saved in the format requested by the option `output_file_type` in the +config-user.yml file. Settings.yml @@ -87,6 +89,7 @@ disk of the metadata.yml file (described below). input_files:[[...]recipe_ocean_bgc_20190118_134855/preproc/diag_timeseries_scalars/mfo/metadata.yml] log_level: debug + output_file_type: png plot_dir: [...]recipe_ocean_bgc_20190118_134855/plots/diag_timeseries_scalars/Scalar_timeseries profile_diagnostic: false recipe: recipe_ocean_bgc.yml diff --git a/esmvalcore/_config.py b/esmvalcore/_config.py index 643717752d..c560777f35 100644 --- a/esmvalcore/_config.py +++ b/esmvalcore/_config.py @@ -44,7 +44,7 @@ def read_config_user_file(config_file, folder_name, options=None): cfg = yaml.safe_load(file) # DEPRECATED: remove in v2.4 - for setting in ('write_plots', 'write_netcdf', 'output_file_type'): + for setting in ('write_plots', 'write_netcdf'): if setting in cfg: msg = ( f"Using '{setting}' in {config_file} is deprecated and will " @@ -61,7 +61,7 @@ def read_config_user_file(config_file, folder_name, options=None): for key, value in options.items(): cfg[key] = value # DEPRECATED: remove in v2.4 - if key in ('write_plots', 'write_netcdf', 'output_file_type'): + if key in ('write_plots', 'write_netcdf'): msg = ( f"Setting '{key}' from the command line is deprecated and " "will be removed in ESMValCore version 2.4. For diagnostics " @@ -74,6 +74,7 @@ def read_config_user_file(config_file, folder_name, options=None): defaults = { 'compress_netcdf': False, 'exit_on_warning': False, + 'output_file_type': 'png', 'output_dir': 'esmvaltool_output', 'auxiliary_data_dir': 'auxiliary_data', 'save_intermediary_cubes': False, @@ -86,7 +87,6 @@ def read_config_user_file(config_file, folder_name, options=None): # DEPRECATED: remove default settings below in v2.4 'write_plots': True, 'write_netcdf': True, - 'output_file_type': 'png', } for key in defaults: diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 8cbb60b087..a368c123fe 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -1217,6 +1217,7 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, if self._cfg['write_ncl_interface']: settings['exit_on_ncl_warning'] = self._cfg['exit_on_warning'] for key in ( + 'output_file_type', 'log_level', 'profile_diagnostic', 'auxiliary_data_dir', @@ -1226,7 +1227,6 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, # Add deprecated settings from configuration file # DEPRECATED: remove in v2.4 for key in ( - 'output_file_type', 'write_plots', 'write_netcdf', ): diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index 6090494308..2a10d00079 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -375,6 +375,7 @@ def _write_ncl_settings(self): 'run_dir', 'plot_dir', 'work_dir', + 'output_file_type', 'log_level', } settings = {'diag_script_info': {}, 'config_user_info': {}} @@ -389,7 +390,7 @@ def _write_ncl_settings(self): # Still add deprecated keys to config_user_info to avoid # crashing the diagnostic script that need this. # DEPRECATED: remove in v2.4 - for key in ('write_plots', 'write_netcdf', 'output_file_type'): + for key in ('write_plots', 'write_netcdf'): if key in self.settings: settings['config_user_info'][key] = self.settings[key] @@ -558,6 +559,7 @@ def _collect_provenance(self): 'exit_on_ncl_warning', 'input_files', 'log_level', + 'output_file_type', 'plot_dir', 'profile_diagnostic', 'recipe', diff --git a/esmvalcore/config-user.yml b/esmvalcore/config-user.yml index 298c35d314..79f3931014 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config-user.yml @@ -6,6 +6,8 @@ log_level: info # Exit on warning (only for NCL diagnostic scripts)? true/[false] exit_on_warning: false +# Plot file format? [png]/pdf/ps/eps/epsi +output_file_type: png # Destination directory output_dir: ./esmvaltool_output # Auxiliary data directory (used for some additional datasets) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index 8910ef5e74..2e5421c78b 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -253,8 +253,6 @@ def deprecate(func, variable, version: str = None): # deprecate in 2.2.0 'write_plots': deprecate(validate_bool, 'write_plots', '2.2.0'), 'write_netcdf': deprecate(validate_bool, 'write_netcdf', '2.2.0'), - 'output_file_type': deprecate(validate_string, 'output_file_type', - '2.2.0'), # From user config 'log_level': validate_string, @@ -268,6 +266,7 @@ def deprecate(func, variable, version: str = None): 'config_developer_file': validate_config_developer, 'profile_diagnostic': validate_bool, 'run_diagnostic': validate_bool, + 'output_file_type': validate_string, # From CLI "skip-nonexistent": validate_bool, diff --git a/tests/integration/test_recipe.py b/tests/integration/test_recipe.py index ae89ae5324..082b264be5 100644 --- a/tests/integration/test_recipe.py +++ b/tests/integration/test_recipe.py @@ -1258,7 +1258,7 @@ def test_derive_with_optional_var_nodata(tmp_path, def create_test_image(basename, cfg): """Get a valid path for saving a diagnostic plot.""" - image = Path(cfg['plot_dir']) / (basename + '.png') + image = Path(cfg['plot_dir']) / (basename + '.' + cfg['output_file_type']) image.parent.mkdir(parents=True) Image.new('RGB', (1, 1)).save(image) return str(image) From c8c7f9bc4b22ee0f47b7976546a3af14d1fcf9fc Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 28 Jan 2021 21:29:43 +0100 Subject: [PATCH 10/10] Correct version in which feature will be removed Co-authored-by: Javier Vegas-Regidor --- esmvalcore/experimental/config/_config_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/experimental/config/_config_validators.py b/esmvalcore/experimental/config/_config_validators.py index 2e5421c78b..d83a26f331 100644 --- a/esmvalcore/experimental/config/_config_validators.py +++ b/esmvalcore/experimental/config/_config_validators.py @@ -251,8 +251,8 @@ def deprecate(func, variable, version: str = None): _validators = { # deprecate in 2.2.0 - 'write_plots': deprecate(validate_bool, 'write_plots', '2.2.0'), - 'write_netcdf': deprecate(validate_bool, 'write_netcdf', '2.2.0'), + 'write_plots': deprecate(validate_bool, 'write_plots', '2.4.0'), + 'write_netcdf': deprecate(validate_bool, 'write_netcdf', '2.4.0'), # From user config 'log_level': validate_string,