diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index 3094c7e2..09dc5c92 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -47,6 +47,8 @@ death_timeout : float Seconds to wait for a scheduler before closing workers extra : list + Deprecated: use ``worker_extra_args`` instead. This parameter will be removed in a future version. + worker_extra_args : list Additional arguments to pass to `dask-worker` env_extra : list Deprecated: use ``job_script_prologue`` instead. This parameter will be removed in a future version. @@ -156,6 +158,7 @@ def __init__( death_timeout=None, local_directory=None, extra=None, + worker_extra_args=None, env_extra=None, job_script_prologue=None, header_skip=None, @@ -208,6 +211,21 @@ def __init__( ) if extra is None: extra = dask.config.get("jobqueue.%s.extra" % self.config_name) + if worker_extra_args is None: + worker_extra_args = dask.config.get( + "jobqueue.%s.worker-extra-args" % self.config_name + ) + if extra is not None: + warn = ( + "extra has been renamed to worker_extra_args. " + "You are still using it (even if only set to []; please also check config files). " + "If you did not set worker_extra_args yet, extra will be respected for now, " + "but it will be removed in a future release. " + "If you already set worker_extra_args, extra is ignored and you can remove it." + ) + warnings.warn(warn, FutureWarning) + if not worker_extra_args: + worker_extra_args = extra if env_extra is None: env_extra = dask.config.get("jobqueue.%s.env-extra" % self.config_name) if job_script_prologue is None: @@ -217,7 +235,7 @@ def __init__( if env_extra is not None: warn = ( "env_extra has been renamed to job_script_prologue. " - "You are still using it (even if only set to []; please also check config files)." + "You are still using it (even if only set to []; please also check config files). " "If you did not set job_script_prologue yet, env_extra will be respected for now, " "but it will be removed in a future release. " "If you already set job_script_prologue, env_extra is ignored and you can remove it." @@ -240,9 +258,9 @@ def __init__( self.job_header = None if interface: - extra = extra + ["--interface", interface] + worker_extra_args += ["--interface", interface] if protocol: - extra = extra + ["--protocol", protocol] + worker_extra_args += ["--protocol", protocol] if security: worker_security_dict = security.get_tls_config_for_role("worker") security_command_line_list = [ @@ -252,7 +270,7 @@ def __init__( if key != "ciphers" ] security_command_line = sum(security_command_line_list, []) - extra = extra + security_command_line + worker_extra_args += security_command_line # Keep information on process, cores, and memory, for use in subclasses self.worker_memory = parse_bytes(memory) if memory is not None else None @@ -283,8 +301,8 @@ def __init__( command_args += ["--death-timeout", death_timeout] if local_directory is not None: command_args += ["--local-directory", local_directory] - if extra is not None: - command_args += extra + if worker_extra_args is not None: + command_args += worker_extra_args self._command_template = " ".join(map(str, command_args)) diff --git a/dask_jobqueue/jobqueue.yaml b/dask_jobqueue/jobqueue.yaml index 568345bc..61f2eccb 100644 --- a/dask_jobqueue/jobqueue.yaml +++ b/dask_jobqueue/jobqueue.yaml @@ -11,7 +11,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # OAR resource manager options shebang: "#!/usr/bin/env bash" @@ -39,7 +40,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # PBS resource manager options shebang: "#!/usr/bin/env bash" @@ -67,7 +69,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # SGE resource manager options shebang: "#!/usr/bin/env bash" @@ -95,7 +98,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # SLURM resource manager options shebang: "#!/usr/bin/env bash" @@ -124,7 +128,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # PBS resource manager options shebang: "#!/usr/bin/env bash" @@ -152,7 +157,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # LSF resource manager options shebang: "#!/usr/bin/env bash" @@ -183,7 +189,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` # HTCondor Resource Manager options disk: null # Total amount of disk per job @@ -209,7 +216,8 @@ jobqueue: death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR shared-temp-directory: null # Shared directory currently used to dump temporary security objects for workers - extra: [] + extra: null # deprecated: use worker-extra-args + worker-extra-args: [] # Additional arguments to pass to `dask-worker` env-extra: null job-script-prologue: [] diff --git a/dask_jobqueue/tests/test_htcondor.py b/dask_jobqueue/tests/test_htcondor.py index 41aa8388..d1228c07 100644 --- a/dask_jobqueue/tests/test_htcondor.py +++ b/dask_jobqueue/tests/test_htcondor.py @@ -139,7 +139,8 @@ def test_config_name_htcondor_takes_custom_config(): "processes": 1, "interface": None, "death-timeout": None, - "extra": [], + "extra": None, + "worker-extra-args": [], "env-extra": None, "job-script-prologue": [], "log-directory": None, diff --git a/dask_jobqueue/tests/test_job.py b/dask_jobqueue/tests/test_job.py index c20b62b7..1ada7b6f 100644 --- a/dask_jobqueue/tests/test_job.py +++ b/dask_jobqueue/tests/test_job.py @@ -145,7 +145,7 @@ def test_docstring_cluster(Cluster): assert Cluster.__name__[: -len("Cluster")] in Cluster.__doc__ -def test_deprecation(Cluster): +def test_deprecation_env_extra(Cluster): import warnings # test issuing of warning @@ -207,3 +207,67 @@ def test_deprecation(Cluster): ) job_script = job.job_script() assert "env_extra" in job_script + + +def test_deprecation_extra(Cluster): + import warnings + + # test issuing of warning + warnings.simplefilter("always") + + job_cls = Cluster.job_cls + with warnings.catch_warnings(record=True) as w: + # should give a warning + job = job_cls(cores=1, memory="1 GB", extra=["old_param"]) + assert len(w) == 1 + assert issubclass(w[0].category, FutureWarning) + assert "extra has been renamed" in str(w[0].message) + with warnings.catch_warnings(record=True) as w: + # should give a warning + job = job_cls( + cores=1, + memory="1 GB", + extra=["old_param"], + worker_extra_args=["new_param"], + ) + assert len(w) == 1 + assert issubclass(w[0].category, FutureWarning) + assert "extra has been renamed" in str(w[0].message) + with warnings.catch_warnings(record=True) as w: + # should not give a warning + job = job_cls( + cores=1, + memory="1 GB", + worker_extra_args=["new_param"], + ) + assert len(w) == 0 + + # the rest is not about the warning but about behaviour: if worker_extra_args is not + # set, extra should still be used if provided + warnings.simplefilter("ignore") + job = job_cls( + cores=1, + memory="1 GB", + extra=["old_param"], + worker_extra_args=["new_param"], + ) + job_script = job.job_script() + assert "old_param" not in job_script + assert "new_param" in job_script + + job = job_cls( + cores=1, + memory="1 GB", + extra=["old_param"], + ) + job_script = job.job_script() + assert "old_param" in job_script + + job = job_cls( + cores=1, + memory="1 GB", + extra=["old_param"], + worker_extra_args=[], + ) + job_script = job.job_script() + assert "old_param" in job_script diff --git a/dask_jobqueue/tests/test_jobqueue_core.py b/dask_jobqueue/tests/test_jobqueue_core.py index 2d781e15..a21303fd 100644 --- a/dask_jobqueue/tests/test_jobqueue_core.py +++ b/dask_jobqueue/tests/test_jobqueue_core.py @@ -43,7 +43,7 @@ def test_command_template(Cluster): memory="4GB", death_timeout=60, local_directory="/scratch", - extra=["--preload", "mymodule"], + worker_extra_args=["--preload", "mymodule"], ) as cluster: assert " --death-timeout 60" in cluster._dummy_job._command_template assert " --local-directory /scratch" in cluster._dummy_job._command_template diff --git a/dask_jobqueue/tests/test_lsf.py b/dask_jobqueue/tests/test_lsf.py index f236ac4e..759de48a 100644 --- a/dask_jobqueue/tests/test_lsf.py +++ b/dask_jobqueue/tests/test_lsf.py @@ -320,7 +320,8 @@ def test_config_name_lsf_takes_custom_config(): "death-timeout": None, "local-directory": "/foo", "shared-temp-directory": None, - "extra": [], + "extra": None, + "worker-extra-args": [], "env-extra": None, "job-script-prologue": [], "log-directory": None, diff --git a/dask_jobqueue/tests/test_oar.py b/dask_jobqueue/tests/test_oar.py index a57b3dd6..3c802ac0 100644 --- a/dask_jobqueue/tests/test_oar.py +++ b/dask_jobqueue/tests/test_oar.py @@ -108,7 +108,8 @@ def test_config_name_oar_takes_custom_config(): "death-timeout": None, "local-directory": "/foo", "shared-temp-directory": None, - "extra": [], + "extra": None, + "worker-extra-args": [], "env-extra": None, "job-script-prologue": [], "log-directory": None, diff --git a/dask_jobqueue/tests/test_pbs.py b/dask_jobqueue/tests/test_pbs.py index b62de964..69027d3e 100644 --- a/dask_jobqueue/tests/test_pbs.py +++ b/dask_jobqueue/tests/test_pbs.py @@ -359,7 +359,8 @@ def test_config_name_pbs_takes_custom_config(): "death-timeout": None, "local-directory": "/foo", "shared-temp-directory": None, - "extra": [], + "extra": None, + "worker-extra-args": [], "env-extra": None, "job-script-prologue": [], "log-directory": None, diff --git a/dask_jobqueue/tests/test_sge.py b/dask_jobqueue/tests/test_sge.py index 4977705e..38639153 100644 --- a/dask_jobqueue/tests/test_sge.py +++ b/dask_jobqueue/tests/test_sge.py @@ -56,7 +56,8 @@ def test_config_name_sge_takes_custom_config(): "death-timeout": None, "local-directory": "/foo", "shared-temp-directory": None, - "extra": [], + "extra": None, + "worker-extra-args": [], "env-extra": None, "job-script-prologue": [], "log-directory": None, diff --git a/dask_jobqueue/tests/test_slurm.py b/dask_jobqueue/tests/test_slurm.py index 1f8701fd..53523c5b 100644 --- a/dask_jobqueue/tests/test_slurm.py +++ b/dask_jobqueue/tests/test_slurm.py @@ -191,7 +191,8 @@ def test_config_name_slurm_takes_custom_config(): "death-timeout": None, "local-directory": "/foo", "shared-temp-directory": None, - "extra": [], + "extra": None, + "worker-extra-args": [], "env-extra": None, "job-script-prologue": [], "log-directory": None, diff --git a/docs/source/advanced-tips-and-tricks.rst b/docs/source/advanced-tips-and-tricks.rst index be635a1a..bd084d1e 100644 --- a/docs/source/advanced-tips-and-tricks.rst +++ b/docs/source/advanced-tips-and-tricks.rst @@ -128,9 +128,16 @@ Here is an example of how to use these parameters: .. code-block:: python - cluster = Cluster(walltime='01:00:00', cores=4, memory='16gb', extra=["--lifetime", "55m", "--lifetime-stagger", "4m"]) + cluster = Cluster( + walltime="01:00:00", + cores=4, + memory="16gb", + worker_extra_args=["--lifetime", "55m", "--lifetime-stagger", "4m"], + ) cluster.adapt(minimum=0, maximum=200) +*Note: the parameter* ``worker_extra_args`` *was named* ``extra`` *until version 0.7.4.* ``extra`` *can still +be used, but is considered deprecated and will be removed in a future version.* Here is an example of a workflow taking advantage of this, if you want to give it a try or adapt it to your use case: diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 55527879..d52dbd8c 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -146,8 +146,12 @@ argument is for the specification of abstract resources, described `here `__. This could be used to specify special hardware availability that the scheduler is not aware of, for example GPUs. Below, the arbitrary resources "ssdGB" and "GPU" are -specified. Notice that the ``extra`` keyword is used to pass through arguments -to the dask-workers. +specified. Notice that the ``worker_extra_args`` keyword is used to pass through +arguments to the dask-workers. + +*Note: the parameter* ``worker_extra_args`` *was named* ``extra`` *until version +0.7.4.* ``extra`` *can still be used, but is considered deprecated and will be +removed in a future version.* .. code-block:: python @@ -155,10 +159,9 @@ to the dask-workers. from distributed import Client from dask import delayed - cluster = SLURMCluster(memory='8g', - processes=1, - cores=2, - extra=['--resources ssdGB=200,GPU=2']) + cluster = SLURMCluster( + memory="8g", processes=1, cores=2, worker_extra_args=["--resources ssdGB=200,GPU=2"] + ) cluster.scale(2) client = Client(cluster)