From 10441016ee9e6d946b4ec16f44cfdf68c6766d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Thu, 2 Apr 2020 17:16:47 +0200 Subject: [PATCH 01/16] Allow scheduler options import from configuration file --- dask_jobqueue/core.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index 994bc886..1a3b4975 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -484,8 +484,15 @@ def __init__( "dashboard_address": ":8787", "security": security, } - # scheduler_options overrides parameters common to both workers and scheduler - scheduler_options = dict(default_scheduler_options, **scheduler_options) + + config_scheduler_options = dask.config.get("jobqueue.%s" % config_name).get('scheduler_options') or {} + + # merge the available scheduler_options + scheduler_options = { + **default_scheduler_options, # least important + **config_scheduler_options, + **scheduler_options # most important + } # Use the same network interface as the workers if scheduler ip has not # been set through scheduler_options via 'host' or 'interface' From ab5571ef1a114655364591e3b7f930f81e18bfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Fri, 3 Apr 2020 20:45:52 +0200 Subject: [PATCH 02/16] First steps to test scheduler options import from config (WIP) --- dask_jobqueue/tests/test_jobqueue_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_jobqueue/tests/test_jobqueue_core.py b/dask_jobqueue/tests/test_jobqueue_core.py index bc9c94e0..3499d7c2 100644 --- a/dask_jobqueue/tests/test_jobqueue_core.py +++ b/dask_jobqueue/tests/test_jobqueue_core.py @@ -1,4 +1,4 @@ -import os +xfimport os import shutil import socket import sys From e408dd664679eadf31699149bfcc920895cccb2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Fri, 3 Apr 2020 20:46:24 +0200 Subject: [PATCH 03/16] Satisfy black --- dask_jobqueue/core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index 1a3b4975..586cf231 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -484,14 +484,16 @@ def __init__( "dashboard_address": ":8787", "security": security, } - - config_scheduler_options = dask.config.get("jobqueue.%s" % config_name).get('scheduler_options') or {} + + config_scheduler_options = ( + dask.config.get("jobqueue.%s" % config_name).get("scheduler_options") or {} + ) # merge the available scheduler_options scheduler_options = { - **default_scheduler_options, # least important + **default_scheduler_options, # least important **config_scheduler_options, - **scheduler_options # most important + **scheduler_options, # most important } # Use the same network interface as the workers if scheduler ip has not From b06788f0b5bf6b70710a96d226b464419963fc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Fri, 3 Apr 2020 20:55:12 +0200 Subject: [PATCH 04/16] Revert "First steps to test scheduler options import from config (WIP)" This reverts commit ab5571ef1a114655364591e3b7f930f81e18bfc1. --- dask_jobqueue/tests/test_jobqueue_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_jobqueue/tests/test_jobqueue_core.py b/dask_jobqueue/tests/test_jobqueue_core.py index 3499d7c2..bc9c94e0 100644 --- a/dask_jobqueue/tests/test_jobqueue_core.py +++ b/dask_jobqueue/tests/test_jobqueue_core.py @@ -1,4 +1,4 @@ -xfimport os +import os import shutil import socket import sys From 331d9676cc64c58fea6524f12678ef5ca5ad9f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Fri, 3 Apr 2020 20:56:06 +0200 Subject: [PATCH 05/16] First steps to test scheduler options import from config (WIP) --- .../test_scheduler_options_from_config.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 dask_jobqueue/tests/test_scheduler_options_from_config.py diff --git a/dask_jobqueue/tests/test_scheduler_options_from_config.py b/dask_jobqueue/tests/test_scheduler_options_from_config.py new file mode 100644 index 00000000..c3416d8b --- /dev/null +++ b/dask_jobqueue/tests/test_scheduler_options_from_config.py @@ -0,0 +1,110 @@ +import pytest, psutil, dask + +from dask_jobqueue import ( + JobQueueCluster, + PBSCluster, + MoabCluster, + SLURMCluster, + SGECluster, + LSFCluster, + OARCluster, +) + + +@pytest.mark.parametrize( + "Cluster", + [PBSCluster, MoabCluster, SLURMCluster, SGECluster, LSFCluster, OARCluster], +) +def test_import_scheduler_options_from_config(Cluster): + + net_if_addrs = psutil.net_if_addrs() + + config_scheduler_interface = list(net_if_addrs.keys())[0] + config_scheduler_host = net_if_addrs[config_scheduler_interface][0].address + config_scheduler_port = 8804 + + pass_scheduler_interface = list(net_if_addrs.keys())[1] + pass_scheduler_host = net_if_addrs[pass_scheduler_interface][0].address + pass_scheduler_port = 8807 + + generic_cluster_conf = { + # configurable entries collected from + # all the available clusters + # they are not actually useful here, + # but are necessary for this to work... + "cores": None, + "memory": None, + "queue": None, + "project": None, + "ncpus": None, + "processes": None, + "walltime": None, + "job-extra": [], + "name": None, + "interface": None, + "death-timeout": None, + "local-directory": None, + "extra": [], + "env-extra": [], + "log-directory": None, + "shebang": None, + "job-cpu": None, + "job-mem": None, + "resource-spec": None, + "mem": None, + "lsf-units": None, + "use-stdin": None, + } + + with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): + with Cluster(cores=2, memory="2GB", config_name="generic-config") as cluster: + assert cluster.scheduler_spec["options"].get("interface") == None + assert cluster.scheduler_spec["options"].get("port") == None + + generic_cluster_conf["scheduler_options"] = { + "interface": config_scheduler_interface, + "port": config_scheduler_port, + } + + with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): + + with Cluster(cores=2, memory="2GB", config_name="generic-config",) as cluster: + assert ( + cluster.scheduler_spec["options"].get("interface") + == config_scheduler_interface + ) + assert ( + cluster.scheduler_spec["options"].get("port") == config_scheduler_port + ) + + with Cluster( + cores=2, + memory="2GB", + config_name="generic-config", + scheduler_options={ + "interface": pass_scheduler_interface, + "port": pass_scheduler_port, + }, + ) as cluster: + assert ( + cluster.scheduler_spec["options"].get("interface") + == pass_scheduler_interface + ) + assert cluster.scheduler_spec["options"].get("port") == pass_scheduler_port + + generic_cluster_conf["scheduler_options"] = {"host": config_scheduler_host} + + with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): + + with Cluster(cores=2, memory="2GB", config_name="generic-config",) as cluster: + assert ( + cluster.scheduler_spec["options"].get("host") == config_scheduler_host + ) + + with Cluster( + cores=2, + memory="2GB", + config_name="generic-config", + scheduler_options={"host": pass_scheduler_host,}, + ) as cluster: + assert cluster.scheduler_spec["options"].get("host") == pass_scheduler_host From 13d1331510b5b1afb1441dc548a925dc6a337bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Tue, 7 Apr 2020 21:46:13 +0200 Subject: [PATCH 06/16] Ensure consistent behaviour with list-type parameters --- dask_jobqueue/core.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index 586cf231..97eb0e4f 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -477,24 +477,17 @@ def __init__( if interface is None: interface = dask.config.get("jobqueue.%s.interface" % config_name) if scheduler_options is None: - scheduler_options = {} + scheduler_options = dask.config.get( + "jobqueue.%s.scheduler-options" % config_name, {} + ) default_scheduler_options = { "protocol": protocol, "dashboard_address": ":8787", "security": security, } - - config_scheduler_options = ( - dask.config.get("jobqueue.%s" % config_name).get("scheduler_options") or {} - ) - - # merge the available scheduler_options - scheduler_options = { - **default_scheduler_options, # least important - **config_scheduler_options, - **scheduler_options, # most important - } + # scheduler_options overrides parameters common to both workers and scheduler + scheduler_options = dict(default_scheduler_options, **scheduler_options) # Use the same network interface as the workers if scheduler ip has not # been set through scheduler_options via 'host' or 'interface' From c4e4ada6834080a4669b2e918f00e592fc8fdeaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Tue, 7 Apr 2020 21:47:29 +0200 Subject: [PATCH 07/16] Clean-up previous draft --- .../test_scheduler_options_from_config.py | 57 ++++--------------- 1 file changed, 10 insertions(+), 47 deletions(-) diff --git a/dask_jobqueue/tests/test_scheduler_options_from_config.py b/dask_jobqueue/tests/test_scheduler_options_from_config.py index c3416d8b..083bba2d 100644 --- a/dask_jobqueue/tests/test_scheduler_options_from_config.py +++ b/dask_jobqueue/tests/test_scheduler_options_from_config.py @@ -1,7 +1,8 @@ -import pytest, psutil, dask +import pytest +import psutil +import dask from dask_jobqueue import ( - JobQueueCluster, PBSCluster, MoabCluster, SLURMCluster, @@ -20,18 +21,11 @@ def test_import_scheduler_options_from_config(Cluster): net_if_addrs = psutil.net_if_addrs() config_scheduler_interface = list(net_if_addrs.keys())[0] - config_scheduler_host = net_if_addrs[config_scheduler_interface][0].address config_scheduler_port = 8804 pass_scheduler_interface = list(net_if_addrs.keys())[1] - pass_scheduler_host = net_if_addrs[pass_scheduler_interface][0].address - pass_scheduler_port = 8807 generic_cluster_conf = { - # configurable entries collected from - # all the available clusters - # they are not actually useful here, - # but are necessary for this to work... "cores": None, "memory": None, "queue": None, @@ -56,11 +50,6 @@ def test_import_scheduler_options_from_config(Cluster): "use-stdin": None, } - with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): - with Cluster(cores=2, memory="2GB", config_name="generic-config") as cluster: - assert cluster.scheduler_spec["options"].get("interface") == None - assert cluster.scheduler_spec["options"].get("port") == None - generic_cluster_conf["scheduler_options"] = { "interface": config_scheduler_interface, "port": config_scheduler_port, @@ -69,42 +58,16 @@ def test_import_scheduler_options_from_config(Cluster): with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): with Cluster(cores=2, memory="2GB", config_name="generic-config",) as cluster: - assert ( - cluster.scheduler_spec["options"].get("interface") - == config_scheduler_interface - ) - assert ( - cluster.scheduler_spec["options"].get("port") == config_scheduler_port - ) - - with Cluster( - cores=2, - memory="2GB", - config_name="generic-config", - scheduler_options={ - "interface": pass_scheduler_interface, - "port": pass_scheduler_port, - }, - ) as cluster: - assert ( - cluster.scheduler_spec["options"].get("interface") - == pass_scheduler_interface - ) - assert cluster.scheduler_spec["options"].get("port") == pass_scheduler_port - - generic_cluster_conf["scheduler_options"] = {"host": config_scheduler_host} - - with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): - - with Cluster(cores=2, memory="2GB", config_name="generic-config",) as cluster: - assert ( - cluster.scheduler_spec["options"].get("host") == config_scheduler_host - ) + scheduler_options = cluster.scheduler_spec["options"] + assert scheduler_options.get("interface") == config_scheduler_interface + assert scheduler_options.get("port") == config_scheduler_port with Cluster( cores=2, memory="2GB", config_name="generic-config", - scheduler_options={"host": pass_scheduler_host,}, + scheduler_options={"interface": pass_scheduler_interface}, ) as cluster: - assert cluster.scheduler_spec["options"].get("host") == pass_scheduler_host + scheduler_options = cluster.scheduler_spec["options"] + assert scheduler_options.get("interface") == pass_scheduler_interface + assert scheduler_options.get("port") is None From cc22dc8cfa19e4c178deaaac52cc78fb3e18a36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Sat, 11 Apr 2020 16:38:23 +0200 Subject: [PATCH 08/16] Indicate configurable scheduler options --- dask_jobqueue/jobqueue.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dask_jobqueue/jobqueue.yaml b/dask_jobqueue/jobqueue.yaml index 14fdb67e..897ebd92 100644 --- a/dask_jobqueue/jobqueue.yaml +++ b/dask_jobqueue/jobqueue.yaml @@ -21,6 +21,9 @@ jobqueue: resource-spec: null job-extra: [] log-directory: null + + # Scheduler options + scheduler-options: pbs: name: dask-worker @@ -44,6 +47,9 @@ jobqueue: resource-spec: null job-extra: [] log-directory: null + + # Scheduler options + scheduler-options: sge: name: dask-worker @@ -68,6 +74,9 @@ jobqueue: log-directory: null resource-spec: null + + # Scheduler options + scheduler-options: slurm: name: dask-worker @@ -92,6 +101,9 @@ jobqueue: job-mem: null job-extra: [] log-directory: null + + # Scheduler options + scheduler-options: moab: name: dask-worker @@ -115,6 +127,9 @@ jobqueue: resource-spec: null job-extra: [] log-directory: null + + # Scheduler options + scheduler-options: lsf: name: dask-worker @@ -141,6 +156,9 @@ jobqueue: log-directory: null lsf-units: null use-stdin: True # (bool) How jobs are launched, i.e. 'bsub jobscript.sh' or 'bsub < jobscript.sh' + + # Scheduler options + scheduler-options: htcondor: name: dask-worker @@ -161,6 +179,9 @@ jobqueue: job-extra: {} # Extra submit attributes log-directory: null shebang: "#!/usr/bin/env condor_submit" + + # Scheduler options + scheduler-options: local: name: dask-worker @@ -177,3 +198,6 @@ jobqueue: env-extra: [] job-extra: [] log-directory: null + + # Scheduler options + scheduler-options: From 43fde67f532ec4eed9fcf8cbb2bd7fb2cea51b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Sat, 11 Apr 2020 17:04:37 +0200 Subject: [PATCH 09/16] Move worker option into worker section --- dask_jobqueue/jobqueue.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dask_jobqueue/jobqueue.yaml b/dask_jobqueue/jobqueue.yaml index 897ebd92..646435a1 100644 --- a/dask_jobqueue/jobqueue.yaml +++ b/dask_jobqueue/jobqueue.yaml @@ -10,13 +10,13 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # OAR resource manager options shebang: "#!/usr/bin/env bash" queue: null project: null walltime: '00:30:00' - extra: [] env-extra: [] resource-spec: null job-extra: [] @@ -36,13 +36,13 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # PBS resource manager options shebang: "#!/usr/bin/env bash" queue: null project: null walltime: '00:30:00' - extra: [] env-extra: [] resource-spec: null job-extra: [] @@ -62,17 +62,16 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # SGE resource manager options shebang: "#!/usr/bin/env bash" queue: null project: null walltime: '00:30:00' - extra: [] env-extra: [] job-extra: [] log-directory: null - resource-spec: null # Scheduler options @@ -89,13 +88,13 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # SLURM resource manager options shebang: "#!/usr/bin/env bash" queue: null project: null walltime: '00:30:00' - extra: [] env-extra: [] job-cpu: null job-mem: null @@ -116,13 +115,13 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # PBS resource manager options shebang: "#!/usr/bin/env bash" queue: null project: null walltime: '00:30:00' - extra: [] env-extra: [] resource-spec: null job-extra: [] @@ -142,13 +141,13 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # LSF resource manager options shebang: "#!/usr/bin/env bash" queue: null project: null walltime: '00:30' - extra: [] env-extra: [] ncpus: null mem: null @@ -171,10 +170,10 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR + extra: [] # HTCondor Resource Manager options disk: null # Total amount of disk per job - extra: [] env-extra: [] job-extra: {} # Extra submit attributes log-directory: null @@ -193,8 +192,8 @@ jobqueue: interface: null # Network interface to use like eth0 or ib0 death-timeout: 60 # Number of seconds to wait if a worker can not find a scheduler local-directory: null # Location of fast local storage like /scratch or $TMPDIR - extra: [] + env-extra: [] job-extra: [] log-directory: null From 2add42a576520d4a2aa18feb8fc9fa68071a91b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Sat, 11 Apr 2020 17:10:37 +0200 Subject: [PATCH 10/16] Prevent NoneType unpacking errors --- dask_jobqueue/jobqueue.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dask_jobqueue/jobqueue.yaml b/dask_jobqueue/jobqueue.yaml index 646435a1..ec8045f8 100644 --- a/dask_jobqueue/jobqueue.yaml +++ b/dask_jobqueue/jobqueue.yaml @@ -23,7 +23,7 @@ jobqueue: log-directory: null # Scheduler options - scheduler-options: + scheduler-options: {} pbs: name: dask-worker @@ -49,7 +49,7 @@ jobqueue: log-directory: null # Scheduler options - scheduler-options: + scheduler-options: {} sge: name: dask-worker @@ -75,7 +75,7 @@ jobqueue: resource-spec: null # Scheduler options - scheduler-options: + scheduler-options: {} slurm: name: dask-worker @@ -102,7 +102,7 @@ jobqueue: log-directory: null # Scheduler options - scheduler-options: + scheduler-options: {} moab: name: dask-worker @@ -128,7 +128,7 @@ jobqueue: log-directory: null # Scheduler options - scheduler-options: + scheduler-options: {} lsf: name: dask-worker @@ -157,7 +157,7 @@ jobqueue: use-stdin: True # (bool) How jobs are launched, i.e. 'bsub jobscript.sh' or 'bsub < jobscript.sh' # Scheduler options - scheduler-options: + scheduler-options: {} htcondor: name: dask-worker @@ -180,7 +180,7 @@ jobqueue: shebang: "#!/usr/bin/env condor_submit" # Scheduler options - scheduler-options: + scheduler-options: {} local: name: dask-worker @@ -199,4 +199,4 @@ jobqueue: log-directory: null # Scheduler options - scheduler-options: + scheduler-options: {} From 20bddf0eae820203ca85033eef4f2b34c570a6ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Thu, 30 Apr 2020 18:40:40 +0200 Subject: [PATCH 11/16] Simplify tests --- .../test_scheduler_options_from_config.py | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/dask_jobqueue/tests/test_scheduler_options_from_config.py b/dask_jobqueue/tests/test_scheduler_options_from_config.py index 083bba2d..5d746f40 100644 --- a/dask_jobqueue/tests/test_scheduler_options_from_config.py +++ b/dask_jobqueue/tests/test_scheduler_options_from_config.py @@ -25,39 +25,18 @@ def test_import_scheduler_options_from_config(Cluster): pass_scheduler_interface = list(net_if_addrs.keys())[1] - generic_cluster_conf = { - "cores": None, - "memory": None, - "queue": None, - "project": None, - "ncpus": None, - "processes": None, - "walltime": None, - "job-extra": [], - "name": None, - "interface": None, - "death-timeout": None, - "local-directory": None, - "extra": [], - "env-extra": [], - "log-directory": None, - "shebang": None, - "job-cpu": None, - "job-mem": None, - "resource-spec": None, - "mem": None, - "lsf-units": None, - "use-stdin": None, - } - - generic_cluster_conf["scheduler_options"] = { + scheduler_options = { "interface": config_scheduler_interface, "port": config_scheduler_port, } - with dask.config.set({"jobqueue.generic-config": generic_cluster_conf}): + default_config_name = Cluster(cores=2, memory="2GB").job_cls.config_name + + with dask.config.set( + {"jobqueue.%s.scheduler-options" % default_config_name: scheduler_options} + ): - with Cluster(cores=2, memory="2GB", config_name="generic-config",) as cluster: + with Cluster(cores=2, memory="2GB") as cluster: scheduler_options = cluster.scheduler_spec["options"] assert scheduler_options.get("interface") == config_scheduler_interface assert scheduler_options.get("port") == config_scheduler_port @@ -65,7 +44,6 @@ def test_import_scheduler_options_from_config(Cluster): with Cluster( cores=2, memory="2GB", - config_name="generic-config", scheduler_options={"interface": pass_scheduler_interface}, ) as cluster: scheduler_options = cluster.scheduler_spec["options"] From e00042fcd66e1795fc587b15d8b4bdfd349bdce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Thu, 30 Apr 2020 19:03:58 +0200 Subject: [PATCH 12/16] Remove obsolete empty dictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Loïc Estève --- dask_jobqueue/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index 97eb0e4f..1d4c9b5e 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -478,7 +478,7 @@ def __init__( interface = dask.config.get("jobqueue.%s.interface" % config_name) if scheduler_options is None: scheduler_options = dask.config.get( - "jobqueue.%s.scheduler-options" % config_name, {} + "jobqueue.%s.scheduler-options" % config_name ) default_scheduler_options = { From e1bf1bdbc9051e3af303ae29d5e7cbbf8fead49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Thu, 30 Apr 2020 19:43:03 +0200 Subject: [PATCH 13/16] Don't instantiate the cluster --- dask_jobqueue/tests/test_scheduler_options_from_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_jobqueue/tests/test_scheduler_options_from_config.py b/dask_jobqueue/tests/test_scheduler_options_from_config.py index 5d746f40..681e1050 100644 --- a/dask_jobqueue/tests/test_scheduler_options_from_config.py +++ b/dask_jobqueue/tests/test_scheduler_options_from_config.py @@ -30,7 +30,7 @@ def test_import_scheduler_options_from_config(Cluster): "port": config_scheduler_port, } - default_config_name = Cluster(cores=2, memory="2GB").job_cls.config_name + default_config_name = Cluster.job_cls.config_name with dask.config.set( {"jobqueue.%s.scheduler-options" % default_config_name: scheduler_options} From df3b7fff76a1cb55e774d28d1312bd468b133885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Mon, 4 May 2020 12:57:35 +0200 Subject: [PATCH 14/16] Restore non-obsolete empty dictionary --- dask_jobqueue/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dask_jobqueue/core.py b/dask_jobqueue/core.py index 1d4c9b5e..97eb0e4f 100644 --- a/dask_jobqueue/core.py +++ b/dask_jobqueue/core.py @@ -478,7 +478,7 @@ def __init__( interface = dask.config.get("jobqueue.%s.interface" % config_name) if scheduler_options is None: scheduler_options = dask.config.get( - "jobqueue.%s.scheduler-options" % config_name + "jobqueue.%s.scheduler-options" % config_name, {} ) default_scheduler_options = { From 9bc806855364b0fe143271b366ea82601edda31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20H=C3=B6flich?= <37177103+kathoef@users.noreply.github.com> Date: Mon, 4 May 2020 13:02:09 +0200 Subject: [PATCH 15/16] Make file name less specific --- .../{test_scheduler_options_from_config.py => test_configs.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dask_jobqueue/tests/{test_scheduler_options_from_config.py => test_configs.py} (100%) diff --git a/dask_jobqueue/tests/test_scheduler_options_from_config.py b/dask_jobqueue/tests/test_configs.py similarity index 100% rename from dask_jobqueue/tests/test_scheduler_options_from_config.py rename to dask_jobqueue/tests/test_configs.py From 29f3491d6836ba31e1aa91899153b687b16c2c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= Date: Tue, 12 May 2020 12:17:43 +0200 Subject: [PATCH 16/16] Put back tests in test_jobqueue_core.py and add changelog entry. --- dask_jobqueue/tests/test_configs.py | 51 ----------------------- dask_jobqueue/tests/test_jobqueue_core.py | 39 +++++++++++++++++ docs/source/changelog.rst | 3 +- 3 files changed, 41 insertions(+), 52 deletions(-) delete mode 100644 dask_jobqueue/tests/test_configs.py diff --git a/dask_jobqueue/tests/test_configs.py b/dask_jobqueue/tests/test_configs.py deleted file mode 100644 index 681e1050..00000000 --- a/dask_jobqueue/tests/test_configs.py +++ /dev/null @@ -1,51 +0,0 @@ -import pytest -import psutil -import dask - -from dask_jobqueue import ( - PBSCluster, - MoabCluster, - SLURMCluster, - SGECluster, - LSFCluster, - OARCluster, -) - - -@pytest.mark.parametrize( - "Cluster", - [PBSCluster, MoabCluster, SLURMCluster, SGECluster, LSFCluster, OARCluster], -) -def test_import_scheduler_options_from_config(Cluster): - - net_if_addrs = psutil.net_if_addrs() - - config_scheduler_interface = list(net_if_addrs.keys())[0] - config_scheduler_port = 8804 - - pass_scheduler_interface = list(net_if_addrs.keys())[1] - - scheduler_options = { - "interface": config_scheduler_interface, - "port": config_scheduler_port, - } - - default_config_name = Cluster.job_cls.config_name - - with dask.config.set( - {"jobqueue.%s.scheduler-options" % default_config_name: scheduler_options} - ): - - with Cluster(cores=2, memory="2GB") as cluster: - scheduler_options = cluster.scheduler_spec["options"] - assert scheduler_options.get("interface") == config_scheduler_interface - assert scheduler_options.get("port") == config_scheduler_port - - with Cluster( - cores=2, - memory="2GB", - scheduler_options={"interface": pass_scheduler_interface}, - ) as cluster: - scheduler_options = cluster.scheduler_spec["options"] - assert scheduler_options.get("interface") == pass_scheduler_interface - assert scheduler_options.get("port") is None diff --git a/dask_jobqueue/tests/test_jobqueue_core.py b/dask_jobqueue/tests/test_jobqueue_core.py index bc9c94e0..db44002b 100644 --- a/dask_jobqueue/tests/test_jobqueue_core.py +++ b/dask_jobqueue/tests/test_jobqueue_core.py @@ -343,3 +343,42 @@ def test_cluster_error_scheduler_arguments_should_use_scheduler_options(Cluster) with pytest.raises(ValueError, match=message): with Cluster(cores=1, memory="1GB", dashboard_address=":8787"): pass + + +@pytest.mark.parametrize( + "Cluster", + [PBSCluster, MoabCluster, SLURMCluster, SGECluster, LSFCluster, OARCluster], +) +def test_import_scheduler_options_from_config(Cluster): + + net_if_addrs = psutil.net_if_addrs() + + config_scheduler_interface = list(net_if_addrs.keys())[0] + config_scheduler_port = 8804 + + pass_scheduler_interface = list(net_if_addrs.keys())[1] + + scheduler_options = { + "interface": config_scheduler_interface, + "port": config_scheduler_port, + } + + default_config_name = Cluster.job_cls.config_name + + with dask.config.set( + {"jobqueue.%s.scheduler-options" % default_config_name: scheduler_options} + ): + + with Cluster(cores=2, memory="2GB") as cluster: + scheduler_options = cluster.scheduler_spec["options"] + assert scheduler_options.get("interface") == config_scheduler_interface + assert scheduler_options.get("port") == config_scheduler_port + + with Cluster( + cores=2, + memory="2GB", + scheduler_options={"interface": pass_scheduler_interface}, + ) as cluster: + scheduler_options = cluster.scheduler_spec["options"] + assert scheduler_options.get("interface") == pass_scheduler_interface + assert scheduler_options.get("port") is None diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index a918dae2..c017867b 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -8,7 +8,8 @@ Development version the Dask scheduler. For example ``scheduler_options={'interface': 'eth0', dashboard_addresses=':12435')`` (:pr:`384`). Breaking change: using ``port`` or ``dashboard_addresses`` arguments raises an error. They have to be passed - through ``scheduler_options``. + through ``scheduler_options``. ``scheduler_options`` can be set through the + config file in the ``scheduler-options`` section (:pr:`405`). - all cluster classes: ``processes`` parameter default has changed. By default, ``processes ~= sqrt(cores)`` so that the number of processes and the number of threads per process is roughly the same. Old default was to use one