From 6e005bf66c088b41141381076a4ad3f522983c21 Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Sat, 27 Mar 2021 13:40:30 +0000 Subject: [PATCH 1/5] Update out-of-date references to `config.yaml` --- distributed/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distributed/client.py b/distributed/client.py index 625d1fb1040..12e5c9c36cc 100644 --- a/distributed/client.py +++ b/distributed/client.py @@ -3494,8 +3494,8 @@ def get_scheduler_logs(self, n=None): Parameters ---------- n : int - Number of logs to retrive. Maxes out at 10000 by default, - confiruable in config.yaml::log-length + Number of logs to retrive. Maxes out at 100000 by default, + confiruable in distributed.yaml::transition-log-length Returns ------- @@ -3509,8 +3509,8 @@ def get_worker_logs(self, n=None, workers=None, nanny=False): Parameters ---------- n : int - Number of logs to retrive. Maxes out at 10000 by default, - confiruable in config.yaml::log-length + Number of logs to retrive. Set to 0 by default, + confiruable in distributed.yaml::recent-messages-log-length workers : iterable List of worker addresses to retrieve. Gets all workers by default. nanny : bool, default False From acc7f544780892840ace4688eb49514dceeb8bf5 Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Sat, 27 Mar 2021 22:04:47 +0000 Subject: [PATCH 2/5] Re-enable logging in default `distributed.yaml` config * `logging` was removed from the config in 399522b95 (distributed PR #1948), and the assumption made in the present commit is that it was done unintentionally, since docs/source/configuration.rst (dask) wasn't updated at the same time with 399522b95. * Later down the line, in response to https://stackoverflow.com/questions/57829474/dask-config-get-cannot-get-anything, (dask PR #5374) was merged, which didn't update the documentation either. * The above two facts, combined together, suggest that `logging` must've been removed unintentionally from the config. --- distributed/distributed.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml index fb01a168d38..bf31fd9d045 100644 --- a/distributed/distributed.yaml +++ b/distributed/distributed.yaml @@ -1,12 +1,12 @@ distributed: version: 2 - # logging: - # distributed: info - # distributed.client: warning - # bokeh: critical - # # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr - # tornado: critical - # tornado.application: error + logging: + distributed: info + distributed.client: warning + bokeh: critical + # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr + tornado: critical + tornado.application: error scheduler: allowed-failures: 3 # number of retries before a task is considered bad From 82ddc13ece98329e0f62c053c00b22a40badcef8 Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Sun, 28 Mar 2021 11:34:57 +0000 Subject: [PATCH 3/5] Update distributed-schema.yaml to accommodate "logging" --- distributed/distributed-schema.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml index 331fdd10ef0..14bd4ae8cf5 100644 --- a/distributed/distributed-schema.yaml +++ b/distributed/distributed-schema.yaml @@ -6,6 +6,13 @@ properties: version: type: integer + logging: + type: object + description: | + The logging levels at which various administrative events get logged using the + Python standard logging module. Both the logging level and logging handlers + are customizable + scheduler: type: object properties: From f7adaea6af621fc101b125bb504e37a129e962b6 Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Sun, 28 Mar 2021 12:43:17 +0000 Subject: [PATCH 4/5] Update test_config.py unit tests (WIP) * OK: * test_logging_simple * test_logging_extended * test_schema_is_complete * Oustanding: test_logging_file_config --- distributed/tests/test_config.py | 82 +++++++++++++++++++------------- 1 file changed, 48 insertions(+), 34 deletions(-) diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py index 74b57b1f011..026025e7fed 100644 --- a/distributed/tests/test_config.py +++ b/distributed/tests/test_config.py @@ -152,7 +152,11 @@ def test_logging_simple(): """ Test simple ("old-style") logging configuration. """ - c = {"logging": {"distributed.foo": "info", "distributed.foo.bar": "error"}} + c = { + "distributed": { + "logging": {"distributed.foo": "info", "distributed.foo.bar": "error",} + } + } # Must test using a subprocess to avoid wrecking pre-existing configuration with new_config_file(c): code = """if 1: @@ -188,29 +192,31 @@ def test_logging_extended(): Test extended ("new-style") logging configuration. """ c = { - "logging": { - "version": "1", - "formatters": { - "simple": {"format": "%(levelname)s: %(name)s: %(message)s"} - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "stream": "ext://sys.stderr", - "formatter": "simple", - } - }, - "loggers": { - "distributed.foo": { - "level": "INFO", - #'handlers': ['console'], + "distributed": { + "logging": { + "version": "1", + "formatters": { + "simple": {"format": "%(levelname)s: %(name)s: %(message)s"} }, - "distributed.foo.bar": { - "level": "ERROR", - #'handlers': ['console'], + "handlers": { + "console": { + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr", + "formatter": "simple", + } }, - }, - "root": {"level": "WARNING", "handlers": ["console"]}, + "loggers": { + "distributed.foo": { + "level": "INFO", + #'handlers': ['console'], + }, + "distributed.foo.bar": { + "level": "ERROR", + #'handlers': ['console'], + }, + }, + "root": {"level": "WARNING", "handlers": ["console"]}, + } } } # Must test using a subprocess to avoid wrecking pre-existing configuration @@ -294,17 +300,25 @@ def test_logging_file_config(): with tempfile.NamedTemporaryFile(mode="w", delete=False) as logging_config: logging_config.write(logging_config_contents) dask_config = {"logging-file-config": logging_config.name} - with new_config_file(dask_config): - code = """if 1: - import logging - from distributed import config - foo = logging.getLogger('foo') - bar = logging.getLogger('foo.bar') - assert logging.INFO == foo.getEffectiveLevel() - assert logging.ERROR == bar.getEffectiveLevel() - """ - subprocess.check_call([sys.executable, "-c", code]) - os.remove(logging_config.name) + + try: + with new_config_file(dask_config): + code = """if 1: + import logging + from distributed import config + + foo = logging.getLogger('foo') + bar = logging.getLogger('foo.bar') + + expected = logging.INFO, logging.ERROR + actual = foo.getEffectiveLevel(), bar.getEffectiveLevel() + + assert expected == actual, "{} != {}".format(expected, actual) + """ + subprocess.check_call([sys.executable, "-c", code]) + + finally: + os.remove(logging_config.name) def test_schema(): @@ -331,7 +345,7 @@ def test_schema_is_complete(): with open(schema_fn) as f: schema = yaml.safe_load(f) - skip = {"default-task-durations", "bokeh-application"} + skip = {"default-task-durations", "bokeh-application", "logging"} def test_matches(c, s): if set(c) != set(s["properties"]): From 57a49cb9fb1e8d8d3bc058c61e8b27ede690e122 Mon Sep 17 00:00:00 2001 From: Hristo Georgiev Date: Sun, 28 Mar 2021 12:59:49 +0000 Subject: [PATCH 5/5] Update test_config as per `black-20.8b1` --- distributed/tests/test_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py index 026025e7fed..507dfcfd258 100644 --- a/distributed/tests/test_config.py +++ b/distributed/tests/test_config.py @@ -154,7 +154,10 @@ def test_logging_simple(): """ c = { "distributed": { - "logging": {"distributed.foo": "info", "distributed.foo.bar": "error",} + "logging": { + "distributed.foo": "info", + "distributed.foo.bar": "error", + } } } # Must test using a subprocess to avoid wrecking pre-existing configuration