From 9926b74bfc88267a76672655eb680e944823b54f Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Mon, 23 Mar 2020 16:44:45 -0700 Subject: [PATCH 1/3] Check `nbytes` and `types` before reading `data` To avoid reading `data` when it is not needed, try checking `nbytes` and `types` beforehand. If the metadata is already there, continue on without reading `data`. Otherwise fallback to the reading `data`, but do make sure to cache the results of that read to avoid doing it again. --- distributed/worker.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/distributed/worker.py b/distributed/worker.py index 247ffc99510..25bacdccac4 100644 --- a/distributed/worker.py +++ b/distributed/worker.py @@ -1836,13 +1836,16 @@ def ensure_communicating(self): def send_task_state_to_scheduler(self, key): if key in self.data or self.actors.get(key): - try: - value = self.data[key] - except KeyError: - value = self.actors[key] - nbytes = self.nbytes[key] or sizeof(value) - typ = self.types.get(key) or type(value) - del value + nbytes = self.nbytes[key] + typ = self.types.get(key) + if nbytes is None or typ is None: + try: + value = self.data[key] + except KeyError: + value = self.actors[key] + nbytes = self.nbytes[key] = sizeof(value) + typ = self.types[key] = type(value) + del value try: typ_serialized = dumps_function(typ) except PicklingError: From ef2bf166a5e654c3a4ddc09f314b338cd04aff31 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Mon, 23 Mar 2020 16:56:21 -0700 Subject: [PATCH 2/3] Use `.get(...)` with `self.nbytes` as well --- distributed/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/worker.py b/distributed/worker.py index 25bacdccac4..191e4df085f 100644 --- a/distributed/worker.py +++ b/distributed/worker.py @@ -1836,7 +1836,7 @@ def ensure_communicating(self): def send_task_state_to_scheduler(self, key): if key in self.data or self.actors.get(key): - nbytes = self.nbytes[key] + nbytes = self.nbytes.get(key) typ = self.types.get(key) if nbytes is None or typ is None: try: From 8d32e300d486612500c4ca575609b91a469bac75 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 24 Mar 2020 01:19:50 -0700 Subject: [PATCH 3/3] Restart GitHub CI Appears GitHub CI failed to checkout the code and clicking restart in the UI does not work. So pushing a dummy commit to restart it.