From b81cb90ffc120e9306a819772538e49e87a26429 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Mon, 29 Apr 2019 17:01:00 +0200 Subject: [PATCH 1/3] Join frames on _deserialize_bytes() --- distributed/protocol/serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py index 3b0a45c8a6f..4ff0fb47a65 100644 --- a/distributed/protocol/serialize.py +++ b/distributed/protocol/serialize.py @@ -473,7 +473,7 @@ def _serialize_bytes(obj): @dask_deserialize.register((bytes, bytearray)) def _deserialize_bytes(header, frames): - return frames[0] + return b"".join(frames) ######################### From ccb8861d67e1561540e6b74df212c10ad5dd0d25 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Mon, 29 Apr 2019 18:44:53 +0200 Subject: [PATCH 2/3] Test for deserialization of bytes larger than 64MB --- distributed/protocol/tests/test_serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py index da43021d550..167e004c7d2 100644 --- a/distributed/protocol/tests/test_serialize.py +++ b/distributed/protocol/tests/test_serialize.py @@ -192,7 +192,7 @@ def test_empty_loads_deep(): def test_serialize_bytes(): - for x in [1, "abc", np.arange(5)]: + for x in [1, "abc", np.arange(5), b"ab" * int(100e6)]: b = serialize_bytes(x) assert isinstance(b, bytes) y = deserialize_bytes(b) From b48c185f4eea36e8fa55b719328511191fbb4baf Mon Sep 17 00:00:00 2001 From: Matthew Rocklin Date: Mon, 29 Apr 2019 20:27:05 +0200 Subject: [PATCH 3/3] Reduce test_serialize_bytes large data sample to decrease runtime Co-Authored-By: pentschev --- distributed/protocol/tests/test_serialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py index 167e004c7d2..4f72ec9a538 100644 --- a/distributed/protocol/tests/test_serialize.py +++ b/distributed/protocol/tests/test_serialize.py @@ -192,7 +192,7 @@ def test_empty_loads_deep(): def test_serialize_bytes(): - for x in [1, "abc", np.arange(5), b"ab" * int(100e6)]: + for x in [1, "abc", np.arange(5), b"ab" * int(40e6)]: b = serialize_bytes(x) assert isinstance(b, bytes) y = deserialize_bytes(b)