From a2dca8ee25bf0728d41cce8ddd271a06dcbcfe15 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 30 Nov 2018 13:16:58 -0500 Subject: [PATCH 1/6] Get bytes from buffer for Python 2's Pickle decode On Python 3, `pickle.loads` is able to work with anything that supports the (new) buffer protocol. Unfortunately Python 2's, `cPickle.loads` is not so flexible and requires a `bytes` object specifically. So this ensures that other objects are coerced to `bytes` objects specifically on Python 2. --- numcodecs/pickles.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/numcodecs/pickles.py b/numcodecs/pickles.py index 2d3ac81d..da996340 100644 --- a/numcodecs/pickles.py +++ b/numcodecs/pickles.py @@ -6,7 +6,7 @@ from .abc import Codec -from .compat import PY2 +from .compat import PY2, ensure_bytes if PY2: # pragma: py3 no cover @@ -48,6 +48,9 @@ def encode(self, buf): return pickle.dumps(buf, protocol=self.protocol) def decode(self, buf, out=None): + if PY2: + buf = ensure_bytes(buf) + dec = pickle.loads(buf) if out is not None: np.copyto(out, dec) From 1c184f8d8120c931c3147380c56efdb00fb0357f Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 30 Nov 2018 13:25:40 -0500 Subject: [PATCH 2/6] Ensure Pickle's decode gets a contiguous buffer On Python 3, `pickle.loads` expects an object that implements the buffer protocol and is contiguous. While it would already raise this error, it helpful for us to check this first. Plus we should be able to handle Fortran order buffers should that be needed. On Python 2, the more strict constraint of a `bytes` object is required, which is still enforced. --- numcodecs/pickles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/numcodecs/pickles.py b/numcodecs/pickles.py index da996340..a0ab2bec 100644 --- a/numcodecs/pickles.py +++ b/numcodecs/pickles.py @@ -6,7 +6,7 @@ from .abc import Codec -from .compat import PY2, ensure_bytes +from .compat import PY2, ensure_bytes, ensure_contiguous_ndarray if PY2: # pragma: py3 no cover @@ -50,6 +50,8 @@ def encode(self, buf): def decode(self, buf, out=None): if PY2: buf = ensure_bytes(buf) + else: + buf = ensure_contiguous_ndarray(buf) dec = pickle.loads(buf) if out is not None: From d4be3dce1e4cf8774954fd679a95183c78eb0e67 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 30 Nov 2018 13:46:31 -0500 Subject: [PATCH 3/6] Test that Pickle can decode an `ndarray` This ensures that Pickle can decode anything that supports the (new) buffer protocol. --- numcodecs/tests/test_pickles.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/numcodecs/tests/test_pickles.py b/numcodecs/tests/test_pickles.py index 9034b10a..437de68a 100644 --- a/numcodecs/tests/test_pickles.py +++ b/numcodecs/tests/test_pickles.py @@ -6,10 +6,11 @@ import numpy as np -from numcodecs.compat import PY2 +from numcodecs.compat import PY2, ensure_ndarray from numcodecs.pickles import Pickle -from numcodecs.tests.common import (check_config, check_repr, check_encode_decode_array, - check_backwards_compatibility, greetings) +from numcodecs.tests.common import (assert_array_items_equal, check_config, check_repr, + check_encode_decode_array, check_backwards_compatibility, + greetings) codecs = [ @@ -43,6 +44,12 @@ def test_encode_decode(): check_encode_decode_array(arr, codec) +def test_decode_ndarray(): + for arr, codec in itertools.product(arrays, codecs): + buf = ensure_ndarray(codec.encode(arr)) + assert_array_items_equal(arr, codec.decode(buf)) + + def test_config(): codec = Pickle(protocol=-1) check_config(codec) From 8d1a4ba1ef6e05dfe36713a8a1941d16ddf187f1 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 30 Nov 2018 14:08:33 -0500 Subject: [PATCH 4/6] Generalize decoding `ndarray` test --- numcodecs/tests/common.py | 4 ++++ numcodecs/tests/test_pickles.py | 13 +++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/numcodecs/tests/common.py b/numcodecs/tests/common.py index a06d4991..cd18491b 100644 --- a/numcodecs/tests/common.py +++ b/numcodecs/tests/common.py @@ -147,6 +147,10 @@ def check_encode_decode_array(arr, codec): codec.decode(enc, out=out) assert_array_items_equal(arr, out) + enc = codec.encode(arr) + dec = codec.decode(ensure_ndarray(enc)) + assert_array_items_equal(arr, dec) + def check_config(codec): config = codec.get_config() diff --git a/numcodecs/tests/test_pickles.py b/numcodecs/tests/test_pickles.py index 437de68a..9034b10a 100644 --- a/numcodecs/tests/test_pickles.py +++ b/numcodecs/tests/test_pickles.py @@ -6,11 +6,10 @@ import numpy as np -from numcodecs.compat import PY2, ensure_ndarray +from numcodecs.compat import PY2 from numcodecs.pickles import Pickle -from numcodecs.tests.common import (assert_array_items_equal, check_config, check_repr, - check_encode_decode_array, check_backwards_compatibility, - greetings) +from numcodecs.tests.common import (check_config, check_repr, check_encode_decode_array, + check_backwards_compatibility, greetings) codecs = [ @@ -44,12 +43,6 @@ def test_encode_decode(): check_encode_decode_array(arr, codec) -def test_decode_ndarray(): - for arr, codec in itertools.product(arrays, codecs): - buf = ensure_ndarray(codec.encode(arr)) - assert_array_items_equal(arr, codec.decode(buf)) - - def test_config(): codec = Pickle(protocol=-1) check_config(codec) From f9eecdfca4819fd021a0fbd190cec196709c07dd Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 30 Nov 2018 14:08:36 -0500 Subject: [PATCH 5/6] Include `Pickle.decode` fix in release notes --- docs/release.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index c55df95b..bfb4c387 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,6 +1,15 @@ Release notes ============= +.. _release_0.6.2: + +0.6.2 +----- + +* Handle (new) buffer protocol conforming types in ``Pickle.decode``. + (by :user:`John Kirkham `, :issue:`143`). + + .. _release_0.6.1: 0.6.1 From f6e6d981ab1ddde011af41135f46110390c37e52 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 30 Nov 2018 15:51:44 -0500 Subject: [PATCH 6/6] Add Python version coverage pragmas to branches --- numcodecs/pickles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numcodecs/pickles.py b/numcodecs/pickles.py index a0ab2bec..7e1c26f7 100644 --- a/numcodecs/pickles.py +++ b/numcodecs/pickles.py @@ -48,9 +48,9 @@ def encode(self, buf): return pickle.dumps(buf, protocol=self.protocol) def decode(self, buf, out=None): - if PY2: + if PY2: # pragma: py3 no cover buf = ensure_bytes(buf) - else: + else: # pragma: py2 no cover buf = ensure_contiguous_ndarray(buf) dec = pickle.loads(buf)