From b51782f9ace837bed904e08563589970702c2351 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 14 Nov 2019 20:28:47 -0500 Subject: [PATCH 1/2] Force binary data to `bytes` in `ensure_text` To handle array types in `ensure_text` that do not support the Python buffer protocol, it can be handy to coerce them to `bytes` using the `tobytes` method. This way we know we have a `bytes` object that can be decoded. Note: This does incur a copy for some types that may not need this. --- numcodecs/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/compat.py b/numcodecs/compat.py index 52827ca6..3d6c9e9e 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -159,7 +159,7 @@ def ensure_bytes(buf): def ensure_text(s, encoding='utf-8'): if not isinstance(s, text_type): - s = ensure_contiguous_ndarray(s) + s = ensure_bytes(s) s = codecs.decode(s, encoding) return s From 95f2f77ba976394b3459365c7ddea7b51170add5 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 14 Nov 2019 20:42:01 -0500 Subject: [PATCH 2/2] Only use `ensure_bytes` when `memoryview` fails Instead of coercing all data to `bytes`, only perform this coercion when the data does not support the Python buffer protocol. IOW when calling `memoryview` on it would raise a `TypeError`. Note: This avoids a copy when the data supports the Python buffer protocol. Otherwise the copy is needed as `codecs.decode` will not know how to work with it, but it can work with `bytes`. --- numcodecs/compat.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/numcodecs/compat.py b/numcodecs/compat.py index 3d6c9e9e..fdf89cf0 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -159,7 +159,13 @@ def ensure_bytes(buf): def ensure_text(s, encoding='utf-8'): if not isinstance(s, text_type): - s = ensure_bytes(s) + s = ensure_contiguous_ndarray(s) + + try: + memoryview(s) + except TypeError: # pragma: no cover + s = ensure_bytes(s) + s = codecs.decode(s, encoding) return s