From ec1058fb6d2791f355aac9b82eb6b0f55bcc5322 Mon Sep 17 00:00:00 2001 From: halehawk Date: Thu, 10 Feb 2022 14:34:14 -0700 Subject: [PATCH 01/13] Optional flatten the encode array --- numcodecs/compat.py | 8 +++++--- numcodecs/tests/test_zfpy.py | 1 - numcodecs/zfpy.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/numcodecs/compat.py b/numcodecs/compat.py index f7f15f11..2f9b50ab 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -50,7 +50,7 @@ def ensure_ndarray(buf): return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None): +def ensure_contiguous_ndarray(buf, max_buffer_size=None, flat=True): """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -91,8 +91,10 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None): # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: - # can flatten without copy - arr = arr.reshape(-1, order='A') + # check if flat flag is on or not + if flat: + # can flatten without copy + arr = arr.reshape(-1, order='A') else: raise ValueError('an array with contiguous memory is required') diff --git a/numcodecs/tests/test_zfpy.py b/numcodecs/tests/test_zfpy.py index b6954dc9..89a951ee 100644 --- a/numcodecs/tests/test_zfpy.py +++ b/numcodecs/tests/test_zfpy.py @@ -38,7 +38,6 @@ np.random.normal(loc=1000, scale=1, size=(100, 10)), np.random.normal(loc=1000, scale=1, size=(10, 10, 10)), np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)), - np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))), np.random.randint(-(2 ** 31), -(2 ** 31) + 20, size=1000, dtype="i4").reshape( 100, 10 ), diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py index 51940399..e90ef4fe 100644 --- a/numcodecs/zfpy.py +++ b/numcodecs/zfpy.py @@ -54,8 +54,14 @@ def __init__( def encode(self, buf): - # normalise inputs - buf = ensure_contiguous_ndarray(buf) + # normalise fortran inputs only + if buf.flags.c_contiguous: + flat = False + else: + print(buf.flags) + print('Do not support Fortran array') + exit() + buf = ensure_contiguous_ndarray(buf, flat=flat) # do compression return _zfpy.compress_numpy( From 80cf19554e7e29b463611bc208b5a4a7fc3ffa22 Mon Sep 17 00:00:00 2001 From: halehawk Date: Thu, 10 Feb 2022 14:47:15 -0700 Subject: [PATCH 02/13] Remove skipping Python 3.9 and later for Zfp --- docs/conf.py | 2 +- requirements_dev.txt | 2 +- requirements_rtfd.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0c978ac2..b2a89606 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ def __getattr__(cls, name): return Mock() -MOCK_MODULES = ['msgpack', 'zfpy'] +MOCK_MODULES = ['msgpack'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/requirements_dev.txt b/requirements_dev.txt index c0d43923..92a3672b 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ Cython==0.29.21 msgpack==1.0.2 numpy==1.19.0 -zfpy==0.5.5; python_version < '3.9' +zfpy==0.5.5 diff --git a/requirements_rtfd.txt b/requirements_rtfd.txt index 72bb7fe5..8854bded 100644 --- a/requirements_rtfd.txt +++ b/requirements_rtfd.txt @@ -6,4 +6,4 @@ numpydoc mock numpy cython -zfpy==0.5.5; python_version < '3.9' +zfpy==0.5.5 From 7038e965a2f9ae7d71f2013defd3a084a29bd282 Mon Sep 17 00:00:00 2001 From: halehawk Date: Thu, 10 Feb 2022 14:55:28 -0700 Subject: [PATCH 03/13] Updated docs/release.rst about ZFPY --- docs/release.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index a8923da3..c04efccc 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -5,6 +5,9 @@ Release notes Unreleased ---------- +* Fix a flatten array error for ZFPY, ZFPY codec is supported on Python 3.9 + and 3.10, the docs about ZFPY is available. + By :user:`Haiying Xu `, .. _release_0.9.1: From 39a58efabd4a3c660c710caa726cf913c0b1b8be Mon Sep 17 00:00:00 2001 From: halehawk Date: Fri, 11 Feb 2022 12:56:32 -0700 Subject: [PATCH 04/13] Removed the trailing whitespace --- numcodecs/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/compat.py b/numcodecs/compat.py index 2f9b50ab..7ed71589 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -92,7 +92,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flat=True): # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: # check if flat flag is on or not - if flat: + if flat: # can flatten without copy arr = arr.reshape(-1, order='A') From 7e274299dcec31d6189637cd8c78ffdbeec93b5d Mon Sep 17 00:00:00 2001 From: halehawk Date: Fri, 11 Feb 2022 13:41:13 -0700 Subject: [PATCH 05/13] Add pip install zfpy to start test_zfpy --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 543bfb43..3fe95a08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ cython numpy msgpack pytest +zfpy From 0f68801c82f9a372e3e5abf4ed7818a227ba4f59 Mon Sep 17 00:00:00 2001 From: halehawk Date: Tue, 15 Feb 2022 11:09:30 -0700 Subject: [PATCH 06/13] Add raise valueerror --- numcodecs/tests/test_zfpy.py | 23 +++++++++++++++++++++++ numcodecs/zfpy.py | 9 +++++---- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/numcodecs/tests/test_zfpy.py b/numcodecs/tests/test_zfpy.py index c8db6092..7766b059 100644 --- a/numcodecs/tests/test_zfpy.py +++ b/numcodecs/tests/test_zfpy.py @@ -83,3 +83,26 @@ def test_err_decode_object_buffer(): def test_err_encode_object_buffer(): check_err_encode_object_buffer(ZFPY()) + + +def test_err_encode_list(): + data = ['foo', 'bar', 'baz'] + for codec in codecs: + with pytest.raises(TypeError): + codec.encode(data) + + +def test_err_encode_non_contiguous(): + # non-contiguous memory + arr = np.arange(1000, dtype='i4')[::2] + for codec in codecs: + with pytest.raises(ValueError): + codec.encode(arr) + + +def test_err_encode_fortran_array(): + # fortran array + arr = np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))) + for codec in codecs: + with pytest.raises(ValueError): + codec.encode(arr) diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py index e90ef4fe..47895c6f 100644 --- a/numcodecs/zfpy.py +++ b/numcodecs/zfpy.py @@ -9,6 +9,7 @@ from .abc import Codec from .compat import ndarray_copy, ensure_contiguous_ndarray, ensure_bytes + import numpy as np # noinspection PyShadowingBuiltins class ZFPY(Codec): @@ -54,13 +55,13 @@ def __init__( def encode(self, buf): - # normalise fortran inputs only + # not flatten c-order array and raise exception for f-order array + if not isinstance(buf, np.ndarray): + raise TypeError(f"The zfp codec does not support none numpy arrays. Your buffers were {type(buf)}.") if buf.flags.c_contiguous: flat = False else: - print(buf.flags) - print('Do not support Fortran array') - exit() + raise ValueError(f"The zfp codec does not support F order arrays. Your arrays flags were {buf.flags}.") buf = ensure_contiguous_ndarray(buf, flat=flat) # do compression From 9bdc0c3ba21fba64174501dad90c5fc8884cc6a6 Mon Sep 17 00:00:00 2001 From: halehawk Date: Tue, 15 Feb 2022 11:25:00 -0700 Subject: [PATCH 07/13] Shorten the raise error lines --- numcodecs/zfpy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py index 47895c6f..d7e5aabd 100644 --- a/numcodecs/zfpy.py +++ b/numcodecs/zfpy.py @@ -57,11 +57,13 @@ def encode(self, buf): # not flatten c-order array and raise exception for f-order array if not isinstance(buf, np.ndarray): - raise TypeError(f"The zfp codec does not support none numpy arrays. Your buffers were {type(buf)}.") + raise TypeError("The zfp codec does not support none numpy arrays." + f" Your buffers were {type(buf)}.") if buf.flags.c_contiguous: flat = False else: - raise ValueError(f"The zfp codec does not support F order arrays. Your arrays flags were {buf.flags}.") + raise ValueError("The zfp codec does not support F order arrays. " + f"Your arrays flags were {buf.flags}.") buf = ensure_contiguous_ndarray(buf, flat=flat) # do compression From 18184d7cfdb384f13ca7381f09a48cd87e44fae2 Mon Sep 17 00:00:00 2001 From: halehawk Date: Fri, 25 Feb 2022 14:44:19 -0700 Subject: [PATCH 08/13] Modified ci-osx clang to 12.0.1 --- .github/workflows/ci-osx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-osx.yaml b/.github/workflows/ci-osx.yaml index a62ffdfa..46dfc481 100644 --- a/.github/workflows/ci-osx.yaml +++ b/.github/workflows/ci-osx.yaml @@ -27,7 +27,7 @@ jobs: - name: Set up env shell: "bash -l {0}" run: | - conda create -n env python==${{matrix.python-version}} wheel pip compilers + conda create -n env python=${{matrix.python-version}} wheel pip compilers 'clang>=12.0.1' conda activate env which pip pip install -r requirements_test.txt -r requirements.txt From 1824166c32735bab278eee6e1679c51d5707d3f3 Mon Sep 17 00:00:00 2001 From: halehawk Date: Fri, 25 Feb 2022 16:17:31 -0700 Subject: [PATCH 09/13] Revert the change in ci-osx.yaml --- .github/workflows/ci-osx.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-osx.yaml b/.github/workflows/ci-osx.yaml index 46dfc481..a62ffdfa 100644 --- a/.github/workflows/ci-osx.yaml +++ b/.github/workflows/ci-osx.yaml @@ -27,7 +27,7 @@ jobs: - name: Set up env shell: "bash -l {0}" run: | - conda create -n env python=${{matrix.python-version}} wheel pip compilers 'clang>=12.0.1' + conda create -n env python==${{matrix.python-version}} wheel pip compilers conda activate env which pip pip install -r requirements_test.txt -r requirements.txt From 91640cdac2a8d02c22ec757f6b597ed04fe590fd Mon Sep 17 00:00:00 2001 From: halehawk Date: Tue, 1 Mar 2022 09:47:49 -0700 Subject: [PATCH 10/13] Update release.rst --- docs/release.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 9f16b7bf..85e0dccd 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -6,8 +6,8 @@ Release notes Unreleased ---------- * Fix a flatten array error for ZFPY, ZFPY codec is supported on Python 3.9 - and 3.10, the docs about ZFPY is available. - By :user:`Haiying Xu `, + and 3.10 on Linux and MacOS, the docs about ZFPY is also available. + By :user:`Haiying Xu `, `John Kirkham `, :issue:`303`. .. _release_0.9.1: From 2a79fd6ea400b6e57dc5448d42a31db9b769b628 Mon Sep 17 00:00:00 2001 From: halehawk Date: Tue, 1 Mar 2022 13:26:24 -0700 Subject: [PATCH 11/13] Change flat to flatten --- docs/release.rst | 3 ++- numcodecs/compat.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 85e0dccd..d9473f65 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -7,7 +7,8 @@ Unreleased ---------- * Fix a flatten array error for ZFPY, ZFPY codec is supported on Python 3.9 and 3.10 on Linux and MacOS, the docs about ZFPY is also available. - By :user:`Haiying Xu `, `John Kirkham `, :issue:`303`. + By :user:`Haiying Xu `, `John Kirkham `, `Ryan Abernathey ` : + issue:`303`. .. _release_0.9.1: diff --git a/numcodecs/compat.py b/numcodecs/compat.py index 1f6893d6..900615bc 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -50,7 +50,7 @@ def ensure_ndarray(buf): return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None, flat=True): +def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True): """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -92,7 +92,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flat=True): # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: # check if flat flag is on or not - if flat: + if flatten: # can flatten without copy arr = arr.reshape(-1, order='A') From cb978d84a459baa33b5b81c94c0814e2694431af Mon Sep 17 00:00:00 2001 From: halehawk Date: Tue, 1 Mar 2022 13:28:09 -0700 Subject: [PATCH 12/13] Missed one flat, changed to flatten --- numcodecs/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/compat.py b/numcodecs/compat.py index 900615bc..b1187838 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -91,7 +91,7 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True): # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: - # check if flat flag is on or not + # check if flatten flag is on or not if flatten: # can flatten without copy arr = arr.reshape(-1, order='A') From 894324d145aeb3679512f5583b0cfbb61fdf063d Mon Sep 17 00:00:00 2001 From: halehawk Date: Tue, 1 Mar 2022 13:52:34 -0700 Subject: [PATCH 13/13] Change flat to flatten in zfpy.py --- numcodecs/zfpy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py index d7e5aabd..899dd330 100644 --- a/numcodecs/zfpy.py +++ b/numcodecs/zfpy.py @@ -60,11 +60,11 @@ def encode(self, buf): raise TypeError("The zfp codec does not support none numpy arrays." f" Your buffers were {type(buf)}.") if buf.flags.c_contiguous: - flat = False + flatten = False else: raise ValueError("The zfp codec does not support F order arrays. " f"Your arrays flags were {buf.flags}.") - buf = ensure_contiguous_ndarray(buf, flat=flat) + buf = ensure_contiguous_ndarray(buf, flatten=flatten) # do compression return _zfpy.compress_numpy(