From 45b5a0c53e77427d09d6bab1e68388e585a055f0 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Tue, 5 Sep 2017 01:46:53 +0000 Subject: [PATCH 1/7] Update guess-chunks so that it works with zero dimensions. See alimanfoo/zarr#150. --- zarr/tests/test_creation.py | 10 ++++++++++ zarr/tests/test_util.py | 5 ++++- zarr/util.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index bb617fff14..1e9f3e85ce 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -383,6 +383,16 @@ def test_create(): with assert_raises(ValueError): create(100, compression=1) +def test_create_zero_len(): + + # Just test defaults. + z = create(0) + assert_is_instance(z, Array) + eq((0,), z.shape) + + n = z[:] + eq(0, len(n)) + def test_compression_args(): diff --git a/zarr/tests/test_util.py b/zarr/tests/test_util.py index fe4d7aaf05..69e6936cb3 100644 --- a/zarr/tests/test_util.py +++ b/zarr/tests/test_util.py @@ -183,12 +183,15 @@ def test_guess_chunks(): (1000, 10000000, 2), (10000, 10000, 10000), (100000, 100000, 100000), + (0,), + (0, 0), + (1, 2, 0, 4, 5), ) for shape in shapes: chunks = guess_chunks(shape, 1) assert_is_instance(chunks, tuple) eq(len(chunks), len(shape)) - assert all([c <= s for c, s in zip(chunks, shape)]) + assert all([c <= max(s, 1) for c, s in zip(chunks, shape)]) # ludicrous itemsize chunks = guess_chunks((1000000,), 40000000) diff --git a/zarr/util.py b/zarr/util.py index 8419d06ae2..ab6337ae5a 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -40,7 +40,7 @@ def guess_chunks(shape, typesize): """ ndims = len(shape) - chunks = np.array(shape, dtype='=f8') + chunks = np.maximum(np.array(shape, dtype='=f8'), 1) # Determine the optimal chunk size in bytes using a PyTables expression. # This is kept as a float. From 96f2558fcbdc9392ef1dd977cf0b53e1b01bd984 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Tue, 5 Sep 2017 15:46:38 +0000 Subject: [PATCH 2/7] Fix util.py to handle zero-length axis. --- zarr/util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zarr/util.py b/zarr/util.py index ab6337ae5a..f14303e4fa 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -153,6 +153,11 @@ def normalize_axis_selection(item, l): stop = l + stop if start < 0 or stop < 0: raise IndexError('index out of bounds: %s, %s' % (start, stop)) + + # Handle zero-length axis. + if start == stop == l == 0: + return slice(0, 0) + if start >= l: raise IndexError('index out of bounds: %s, %s' % (start, stop)) if stop > l: From 177b4fdeac115ee767cad8eea2b437b0616b8bc5 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Tue, 5 Sep 2017 16:01:30 +0000 Subject: [PATCH 3/7] Fix flake8 issue. --- zarr/tests/test_creation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 1e9f3e85ce..8f70bd177b 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -383,6 +383,7 @@ def test_create(): with assert_raises(ValueError): create(100, compression=1) + def test_create_zero_len(): # Just test defaults. From 9cde86e2e1be45b15baad7bd4062e8f75f3cab26 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Mon, 9 Oct 2017 13:31:44 +0000 Subject: [PATCH 4/7] Added in changes to handle the no-dimension case as well. --- zarr/core.py | 4 ++-- zarr/tests/test_creation.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/zarr/core.py b/zarr/core.py index b7f416d3f8..e4fb1a2e6d 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -265,7 +265,7 @@ def ndim(self): @property def _size(self): - return reduce(operator.mul, self._shape) + return reduce(operator.mul, self._shape, 1) @property def size(self): @@ -322,7 +322,7 @@ def cdata_shape(self): @property def _nchunks(self): - return reduce(operator.mul, self._cdata_shape) + return reduce(operator.mul, self._cdata_shape, 1) @property def nchunks(self): diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 8f70bd177b..8e6b77cc2b 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -394,6 +394,11 @@ def test_create_zero_len(): n = z[:] eq(0, len(n)) +def test_create_no_dims(): + ar = np.ndarray(()) + ar[()] = 100 + z = array(ar) + assert_array_equal(ar, z[:]) def test_compression_args(): From 0c258db69e8bb64b2a084b94180f37d86e64b3e5 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Mon, 9 Oct 2017 13:49:14 +0000 Subject: [PATCH 5/7] Added in changes to make no-dims arrays work well with the DirectoryStores. The problem was the empty file name. --- zarr/core.py | 6 +++++- zarr/tests/test_creation.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/zarr/core.py b/zarr/core.py index e4fb1a2e6d..52eec5329a 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -764,7 +764,11 @@ def _chunk_setitem_nosync(self, cidx, item, value): self._chunk_store[ckey] = cdata def _chunk_key(self, cidx): - return self._key_prefix + '.'.join(map(str, cidx)) + # Empty keys don't play well with file-systems... + if len(cidx) == 0: + return self._key_prefix + 'null' + else: + return self._key_prefix + '.'.join(map(str, cidx)) def _decode_chunk(self, cdata): diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 8e6b77cc2b..52ef1f45cb 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -400,6 +400,18 @@ def test_create_no_dims(): z = array(ar) assert_array_equal(ar, z[:]) +def test_create_no_dims_dirstore(): + ar = np.ndarray(()) + ar[()] = 100 + + path = tempfile.mkdtemp() + try: + store = DirectoryStore(path) + z = array(ar, store = store) + assert_array_equal(ar, z[:]) + finally: + shutil.rmtree(path) + def test_compression_args(): z = create(100, compression='zlib', compression_opts=9) From 9153a3fdfd44c06965b456548fcc651cadd12c47 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Mon, 9 Oct 2017 15:41:54 +0000 Subject: [PATCH 6/7] Fixed flake8 issues. --- zarr/tests/test_creation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 52ef1f45cb..485c17b7de 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -394,12 +394,14 @@ def test_create_zero_len(): n = z[:] eq(0, len(n)) + def test_create_no_dims(): ar = np.ndarray(()) ar[()] = 100 z = array(ar) assert_array_equal(ar, z[:]) + def test_create_no_dims_dirstore(): ar = np.ndarray(()) ar[()] = 100 @@ -407,11 +409,12 @@ def test_create_no_dims_dirstore(): path = tempfile.mkdtemp() try: store = DirectoryStore(path) - z = array(ar, store = store) + z = array(ar, store=store) assert_array_equal(ar, z[:]) finally: shutil.rmtree(path) + def test_compression_args(): z = create(100, compression='zlib', compression_opts=9) From 9b96ad383f4488b4c89923a49d218293b83357d8 Mon Sep 17 00:00:00 2001 From: Prakhar Goel Date: Mon, 9 Oct 2017 22:25:54 +0000 Subject: [PATCH 7/7] Use "0" instead of "null" for the chunk key for 0-d arrays. --- zarr/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/core.py b/zarr/core.py index 52eec5329a..4c48021b63 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -766,7 +766,7 @@ def _chunk_setitem_nosync(self, cidx, item, value): def _chunk_key(self, cidx): # Empty keys don't play well with file-systems... if len(cidx) == 0: - return self._key_prefix + 'null' + return self._key_prefix + '0' else: return self._key_prefix + '.'.join(map(str, cidx))